diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6e13a4b5c..1fd613229 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- fix(grammar): decode JSON Pointer escapes in schema references by @YaoxinHuang in #2368
 - feat: update llama.cpp to ggml-org/llama.cpp@v0.4.0
 
 ## [0.3.35]
diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py
index ba34dda83..93f7b7d8a 100644
--- a/llama_cpp/llama_grammar.py
+++ b/llama_cpp/llama_grammar.py
@@ -475,6 +475,7 @@ def visit(n: dict):
                         raise ValueError(f"Unsupported ref {ref}")
 
                     for sel in ref.split("#")[-1].split("/")[1:]:
+                        sel = sel.replace("~1", "/").replace("~0", "~")
                         assert target is not None and sel in target, (
                             f"Error resolving ref {ref}: {sel} not in {target}"
                         )
diff --git a/tests/test_llama_grammar.py b/tests/test_llama_grammar.py
index 34ef2874d..65859dfa8 100644
--- a/tests/test_llama_grammar.py
+++ b/tests/test_llama_grammar.py
@@ -1,5 +1,6 @@
 import llama_cpp
 import json
+import pytest
 
 tree = """
 leaf ::= "."
@@ -76,3 +77,23 @@ def test_grammar_anyof():
     grammar = llama_cpp.LlamaGrammar.from_json_schema(json.dumps(sch))
 
     # assert grammar.grammar is not None
+
+
+@pytest.mark.parametrize(
+    ("definition_name", "reference_token"),
+    [
+        ("a/b", "a~1b"),
+        ("a~b", "a~0b"),
+        ("a~1b", "a~01b"),
+        ("a~/b", "a~0~1b"),
+    ],
+)
+def test_grammar_json_pointer_escapes(definition_name, reference_token):
+    schema = {
+        "$defs": {definition_name: {"const": 42}},
+        "$ref": f"#/$defs/{reference_token}",
+    }
+
+    grammar = llama_cpp.LlamaGrammar.from_json_schema(json.dumps(schema))
+
+    assert '::= "42"' in grammar._grammar