From 955b1d37acc8fff30424f9043103c4861095dfa3 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 10:49:19 +0700 Subject: [PATCH 01/72] Preserve Python class namespace semantics --- src/aiython/runtime.py | 90 ++++++++++++++++-------- src/aiython/typed_runtime.py | 63 ++++++++++++----- tests/test_python_compatibility.py | 109 +++++++++++++++++++++++++++++ tests/test_runtime.py | 20 ++++++ 4 files changed, 237 insertions(+), 45 deletions(-) create mode 100644 tests/test_python_compatibility.py diff --git a/src/aiython/runtime.py b/src/aiython/runtime.py index 3fe6631..36c7349 100644 --- a/src/aiython/runtime.py +++ b/src/aiython/runtime.py @@ -525,31 +525,46 @@ def install_checkpoint(runtime, unit, node, key, *, retry_allowed=True, cleanup= runtime.checkpoints[key] = Checkpoint(unit, span, statement, target, ast.unparse(node.annotation) if isinstance(node, ast.AnnAssign) else None, retry_allowed) - counter = '__aiython_recovery_attempt_' + hashlib.sha256(key.encode()).hexdigest()[:16] - template = ast.parse( - "if True:\n" - f" {counter} = 0\n" - " while True:\n" - " try:\n" - " pass\n" - f" except {RUNTIME_NAME}.error_type as __aiython_error__:\n" - f" {counter} += 1\n" - f" if {RUNTIME_NAME}.recover({key!r}, __aiython_error__, {counter}):\n" - " continue\n" - " break\n" - " else:\n" - " break\n" - ).body[0] + if cleanup: + # Class __prepare__ mappings need not support deletion, so keep retry + # counts in the out-of-band class scope instead of their namespace. + template = ast.parse( + "while True:\n" + " try:\n" + " pass\n" + f" except {RUNTIME_NAME}.error_type:\n" + f" if {RUNTIME_NAME}.recover({key!r}, {RUNTIME_NAME}.current_exception()):\n" + " continue\n" + " break\n" + " else:\n" + f" {RUNTIME_NAME}.clear_recovery_count({key!r})\n" + " break\n" + ).body[0] + attempt = template.body[0] + else: + counter = '__aiython_recovery_attempt_' + hashlib.sha256(key.encode()).hexdigest()[:16] + template = ast.parse( + "if True:\n" + f" {counter} = 0\n" + " while True:\n" + " try:\n" + " pass\n" + f" except {RUNTIME_NAME}.error_type:\n" + f" {counter} += 1\n" + f" if {RUNTIME_NAME}.recover({key!r}, {RUNTIME_NAME}.current_exception(), {counter}):\n" + " continue\n" + " break\n" + " else:\n" + " break\n" + ).body[0] + attempt = template.body[1].body[0] for generated in ast.walk(template): if hasattr(generated, "lineno"): generated.lineno = node.lineno generated.end_lineno = node.end_lineno generated.col_offset = node.col_offset generated.end_col_offset = node.end_col_offset - template.body[1].body[0].body = [node] - if cleanup: - deletion = ast.Delete(targets=[ast.Name(id=counter, ctx=ast.Del())]) - template.body.append(ast.copy_location(deletion, node)) + attempt.body = [node] return template @@ -566,6 +581,7 @@ class NestedCheckpoints(ast.NodeTransformer): def __init__(self, runtime, unit): self.runtime, self.unit, self.serial = runtime, unit, 0 + self.in_class = False @staticmethod def generated(node): @@ -598,18 +614,24 @@ def visit_Module(self, node): return node def visit_FunctionDef(self, node): + previous = self.in_class + self.in_class = False node.body = self.body(node.body, preserve_docstring=True) + self.in_class = previous return node visit_AsyncFunctionDef = visit_FunctionDef def visit_ClassDef(self, node): + previous = self.in_class + self.in_class = True node.body = self.body(node.body, preserve_docstring=True, class_body=True) + self.in_class = previous return node def visit_For(self, node): - node.body = self.body(node.body) - node.orelse = self.body(node.orelse) + node.body = self.body(node.body, class_body=self.in_class) + node.orelse = self.body(node.orelse, class_body=self.in_class) return node visit_AsyncFor = visit_For @@ -618,14 +640,14 @@ def visit_For(self, node): def visit_Match(self, node): for case in node.cases: - case.body = self.body(case.body) + case.body = self.body(case.body, class_body=self.in_class) return node def visit_Try(self, node): # User try/except/finally and with managers must see exceptions first. # TypedTransformer's outer function guard is an implementation detail. - if getattr(node, '_aiython_type_guard', False): - node.body = self.body(node.body) + if getattr(node, '_aiython_type_guard', False) or getattr(node, '_aiython_class_guard', False): + node.body = self.body(node.body, class_body=self.in_class) return node visit_TryStar = visit_Try @@ -634,6 +656,10 @@ def visit_Try(self, node): class Runtime: error_type = BaseException + @staticmethod + def current_exception(): + return sys.exception() + def __init__(self, config: ResolvedConfig, *, agent_factory=None, stats=False, trace_plan=False): self._lock = threading.RLock() self.config = config @@ -906,12 +932,12 @@ def recover(self, key: str, error: BaseException, attempt: int | None = None) -> raise error checkpoint = self.checkpoints[key] frame = inspect.currentframe().f_back - # Compiled boundaries pass a frame-local counter reset for each visit. - # Keep the old frame-local fallback for direct/custom callers. + # Function/module boundaries pass their own counter. Class boundaries + # keep theirs outside the metaclass's namespace. counts = None if attempt is None: - from .typed_runtime import SCOPE, Scope - scope = frame.f_locals.get(SCOPE) + from .typed_runtime import SCOPE, Scope, class_scope + scope = class_scope(frame) or frame.f_locals.get(SCOPE) counts = (scope.recovery_counts if isinstance(scope, Scope) else frame.f_locals.setdefault("__aiython_recovery_counts__", {})) attempt = counts.get(key, 0) + 1 @@ -959,3 +985,11 @@ def recover(self, key: str, error: BaseException, attempt: int | None = None) -> raise exc from error finally: del frame + + def clear_recovery_count(self, key): + from .typed_runtime import class_scope + frame = inspect.currentframe().f_back + try: + class_scope(frame).recovery_counts.pop(key, None) + finally: + del frame diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 30e7a30..d062caa 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -2,6 +2,7 @@ from __future__ import annotations import ast +from contextvars import ContextVar from dataclasses import dataclass, field import inspect import threading @@ -11,6 +12,12 @@ from .type_constraints import Contract, ContractCache, TypeViolation, compile_contract, annotations_of, Compiler SCOPE = '__aiython_type_scope__' +_CLASS_SCOPES = ContextVar('aiython_class_scopes', default=()) + + +def class_scope(frame): + return next((scope for active, scope in reversed(_CLASS_SCOPES.get()) + if active is frame), None) @dataclass @@ -54,31 +61,46 @@ def register_class(self, cls): @staticmethod def namespace(frame): namespace = dict(frame.f_globals) | dict(frame.f_locals) - scope = frame.f_locals.get(SCOPE) + scope = class_scope(frame) or frame.f_locals.get(SCOPE) if isinstance(scope,Scope): namespace.update({parameter.__name__: bound for parameter,bound in scope.bindings.items()}) return namespace + def _initialize(self, frame, declarations, parameters, returns): + scope = Scope(declarations=declarations) + namespace = self.namespace(frame) + for name, (source, mode) in (parameters or {}).items(): + contract = self.contract(source,namespace) + if mode == 'args': contract = Contract('tuple_many',source,(contract,)) + elif mode == 'kwargs': contract = Contract('dict',source,(compile_contract('str',namespace),contract)) + contract.validate(frame.f_locals[name],name,bindings=scope.bindings) + scope.contracts[name] = contract + if returns: + scope.returned = self.contract(returns,namespace) + return scope + def initialize(self, declarations, parameters=None, returns=None): frame = inspect.currentframe().f_back try: - scope = Scope(declarations=declarations) - namespace = self.namespace(frame) - for name, (source, mode) in (parameters or {}).items(): - contract = self.contract(source,namespace) - if mode == 'args': contract = Contract('tuple_many',source,(contract,)) - elif mode == 'kwargs': contract = Contract('dict',source,(compile_contract('str',namespace),contract)) - contract.validate(frame.f_locals[name],name,bindings=scope.bindings) - scope.contracts[name] = contract - if returns: - scope.returned = self.contract(returns,namespace) - return scope + return self._initialize(frame, declarations, parameters, returns) finally: del frame + def enter_class_scope(self, declarations): + frame = inspect.currentframe().f_back + try: + scope = self._initialize(frame, declarations, None, None) + _CLASS_SCOPES.set(_CLASS_SCOPES.get() + ((frame, scope),)) + finally: + del frame + + def exit_class_scope(self): + stack = _CLASS_SCOPES.get() + _CLASS_SCOPES.set(stack[:-1]) + @staticmethod def scopes(frame): - local = frame.f_locals.get(SCOPE) + local = class_scope(frame) or frame.f_locals.get(SCOPE) global_scope = frame.f_globals.get(SCOPE) scopes = [(local,frame.f_locals)] if isinstance(local,Scope) else [] if isinstance(global_scope,Scope) and global_scope is not local: @@ -171,7 +193,7 @@ def checkpoint(self): # Active enclosing scopes can hold annotated aliases to mutated values. parent = frame.f_back while parent: - if SCOPE in parent.f_locals and parent.f_globals.get(RUNTIME_NAME) is frame.f_globals.get(RUNTIME_NAME): + if (class_scope(parent) or SCOPE in parent.f_locals) and parent.f_globals.get(RUNTIME_NAME) is frame.f_globals.get(RUNTIME_NAME): self.check_frame(parent) parent = parent.f_back finally: del frame @@ -314,7 +336,7 @@ def collect(node): for statement in body: collect(statement) return result - def body(self,body,*,parameters=None,returns=None,initialize=True,inherited=None): + def body(self,body,*,parameters=None,returns=None,initialize=True,inherited=None,class_scope=False): previous = self.declarations self.declarations = dict(inherited or {}) | self.declarations_in(body) output = [] @@ -324,7 +346,8 @@ def body(self,body,*,parameters=None,returns=None,initialize=True,inherited=None while body and isinstance(body[0],ast.ImportFrom) and body[0].module == '__future__': header.append(body[0]); body = body[1:] if initialize: - initial = ast.Assign([ast.Name(SCOPE,ast.Store())],helper('initialize',literal(self.declarations),literal(parameters),literal(returns))) + initial = (ast.Expr(helper('enter_class_scope',literal(self.declarations))) if class_scope else + ast.Assign([ast.Name(SCOPE,ast.Store())],helper('initialize',literal(self.declarations),literal(parameters),literal(returns)))) ast.copy_location(initial,body[0] if body else header[-1] if header else ast.Constant(None,lineno=1,col_offset=0)) output.append(initial) # Capture lexical types used only in stringified contracts without executing them. @@ -391,7 +414,13 @@ def visit_ClassDef(self,node): node.decorator_list.insert(0, ast.Attribute(ast.Attribute(ast.Name(RUNTIME_NAME,ast.Load()),'types',ast.Load()),'register_class',ast.Load())) previous = self.function self.function = False - node.body = self.body(node.body) + body = self.body(node.body,class_scope=True) + header = body[:1] if isinstance(body[0],ast.Expr) and isinstance(body[0].value,ast.Constant) and isinstance(body[0].value.value,str) else [] + enter, *statements = body[len(header):] + exit_call = ast.copy_location(ast.Expr(helper('exit_class_scope')), node) + guard = ast.copy_location(ast.Try(statements, [], [], [exit_call]), node) + guard._aiython_class_guard = True + node.body = header + [enter, guard] self.function = previous return node diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py new file mode 100644 index 0000000..c0a0133 --- /dev/null +++ b/tests/test_python_compatibility.py @@ -0,0 +1,109 @@ +"""Compare ordinary Python execution with Aiython in separate processes.""" + +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +class PythonCompatibilityTests(unittest.TestCase): + def test_language_constructs_match_cpython(self): + cases = { + 'metaclass namespace': '''class Meta(type): + def __new__(meta, name, bases, namespace): + print(sorted(namespace)) + return super().__new__(meta, name, bases, namespace) +class Example(metaclass=Meta): + value: int = 3 +print(Example.value) +''', + 'custom class namespace': '''class Namespace(dict): + def pop(self, *args): raise TypeError('removal forbidden') + def __delitem__(self, key): raise TypeError('removal forbidden') +class Meta(type): + @classmethod + def __prepare__(meta, name, bases): return Namespace() + def __new__(meta, name, bases, namespace): + print(sorted(namespace)) + return super().__new__(meta, name, bases, namespace) +class Example(metaclass=Meta): + for index in range(2): + value: int = index +print(Example.value) +''', + 'class locals': '''class Example: + snapshot = sorted(locals()) +print(Example.snapshot) +''', + 'failed class body leaves no scope': '''try: + class Broken: + value: int = 1 + raise ValueError('stop') +except ValueError: + pass +class Working: + value: int = 2 +print(Working.value) +''', + 'nested class and zero argument super': '''class Base: + def value(self): return 1 +class Outer: + class Child(Base): + def value(self): return super().value() + 1 +print(Outer.Child().value()) +''', + 'decorated slotted dataclass': '''from dataclasses import dataclass +@dataclass(slots=True) +class Point: + x: int +print(Point(2).x, hasattr(Point(2), '__dict__')) +''', + 'enum': '''from enum import Enum +class Color(Enum): + RED = 1 + BLUE = 2 +print([item.name for item in Color]) +''', + 'pattern matching': '''value = ('ok', 3) +match value: + case ('ok', number): print(number) + case _: print('missing') +''', + 'exception groups': '''try: + raise ExceptionGroup('both', [ValueError('a'), TypeError('b')]) +except* ValueError as error: + print(type(error.exceptions[0]).__name__) +except* TypeError as error: + print(type(error.exceptions[0]).__name__) +''', + 'generator delegation': '''def inner(): + received = yield 1 + return received +def outer(): + result = yield from inner() + print(result) +item = outer() +print(next(item)) +try: item.send(7) +except StopIteration: pass +''', + 'async generator': '''import asyncio +async def numbers(): + yield 1 + yield 2 +async def run(): + print([number async for number in numbers()]) +asyncio.run(run()) +''', + } + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'main.py' + for name, source in cases.items(): + with self.subTest(name=name): + path.write_text(source) + python = subprocess.run([sys.executable, str(path)], capture_output=True, text=True) + aiython = subprocess.run([sys.executable, '-m', 'aiython', str(path)], + capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 259ba65..be3594d 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -52,6 +52,26 @@ def test_plain_python_and_handled_error_never_call_agent(self): self.assertEqual(result["__doc__"], "module doc") self.assertFalse(agent.requests or agent.errors) + def test_class_recovery_keeps_scope_outside_namespace(self): + agent = FakeAgent(recover=lambda request, runtime: RecoveryDecision('complete', 7, True)) + result = self.run_source('''class Namespace(dict): + def __delitem__(self, key): raise TypeError('removal forbidden') +class Meta(type): + @classmethod + def __prepare__(meta, name, bases): return Namespace() + def __new__(meta, name, bases, namespace): + assert not any(key.startswith('__aiython_') for key in namespace) + return super().__new__(meta, name, bases, namespace) +class Example(metaclass=Meta): + values = [] + for index in range(2): + value = missing + values.append(value) +answer = Example.values +''', agent) + self.assertEqual(result['answer'], [7, 7]) + self.assertEqual([request.attempt for request in agent.errors], [1, 1]) + def test_grouped_statement_module_and_identity(self): def execute(request, runtime): self.assertEqual(request.related_objects["items"], []) From d3ba9b39962d267371f56b96ba7123bdbb1d9218 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 11:19:11 +0700 Subject: [PATCH 02/72] Align entry metadata and scope visibility with CPython --- src/aiython/cli.py | 19 ++++-- src/aiython/runtime.py | 86 ++++++++++++------------ src/aiython/typed_runtime.py | 85 +++++++++++++++--------- tests/test_python_compatibility.py | 103 +++++++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 83 deletions(-) diff --git a/src/aiython/cli.py b/src/aiython/cli.py index 47a79ac..33953bd 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import builtins import importlib.abc import importlib.machinery import importlib.metadata @@ -59,16 +60,22 @@ def find_spec(self, fullname, path=None, target=None): def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, stats=False, trace_plan=False, config_path=None, profile=None, force_profile=None): started = perf_counter() + argv0 = str(path) + display_path = path.absolute() path = path.resolve() config = config or resolve(path, config_path=config_path, profile=profile, force_profile=force_profile) config_seconds = perf_counter() - started runtime = Runtime(config, agent_factory=agent_factory, stats=stats, trace_plan=trace_plan) - code = runtime.compile_source(read_source(path), str(path), entry=True) + code = runtime.compile_source(read_source(path), str(display_path), entry=True) module = types.ModuleType("__main__") - module.__dict__.update({"__file__": str(path), "__package__": None, + module.__dict__.update({"__file__": str(display_path), "__package__": None, "__spec__": None, - "__cached__": None, RUNTIME_NAME: runtime, - "__builtins__": __builtins__}) + "__cached__": None, + "__loader__": importlib.machinery.SourceFileLoader("__main__", str(display_path)), + RUNTIME_NAME: runtime, + "__builtins__": builtins}) + if sys.version_info < (3, 14): + module.__annotations__ = {} old_main = sys.modules.get("__main__") old_argv, old_path = sys.argv, sys.path[:] old_spawn_entry = os.environ.get("AIYTHON_SPAWN_ENTRY") @@ -76,10 +83,10 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta execution_started = None try: sys.modules["__main__"] = module - sys.argv = [str(path), *arguments] + sys.argv = [argv0, *arguments] sys.path.insert(0, str(path.parent)) sys.meta_path.insert(0, finder) - os.environ["AIYTHON_SPAWN_ENTRY"] = str(path) + os.environ["AIYTHON_SPAWN_ENTRY"] = str(display_path) execution_started = perf_counter() exec(code, module.__dict__) return module.__dict__ diff --git a/src/aiython/runtime.py b/src/aiython/runtime.py index 36c7349..a18edd2 100644 --- a/src/aiython/runtime.py +++ b/src/aiython/runtime.py @@ -511,7 +511,7 @@ def visit_Call(self, node): return node -def install_checkpoint(runtime, unit, node, key, *, retry_allowed=True, cleanup=False): +def install_checkpoint(runtime, unit, node, key, *, retry_allowed=True, scoped_retries=False): """Catch one statement without replaying statements before it.""" target = None if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): @@ -525,9 +525,8 @@ def install_checkpoint(runtime, unit, node, key, *, retry_allowed=True, cleanup= runtime.checkpoints[key] = Checkpoint(unit, span, statement, target, ast.unparse(node.annotation) if isinstance(node, ast.AnnAssign) else None, retry_allowed) - if cleanup: - # Class __prepare__ mappings need not support deletion, so keep retry - # counts in the out-of-band class scope instead of their namespace. + if scoped_retries: + # Store retry counts in the type scope, away from user bindings. template = ast.parse( "while True:\n" " try:\n" @@ -581,7 +580,7 @@ class NestedCheckpoints(ast.NodeTransformer): def __init__(self, runtime, unit): self.runtime, self.unit, self.serial = runtime, unit, 0 - self.in_class = False + self.scoped_retries = False @staticmethod def generated(node): @@ -595,7 +594,7 @@ def generated(node): func = func.value return isinstance(func, ast.Name) and func.id == RUNTIME_NAME - def body(self, statements, *, nested=True, preserve_docstring=False, class_body=False): + def body(self, statements, *, nested=True, preserve_docstring=False, scoped_retries=False): result = [] for index, node in enumerate(statements): node = self.visit(node) @@ -605,7 +604,7 @@ def body(self, statements, *, nested=True, preserve_docstring=False, class_body= and not self.generated(node)): self.serial += 1 key = f'{self.unit.filename}:nested-checkpoint:{self.serial}' - node = install_checkpoint(self.runtime, self.unit, node, key, cleanup=class_body) + node = install_checkpoint(self.runtime, self.unit, node, key, scoped_retries=scoped_retries) result.append(node) return result @@ -614,24 +613,25 @@ def visit_Module(self, node): return node def visit_FunctionDef(self, node): - previous = self.in_class - self.in_class = False - node.body = self.body(node.body, preserve_docstring=True) - self.in_class = previous + previous = self.scoped_retries + self.scoped_retries = True + node.body = self.body(node.body, preserve_docstring=True, + scoped_retries=True) + self.scoped_retries = previous return node visit_AsyncFunctionDef = visit_FunctionDef def visit_ClassDef(self, node): - previous = self.in_class - self.in_class = True - node.body = self.body(node.body, preserve_docstring=True, class_body=True) - self.in_class = previous + previous = self.scoped_retries + self.scoped_retries = True + node.body = self.body(node.body, preserve_docstring=True, scoped_retries=True) + self.scoped_retries = previous return node def visit_For(self, node): - node.body = self.body(node.body, class_body=self.in_class) - node.orelse = self.body(node.orelse, class_body=self.in_class) + node.body = self.body(node.body, scoped_retries=self.scoped_retries) + node.orelse = self.body(node.orelse, scoped_retries=self.scoped_retries) return node visit_AsyncFor = visit_For @@ -640,14 +640,16 @@ def visit_For(self, node): def visit_Match(self, node): for case in node.cases: - case.body = self.body(case.body, class_body=self.in_class) + case.body = self.body(case.body, scoped_retries=self.scoped_retries) return node def visit_Try(self, node): # User try/except/finally and with managers must see exceptions first. # TypedTransformer's outer function guard is an implementation detail. - if getattr(node, '_aiython_type_guard', False) or getattr(node, '_aiython_class_guard', False): - node.body = self.body(node.body, class_body=self.in_class) + if getattr(node, '_aiython_module_guard', False): + node.body = self.body(node.body, nested=False) + elif getattr(node, '_aiython_type_guard', False) or getattr(node, '_aiython_class_guard', False): + node.body = self.body(node.body, scoped_retries=self.scoped_retries) return node visit_TryStar = visit_Try @@ -802,26 +804,22 @@ def _prepare(self, unit: Unit, *, entry: bool = False): tree = NestedCheckpoints(self, unit).visit(tree) ast.fix_missing_locations(tree) if entry: - body = [] - for index, node in enumerate(tree.body): - from .typed_runtime import SCOPE - internal_scope = isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == SCOPE for t in node.targets) - internal_check = (isinstance(node, ast.Expr) and isinstance(node.value, ast.Call) - and isinstance(node.value.func, ast.Attribute) and node.value.func.attr == 'checkpoint') - if internal_scope or internal_check: - body.append(node) - continue - if (isinstance(node, ast.ImportFrom) and node.module == "__future__") or ( - index == 0 and isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) - and isinstance(node.value.value, str)): - body.append(node) - continue - key = f"{unit.filename}:checkpoint:{index}" - retry_allowed = not isinstance(node, (ast.For, ast.AsyncFor, ast.While, - ast.If, ast.Try, ast.TryStar, ast.With, ast.AsyncWith, ast.Match, - ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)) - body.append(install_checkpoint(self, unit, node, key, retry_allowed=retry_allowed)) - tree.body = body + guard = next((node for node in tree.body if getattr(node, '_aiython_module_guard', False)), None) + if guard is not None: + body = [] + for index, node in enumerate(guard.body): + internal_check = (isinstance(node, ast.Expr) and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Attribute) and node.value.func.attr == 'checkpoint') + if internal_check: + body.append(node) + continue + key = f"{unit.filename}:checkpoint:{index}" + retry_allowed = not isinstance(node, (ast.For, ast.AsyncFor, ast.While, + ast.If, ast.Try, ast.TryStar, ast.With, ast.AsyncWith, ast.Match, + ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)) + body.append(install_checkpoint(self, unit, node, key, retry_allowed=retry_allowed, + scoped_retries=True)) + guard.body = body ast.fix_missing_locations(tree) return compile(tree, unit.filename, "exec", dont_inherit=True) @@ -936,8 +934,8 @@ def recover(self, key: str, error: BaseException, attempt: int | None = None) -> # keep theirs outside the metaclass's namespace. counts = None if attempt is None: - from .typed_runtime import SCOPE, Scope, class_scope - scope = class_scope(frame) or frame.f_locals.get(SCOPE) + from .typed_runtime import SCOPE, Scope, frame_scope + scope = frame_scope(frame) or frame.f_locals.get(SCOPE) counts = (scope.recovery_counts if isinstance(scope, Scope) else frame.f_locals.setdefault("__aiython_recovery_counts__", {})) attempt = counts.get(key, 0) + 1 @@ -987,9 +985,9 @@ def recover(self, key: str, error: BaseException, attempt: int | None = None) -> del frame def clear_recovery_count(self, key): - from .typed_runtime import class_scope + from .typed_runtime import SCOPE, frame_scope frame = inspect.currentframe().f_back try: - class_scope(frame).recovery_counts.pop(key, None) + (frame_scope(frame) or frame.f_locals[SCOPE]).recovery_counts.pop(key, None) finally: del frame diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index d062caa..f1a39d3 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -12,11 +12,11 @@ from .type_constraints import Contract, ContractCache, TypeViolation, compile_contract, annotations_of, Compiler SCOPE = '__aiython_type_scope__' -_CLASS_SCOPES = ContextVar('aiython_class_scopes', default=()) +_FRAME_SCOPES = ContextVar('aiython_frame_scopes', default=()) -def class_scope(frame): - return next((scope for active, scope in reversed(_CLASS_SCOPES.get()) +def frame_scope(frame): + return next((scope for active, scope in reversed(_FRAME_SCOPES.get()) if active is frame), None) @@ -61,7 +61,7 @@ def register_class(self, cls): @staticmethod def namespace(frame): namespace = dict(frame.f_globals) | dict(frame.f_locals) - scope = class_scope(frame) or frame.f_locals.get(SCOPE) + scope = frame_scope(frame) or frame.f_locals.get(SCOPE) if isinstance(scope,Scope): namespace.update({parameter.__name__: bound for parameter,bound in scope.bindings.items()}) return namespace @@ -86,21 +86,21 @@ def initialize(self, declarations, parameters=None, returns=None): finally: del frame - def enter_class_scope(self, declarations): + def enter_scope(self, declarations, parameters=None, returns=None): frame = inspect.currentframe().f_back try: - scope = self._initialize(frame, declarations, None, None) - _CLASS_SCOPES.set(_CLASS_SCOPES.get() + ((frame, scope),)) + scope = self._initialize(frame, declarations, parameters, returns) + _FRAME_SCOPES.set(_FRAME_SCOPES.get() + ((frame, scope),)) finally: del frame - def exit_class_scope(self): - stack = _CLASS_SCOPES.get() - _CLASS_SCOPES.set(stack[:-1]) + def exit_scope(self): + stack = _FRAME_SCOPES.get() + _FRAME_SCOPES.set(stack[:-1]) @staticmethod def scopes(frame): - local = class_scope(frame) or frame.f_locals.get(SCOPE) + local = frame_scope(frame) or frame.f_locals.get(SCOPE) global_scope = frame.f_globals.get(SCOPE) scopes = [(local,frame.f_locals)] if isinstance(local,Scope) else [] if isinstance(global_scope,Scope) and global_scope is not local: @@ -193,7 +193,7 @@ def checkpoint(self): # Active enclosing scopes can hold annotated aliases to mutated values. parent = frame.f_back while parent: - if (class_scope(parent) or SCOPE in parent.f_locals) and parent.f_globals.get(RUNTIME_NAME) is frame.f_globals.get(RUNTIME_NAME): + if (frame_scope(parent) or SCOPE in parent.f_locals) and parent.f_globals.get(RUNTIME_NAME) is frame.f_globals.get(RUNTIME_NAME): self.check_frame(parent) parent = parent.f_back finally: del frame @@ -202,7 +202,7 @@ def returned(self,value): frame = inspect.currentframe().f_back try: self.check_frame(frame) - scope = frame.f_locals.get(SCOPE) + scope = frame_scope(frame) or frame.f_locals.get(SCOPE) if isinstance(scope,Scope) and scope.returned: contract = scope.returned.args[2] if scope.returned.kind in ('generator','async_generator') and frame.f_code.co_flags & (inspect.CO_GENERATOR | inspect.CO_ASYNC_GENERATOR) else scope.returned contract.validate(value,'return',bindings=scope.bindings) @@ -214,14 +214,14 @@ def returned(self,value): def aborted(self): frame = inspect.currentframe().f_back try: - scope = frame.f_locals.get(SCOPE) + scope = frame_scope(frame) or frame.f_locals.get(SCOPE) if isinstance(scope,Scope): scope.failed = True finally: del frame def leaving(self): frame = inspect.currentframe().f_back try: - scope = frame.f_locals.get(SCOPE) + scope = frame_scope(frame) or frame.f_locals.get(SCOPE) if not isinstance(scope,Scope) or scope.failed: return self.check_frame(frame) @@ -234,7 +234,7 @@ def yielded(self,value): frame = inspect.currentframe().f_back try: self.check_frame(frame) - scope = frame.f_locals.get(SCOPE) + scope = frame_scope(frame) or frame.f_locals.get(SCOPE) if isinstance(scope,Scope) and scope.returned: if scope.returned.kind not in ('generator','async_generator'): raise TypeViolation('Generator return annotation must describe yielded values') @@ -245,7 +245,7 @@ def yielded(self,value): def sent(self,value): frame = inspect.currentframe().f_back try: - scope = frame.f_locals.get(SCOPE) + scope = frame_scope(frame) or frame.f_locals.get(SCOPE) if isinstance(scope,Scope) and scope.returned: scope.returned.args[1].validate(value,'send',bindings=scope.bindings) return value @@ -253,7 +253,7 @@ def sent(self,value): def delegate(self,iterable): frame = inspect.currentframe().f_back - scope = frame.f_locals.get(SCOPE) + scope = frame_scope(frame) or frame.f_locals.get(SCOPE) del frame contract = scope.returned if isinstance(scope,Scope) else None iterator = iter(iterable) @@ -323,7 +323,6 @@ def __init__(self, *, snippet=False): self.snippet = snippet self.declarations = {} self.function = False - self.temp = 0 @staticmethod def declarations_in(body): @@ -336,7 +335,7 @@ def collect(node): for statement in body: collect(statement) return result - def body(self,body,*,parameters=None,returns=None,initialize=True,inherited=None,class_scope=False): + def body(self,body,*,parameters=None,returns=None,initialize=True,inherited=None,external_scope=False): previous = self.declarations self.declarations = dict(inherited or {}) | self.declarations_in(body) output = [] @@ -346,9 +345,10 @@ def body(self,body,*,parameters=None,returns=None,initialize=True,inherited=None while body and isinstance(body[0],ast.ImportFrom) and body[0].module == '__future__': header.append(body[0]); body = body[1:] if initialize: - initial = (ast.Expr(helper('enter_class_scope',literal(self.declarations))) if class_scope else + initial = (ast.Expr(helper('enter_scope',literal(self.declarations),literal(parameters),literal(returns))) if external_scope else ast.Assign([ast.Name(SCOPE,ast.Store())],helper('initialize',literal(self.declarations),literal(parameters),literal(returns)))) ast.copy_location(initial,body[0] if body else header[-1] if header else ast.Constant(None,lineno=1,col_offset=0)) + initial._aiython_scope_initializer = True output.append(initial) # Capture lexical types used only in stringified contracts without executing them. names = set() @@ -367,7 +367,19 @@ def body(self,body,*,parameters=None,returns=None,initialize=True,inherited=None return header+output def visit_Module(self,node): - node.body = self.body(node.body,initialize=not self.snippet) + if self.snippet: + node.body = self.body(node.body,initialize=False) + return node + body = self.body(node.body,external_scope=True) + initial = next(i for i,item in enumerate(body) if getattr(item,'_aiython_scope_initializer',False)) + header, enter, statements = body[:initial], body[initial], body[initial+1:] + exit_call = ast.copy_location(ast.Expr(helper('exit_scope')), enter) + if statements: + guard = ast.copy_location(ast.Try(statements, [], [], [exit_call]), enter) + guard._aiython_module_guard = True + node.body = header + [enter, guard] + else: + node.body = header + [enter, exit_call] return node def visit_FunctionDef(self,node): @@ -382,19 +394,24 @@ def visit_FunctionDef(self,node): returns = ast.unparse(node.returns) if node.returns else None # Generator annotations require yield/send checks, not return-only checks. is_generator = any(isinstance(n,(ast.Yield,ast.YieldFrom)) for n in self.function_nodes(node)) + external_scope = not is_generator used = {n.id for n in self.function_nodes(node) if isinstance(n,ast.Name) and isinstance(n.ctx,ast.Load)} nonlocal_names = {name for n in self.function_nodes(node) if isinstance(n,ast.Nonlocal) for name in n.names} assigned = {n.id for n in self.function_nodes(node) if isinstance(n,ast.Name) and isinstance(n.ctx,ast.Store)} - nonlocal_names inherited = {name:source for name,source in parent_declarations.items() if name in (used|nonlocal_names) and name not in assigned and name not in parameters} - node.body = self.body(node.body,parameters=parameters,returns=returns,inherited=inherited) + node.body = self.body(node.body,parameters=parameters,returns=returns, + inherited=inherited,external_scope=external_scope) if isinstance(node,ast.AsyncFunctionDef) and is_generator: node.body.append(ast.copy_location(ast.Expr(helper('returned',ast.Constant(None))),node)) else: node.body.append(ast.copy_location(ast.Return(helper('returned',ast.Constant(None))),node)) - initial = next(i for i,n in enumerate(node.body) if isinstance(n,ast.Assign) and any(isinstance(t,ast.Name) and t.id == SCOPE for t in n.targets)) + initial = next(i for i,n in enumerate(node.body) if getattr(n,'_aiython_scope_initializer',False)) handler = ast.ExceptHandler(ast.Attribute(ast.Name(RUNTIME_NAME,ast.Load()),'error_type',ast.Load()),None, [ast.Expr(helper('aborted')),ast.Raise()]) - guarded = ast.Try(node.body[initial+1:],[handler],[],[ast.Expr(helper('leaving'))]) + final = ast.Expr(helper('leaving')) + if external_scope: + final = ast.Try([final],[],[],[ast.Expr(helper('exit_scope'))]) + guarded = ast.Try(node.body[initial+1:],[handler],[],[final]) guarded._aiython_type_guard = True ast.copy_location(guarded,node) node.body = node.body[:initial+1] + [guarded] @@ -414,10 +431,10 @@ def visit_ClassDef(self,node): node.decorator_list.insert(0, ast.Attribute(ast.Attribute(ast.Name(RUNTIME_NAME,ast.Load()),'types',ast.Load()),'register_class',ast.Load())) previous = self.function self.function = False - body = self.body(node.body,class_scope=True) + body = self.body(node.body,external_scope=True) header = body[:1] if isinstance(body[0],ast.Expr) and isinstance(body[0].value,ast.Constant) and isinstance(body[0].value.value,str) else [] enter, *statements = body[len(header):] - exit_call = ast.copy_location(ast.Expr(helper('exit_class_scope')), node) + exit_call = ast.copy_location(ast.Expr(helper('exit_scope')), node) guard = ast.copy_location(ast.Try(statements, [], [], [exit_call]), node) guard._aiython_class_guard = True node.body = header + [enter, guard] @@ -452,12 +469,14 @@ def visit_Assign(self,node): node.value = self.visit(node.value) if len(node.targets) == 1 and isinstance(node.targets[0],ast.Attribute): target = node.targets[0] - self.temp += 1 - temporary = f'__aiython_typed_value_{self.temp}' - store = ast.copy_location(ast.Assign([ast.Name(temporary,ast.Store())],node.value),node) - assign = ast.copy_location(ast.Expr(helper('assign_attribute',self.visit(target.value),ast.Constant(target.attr),ast.Name(temporary,ast.Load()))),node) - clean = ast.copy_location(ast.Delete([ast.Name(temporary,ast.Del())]),node) - return [store,assign,clean] + assign = ast.Call( + ast.Attribute(ast.Attribute(ast.Name(RUNTIME_NAME,ast.Load()),'types',ast.Load()), + 'assign_attribute',ast.Load()), [], [ + ast.keyword(arg='value',value=node.value), + ast.keyword(arg='owner',value=self.visit(target.value)), + ast.keyword(arg='name',value=ast.Constant(target.attr)), + ]) + return ast.copy_location(ast.Expr(assign),node) for target in node.targets: if isinstance(target,ast.Name) and not target.id.startswith('__aiython_'): node.value = helper('assignment',node.value,ast.Constant(target.id)) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index c0a0133..e2e8caa 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -5,11 +5,61 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch + +from aiython.cli import run_script class PythonCompatibilityTests(unittest.TestCase): + def test_entry_annotations_follow_python_version(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'main.py' + path.write_text('') + for version, present in (((3, 13), True), ((3, 14), False)): + with self.subTest(version=version), patch('aiython.cli.sys.version_info', version): + namespace = run_script(path) + self.assertEqual('__annotations__' in namespace, present) + + def test_symlink_script_metadata_matches_cpython(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'source').mkdir() + (root / 'links').mkdir() + target = root / 'source' / 'target.py' + target.write_text('import inspect, sys\n' + 'print(sys.argv[0], __file__, __loader__.path, ' + 'inspect.currentframe().f_code.co_filename, sys.path[0], sep="\\n")\n') + (root / 'links' / 'alias.py').symlink_to(target) + python = subprocess.run([sys.executable, 'links/alias.py'], cwd=root, + capture_output=True, text=True) + aiython = subprocess.run([sys.executable, '-m', 'aiython', 'links/alias.py'], + cwd=root, capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + + def test_relative_script_argument_matches_cpython(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'main.py' + path.write_text('import sys\nprint(sys.argv[0])\nprint(__file__)\n') + python = subprocess.run([sys.executable, 'main.py'], cwd=directory, + capture_output=True, text=True) + aiython = subprocess.run([sys.executable, '-m', 'aiython', 'main.py'], + cwd=directory, capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + def test_language_constructs_match_cpython(self): cases = { + 'empty script': '', + 'entry builtins module': '''import builtins +print(type(__builtins__).__name__, __builtins__ is builtins) +''', + 'entry module metadata': '''print(type(__loader__).__name__, __loader__.name, __loader__.path == __file__) +print('__annotations__' in globals(), '__annotate__' in globals()) +''', + 'module annotation metadata': '''value: int = 2 +print('__annotations__' in globals(), '__annotate__' in globals()) +''', 'metaclass namespace': '''class Meta(type): def __new__(meta, name, bases, namespace): print(sorted(namespace)) @@ -35,6 +85,59 @@ class Example(metaclass=Meta): 'class locals': '''class Example: snapshot = sorted(locals()) print(Example.snapshot) +''', + 'function locals': '''def run(value): + result = value + 1 + print(sorted(locals())) + return result +print(run(2)) +''', + 'function frame locals': '''import inspect +def run(value): + result = value + 1 + print(sorted(inspect.currentframe().f_locals)) + return result +print(run(2)) +''', + 'attribute assignment locals and order': '''events = [] +class Box: + def __setattr__(self, name, value): + events.append(('set', name, value)) + super().__setattr__(name, value) +box = Box() +def produce(): + events.append('value') + return 3 +def pick(): + events.append('target') + return box +def run(): + pick().value = produce() + print(run.__code__.co_varnames, sorted(locals())) +run() +print(events) +''', + 'async function locals': '''import asyncio +async def run(value): + result = value + 1 + await asyncio.sleep(0) + print(sorted(locals())) + return result +print(asyncio.run(run(2))) +''', + 'recursive function scopes': '''def factorial(value: int) -> int: + if value == 0: + return 1 + return value * factorial(value - 1) +print(factorial(6)) +''', + 'concurrent async function scopes': '''import asyncio +async def worker(value: int) -> tuple[int, list[str]]: + await asyncio.sleep(0) + return value, sorted(locals()) +async def run(): + print(await asyncio.gather(worker(1), worker(2))) +asyncio.run(run()) ''', 'failed class body leaves no scope': '''try: class Broken: From abb1d50f7d34f32d84febb7f6f9cfc06556fb735 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 11:24:29 +0700 Subject: [PATCH 03/72] Hide generator type scopes from Python locals --- src/aiython/runtime.py | 3 -- src/aiython/typed_runtime.py | 49 +++++++++++++++++++----------- tests/test_python_compatibility.py | 35 +++++++++++++++++++++ 3 files changed, 66 insertions(+), 21 deletions(-) diff --git a/src/aiython/runtime.py b/src/aiython/runtime.py index a18edd2..ff0e0bb 100644 --- a/src/aiython/runtime.py +++ b/src/aiython/runtime.py @@ -584,9 +584,6 @@ def __init__(self, runtime, unit): @staticmethod def generated(node): - if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id.startswith('__aiython_') - for t in node.targets): - return True if not isinstance(node, ast.Expr) or not isinstance(node.value, ast.Call): return False func = node.value.func diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index f1a39d3..0e8b87f 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -13,11 +13,16 @@ SCOPE = '__aiython_type_scope__' _FRAME_SCOPES = ContextVar('aiython_frame_scopes', default=()) +_GENERATOR_SCOPES = {} +_GENERATOR_SCOPES_LOCK = threading.Lock() def frame_scope(frame): - return next((scope for active, scope in reversed(_FRAME_SCOPES.get()) - if active is frame), None) + for active, scope in reversed(_FRAME_SCOPES.get()): + if active is frame: + return scope + with _GENERATOR_SCOPES_LOCK: + return _GENERATOR_SCOPES.get(id(frame)) @dataclass @@ -79,13 +84,6 @@ def _initialize(self, frame, declarations, parameters, returns): scope.returned = self.contract(returns,namespace) return scope - def initialize(self, declarations, parameters=None, returns=None): - frame = inspect.currentframe().f_back - try: - return self._initialize(frame, declarations, parameters, returns) - finally: - del frame - def enter_scope(self, declarations, parameters=None, returns=None): frame = inspect.currentframe().f_back try: @@ -98,6 +96,23 @@ def exit_scope(self): stack = _FRAME_SCOPES.get() _FRAME_SCOPES.set(stack[:-1]) + def enter_generator_scope(self, declarations, parameters=None, returns=None): + frame = inspect.currentframe().f_back + try: + scope = self._initialize(frame, declarations, parameters, returns) + with _GENERATOR_SCOPES_LOCK: + _GENERATOR_SCOPES[id(frame)] = scope + finally: + del frame + + def exit_generator_scope(self): + frame = inspect.currentframe().f_back + try: + with _GENERATOR_SCOPES_LOCK: + _GENERATOR_SCOPES.pop(id(frame), None) + finally: + del frame + @staticmethod def scopes(frame): local = frame_scope(frame) or frame.f_locals.get(SCOPE) @@ -335,7 +350,7 @@ def collect(node): for statement in body: collect(statement) return result - def body(self,body,*,parameters=None,returns=None,initialize=True,inherited=None,external_scope=False): + def body(self,body,*,parameters=None,returns=None,initialize=True,inherited=None,generator_scope=False): previous = self.declarations self.declarations = dict(inherited or {}) | self.declarations_in(body) output = [] @@ -345,8 +360,8 @@ def body(self,body,*,parameters=None,returns=None,initialize=True,inherited=None while body and isinstance(body[0],ast.ImportFrom) and body[0].module == '__future__': header.append(body[0]); body = body[1:] if initialize: - initial = (ast.Expr(helper('enter_scope',literal(self.declarations),literal(parameters),literal(returns))) if external_scope else - ast.Assign([ast.Name(SCOPE,ast.Store())],helper('initialize',literal(self.declarations),literal(parameters),literal(returns)))) + initial = ast.Expr(helper('enter_generator_scope' if generator_scope else 'enter_scope', + literal(self.declarations),literal(parameters),literal(returns))) ast.copy_location(initial,body[0] if body else header[-1] if header else ast.Constant(None,lineno=1,col_offset=0)) initial._aiython_scope_initializer = True output.append(initial) @@ -370,7 +385,7 @@ def visit_Module(self,node): if self.snippet: node.body = self.body(node.body,initialize=False) return node - body = self.body(node.body,external_scope=True) + body = self.body(node.body) initial = next(i for i,item in enumerate(body) if getattr(item,'_aiython_scope_initializer',False)) header, enter, statements = body[:initial], body[initial], body[initial+1:] exit_call = ast.copy_location(ast.Expr(helper('exit_scope')), enter) @@ -394,13 +409,12 @@ def visit_FunctionDef(self,node): returns = ast.unparse(node.returns) if node.returns else None # Generator annotations require yield/send checks, not return-only checks. is_generator = any(isinstance(n,(ast.Yield,ast.YieldFrom)) for n in self.function_nodes(node)) - external_scope = not is_generator used = {n.id for n in self.function_nodes(node) if isinstance(n,ast.Name) and isinstance(n.ctx,ast.Load)} nonlocal_names = {name for n in self.function_nodes(node) if isinstance(n,ast.Nonlocal) for name in n.names} assigned = {n.id for n in self.function_nodes(node) if isinstance(n,ast.Name) and isinstance(n.ctx,ast.Store)} - nonlocal_names inherited = {name:source for name,source in parent_declarations.items() if name in (used|nonlocal_names) and name not in assigned and name not in parameters} node.body = self.body(node.body,parameters=parameters,returns=returns, - inherited=inherited,external_scope=external_scope) + inherited=inherited,generator_scope=is_generator) if isinstance(node,ast.AsyncFunctionDef) and is_generator: node.body.append(ast.copy_location(ast.Expr(helper('returned',ast.Constant(None))),node)) else: @@ -409,8 +423,7 @@ def visit_FunctionDef(self,node): handler = ast.ExceptHandler(ast.Attribute(ast.Name(RUNTIME_NAME,ast.Load()),'error_type',ast.Load()),None, [ast.Expr(helper('aborted')),ast.Raise()]) final = ast.Expr(helper('leaving')) - if external_scope: - final = ast.Try([final],[],[],[ast.Expr(helper('exit_scope'))]) + final = ast.Try([final],[],[],[ast.Expr(helper('exit_generator_scope' if is_generator else 'exit_scope'))]) guarded = ast.Try(node.body[initial+1:],[handler],[],[final]) guarded._aiython_type_guard = True ast.copy_location(guarded,node) @@ -431,7 +444,7 @@ def visit_ClassDef(self,node): node.decorator_list.insert(0, ast.Attribute(ast.Attribute(ast.Name(RUNTIME_NAME,ast.Load()),'types',ast.Load()),'register_class',ast.Load())) previous = self.function self.function = False - body = self.body(node.body,external_scope=True) + body = self.body(node.body) header = body[:1] if isinstance(body[0],ast.Expr) and isinstance(body[0].value,ast.Constant) and isinstance(body[0].value.value,str) else [] enter, *statements = body[len(header):] exit_call = ast.copy_location(ast.Expr(helper('exit_scope')), node) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index e2e8caa..4889a1b 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -190,6 +190,33 @@ def outer(): print(next(item)) try: item.send(7) except StopIteration: pass +''', + 'generator locals': '''import inspect +def generate(value): + yield sorted(locals()) + yield sorted(inspect.currentframe().f_locals) + yield generate.__code__.co_varnames +item = generate(3) +print(next(item), next(item), next(item)) +''', + 'generator collected after suspension': '''import gc, weakref +def generate(): + yield 1 +item = generate() +reference = weakref.ref(item) +next(item) +del item +gc.collect() +print(reference() is None) +''', + 'generator resumed on another thread': '''from concurrent.futures import ThreadPoolExecutor +def generate(value: int): + yield value + yield value + 1 +item = generate(2) +print(next(item)) +with ThreadPoolExecutor(max_workers=1) as pool: + print(pool.submit(next, item).result()) ''', 'async generator': '''import asyncio async def numbers(): @@ -198,6 +225,14 @@ async def numbers(): async def run(): print([number async for number in numbers()]) asyncio.run(run()) +''', + 'async generator locals': '''import asyncio, inspect +async def generate(value): + yield sorted(locals()) + yield sorted(inspect.currentframe().f_locals) +async def run(): + print([item async for item in generate(3)]) +asyncio.run(run()) ''', } with tempfile.TemporaryDirectory() as directory: From e4a016a28ec04327b803c8960e20fc94c43d6585 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 11:46:03 +0700 Subject: [PATCH 04/72] Compile docstring-only classes and check stdlib syntax --- .github/workflows/tests.yml | 2 + benchmarks/README.md | 9 +++++ benchmarks/stdlib_syntax.py | 59 ++++++++++++++++++++++++++++++ src/aiython/typed_runtime.py | 9 +++-- tests/test_python_compatibility.py | 4 ++ 5 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 benchmarks/stdlib_syntax.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d28d7d3..2cae1e4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,6 +47,8 @@ jobs: uv run --locked --python ${{ matrix.python-version }} --extra a2a python -m coverage report uv run --locked --python ${{ matrix.python-version }} --extra a2a python -m coverage json -o coverage.json uv run --locked --python ${{ matrix.python-version }} python scripts/coverage_summary.py + - name: Compile standard library syntax + run: uv run --locked --python ${{ matrix.python-version }} python benchmarks/stdlib_syntax.py - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-${{ matrix.python-version }} diff --git a/benchmarks/README.md b/benchmarks/README.md index 6beb66e..d5f27f7 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -86,3 +86,12 @@ For live task comparisons against agent baselines, keep acceptance tests, input snapshots, model settings, tool permissions and budgets fixed; count failures as well as successes. Follow the [evaluation guide](../docs/performance-evaluation.md) and collect run wall time, human intervention, token usage and cost separately. + +## Standard library syntax check + +Run `uv run --locked --python 3.13 python benchmarks/stdlib_syntax.py` from the +repository root, changing `--python` to test another supported version. The +checker compares CPython compilation with Aiython's source transformation for +standard library modules. It processes files in batches to bound memory use. +Passing this check establishes syntax transformation only; it does not execute +the modules or establish behavioral compatibility. diff --git a/benchmarks/stdlib_syntax.py b/benchmarks/stdlib_syntax.py new file mode 100644 index 0000000..bcc1100 --- /dev/null +++ b/benchmarks/stdlib_syntax.py @@ -0,0 +1,59 @@ +"""Check that Aiython compiles the current interpreter's standard library. + +Run: uv run python benchmarks/stdlib_syntax.py +This checks syntax transformation only; it does not execute the modules. +""" + +import argparse +import gc +from pathlib import Path +import sysconfig +import tokenize + +from aiython.models import ProfileConfig, ResolvedConfig +from aiython.runtime import Runtime + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--stdlib', type=Path, default=Path(sysconfig.get_path('stdlib'))) + parser.add_argument('--batch-size', type=int, default=40) + args = parser.parse_args() + if args.batch_size < 1: + parser.error('--batch-size must be positive') + + root = args.stdlib.resolve() + paths = sorted(path for path in root.rglob('*.py') + if not {'site-packages', 'test', 'tests'} & set(path.parts)) + profile = ProfileConfig('offline', 'fake', 'offline') + config = ResolvedConfig(None, root, 'offline', {'offline': profile}) + failures = [] + runtime = None + for index, path in enumerate(paths): + if index % args.batch_size == 0: + if runtime is not None: + runtime.capabilities.close() + del runtime + gc.collect() + runtime = Runtime(config) + try: + with tokenize.open(path) as file: + source = file.read() + compile(source, str(path), 'exec', dont_inherit=True) + runtime.compile_source(source, str(path)) + except Exception as error: + failures.append((path.relative_to(root), error)) + if (index + 1) % 100 == 0: + print(f'Checked {index + 1}/{len(paths)}; failures: {len(failures)}', flush=True) + if runtime is not None: + runtime.capabilities.close() + print(f'Checked {len(paths)} standard library files; failures: {len(failures)}') + for path, error in failures[:30]: + print(f'{path}: {type(error).__name__}: {error}') + if len(failures) > 30: + print(f'... and {len(failures) - 30} more failures') + return bool(failures) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 0e8b87f..f2014ee 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -448,9 +448,12 @@ def visit_ClassDef(self,node): header = body[:1] if isinstance(body[0],ast.Expr) and isinstance(body[0].value,ast.Constant) and isinstance(body[0].value.value,str) else [] enter, *statements = body[len(header):] exit_call = ast.copy_location(ast.Expr(helper('exit_scope')), node) - guard = ast.copy_location(ast.Try(statements, [], [], [exit_call]), node) - guard._aiython_class_guard = True - node.body = header + [enter, guard] + if statements: + guard = ast.copy_location(ast.Try(statements, [], [], [exit_call]), node) + guard._aiython_class_guard = True + node.body = header + [enter, guard] + else: + node.body = header + [enter, exit_call] self.function = previous return node diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 4889a1b..32870b0 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -85,6 +85,10 @@ class Example(metaclass=Meta): 'class locals': '''class Example: snapshot = sorted(locals()) print(Example.snapshot) +''', + 'class with only a docstring': '''class Example: + "documentation" +print(Example.__doc__, sorted(Example.__dict__)) ''', 'function locals': '''def run(value): result = value + 1 From 8b84cfb1f7745701090663bdaa3f0b5a10061f3f Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 12:00:14 +0700 Subject: [PATCH 05/72] Preserve entry script state through process shutdown --- src/aiython/cli.py | 52 +++++++++++++++++++----------- tests/test_python_compatibility.py | 26 +++++++++++++++ 2 files changed, 59 insertions(+), 19 deletions(-) diff --git a/src/aiython/cli.py b/src/aiython/cli.py index 33953bd..a942f1d 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import atexit import builtins import importlib.abc import importlib.machinery @@ -58,7 +59,7 @@ def find_spec(self, fullname, path=None, target=None): def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, stats=False, trace_plan=False, - config_path=None, profile=None, force_profile=None): + config_path=None, profile=None, force_profile=None, restore_state=True): started = perf_counter() argv0 = str(path) display_path = path.absolute() @@ -81,6 +82,18 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta old_spawn_entry = os.environ.get("AIYTHON_SPAWN_ENTRY") finder = ProjectFinder(runtime) execution_started = None + def finish(): + execution_seconds = perf_counter() - execution_started if execution_started is not None else 0 + runtime.capabilities.close() + if stats: + runtime.stats.run = {'total_seconds': perf_counter() - started, + 'config_seconds': config_seconds, + 'execution_seconds': execution_seconds} + runtime.stats.report() + if not restore_state: + # Register before user code so its atexit callbacks run while the + # script's argv, import path, and runtime are still available. + atexit.register(finish) try: sys.modules["__main__"] = module sys.argv = [argv0, *arguments] @@ -91,24 +104,24 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta exec(code, module.__dict__) return module.__dict__ finally: - execution_seconds = perf_counter() - execution_started if execution_started is not None else 0 - sys.argv = old_argv - sys.path[:] = old_path - sys.meta_path.remove(finder) - if old_spawn_entry is None: - os.environ.pop("AIYTHON_SPAWN_ENTRY", None) - else: - os.environ["AIYTHON_SPAWN_ENTRY"] = old_spawn_entry - if old_main is not None: - sys.modules["__main__"] = old_main + if restore_state: + sys.argv = old_argv + sys.path[:] = old_path + sys.meta_path.remove(finder) + if old_spawn_entry is None: + os.environ.pop("AIYTHON_SPAWN_ENTRY", None) + else: + os.environ["AIYTHON_SPAWN_ENTRY"] = old_spawn_entry + if old_main is not None: + sys.modules["__main__"] = old_main + else: + sys.modules.pop("__main__", None) + finish() else: - sys.modules.pop("__main__", None) - runtime.capabilities.close() - if stats: - runtime.stats.run = {'total_seconds': perf_counter() - started, - 'config_seconds': config_seconds, - 'execution_seconds': execution_seconds} - runtime.stats.report() + # CPython removes these entry-script attributes before waiting for + # non-daemon threads and running atexit callbacks. + module.__dict__.pop("__file__", None) + module.__dict__.pop("__cached__", None) def parser(): @@ -185,7 +198,8 @@ def main(argv=None): for c in runtime.checkpoints.values()]}, ensure_ascii=False, indent=2)) return run_script(path, args.args, config_path=args.config, profile=args.profile, - force_profile=args.force_profile, stats=args.stats, trace_plan=args.trace_plan) + force_profile=args.force_profile, stats=args.stats, trace_plan=args.trace_plan, + restore_state=argv is not None) except AiythonError as exc: # Keep the original runtime cause visible without leaking provider internals. if exc.__cause__: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 32870b0..2175fd2 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -11,6 +11,18 @@ class PythonCompatibilityTests(unittest.TestCase): + def test_embedded_run_restores_host_process_state(self): + original_argv = sys.argv + original_path = sys.path[:] + original_main = sys.modules.get('__main__') + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'main.py' + path.write_text('value = 2\n') + run_script(path) + self.assertIs(sys.argv, original_argv) + self.assertEqual(sys.path, original_path) + self.assertIs(sys.modules.get('__main__'), original_main) + def test_entry_annotations_follow_python_version(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / 'main.py' @@ -56,6 +68,20 @@ def test_language_constructs_match_cpython(self): ''', 'entry module metadata': '''print(type(__loader__).__name__, __loader__.name, __loader__.path == __file__) print('__annotations__' in globals(), '__annotate__' in globals()) +''', + 'atexit script state': '''import atexit, sys +def report(): + main = sys.modules.get('__main__') + print(sys.argv[0], sys.path[0], getattr(main, '__file__', None), + '__cached__' in globals(), sep=' | ') +atexit.register(report) +''', + 'worker script state': '''import sys, threading, time +def worker(): + time.sleep(0.05) + main = sys.modules.get('__main__') + print(sys.argv[0], sys.path[0], getattr(main, '__file__', None), sep=' | ') +threading.Thread(target=worker).start() ''', 'module annotation metadata': '''value: int = 2 print('__annotations__' in globals(), '__annotate__' in globals()) From 787f999a1d9a832d85bd1f37012e3386af369264 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 12:11:04 +0700 Subject: [PATCH 06/72] Match Python original arguments and safe import path --- src/aiython/cli.py | 13 ++++++++-- tests/test_python_compatibility.py | 40 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/aiython/cli.py b/src/aiython/cli.py index a942f1d..33005d9 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -78,7 +78,13 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta if sys.version_info < (3, 14): module.__annotations__ = {} old_main = sys.modules.get("__main__") - old_argv, old_path = sys.argv, sys.path[:] + old_argv, old_orig_argv, old_path = sys.argv, sys.orig_argv, sys.path[:] + interpreter_args = [sys.executable] + script_position = old_orig_argv.index(argv0) if argv0 in old_orig_argv else len(old_orig_argv) + for index in range(min(script_position, len(old_orig_argv) - 1)): + if old_orig_argv[index] == "-m" and old_orig_argv[index + 1] in ("aiython", "aiython.__main__"): + interpreter_args = old_orig_argv[:index] + break old_spawn_entry = os.environ.get("AIYTHON_SPAWN_ENTRY") finder = ProjectFinder(runtime) execution_started = None @@ -97,7 +103,9 @@ def finish(): try: sys.modules["__main__"] = module sys.argv = [argv0, *arguments] - sys.path.insert(0, str(path.parent)) + sys.orig_argv = [*interpreter_args, argv0, *arguments] + if not sys.flags.safe_path: + sys.path.insert(0, str(path.parent)) sys.meta_path.insert(0, finder) os.environ["AIYTHON_SPAWN_ENTRY"] = str(display_path) execution_started = perf_counter() @@ -106,6 +114,7 @@ def finish(): finally: if restore_state: sys.argv = old_argv + sys.orig_argv = old_orig_argv sys.path[:] = old_path sys.meta_path.remove(finder) if old_spawn_entry is None: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 2175fd2..25d4a3a 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -11,8 +11,42 @@ class PythonCompatibilityTests(unittest.TestCase): + def test_safe_path_modes_match_cpython(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'main.py' + (Path(directory) / 'sibling.py').write_text('value = 2\n') + path.write_text('''import sys +print(sys.flags.safe_path, sys.path[:2], sys.orig_argv) +try: + import sibling +except ModuleNotFoundError: + print('sibling unavailable') +else: + print(sibling.value) +''') + for flag in ('-I', '-P'): + with self.subTest(flag=flag): + python = subprocess.run([sys.executable, flag, str(path)], + capture_output=True, text=True) + aiython = subprocess.run([sys.executable, flag, '-m', 'aiython', str(path)], + capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + + def test_original_arguments_preserve_interpreter_flags(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'main.py' + path.write_text('import sys\nprint(sys.orig_argv)\n') + python = subprocess.run([sys.executable, '-B', str(path), 'one'], + capture_output=True, text=True) + aiython = subprocess.run([sys.executable, '-B', '-m', 'aiython', str(path), 'one'], + capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + def test_embedded_run_restores_host_process_state(self): original_argv = sys.argv + original_orig_argv = sys.orig_argv original_path = sys.path[:] original_main = sys.modules.get('__main__') with tempfile.TemporaryDirectory() as directory: @@ -20,6 +54,7 @@ def test_embedded_run_restores_host_process_state(self): path.write_text('value = 2\n') run_script(path) self.assertIs(sys.argv, original_argv) + self.assertIs(sys.orig_argv, original_orig_argv) self.assertEqual(sys.path, original_path) self.assertIs(sys.modules.get('__main__'), original_main) @@ -68,12 +103,17 @@ def test_language_constructs_match_cpython(self): ''', 'entry module metadata': '''print(type(__loader__).__name__, __loader__.name, __loader__.path == __file__) print('__annotations__' in globals(), '__annotate__' in globals()) +''', + 'original argument vector': '''import sys +print(sys.argv) +print(sys.orig_argv) ''', 'atexit script state': '''import atexit, sys def report(): main = sys.modules.get('__main__') print(sys.argv[0], sys.path[0], getattr(main, '__file__', None), '__cached__' in globals(), sep=' | ') + print(sys.orig_argv) atexit.register(report) ''', 'worker script state': '''import sys, threading, time From 7da0b8d71536368c780a9fe865a748b6cf263379 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 12:40:16 +0700 Subject: [PATCH 07/72] Run Python modules with CPython-compatible startup --- README.md | 2 + README.pypi.md | 2 + docs/configuration.md | 2 + src/aiython/cli.py | 127 ++++++++++++++++------ tests/test_python_compatibility.py | 162 ++++++++++++++++++++++++++++- 5 files changed, 264 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 4b78072..57a9c52 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ aiython tickets.py Use a project environment when your script imports other project dependencies; a uv tool has its own isolated environment. [Setup and configuration](docs/configuration.md) covers profiles, custom endpoints, and capability routes. +Run an importable module or package with `aiython -m package.module` or `aiython -m package`; pass its arguments after the module name. + Ordinary Python runs without loading LiteLLM or contacting a provider. Reasoning calls use the [LiteLLM Python SDK](https://docs.litellm.ai/docs/) in process; no proxy service is needed. ## The execution boundary diff --git a/README.pypi.md b/README.pypi.md index 927447a..20e132e 100644 --- a/README.pypi.md +++ b/README.pypi.md @@ -48,6 +48,8 @@ Python owns statement order, loops, assignments, and side effects. AI works at t Use `aiython --help` to see the CLI, `aiython config show` to inspect configuration, and `aiython --stats tickets.py` to see model calls and timings. Additional routes for vision, documents, embeddings, reranking, speech, images, and video are configured only when needed. +Run an importable module or package with `aiython -m package.module` or `aiython -m package`; pass its arguments after the module name. + Aiython is not a sandbox. Frame tools can use `eval` and `exec` with your process permissions, and relevant source or object data may be sent to your configured provider. Use trusted code and review provider data handling. Licensed under MIT. diff --git a/docs/configuration.md b/docs/configuration.md index 9f24684..e3d4245 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -64,6 +64,8 @@ Model IDs use LiteLLM syntax. A route can also be `{ model = "...", api_base = " Run `aiython config show --script PATH` to inspect resolved settings. `--config` chooses an exact file. `--profile` selects a default named profile; `--force-profile` overrides source directives. CLI options for script execution go before the script path, while later arguments go to the program. +Run an importable module or package with `aiython -m package.module` or `aiython -m package`. Put Aiython options before `-m`; arguments after the module name go to that module. Packages need a `__main__.py`, as with `python -m`. Source files get Aiython's source transformation; modules available only as bytecode run as Python bytecode. Resolving a dotted name for `--explain -m` imports its parent package. + Version 1 and 2 configs are rejected with migration guidance and never overwritten. Move the old reasoning model to top-level `model`, put other routes under `[capabilities]`, and replace direct keys with `api_key_env` plus a private env file. Runtime state starts fresh in `.aiython/runtime-v3.sqlite`; older databases remain untouched. ## Deadlines and jobs diff --git a/src/aiython/cli.py b/src/aiython/cli.py index 33005d9..3d187f4 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -8,6 +8,7 @@ import importlib.metadata import json import os +import runpy import sys import tokenize import types @@ -58,33 +59,71 @@ def find_spec(self, fullname, path=None, target=None): return spec +def interpreter_arguments(): + original = sys.orig_argv + for index in range(len(original) - 1): + if original[index] == "-m" and original[index + 1] in ("aiython", "aiython.__main__"): + return original[:index] + return [sys.executable] + + +def module_source(name, arguments=()): + old_main = sys.modules.get("__main__") + old_argv, old_orig_argv = sys.argv, sys.orig_argv + interpreter_args = interpreter_arguments() + initial_main = types.ModuleType("__main__") + initial_main.__loader__ = importlib.machinery.BuiltinImporter + initial_main.__builtins__ = builtins + if sys.version_info < (3, 14): + initial_main.__annotations__ = {} + try: + sys.modules["__main__"] = initial_main + sys.argv = ["-m", *arguments] + sys.orig_argv = [*interpreter_args, "-m", name, *arguments] + if not sys.flags.safe_path: + sys.path[:1] = [str(Path.cwd())] + actual_name, spec, code = runpy._get_module_details(name) + except ImportError as exc: + raise AiythonError(str(exc)) from None + finally: + sys.argv = old_argv + sys.orig_argv = old_orig_argv + if old_main is not None: + sys.modules["__main__"] = old_main + else: + sys.modules.pop("__main__", None) + loader = spec.loader + source = getattr(loader, "get_source", lambda _: None)(actual_name) + return spec, source, code, initial_main + + def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, stats=False, trace_plan=False, - config_path=None, profile=None, force_profile=None, restore_state=True): + config_path=None, profile=None, force_profile=None, restore_state=True, + source=None, module_spec=None, module_invocation=None, compiled_code=None, initial_main=None): started = perf_counter() - argv0 = str(path) - display_path = path.absolute() + argv0 = str(path) if module_spec is None else module_spec.origin + display_path = path.absolute() if module_spec is None else path path = path.resolve() - config = config or resolve(path, config_path=config_path, profile=profile, force_profile=force_profile) + config_source = Path.cwd() / "__main__.py" if module_spec else path + config = config or resolve(config_source, config_path=config_path, profile=profile, force_profile=force_profile) config_seconds = perf_counter() - started runtime = Runtime(config, agent_factory=agent_factory, stats=stats, trace_plan=trace_plan) - code = runtime.compile_source(read_source(path), str(display_path), entry=True) - module = types.ModuleType("__main__") - module.__dict__.update({"__file__": str(display_path), "__package__": None, - "__spec__": None, - "__cached__": None, - "__loader__": importlib.machinery.SourceFileLoader("__main__", str(display_path)), + code = (compiled_code if compiled_code is not None else + runtime.compile_source(read_source(path) if source is None else source, str(display_path), entry=True)) + module = initial_main or types.ModuleType("__main__") + module.__dict__.update({"__file__": str(display_path) if module_spec is None else module_spec.origin, + "__package__": module_spec.parent if module_spec else None, + "__spec__": module_spec, + "__cached__": module_spec.cached if module_spec else None, + "__loader__": module_spec.loader if module_spec else + importlib.machinery.SourceFileLoader("__main__", str(display_path)), RUNTIME_NAME: runtime, "__builtins__": builtins}) if sys.version_info < (3, 14): - module.__annotations__ = {} + module.__dict__.setdefault("__annotations__", {}) old_main = sys.modules.get("__main__") old_argv, old_orig_argv, old_path = sys.argv, sys.orig_argv, sys.path[:] - interpreter_args = [sys.executable] - script_position = old_orig_argv.index(argv0) if argv0 in old_orig_argv else len(old_orig_argv) - for index in range(min(script_position, len(old_orig_argv) - 1)): - if old_orig_argv[index] == "-m" and old_orig_argv[index + 1] in ("aiython", "aiython.__main__"): - interpreter_args = old_orig_argv[:index] - break + interpreter_args = interpreter_arguments() old_spawn_entry = os.environ.get("AIYTHON_SPAWN_ENTRY") finder = ProjectFinder(runtime) execution_started = None @@ -103,8 +142,9 @@ def finish(): try: sys.modules["__main__"] = module sys.argv = [argv0, *arguments] - sys.orig_argv = [*interpreter_args, argv0, *arguments] - if not sys.flags.safe_path: + sys.orig_argv = ([*interpreter_args, "-m", module_invocation, *arguments] if module_spec else + [*interpreter_args, argv0, *arguments]) + if module_spec is None and not sys.flags.safe_path: sys.path.insert(0, str(path.parent)) sys.meta_path.insert(0, finder) os.environ["AIYTHON_SPAWN_ENTRY"] = str(display_path) @@ -129,14 +169,15 @@ def finish(): else: # CPython removes these entry-script attributes before waiting for # non-daemon threads and running atexit callbacks. - module.__dict__.pop("__file__", None) - module.__dict__.pop("__cached__", None) + if module_spec is None: + module.__dict__.pop("__file__", None) + module.__dict__.pop("__cached__", None) def parser(): result = argparse.ArgumentParser( prog="aiython", description="Python with project-scoped AI execution", - epilog="Get started: aiython setup; aiython --explain script.py; aiython script.py. " + epilog="Get started: aiython setup; aiython --explain script.py; aiython script.py; aiython -m package.module. " "Other commands: aiython config show; aiython jobs list.") result.add_argument("--version", action="version", version=f"%(prog)s {importlib.metadata.version('aiython')}") @@ -147,13 +188,16 @@ def parser(): result.add_argument("--explain", action="store_true", help="Show blocks/checkpoints without executing code") result.add_argument("--trace-plan", action="store_true", help="Trace actual capability routes, cache and timings") result.add_argument("--stats", action="store_true", help="Report model calls, tools, request bytes and timings on stderr") - result.add_argument("script") + result.add_argument("-m", "--module", dest="module_args", nargs=argparse.REMAINDER, + help="Run a Python module as __main__") + result.add_argument("script", nargs="?") result.add_argument("args", nargs=argparse.REMAINDER) return result def main(argv=None): arguments = list(sys.argv[1:] if argv is None else argv) + original_path = sys.path[:] if argv is not None else None try: if not arguments: parser().print_help() @@ -191,12 +235,30 @@ def main(argv=None): args = command.parse_args(arguments[2:]) print(json.dumps(describe(resolve(Path(args.script), config_path=args.config)), indent=2, ensure_ascii=False)) return - args = parser().parse_args(arguments) - path = Path(args.script) + argument_parser = parser() + args = argument_parser.parse_args(arguments) + module_spec = source = module_invocation = compiled_code = initial_main = None + if args.module_args is not None: + if not args.module_args: + argument_parser.error("-m requires a module name") + module_invocation, *script_args = args.module_args + module_spec, source, original_code, initial_main = module_source(module_invocation, script_args) + path = Path(module_spec.origin or original_code.co_filename) + if source is None: + compiled_code = original_code + else: + if args.script is None: + argument_parser.error("a script path or -m module is required") + path = Path(args.script) + script_args = args.args if args.explain: - config = resolve(path, config_path=args.config, profile=args.profile, force_profile=args.force_profile) - path = path.resolve() - unit = parse(read_source(path), str(path)) + if source is None and compiled_code is not None: + raise AiythonError("Cannot explain a module without Python source") + config_source = Path.cwd() / "__main__.py" if module_spec else path + config = resolve(config_source, config_path=args.config, profile=args.profile, + force_profile=args.force_profile) + filename = str(path if module_spec else path.resolve()) + unit = parse(read_source(path.resolve()) if source is None else source, filename) runtime = Runtime(config) runtime.prepare(unit, entry=True) print(json.dumps({"config": describe(config), "blocks": [ @@ -206,9 +268,11 @@ def main(argv=None): "checkpoints": [{"span": vars(c.span), "statement": c.statement} for c in runtime.checkpoints.values()]}, ensure_ascii=False, indent=2)) return - run_script(path, args.args, config_path=args.config, profile=args.profile, + run_script(path, script_args, config_path=args.config, profile=args.profile, force_profile=args.force_profile, stats=args.stats, trace_plan=args.trace_plan, - restore_state=argv is not None) + restore_state=argv is not None, source=source, + module_spec=module_spec, module_invocation=module_invocation, + compiled_code=compiled_code, initial_main=initial_main) except AiythonError as exc: # Keep the original runtime cause visible without leaking provider internals. if exc.__cause__: @@ -216,3 +280,6 @@ def main(argv=None): traceback.print_exception(exc.__cause__) print(f"aiython: {exc}", file=sys.stderr) raise SystemExit(1) from None + finally: + if original_path is not None: + sys.path[:] = original_path diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 25d4a3a..178e39e 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1,16 +1,176 @@ """Compare ordinary Python execution with Aiython in separate processes.""" +import json +import os +import py_compile import subprocess import sys import tempfile import unittest +import zipfile from pathlib import Path from unittest.mock import patch -from aiython.cli import run_script +from aiython.cli import main, module_source, run_script class PythonCompatibilityTests(unittest.TestCase): + def test_module_startup_without_existing_main_restores_modules(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'standalone.py').write_text('value = 1\n') + for version, annotations_present in (((3, 13), True), ((3, 14), False)): + with self.subTest(version=version), patch.dict(sys.modules, {'__main__': None}), \ + patch('pathlib.Path.cwd', return_value=root), patch.object(sys, 'path', sys.path[:]), \ + patch('aiython.cli.sys.version_info', version): + spec, source, code, initial_main = module_source('standalone') + self.assertNotIn('__main__', sys.modules) + self.assertEqual(spec.name, 'standalone') + self.assertEqual(source, 'value = 1\n') + self.assertEqual(code.co_name, '') + self.assertEqual(initial_main.__name__, '__main__') + self.assertEqual('__annotations__' in vars(initial_main), annotations_present) + + def test_safe_path_module_mode_matches_cpython(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'safemodule.py').write_text('import sys\nprint(sys.flags.safe_path, sys.path[:2])\n') + environment = {**os.environ, 'PYTHONPATH': str(root) + os.pathsep + os.environ.get('PYTHONPATH', '')} + python = subprocess.run([sys.executable, '-P', '-m', 'safemodule'], cwd=root, env=environment, + capture_output=True, text=True) + aiython = subprocess.run([sys.executable, '-P', '-m', 'aiython', '-m', 'safemodule'], cwd=root, + env=environment, capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + + def test_parent_package_startup_matches_cpython(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + package = root / 'startup' + package.mkdir() + (package / '__init__.py').write_text('''import sys +initial_main = sys.modules['__main__'] +if hasattr(initial_main, '__annotations__'): + initial_main.__annotations__['from_init'] = int +print(sys.argv, sys.orig_argv, initial_main.__spec__, '__file__' in vars(initial_main)) +''') + (package / 'module.py').write_text('''import sys +from . import initial_main +print(initial_main is sys.modules['__main__'], 'from_init' in globals().get('__annotations__', {}), sys.path[0]) +''') + python = subprocess.run([sys.executable, '-B', '-m', 'startup.module', 'arg'], cwd=root, + capture_output=True, text=True) + aiython = subprocess.run([sys.executable, '-B', '-m', 'aiython', '-m', 'startup.module', 'arg'], + cwd=root, capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + + def test_console_entry_point_finds_current_directory_modules(self): + console = Path(sys.executable).with_name('aiython') + self.assertTrue(console.is_file()) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'localmodule.py').write_text('import sys\nprint(sys.argv, sys.orig_argv, sys.path[0])\n') + python = subprocess.run([sys.executable, '-m', 'localmodule', 'arg'], cwd=root, + capture_output=True, text=True) + aiython = subprocess.run([str(console), '-m', 'localmodule', 'arg'], cwd=root, + capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + + def test_embedded_module_run_restores_host_import_path(self): + original_path = sys.path[:] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'embedded_module.py').write_text('value = 1\n') + with patch('pathlib.Path.cwd', return_value=root): + main(['-m', 'embedded_module']) + self.assertEqual(sys.path, original_path) + + def test_module_and_package_execution_match_cpython(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + package = root / 'example' + package.mkdir() + (package / '__init__.py').write_text('value = 7\n') + source = '''import atexit, sys +from . import value +print(value, __name__, __package__, __spec__.name, __file__, __cached__) +print(sys.argv, sys.orig_argv, sys.path[0]) +atexit.register(lambda: print('EXIT', sys.modules['__main__'].__file__, + sys.modules['__main__'].__cached__)) +''' + (package / 'module.py').write_text(source) + (package / '__main__.py').write_text(source) + for name in ('example.module', 'example'): + with self.subTest(name=name): + python = subprocess.run([sys.executable, '-m', name, 'one', '-x'], cwd=root, + capture_output=True, text=True) + aiython = subprocess.run([sys.executable, '-m', 'aiython', '-m', name, 'one', '-x'], cwd=root, + capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + + def test_zip_and_sourceless_module_execution_match_cpython(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive = root / 'modules.zip' + with zipfile.ZipFile(archive, 'w') as package: + package.writestr('zipmodule.py', + 'import sys\nprint(__name__, __spec__.name, __file__, sys.argv)\n') + source = root / 'sourceless.py' + source.write_text('import sys\nprint(__file__, __cached__, sys.argv, sys.orig_argv)\n') + py_compile.compile(str(source), cfile=str(root / 'sourceless.pyc'), doraise=True) + source.unlink() + environment = {**os.environ, 'PYTHONPATH': str(archive) + os.pathsep + os.environ.get('PYTHONPATH', '')} + for name in ('zipmodule', 'sourceless'): + with self.subTest(name=name): + python = subprocess.run([sys.executable, '-m', name, 'arg'], cwd=root, env=environment, + capture_output=True, text=True) + aiython = subprocess.run([sys.executable, '-m', 'aiython', '-m', name, 'arg'], cwd=root, + env=environment, capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + + def test_module_cli_reports_invalid_requests(self): + for arguments, expected in ((['-m'], '-m requires a module name'), + (['-m', 'module_that_does_not_exist'], 'No module named'), + (['--stats'], 'a script path or -m module is required')): + with self.subTest(arguments=arguments): + result = subprocess.run([sys.executable, '-m', 'aiython', *arguments], + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected, result.stderr) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'sourceless.py' + source.write_text('value = 1\n') + py_compile.compile(str(source), cfile=str(root / 'sourceless.pyc'), doraise=True) + source.unlink() + result = subprocess.run([sys.executable, '-m', 'aiython', '--explain', '-m', 'sourceless'], + cwd=root, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('Cannot explain a module without Python source', result.stderr) + + def test_module_config_comes_from_invoking_project(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + project = root / 'project' + project.mkdir() + config = project / 'aiython.toml' + config.write_text('version=3\nmodel="openai/test"\n') + external = root / 'external' + external.mkdir() + (external / 'externalmod.py').write_text('print("ok")\n') + environment = {**os.environ, 'PYTHONPATH': str(external) + os.pathsep + os.environ.get('PYTHONPATH', '')} + result = subprocess.run([sys.executable, '-m', 'aiython', '--explain', '-m', 'externalmod'], + cwd=project, env=environment, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout)['config']['config'], str(config)) + result = subprocess.run([sys.executable, '-m', 'aiython', '-m', 'externalmod'], + cwd=project, env=environment, capture_output=True, text=True) + self.assertEqual((result.returncode, result.stdout), (0, 'ok\n'), result.stderr) + def test_safe_path_modes_match_cpython(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / 'main.py' From ac6fd22d1841ae69664a4193d75ff59002797c53 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 13:01:23 +0700 Subject: [PATCH 08/72] Support Python command stdin and archive entry modes --- README.md | 1 + README.pypi.md | 1 + docs/configuration.md | 2 + src/aiython/cli.py | 90 +++++++++++++++---- tests/test_python_compatibility.py | 139 ++++++++++++++++++++++++++++- 5 files changed, 215 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 57a9c52..541af32 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ aiython tickets.py Use a project environment when your script imports other project dependencies; a uv tool has its own isolated environment. [Setup and configuration](docs/configuration.md) covers profiles, custom endpoints, and capability routes. Run an importable module or package with `aiython -m package.module` or `aiython -m package`; pass its arguments after the module name. +The CLI also accepts `aiython -c 'print(1)'`, `aiython -` for stdin, and a directory or zipapp with `__main__.py`. Ordinary Python runs without loading LiteLLM or contacting a provider. Reasoning calls use the [LiteLLM Python SDK](https://docs.litellm.ai/docs/) in process; no proxy service is needed. diff --git a/README.pypi.md b/README.pypi.md index 20e132e..1c0cb3c 100644 --- a/README.pypi.md +++ b/README.pypi.md @@ -49,6 +49,7 @@ Python owns statement order, loops, assignments, and side effects. AI works at t Use `aiython --help` to see the CLI, `aiython config show` to inspect configuration, and `aiython --stats tickets.py` to see model calls and timings. Additional routes for vision, documents, embeddings, reranking, speech, images, and video are configured only when needed. Run an importable module or package with `aiython -m package.module` or `aiython -m package`; pass its arguments after the module name. +The CLI also accepts `aiython -c 'print(1)'`, `aiython -` for stdin, and a directory or zipapp with `__main__.py`. Aiython is not a sandbox. Frame tools can use `eval` and `exec` with your process permissions, and relevant source or object data may be sent to your configured provider. Use trusted code and review provider data handling. diff --git a/docs/configuration.md b/docs/configuration.md index e3d4245..b30e766 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -66,6 +66,8 @@ Run `aiython config show --script PATH` to inspect resolved settings. `--config` Run an importable module or package with `aiython -m package.module` or `aiython -m package`. Put Aiython options before `-m`; arguments after the module name go to that module. Packages need a `__main__.py`, as with `python -m`. Source files get Aiython's source transformation; modules available only as bytecode run as Python bytecode. Resolving a dotted name for `--explain -m` imports its parent package. +The CLI also accepts `aiython -c 'print(1)'`, `aiython -` for source from stdin, and a directory or zipapp containing `__main__.py`. Arguments after the command, `-`, or path go to the program. For `-c` and stdin, Aiython searches for `aiython.toml` from the current directory. A direct `.pyc` file and source-less `__main__.pyc` run as Python bytecode without source transformation or `--explain` output. + Version 1 and 2 configs are rejected with migration guidance and never overwritten. Move the old reasoning model to top-level `model`, put other routes under `[capabilities]`, and replace direct keys with `api_key_env` plus a private env file. Runtime state starts fresh in `.aiython/runtime-v3.sqlite`; older databases remain untouched. ## Deadlines and jobs diff --git a/src/aiython/cli.py b/src/aiython/cli.py index 3d187f4..e034f0d 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -12,6 +12,7 @@ import sys import tokenize import types +import zipfile from pathlib import Path from time import perf_counter @@ -97,28 +98,45 @@ def module_source(name, arguments=()): return spec, source, code, initial_main +def path_source(path): + spec = importlib.machinery.PathFinder.find_spec("__main__", [str(path.absolute())]) + if spec is None or spec.loader is None: + raise AiythonError(f"Cannot find __main__ in: {path}") + source = getattr(spec.loader, "get_source", lambda _: None)("__main__") + return spec, source, spec.loader.get_code("__main__") + + def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, stats=False, trace_plan=False, config_path=None, profile=None, force_profile=None, restore_state=True, - source=None, module_spec=None, module_invocation=None, compiled_code=None, initial_main=None): + source=None, module_spec=None, module_invocation=None, compiled_code=None, initial_main=None, + entry_kind="file", entry_argument=None): started = perf_counter() - argv0 = str(path) if module_spec is None else module_spec.origin - display_path = path.absolute() if module_spec is None else path + argv0 = ("-c" if entry_kind == "command" else "-" if entry_kind == "stdin" else + module_spec.origin if entry_kind == "module" else entry_argument if entry_kind == "path" else str(path)) + display_path = ("" if entry_kind == "command" else "" if entry_kind == "stdin" else + path if module_spec else path.absolute()) path = path.resolve() - config_source = Path.cwd() / "__main__.py" if module_spec else path + config_source = (Path.cwd() / "__main__.py" if entry_kind in ("module", "command", "stdin") else + Path(entry_argument).absolute() / "__main__.py" if entry_kind == "path" and + Path(entry_argument).is_dir() else path) config = config or resolve(config_source, config_path=config_path, profile=profile, force_profile=force_profile) config_seconds = perf_counter() - started runtime = Runtime(config, agent_factory=agent_factory, stats=stats, trace_plan=trace_plan) code = (compiled_code if compiled_code is not None else runtime.compile_source(read_source(path) if source is None else source, str(display_path), entry=True)) module = initial_main or types.ModuleType("__main__") - module.__dict__.update({"__file__": str(display_path) if module_spec is None else module_spec.origin, - "__package__": module_spec.parent if module_spec else None, + module.__dict__.update({"__package__": module_spec.parent if module_spec else None, "__spec__": module_spec, - "__cached__": module_spec.cached if module_spec else None, "__loader__": module_spec.loader if module_spec else + importlib.machinery.BuiltinImporter if entry_kind in ("command", "stdin") else + importlib.machinery.SourcelessFileLoader("__main__", str(display_path)) + if entry_kind == "bytecode" else importlib.machinery.SourceFileLoader("__main__", str(display_path)), RUNTIME_NAME: runtime, "__builtins__": builtins}) + if entry_kind != "command": + module.__dict__.update({"__file__": module_spec.origin if module_spec else str(display_path), + "__cached__": module_spec.cached if module_spec else None}) if sys.version_info < (3, 14): module.__dict__.setdefault("__annotations__", {}) old_main = sys.modules.get("__main__") @@ -142,12 +160,23 @@ def finish(): try: sys.modules["__main__"] = module sys.argv = [argv0, *arguments] - sys.orig_argv = ([*interpreter_args, "-m", module_invocation, *arguments] if module_spec else + sys.orig_argv = ([*interpreter_args, "-m", module_invocation, *arguments] if entry_kind == "module" else + [*interpreter_args, "-c", source, *arguments] if entry_kind == "command" else [*interpreter_args, argv0, *arguments]) - if module_spec is None and not sys.flags.safe_path: - sys.path.insert(0, str(path.parent)) + if entry_kind == "path": + entry_path = str(Path(entry_argument).absolute()) + if sys.flags.safe_path: + sys.path.insert(0, entry_path) + else: + sys.path[:1] = [entry_path] + elif not sys.flags.safe_path: + if entry_kind in ("file", "bytecode"): + sys.path[:1] = [str(path.parent)] + elif entry_kind in ("command", "stdin"): + sys.path[:1] = [""] sys.meta_path.insert(0, finder) - os.environ["AIYTHON_SPAWN_ENTRY"] = str(display_path) + if entry_kind in ("file", "module"): + os.environ["AIYTHON_SPAWN_ENTRY"] = str(display_path) execution_started = perf_counter() exec(code, module.__dict__) return module.__dict__ @@ -169,7 +198,7 @@ def finish(): else: # CPython removes these entry-script attributes before waiting for # non-daemon threads and running atexit callbacks. - if module_spec is None: + if entry_kind in ("file", "bytecode", "stdin"): module.__dict__.pop("__file__", None) module.__dict__.pop("__cached__", None) @@ -190,6 +219,8 @@ def parser(): result.add_argument("--stats", action="store_true", help="Report model calls, tools, request bytes and timings on stderr") result.add_argument("-m", "--module", dest="module_args", nargs=argparse.REMAINDER, help="Run a Python module as __main__") + result.add_argument("-c", dest="command_args", nargs=argparse.REMAINDER, + help="Run Python source from a command string") result.add_argument("script", nargs="?") result.add_argument("args", nargs=argparse.REMAINDER) return result @@ -237,27 +268,51 @@ def main(argv=None): return argument_parser = parser() args = argument_parser.parse_args(arguments) - module_spec = source = module_invocation = compiled_code = initial_main = None + module_spec = source = module_invocation = compiled_code = initial_main = entry_argument = None + entry_kind = "file" if args.module_args is not None: if not args.module_args: argument_parser.error("-m requires a module name") module_invocation, *script_args = args.module_args module_spec, source, original_code, initial_main = module_source(module_invocation, script_args) path = Path(module_spec.origin or original_code.co_filename) + entry_kind = "module" if source is None: compiled_code = original_code + elif args.command_args is not None: + if not args.command_args: + argument_parser.error("-c requires a command string") + source, *script_args = args.command_args + path = Path.cwd() / "__main__.py" + entry_kind = "command" else: if args.script is None: argument_parser.error("a script path or -m module is required") path = Path(args.script) script_args = args.args + if args.script == "-": + source = sys.stdin.read() + path = Path.cwd() / "__main__.py" + entry_kind = "stdin" + elif path.suffix == ".pyc" and path.is_file(): + compiled_code = importlib.machinery.SourcelessFileLoader( + "__main__", str(path.absolute())).get_code("__main__") + entry_kind = "bytecode" + elif path.is_dir() or zipfile.is_zipfile(path): + entry_argument = args.script + module_spec, source, original_code = path_source(path) + path = Path(module_spec.origin or original_code.co_filename) + entry_kind = "path" + if source is None: + compiled_code = original_code if args.explain: if source is None and compiled_code is not None: - raise AiythonError("Cannot explain a module without Python source") - config_source = Path.cwd() / "__main__.py" if module_spec else path + raise AiythonError("Cannot explain an entry without Python source") + config_source = Path.cwd() / "__main__.py" if entry_kind in ("module", "command", "stdin") else path config = resolve(config_source, config_path=args.config, profile=args.profile, force_profile=args.force_profile) - filename = str(path if module_spec else path.resolve()) + filename = ("" if entry_kind == "command" else "" if entry_kind == "stdin" else + str(path if module_spec else path.resolve())) unit = parse(read_source(path.resolve()) if source is None else source, filename) runtime = Runtime(config) runtime.prepare(unit, entry=True) @@ -272,7 +327,8 @@ def main(argv=None): force_profile=args.force_profile, stats=args.stats, trace_plan=args.trace_plan, restore_state=argv is not None, source=source, module_spec=module_spec, module_invocation=module_invocation, - compiled_code=compiled_code, initial_main=initial_main) + compiled_code=compiled_code, initial_main=initial_main, + entry_kind=entry_kind, entry_argument=entry_argument) except AiythonError as exc: # Keep the original runtime cause visible without leaking provider internals. if exc.__cause__: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 178e39e..552a8a4 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -15,6 +15,143 @@ class PythonCompatibilityTests(unittest.TestCase): + def test_command_and_stdin_execution_match_cpython(self): + source = '''import atexit, inspect, sys +def report(stage): + main = sys.modules['__main__'] + print(stage, sys.argv, sys.orig_argv, sys.path[:2]) + print(__name__, __package__, __spec__, getattr(__loader__, '__name__', type(__loader__).__name__)) + print(vars(main).get('__file__', 'ABSENT'), vars(main).get('__cached__', 'ABSENT')) + print(inspect.currentframe().f_code.co_filename) +report('RUN') +atexit.register(lambda: report('EXIT')) +''' + with tempfile.TemporaryDirectory() as directory: + for name, arguments, input_source in (('command', ['-c', source, 'one', '-x'], None), + ('stdin', ['-', 'one', '-x'], source)): + with self.subTest(name=name): + python = subprocess.run([sys.executable, *arguments], input=input_source, + cwd=directory, capture_output=True, text=True) + aiython = subprocess.run([sys.executable, '-m', 'aiython', *arguments], input=input_source, + cwd=directory, capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + + def test_directory_and_zipapp_execution_match_cpython(self): + source = '''import atexit, inspect, sys +from helper import value +def report(stage): + main = sys.modules['__main__'] + print(stage, value, sys.argv, sys.orig_argv, sys.path[:2]) + print(__name__, __package__, __spec__.name, __spec__.origin) + print(vars(main).get('__file__', 'ABSENT'), vars(main).get('__cached__', 'ABSENT')) + print(type(__loader__).__name__, inspect.currentframe().f_code.co_filename) +report('RUN') +atexit.register(lambda: report('EXIT')) +''' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + app = root / 'app' + app.mkdir() + (app / '__main__.py').write_text(source) + (app / 'helper.py').write_text('value = 7\n') + archive = root / 'app.pyz' + with zipfile.ZipFile(archive, 'w') as package: + package.writestr('__main__.py', source) + package.writestr('helper.py', 'value = 7\n') + for name in ('app', 'app.pyz'): + with self.subTest(name=name): + python = subprocess.run([sys.executable, name, 'one', '-x'], cwd=root, + capture_output=True, text=True) + aiython = subprocess.run([sys.executable, '-m', 'aiython', name, 'one', '-x'], cwd=root, + capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + + def test_sourceless_directory_execution_matches_cpython(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + app = root / 'app' + app.mkdir() + source = app / '__main__.py' + source.write_text('import sys\nprint(__file__, __cached__, sys.argv, sys.path[0])\n') + py_compile.compile(str(source), cfile=str(app / '__main__.pyc'), doraise=True) + source.unlink() + python = subprocess.run([sys.executable, 'app'], cwd=root, capture_output=True, text=True) + aiython = subprocess.run([sys.executable, '-m', 'aiython', 'app'], cwd=root, + capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + explanation = subprocess.run([sys.executable, '-m', 'aiython', '--explain', 'app'], + cwd=root, capture_output=True, text=True) + self.assertNotEqual(explanation.returncode, 0) + self.assertIn('Cannot explain an entry without Python source', explanation.stderr) + + def test_direct_bytecode_execution_matches_cpython(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'source.py' + source.write_text('''import atexit, inspect, sys +def report(): + main = sys.modules['__main__'] + print(sys.argv, sys.orig_argv, sys.path[:2]) + print(vars(main).get('__file__', 'ABSENT'), vars(main).get('__cached__', 'ABSENT')) + print(type(__loader__).__name__, __package__, __spec__, inspect.currentframe().f_code.co_filename) +report() +atexit.register(report) +''') + bytecode = root / 'direct.pyc' + py_compile.compile(str(source), cfile=str(bytecode), doraise=True) + source.unlink() + python = subprocess.run([sys.executable, 'direct.pyc', 'arg'], cwd=root, + capture_output=True, text=True) + aiython = subprocess.run([sys.executable, '-m', 'aiython', 'direct.pyc', 'arg'], cwd=root, + capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + + def test_new_entry_modes_report_invalid_requests(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + empty = root / 'empty' + empty.mkdir() + for arguments, expected in ((['-c'], '-c requires a command string'), + (['empty'], 'Cannot find __main__')): + with self.subTest(arguments=arguments): + result = subprocess.run([sys.executable, '-m', 'aiython', *arguments], cwd=root, + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected, result.stderr) + for arguments, input_source in ((['--explain', '-c', 'value = 1'], None), + (['--explain', '-'], 'value = 1\n')): + with self.subTest(arguments=arguments): + result = subprocess.run([sys.executable, '-m', 'aiython', *arguments], cwd=root, + input=input_source, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout)['blocks'], []) + + def test_safe_path_modes_for_new_entries_match_cpython(self): + source = 'import sys\nprint(sys.flags.safe_path, sys.path[:2])\n' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + app = root / 'app' + app.mkdir() + (app / '__main__.py').write_text(source) + with zipfile.ZipFile(root / 'app.pyz', 'w') as package: + package.writestr('__main__.py', source) + for flag in ('-I', '-P'): + for name, arguments, input_source in (('command', ['-c', source], None), + ('stdin', ['-'], source), + ('directory', ['app'], None), + ('zipapp', ['app.pyz'], None)): + with self.subTest(flag=flag, name=name): + python = subprocess.run([sys.executable, flag, *arguments], input=input_source, + cwd=root, capture_output=True, text=True) + aiython = subprocess.run([sys.executable, flag, '-m', 'aiython', *arguments], + input=input_source, cwd=root, capture_output=True, text=True) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + def test_module_startup_without_existing_main_restores_modules(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -150,7 +287,7 @@ def test_module_cli_reports_invalid_requests(self): result = subprocess.run([sys.executable, '-m', 'aiython', '--explain', '-m', 'sourceless'], cwd=root, capture_output=True, text=True) self.assertNotEqual(result.returncode, 0) - self.assertIn('Cannot explain a module without Python source', result.stderr) + self.assertIn('Cannot explain an entry without Python source', result.stderr) def test_module_config_comes_from_invoking_project(self): with tempfile.TemporaryDirectory() as directory: From 853d8caa7c2472e81e18ca864cd2054be39487af Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 13:30:41 +0700 Subject: [PATCH 09/72] Compile AI source during module startup --- docs/configuration.md | 2 +- src/aiython/cli.py | 99 +++++++++++++++++++++---- tests/test_python_compatibility.py | 111 ++++++++++++++++++++++++++++- 3 files changed, 194 insertions(+), 18 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b30e766..2bd6b0f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -64,7 +64,7 @@ Model IDs use LiteLLM syntax. A route can also be `{ model = "...", api_base = " Run `aiython config show --script PATH` to inspect resolved settings. `--config` chooses an exact file. `--profile` selects a default named profile; `--force-profile` overrides source directives. CLI options for script execution go before the script path, while later arguments go to the program. -Run an importable module or package with `aiython -m package.module` or `aiython -m package`. Put Aiython options before `-m`; arguments after the module name go to that module. Packages need a `__main__.py`, as with `python -m`. Source files get Aiython's source transformation; modules available only as bytecode run as Python bytecode. Resolving a dotted name for `--explain -m` imports its parent package. +Run an importable module or package with `aiython -m package.module` or `aiython -m package`. Put Aiython options before `-m`; arguments after the module name go to that module. Packages need a `__main__.py`, as with `python -m`. Source files get Aiython's source transformation; modules available only as bytecode run as Python bytecode. During execution, project parent packages with AI statements are transformed too. `--explain -m` imports the parent package using ordinary Python, so it cannot explain a module whose parent package requires AI syntax. The CLI also accepts `aiython -c 'print(1)'`, `aiython -` for source from stdin, and a directory or zipapp containing `__main__.py`. Arguments after the command, `-`, or path go to the program. For `-c` and stdin, Aiython searches for `aiython.toml` from the current directory. A direct `.pyc` file and source-less `__main__.pyc` run as Python bytecode without source transformation or `--explain` output. diff --git a/src/aiython/cli.py b/src/aiython/cli.py index e034f0d..9a2d9f1 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -6,12 +6,13 @@ import importlib.abc import importlib.machinery import importlib.metadata +import importlib.util import json import os -import runpy import sys import tokenize import types +import warnings import zipfile from pathlib import Path from time import perf_counter @@ -60,6 +61,23 @@ def find_spec(self, fullname, path=None, target=None): return spec +class ModuleStartFinder(ProjectFinder): + """Transform project imports with AI syntax while -m resolves its target.""" + + def find_spec(self, fullname, path=None, target=None): + spec = super().find_spec(fullname, path, target) + if spec is None: + return None + source = spec.loader.get_source(fullname) + if source is None: + return None + try: + compile(source, spec.origin, "exec", dont_inherit=True) + except SyntaxError: + return spec + return None + + def interpreter_arguments(): original = sys.orig_argv for index in range(len(original) - 1): @@ -68,7 +86,40 @@ def interpreter_arguments(): return [sys.executable] -def module_source(name, arguments=()): +def module_details(name): + if name.startswith("."): + raise AiythonError("Relative module names not supported") + try: + spec = importlib.util.find_spec(name) + except (ImportError, AttributeError, TypeError, ValueError) as exc: + raise AiythonError(f"Error while finding module specification for {name!r} " + f"({type(exc).__name__}: {exc})") from None + parent = name.rpartition(".")[0] + existing = sys.modules.get(name) + if parent and existing is not None and not hasattr(existing, "__path__"): + warnings.warn(f"{name!r} found in sys.modules after import of package {parent!r}, " + f"but prior to execution of {name!r}; this may result in unpredictable behaviour", + RuntimeWarning, stacklevel=2) + if spec is None: + raise AiythonError(f"No module named {name}") + if spec.submodule_search_locations is not None: + if name == "__main__" or name.endswith(".__main__"): + raise AiythonError("Cannot use package as __main__ module") + try: + return module_details(name + ".__main__") + except AiythonError as exc: + raise AiythonError(f"{exc}; {name!r} is a package and cannot be directly executed") from None + loader = spec.loader + if loader is None: + raise AiythonError(f"{name!r} is a namespace package and cannot be executed") + source = getattr(loader, "get_source", lambda _: None)(name) + code = loader.get_code(name) if source is None else None + if code is None and source is None: + raise AiythonError(f"No code object available for {name}") + return spec, source, code + + +def module_source(name, arguments=(), *, runtime=None): old_main = sys.modules.get("__main__") old_argv, old_orig_argv = sys.argv, sys.orig_argv interpreter_args = interpreter_arguments() @@ -77,24 +128,27 @@ def module_source(name, arguments=()): initial_main.__builtins__ = builtins if sys.version_info < (3, 14): initial_main.__annotations__ = {} + finder = ModuleStartFinder(runtime) if runtime is not None else None try: sys.modules["__main__"] = initial_main sys.argv = ["-m", *arguments] sys.orig_argv = [*interpreter_args, "-m", name, *arguments] if not sys.flags.safe_path: sys.path[:1] = [str(Path.cwd())] - actual_name, spec, code = runpy._get_module_details(name) - except ImportError as exc: - raise AiythonError(str(exc)) from None + if finder is not None: + sys.meta_path.insert(0, finder) + spec, source, code = module_details(name) finally: + if finder is not None: + sys.meta_path.remove(finder) sys.argv = old_argv sys.orig_argv = old_orig_argv if old_main is not None: sys.modules["__main__"] = old_main else: sys.modules.pop("__main__", None) - loader = spec.loader - source = getattr(loader, "get_source", lambda _: None)(actual_name) + if isinstance(spec.loader, ProjectLoader): + spec.loader = importlib.machinery.SourceFileLoader(spec.name, spec.origin) return spec, source, code, initial_main @@ -109,8 +163,9 @@ def path_source(path): def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, stats=False, trace_plan=False, config_path=None, profile=None, force_profile=None, restore_state=True, source=None, module_spec=None, module_invocation=None, compiled_code=None, initial_main=None, - entry_kind="file", entry_argument=None): - started = perf_counter() + entry_kind="file", entry_argument=None, runtime=None, preparation_started=None, + resolved_seconds=None): + started = preparation_started if preparation_started is not None else perf_counter() argv0 = ("-c" if entry_kind == "command" else "-" if entry_kind == "stdin" else module_spec.origin if entry_kind == "module" else entry_argument if entry_kind == "path" else str(path)) display_path = ("" if entry_kind == "command" else "" if entry_kind == "stdin" else @@ -120,8 +175,8 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta Path(entry_argument).absolute() / "__main__.py" if entry_kind == "path" and Path(entry_argument).is_dir() else path) config = config or resolve(config_source, config_path=config_path, profile=profile, force_profile=force_profile) - config_seconds = perf_counter() - started - runtime = Runtime(config, agent_factory=agent_factory, stats=stats, trace_plan=trace_plan) + config_seconds = resolved_seconds if resolved_seconds is not None else perf_counter() - started + runtime = runtime or Runtime(config, agent_factory=agent_factory, stats=stats, trace_plan=trace_plan) code = (compiled_code if compiled_code is not None else runtime.compile_source(read_source(path) if source is None else source, str(display_path), entry=True)) module = initial_main or types.ModuleType("__main__") @@ -269,13 +324,26 @@ def main(argv=None): argument_parser = parser() args = argument_parser.parse_args(arguments) module_spec = source = module_invocation = compiled_code = initial_main = entry_argument = None + module_config = module_runtime = module_started = module_config_seconds = None entry_kind = "file" if args.module_args is not None: if not args.module_args: argument_parser.error("-m requires a module name") module_invocation, *script_args = args.module_args - module_spec, source, original_code, initial_main = module_source(module_invocation, script_args) - path = Path(module_spec.origin or original_code.co_filename) + if not args.explain: + module_started = perf_counter() + module_config = resolve(Path.cwd() / "__main__.py", config_path=args.config, + profile=args.profile, force_profile=args.force_profile) + module_config_seconds = perf_counter() - module_started + module_runtime = Runtime(module_config, stats=args.stats, trace_plan=args.trace_plan) + try: + module_spec, source, original_code, initial_main = module_source( + module_invocation, script_args, runtime=module_runtime) + except BaseException: + if module_runtime is not None: + module_runtime.capabilities.close() + raise + path = Path(module_spec.origin or (original_code.co_filename if original_code else module_invocation)) entry_kind = "module" if source is None: compiled_code = original_code @@ -323,12 +391,13 @@ def main(argv=None): "checkpoints": [{"span": vars(c.span), "statement": c.statement} for c in runtime.checkpoints.values()]}, ensure_ascii=False, indent=2)) return - run_script(path, script_args, config_path=args.config, profile=args.profile, + run_script(path, script_args, config=module_config, config_path=args.config, profile=args.profile, force_profile=args.force_profile, stats=args.stats, trace_plan=args.trace_plan, restore_state=argv is not None, source=source, module_spec=module_spec, module_invocation=module_invocation, compiled_code=compiled_code, initial_main=initial_main, - entry_kind=entry_kind, entry_argument=entry_argument) + entry_kind=entry_kind, entry_argument=entry_argument, runtime=module_runtime, + preparation_started=module_started, resolved_seconds=module_config_seconds) except AiythonError as exc: # Keep the original runtime cause visible without leaking provider internals. if exc.__cause__: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 552a8a4..2e058a1 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1,5 +1,6 @@ """Compare ordinary Python execution with Aiython in separate processes.""" +import importlib import json import os import py_compile @@ -7,14 +8,70 @@ import sys import tempfile import unittest +import warnings import zipfile from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch -from aiython.cli import main, module_source, run_script +from aiython.cli import ModuleStartFinder, main, module_details, module_source, run_script +from aiython.models import AiythonError, ProfileConfig, ResolvedConfig +from aiython.runtime import Runtime class PythonCompatibilityTests(unittest.TestCase): + def test_module_resolution_errors_are_explicit(self): + with self.assertRaisesRegex(AiythonError, 'Relative module names not supported'): + module_details('.relative') + with self.assertRaisesRegex(AiythonError, 'Error while finding module specification'): + module_details('missing_parent_for_aiython.child') + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + package = root / 'empty_package_for_aiython' + package.mkdir() + (package / '__init__.py').write_text('') + with patch.object(sys, 'path', [str(root), *sys.path]): + with self.assertRaisesRegex(AiythonError, 'is a package and cannot be directly executed'): + module_details('empty_package_for_aiython') + nested_main = package / '__main__' + nested_main.mkdir() + (nested_main / '__init__.py').write_text('') + importlib.invalidate_caches() + with self.assertRaisesRegex(AiythonError, 'Cannot use package as __main__ module'): + module_details('empty_package_for_aiython.__main__') + no_loader = SimpleNamespace(submodule_search_locations=None, loader=None) + no_code = SimpleNamespace(submodule_search_locations=None, + loader=SimpleNamespace(get_source=lambda _: None, get_code=lambda _: None)) + for spec, expected in ((no_loader, 'namespace package and cannot be executed'), + (no_code, 'No code object available')): + with self.subTest(expected=expected), patch('aiython.cli.importlib.util.find_spec', return_value=spec): + with self.assertRaisesRegex(AiythonError, expected): + module_details('custom_loader') + + def test_module_start_finder_leaves_sourceless_loaders_unchanged(self): + runtime = Runtime(ResolvedConfig(None, Path.cwd())) + loader = SimpleNamespace(get_source=lambda _: None) + spec = SimpleNamespace(loader=loader, origin='sourceless.pyc') + with patch('aiython.cli.ProjectFinder.find_spec', return_value=spec): + self.assertIsNone(ModuleStartFinder(runtime).find_spec('sourceless')) + runtime.capabilities.close() + + def test_module_resolution_warns_when_parent_imports_target(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + package = root / 'early_import_package' + package.mkdir() + (package / '__init__.py').write_text('from . import task\n') + (package / 'task.py').write_text('value = 1\n') + with patch.object(sys, 'path', [str(root), *sys.path]), warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always', RuntimeWarning) + spec, source, code = module_details('early_import_package.task') + self.assertEqual(spec.name, 'early_import_package.task') + self.assertEqual(source, 'value = 1\n') + self.assertIsNone(code) + self.assertEqual(len(caught), 1) + self.assertIn('found in sys.modules after import of package', str(caught[0].message)) + def test_command_and_stdin_execution_match_cpython(self): source = '''import atexit, inspect, sys def report(stage): @@ -164,10 +221,59 @@ def test_module_startup_without_existing_main_restores_modules(self): self.assertNotIn('__main__', sys.modules) self.assertEqual(spec.name, 'standalone') self.assertEqual(source, 'value = 1\n') - self.assertEqual(code.co_name, '') + self.assertIsNone(code) self.assertEqual(initial_main.__name__, '__main__') self.assertEqual('__annotations__' in vars(initial_main), annotations_present) + def test_module_with_ai_source_uses_aiython_compiler(self): + class Agent: + def execute(self, request, runtime): + return 7 + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + package = root / 'ai_source_package' + package.mkdir() + (package / '__init__.py').write_text('value = 1\n') + (package / 'task.py').write_text('answer = compute the answer\n') + with patch('pathlib.Path.cwd', return_value=root), patch.object(sys, 'path', sys.path[:]): + spec, source, code, initial_main = module_source('ai_source_package.task') + self.assertIsNone(code) + self.assertIn('compute the answer', source) + config = ResolvedConfig(None, root, 'default', + {'default': ProfileConfig('default', 'fake', 'test')}) + namespace = run_script(Path(spec.origin), config=config, agent_factory=lambda _: Agent(), + source=source, module_spec=spec, module_invocation='ai_source_package.task', + initial_main=initial_main, entry_kind='module') + self.assertEqual(namespace['answer'], 7) + result = subprocess.run([sys.executable, '-m', 'aiython', '--explain', '-m', 'ai_source_package.task'], + cwd=root, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(json.loads(result.stdout)['blocks']), 1) + + def test_module_parent_package_with_ai_source_uses_same_runtime(self): + class Agent: + def execute(self, request, runtime): + return 7 + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + package = root / 'ai_parent_package' + package.mkdir() + (package / '__init__.py').write_text('prefix = decide a number\n') + (package / 'task.py').write_text('from . import prefix\nanswer = decide another number\n') + config = ResolvedConfig(None, root, 'default', + {'default': ProfileConfig('default', 'fake', 'test')}) + runtime = Runtime(config, agent_factory=lambda _: Agent()) + with patch('pathlib.Path.cwd', return_value=root), patch.object(sys, 'path', sys.path[:]): + spec, source, code, initial_main = module_source('ai_parent_package.task', runtime=runtime) + self.assertIsNone(code) + self.assertEqual(type(spec.loader).__name__, 'SourceFileLoader') + namespace = run_script(Path(spec.origin), config=config, runtime=runtime, + source=source, module_spec=spec, module_invocation='ai_parent_package.task', + initial_main=initial_main, entry_kind='module') + self.assertEqual((namespace['prefix'], namespace['answer']), (7, 7)) + def test_safe_path_module_mode_matches_cpython(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -272,6 +378,7 @@ def test_zip_and_sourceless_module_execution_match_cpython(self): def test_module_cli_reports_invalid_requests(self): for arguments, expected in ((['-m'], '-m requires a module name'), (['-m', 'module_that_does_not_exist'], 'No module named'), + (['--explain', '-m', 'module_that_does_not_exist'], 'No module named'), (['--stats'], 'a script path or -m module is required')): with self.subTest(arguments=arguments): result = subprocess.run([sys.executable, '-m', 'aiython', *arguments], From 8fec650f09e97aada87ef2697b22a1e9f67a3258 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 13:37:50 +0700 Subject: [PATCH 10/72] Resolve modules before loading project configuration --- src/aiython/cli.py | 25 ++++++++--- tests/test_python_compatibility.py | 69 ++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 7 deletions(-) diff --git a/src/aiython/cli.py b/src/aiython/cli.py index 9a2d9f1..6be3e8b 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -330,19 +330,30 @@ def main(argv=None): if not args.module_args: argument_parser.error("-m requires a module name") module_invocation, *script_args = args.module_args - if not args.explain: + try: + module_spec, source, original_code, initial_main = module_source(module_invocation, script_args) + except SyntaxError as exc: + # A source file with AI syntax cannot be imported as an ordinary + # parent package. A SyntaxError raised *by* package code is a + # program error and must not cause that package to run twice. + if args.explain or exc.filename is None: + raise + traceback = exc.__traceback__ + while traceback is not None: + if traceback.tb_frame.f_code.co_filename == exc.filename: + raise + traceback = traceback.tb_next module_started = perf_counter() module_config = resolve(Path.cwd() / "__main__.py", config_path=args.config, profile=args.profile, force_profile=args.force_profile) module_config_seconds = perf_counter() - module_started module_runtime = Runtime(module_config, stats=args.stats, trace_plan=args.trace_plan) - try: - module_spec, source, original_code, initial_main = module_source( - module_invocation, script_args, runtime=module_runtime) - except BaseException: - if module_runtime is not None: + try: + module_spec, source, original_code, initial_main = module_source( + module_invocation, script_args, runtime=module_runtime) + except BaseException: module_runtime.capabilities.close() - raise + raise path = Path(module_spec.origin or (original_code.co_filename if original_code else module_invocation)) entry_kind = "module" if source is None: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 2e058a1..b66e390 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1,6 +1,8 @@ """Compare ordinary Python execution with Aiython in separate processes.""" +import contextlib import importlib +import io import json import os import py_compile @@ -15,6 +17,7 @@ from unittest.mock import patch from aiython.cli import ModuleStartFinder, main, module_details, module_source, run_script +from aiython.config import resolve from aiython.models import AiythonError, ProfileConfig, ResolvedConfig from aiython.runtime import Runtime @@ -50,6 +53,8 @@ def test_module_resolution_errors_are_explicit(self): def test_module_start_finder_leaves_sourceless_loaders_unchanged(self): runtime = Runtime(ResolvedConfig(None, Path.cwd())) + with patch('aiython.cli.ProjectFinder.find_spec', return_value=None): + self.assertIsNone(ModuleStartFinder(runtime).find_spec('missing')) loader = SimpleNamespace(get_source=lambda _: None) spec = SimpleNamespace(loader=loader, origin='sourceless.pyc') with patch('aiython.cli.ProjectFinder.find_spec', return_value=spec): @@ -273,6 +278,70 @@ def execute(self, request, runtime): source=source, module_spec=spec, module_invocation='ai_parent_package.task', initial_main=initial_main, entry_kind='module') self.assertEqual((namespace['prefix'], namespace['answer']), (7, 7)) + explanation = subprocess.run([sys.executable, '-m', 'aiython', '--explain', '-m', + 'ai_parent_package.task'], cwd=root, capture_output=True, text=True) + self.assertNotEqual(explanation.returncode, 0) + self.assertIn('SyntaxError', explanation.stderr) + + def test_cli_retries_parent_ai_source_with_one_runtime(self): + class Agent: + def execute(self, request, runtime): + return 7 + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'aiython.toml').write_text('version=3\nmodel="openai/test"\n') + package = root / 'retry_parent_package' + package.mkdir() + (package / '__init__.py').write_text('prefix = decide a number\n') + (package / 'task.py').write_text('from . import prefix\nprint(prefix)\n') + runtime = Runtime(resolve(root / '__main__.py'), agent_factory=lambda _: Agent()) + output = io.StringIO() + with patch('pathlib.Path.cwd', return_value=root), patch.object(sys, 'path', sys.path[:]), \ + patch('aiython.cli.Runtime', return_value=runtime), contextlib.redirect_stdout(output): + main(['-m', 'retry_parent_package.task']) + self.assertEqual(output.getvalue(), '7\n') + + def test_cli_does_not_repeat_package_that_raises_syntax_error(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + package = root / 'raised_syntax_package' + package.mkdir() + marker = root / 'count.txt' + (package / '__init__.py').write_text( + 'from pathlib import Path\n' + f'path = Path({str(marker)!r})\n' + 'path.write_text(path.read_text() + "x" if path.exists() else "x")\n' + 'raise SyntaxError("raised by package", (__file__, 1, 1, "x"))\n') + (package / 'task.py').write_text('print("unreachable")\n') + result = subprocess.run([sys.executable, '-m', 'aiython', '-m', 'raised_syntax_package.task'], + cwd=root, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('raised by package', result.stderr) + self.assertEqual(marker.read_text(), 'x') + + def test_missing_module_error_precedes_invalid_project_config(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'aiython.toml').write_text('version=3\nmodel="openai/test"\n' + 'env_file=".aiython/missing.env"\n') + result = subprocess.run([sys.executable, '-m', 'aiython', '-m', 'module_that_does_not_exist'], + cwd=root, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('No module named', result.stderr) + self.assertNotIn('env_file', result.stderr) + + def test_failed_parent_ai_retry_closes_runtime(self): + config = ResolvedConfig(None, Path.cwd()) + runtime = unittest.mock.Mock() + first_error = SyntaxError('invalid', ('parent_source.py', 1, 1, 'invalid')) + with patch('aiython.cli.module_source', side_effect=[first_error, AiythonError('retry failed')]), \ + patch('aiython.cli.resolve', return_value=config), \ + patch('aiython.cli.Runtime', return_value=runtime), \ + contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + main(['-m', 'failing_parent.task']) + runtime.capabilities.close.assert_called_once_with() def test_safe_path_module_mode_matches_cpython(self): with tempfile.TemporaryDirectory() as directory: From 6b51a11bfeebc4d618c84d75007ccecd3fb7c546 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 13:59:46 +0700 Subject: [PATCH 11/72] Preserve Python generator delegation and shutdown behavior --- src/aiython/runtime.py | 4 ++- src/aiython/typed_runtime.py | 20 +++++++++++++-- tests/test_python_compatibility.py | 40 ++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/aiython/runtime.py b/src/aiython/runtime.py index ff0e0bb..8a66a13 100644 --- a/src/aiython/runtime.py +++ b/src/aiython/runtime.py @@ -653,7 +653,9 @@ def visit_Try(self, node): class Runtime: - error_type = BaseException + # Recovery handles ordinary failures; scope cleanup also sees control signals. + error_type = Exception + scope_error_type = BaseException @staticmethod def current_exception(): diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index f2014ee..ead2f5e 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -338,6 +338,7 @@ def __init__(self, *, snippet=False): self.snippet = snippet self.declarations = {} self.function = False + self.delegation_contract = False @staticmethod def declarations_in(body): @@ -399,6 +400,7 @@ def visit_Module(self,node): def visit_FunctionDef(self,node): previous = self.function + previous_contract = self.delegation_contract parent_declarations = dict(self.declarations) if previous else {} self.function = True parameters = {} @@ -407,6 +409,7 @@ def visit_FunctionDef(self,node): for arg, mode in ((node.args.vararg,'args'),(node.args.kwarg,'kwargs')): if arg and arg.annotation: parameters[arg.arg] = (ast.unparse(arg.annotation),mode) returns = ast.unparse(node.returns) if node.returns else None + self.delegation_contract = returns is not None # Generator annotations require yield/send checks, not return-only checks. is_generator = any(isinstance(n,(ast.Yield,ast.YieldFrom)) for n in self.function_nodes(node)) used = {n.id for n in self.function_nodes(node) if isinstance(n,ast.Name) and isinstance(n.ctx,ast.Load)} @@ -420,7 +423,7 @@ def visit_FunctionDef(self,node): else: node.body.append(ast.copy_location(ast.Return(helper('returned',ast.Constant(None))),node)) initial = next(i for i,n in enumerate(node.body) if getattr(n,'_aiython_scope_initializer',False)) - handler = ast.ExceptHandler(ast.Attribute(ast.Name(RUNTIME_NAME,ast.Load()),'error_type',ast.Load()),None, + handler = ast.ExceptHandler(ast.Attribute(ast.Name(RUNTIME_NAME,ast.Load()),'scope_error_type',ast.Load()),None, [ast.Expr(helper('aborted')),ast.Raise()]) final = ast.Expr(helper('leaving')) final = ast.Try([final],[],[],[ast.Expr(helper('exit_generator_scope' if is_generator else 'exit_scope'))]) @@ -429,6 +432,7 @@ def visit_FunctionDef(self,node): ast.copy_location(guarded,node) node.body = node.body[:initial+1] + [guarded] self.function = previous + self.delegation_contract = previous_contract return node visit_AsyncFunctionDef = visit_FunctionDef @@ -443,7 +447,9 @@ def function_nodes(node): def visit_ClassDef(self,node): node.decorator_list.insert(0, ast.Attribute(ast.Attribute(ast.Name(RUNTIME_NAME,ast.Load()),'types',ast.Load()),'register_class',ast.Load())) previous = self.function + previous_contract = self.delegation_contract self.function = False + self.delegation_contract = False body = self.body(node.body) header = body[:1] if isinstance(body[0],ast.Expr) and isinstance(body[0].value,ast.Constant) and isinstance(body[0].value.value,str) else [] enter, *statements = body[len(header):] @@ -455,6 +461,14 @@ def visit_ClassDef(self,node): else: node.body = header + [enter, exit_call] self.function = previous + self.delegation_contract = previous_contract + return node + + def visit_Lambda(self,node): + previous_contract = self.delegation_contract + self.delegation_contract = False + node = self.generic_visit(node) + self.delegation_contract = previous_contract return node def visit_Return(self,node): @@ -469,7 +483,9 @@ def visit_Yield(self,node): return ast.copy_location(helper('sent',node),node) def visit_YieldFrom(self,node): - node.value = helper('delegate',self.visit(node.value)) + node.value = self.visit(node.value) + if self.delegation_contract: + node.value = helper('delegate',node.value) return node def visit_AnnAssign(self,node): diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index b66e390..f1bb2db 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -733,6 +733,46 @@ def outer(): print(next(item)) try: item.send(7) except StopIteration: pass +''', + 'generator delegation identity': '''from collections.abc import Generator +def child(): + received = yield 1 + return received +def plain(): + yield from child() +def typed() -> Generator[int, None, None]: + def nested(): + yield from child() + nested_gen = nested() + print(next(nested_gen), nested_gen.gi_yieldfrom.gi_code.co_name) + nested_gen.close() + through_lambda = lambda: (yield from child()) + lambda_gen = through_lambda() + print(next(lambda_gen), lambda_gen.gi_yieldfrom.gi_code.co_name) + lambda_gen.close() + class Local: + def method(self): + yield from child() + method_gen = Local().method() + print(next(method_gen), method_gen.gi_yieldfrom.gi_code.co_name) + method_gen.close() + yield from child() +plain_gen = plain() +print(next(plain_gen), plain_gen.gi_yieldfrom.gi_code.co_name) +plain_gen.close() +typed_gen = typed() +print(next(typed_gen)) +typed_gen.close() +''', + 'generator shutdown delegation': '''from collections.abc import Generator +def generate(): + yield from [1, 2] +item = generate() +next(item) +def typed() -> Generator[int, None, None]: + yield from [1, 2] +typed_item = typed() +next(typed_item) ''', 'generator locals': '''import inspect def generate(value): From 6a64173e5b08cb57fda44650eff410540a59f06e Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 14:18:56 +0700 Subject: [PATCH 12/72] Avoid user attribute hooks during type inspection --- src/aiython/type_constraints.py | 3 ++- src/aiython/typed_runtime.py | 16 ++++++++++++---- tests/test_python_compatibility.py | 17 +++++++++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index d8a72ff..f43df03 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -321,7 +321,8 @@ def class_contract(self,target,names,label=None): if target in VALIDATORS: return result scope = self.module_names(target,names) | {target.__name__:target} scope.update({p.__name__:p for p in (getattr(target,'__type_params__',()) or getattr(target,'__parameters__',()))}) - scope.update({k:v for k,v in names.items() if isinstance(v,Contract)}) + # isinstance can call a user's __getattribute__('__class__') here. + scope.update({k:v for k,v in names.items() if issubclass(type(v), Contract)}) fields = {} for base in reversed(target.__mro__): if base in (object,dict): continue diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index ead2f5e..6ce54eb 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -25,6 +25,14 @@ def frame_scope(frame): return _GENERATOR_SCOPES.get(id(frame)) +def original_class(value): + # Bypass user __getattribute__ while reading typing's instance metadata. + try: + return object.__getattribute__(value, '__orig_class__') + except AttributeError: + return type(value) + + @dataclass class Scope: declarations: dict = field(default_factory=dict) @@ -145,7 +153,7 @@ def assignment_in(self, frame, value, name, annotation=None): with self._classes_lock: registered = type(value) in self.classes if registered: - compile_contract(getattr(value,'__orig_class__',type(value)),self.namespace(frame)).validate(value,name) + compile_contract(original_class(value),self.namespace(frame)).validate(value,name) return value def assignment(self,value,name,annotation=None): @@ -181,7 +189,7 @@ def check_instances(candidate): seen.add(id(candidate)) cls = type(candidate) if cls in classes: - compile_contract(getattr(candidate,'__orig_class__',cls),self.namespace(frame)).validate(candidate,cls.__qualname__) + compile_contract(original_class(candidate),self.namespace(frame)).validate(candidate,cls.__qualname__) try: state = object.__getattribute__(candidate,'__dict__') except AttributeError: state = {} check_instances(state) @@ -305,7 +313,7 @@ def assign_attribute(self,owner,name,value,annotation=None): if annotation: contract = compile_contract(annotation,self.namespace(frame)) else: - target = owner if isinstance(owner,type) else type(owner) + target = owner if issubclass(type(owner), type) else type(owner) fields = {} for base in reversed(target.__mro__): fields.update(annotations_of(base)) @@ -314,7 +322,7 @@ def assign_attribute(self,owner,name,value,annotation=None): namespace.update({p.__name__:p for p in (getattr(target,'__type_params__',()) or getattr(target,'__parameters__',()))}) contract = compile_contract(source,namespace) if source else None if contract: - if contract.marker == 'ClassVar' and not isinstance(owner,type): + if contract.marker == 'ClassVar' and not issubclass(type(owner), type): raise TypeViolation(f'{name}: ClassVar must be assigned on the class') if contract.marker == 'Final': try: inspect.getattr_static(owner,name) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index f1bb2db..1fe868f 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -659,6 +659,23 @@ def run(): print(run.__code__.co_varnames, sorted(locals())) run() print(events) +''', + 'custom attribute access during assignment': '''events = [] +class Item: + def __iadd__(self, other): + events.append(('iadd', other)) + return self +class Box: + def __init__(self): self.value = Item() + def __getattribute__(self, name): + events.append(('get', name)) + return object.__getattribute__(self, name) + def __setattr__(self, name, value): + events.append(('set', name)) + object.__setattr__(self, name, value) +box = Box() +box.value += 1 +print(events) ''', 'async function locals': '''import asyncio async def run(value): From bb487eca23c0df3d27a6d26de9589bcc6303befb Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 14:23:48 +0700 Subject: [PATCH 13/72] Allow tracing callbacks to reenter type checks --- src/aiython/typed_runtime.py | 4 ++-- tests/test_python_compatibility.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 6ce54eb..1002a2b 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -14,7 +14,7 @@ SCOPE = '__aiython_type_scope__' _FRAME_SCOPES = ContextVar('aiython_frame_scopes', default=()) _GENERATOR_SCOPES = {} -_GENERATOR_SCOPES_LOCK = threading.Lock() +_GENERATOR_SCOPES_LOCK = threading.RLock() def frame_scope(frame): @@ -49,7 +49,7 @@ class Scope: class TypeRuntime: def __init__(self): self.classes = weakref.WeakSet() - self._classes_lock = threading.Lock() + self._classes_lock = threading.RLock() self._contract_cache = threading.local() def contract(self, annotation, namespace): diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 1fe868f..acc8bf2 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -568,6 +568,29 @@ def test_relative_script_argument_matches_cpython(self): self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), (python.returncode, python.stdout, python.stderr)) + def test_trace_and_profile_callbacks_match_cpython(self): + with tempfile.TemporaryDirectory() as directory: + for callback in ('trace', 'profile'): + with self.subTest(callback=callback): + source = f'''import sys +events = [] +def callback(frame, event, arg): + if frame.f_code.co_name == 'target' and event == 'call': + events.append('target') + return callback +sys.set{callback}(callback) +def target(): return 1 +target() +sys.set{callback}(None) +print(events) +''' + python = subprocess.run([sys.executable, '-c', source], cwd=directory, + capture_output=True, text=True, timeout=10) + aiython = subprocess.run([sys.executable, '-m', 'aiython', '-c', source], + cwd=directory, capture_output=True, text=True, timeout=10) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + def test_language_constructs_match_cpython(self): cases = { 'empty script': '', From 1596c809fe524e600838437bdcd590a469427f88 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 14:59:26 +0700 Subject: [PATCH 14/72] Avoid collisions with user Python bindings --- src/aiython/_worker_main.py | 7 +- src/aiython/cli.py | 19 +++-- src/aiython/collaboration.py | 5 +- src/aiython/frontend.py | 25 ++++-- src/aiython/runtime.py | 69 +++++++++------- src/aiython/typed_runtime.py | 75 +++++++++-------- tests/test_collaboration.py | 5 +- tests/test_python_compatibility.py | 125 +++++++++++++++++++++++++++++ tests/test_runtime_edges.py | 2 +- 9 files changed, 253 insertions(+), 79 deletions(-) diff --git a/src/aiython/_worker_main.py b/src/aiython/_worker_main.py index f05b4f1..c42981c 100644 --- a/src/aiython/_worker_main.py +++ b/src/aiython/_worker_main.py @@ -13,13 +13,14 @@ from aiython.cli import ProjectFinder, read_source from aiython.config import resolve - from aiython.frontend import RUNTIME_NAME + from aiython.frontend import runtime_binding_name from aiython.runtime import Runtime source_path = Path(os.environ["AIYTHON_SPAWN_ENTRY"]) runtime = Runtime(resolve(source_path)) + source = read_source(source_path) globals().update({"__file__": str(source_path), "__package__": None, - RUNTIME_NAME: runtime}) + runtime_binding_name(source): runtime}) sys.path.insert(0, str(source_path.parent)) sys.meta_path.insert(0, ProjectFinder(runtime)) - exec(runtime.compile_source(read_source(source_path), str(source_path)), globals()) + exec(runtime.compile_source(source, str(source_path)), globals()) diff --git a/src/aiython/cli.py b/src/aiython/cli.py index 6be3e8b..6e5ccd0 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -18,7 +18,7 @@ from time import perf_counter from .config import describe, resolve -from .frontend import RUNTIME_NAME, parse +from .frontend import parse, runtime_binding_name from .models import AiythonError, ConfigError from .runtime import Runtime @@ -37,8 +37,9 @@ def __init__(self, name, path, runtime): self.runtime = runtime def exec_module(self, module): - module.__dict__[RUNTIME_NAME] = self.runtime - exec(self.runtime.compile_source(read_source(self.path), self.path), module.__dict__) + source = read_source(self.path) + module.__dict__[runtime_binding_name(source)] = self.runtime + exec(self.runtime.compile_source(source, self.path), module.__dict__) class ProjectFinder(importlib.abc.MetaPathFinder): @@ -177,8 +178,13 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta config = config or resolve(config_source, config_path=config_path, profile=profile, force_profile=force_profile) config_seconds = resolved_seconds if resolved_seconds is not None else perf_counter() - started runtime = runtime or Runtime(config, agent_factory=agent_factory, stats=stats, trace_plan=trace_plan) - code = (compiled_code if compiled_code is not None else - runtime.compile_source(read_source(path) if source is None else source, str(display_path), entry=True)) + if compiled_code is None: + source = read_source(path) if source is None else source + runtime_name = runtime_binding_name(source) + code = runtime.compile_source(source, str(display_path), entry=True) + else: + runtime_name = None + code = compiled_code module = initial_main or types.ModuleType("__main__") module.__dict__.update({"__package__": module_spec.parent if module_spec else None, "__spec__": module_spec, @@ -187,8 +193,9 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta importlib.machinery.SourcelessFileLoader("__main__", str(display_path)) if entry_kind == "bytecode" else importlib.machinery.SourceFileLoader("__main__", str(display_path)), - RUNTIME_NAME: runtime, "__builtins__": builtins}) + if runtime_name is not None: + module.__dict__[runtime_name] = runtime if entry_kind != "command": module.__dict__.update({"__file__": module_spec.origin if module_spec else str(display_path), "__cached__": module_spec.cached if module_spec else None}) diff --git a/src/aiython/collaboration.py b/src/aiython/collaboration.py index 548e0c5..8c75435 100644 --- a/src/aiython/collaboration.py +++ b/src/aiython/collaboration.py @@ -423,7 +423,10 @@ class Group: def __init__(self, project_root=None): if project_root is None: main_module = sys.modules.get("__main__") - runtime = getattr(main_module, "__aiython_runtime__", None) + from .runtime import Runtime + runtime = next((value for key, value in vars(main_module).items() + if key.startswith("__aiython_runtime") and key.endswith("__") + and issubclass(type(value), Runtime)), None) if main_module is not None else None project_root = runtime.config.project_root if runtime is not None else Path.cwd() self.project_root = str(Path(project_root).resolve()) self._ticket: Ticket | None = None diff --git a/src/aiython/frontend.py b/src/aiython/frontend.py index d4dc392..e261974 100644 --- a/src/aiython/frontend.py +++ b/src/aiython/frontend.py @@ -13,6 +13,15 @@ RUNTIME_NAME = "__aiython_runtime__" +def runtime_binding_name(source: str, reserved=()) -> str: + name = RUNTIME_NAME + index = 1 + while name in source or name in reserved: + name = f"__aiython_runtime_{index}__" + index += 1 + return name + + @dataclass class Block: id: str @@ -32,6 +41,7 @@ class Unit: directives: Directives blocks: dict[str, Block] transformed: str + runtime_name: str = RUNTIME_NAME class Frontend: @@ -44,6 +54,7 @@ class Frontend: def __init__(self, source: str, filename: str): self.source = source self.filename = filename + self.runtime_name = runtime_binding_name(source) self.lines = source.splitlines(keepends=True) self.offsets = [0] for line in self.lines: @@ -66,7 +77,7 @@ def render(self, blocks: list[Block]) -> tuple[str, list[int]]: for block in sorted(blocks, key=lambda b: b.start): pieces.append(self.source[cursor:block.start]) mapping.extend(range(cursor, block.start)) - call = f"{RUNTIME_NAME}.execute({block.id!r})" + call = f"{self.runtime_name}.execute({block.id!r})" count = self.source[block.start:block.end].count("\n") replacement = "(" + call + "\n" * count + ")" if count else call pieces.append(replacement) @@ -261,7 +272,7 @@ def build(self) -> Unit: text, source_map = self.render(proposed) # A syntactically valid edit inside string text is not a runtime # call (notably when deleting an f-string brace). Reject it. - real_calls = sum(t.type == tokenize.NAME and t.string == RUNTIME_NAME + real_calls = sum(t.type == tokenize.NAME and t.string == self.runtime_name for t in tolerant_tokens(text)) if sys.version_info < (3, 12): legacy_fields = set(self.legacy_fstring_candidates()) @@ -287,7 +298,8 @@ def build(self) -> Unit: if not blocks: tree = ast.parse(self.source, self.filename) self.directives.bind(tree) - return Unit(self.source, self.filename, tree, self.directives, {}, self.source) + return Unit(self.source, self.filename, tree, self.directives, {}, self.source, + self.runtime_name) # Mark standalone calls, then combine consecutive invalid statements. tree = ast.parse(rendered, self.filename) standalone = {} @@ -341,12 +353,11 @@ def build(self) -> Unit: child.end_col_offset = len(self.lines[block.span.end_line - 1][:block.span.end_column].encode("utf-8")) self.directives.bind(tree) return Unit(self.source, self.filename, tree, self.directives, - {b.id: b for b in merged}, rendered) + {b.id: b for b in merged}, rendered, self.runtime_name) - @staticmethod - def is_call(node: ast.AST) -> bool: + def is_call(self, node: ast.AST) -> bool: return (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) - and isinstance(node.func.value, ast.Name) and node.func.value.id == RUNTIME_NAME + and isinstance(node.func.value, ast.Name) and node.func.value.id == self.runtime_name and node.func.attr == "execute" and bool(node.args) and isinstance(node.args[0], ast.Constant)) diff --git a/src/aiython/runtime.py b/src/aiython/runtime.py index 8a66a13..4b22ad7 100644 --- a/src/aiython/runtime.py +++ b/src/aiython/runtime.py @@ -18,7 +18,7 @@ from collections.abc import MutableMapping from typing import Any -from .frontend import RUNTIME_NAME, Unit +from .frontend import RUNTIME_NAME, Unit, runtime_binding_name from .models import (AgentRequest, AiythonError, ConfigError, DirectiveContext, RecoveryDecision, RecoveryRequest, ResolvedConfig, SourceSpan) from .stats import Stats @@ -115,13 +115,18 @@ def expression_code(source): @lru_cache(maxsize=128) -def snippet_code(source): +def snippet_code(source, runtime_name=RUNTIME_NAME): from .typed_runtime import TypedTransformer - tree = TypedTransformer(snippet=True).visit(ast.parse(source, '')) + tree = TypedTransformer(snippet=True, runtime_name=runtime_name).visit(ast.parse(source, '')) ast.fix_missing_locations(tree) return compile(tree, '', 'exec') +def internal_binding(name, runtime_name): + return (name == runtime_name or name == runtime_name + 'recovery_counts' + or name.startswith(runtime_name + 'recovery_attempt_')) + + # Calling type's built-in descriptors bypasses user metaclass __getattribute__ # and descriptors shadowing __name__/__module__/__qualname__. _TYPE_FIELDS = {name: type.__dict__[name] for name in ("__name__", "__module__", "__qualname__", "__mro__", "__dict__")} @@ -210,8 +215,11 @@ def eval(self, code: str) -> Any: def exec(self, code: str) -> None: prepare = snippet_code if len(code) <= 64 * 1024 else snippet_code.__wrapped__ - compiled = prepare(code) - self.namespace()[RUNTIME_NAME] = self.manager + occupied = {name for namespace in (self.frame.f_globals, self.frame.f_locals) + for name, value in namespace.items() if value is not self.manager} + runtime_name = runtime_binding_name(code, occupied) + compiled = prepare(code, runtime_name) + self.frame.f_globals[runtime_name] = self.manager from .source_guard import protect_source with protect_source(self.manager): exec(compiled, self.frame.f_globals, self.namespace()) @@ -221,7 +229,9 @@ def get(self, name: str) -> Any: return self.eval(name) def set(self, name: str, value: Any) -> None: - if not name.isidentifier() or name.startswith("__aiython_"): + unit = self.manager.units.get(self.frame.f_code.co_filename) + runtime_name = unit.runtime_name if unit else RUNTIME_NAME + if not name.isidentifier() or internal_binding(name, runtime_name): raise ValueError("Binding must be a non-reserved Python identifier") self.manager.types.assignment_in(self.frame, value, name) self.namespace()[name] = value @@ -366,10 +376,12 @@ def frames(self) -> list[dict]: while tb: frame = tb.tb_frame if frame.f_code.co_filename in self.manager.units: + unit = self.manager.units[frame.f_code.co_filename] + runtime_name = unit.runtime_name if unit is not None else RUNTIME_NAME result.append({"filename": frame.f_code.co_filename, "line": tb.tb_lineno, "name": frame.f_code.co_name, "active": frame is self.frame, "locals": {k: self.handle(v) for k, v in frame.f_locals.items() - if not k.startswith("__aiython_")}}) + if not internal_binding(k, runtime_name)}}) tb = tb.tb_next return result @@ -377,6 +389,7 @@ def frames(self) -> list[dict]: class DynamicNames(ast.NodeTransformer): def __init__(self, unit: Unit): self.table = symtable.symtable(unit.transformed, unit.filename, "exec") + self.runtime_name = unit.runtime_name self.used_tables: set[int] = set() def in_scope(self, name: str, line: int, body): @@ -446,7 +459,7 @@ def visit_AnnAssign(self, node): return node def visit_Name(self, node): - if not isinstance(node.ctx, ast.Load) or node.id.startswith("__aiython_") or node.id == "super": + if not isinstance(node.ctx, ast.Load) or node.id == self.runtime_name or node.id == "super": # Keep CPython's compiler recognition of zero-argument super(), # which creates the implicit __class__ closure cell. return node @@ -457,7 +470,7 @@ def visit_Name(self, node): except KeyError: return node if symbol.is_global() and not symbol.is_declared_global(): - call = ast.Call(ast.Attribute(ast.Name(RUNTIME_NAME, ast.Load()), "lookup", ast.Load()), + call = ast.Call(ast.Attribute(ast.Name(self.runtime_name, ast.Load()), "lookup", ast.Load()), [ast.Constant(node.id)], []) return ast.copy_location(call, node) return node @@ -466,8 +479,9 @@ def visit_Name(self, node): class AsyncCalls(ast.NodeTransformer): """Await suspended AI calls in coroutines without changing Python scheduling.""" - def __init__(self): + def __init__(self, runtime_name=RUNTIME_NAME): self.in_async = False + self.runtime_name = runtime_name def _body(self, node, active): previous = self.in_async @@ -505,7 +519,7 @@ def visit_Call(self, node): node = self.generic_visit(node) if (self.in_async and isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) - and node.func.value.id == RUNTIME_NAME and node.func.attr == "execute"): + and node.func.value.id == self.runtime_name and node.func.attr == "execute"): node.func.attr = "aexecute" return ast.copy_location(ast.Await(value=node), node) return node @@ -513,6 +527,7 @@ def visit_Call(self, node): def install_checkpoint(runtime, unit, node, key, *, retry_allowed=True, scoped_retries=False): """Catch one statement without replaying statements before it.""" + runtime_name = unit.runtime_name target = None if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): target = node.targets[0].id @@ -531,26 +546,26 @@ def install_checkpoint(runtime, unit, node, key, *, retry_allowed=True, scoped_r "while True:\n" " try:\n" " pass\n" - f" except {RUNTIME_NAME}.error_type:\n" - f" if {RUNTIME_NAME}.recover({key!r}, {RUNTIME_NAME}.current_exception()):\n" + f" except {runtime_name}.error_type:\n" + f" if {runtime_name}.recover({key!r}, {runtime_name}.current_exception()):\n" " continue\n" " break\n" " else:\n" - f" {RUNTIME_NAME}.clear_recovery_count({key!r})\n" + f" {runtime_name}.clear_recovery_count({key!r})\n" " break\n" ).body[0] attempt = template.body[0] else: - counter = '__aiython_recovery_attempt_' + hashlib.sha256(key.encode()).hexdigest()[:16] + counter = runtime_name + 'recovery_attempt_' + hashlib.sha256(key.encode()).hexdigest()[:16] template = ast.parse( "if True:\n" f" {counter} = 0\n" " while True:\n" " try:\n" " pass\n" - f" except {RUNTIME_NAME}.error_type:\n" + f" except {runtime_name}.error_type:\n" f" {counter} += 1\n" - f" if {RUNTIME_NAME}.recover({key!r}, {RUNTIME_NAME}.current_exception(), {counter}):\n" + f" if {runtime_name}.recover({key!r}, {runtime_name}.current_exception(), {counter}):\n" " continue\n" " break\n" " else:\n" @@ -582,14 +597,13 @@ def __init__(self, runtime, unit): self.runtime, self.unit, self.serial = runtime, unit, 0 self.scoped_retries = False - @staticmethod - def generated(node): + def generated(self, node): if not isinstance(node, ast.Expr) or not isinstance(node.value, ast.Call): return False func = node.value.func while isinstance(func, ast.Attribute): func = func.value - return isinstance(func, ast.Name) and func.id == RUNTIME_NAME + return isinstance(func, ast.Name) and func.id == self.unit.runtime_name def body(self, statements, *, nested=True, preserve_docstring=False, scoped_retries=False): result = [] @@ -665,7 +679,7 @@ def __init__(self, config: ResolvedConfig, *, agent_factory=None, stats=False, t self._lock = threading.RLock() self.config = config from .typed_runtime import TypeRuntime - self.types = TypeRuntime() + self.types = TypeRuntime(self) from .capabilities import CapabilityRuntime self.capabilities = CapabilityRuntime(config, trace=trace_plan) self.agent_factory = agent_factory @@ -784,7 +798,7 @@ def _prepare(self, unit: Unit, *, entry: bool = False): linecache.cache[unit.filename] = (len(unit.source), None, unit.source.splitlines(True), unit.filename) tree = unit.tree from .typed_runtime import ExpectedTypes, TypedTransformer - ExpectedTypes(unit.blocks).visit(tree) + ExpectedTypes(unit.blocks, unit.runtime_name).visit(tree) tables = [symtable.symtable(unit.transformed, unit.filename, "exec")] definitions = {(n.name, n.lineno): min([n.lineno, *(d.lineno for d in n.decorator_list)]) for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))} @@ -797,9 +811,9 @@ def _prepare(self, unit: Unit, *, entry: bool = False): s.get_name() for s in table.get_symbols() if s.is_declared_global()} if unit.blocks: tree = DynamicNames(unit).visit(tree) - tree = TypedTransformer().visit(tree) + tree = TypedTransformer(runtime_name=unit.runtime_name).visit(tree) if unit.blocks and any(isinstance(node, ast.AsyncFunctionDef) for node in ast.walk(tree)): - tree = AsyncCalls().visit(tree) + tree = AsyncCalls(unit.runtime_name).visit(tree) tree = NestedCheckpoints(self, unit).visit(tree) ast.fix_missing_locations(tree) if entry: @@ -859,11 +873,12 @@ def selection(self, context: DirectiveContext): def request(self, statement, span, unit, frame, context): profile, prompts, agent = self.selection(context) - objects = {k: v for k, v in frame.f_locals.items() if not k.startswith("__aiython_")} + objects = {k: v for k, v in frame.f_locals.items() + if not internal_binding(k, unit.runtime_name)} names = statement_names(statement) related = {k: v for k, v in objects.items() if k in names} for name in names - related.keys(): - if name in frame.f_globals and not name.startswith("__aiython_"): + if name in frame.f_globals and not internal_binding(name, unit.runtime_name): related[name] = frame.f_globals[name] frame_code = self.frame_source(frame).get("code", unit.source) return AgentRequest(statement, frame_code, related, @@ -936,7 +951,7 @@ def recover(self, key: str, error: BaseException, attempt: int | None = None) -> from .typed_runtime import SCOPE, Scope, frame_scope scope = frame_scope(frame) or frame.f_locals.get(SCOPE) counts = (scope.recovery_counts if isinstance(scope, Scope) else - frame.f_locals.setdefault("__aiython_recovery_counts__", {})) + frame.f_locals.setdefault(checkpoint.unit.runtime_name + 'recovery_counts', {})) attempt = counts.get(key, 0) + 1 counts[key] = attempt if attempt > 2: diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 1002a2b..6b99c6f 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -47,7 +47,8 @@ class Scope: class TypeRuntime: - def __init__(self): + def __init__(self, manager=None): + self.manager = manager self.classes = weakref.WeakSet() self._classes_lock = threading.RLock() self._contract_cache = threading.local() @@ -216,7 +217,10 @@ def checkpoint(self): # Active enclosing scopes can hold annotated aliases to mutated values. parent = frame.f_back while parent: - if (frame_scope(parent) or SCOPE in parent.f_locals) and parent.f_globals.get(RUNTIME_NAME) is frame.f_globals.get(RUNTIME_NAME): + unit = self.manager.units.get(parent.f_code.co_filename) if self.manager is not None else None + same_runtime = (parent.f_globals.get(unit.runtime_name) is self.manager if unit else + self.manager is None and parent.f_globals.get(RUNTIME_NAME) is frame.f_globals.get(RUNTIME_NAME)) + if (frame_scope(parent) or SCOPE in parent.f_locals) and same_runtime: self.check_frame(parent) parent = parent.f_back finally: del frame @@ -333,8 +337,8 @@ def assign_attribute(self,owner,name,value,annotation=None): finally: del frame -def helper(name,*args): - return ast.Call(ast.Attribute(ast.Attribute(ast.Name(RUNTIME_NAME,ast.Load()),'types',ast.Load()),name,ast.Load()),list(args),[]) +def helper(name,*args,runtime_name=RUNTIME_NAME): + return ast.Call(ast.Attribute(ast.Attribute(ast.Name(runtime_name,ast.Load()),'types',ast.Load()),name,ast.Load()),list(args),[]) def literal(value): @@ -342,12 +346,16 @@ def literal(value): class TypedTransformer(ast.NodeTransformer): - def __init__(self, *, snippet=False): + def __init__(self, *, snippet=False, runtime_name=RUNTIME_NAME): self.snippet = snippet + self.runtime_name = runtime_name self.declarations = {} self.function = False self.delegation_contract = False + def helper(self, name, *args): + return helper(name, *args, runtime_name=self.runtime_name) + @staticmethod def declarations_in(body): result = {} @@ -369,7 +377,7 @@ def body(self,body,*,parameters=None,returns=None,initialize=True,inherited=None while body and isinstance(body[0],ast.ImportFrom) and body[0].module == '__future__': header.append(body[0]); body = body[1:] if initialize: - initial = ast.Expr(helper('enter_generator_scope' if generator_scope else 'enter_scope', + initial = ast.Expr(self.helper('enter_generator_scope' if generator_scope else 'enter_scope', literal(self.declarations),literal(parameters),literal(returns))) ast.copy_location(initial,body[0] if body else header[-1] if header else ast.Constant(None,lineno=1,col_offset=0)) initial._aiython_scope_initializer = True @@ -386,7 +394,7 @@ def body(self,body,*,parameters=None,returns=None,initialize=True,inherited=None transformed = self.visit(statement) output.extend(transformed if isinstance(transformed,list) else [transformed]) if not isinstance(statement,(ast.Return,ast.Raise,ast.Break,ast.Continue)): - output.append(ast.copy_location(ast.Expr(helper('checkpoint')),statement)) + output.append(ast.copy_location(ast.Expr(self.helper('checkpoint')),statement)) self.declarations = previous return header+output @@ -397,7 +405,7 @@ def visit_Module(self,node): body = self.body(node.body) initial = next(i for i,item in enumerate(body) if getattr(item,'_aiython_scope_initializer',False)) header, enter, statements = body[:initial], body[initial], body[initial+1:] - exit_call = ast.copy_location(ast.Expr(helper('exit_scope')), enter) + exit_call = ast.copy_location(ast.Expr(self.helper('exit_scope')), enter) if statements: guard = ast.copy_location(ast.Try(statements, [], [], [exit_call]), enter) guard._aiython_module_guard = True @@ -427,14 +435,14 @@ def visit_FunctionDef(self,node): node.body = self.body(node.body,parameters=parameters,returns=returns, inherited=inherited,generator_scope=is_generator) if isinstance(node,ast.AsyncFunctionDef) and is_generator: - node.body.append(ast.copy_location(ast.Expr(helper('returned',ast.Constant(None))),node)) + node.body.append(ast.copy_location(ast.Expr(self.helper('returned',ast.Constant(None))),node)) else: - node.body.append(ast.copy_location(ast.Return(helper('returned',ast.Constant(None))),node)) + node.body.append(ast.copy_location(ast.Return(self.helper('returned',ast.Constant(None))),node)) initial = next(i for i,n in enumerate(node.body) if getattr(n,'_aiython_scope_initializer',False)) - handler = ast.ExceptHandler(ast.Attribute(ast.Name(RUNTIME_NAME,ast.Load()),'scope_error_type',ast.Load()),None, - [ast.Expr(helper('aborted')),ast.Raise()]) - final = ast.Expr(helper('leaving')) - final = ast.Try([final],[],[],[ast.Expr(helper('exit_generator_scope' if is_generator else 'exit_scope'))]) + handler = ast.ExceptHandler(ast.Attribute(ast.Name(self.runtime_name,ast.Load()),'scope_error_type',ast.Load()),None, + [ast.Expr(self.helper('aborted')),ast.Raise()]) + final = ast.Expr(self.helper('leaving')) + final = ast.Try([final],[],[],[ast.Expr(self.helper('exit_generator_scope' if is_generator else 'exit_scope'))]) guarded = ast.Try(node.body[initial+1:],[handler],[],[final]) guarded._aiython_type_guard = True ast.copy_location(guarded,node) @@ -453,7 +461,7 @@ def function_nodes(node): yield from TypedTransformer.function_nodes(child) def visit_ClassDef(self,node): - node.decorator_list.insert(0, ast.Attribute(ast.Attribute(ast.Name(RUNTIME_NAME,ast.Load()),'types',ast.Load()),'register_class',ast.Load())) + node.decorator_list.insert(0, ast.Attribute(ast.Attribute(ast.Name(self.runtime_name,ast.Load()),'types',ast.Load()),'register_class',ast.Load())) previous = self.function previous_contract = self.delegation_contract self.function = False @@ -461,7 +469,7 @@ def visit_ClassDef(self,node): body = self.body(node.body) header = body[:1] if isinstance(body[0],ast.Expr) and isinstance(body[0].value,ast.Constant) and isinstance(body[0].value.value,str) else [] enter, *statements = body[len(header):] - exit_call = ast.copy_location(ast.Expr(helper('exit_scope')), node) + exit_call = ast.copy_location(ast.Expr(self.helper('exit_scope')), node) if statements: guard = ast.copy_location(ast.Try(statements, [], [], [exit_call]), node) guard._aiython_class_guard = True @@ -481,28 +489,28 @@ def visit_Lambda(self,node): def visit_Return(self,node): if node.value is None: - return [ast.copy_location(ast.Expr(helper('returned',ast.Constant(None))),node),node] - node.value = helper('returned',self.visit(node.value)) + return [ast.copy_location(ast.Expr(self.helper('returned',ast.Constant(None))),node),node] + node.value = self.helper('returned',self.visit(node.value)) return node def visit_Yield(self,node): value = self.visit(node.value) if node.value else ast.Constant(None) - node.value = helper('yielded',value) - return ast.copy_location(helper('sent',node),node) + node.value = self.helper('yielded',value) + return ast.copy_location(self.helper('sent',node),node) def visit_YieldFrom(self,node): node.value = self.visit(node.value) if self.delegation_contract: - node.value = helper('delegate',node.value) + node.value = self.helper('delegate',node.value) return node def visit_AnnAssign(self,node): if node.value is None: return node annotation = ast.unparse(node.annotation) if isinstance(node.target,ast.Name): - node.value = helper('assignment',self.visit(node.value),ast.Constant(node.target.id),ast.Constant(annotation)) + node.value = self.helper('assignment',self.visit(node.value),ast.Constant(node.target.id),ast.Constant(annotation)) return node - node.value = helper('expression',self.visit(node.value),ast.Constant(annotation)) + node.value = self.helper('expression',self.visit(node.value),ast.Constant(annotation)) return node def visit_Assign(self,node): @@ -510,7 +518,7 @@ def visit_Assign(self,node): if len(node.targets) == 1 and isinstance(node.targets[0],ast.Attribute): target = node.targets[0] assign = ast.Call( - ast.Attribute(ast.Attribute(ast.Name(RUNTIME_NAME,ast.Load()),'types',ast.Load()), + ast.Attribute(ast.Attribute(ast.Name(self.runtime_name,ast.Load()),'types',ast.Load()), 'assign_attribute',ast.Load()), [], [ ast.keyword(arg='value',value=node.value), ast.keyword(arg='owner',value=self.visit(target.value)), @@ -518,16 +526,16 @@ def visit_Assign(self,node): ]) return ast.copy_location(ast.Expr(assign),node) for target in node.targets: - if isinstance(target,ast.Name) and not target.id.startswith('__aiython_'): - node.value = helper('assignment',node.value,ast.Constant(target.id)) + if isinstance(target,ast.Name): + node.value = self.helper('assignment',node.value,ast.Constant(target.id)) return node def visit_AugAssign(self,node): names = [node.target.id] if isinstance(node.target,ast.Name) else [] - return [ast.copy_location(ast.Expr(helper('reassigning',literal(names))),node),node] + return [ast.copy_location(ast.Expr(self.helper('reassigning',literal(names))),node),node] def visit_NamedExpr(self,node): - node.value = helper('assignment',self.visit(node.value),ast.Constant(node.target.id)) + node.value = self.helper('assignment',self.visit(node.value),ast.Constant(node.target.id)) return node def visit_If(self,node): @@ -542,12 +550,12 @@ def nested(self,body): result = self.visit(statement) output.extend(result if isinstance(result,list) else [result]) if not isinstance(statement,(ast.Return,ast.Raise,ast.Break,ast.Continue)): - output.append(ast.copy_location(ast.Expr(helper('checkpoint')),statement)) + output.append(ast.copy_location(ast.Expr(self.helper('checkpoint')),statement)) return output def visit_For(self,node): node.iter = self.visit(node.iter) - node.body = [ast.copy_location(ast.Expr(helper('checkpoint')),node)] + self.nested(node.body) + node.body = [ast.copy_location(ast.Expr(self.helper('checkpoint')),node)] + self.nested(node.body) node.orelse = self.nested(node.orelse) return node visit_AsyncFor = visit_For @@ -559,7 +567,7 @@ def visit_While(self,node): def visit_With(self,node): node.items = [self.visit(item) for item in node.items] - node.body = [ast.copy_location(ast.Expr(helper('checkpoint')),node)] + self.nested(node.body) + node.body = [ast.copy_location(ast.Expr(self.helper('checkpoint')),node)] + self.nested(node.body) return node visit_AsyncWith = visit_With @@ -573,8 +581,9 @@ def visit_Try(self,node): class ExpectedTypes(ast.NodeVisitor): """Propagate declared contracts to direct AI values before code generation.""" - def __init__(self,blocks): + def __init__(self,blocks,runtime_name=RUNTIME_NAME): self.blocks = blocks + self.runtime_name = runtime_name self.declarations = {} self.returns = None self.functions = {} @@ -587,7 +596,7 @@ def signatures(body): def apply(self,node,annotation): if node is None or not annotation: return if (isinstance(node,ast.Call) and isinstance(node.func,ast.Attribute) - and isinstance(node.func.value,ast.Name) and node.func.value.id == RUNTIME_NAME + and isinstance(node.func.value,ast.Name) and node.func.value.id == self.runtime_name and node.func.attr == 'execute' and node.args and isinstance(node.args[0],ast.Constant)): self.blocks[node.args[0].value].output_type = annotation elif isinstance(node,ast.IfExp): diff --git a/tests/test_collaboration.py b/tests/test_collaboration.py index 38f0041..787c05b 100644 --- a/tests/test_collaboration.py +++ b/tests/test_collaboration.py @@ -107,19 +107,22 @@ def test_process_start_methods_import_aiython_source_and_exchange_messages(self) (root / "aiython.toml").write_text('version = 3\nmodel = "openai/test"\n') (root / "worker.py").write_text( 'from aiython import join\n' + '__aiython_runtime__ = 12\n' 'def work(ticket):\n' ' with join(ticket) as me:\n' ' me.send("main", {"ok": True})\n' - ' return 12\n' + ' return __aiython_runtime__\n' 'def unused():\n' ' return choose a value\n') (root / "main.py").write_text( + '__aiython_runtime__ = 7\n' 'from concurrent.futures import ProcessPoolExecutor\n' 'from multiprocessing import get_all_start_methods, get_context\n' 'from pathlib import Path\n' 'from aiython import group, worker_entry\n' 'if __name__ == "__main__":\n' ' with group() as team:\n' + ' assert __aiython_runtime__ == 7\n' ' assert team.project_root == str(Path(__file__).parent)\n' ' answer = []\n' ' for method in ("spawn", "forkserver"):\n' diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index acc8bf2..ff761ab 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -99,6 +99,106 @@ def report(stage): self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), (python.returncode, python.stdout, python.stderr)) + def test_runtime_name_collision_in_entry_and_imports(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'helper.py').write_text( + '__aiython_runtime__ = 11\n' + 'def read(): return __aiython_runtime__\n') + (root / 'main.py').write_text( + '__aiython_runtime__ = 7\n' + '__aiython_runtime_1__ = 8\n' + 'import helper\n' + 'print(__aiython_runtime__, __aiython_runtime_1__, helper.read())\n') + package = root / 'runtime_package' + package.mkdir() + (package / '__init__.py').write_text('__aiython_runtime__ = 5\n') + (package / '__main__.py').write_text( + 'from . import __aiython_runtime__ as parent_value\n' + '__aiython_runtime__ = 7\n' + 'print(parent_value, __aiython_runtime__)\n') + for python_args, aiython_args in ((['main.py'], ['main.py']), + (['-m', 'runtime_package'], ['-m', 'runtime_package'])): + with self.subTest(python_args=python_args): + python = subprocess.run([sys.executable, *python_args], cwd=root, + capture_output=True, text=True, timeout=10) + aiython = subprocess.run([sys.executable, '-m', 'aiython', *aiython_args], cwd=root, + capture_output=True, text=True, timeout=10) + self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), + (python.returncode, python.stdout, python.stderr)) + + def test_ai_source_and_bridge_preserve_user_runtime_name(self): + from aiython.frontend import parse + + class Agent: + def __init__(self): + self.observed = [] + + def execute(self, request, bridge): + bridge.exec('global __aiython_runtime__\n__aiython_runtime__ += 1\n' + 'def generated(): return __aiython_runtime__') + self.observed.append(bridge.eval('generated()')) + return 7 + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / 'main.py' + source = ('import asyncio\n' + 'from aiython.collaboration import Group\n' + '__aiython_runtime__ = 7\n' + '__aiython_runtime_1__ = 8\n' + 'answer: int = choose seven\n' + 'async def choose() -> int:\n' + ' return choose seven\n' + 'async_answer = asyncio.run(choose())\n' + 'result = (__aiython_runtime__, __aiython_runtime_1__, answer, async_answer, Group().project_root)\n') + path.write_text(source) + self.assertEqual(parse(source, str(path)).runtime_name, '__aiython_runtime_2__') + config = ResolvedConfig(None, root, 'default', + {'default': ProfileConfig('default', 'fake', 'test')}) + agent = Agent() + namespace = run_script(path, config=config, agent_factory=lambda _: agent) + self.assertEqual(namespace['result'], (9, 8, 7, 7, str(root))) + self.assertEqual(agent.observed, [8, 9]) + self.assertEqual([name for name in namespace if name.startswith('__aiython_runtime')], + ['__aiython_runtime_2__', '__aiython_runtime__', '__aiython_runtime_1__']) + + def test_ai_request_and_bridge_expose_user_prefixed_names(self): + case = self + + class Agent: + def execute(self, request, bridge): + case.assertEqual( + {key: request.related_objects[key] + for key in ('__aiython_runtime__', '__aiython_user', + '__aiython_recovery_counts__', '__aiython_recovery_attempt_user')}, + {'__aiython_runtime__': 7, '__aiython_user': 5, + '__aiython_recovery_counts__': 3, '__aiython_recovery_attempt_user': 4}) + bridge.set('__aiython_runtime__', 8) + bridge.set('__aiython_user', 6) + bridge.set('__aiython_recovery_counts__', 9) + bridge.set('__aiython_recovery_attempt_user', 10) + with case.assertRaises(ValueError): + bridge.set('__aiython_runtime_1__', None) + return 7 + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / 'main.py' + path.write_text('__aiython_runtime__ = 7\n' + '__aiython_user = 5\n' + '__aiython_recovery_counts__ = 3\n' + '__aiython_recovery_attempt_user = 4\n' + 'answer = choose seven using __aiython_runtime__ and __aiython_user ' + 'and __aiython_recovery_counts__ and __aiython_recovery_attempt_user\n' + 'result = (__aiython_runtime__, __aiython_user, ' + '__aiython_recovery_counts__, __aiython_recovery_attempt_user, answer)\n') + agent = Agent() + config = ResolvedConfig(None, root, 'default', + {'default': ProfileConfig('default', 'fake', 'test')}) + namespace = run_script(path, config=config, agent_factory=lambda _: agent) + self.assertEqual(namespace['result'], (8, 6, 9, 10, 7)) + def test_directory_and_zipapp_execution_match_cpython(self): source = '''import atexit, inspect, sys from helper import value @@ -159,6 +259,7 @@ def report(): print(sys.argv, sys.orig_argv, sys.path[:2]) print(vars(main).get('__file__', 'ABSENT'), vars(main).get('__cached__', 'ABSENT')) print(type(__loader__).__name__, __package__, __spec__, inspect.currentframe().f_code.co_filename) + print(sorted(name for name in globals() if name.startswith('__aiython_runtime'))) report() atexit.register(report) ''') @@ -596,6 +697,30 @@ def test_language_constructs_match_cpython(self): 'empty script': '', 'entry builtins module': '''import builtins print(type(__builtins__).__name__, __builtins__ is builtins) +''', + 'runtime binding collision': '''__aiython_runtime__ = 7 +__aiython_runtime_1__ = 8 +print(__aiython_runtime__, __aiython_runtime_1__, f'{__aiython_runtime__}') +globals()['__aiython_runtime__'] += 1 +print(__aiython_runtime__) +def local(): + __aiython_runtime__ = 9 + return __aiython_runtime__ +class Box: + __aiython_runtime__ = 10 + def read(self): return self.__aiython_runtime__ +print(local(), Box().read()) +''', + 'other internal-looking user names': '''__aiython_type_scope__ = 3 +__aiython_recovery_counts__ = 4 +__aiython_recovery_attempt_user = 5 +def values(): + __aiython_type_scope__ = 6 + return __aiython_type_scope__ +class Box: + __aiython_type_scope__ = 7 +print(__aiython_type_scope__, __aiython_recovery_counts__, + __aiython_recovery_attempt_user, values(), Box.__aiython_type_scope__) ''', 'entry module metadata': '''print(type(__loader__).__name__, __loader__.name, __loader__.path == __file__) print('__annotations__' in globals(), '__annotate__' in globals()) diff --git a/tests/test_runtime_edges.py b/tests/test_runtime_edges.py index 075ade0..1832139 100644 --- a/tests/test_runtime_edges.py +++ b/tests/test_runtime_edges.py @@ -40,7 +40,7 @@ def setUp(self): def test_eval_code_object_binding_and_unknown_handle_validation(self): self.assertEqual(self.bridge.eval(compile('1 + 2', '', 'eval')), 3) - for name in ('not a name', '__aiython_hidden'): + for name in ('not a name', '__aiython_runtime__'): with self.subTest(name=name), self.assertRaisesRegex(ValueError, 'non-reserved'): self.bridge.set(name, 1) with self.assertRaisesRegex(ValueError, 'Unknown object handle'): From 4dcafc6302e0d93e014fde3221dcbc7f366df6d0 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 15:14:12 +0700 Subject: [PATCH 15/72] Hide runtime bindings from Python globals --- src/aiython/_worker_main.py | 4 +- src/aiython/cli.py | 7 +--- src/aiython/collaboration.py | 8 ++-- src/aiython/runtime.py | 62 +++++++++++++++++++++++++----- src/aiython/typed_runtime.py | 2 +- tests/test_overhead.py | 11 +++--- tests/test_python_compatibility.py | 29 ++++++++++---- tests/test_runtime_edges.py | 4 +- 8 files changed, 89 insertions(+), 38 deletions(-) diff --git a/src/aiython/_worker_main.py b/src/aiython/_worker_main.py index c42981c..3f6d836 100644 --- a/src/aiython/_worker_main.py +++ b/src/aiython/_worker_main.py @@ -13,14 +13,12 @@ from aiython.cli import ProjectFinder, read_source from aiython.config import resolve - from aiython.frontend import runtime_binding_name from aiython.runtime import Runtime source_path = Path(os.environ["AIYTHON_SPAWN_ENTRY"]) runtime = Runtime(resolve(source_path)) source = read_source(source_path) - globals().update({"__file__": str(source_path), "__package__": None, - runtime_binding_name(source): runtime}) + globals().update({"__file__": str(source_path), "__package__": None}) sys.path.insert(0, str(source_path.parent)) sys.meta_path.insert(0, ProjectFinder(runtime)) exec(runtime.compile_source(source, str(source_path)), globals()) diff --git a/src/aiython/cli.py b/src/aiython/cli.py index 6e5ccd0..1cbe744 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -18,7 +18,7 @@ from time import perf_counter from .config import describe, resolve -from .frontend import parse, runtime_binding_name +from .frontend import parse from .models import AiythonError, ConfigError from .runtime import Runtime @@ -38,7 +38,6 @@ def __init__(self, name, path, runtime): def exec_module(self, module): source = read_source(self.path) - module.__dict__[runtime_binding_name(source)] = self.runtime exec(self.runtime.compile_source(source, self.path), module.__dict__) @@ -180,10 +179,8 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta runtime = runtime or Runtime(config, agent_factory=agent_factory, stats=stats, trace_plan=trace_plan) if compiled_code is None: source = read_source(path) if source is None else source - runtime_name = runtime_binding_name(source) code = runtime.compile_source(source, str(display_path), entry=True) else: - runtime_name = None code = compiled_code module = initial_main or types.ModuleType("__main__") module.__dict__.update({"__package__": module_spec.parent if module_spec else None, @@ -194,8 +191,6 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta if entry_kind == "bytecode" else importlib.machinery.SourceFileLoader("__main__", str(display_path)), "__builtins__": builtins}) - if runtime_name is not None: - module.__dict__[runtime_name] = runtime if entry_kind != "command": module.__dict__.update({"__file__": module_spec.origin if module_spec else str(display_path), "__cached__": module_spec.cached if module_spec else None}) diff --git a/src/aiython/collaboration.py b/src/aiython/collaboration.py index 8c75435..d785e95 100644 --- a/src/aiython/collaboration.py +++ b/src/aiython/collaboration.py @@ -422,11 +422,9 @@ async def send_a2a(self, url, payload, *, timeout=120, api_key_env=None): class Group: def __init__(self, project_root=None): if project_root is None: - main_module = sys.modules.get("__main__") - from .runtime import Runtime - runtime = next((value for key, value in vars(main_module).items() - if key.startswith("__aiython_runtime") and key.endswith("__") - and issubclass(type(value), Runtime)), None) if main_module is not None else None + from .cli import ProjectFinder + runtime = next((finder.runtime for finder in sys.meta_path + if isinstance(finder, ProjectFinder)), None) project_root = runtime.config.project_root if runtime is not None else Path.cwd() self.project_root = str(Path(project_root).resolve()) self._ticket: Ticket | None = None diff --git a/src/aiython/runtime.py b/src/aiython/runtime.py index 4b22ad7..36c23e7 100644 --- a/src/aiython/runtime.py +++ b/src/aiython/runtime.py @@ -118,12 +118,37 @@ def expression_code(source): def snippet_code(source, runtime_name=RUNTIME_NAME): from .typed_runtime import TypedTransformer tree = TypedTransformer(snippet=True, runtime_name=runtime_name).visit(ast.parse(source, '')) + tree = RuntimeReferences(runtime_name).visit(tree) ast.fix_missing_locations(tree) return compile(tree, '', 'exec') +class RuntimeReferences(ast.NodeTransformer): + """Keep generated runtime access out of Python's user-visible namespaces.""" + + def __init__(self, runtime_name): + self.runtime_name = runtime_name + + def visit_Name(self, node): + if node.id == self.runtime_name and isinstance(node.ctx, ast.Load): + return ast.copy_location(ast.Constant(self.runtime_name + 'constant'), node) + return node + + +def bind_runtime(code, runtime_name, runtime): + """Bind an execution's runtime after compiling or retrieving cached code.""" + marker = runtime_name + 'constant' + constants = tuple(bind_runtime(value, runtime_name, runtime) + if isinstance(value, types.CodeType) else + runtime if isinstance(value, str) and value == marker else value + for value in code.co_consts) + return (code.replace(co_consts=constants) + if any(bound is not original for bound, original in zip(constants, code.co_consts)) + else code) + + def internal_binding(name, runtime_name): - return (name == runtime_name or name == runtime_name + 'recovery_counts' + return (name == runtime_name + 'recovery_counts' or name.startswith(runtime_name + 'recovery_attempt_')) @@ -215,11 +240,8 @@ def eval(self, code: str) -> Any: def exec(self, code: str) -> None: prepare = snippet_code if len(code) <= 64 * 1024 else snippet_code.__wrapped__ - occupied = {name for namespace in (self.frame.f_globals, self.frame.f_locals) - for name, value in namespace.items() if value is not self.manager} - runtime_name = runtime_binding_name(code, occupied) - compiled = prepare(code, runtime_name) - self.frame.f_globals[runtime_name] = self.manager + runtime_name = runtime_binding_name(code) + compiled = bind_runtime(prepare(code, runtime_name), runtime_name, self.manager) from .source_guard import protect_source with protect_source(self.manager): exec(compiled, self.frame.f_globals, self.namespace()) @@ -620,7 +642,12 @@ def body(self, statements, *, nested=True, preserve_docstring=False, scoped_retr return result def visit_Module(self, node): - node.body = self.body(node.body, nested=False) + previous = self.scoped_retries + self.scoped_retries = True + try: + node.body = self.body(node.body, nested=False) + finally: + self.scoped_retries = previous return node def visit_FunctionDef(self, node): @@ -677,6 +704,7 @@ def current_exception(): def __init__(self, config: ResolvedConfig, *, agent_factory=None, stats=False, trace_plan=False): self._lock = threading.RLock() + self._bound_codes = OrderedDict() self.config = config from .typed_runtime import TypeRuntime self.types = TypeRuntime(self) @@ -728,7 +756,7 @@ def compile_source(self, source, filename, *, entry=False): if self.stats.enabled: self.stats.preparation_cache_hits += 1 self.stats.prepare_seconds += perf_counter() - started - return code + return self._bind_compiled(code, unit.runtime_name) unit = parse(source, filename) if self.stats.enabled: self.stats.parse_seconds += perf_counter() - started @@ -749,7 +777,21 @@ def compile_source(self, source, filename, *, entry=False): _PREPARED[key] = (code, packet) while len(_PREPARED) > _PREPARED_LIMIT or sum(len(v[1]) for v in _PREPARED.values()) > _PREPARED_BYTES: _PREPARED.popitem(last=False) - return code + return self._bind_compiled(code, unit.runtime_name, cache=len(source) <= 256 * 1024) + + def _bind_compiled(self, code, runtime_name, *, cache=True): + if not cache: + return bind_runtime(code, runtime_name, self) + with self._lock: + bound = self._bound_codes.get(code) + if bound is None: + bound = bind_runtime(code, runtime_name, self) + self._bound_codes[code] = bound + if len(self._bound_codes) > _PREPARED_LIMIT: + self._bound_codes.popitem(last=False) + else: + self._bound_codes.move_to_end(code) + return bound def register(self, unit): self.units[unit.filename] = unit @@ -834,6 +876,8 @@ def _prepare(self, unit: Unit, *, entry: bool = False): scoped_retries=True)) guard.body = body ast.fix_missing_locations(tree) + tree = RuntimeReferences(unit.runtime_name).visit(tree) + ast.fix_missing_locations(tree) return compile(tree, unit.filename, "exec", dont_inherit=True) def lookup(self, name: str): diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 6b99c6f..17d61cd 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -218,7 +218,7 @@ def checkpoint(self): parent = frame.f_back while parent: unit = self.manager.units.get(parent.f_code.co_filename) if self.manager is not None else None - same_runtime = (parent.f_globals.get(unit.runtime_name) is self.manager if unit else + same_runtime = (any(value is self.manager for value in parent.f_code.co_consts) if unit else self.manager is None and parent.f_globals.get(RUNTIME_NAME) is frame.f_globals.get(RUNTIME_NAME)) if (frame_scope(parent) or SCOPE in parent.f_locals) and same_runtime: self.check_frame(parent) diff --git a/tests/test_overhead.py b/tests/test_overhead.py index e2508f0..fbc24cf 100644 --- a/tests/test_overhead.py +++ b/tests/test_overhead.py @@ -15,7 +15,6 @@ from aiython.capabilities import Store, CapabilityResult, Embeddings from aiython.cli import run_script -from aiython.frontend import RUNTIME_NAME from aiython.models import ProfileConfig, ResolvedConfig from aiython.providers import LiteLLMProvider from aiython.runtime import Runtime @@ -65,16 +64,18 @@ def test_preparation_cache_preserves_fresh_state_and_source_changes(self): first, second = Runtime(config, stats=True), Runtime(config, stats=True) code_a = first.compile_source(source, str(path), entry=True) code_b = second.compile_source(source, str(path), entry=True) - self.assertIs(code_a, code_b) + self.assertIsNot(code_a, code_b) + self.assertIs(code_a, first.compile_source(source, str(path), entry=True)) self.assertEqual(second.stats.preparation_cache_hits, 1) self.assertIsNot(first.units[str(path)], second.units[str(path)]) self.assertEqual(set(first.checkpoints), set(second.checkpoints)) - left, right = {RUNTIME_NAME: first}, {RUNTIME_NAME: second} + left, right = {}, {} exec(code_a, left) exec(code_b, right) + self.assertFalse(any(name.startswith('__aiython_runtime') for name in left | right)) self.assertIsNot(left['answer'], right['answer']) changed = Runtime(config) - namespace = {RUNTIME_NAME: changed} + namespace = {} exec(changed.compile_source(source.replace('append(1)', 'append(2)'), str(path), entry=True), namespace) self.assertEqual(namespace['answer'], [2]) self.assertEqual(left['answer'], [1]) @@ -105,7 +106,7 @@ def test_warm_program_still_catches_foreign_alias_mutations(self): source = 'items: list[int] = [1, 2]\nalias = items\nmutate(alias)\nanswer = items\n' for _ in range(2): runtime = Runtime(config) - namespace = {RUNTIME_NAME: runtime, 'mutate': lambda values: values.__setitem__(0, True)} + namespace = {'mutate': lambda values: values.__setitem__(0, True)} with self.assertRaises(TypeViolation): exec(runtime.compile_source(source, str(path), entry=True), namespace) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index ff761ab..f6e81b8 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -104,19 +104,24 @@ def test_runtime_name_collision_in_entry_and_imports(self): root = Path(directory) (root / 'helper.py').write_text( '__aiython_runtime__ = 11\n' - 'def read(): return __aiython_runtime__\n') + 'def read(): return __aiython_runtime__\n' + 'def internal_names(): return sorted(name for name in globals() ' + 'if name.startswith("__aiython_runtime"))\n') (root / 'main.py').write_text( '__aiython_runtime__ = 7\n' '__aiython_runtime_1__ = 8\n' 'import helper\n' - 'print(__aiython_runtime__, __aiython_runtime_1__, helper.read())\n') + 'print(__aiython_runtime__, __aiython_runtime_1__, helper.read())\n' + 'print(sorted(name for name in globals() if name.startswith("__aiython_runtime")), ' + 'helper.internal_names())\n') package = root / 'runtime_package' package.mkdir() (package / '__init__.py').write_text('__aiython_runtime__ = 5\n') (package / '__main__.py').write_text( 'from . import __aiython_runtime__ as parent_value\n' '__aiython_runtime__ = 7\n' - 'print(parent_value, __aiython_runtime__)\n') + 'print(parent_value, __aiython_runtime__)\n' + 'print(sorted(name for name in globals() if name.startswith("__aiython_runtime")))\n') for python_args, aiython_args in ((['main.py'], ['main.py']), (['-m', 'runtime_package'], ['-m', 'runtime_package'])): with self.subTest(python_args=python_args): @@ -161,7 +166,7 @@ def execute(self, request, bridge): self.assertEqual(namespace['result'], (9, 8, 7, 7, str(root))) self.assertEqual(agent.observed, [8, 9]) self.assertEqual([name for name in namespace if name.startswith('__aiython_runtime')], - ['__aiython_runtime_2__', '__aiython_runtime__', '__aiython_runtime_1__']) + ['__aiython_runtime__', '__aiython_runtime_1__']) def test_ai_request_and_bridge_expose_user_prefixed_names(self): case = self @@ -178,8 +183,7 @@ def execute(self, request, bridge): bridge.set('__aiython_user', 6) bridge.set('__aiython_recovery_counts__', 9) bridge.set('__aiython_recovery_attempt_user', 10) - with case.assertRaises(ValueError): - bridge.set('__aiython_runtime_1__', None) + bridge.set('__aiython_runtime_1__', 11) return 7 with tempfile.TemporaryDirectory() as directory: @@ -192,12 +196,13 @@ def execute(self, request, bridge): 'answer = choose seven using __aiython_runtime__ and __aiython_user ' 'and __aiython_recovery_counts__ and __aiython_recovery_attempt_user\n' 'result = (__aiython_runtime__, __aiython_user, ' - '__aiython_recovery_counts__, __aiython_recovery_attempt_user, answer)\n') + '__aiython_recovery_counts__, __aiython_recovery_attempt_user, ' + '__aiython_runtime_1__, answer)\n') agent = Agent() config = ResolvedConfig(None, root, 'default', {'default': ProfileConfig('default', 'fake', 'test')}) namespace = run_script(path, config=config, agent_factory=lambda _: agent) - self.assertEqual(namespace['result'], (8, 6, 9, 10, 7)) + self.assertEqual(namespace['result'], (8, 6, 9, 10, 11, 7)) def test_directory_and_zipapp_execution_match_cpython(self): source = '''import atexit, inspect, sys @@ -695,6 +700,14 @@ def target(): return 1 def test_language_constructs_match_cpython(self): cases = { 'empty script': '', + 'no runtime binding in globals': '''print(sorted(name for name in globals() if name.startswith('__aiython_'))) +''', + 'nested statements leave globals unchanged': '''if True: + value = 1 +for item in range(2): + value += item +print(value, sorted(name for name in globals() if name.startswith('__aiython_'))) +''', 'entry builtins module': '''import builtins print(type(__builtins__).__name__, __builtins__ is builtins) ''', diff --git a/tests/test_runtime_edges.py b/tests/test_runtime_edges.py index 1832139..8211c22 100644 --- a/tests/test_runtime_edges.py +++ b/tests/test_runtime_edges.py @@ -40,9 +40,11 @@ def setUp(self): def test_eval_code_object_binding_and_unknown_handle_validation(self): self.assertEqual(self.bridge.eval(compile('1 + 2', '', 'eval')), 3) - for name in ('not a name', '__aiython_runtime__'): + for name in ('not a name',): with self.subTest(name=name), self.assertRaisesRegex(ValueError, 'non-reserved'): self.bridge.set(name, 1) + self.bridge.set('__aiython_runtime__', 1) + self.assertEqual(self.bridge.eval('__aiython_runtime__'), 1) with self.assertRaisesRegex(ValueError, 'Unknown object handle'): self.bridge.dereference('missing') for depth, limit in ((0, 1), (1, 101), (True, 1)): From 9eb91799bda71754973a0e243367a75b7969de29 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 15:20:27 +0700 Subject: [PATCH 16/72] Preserve Python bytecode cache on source imports --- src/aiython/cli.py | 8 +++++++- tests/test_python_compatibility.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/aiython/cli.py b/src/aiython/cli.py index 1cbe744..ebca8c9 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -38,7 +38,10 @@ def __init__(self, name, path, runtime): def exec_module(self, module): source = read_source(self.path) - exec(self.runtime.compile_source(source, self.path), module.__dict__) + code = self.runtime.compile_source(source, self.path) + if not sys.dont_write_bytecode and not self.runtime.units[self.path].blocks: + self.get_code(module.__name__) + exec(code, module.__dict__) class ProjectFinder(importlib.abc.MetaPathFinder): @@ -180,6 +183,9 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta if compiled_code is None: source = read_source(path) if source is None else source code = runtime.compile_source(source, str(display_path), entry=True) + if (entry_kind == "module" and module_spec is not None and not sys.dont_write_bytecode + and not runtime.units[str(display_path)].blocks): + module_spec.loader.get_code(module_spec.name) else: code = compiled_code module = initial_main or types.ModuleType("__main__") diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index f6e81b8..f4a2422 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -132,6 +132,35 @@ def test_runtime_name_collision_in_entry_and_imports(self): self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), (python.returncode, python.stdout, python.stderr)) + def test_source_import_and_module_bytecode_cache_match_cpython(self): + with tempfile.TemporaryDirectory() as directory: + for mode in ('import', 'module'): + for disabled in (False, True): + with self.subTest(mode=mode, disabled=disabled): + outputs = [] + for executable in ('python', 'aiython'): + root = Path(directory) / f'{mode}-{disabled}-{executable}' + root.mkdir() + (root / 'helper.py').write_text('value = 3\n') + (root / 'main.py').write_text( + 'from pathlib import Path\nimport helper\n' + 'print(Path(helper.__cached__).is_file())\n') + (root / 'target.py').write_text( + 'from pathlib import Path\nprint(Path(__cached__).is_file())\n') + command = (['main.py'] if mode == 'import' else ['-m', 'target']) + if executable == 'aiython': + command.insert(0, '-m') + command.insert(1, 'aiython') + env = os.environ.copy() + if disabled: + env['PYTHONDONTWRITEBYTECODE'] = '1' + else: + env.pop('PYTHONDONTWRITEBYTECODE', None) + result = subprocess.run([sys.executable, *command], cwd=root, + env=env, capture_output=True, text=True) + outputs.append((result.returncode, result.stdout, result.stderr)) + self.assertEqual(outputs[1], outputs[0]) + def test_ai_source_and_bridge_preserve_user_runtime_name(self): from aiython.frontend import parse From e630a5de53c199bbc127ea13222ae736445e414a Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 15:25:17 +0700 Subject: [PATCH 17/72] Explain modules beneath packages with AI syntax --- docs/configuration.md | 2 +- src/aiython/cli.py | 43 +++++++++++++++++++++++++++--- tests/test_python_compatibility.py | 35 ++++++++++++++++++++++-- 3 files changed, 73 insertions(+), 7 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2bd6b0f..c927b41 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -64,7 +64,7 @@ Model IDs use LiteLLM syntax. A route can also be `{ model = "...", api_base = " Run `aiython config show --script PATH` to inspect resolved settings. `--config` chooses an exact file. `--profile` selects a default named profile; `--force-profile` overrides source directives. CLI options for script execution go before the script path, while later arguments go to the program. -Run an importable module or package with `aiython -m package.module` or `aiython -m package`. Put Aiython options before `-m`; arguments after the module name go to that module. Packages need a `__main__.py`, as with `python -m`. Source files get Aiython's source transformation; modules available only as bytecode run as Python bytecode. During execution, project parent packages with AI statements are transformed too. `--explain -m` imports the parent package using ordinary Python, so it cannot explain a module whose parent package requires AI syntax. +Run an importable module or package with `aiython -m package.module` or `aiython -m package`. Put Aiython options before `-m`; arguments after the module name go to that module. Packages need a `__main__.py`, as with `python -m`. Source files get Aiython's source transformation; modules available only as bytecode run as Python bytecode. During execution, project parent packages with AI statements are transformed too. `--explain -m` resolves parent packages with AI syntax without executing them. The CLI also accepts `aiython -c 'print(1)'`, `aiython -` for source from stdin, and a directory or zipapp containing `__main__.py`. Arguments after the command, `-`, or path go to the program. For `-c` and stdin, Aiython searches for `aiython.toml` from the current directory. A direct `.pyc` file and source-less `__main__.pyc` run as Python bytecode without source transformation or `--explain` output. diff --git a/src/aiython/cli.py b/src/aiython/cli.py index ebca8c9..8e0a157 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -44,6 +44,21 @@ def exec_module(self, module): exec(code, module.__dict__) +class StaticProjectLoader(ProjectLoader): + """Expose a package path during --explain without running its AI source.""" + + def __init__(self, name, path, runtime, loaded): + super().__init__(name, path, runtime) + self.loaded = loaded + + def exec_module(self, module): + parent_name, _, child_name = module.__name__.rpartition(".") + parent = sys.modules.get(parent_name) if parent_name else None + old_value = vars(parent).get(child_name) if parent is not None else None + had_value = parent is not None and child_name in vars(parent) + self.loaded.append((module, parent, child_name, had_value, old_value)) + + class ProjectFinder(importlib.abc.MetaPathFinder): def __init__(self, runtime): self.runtime = runtime @@ -67,6 +82,11 @@ def find_spec(self, fullname, path=None, target=None): class ModuleStartFinder(ProjectFinder): """Transform project imports with AI syntax while -m resolves its target.""" + def __init__(self, runtime, *, explain=False): + super().__init__(runtime) + self.explain = explain + self.static_modules = [] + def find_spec(self, fullname, path=None, target=None): spec = super().find_spec(fullname, path, target) if spec is None: @@ -77,6 +97,9 @@ def find_spec(self, fullname, path=None, target=None): try: compile(source, spec.origin, "exec", dont_inherit=True) except SyntaxError: + if self.explain: + spec.loader = StaticProjectLoader(fullname, spec.origin, self.runtime, + self.static_modules) return spec return None @@ -122,7 +145,7 @@ def module_details(name): return spec, source, code -def module_source(name, arguments=(), *, runtime=None): +def module_source(name, arguments=(), *, runtime=None, explain=False): old_main = sys.modules.get("__main__") old_argv, old_orig_argv = sys.argv, sys.orig_argv interpreter_args = interpreter_arguments() @@ -131,7 +154,7 @@ def module_source(name, arguments=(), *, runtime=None): initial_main.__builtins__ = builtins if sys.version_info < (3, 14): initial_main.__annotations__ = {} - finder = ModuleStartFinder(runtime) if runtime is not None else None + finder = ModuleStartFinder(runtime, explain=explain) if runtime is not None else None try: sys.modules["__main__"] = initial_main sys.argv = ["-m", *arguments] @@ -144,6 +167,14 @@ def module_source(name, arguments=(), *, runtime=None): finally: if finder is not None: sys.meta_path.remove(finder) + for module, parent, child_name, had_value, old_value in reversed(finder.static_modules): + if sys.modules.get(module.__name__) is module: + sys.modules.pop(module.__name__) + if parent is not None and vars(parent).get(child_name) is module: + if had_value: + setattr(parent, child_name, old_value) + else: + delattr(parent, child_name) sys.argv = old_argv sys.orig_argv = old_orig_argv if old_main is not None: @@ -344,7 +375,7 @@ def main(argv=None): # A source file with AI syntax cannot be imported as an ordinary # parent package. A SyntaxError raised *by* package code is a # program error and must not cause that package to run twice. - if args.explain or exc.filename is None: + if exc.filename is None: raise traceback = exc.__traceback__ while traceback is not None: @@ -358,10 +389,14 @@ def main(argv=None): module_runtime = Runtime(module_config, stats=args.stats, trace_plan=args.trace_plan) try: module_spec, source, original_code, initial_main = module_source( - module_invocation, script_args, runtime=module_runtime) + module_invocation, script_args, runtime=module_runtime, + explain=args.explain) except BaseException: module_runtime.capabilities.close() raise + if args.explain: + module_runtime.capabilities.close() + module_runtime = None path = Path(module_spec.origin or (original_code.co_filename if original_code else module_invocation)) entry_kind = "module" if source is None: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index f4a2422..0f37518 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -415,8 +415,39 @@ def execute(self, request, runtime): self.assertEqual((namespace['prefix'], namespace['answer']), (7, 7)) explanation = subprocess.run([sys.executable, '-m', 'aiython', '--explain', '-m', 'ai_parent_package.task'], cwd=root, capture_output=True, text=True) - self.assertNotEqual(explanation.returncode, 0) - self.assertIn('SyntaxError', explanation.stderr) + self.assertEqual(explanation.returncode, 0, explanation.stderr) + self.assertEqual([block['statement'] for block in json.loads(explanation.stdout)['blocks']], + ['decide another number']) + + def test_explain_nested_ai_packages_does_not_execute_parents(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + parent = root / 'outer_package' + child = parent / 'inner_package' + child.mkdir(parents=True) + (parent / '__init__.py').write_text( + 'raise RuntimeError("outer package executed")\nvalue = decide a value\n') + (child / '__init__.py').write_text( + 'raise RuntimeError("inner package executed")\nvalue = decide another value\n') + (child / 'task.py').write_text('answer = determine the answer\n') + result = subprocess.run([sys.executable, '-m', 'aiython', '--explain', '-m', + 'outer_package.inner_package.task'], cwd=root, + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual([block['statement'] for block in json.loads(result.stdout)['blocks']], + ['determine the answer']) + runtime = Runtime(ResolvedConfig(None, root)) + try: + with patch('pathlib.Path.cwd', return_value=root), patch.object(sys, 'path', sys.path[:]): + spec, source, code, _ = module_source( + 'outer_package.inner_package.task', runtime=runtime, explain=True) + self.assertEqual(spec.name, 'outer_package.inner_package.task') + self.assertIn('determine the answer', source) + self.assertIsNone(code) + self.assertNotIn('outer_package', sys.modules) + self.assertNotIn('outer_package.inner_package', sys.modules) + finally: + runtime.capabilities.close() def test_cli_retries_parent_ai_source_with_one_runtime(self): class Agent: From 609e65479094ce046d002ddecbedfba24d3b6aad Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 15:36:17 +0700 Subject: [PATCH 18/72] Support variadic generic parameter and tuple annotations --- docs/type-safety.md | 2 +- src/aiython/type_constraints.py | 29 ++++++++++++++++++++++++ src/aiython/typed_runtime.py | 33 ++++++++++++++++++++++++++-- tests/test_python_compatibility.py | 16 ++++++++++++++ tests/test_type_constraints_edges.py | 19 +++++++++++++++- 5 files changed, 95 insertions(+), 4 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 13df575..c19817c 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -42,7 +42,7 @@ Annotations are interpreted rather than passed to `eval`. On Python 3.14+, Aiyth ## Limits are explicit -Runtime checking is not a complete static type proof. Unsupported annotations fail with `UnsupportedType`; they are not silently reduced to `Any`. In particular, Callable signatures, ParamSpec/TypeVarTuple, unregistered Protocols, LiteralString provenance, ReadOnly mutation contracts and arbitrary annotation calls are not automatically proven. Lazy iterator objects supplied by other code are not consumed or wrapped merely to guess their element type; use a typed generator function to check values as they pass yield/send boundaries. +Runtime checking is not a complete static type proof. Unsupported annotations fail with `UnsupportedType`; they are not silently reduced to `Any`. In particular, Callable signatures, unregistered Protocols, LiteralString provenance, ReadOnly mutation contracts and arbitrary annotation calls are not automatically proven. Variadic `TypeVarTuple` and `ParamSpec` parameters run with normal Python values, but their per-call type substitutions are not proven. Lazy iterator objects supplied by other code are not consumed or wrapped merely to guess their element type; use a typed generator function to check values as they pass yield/send boundaries. Python object identity and side effects are preserved. A failed mutation check does **not** roll back `append`, an external API call, a property setter, or arbitrary native code. Foreign code/threads are not instrumented internally; Aiython checks its own boundaries. Objects can be temporarily invalid before the next boundary check. For a guarantee that invalid values can never enter an object, a different object model or isolation boundary is needed. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index f43df03..7b47b94 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -89,6 +89,11 @@ def schema(self, seen=None): elif kind == 'tuple': result = {'type': 'array', 'prefixItems': [a.schema(seen) for a in self.args], 'minItems': len(self.args), 'maxItems': len(self.args)} + elif kind == 'tuple_unpacked': + pivot = next(index for index, item in enumerate(self.args) if item.kind == 'unpack_any') + result = {'type': 'array', 'minItems': len(self.args) - 1, + 'prefixItems': [item.schema(seen) for item in self.args[:pivot]], + 'x-python-suffixItems': [item.schema(seen) for item in self.args[pivot + 1:]]} elif kind == 'tuple_many': result = {'type':'array', 'items':self.args[0].schema(seen)} elif kind in ('dict', 'mapping'): result = {'type':'object', 'additionalProperties':self.args[1].schema(seen), 'x-key-schema':self.args[0].schema(seen)} @@ -158,6 +163,16 @@ def child(contract, item, suffix): return for index, item in enumerate(value): child(self.args[index] if kind == 'tuple' else self.args[0], item, f'[{index}]') + elif kind == 'tuple_unpacked': + if type(value) is not tuple: fail() + pivot = next(index for index, item in enumerate(self.args) if item.kind == 'unpack_any') + suffix = len(self.args) - pivot - 1 + if len(value) < pivot + suffix: fail('wrong tuple length') + for index, contract in enumerate(self.args[:pivot]): + child(contract, value[index], f'[{index}]') + for index, contract in enumerate(self.args[pivot + 1:]): + position = len(value) - suffix + index + child(contract, value[position], f'[{position}]') elif kind in ('dict','mapping'): if type(value) is not dict: fail('a concrete dict is required for deep checking') for index, (key,item) in enumerate(value.items()): @@ -229,9 +244,16 @@ def node(self,node,names): if isinstance(node.value,str): return self.compile(node.value,names) if isinstance(node,ast.BinOp) and isinstance(node.op,ast.BitOr): return Contract('union',ast.unparse(node),(self.node(node.left,names),self.node(node.right,names))) + if isinstance(node, ast.Starred) and isinstance(node.value, ast.Name): + parameter = names.get(node.value.id) + if isinstance(parameter, typing.TypeVarTuple): + return Contract('unpack_any', ast.unparse(node), python_type=parameter) if isinstance(node,ast.Subscript): base = self.lookup(node.value,names) nodes = node.slice.elts if isinstance(node.slice,ast.Tuple) else [node.slice] + if (base is typing.Unpack and len(nodes) == 1 and isinstance(nodes[0], ast.Name) + and isinstance(names.get(nodes[0].id), typing.TypeVarTuple)): + return Contract('unpack_any', ast.unparse(node), python_type=names[nodes[0].id]) if base is typing.Literal: values = tuple(self.lookup(n,names) if isinstance(n,ast.Attribute) else ast.literal_eval(n) for n in nodes) if any(type(v) not in (str,int,bool,bytes,type(None)) and not isinstance(v,enum.Enum) for v in values): @@ -266,6 +288,13 @@ def generic(self,base,args,label,names): return Contract(containers[origin],label,args) if origin is tuple: if len(args) == 2 and args[1] is Ellipsis: return Contract('tuple_many',label,args[:1]) + unpacked = [index for index, arg in enumerate(args) + if isinstance(arg, Contract) and arg.kind == 'unpack_any'] + if len(unpacked) == 1: + if len(args) == 1: return Contract('tuple_many',label,(Contract('any','Any'),)) + return Contract('tuple_unpacked',label,args) + if unpacked: + raise UnsupportedType('Only one variadic tuple parameter can be checked') return Contract('tuple',label,args) if origin in (abc.Generator,abc.Iterator,abc.Iterable,abc.AsyncGenerator,abc.AsyncIterator,abc.AsyncIterable): async_kind = origin in (abc.AsyncGenerator,abc.AsyncIterator,abc.AsyncIterable) diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 17d61cd..b7d7f56 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -6,6 +6,7 @@ from dataclasses import dataclass, field import inspect import threading +import typing import weakref from .frontend import RUNTIME_NAME @@ -33,6 +34,25 @@ def original_class(value): return type(value) +def unconstrained_variadic(source, mode, namespace): + if mode == 'args' and source.startswith('*'): + return True + parameter_name, separator, attribute = source.partition('.') + if (mode in ('args', 'kwargs') and separator and attribute == mode and + isinstance(namespace.get(parameter_name), typing.ParamSpec)): + return True + if mode != 'args': + return False + annotation = ast.parse(source, mode='eval').body + if not isinstance(annotation, ast.Subscript) or not isinstance(annotation.slice, ast.Name): + return False + unpack = annotation.value + is_unpack = ((isinstance(unpack, ast.Name) and namespace.get(unpack.id) is typing.Unpack) or + (isinstance(unpack, ast.Attribute) and unpack.attr == 'Unpack' and + isinstance(unpack.value, ast.Name) and namespace.get(unpack.value.id) is typing)) + return is_unpack and isinstance(namespace.get(annotation.slice.id), typing.TypeVarTuple) + + @dataclass class Scope: declarations: dict = field(default_factory=dict) @@ -84,7 +104,10 @@ def _initialize(self, frame, declarations, parameters, returns): scope = Scope(declarations=declarations) namespace = self.namespace(frame) for name, (source, mode) in (parameters or {}).items(): - contract = self.contract(source,namespace) + # Unbound variadic type parameters have no concrete element types + # to check at this call boundary. + contract = (Contract('any', 'Any') if unconstrained_variadic(source, mode, namespace) + else self.contract(source,namespace)) if mode == 'args': contract = Contract('tuple_many',source,(contract,)) elif mode == 'kwargs': contract = Contract('dict',source,(compile_contract('str',namespace),contract)) contract.validate(frame.f_locals[name],name,bindings=scope.bindings) @@ -385,7 +408,13 @@ def body(self,body,*,parameters=None,returns=None,initialize=True,inherited=None # Capture lexical types used only in stringified contracts without executing them. names = set() for source in list(self.declarations.values()) + [p[0] for p in (parameters or {}).values()] + ([returns] if returns else []): - names.update(n.id for n in ast.walk(ast.parse(source,mode='eval')) if isinstance(n,ast.Name)) + try: + annotation = ast.parse(source, mode='eval') + except SyntaxError: + # Variadic parameter annotations such as *args: *Ts need a + # subscription context; their unparsed text is not an expression. + annotation = ast.parse(f'tuple[{source}]', mode='eval') + names.update(n.id for n in ast.walk(annotation) if isinstance(n,ast.Name)) if self.function and names: capture = ast.If(ast.Constant(False),[ast.Expr(ast.Tuple([ast.Name(n,ast.Load()) for n in sorted(names)],ast.Load()))],[]) ast.copy_location(capture,body[0] if body else output[0]) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 0f37518..111b7c3 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1054,8 +1054,24 @@ async def generate(value): async def run(): print([item async for item in generate(3)]) asyncio.run(run()) +''', + 'typing variadic parameters': '''import typing +from typing import ParamSpec, TypeVarTuple, Unpack +P = ParamSpec('P') +Ts = TypeVarTuple('Ts') +def collect(*args: Unpack[Ts]) -> tuple[Unpack[Ts]]: return args +def collect_qualified(*args: typing.Unpack[Ts]) -> tuple[*Ts]: return args +def forward(*args: P.args, **kwargs: P.kwargs): return args, kwargs +print(collect(1, 'x'), collect_qualified(2, 'y'), forward(1, x=2)) ''', } + if sys.version_info >= (3, 12): + cases['generic variadic parameters'] = '''def collect[*Ts](*args: *Ts) -> tuple[*Ts]: return args +def mixed[*Ts](first: int, *args: *Ts) -> tuple[int, *Ts, str]: + return (first, *args, 'done') +def forward[**P](*args: P.args, **kwargs: P.kwargs): return args, kwargs +print(collect(1, 'x'), mixed(1, 2, 3), forward(1, x=2)) +''' with tempfile.TemporaryDirectory() as directory: path = Path(directory) / 'main.py' for name, source in cases.items(): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 6aaa7f3..fee10c5 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -6,7 +6,7 @@ import sys import types from types import SimpleNamespace -from typing import Any, Annotated, ClassVar, Final, Generic, Literal, NewType, Optional, Protocol, Required, Self, TypeVar, TypedDict +from typing import Any, Annotated, ClassVar, Final, Generic, Literal, NewType, Optional, Protocol, Required, Self, TypeVar, TypeVarTuple, TypedDict, Unpack import unittest from unittest.mock import patch @@ -14,6 +14,23 @@ class ContractEdgeTests(unittest.TestCase): + def test_variadic_tuple_contract_keeps_fixed_members(self): + parameters = {'Ts': TypeVarTuple('Ts'), 'Unpack': Unpack} + for annotation in ('tuple[*Ts]', 'tuple[Unpack[Ts]]'): + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, parameters) + self.assertEqual(contract.kind, 'tuple_many') + contract.validate((1, 'x')) + contract = tc.compile_contract('tuple[int, *Ts, str]', parameters) + schema = contract.schema() + self.assertEqual(schema['minItems'], 2) + self.assertEqual(schema['prefixItems'][0]['type'], 'integer') + self.assertEqual(schema['x-python-suffixItems'][0]['type'], 'string') + contract.validate((1, 2, 'x')) + for value in ((1,), ('bad', 2, 'x'), (1, 2, 3), [1, 2, 'x']): + with self.subTest(value=value), self.assertRaises(tc.TypeViolation): + contract.validate(value) + def test_schema_variants_and_recursive_contract(self): integer = tc.compile_contract('int', {}) cases = [ From 4ca7bd32861156c9e7a19b6cc3ff65ee91bfaeda Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 15:43:24 +0700 Subject: [PATCH 19/72] Allow callable annotations at runtime boundaries --- docs/type-safety.md | 2 +- src/aiython/type_constraints.py | 13 +++++++++++++ tests/test_python_compatibility.py | 8 ++++++++ tests/test_type_constraints_edges.py | 13 +++++++++++-- tests/test_type_safety.py | 2 +- 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index c19817c..3cbc67d 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -42,7 +42,7 @@ Annotations are interpreted rather than passed to `eval`. On Python 3.14+, Aiyth ## Limits are explicit -Runtime checking is not a complete static type proof. Unsupported annotations fail with `UnsupportedType`; they are not silently reduced to `Any`. In particular, Callable signatures, unregistered Protocols, LiteralString provenance, ReadOnly mutation contracts and arbitrary annotation calls are not automatically proven. Variadic `TypeVarTuple` and `ParamSpec` parameters run with normal Python values, but their per-call type substitutions are not proven. Lazy iterator objects supplied by other code are not consumed or wrapped merely to guess their element type; use a typed generator function to check values as they pass yield/send boundaries. +Runtime checking is not a complete static type proof. Unsupported annotations fail with `UnsupportedType`; they are not silently reduced to `Any`. A `Callable` contract checks that a value is callable, but does not prove its parameter or return signature. Unregistered Protocols, LiteralString provenance, ReadOnly mutation contracts and arbitrary annotation calls are not automatically proven. Variadic `TypeVarTuple` and `ParamSpec` parameters run with normal Python values, but their per-call type substitutions are not proven. Lazy iterator objects supplied by other code are not consumed or wrapped merely to guess their element type; use a typed generator function to check values as they pass yield/send boundaries. Python object identity and side effects are preserved. A failed mutation check does **not** roll back `append`, an external API call, a property setter, or arbitrary native code. Foreign code/threads are not instrumented internally; Aiython checks its own boundaries. Objects can be temporarily invalid before the next boundary check. For a guarantee that invalid values can never enter an object, a different object model or isolation boundary is needed. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 7b47b94..db29875 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -82,6 +82,7 @@ def schema(self, seen=None): members = [type(v).__qualname__ + '.' + v.name for v in self.args if isinstance(v,enum.Enum)] if members: result['x-python-enum-members'] = members elif kind == 'null': result = {'type': 'null'} + elif kind == 'callable': result = {'x-python-callable': True} elif kind in ('str', 'int', 'float', 'bool'): result = {'type': {'str':'string', 'int':'integer', 'float':'number', 'bool':'boolean'}[kind]} elif kind in ('list', 'set', 'frozenset', 'sequence'): @@ -120,6 +121,10 @@ def validate(self, value, path='value', *, bindings=None, seen=None): kind = self.kind if kind == 'any' or (kind == 'null' and value is None): return + if kind == 'callable': + if not callable(value): + raise TypeViolation(f'{path}: expected {self.name}, got {type(value).__name__}') + return if kind in PRIMITIVE_KINDS and type(value) is self.python_type: return bindings = {} if bindings is None else bindings @@ -251,6 +256,10 @@ def node(self,node,names): if isinstance(node,ast.Subscript): base = self.lookup(node.value,names) nodes = node.slice.elts if isinstance(node.slice,ast.Tuple) else [node.slice] + if base in (typing.Callable, abc.Callable): + if len(nodes) != 2: + raise UnsupportedType('Callable requires parameters and a return type') + return Contract('callable', ast.unparse(node)) if (base is typing.Unpack and len(nodes) == 1 and isinstance(nodes[0], ast.Name) and isinstance(names.get(nodes[0].id), typing.TypeVarTuple)): return Contract('unpack_any', ast.unparse(node), python_type=names[nodes[0].id]) @@ -269,6 +278,8 @@ def node(self,node,names): def generic(self,base,args,label,names): origin = typing.get_origin(base) or base + if origin is abc.Callable: + return Contract('callable',label) if origin in (typing.Union,types.UnionType): return Contract('union',label,args) if base is typing.Optional: @@ -376,6 +387,8 @@ def value(self,target,names): if isinstance(target,Contract): return target if target is None or target is type(None): return Contract('null','None') if target is typing.Any: return Contract('any','Any') + if target in (typing.Callable, abc.Callable): + return Contract('callable',str(target)) if target in (typing.Final,typing.ClassVar): return Contract('qualifier',str(target),(Contract('any','Any'),),qualifier=target._name) if target in (typing.Never,typing.NoReturn): return Contract('never',str(target)) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 111b7c3..efab1b7 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1063,6 +1063,14 @@ def collect(*args: Unpack[Ts]) -> tuple[Unpack[Ts]]: return args def collect_qualified(*args: typing.Unpack[Ts]) -> tuple[*Ts]: return args def forward(*args: P.args, **kwargs: P.kwargs): return args, kwargs print(collect(1, 'x'), collect_qualified(2, 'y'), forward(1, x=2)) +''', + 'callable annotations': '''from typing import Callable +from collections.abc import Callable as AbstractCallable +def apply(fn: Callable[[int], str], value: int) -> str: + return fn(value) +def invoke(fn: AbstractCallable): + return fn() +print(apply(str, 3), invoke(lambda: 'ok')) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index fee10c5..c08d401 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -6,7 +6,7 @@ import sys import types from types import SimpleNamespace -from typing import Any, Annotated, ClassVar, Final, Generic, Literal, NewType, Optional, Protocol, Required, Self, TypeVar, TypeVarTuple, TypedDict, Unpack +from typing import Any, Annotated, Callable, ClassVar, Final, Generic, Literal, NewType, Optional, Protocol, Required, Self, TypeVar, TypeVarTuple, TypedDict, Unpack import unittest from unittest.mock import patch @@ -14,6 +14,15 @@ class ContractEdgeTests(unittest.TestCase): + def test_callable_contract_checks_callable_without_claiming_signature(self): + for annotation in ('Callable[[int], str]', 'Callable[..., str]', 'Callable'): + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, {'Callable': Callable}) + self.assertTrue(contract.schema()['x-python-callable']) + contract.validate(str) + with self.assertRaises(tc.TypeViolation): + contract.validate(3) + def test_variadic_tuple_contract_keeps_fixed_members(self): parameters = {'Ts': TypeVarTuple('Ts'), 'Unpack': Unpack} for annotation in ('tuple[*Ts]', 'tuple[Unpack[Ts]]'): @@ -223,7 +232,7 @@ class Sample: def test_compiler_rejects_unsafe_and_unresolved_annotations(self): cases = [ 'Missing', 'danger()', 'Unknown.attribute', 'Literal[1.5]', - 'ReadOnly[int]', 'Callable[[int], str]', 'Self', 'LiteralString', + 'ReadOnly[int]', 'Self', 'LiteralString', 'list[int, str]', 'dict[str]', 'Generator[int, str]', 'type[int, str]', ] for source in cases: diff --git a/tests/test_type_safety.py b/tests/test_type_safety.py index 75d2be0..a6bf2d1 100644 --- a/tests/test_type_safety.py +++ b/tests/test_type_safety.py @@ -266,7 +266,7 @@ class Level(Enum): def test_unsupported_contract_fails_before_model_call(self): provider = Mock() with self.assertRaises(UnsupportedType): - self.run_source('from typing import Callable\nanswer: Callable[[int], str] = choose a function',ToolAgent(provider)) + self.run_source('from typing import LiteralString\nanswer: LiteralString = choose a string',ToolAgent(provider)) provider.complete.assert_not_called() def test_ai_expected_type_from_function_argument_and_generic_return(self): From a3ee0ca7ea05c35a76d31507d0635da013e5d9ce Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 15:46:00 +0700 Subject: [PATCH 20/72] Cover Python 3.14 syntax in differential tests --- tests/test_python_compatibility.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index efab1b7..5bbaaf2 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1079,6 +1079,30 @@ def mixed[*Ts](first: int, *args: *Ts) -> tuple[int, *Ts, str]: return (first, *args, 'done') def forward[**P](*args: P.args, **kwargs: P.kwargs): return args, kwargs print(collect(1, 'x'), mixed(1, 2, 3), forward(1, x=2)) +''' + if sys.version_info >= (3, 14): + cases['template string interpolation'] = '''events = [] +def pick(): + events.append('called') + return 3 +template = t"value {pick()!r:>8}" +print(template.strings, [(item.value, item.expression, item.conversion, item.format_spec) + for item in template.interpolations], events) +''' + cases['deferred annotations and type defaults'] = '''def future(value: Missing) -> Unknown: return value +class Box[T = int]: + item: T +type Alias = list[Missing] +print(Box.__type_params__[0].__default__, Alias.__name__, + '__annotate__' in Box.__dict__, '__annotate__' in dir(future)) +for target in (future, Box): + try: print(target.__annotations__) + except NameError as error: print(type(error).__name__) +''' + cases['unparenthesized exception tuple'] = '''try: + raise ValueError('bad') +except ValueError, TypeError: + print('caught') ''' with tempfile.TemporaryDirectory() as directory: path = Path(directory) / 'main.py' From 9f345392b2e34195abb9f32140a42af67ca82629 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 15:51:18 +0700 Subject: [PATCH 21/72] Expand variadic generic type aliases in contracts --- docs/type-safety.md | 2 +- src/aiython/type_constraints.py | 47 +++++++++++++++++++++++++--- tests/test_python_compatibility.py | 6 ++++ tests/test_type_constraints_edges.py | 24 ++++++++++++-- 4 files changed, 72 insertions(+), 7 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 3cbc67d..9feda74 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -42,7 +42,7 @@ Annotations are interpreted rather than passed to `eval`. On Python 3.14+, Aiyth ## Limits are explicit -Runtime checking is not a complete static type proof. Unsupported annotations fail with `UnsupportedType`; they are not silently reduced to `Any`. A `Callable` contract checks that a value is callable, but does not prove its parameter or return signature. Unregistered Protocols, LiteralString provenance, ReadOnly mutation contracts and arbitrary annotation calls are not automatically proven. Variadic `TypeVarTuple` and `ParamSpec` parameters run with normal Python values, but their per-call type substitutions are not proven. Lazy iterator objects supplied by other code are not consumed or wrapped merely to guess their element type; use a typed generator function to check values as they pass yield/send boundaries. +Runtime checking is not a complete static type proof. Unsupported annotations fail with `UnsupportedType`; they are not silently reduced to `Any`. A `Callable` contract checks that a value is callable, but does not prove its parameter or return signature. Unregistered Protocols, LiteralString provenance, ReadOnly mutation contracts and arbitrary annotation calls are not automatically proven. Variadic `TypeVarTuple` and `ParamSpec` parameters run with normal Python values, but their per-call type substitutions are not proven. Specialized variadic type aliases expand their declared member types. Lazy iterator objects supplied by other code are not consumed or wrapped merely to guess their element type; use a typed generator function to check values as they pass yield/send boundaries. Python object identity and side effects are preserved. A failed mutation check does **not** roll back `append`, an external API call, a property setter, or arbitrary native code. Foreign code/threads are not instrumented internally; Aiython checks its own boundaries. Objects can be temporarily invalid before the next boundary check. For a guarantee that invalid values can never enter an object, a different object model or isolation boundary is needed. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index db29875..939743f 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -253,6 +253,8 @@ def node(self,node,names): parameter = names.get(node.value.id) if isinstance(parameter, typing.TypeVarTuple): return Contract('unpack_any', ast.unparse(node), python_type=parameter) + if isinstance(parameter, tuple) and all(isinstance(item, Contract) for item in parameter): + return Contract('unpack_fixed', ast.unparse(node), parameter) if isinstance(node,ast.Subscript): base = self.lookup(node.value,names) nodes = node.slice.elts if isinstance(node.slice,ast.Tuple) else [node.slice] @@ -263,6 +265,10 @@ def node(self,node,names): if (base is typing.Unpack and len(nodes) == 1 and isinstance(nodes[0], ast.Name) and isinstance(names.get(nodes[0].id), typing.TypeVarTuple)): return Contract('unpack_any', ast.unparse(node), python_type=names[nodes[0].id]) + if (base is typing.Unpack and len(nodes) == 1 and isinstance(nodes[0], ast.Name) + and isinstance(names.get(nodes[0].id), tuple) + and all(isinstance(item, Contract) for item in names[nodes[0].id])): + return Contract('unpack_fixed', ast.unparse(node), names[nodes[0].id]) if base is typing.Literal: values = tuple(self.lookup(n,names) if isinstance(n,ast.Attribute) else ast.literal_eval(n) for n in nodes) if any(type(v) not in (str,int,bool,bytes,type(None)) and not isinstance(v,enum.Enum) for v in values): @@ -278,6 +284,8 @@ def node(self,node,names): def generic(self,base,args,label,names): origin = typing.get_origin(base) or base + if origin is typing.Unpack and len(args) == 1 and args[0].kind in ('unpack_any', 'unpack_fixed'): + return args[0] if origin is abc.Callable: return Contract('callable',label) if origin in (typing.Union,types.UnionType): @@ -299,6 +307,13 @@ def generic(self,base,args,label,names): return Contract(containers[origin],label,args) if origin is tuple: if len(args) == 2 and args[1] is Ellipsis: return Contract('tuple_many',label,args[:1]) + expanded = [] + for arg in args: + if isinstance(arg, Contract) and arg.kind == 'unpack_fixed': + expanded.extend(arg.args) + else: + expanded.append(arg) + args = tuple(expanded) unpacked = [index for index, arg in enumerate(args) if isinstance(arg, Contract) and arg.kind == 'unpack_any'] if len(unpacked) == 1: @@ -328,10 +343,29 @@ def generic(self,base,args,label,names): def alias(self,alias,names,args=(),label=None): parameters = alias.__type_params__ - if parameters and len(args) != len(parameters): - raise UnsupportedType('Generic alias requires its type arguments') - scope = self.module_names(alias,names) | {p.__name__:a for p,a in zip(parameters,args)} - key = (id(alias),tuple(id(a) for a in args)) + if not args and label is None: + bindings = {parameter.__name__: parameter for parameter in parameters} + else: + variadic = [index for index, parameter in enumerate(parameters) + if isinstance(parameter, typing.TypeVarTuple)] + if len(variadic) > 1: + raise UnsupportedType('Only one TypeVarTuple can be specialized') + if variadic: + pivot = variadic[0] + suffix = len(parameters) - pivot - 1 + if len(args) < len(parameters) - 1: + raise UnsupportedType('Generic alias requires its type arguments') + bindings = {parameter.__name__: arg + for parameter, arg in zip(parameters[:pivot], args[:pivot])} + bindings[parameters[pivot].__name__] = args[pivot:len(args) - suffix] + bindings.update({parameter.__name__: arg for parameter, arg in + zip(parameters[pivot + 1:], args[len(args) - suffix:])}) + else: + if len(args) != len(parameters): + raise UnsupportedType('Generic alias requires its type arguments') + bindings = {parameter.__name__: arg for parameter, arg in zip(parameters,args)} + scope = self.module_names(alias,names) | bindings + key = (id(alias), tuple(id(a) for a in args), label is None) if key in self.cache: return self.cache[key] result = Contract('alias',label or alias.__name__) @@ -387,6 +421,11 @@ def value(self,target,names): if isinstance(target,Contract): return target if target is None or target is type(None): return Contract('null','None') if target is typing.Any: return Contract('any','Any') + if isinstance(target, typing.TypeVarTuple): + bound = names.get(target.__name__) + return (Contract('unpack_fixed', target.__name__, bound) + if isinstance(bound, tuple) and all(isinstance(item, Contract) for item in bound) + else Contract('unpack_any', target.__name__, python_type=target)) if target in (typing.Callable, abc.Callable): return Contract('callable',str(target)) if target in (typing.Final,typing.ClassVar): diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 5bbaaf2..24acf68 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1079,6 +1079,12 @@ def mixed[*Ts](first: int, *args: *Ts) -> tuple[int, *Ts, str]: return (first, *args, 'done') def forward[**P](*args: P.args, **kwargs: P.kwargs): return args, kwargs print(collect(1, 'x'), mixed(1, 2, 3), forward(1, x=2)) +''' + cases['variadic type aliases'] = '''type TupleAlias[*Ts] = tuple[*Ts] +type Mixed[T, *Ts, U] = tuple[T, *Ts, U] +first: TupleAlias[int, str] = (1, 'x') +second: Mixed[int, str, bool] = (1, 'x', True) +print(first, second) ''' if sys.version_info >= (3, 14): cases['template string interpolation'] = '''events = [] diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index c08d401..633cee1 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -14,6 +14,25 @@ class ContractEdgeTests(unittest.TestCase): + @unittest.skipIf(sys.version_info < (3, 12), 'type statements require Python 3.12') + def test_variadic_type_alias_specialization(self): + namespace = {'__name__': __name__} + exec('type TupleAlias[*Ts] = tuple[*Ts]\n' + 'type Mixed[T, *Ts, U] = tuple[T, *Ts, U]\n', namespace) + specialized = tc.compile_contract('TupleAlias[int, str]', namespace) + specialized.validate((1, 'x')) + with self.assertRaises(tc.TypeViolation): + specialized.validate((1, 2)) + tc.compile_contract('TupleAlias', namespace).validate((1, 'x', True)) + empty = tc.compile_contract('TupleAlias[()]', namespace) + empty.validate(()) + with self.assertRaises(tc.TypeViolation): + empty.validate((1,)) + mixed = tc.compile_contract('Mixed[int, str, bool]', namespace) + mixed.validate((1, 'x', True)) + with self.assertRaises(tc.TypeViolation): + mixed.validate((1, 'x', 3)) + def test_callable_contract_checks_callable_without_claiming_signature(self): for annotation in ('Callable[[int], str]', 'Callable[..., str]', 'Callable'): with self.subTest(annotation=annotation): @@ -165,8 +184,9 @@ def test_alias_generic_protocol_and_class_field_paths(self): variable = TypeVar('T') alias = tc.TypeAliasType('Items', list[variable], type_params=(variable,)) compiler = tc.Compiler({'Items': alias}) - with self.assertRaisesRegex(tc.UnsupportedType, 'Generic alias requires'): - compiler.compile(alias) + unspecialized = compiler.compile(alias) + self.assertEqual(unspecialized.schema()['type'], 'array') + unspecialized.validate([1, 2]) with patch.object(tc, 'annotationlib', None): compiled = compiler.compile('Items[int]') self.assertEqual(compiled.schema()['type'], 'array') From 04a8d29f9d97697228e0c62c8dd86b0d0e3869f9 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 15:55:06 +0700 Subject: [PATCH 22/72] Resolve defaulted generic alias parameters --- src/aiython/type_constraints.py | 27 +++++++++++++++++++++++---- tests/test_python_compatibility.py | 8 ++++++++ tests/test_type_constraints_edges.py | 20 ++++++++++++++++++++ 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 939743f..b4de562 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -353,17 +353,36 @@ def alias(self,alias,names,args=(),label=None): if variadic: pivot = variadic[0] suffix = len(parameters) - pivot - 1 - if len(args) < len(parameters) - 1: + no_default = getattr(typing, 'NoDefault', None) + suffix_parameters = parameters[pivot + 1:] + required_suffix = sum(getattr(parameter, '__default__', no_default) is no_default + for parameter in suffix_parameters) + if len(args) < pivot + required_suffix: raise UnsupportedType('Generic alias requires its type arguments') bindings = {parameter.__name__: arg for parameter, arg in zip(parameters[:pivot], args[:pivot])} - bindings[parameters[pivot].__name__] = args[pivot:len(args) - suffix] + supplied_suffix = min(suffix, len(args) - pivot) + middle_end = len(args) - supplied_suffix + bindings[parameters[pivot].__name__] = args[pivot:middle_end] bindings.update({parameter.__name__: arg for parameter, arg in - zip(parameters[pivot + 1:], args[len(args) - suffix:])}) + zip(suffix_parameters, args[middle_end:])}) + for parameter in suffix_parameters[supplied_suffix:]: + default = getattr(parameter, '__default__', no_default) + if default is no_default: + raise UnsupportedType('Generic alias requires its type arguments') + bindings[parameter.__name__] = self.compile( + default, self.module_names(alias,names) | bindings) else: - if len(args) != len(parameters): + if len(args) > len(parameters): raise UnsupportedType('Generic alias requires its type arguments') bindings = {parameter.__name__: arg for parameter, arg in zip(parameters,args)} + no_default = getattr(typing, 'NoDefault', None) + for parameter in parameters[len(args):]: + default = getattr(parameter, '__default__', no_default) + if default is no_default: + raise UnsupportedType('Generic alias requires its type arguments') + bindings[parameter.__name__] = self.compile( + default, self.module_names(alias,names) | bindings) scope = self.module_names(alias,names) | bindings key = (id(alias), tuple(id(a) for a in args), label is None) if key in self.cache: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 24acf68..82f81e0 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1085,6 +1085,14 @@ def forward[**P](*args: P.args, **kwargs: P.kwargs): return args, kwargs first: TupleAlias[int, str] = (1, 'x') second: Mixed[int, str, bool] = (1, 'x', True) print(first, second) +''' + if sys.version_info >= (3, 13): + cases['defaulted type aliases'] = '''type Pair[T, U = str] = tuple[T, U] +type Variadic[T, *Ts, U = str] = tuple[T, *Ts, U] +first: Pair[int] = (1, 'x') +second: Variadic[int] = (1, 'x') +third: Variadic[int, bool, str] = (1, True, 'x') +print(first, second, third) ''' if sys.version_info >= (3, 14): cases['template string interpolation'] = '''events = [] diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 633cee1..8bb5759 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -14,6 +14,26 @@ class ContractEdgeTests(unittest.TestCase): + @unittest.skipIf(sys.version_info < (3, 13), 'type parameter defaults require Python 3.13') + def test_defaulted_type_alias_parameters(self): + namespace = {'__name__': __name__} + exec('type Pair[T, U = str] = tuple[T, U]\n' + 'type Again[T, U = T] = tuple[T, U]\n' + 'type Variadic[T, *Ts, U = str] = tuple[T, *Ts, U]\n', namespace) + pair = tc.compile_contract('Pair[int]', namespace) + pair.validate((1, 'x')) + with self.assertRaises(tc.TypeViolation): + pair.validate((1, 2)) + again = tc.compile_contract('Again[int]', namespace) + again.validate((1, 2)) + with self.assertRaises(tc.TypeViolation): + again.validate((1, 'x')) + for annotation, value in (('Variadic[int]', (1, 'x')), + ('Variadic[int, bool]', (1, True)), + ('Variadic[int, bool, str]', (1, True, 'x'))): + with self.subTest(annotation=annotation): + tc.compile_contract(annotation, namespace).validate(value) + @unittest.skipIf(sys.version_info < (3, 12), 'type statements require Python 3.12') def test_variadic_type_alias_specialization(self): namespace = {'__name__': __name__} From e58bdcea282b7b98b2ffe8e6347378280e89fe41 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:00:39 +0700 Subject: [PATCH 23/72] Bind variadic and defaulted generic class parameters --- src/aiython/type_constraints.py | 94 +++++++++++++++------------- tests/test_python_compatibility.py | 14 +++++ tests/test_type_constraints_edges.py | 24 +++++++ 3 files changed, 87 insertions(+), 45 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index b4de562..ecfa90d 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -336,54 +336,56 @@ def generic(self,base,args,label,names): if isinstance(base,TYPE_ALIAS_TYPES): return self.alias(base,names,args,label) if isinstance(base,type) and (getattr(base,'__type_params__',()) or getattr(base,'__parameters__',())): parameters = getattr(base,'__type_params__',()) or base.__parameters__ - if len(parameters) != len(args): - raise UnsupportedType('Generic type argument count mismatch') - return self.class_contract(base,names | {p.__name__: a for p,a in zip(parameters,args)},label) + scope = self.module_names(base,names) + bindings = self.parameter_bindings(parameters, args, scope, + 'Generic type argument count mismatch') + return self.class_contract(base,scope | bindings,label) raise UnsupportedType(f'Unsupported generic output type: {label}; no unchecked fallback is allowed') + def parameter_bindings(self, parameters, args, names, error): + variadic = [index for index, parameter in enumerate(parameters) + if isinstance(parameter, typing.TypeVarTuple)] + if len(variadic) > 1: + raise UnsupportedType('Only one TypeVarTuple can be specialized') + no_default = getattr(typing, 'NoDefault', None) + if variadic: + pivot = variadic[0] + suffix_parameters = parameters[pivot + 1:] + required_suffix = sum(getattr(parameter, '__default__', no_default) is no_default + for parameter in suffix_parameters) + if len(args) < pivot + required_suffix: + raise UnsupportedType(error) + bindings = {parameter.__name__: arg + for parameter, arg in zip(parameters[:pivot], args[:pivot])} + supplied_suffix = min(len(suffix_parameters), len(args) - pivot) + middle_end = len(args) - supplied_suffix + bindings[parameters[pivot].__name__] = args[pivot:middle_end] + bindings.update({parameter.__name__: arg for parameter, arg in + zip(suffix_parameters, args[middle_end:])}) + for parameter in suffix_parameters[supplied_suffix:]: + default = getattr(parameter, '__default__', no_default) + if default is no_default: + raise UnsupportedType(error) + bindings[parameter.__name__] = self.compile(default, names | bindings) + return bindings + if len(args) > len(parameters): + raise UnsupportedType(error) + bindings = {parameter.__name__: arg for parameter, arg in zip(parameters,args)} + for parameter in parameters[len(args):]: + default = getattr(parameter, '__default__', no_default) + if default is no_default: + raise UnsupportedType(error) + bindings[parameter.__name__] = self.compile(default, names | bindings) + return bindings + def alias(self,alias,names,args=(),label=None): parameters = alias.__type_params__ - if not args and label is None: - bindings = {parameter.__name__: parameter for parameter in parameters} - else: - variadic = [index for index, parameter in enumerate(parameters) - if isinstance(parameter, typing.TypeVarTuple)] - if len(variadic) > 1: - raise UnsupportedType('Only one TypeVarTuple can be specialized') - if variadic: - pivot = variadic[0] - suffix = len(parameters) - pivot - 1 - no_default = getattr(typing, 'NoDefault', None) - suffix_parameters = parameters[pivot + 1:] - required_suffix = sum(getattr(parameter, '__default__', no_default) is no_default - for parameter in suffix_parameters) - if len(args) < pivot + required_suffix: - raise UnsupportedType('Generic alias requires its type arguments') - bindings = {parameter.__name__: arg - for parameter, arg in zip(parameters[:pivot], args[:pivot])} - supplied_suffix = min(suffix, len(args) - pivot) - middle_end = len(args) - supplied_suffix - bindings[parameters[pivot].__name__] = args[pivot:middle_end] - bindings.update({parameter.__name__: arg for parameter, arg in - zip(suffix_parameters, args[middle_end:])}) - for parameter in suffix_parameters[supplied_suffix:]: - default = getattr(parameter, '__default__', no_default) - if default is no_default: - raise UnsupportedType('Generic alias requires its type arguments') - bindings[parameter.__name__] = self.compile( - default, self.module_names(alias,names) | bindings) - else: - if len(args) > len(parameters): - raise UnsupportedType('Generic alias requires its type arguments') - bindings = {parameter.__name__: arg for parameter, arg in zip(parameters,args)} - no_default = getattr(typing, 'NoDefault', None) - for parameter in parameters[len(args):]: - default = getattr(parameter, '__default__', no_default) - if default is no_default: - raise UnsupportedType('Generic alias requires its type arguments') - bindings[parameter.__name__] = self.compile( - default, self.module_names(alias,names) | bindings) - scope = self.module_names(alias,names) | bindings + scope = self.module_names(alias,names) + bindings = ({parameter.__name__: parameter for parameter in parameters} + if not args and label is None else + self.parameter_bindings(parameters, args, scope, + 'Generic alias requires its type arguments')) + scope |= bindings key = (id(alias), tuple(id(a) for a in args), label is None) if key in self.cache: return self.cache[key] @@ -415,7 +417,9 @@ def class_contract(self,target,names,label=None): scope = self.module_names(target,names) | {target.__name__:target} scope.update({p.__name__:p for p in (getattr(target,'__type_params__',()) or getattr(target,'__parameters__',()))}) # isinstance can call a user's __getattribute__('__class__') here. - scope.update({k:v for k,v in names.items() if issubclass(type(v), Contract)}) + scope.update({k:v for k,v in names.items() + if (issubclass(type(v), Contract) or + (type(v) is tuple and all(issubclass(type(item), Contract) for item in v)))}) fields = {} for base in reversed(target.__mro__): if base in (object,dict): continue diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 82f81e0..d7e9b3e 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1085,6 +1085,12 @@ def forward[**P](*args: P.args, **kwargs: P.kwargs): return args, kwargs first: TupleAlias[int, str] = (1, 'x') second: Mixed[int, str, bool] = (1, 'x', True) print(first, second) +''' + cases['variadic generic class'] = '''class Box[*Ts]: + value: tuple[*Ts] + def __init__(self, value: tuple[*Ts]): self.value = value +box: Box[int, str] = Box((1, 'x')) +print(box.value) ''' if sys.version_info >= (3, 13): cases['defaulted type aliases'] = '''type Pair[T, U = str] = tuple[T, U] @@ -1093,6 +1099,14 @@ def forward[**P](*args: P.args, **kwargs: P.kwargs): return args, kwargs second: Variadic[int] = (1, 'x') third: Variadic[int, bool, str] = (1, True, 'x') print(first, second, third) +''' + cases['defaulted generic class'] = '''class Pair[T, U = str]: + left: T + right: U + def __init__(self, left: T, right: U): + self.left, self.right = left, right +pair: Pair[int] = Pair(1, 'x') +print(pair.left, pair.right) ''' if sys.version_info >= (3, 14): cases['template string interpolation'] = '''events = [] diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 8bb5759..7d26418 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -14,6 +14,17 @@ class ContractEdgeTests(unittest.TestCase): + @unittest.skipIf(sys.version_info < (3, 12), 'generic class syntax requires Python 3.12') + def test_variadic_generic_class_specialization(self): + namespace = {'__name__': __name__} + exec('class Box[*Ts]:\n' + ' value: tuple[*Ts]\n' + ' def __init__(self, value): self.value = value\n', namespace) + contract = tc.compile_contract('Box[int, str]', namespace) + contract.validate(namespace['Box']((1, 'x'))) + with self.assertRaises(tc.TypeViolation): + contract.validate(namespace['Box']((1, 2))) + @unittest.skipIf(sys.version_info < (3, 13), 'type parameter defaults require Python 3.13') def test_defaulted_type_alias_parameters(self): namespace = {'__name__': __name__} @@ -34,6 +45,19 @@ def test_defaulted_type_alias_parameters(self): with self.subTest(annotation=annotation): tc.compile_contract(annotation, namespace).validate(value) + @unittest.skipIf(sys.version_info < (3, 13), 'type parameter defaults require Python 3.13') + def test_defaulted_generic_class_parameters(self): + namespace = {'__name__': __name__} + exec('class Pair[T, U = str]:\n' + ' left: T\n' + ' right: U\n' + ' def __init__(self, left, right):\n' + ' self.left, self.right = left, right\n', namespace) + contract = tc.compile_contract('Pair[int]', namespace) + contract.validate(namespace['Pair'](1, 'x')) + with self.assertRaises(tc.TypeViolation): + contract.validate(namespace['Pair'](1, 2)) + @unittest.skipIf(sys.version_info < (3, 12), 'type statements require Python 3.12') def test_variadic_type_alias_specialization(self): namespace = {'__name__': __name__} From ef5fed9e95f732b22332c549a81f86e2195217a6 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:05:29 +0700 Subject: [PATCH 24/72] Resolve inherited generic field types --- src/aiython/type_constraints.py | 29 +++++++++++++++++++--- tests/test_python_compatibility.py | 8 +++++++ tests/test_type_constraints_edges.py | 36 ++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index ecfa90d..de5c967 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -420,12 +420,35 @@ def class_contract(self,target,names,label=None): scope.update({k:v for k,v in names.items() if (issubclass(type(v), Contract) or (type(v) is tuple and all(issubclass(type(item), Contract) for item in v)))}) + # Each class owns its annotation namespace. A subclass can bind an + # inherited parameter through Base[int] or through several generic + # intermediate classes, even when it has no parameters of its own. + scopes = {target: scope} + pending = [target] + while pending: + current = pending.pop(0) + current_scope = scopes[current] + for original in vars(current).get('__orig_bases__', current.__bases__): + base = typing.get_origin(original) or original + if not isinstance(base, type) or base in scopes or base not in current.__bases__: + continue + base_scope = self.module_names(base, current_scope) | {base.__name__: base} + parameters = getattr(base, '__type_params__', ()) or getattr(base, '__parameters__', ()) + arguments = typing.get_args(original) + if parameters and arguments: + compiled = tuple(self.compile(arg, current_scope) for arg in arguments) + base_scope.update(self.parameter_bindings(parameters, compiled, current_scope, + 'Generic type argument count mismatch')) + else: + base_scope.update({parameter.__name__: parameter for parameter in parameters}) + scopes[base] = base_scope + pending.append(base) fields = {} for base in reversed(target.__mro__): if base in (object,dict): continue - fields.update(annotations_of(base)) - for name, source in fields.items(): - contract = self.compile(source,scope) + fields.update({name: (source, base) for name, source in annotations_of(base).items()}) + for name, (source, owner) in fields.items(): + contract = self.compile(source, scopes.get(owner, scope)) if contract.marker == 'ClassVar': continue result.fields[name] = contract diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index d7e9b3e..438ea3f 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1091,6 +1091,14 @@ def forward[**P](*args: P.args, **kwargs: P.kwargs): return args, kwargs def __init__(self, value: tuple[*Ts]): self.value = value box: Box[int, str] = Box((1, 'x')) print(box.value) +''' + cases['inherited generic fields'] = '''class Base[T]: + value: T + def __init__(self, value): self.value = value +class Middle[U](Base[list[U]]): pass +class Leaf(Middle[int]): pass +leaf: Leaf = Leaf([1, 2]) +print(leaf.value) ''' if sys.version_info >= (3, 13): cases['defaulted type aliases'] = '''type Pair[T, U = str] = tuple[T, U] diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 7d26418..5d56054 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -14,6 +14,42 @@ class ContractEdgeTests(unittest.TestCase): + def test_inherited_generic_fields_use_base_type_arguments(self): + variable = TypeVar('T') + item = TypeVar('Item') + + class Base(Generic[variable]): + value: variable + def __init__(self, value): self.value = value + + class Middle(Base[list[item]], Generic[item]): + pass + + class Leaf(Middle[int]): + pass + + contract = tc.compile_contract(Leaf, locals()) + contract.validate(Leaf([1, 2])) + with self.assertRaises(tc.TypeViolation): + contract.validate(Leaf(['wrong'])) + + @unittest.skipIf(sys.version_info < (3, 12), 'generic class syntax requires Python 3.12') + def test_inherited_pep695_fields_keep_each_class_parameter_scope(self): + namespace = {'__name__': __name__} + exec('class Base[T]:\n' + ' value: T\n' + ' def __init__(self, value): self.value = value\n' + 'class Child[T](Base[str]):\n' + ' other: T\n' + ' def __init__(self, value, other):\n' + ' super().__init__(value)\n' + ' self.other = other\n', namespace) + contract = tc.compile_contract('Child[int]', namespace) + contract.validate(namespace['Child']('ok', 1)) + for value, other in ((1, 1), ('ok', 'wrong')): + with self.subTest(value=value, other=other), self.assertRaises(tc.TypeViolation): + contract.validate(namespace['Child'](value, other)) + @unittest.skipIf(sys.version_info < (3, 12), 'generic class syntax requires Python 3.12') def test_variadic_generic_class_specialization(self): namespace = {'__name__': __name__} From c867115d22f4f8baa6fcd96059d61e27e1e94e79 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:08:15 +0700 Subject: [PATCH 25/72] Expand inherited variadic generic arguments --- src/aiython/type_constraints.py | 10 ++++++++-- tests/test_python_compatibility.py | 4 +++- tests/test_type_constraints_edges.py | 16 ++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index de5c967..8b7c3d8 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -436,8 +436,14 @@ def class_contract(self,target,names,label=None): parameters = getattr(base, '__type_params__', ()) or getattr(base, '__parameters__', ()) arguments = typing.get_args(original) if parameters and arguments: - compiled = tuple(self.compile(arg, current_scope) for arg in arguments) - base_scope.update(self.parameter_bindings(parameters, compiled, current_scope, + compiled = [] + for arg in arguments: + contract = self.compile(arg, current_scope) + if contract.kind == 'unpack_fixed': + compiled.extend(contract.args) + else: + compiled.append(contract) + base_scope.update(self.parameter_bindings(parameters, tuple(compiled), current_scope, 'Generic type argument count mismatch')) else: base_scope.update({parameter.__name__: parameter for parameter in parameters}) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 438ea3f..58a55a5 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1090,7 +1090,9 @@ def forward[**P](*args: P.args, **kwargs: P.kwargs): return args, kwargs value: tuple[*Ts] def __init__(self, value: tuple[*Ts]): self.value = value box: Box[int, str] = Box((1, 'x')) -print(box.value) +class Child[*Us](Box[*Us]): pass +child: Child[int, str] = Child((2, 'y')) +print(box.value, child.value) ''' cases['inherited generic fields'] = '''class Base[T]: value: T diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 5d56054..c39e2bd 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -50,6 +50,22 @@ def test_inherited_pep695_fields_keep_each_class_parameter_scope(self): with self.subTest(value=value, other=other), self.assertRaises(tc.TypeViolation): contract.validate(namespace['Child'](value, other)) + @unittest.skipIf(sys.version_info < (3, 12), 'variadic class syntax requires Python 3.12') + def test_inherited_variadic_generic_fields_expand_arguments(self): + namespace = {'__name__': __name__} + exec('class Base[*Ts]:\n' + ' value: tuple[*Ts]\n' + ' def __init__(self, value): self.value = value\n' + 'class Child[*Us](Base[*Us]): pass\n', namespace) + child = namespace['Child'] + for annotation, valid, invalid in (('Child[int, str]', (1, 'x'), (1, 2)), + ('Child[()]', (), (1,))): + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, namespace) + contract.validate(child(valid)) + with self.assertRaises(tc.TypeViolation): + contract.validate(child(invalid)) + @unittest.skipIf(sys.version_info < (3, 12), 'generic class syntax requires Python 3.12') def test_variadic_generic_class_specialization(self): namespace = {'__name__': __name__} From aed6dd30d37fa35517e008d4fb8c7bef67207832 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:11:58 +0700 Subject: [PATCH 26/72] Resolve inherited generic defaults by parameter identity --- src/aiython/type_constraints.py | 19 ++++++++++++------- tests/test_type_constraints_edges.py | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 8b7c3d8..32a27b1 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -437,12 +437,17 @@ def class_contract(self,target,names,label=None): arguments = typing.get_args(original) if parameters and arguments: compiled = [] + argument_scope = dict(current_scope) + variadic = any(isinstance(parameter, typing.TypeVarTuple) + for parameter in parameters) for arg in arguments: - contract = self.compile(arg, current_scope) - if contract.kind == 'unpack_fixed': - compiled.extend(contract.args) - else: - compiled.append(contract) + contract = self.compile(arg, argument_scope) + members = contract.args if contract.kind == 'unpack_fixed' else (contract,) + for member in members: + position = len(compiled) + compiled.append(member) + if not variadic and position < len(parameters): + argument_scope[parameters[position]] = member base_scope.update(self.parameter_bindings(parameters, tuple(compiled), current_scope, 'Generic type argument count mismatch')) else: @@ -474,7 +479,7 @@ def value(self,target,names): if target is None or target is type(None): return Contract('null','None') if target is typing.Any: return Contract('any','Any') if isinstance(target, typing.TypeVarTuple): - bound = names.get(target.__name__) + bound = names.get(target, names.get(target.__name__)) return (Contract('unpack_fixed', target.__name__, bound) if isinstance(bound, tuple) and all(isinstance(item, Contract) for item in bound) else Contract('unpack_any', target.__name__, python_type=target)) @@ -492,7 +497,7 @@ def value(self,target,names): raise UnsupportedType('LiteralString requires static provenance checking; use str for a runtime string contract') if isinstance(target,TYPE_ALIAS_TYPES): return self.alias(target,names) if isinstance(target,typing.TypeVar): - substituted = names.get(target.__name__) + substituted = names.get(target, names.get(target.__name__)) if isinstance(substituted,Contract): return substituted choices = target.__constraints__ or ((target.__bound__,) if target.__bound__ else ()) diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index c39e2bd..e94ecdb 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -50,6 +50,31 @@ def test_inherited_pep695_fields_keep_each_class_parameter_scope(self): with self.subTest(value=value, other=other), self.assertRaises(tc.TypeViolation): contract.validate(namespace['Child'](value, other)) + @unittest.skipIf(sys.version_info < (3, 12), 'generic class syntax requires Python 3.12') + def test_inherited_parameters_with_matching_names_keep_their_identity(self): + namespace = {'__name__': __name__} + exec('class Base[T, U]:\n' + ' left: T\n' + ' right: U\n' + ' def __init__(self, left, right): self.left, self.right = left, right\n' + 'class Child[T, U](Base[U, T]): pass\n', namespace) + contract = tc.compile_contract('Child[int, str]', namespace) + contract.validate(namespace['Child']('left', 1)) + with self.assertRaises(tc.TypeViolation): + contract.validate(namespace['Child'](1, 'right')) + + @unittest.skipIf(sys.version_info < (3, 13), 'type parameter defaults require Python 3.13') + def test_inherited_default_can_reference_earlier_base_parameter(self): + namespace = {'__name__': __name__} + exec('class Base[T, U = list[T]]:\n' + ' value: U\n' + ' def __init__(self, value): self.value = value\n' + 'class Child(Base[int]): pass\n', namespace) + contract = tc.compile_contract('Child', namespace) + contract.validate(namespace['Child']([1])) + with self.assertRaises(tc.TypeViolation): + contract.validate(namespace['Child'](['wrong'])) + @unittest.skipIf(sys.version_info < (3, 12), 'variadic class syntax requires Python 3.12') def test_inherited_variadic_generic_fields_expand_arguments(self): namespace = {'__name__': __name__} From d654b3be97c855a62af7fa8b4331c64efe18f781 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:19:03 +0700 Subject: [PATCH 27/72] Support TypeGuard and TypeIs return contracts --- docs/type-safety.md | 2 +- src/aiython/type_constraints.py | 10 +++++++++- tests/test_python_compatibility.py | 13 +++++++++++++ tests/test_type_constraints_edges.py | 15 ++++++++++++++- 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 9feda74..675d3aa 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -32,7 +32,7 @@ Expected types reach direct AI expressions in annotated assignments, later assig ## Type forms -Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, concrete Sequence/Mapping values, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. +Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, concrete Sequence/Mapping values, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. Primitive checks are strict: no string-to-number conversion, and bool does not pass an int contract. `Any` is an explicit escape from value checking. Bare containers have unconstrained elements. `Final` bindings reject reassignment; `ClassVar` direct writes must target the class. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 32a27b1..5c3e027 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -16,7 +16,7 @@ import sys import types import typing -from typing_extensions import ReadOnly, TypeAliasType +from typing_extensions import ReadOnly, TypeAliasType, TypeGuard, TypeIs try: import annotationlib @@ -29,6 +29,8 @@ PRIMITIVE_KINDS = frozenset(('str', 'int', 'float', 'bool', 'bytes', 'complex')) TYPE_ALIAS_TYPES = tuple({TypeAliasType, getattr(typing, "TypeAliasType", TypeAliasType)}) READ_ONLY_TYPES = tuple({ReadOnly, getattr(typing, "ReadOnly", ReadOnly)}) +TYPE_NARROWING_TYPES = tuple({TypeGuard, TypeIs, typing.TypeGuard, + getattr(typing, 'TypeIs', TypeIs)}) class TypeViolation(CapabilityError, TypeError): @@ -278,6 +280,10 @@ def node(self,node,names): metadata = [ast.literal_eval(n) for n in nodes[1:]] return Contract('annotated',ast.unparse(node),(self.node(nodes[0],names),), description='; '.join(v for v in metadata if isinstance(v,str))) + if base in TYPE_NARROWING_TYPES: + if len(nodes) != 1: + raise UnsupportedType('Type narrowing requires one target type') + return Contract('bool',ast.unparse(node),python_type=bool) args = tuple(Ellipsis if isinstance(n,ast.Constant) and n.value is Ellipsis else self.node(n,names) for n in nodes) return self.generic(base,args,ast.unparse(node),names) return self.value(self.lookup(node,names),names) @@ -509,6 +515,8 @@ def value(self,target,names): if origin is typing.Literal: return Contract('literal',str(target),args) if origin is typing.Annotated: return Contract('annotated',str(target),(self.compile(args[0],names),),description='; '.join(v for v in args[1:] if isinstance(v,str))) + if origin in TYPE_NARROWING_TYPES: + return Contract('bool',str(target),python_type=bool) if origin is not None: return self.generic(origin,tuple(Ellipsis if a is Ellipsis else self.compile(a,names) for a in args),str(target),names) if target in (int,str,float,bool,bytes,complex): return Contract(target.__name__,target.__name__,python_type=target) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 58a55a5..63b74e3 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1071,6 +1071,14 @@ def apply(fn: Callable[[int], str], value: int) -> str: def invoke(fn: AbstractCallable): return fn() print(apply(str, 3), invoke(lambda: 'ok')) +''', + 'type narrowing return': '''from __future__ import annotations +from typing import TypeGuard +def is_int(value: object) -> TypeGuard[int]: + return isinstance(value, int) +def deferred(value) -> TypeGuard[Undefined]: + return bool(value) +print(is_int(1), is_int('x'), deferred('yes')) ''', } if sys.version_info >= (3, 12): @@ -1103,6 +1111,11 @@ class Leaf(Middle[int]): pass print(leaf.value) ''' if sys.version_info >= (3, 13): + cases['type is return'] = '''from typing import TypeIs +def is_int(value: object) -> TypeIs[int]: + return isinstance(value, int) +print(is_int(1), is_int('x')) +''' cases['defaulted type aliases'] = '''type Pair[T, U = str] = tuple[T, U] type Variadic[T, *Ts, U = str] = tuple[T, *Ts, U] first: Pair[int] = (1, 'x') diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index e94ecdb..995c2d7 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -6,14 +6,27 @@ import sys import types from types import SimpleNamespace -from typing import Any, Annotated, Callable, ClassVar, Final, Generic, Literal, NewType, Optional, Protocol, Required, Self, TypeVar, TypeVarTuple, TypedDict, Unpack +from typing import Any, Annotated, Callable, ClassVar, Final, Generic, Literal, NewType, Optional, Protocol, Required, Self, TypeGuard, TypeVar, TypeVarTuple, TypedDict, Unpack import unittest from unittest.mock import patch +from typing_extensions import TypeIs from aiython import type_constraints as tc class ContractEdgeTests(unittest.TestCase): + def test_type_narrowing_annotations_check_boolean_results(self): + namespace = {'TypeGuard': TypeGuard, 'TypeIs': TypeIs} + for annotation in ('TypeGuard[int]', TypeGuard[int], + 'TypeIs[int]', TypeIs[int], 'TypeGuard[Undefined]'): + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, namespace) + self.assertEqual(contract.schema()['type'], 'boolean') + contract.validate(True) + contract.validate(False) + with self.assertRaises(tc.TypeViolation): + contract.validate(1) + def test_inherited_generic_fields_use_base_type_arguments(self): variable = TypeVar('T') item = TypeVar('Item') From ec8c56bd5a829c7d1fb245090da9a2b562db31ff Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:20:49 +0700 Subject: [PATCH 28/72] Accept TypeAlias declaration markers --- docs/type-safety.md | 2 ++ src/aiython/type_constraints.py | 4 +++- tests/test_python_compatibility.py | 7 +++++++ tests/test_type_constraints_edges.py | 12 +++++++++++- 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 675d3aa..54807ff 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -36,6 +36,8 @@ Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, An Primitive checks are strict: no string-to-number conversion, and bool does not pass an int contract. `Any` is an explicit escape from value checking. Bare containers have unconstrained elements. `Final` bindings reject reassignment; `ClassVar` direct writes must target the class. +`TypeAlias` marks an alias declaration and does not constrain the alias object itself. Values annotated with that alias are checked against its target type. + Annotations are interpreted rather than passed to `eval`. On Python 3.14+, Aiython uses string-format annotation introspection; on 3.11–3.13, it reads stored annotations without evaluating strings. Type aliases on 3.12–3.13 use Python's lazy alias value machinery, which can evaluate code supplied by the alias author. Custom annotation machinery and custom validators are trusted Python code, not sandboxed code. `aiython.type_constraints.register_validator(Class, validator)` supplies a custom runtime predicate. A class validator returns true for a valid value. It can also provide the structural check for a Protocol that Aiython cannot prove automatically. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 5c3e027..3f93d94 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -16,7 +16,7 @@ import sys import types import typing -from typing_extensions import ReadOnly, TypeAliasType, TypeGuard, TypeIs +from typing_extensions import ReadOnly, TypeAlias, TypeAliasType, TypeGuard, TypeIs try: import annotationlib @@ -484,6 +484,8 @@ def value(self,target,names): if isinstance(target,Contract): return target if target is None or target is type(None): return Contract('null','None') if target is typing.Any: return Contract('any','Any') + if target is typing.TypeAlias or target is TypeAlias: + return Contract('any', 'TypeAlias') if isinstance(target, typing.TypeVarTuple): bound = names.get(target, names.get(target.__name__)) return (Contract('unpack_fixed', target.__name__, bound) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 63b74e3..860b861 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1079,6 +1079,13 @@ def is_int(value: object) -> TypeGuard[int]: def deferred(value) -> TypeGuard[Undefined]: return bool(value) print(is_int(1), is_int('x'), deferred('yes')) +''', + 'type alias annotation': '''from typing import TypeAlias +Numbers: TypeAlias = list[int] +Forward: TypeAlias = 'dict[str, int]' +numbers: Numbers = [1, 2] +mapping: Forward = {'one': 1} +print(numbers, mapping) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 995c2d7..8497e06 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -9,12 +9,22 @@ from typing import Any, Annotated, Callable, ClassVar, Final, Generic, Literal, NewType, Optional, Protocol, Required, Self, TypeGuard, TypeVar, TypeVarTuple, TypedDict, Unpack import unittest from unittest.mock import patch -from typing_extensions import TypeIs +from typing_extensions import TypeAlias, TypeIs from aiython import type_constraints as tc class ContractEdgeTests(unittest.TestCase): + def test_type_alias_marker_allows_alias_declaration(self): + marker = tc.compile_contract('TypeAlias', {'TypeAlias': TypeAlias}) + self.assertEqual(marker.kind, 'any') + marker.validate(list[int]) + marker.validate('list[int]') + alias = tc.compile_contract('Values', {'Values': list[int]}) + alias.validate([1, 2]) + with self.assertRaises(tc.TypeViolation): + alias.validate(['wrong']) + def test_type_narrowing_annotations_check_boolean_results(self): namespace = {'TypeGuard': TypeGuard, 'TypeIs': TypeIs} for annotation in ('TypeGuard[int]', TypeGuard[int], From 05750d32798b6b3fd61c5b6368875eb7d471a0db Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:23:18 +0700 Subject: [PATCH 29/72] Validate standard generic collection annotations --- docs/type-safety.md | 2 +- src/aiython/type_constraints.py | 30 ++++++++++++++++++---------- tests/test_python_compatibility.py | 8 ++++++++ tests/test_type_constraints_edges.py | 26 +++++++++++++++++++++++- 4 files changed, 54 insertions(+), 12 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 54807ff..1e45254 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -32,7 +32,7 @@ Expected types reach direct AI expressions in annotated assignments, later assig ## Type forms -Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, concrete Sequence/Mapping values, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. +Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, concrete Sequence/Mapping values, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. Primitive checks are strict: no string-to-number conversion, and bool does not pass an int contract. `Any` is an explicit escape from value checking. Bare containers have unconstrained elements. `Final` bindings reject reassignment; `ClassVar` direct writes must target the class. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 3f93d94..50d8976 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -8,6 +8,7 @@ import ast import builtins import collections.abc as abc +from collections import Counter, OrderedDict, defaultdict, deque from dataclasses import dataclass, field import dataclasses import enum @@ -31,6 +32,13 @@ READ_ONLY_TYPES = tuple({ReadOnly, getattr(typing, "ReadOnly", ReadOnly)}) TYPE_NARROWING_TYPES = tuple({TypeGuard, TypeIs, typing.TypeGuard, getattr(typing, 'TypeIs', TypeIs)}) +CONCRETE_SEQUENCE_TYPES = {'list':list, 'set':set, 'frozenset':frozenset, + 'deque':deque, 'tuple':tuple, 'tuple_many':tuple} +CONCRETE_MAPPING_TYPES = {'dict':dict, 'mapping':dict, 'defaultdict':defaultdict, + 'ordered_dict':OrderedDict, 'counter':Counter} +CONTAINER_KINDS = {list:'list', set:'set', frozenset:'frozenset', dict:'dict', + deque:'deque', defaultdict:'defaultdict', OrderedDict:'ordered_dict', + Counter:'counter', abc.Sequence:'sequence', abc.Mapping:'mapping'} class TypeViolation(CapabilityError, TypeError): @@ -87,7 +95,7 @@ def schema(self, seen=None): elif kind == 'callable': result = {'x-python-callable': True} elif kind in ('str', 'int', 'float', 'bool'): result = {'type': {'str':'string', 'int':'integer', 'float':'number', 'bool':'boolean'}[kind]} - elif kind in ('list', 'set', 'frozenset', 'sequence'): + elif kind in ('list', 'set', 'frozenset', 'sequence', 'deque'): result = {'type': 'array', 'items': self.args[0].schema(seen)} elif kind == 'tuple': result = {'type': 'array', 'prefixItems': [a.schema(seen) for a in self.args], @@ -98,7 +106,7 @@ def schema(self, seen=None): 'prefixItems': [item.schema(seen) for item in self.args[:pivot]], 'x-python-suffixItems': [item.schema(seen) for item in self.args[pivot + 1:]]} elif kind == 'tuple_many': result = {'type':'array', 'items':self.args[0].schema(seen)} - elif kind in ('dict', 'mapping'): + elif kind in ('dict', 'mapping', 'defaultdict', 'ordered_dict', 'counter'): result = {'type':'object', 'additionalProperties':self.args[1].schema(seen), 'x-key-schema':self.args[0].schema(seen)} elif kind == 'class' and isinstance(self.python_type,type) and issubclass(self.python_type,enum.Enum): values = [v.value for v in self.python_type] @@ -155,8 +163,8 @@ def child(contract, item, suffix): if value is not None: fail() elif kind in ('str','int','float','bool','bytes','complex'): if type(value) is not self.python_type: fail() - elif kind in ('list','set','frozenset','sequence','tuple_many','tuple'): - expected = {'list':list,'set':set,'frozenset':frozenset,'tuple':tuple,'tuple_many':tuple}.get(kind) + elif kind in ('list','set','frozenset','sequence','deque','tuple_many','tuple'): + expected = CONCRETE_SEQUENCE_TYPES.get(kind) if expected is not None and type(value) is not expected: fail() if kind == 'sequence' and type(value) not in (list,tuple,str,bytes,range): fail('only non-consuming concrete sequences can be checked') @@ -180,8 +188,9 @@ def child(contract, item, suffix): for index, contract in enumerate(self.args[pivot + 1:]): position = len(value) - suffix + index child(contract, value[position], f'[{position}]') - elif kind in ('dict','mapping'): - if type(value) is not dict: fail('a concrete dict is required for deep checking') + elif kind in ('dict','mapping','defaultdict','ordered_dict','counter'): + expected = CONCRETE_MAPPING_TYPES[kind] + if type(value) is not expected: fail(f'a concrete {expected.__name__} is required for deep checking') for index, (key,item) in enumerate(value.items()): child(self.args[0],key,f'.keys[{index}]') child(self.args[1],item,f'[{key!r}]' if type(key) in (str,int) else f'.values[{index}]') @@ -305,12 +314,13 @@ def generic(self,base,args,label,names): if len(args) != 1: raise UnsupportedType('Qualifier requires one type') return Contract('qualifier',label,args,qualifier=base._name) - containers = {list:'list',set:'set',frozenset:'frozenset',dict:'dict',abc.Sequence:'sequence',abc.Mapping:'mapping'} - if origin in containers: - expected = 2 if origin in (dict,abc.Mapping) else 1 + if origin in CONTAINER_KINDS: + expected = 2 if origin in (dict,abc.Mapping,defaultdict,OrderedDict) else 1 if len(args) != expected: raise UnsupportedType(f'{label}: wrong number of type parameters') - return Contract(containers[origin],label,args) + if origin is Counter: + args += (Contract('int','int',python_type=int),) + return Contract(CONTAINER_KINDS[origin],label,args) if origin is tuple: if len(args) == 2 and args[1] is Ellipsis: return Contract('tuple_many',label,args[:1]) expanded = [] diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 860b861..08ada25 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1086,6 +1086,14 @@ def deferred(value) -> TypeGuard[Undefined]: numbers: Numbers = [1, 2] mapping: Forward = {'one': 1} print(numbers, mapping) +''', + 'generic collections': '''from collections import Counter, OrderedDict, defaultdict, deque +from typing import Deque, DefaultDict +numbers: Deque[int] = deque([1, 2]) +mapping: DefaultDict[str, int] = defaultdict(int, {'x': 3}) +ordered: OrderedDict[str, int] = OrderedDict([('a', 4)]) +counts: Counter[str] = Counter({'a': 2}) +print(list(numbers), dict(mapping), list(ordered.items()), dict(counts)) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 8497e06..918323b 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -1,9 +1,10 @@ import ast -from collections import OrderedDict +from collections import Counter, OrderedDict, defaultdict, deque from dataclasses import dataclass import enum from pathlib import Path import sys +import typing import types from types import SimpleNamespace from typing import Any, Annotated, Callable, ClassVar, Final, Generic, Literal, NewType, Optional, Protocol, Required, Self, TypeGuard, TypeVar, TypeVarTuple, TypedDict, Unpack @@ -15,6 +16,29 @@ class ContractEdgeTests(unittest.TestCase): + def test_concrete_collections_preserve_generic_member_checks(self): + namespace = {'typing': typing, 'deque': deque, 'defaultdict': defaultdict, + 'OrderedDict': OrderedDict, 'Counter': Counter} + cases = ( + ('deque[int]', deque([1]), deque(['wrong']), 'array'), + ('typing.Deque[int]', deque([1]), deque(['wrong']), 'array'), + ('defaultdict[str, int]', defaultdict(int, {'x': 1}), + defaultdict(int, {'x': 'wrong'}), 'object'), + ('typing.DefaultDict[str, int]', defaultdict(int, {'x': 1}), + defaultdict(int, {1: 2}), 'object'), + ('OrderedDict[str, int]', OrderedDict([('x', 1)]), + OrderedDict([('x', 'wrong')]), 'object'), + ('Counter[str]', Counter({'x': 2}), Counter({'x': 'wrong'}), 'object'), + ('typing.Counter[str]', Counter({'x': 2}), Counter({1: 2}), 'object'), + ) + for annotation, valid, invalid, schema_type in cases: + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, namespace) + self.assertEqual(contract.schema()['type'], schema_type) + contract.validate(valid) + with self.assertRaises(tc.TypeViolation): + contract.validate(invalid) + def test_type_alias_marker_allows_alias_declaration(self): marker = tc.compile_contract('TypeAlias', {'TypeAlias': TypeAlias}) self.assertEqual(marker.kind, 'any') From 561a2782c13c80d41f01cc2654c770b6b67835d5 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:25:14 +0700 Subject: [PATCH 30/72] Validate typed regular expression objects --- docs/type-safety.md | 2 +- src/aiython/type_constraints.py | 13 +++++++++++++ tests/test_python_compatibility.py | 6 ++++++ tests/test_type_constraints_edges.py | 17 +++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 1e45254..ac8848a 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -32,7 +32,7 @@ Expected types reach direct AI expressions in annotated assignments, later assig ## Type forms -Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, concrete Sequence/Mapping values, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. +Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, `Pattern[str | bytes]`, `Match[str | bytes]`, concrete Sequence/Mapping values, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. Primitive checks are strict: no string-to-number conversion, and bool does not pass an int contract. `Any` is an explicit escape from value checking. Bare containers have unconstrained elements. `Final` bindings reject reassignment; `ClassVar` direct writes must target the class. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 50d8976..d6e88a4 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -14,6 +14,7 @@ import enum from functools import lru_cache import inspect +import re import sys import types import typing @@ -93,6 +94,8 @@ def schema(self, seen=None): if members: result['x-python-enum-members'] = members elif kind == 'null': result = {'type': 'null'} elif kind == 'callable': result = {'x-python-callable': True} + elif kind in ('pattern', 'match'): + result = {'x-python-regex': kind, 'x-python-input-type': self.args[0].schema(seen)} elif kind in ('str', 'int', 'float', 'bool'): result = {'type': {'str':'string', 'int':'integer', 'float':'number', 'bool':'boolean'}[kind]} elif kind in ('list', 'set', 'frozenset', 'sequence', 'deque'): @@ -159,6 +162,11 @@ def child(contract, item, suffix): fail() elif kind == 'literal': if not any(type(value) is type(v) and value == v for v in self.args): fail('not an allowed literal') + elif kind in ('pattern', 'match'): + expected = re.Pattern if kind == 'pattern' else re.Match + if type(value) is not expected: fail() + child(self.args[0], value.pattern if kind == 'pattern' else value.string, + '.pattern' if kind == 'pattern' else '.string') elif kind == 'null': if value is not None: fail() elif kind in ('str','int','float','bool','bytes','complex'): @@ -303,6 +311,11 @@ def generic(self,base,args,label,names): return args[0] if origin is abc.Callable: return Contract('callable',label) + if origin in (re.Pattern, re.Match): + if len(args) > 1: + raise UnsupportedType('Regex type requires one input type') + return Contract('pattern' if origin is re.Pattern else 'match', label, + args or (Contract('any', 'Any'),)) if origin in (typing.Union,types.UnionType): return Contract('union',label,args) if base is typing.Optional: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 08ada25..086c487 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1094,6 +1094,12 @@ def deferred(value) -> TypeGuard[Undefined]: ordered: OrderedDict[str, int] = OrderedDict([('a', 4)]) counts: Counter[str] = Counter({'a': 2}) print(list(numbers), dict(mapping), list(ordered.items()), dict(counts)) +''', + 'generic regex': '''import re +from typing import Pattern, Match +pattern: Pattern[str] = re.compile('a+') +match: Match[str] | None = pattern.search('aa') +print(pattern.pattern, match.group() if match else None) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 918323b..daf1b61 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -3,6 +3,7 @@ from dataclasses import dataclass import enum from pathlib import Path +import re import sys import typing import types @@ -16,6 +17,22 @@ class ContractEdgeTests(unittest.TestCase): + def test_regex_generic_annotations_check_input_type(self): + namespace = {'re': re, 'typing': typing} + cases = ( + ('re.Pattern[str]', re.compile('a'), re.compile(b'a'), 'pattern'), + ('typing.Pattern[str]', re.compile('a'), re.compile(b'a'), 'pattern'), + ('re.Match[bytes]', re.match(b'a', b'a'), re.match('a', 'a'), 'match'), + ('typing.Match[str]', re.match('a', 'a'), re.match(b'a', b'a'), 'match'), + ) + for annotation, valid, invalid, kind in cases: + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, namespace) + self.assertEqual(contract.schema()['x-python-regex'], kind) + contract.validate(valid) + with self.assertRaises(tc.TypeViolation): + contract.validate(invalid) + def test_concrete_collections_preserve_generic_member_checks(self): namespace = {'typing': typing, 'deque': deque, 'defaultdict': defaultdict, 'OrderedDict': OrderedDict, 'Counter': Counter} From 4a54907015a8ca4a9780492d5712c7d3061281ec Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:27:50 +0700 Subject: [PATCH 31/72] Accept standard typed IO streams --- docs/type-safety.md | 2 +- src/aiython/type_constraints.py | 18 ++++++++++++++++++ tests/test_python_compatibility.py | 7 +++++++ tests/test_type_constraints_edges.py | 19 +++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index ac8848a..7183f19 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -32,7 +32,7 @@ Expected types reach direct AI expressions in annotated assignments, later assig ## Type forms -Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, `Pattern[str | bytes]`, `Match[str | bytes]`, concrete Sequence/Mapping values, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. +Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, typed regular expression objects, standard text/binary `IO` streams, concrete Sequence/Mapping values, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. Primitive checks are strict: no string-to-number conversion, and bool does not pass an int contract. `Any` is an explicit escape from value checking. Bare containers have unconstrained elements. `Final` bindings reject reassignment; `ClassVar` direct writes must target the class. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index d6e88a4..e978b5d 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -13,6 +13,7 @@ import dataclasses import enum from functools import lru_cache +import io import inspect import re import sys @@ -94,6 +95,8 @@ def schema(self, seen=None): if members: result['x-python-enum-members'] = members elif kind == 'null': result = {'type': 'null'} elif kind == 'callable': result = {'x-python-callable': True} + elif kind == 'io': + result = {'x-python-io': self.args[0].name} elif kind in ('pattern', 'match'): result = {'x-python-regex': kind, 'x-python-input-type': self.args[0].schema(seen)} elif kind in ('str', 'int', 'float', 'bool'): @@ -162,6 +165,11 @@ def child(contract, item, suffix): fail() elif kind == 'literal': if not any(type(value) is type(v) and value == v for v in self.args): fail('not an allowed literal') + elif kind == 'io': + target = self.args[0] + if not issubclass(type(value), io.IOBase): fail() + if target.kind == 'str' and not issubclass(type(value), io.TextIOBase): fail() + if target.kind == 'bytes' and not issubclass(type(value), (io.BufferedIOBase, io.RawIOBase)): fail() elif kind in ('pattern', 'match'): expected = re.Pattern if kind == 'pattern' else re.Match if type(value) is not expected: fail() @@ -311,6 +319,10 @@ def generic(self,base,args,label,names): return args[0] if origin is abc.Callable: return Contract('callable',label) + if origin is typing.IO: + if len(args) != 1 or args[0].kind not in ('str', 'bytes', 'any'): + raise UnsupportedType('IO requires str, bytes, or Any as its stream type') + return Contract('io', label, args) if origin in (re.Pattern, re.Match): if len(args) > 1: raise UnsupportedType('Regex type requires one input type') @@ -507,6 +519,12 @@ def value(self,target,names): if isinstance(target,Contract): return target if target is None or target is type(None): return Contract('null','None') if target is typing.Any: return Contract('any','Any') + if target is typing.IO or target is typing.TextIO or target is typing.BinaryIO: + stream_type = ('str' if target is typing.TextIO else + 'bytes' if target is typing.BinaryIO else 'any') + return Contract('io', target.__name__, + (Contract(stream_type, stream_type, + python_type={'str':str, 'bytes':bytes}.get(stream_type)),)) if target is typing.TypeAlias or target is TypeAlias: return Contract('any', 'TypeAlias') if isinstance(target, typing.TypeVarTuple): diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 086c487..27950d2 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1100,6 +1100,13 @@ def deferred(value) -> TypeGuard[Undefined]: pattern: Pattern[str] = re.compile('a+') match: Match[str] | None = pattern.search('aa') print(pattern.pattern, match.group() if match else None) +''', + 'typed streams': '''import io +from typing import IO, TextIO, BinaryIO +text: IO[str] = io.StringIO('alpha') +binary: BinaryIO = io.BytesIO(b'beta') +def read_text(stream: TextIO) -> str: return stream.read() +print(read_text(text), binary.read()) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index daf1b61..823c826 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -2,6 +2,7 @@ from collections import Counter, OrderedDict, defaultdict, deque from dataclasses import dataclass import enum +import io from pathlib import Path import re import sys @@ -17,6 +18,24 @@ class ContractEdgeTests(unittest.TestCase): + def test_io_annotations_use_standard_stream_classes(self): + text = io.StringIO('alpha') + binary = io.BytesIO(b'beta') + for annotation, valid, invalid, stream_type in ( + ('typing.IO[str]', text, binary, 'str'), + ('typing.IO[bytes]', binary, text, 'bytes'), + ('typing.TextIO', text, binary, 'str'), + ('typing.BinaryIO', binary, text, 'bytes'), + ): + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, {'typing': typing}) + self.assertEqual(contract.schema()['x-python-io'], stream_type) + contract.validate(valid) + with self.assertRaises(tc.TypeViolation): + contract.validate(invalid) + tc.compile_contract('typing.IO', {'typing': typing}).validate(text) + tc.compile_contract('typing.IO[Any]', {'typing': typing}).validate(binary) + def test_regex_generic_annotations_check_input_type(self): namespace = {'re': re, 'typing': typing} cases = ( From e817db4b766801efee3e4c65e861ebff8cabac28 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:30:28 +0700 Subject: [PATCH 32/72] Bind AnyStr through typed IO streams --- docs/type-safety.md | 2 ++ src/aiython/type_constraints.py | 12 ++++++++---- tests/test_python_compatibility.py | 5 +++-- tests/test_type_constraints_edges.py | 10 ++++++++++ 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 7183f19..7c3ddef 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -38,6 +38,8 @@ Primitive checks are strict: no string-to-number conversion, and bool does not p `TypeAlias` marks an alias declaration and does not constrain the alias object itself. Values annotated with that alias are checked against its target type. +`IO[AnyStr]` binds `AnyStr` from the stream's standard text or binary base class without reading the stream. + Annotations are interpreted rather than passed to `eval`. On Python 3.14+, Aiython uses string-format annotation introspection; on 3.11–3.13, it reads stored annotations without evaluating strings. Type aliases on 3.12–3.13 use Python's lazy alias value machinery, which can evaluate code supplied by the alias author. Custom annotation machinery and custom validators are trusted Python code, not sandboxed code. `aiython.type_constraints.register_validator(Class, validator)` supplies a custom runtime predicate. A class validator returns true for a valid value. It can also provide the structural check for a Protocol that Aiython cannot prove automatically. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index e978b5d..4023ceb 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -168,8 +168,12 @@ def child(contract, item, suffix): elif kind == 'io': target = self.args[0] if not issubclass(type(value), io.IOBase): fail() - if target.kind == 'str' and not issubclass(type(value), io.TextIOBase): fail() - if target.kind == 'bytes' and not issubclass(type(value), (io.BufferedIOBase, io.RawIOBase)): fail() + if issubclass(type(value), io.TextIOBase): + child(target, '', '.read()') + elif issubclass(type(value), (io.BufferedIOBase, io.RawIOBase)): + child(target, b'', '.read()') + elif target.kind != 'any': + fail('stream data type cannot be determined without reading it') elif kind in ('pattern', 'match'): expected = re.Pattern if kind == 'pattern' else re.Match if type(value) is not expected: fail() @@ -320,8 +324,8 @@ def generic(self,base,args,label,names): if origin is abc.Callable: return Contract('callable',label) if origin is typing.IO: - if len(args) != 1 or args[0].kind not in ('str', 'bytes', 'any'): - raise UnsupportedType('IO requires str, bytes, or Any as its stream type') + if len(args) != 1: + raise UnsupportedType('IO requires one stream type') return Contract('io', label, args) if origin in (re.Pattern, re.Match): if len(args) > 1: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 27950d2..a395981 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1102,11 +1102,12 @@ def deferred(value) -> TypeGuard[Undefined]: print(pattern.pattern, match.group() if match else None) ''', 'typed streams': '''import io -from typing import IO, TextIO, BinaryIO +from typing import AnyStr, IO, TextIO, BinaryIO text: IO[str] = io.StringIO('alpha') binary: BinaryIO = io.BytesIO(b'beta') def read_text(stream: TextIO) -> str: return stream.read() -print(read_text(text), binary.read()) +def read_any(stream: IO[AnyStr]) -> AnyStr: return stream.read() +print(read_text(text), read_any(io.StringIO('gamma')), read_any(binary)) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 823c826..0a9abed 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -35,6 +35,16 @@ def test_io_annotations_use_standard_stream_classes(self): contract.validate(invalid) tc.compile_contract('typing.IO', {'typing': typing}).validate(text) tc.compile_contract('typing.IO[Any]', {'typing': typing}).validate(binary) + generic = tc.compile_contract('typing.IO[typing.AnyStr]', {'typing': typing}) + returned = tc.compile_contract('typing.AnyStr', {'typing': typing}) + for stream, matching, wrong in ((text, 'text', b'wrong'), + (binary, b'binary', 'wrong')): + with self.subTest(stream=type(stream).__name__): + bindings = {} + generic.validate(stream, bindings=bindings) + returned.validate(matching, bindings=bindings) + with self.assertRaises(tc.TypeViolation): + returned.validate(wrong, bindings=bindings) def test_regex_generic_annotations_check_input_type(self): namespace = {'re': re, 'typing': typing} From 0a1246ed6467d0dfe868defb408e6654f26d5a10 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:33:10 +0700 Subject: [PATCH 33/72] Check concrete implementations of mapping annotations --- docs/type-safety.md | 2 ++ src/aiython/type_constraints.py | 21 ++++++++++++++------- tests/test_python_compatibility.py | 10 ++++++++++ tests/test_type_constraints_edges.py | 14 ++++++++++++++ 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 7c3ddef..6a55f15 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -40,6 +40,8 @@ Primitive checks are strict: no string-to-number conversion, and bool does not p `IO[AnyStr]` binds `AnyStr` from the stream's standard text or binary base class without reading the stream. +`Mapping[K, V]` and `MutableMapping[K, V]` check concrete `dict`, `defaultdict`, `OrderedDict`, and `Counter` values deeply. + Annotations are interpreted rather than passed to `eval`. On Python 3.14+, Aiython uses string-format annotation introspection; on 3.11–3.13, it reads stored annotations without evaluating strings. Type aliases on 3.12–3.13 use Python's lazy alias value machinery, which can evaluate code supplied by the alias author. Custom annotation machinery and custom validators are trusted Python code, not sandboxed code. `aiython.type_constraints.register_validator(Class, validator)` supplies a custom runtime predicate. A class validator returns true for a valid value. It can also provide the structural check for a Protocol that Aiython cannot prove automatically. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 4023ceb..5aa5623 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -36,11 +36,13 @@ getattr(typing, 'TypeIs', TypeIs)}) CONCRETE_SEQUENCE_TYPES = {'list':list, 'set':set, 'frozenset':frozenset, 'deque':deque, 'tuple':tuple, 'tuple_many':tuple} -CONCRETE_MAPPING_TYPES = {'dict':dict, 'mapping':dict, 'defaultdict':defaultdict, +CONCRETE_MAPPING_TYPES = {'dict':dict, 'defaultdict':defaultdict, 'ordered_dict':OrderedDict, 'counter':Counter} +SAFE_MAPPING_TYPES = (dict, defaultdict, OrderedDict, Counter) CONTAINER_KINDS = {list:'list', set:'set', frozenset:'frozenset', dict:'dict', deque:'deque', defaultdict:'defaultdict', OrderedDict:'ordered_dict', - Counter:'counter', abc.Sequence:'sequence', abc.Mapping:'mapping'} + Counter:'counter', abc.Sequence:'sequence', abc.Mapping:'mapping', + abc.MutableMapping:'mutable_mapping'} class TypeViolation(CapabilityError, TypeError): @@ -112,7 +114,7 @@ def schema(self, seen=None): 'prefixItems': [item.schema(seen) for item in self.args[:pivot]], 'x-python-suffixItems': [item.schema(seen) for item in self.args[pivot + 1:]]} elif kind == 'tuple_many': result = {'type':'array', 'items':self.args[0].schema(seen)} - elif kind in ('dict', 'mapping', 'defaultdict', 'ordered_dict', 'counter'): + elif kind in ('dict', 'mapping', 'mutable_mapping', 'defaultdict', 'ordered_dict', 'counter'): result = {'type':'object', 'additionalProperties':self.args[1].schema(seen), 'x-key-schema':self.args[0].schema(seen)} elif kind == 'class' and isinstance(self.python_type,type) and issubclass(self.python_type,enum.Enum): values = [v.value for v in self.python_type] @@ -208,9 +210,14 @@ def child(contract, item, suffix): for index, contract in enumerate(self.args[pivot + 1:]): position = len(value) - suffix + index child(contract, value[position], f'[{position}]') - elif kind in ('dict','mapping','defaultdict','ordered_dict','counter'): - expected = CONCRETE_MAPPING_TYPES[kind] - if type(value) is not expected: fail(f'a concrete {expected.__name__} is required for deep checking') + elif kind in ('dict','mapping','mutable_mapping','defaultdict','ordered_dict','counter'): + if kind in ('mapping', 'mutable_mapping'): + if type(value) not in SAFE_MAPPING_TYPES: + fail('only concrete mappings can be checked deeply') + else: + expected = CONCRETE_MAPPING_TYPES[kind] + if type(value) is not expected: + fail(f'a concrete {expected.__name__} is required for deep checking') for index, (key,item) in enumerate(value.items()): child(self.args[0],key,f'.keys[{index}]') child(self.args[1],item,f'[{key!r}]' if type(key) in (str,int) else f'.values[{index}]') @@ -344,7 +351,7 @@ def generic(self,base,args,label,names): raise UnsupportedType('Qualifier requires one type') return Contract('qualifier',label,args,qualifier=base._name) if origin in CONTAINER_KINDS: - expected = 2 if origin in (dict,abc.Mapping,defaultdict,OrderedDict) else 1 + expected = 2 if origin in (dict,abc.Mapping,abc.MutableMapping,defaultdict,OrderedDict) else 1 if len(args) != expected: raise UnsupportedType(f'{label}: wrong number of type parameters') if origin is Counter: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index a395981..d8a179c 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1108,6 +1108,16 @@ def deferred(value) -> TypeGuard[Undefined]: def read_text(stream: TextIO) -> str: return stream.read() def read_any(stream: IO[AnyStr]) -> AnyStr: return stream.read() print(read_text(text), read_any(io.StringIO('gamma')), read_any(binary)) +''', + 'abstract mapping implementations': '''from collections import OrderedDict, defaultdict +from typing import Mapping, MutableMapping +def total(values: Mapping[str, int]) -> int: return sum(values.values()) +def increment(values: MutableMapping[str, int]) -> int: + values['x'] += 1 + return values['x'] +ordered = OrderedDict(x=2) +defaulted = defaultdict(int, x=3) +print(total(ordered), increment(defaulted)) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 0a9abed..8eea4a3 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -18,6 +18,20 @@ class ContractEdgeTests(unittest.TestCase): + def test_abstract_mappings_accept_known_concrete_implementations(self): + namespace = {'typing': typing} + for annotation in ('typing.Mapping[str, int]', 'typing.MutableMapping[str, int]'): + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, namespace) + self.assertEqual(contract.schema()['type'], 'object') + for value in (dict(x=1), OrderedDict(x=1), + defaultdict(int, x=1), Counter(x=1)): + contract.validate(value) + with self.assertRaises(tc.TypeViolation): + contract.validate(OrderedDict(x='wrong')) + with self.assertRaises(tc.TypeViolation): + tc.compile_contract('dict[str, int]', {}).validate(OrderedDict(x=1)) + def test_io_annotations_use_standard_stream_classes(self): text = io.StringIO('alpha') binary = io.BytesIO(b'beta') From 90a2bcf33a38ff12f6a11ad171df6b5f3b46da6e Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:36:59 +0700 Subject: [PATCH 34/72] Support abstract concrete collections and bare typing aliases --- docs/type-safety.md | 2 +- src/aiython/type_constraints.py | 27 ++++++++++++++++++++++----- tests/test_python_compatibility.py | 11 +++++++++++ tests/test_type_constraints_edges.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 6a55f15..c6ad06c 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -32,7 +32,7 @@ Expected types reach direct AI expressions in annotated assignments, later assig ## Type forms -Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, typed regular expression objects, standard text/binary `IO` streams, concrete Sequence/Mapping values, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. +Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, typed regular expression objects, standard text/binary `IO` streams, concrete Sequence/Mapping/Collection and mutable or set variants, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. Primitive checks are strict: no string-to-number conversion, and bool does not pass an int contract. `Any` is an explicit escape from value checking. Bare containers have unconstrained elements. `Final` bindings reject reassignment; `ClassVar` direct writes must target the class. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 5aa5623..6f89d36 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -35,14 +35,19 @@ TYPE_NARROWING_TYPES = tuple({TypeGuard, TypeIs, typing.TypeGuard, getattr(typing, 'TypeIs', TypeIs)}) CONCRETE_SEQUENCE_TYPES = {'list':list, 'set':set, 'frozenset':frozenset, - 'deque':deque, 'tuple':tuple, 'tuple_many':tuple} + 'deque':deque, 'tuple':tuple, 'tuple_many':tuple, + 'mutable_set':set} CONCRETE_MAPPING_TYPES = {'dict':dict, 'defaultdict':defaultdict, 'ordered_dict':OrderedDict, 'counter':Counter} SAFE_MAPPING_TYPES = (dict, defaultdict, OrderedDict, Counter) +SAFE_COLLECTION_TYPES = (list, tuple, set, frozenset, dict, str, bytes, + bytearray, memoryview, range, + deque, defaultdict, OrderedDict, Counter) CONTAINER_KINDS = {list:'list', set:'set', frozenset:'frozenset', dict:'dict', deque:'deque', defaultdict:'defaultdict', OrderedDict:'ordered_dict', Counter:'counter', abc.Sequence:'sequence', abc.Mapping:'mapping', - abc.MutableMapping:'mutable_mapping'} + abc.MutableMapping:'mutable_mapping', abc.MutableSequence:'mutable_sequence', + abc.Set:'abstract_set', abc.MutableSet:'mutable_set', abc.Collection:'collection'} class TypeViolation(CapabilityError, TypeError): @@ -103,8 +108,11 @@ def schema(self, seen=None): result = {'x-python-regex': kind, 'x-python-input-type': self.args[0].schema(seen)} elif kind in ('str', 'int', 'float', 'bool'): result = {'type': {'str':'string', 'int':'integer', 'float':'number', 'bool':'boolean'}[kind]} - elif kind in ('list', 'set', 'frozenset', 'sequence', 'deque'): + elif kind in ('list', 'set', 'frozenset', 'sequence', 'deque', + 'mutable_sequence', 'abstract_set', 'mutable_set'): result = {'type': 'array', 'items': self.args[0].schema(seen)} + elif kind == 'collection': + result = {'x-python-collection-items': self.args[0].schema(seen)} elif kind == 'tuple': result = {'type': 'array', 'prefixItems': [a.schema(seen) for a in self.args], 'minItems': len(self.args), 'maxItems': len(self.args)} @@ -185,11 +193,16 @@ def child(contract, item, suffix): if value is not None: fail() elif kind in ('str','int','float','bool','bytes','complex'): if type(value) is not self.python_type: fail() - elif kind in ('list','set','frozenset','sequence','deque','tuple_many','tuple'): + elif kind in ('list','set','frozenset','sequence','deque','tuple_many','tuple', + 'mutable_sequence','abstract_set','mutable_set','collection'): expected = CONCRETE_SEQUENCE_TYPES.get(kind) if expected is not None and type(value) is not expected: fail() - if kind == 'sequence' and type(value) not in (list,tuple,str,bytes,range): + if kind == 'sequence' and type(value) not in (list,tuple,str,bytes,bytearray,memoryview,range): fail('only non-consuming concrete sequences can be checked') + if kind == 'mutable_sequence' and type(value) not in (list,bytearray): fail() + if kind == 'abstract_set' and type(value) not in (set,frozenset): fail() + if kind == 'collection' and type(value) not in SAFE_COLLECTION_TYPES: + fail('only non-consuming concrete collections can be checked') if kind == 'tuple' and len(value) != len(self.args): fail('wrong tuple length') if kind != 'tuple' and self.args[0].kind in PRIMITIVE_KINDS: member = self.args[0] @@ -352,6 +365,8 @@ def generic(self,base,args,label,names): return Contract('qualifier',label,args,qualifier=base._name) if origin in CONTAINER_KINDS: expected = 2 if origin in (dict,abc.Mapping,abc.MutableMapping,defaultdict,OrderedDict) else 1 + if not args: + args = (Contract('any', 'Any'),) * expected if len(args) != expected: raise UnsupportedType(f'{label}: wrong number of type parameters') if origin is Counter: @@ -530,6 +545,8 @@ def value(self,target,names): if isinstance(target,Contract): return target if target is None or target is type(None): return Contract('null','None') if target is typing.Any: return Contract('any','Any') + if target is typing.Tuple: + return Contract('tuple_many', 'typing.Tuple', (Contract('any', 'Any'),)) if target is typing.IO or target is typing.TextIO or target is typing.BinaryIO: stream_type = ('str' if target is typing.TextIO else 'bytes' if target is typing.BinaryIO else 'any') diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index d8a179c..7662681 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1118,6 +1118,17 @@ def increment(values: MutableMapping[str, int]) -> int: ordered = OrderedDict(x=2) defaulted = defaultdict(int, x=3) print(total(ordered), increment(defaulted)) +''', + 'abstract collections and bare aliases': '''from typing import AbstractSet, Collection, List, MutableSequence, Sequence, Tuple +def count(values: Collection[int]) -> int: return len(values) +def extend(values: MutableSequence[int]) -> list[int]: + values.append(2) + return list(values) +def size(values: AbstractSet[str]) -> int: return len(values) +bare: List = [1, 'x'] +bare_tuple: Tuple = (1, 'x') +view: Sequence[int] = memoryview(b'A') +print(count({1: 'one'}), extend(bytearray(b'A')), size(frozenset({'x'})), bare, bare_tuple, list(view)) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 8eea4a3..2d29ced 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -18,6 +18,34 @@ class ContractEdgeTests(unittest.TestCase): + def test_abstract_collections_check_known_concrete_values(self): + namespace = {'typing': typing} + cases = ( + ('typing.MutableSequence[int]', [1, 2], (1, 2)), + ('typing.AbstractSet[str]', frozenset({'x'}), frozenset({1})), + ('typing.MutableSet[int]', {1}, frozenset({1})), + ('typing.Collection[int]', {1: 'value'}, ['wrong']), + ('typing.Sequence[int]', memoryview(b'abc'), ['wrong']), + ) + for annotation, valid, invalid in cases: + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, namespace) + contract.validate(valid) + with self.assertRaises(tc.TypeViolation): + contract.validate(invalid) + tc.compile_contract('typing.MutableSequence[int]', namespace).validate(bytearray(b'a')) + for annotation, value in (('typing.List', [1, 'x']), + ('typing.Dict', {'x': 1}), + ('typing.Collection', {1: 2}), + ('typing.Sequence', memoryview(b'a')), + ('typing.Tuple', (1, 'x'))): + with self.subTest(annotation=annotation): + tc.compile_contract(annotation, namespace).validate(value) + empty = tc.compile_contract('typing.Tuple[()]', namespace) + empty.validate(()) + with self.assertRaises(tc.TypeViolation): + empty.validate((1,)) + def test_abstract_mappings_accept_known_concrete_implementations(self): namespace = {'typing': typing} for annotation in ('typing.Mapping[str, int]', 'typing.MutableMapping[str, int]'): From 5fa6aa05571cf5c0924c6d85cb9bb0e7d9a04fb8 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:41:24 +0700 Subject: [PATCH 35/72] Validate Hashable Sized and ChainMap annotations --- docs/type-safety.md | 2 +- src/aiython/type_constraints.py | 25 +++++++++++++++++++++---- tests/test_python_compatibility.py | 10 ++++++++++ tests/test_type_constraints_edges.py | 26 +++++++++++++++++++++++++- 4 files changed, 57 insertions(+), 6 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index c6ad06c..4e45de2 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -32,7 +32,7 @@ Expected types reach direct AI expressions in annotated assignments, later assig ## Type forms -Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, typed regular expression objects, standard text/binary `IO` streams, concrete Sequence/Mapping/Collection and mutable or set variants, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. +Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, `ChainMap`, typed regular expression objects, standard text/binary `IO` streams, concrete Sequence/Mapping/Collection and mutable or set variants, `Hashable`, `Sized`, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. Primitive checks are strict: no string-to-number conversion, and bool does not pass an int contract. `Any` is an explicit escape from value checking. Bare containers have unconstrained elements. `Final` bindings reject reassignment; `ClassVar` direct writes must target the class. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 6f89d36..3e7f08e 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -8,7 +8,7 @@ import ast import builtins import collections.abc as abc -from collections import Counter, OrderedDict, defaultdict, deque +from collections import ChainMap, Counter, OrderedDict, defaultdict, deque from dataclasses import dataclass, field import dataclasses import enum @@ -45,7 +45,7 @@ deque, defaultdict, OrderedDict, Counter) CONTAINER_KINDS = {list:'list', set:'set', frozenset:'frozenset', dict:'dict', deque:'deque', defaultdict:'defaultdict', OrderedDict:'ordered_dict', - Counter:'counter', abc.Sequence:'sequence', abc.Mapping:'mapping', + Counter:'counter', ChainMap:'chainmap', abc.Sequence:'sequence', abc.Mapping:'mapping', abc.MutableMapping:'mutable_mapping', abc.MutableSequence:'mutable_sequence', abc.Set:'abstract_set', abc.MutableSet:'mutable_set', abc.Collection:'collection'} @@ -102,6 +102,7 @@ def schema(self, seen=None): if members: result['x-python-enum-members'] = members elif kind == 'null': result = {'type': 'null'} elif kind == 'callable': result = {'x-python-callable': True} + elif kind == 'abc': result = {'x-python-abc': self.name} elif kind == 'io': result = {'x-python-io': self.args[0].name} elif kind in ('pattern', 'match'): @@ -122,7 +123,7 @@ def schema(self, seen=None): 'prefixItems': [item.schema(seen) for item in self.args[:pivot]], 'x-python-suffixItems': [item.schema(seen) for item in self.args[pivot + 1:]]} elif kind == 'tuple_many': result = {'type':'array', 'items':self.args[0].schema(seen)} - elif kind in ('dict', 'mapping', 'mutable_mapping', 'defaultdict', 'ordered_dict', 'counter'): + elif kind in ('dict', 'mapping', 'mutable_mapping', 'defaultdict', 'ordered_dict', 'counter', 'chainmap'): result = {'type':'object', 'additionalProperties':self.args[1].schema(seen), 'x-key-schema':self.args[0].schema(seen)} elif kind == 'class' and isinstance(self.python_type,type) and issubclass(self.python_type,enum.Enum): values = [v.value for v in self.python_type] @@ -175,6 +176,8 @@ def child(contract, item, suffix): fail() elif kind == 'literal': if not any(type(value) is type(v) and value == v for v in self.args): fail('not an allowed literal') + elif kind == 'abc': + if not issubclass(type(value), self.python_type): fail() elif kind == 'io': target = self.args[0] if not issubclass(type(value), io.IOBase): fail() @@ -234,6 +237,16 @@ def child(contract, item, suffix): for index, (key,item) in enumerate(value.items()): child(self.args[0],key,f'.keys[{index}]') child(self.args[1],item,f'[{key!r}]' if type(key) in (str,int) else f'.values[{index}]') + elif kind == 'chainmap': + if type(value) is not ChainMap: fail() + if type(value.maps) not in (list, tuple): fail('ChainMap maps must be concrete') + for map_index, mapping in enumerate(value.maps): + if type(mapping) not in SAFE_MAPPING_TYPES: + fail('only concrete ChainMap members can be checked deeply') + for index, (key, item) in enumerate(mapping.items()): + child(self.args[0], key, f'.maps[{map_index}].keys[{index}]') + child(self.args[1], item, f'.maps[{map_index}][{key!r}]' + if type(key) in (str,int) else f'.maps[{map_index}].values[{index}]') elif kind == 'typeddict': if type(value) is not dict: fail() missing = self.required - value.keys() @@ -343,6 +356,10 @@ def generic(self,base,args,label,names): return args[0] if origin is abc.Callable: return Contract('callable',label) + if origin in (abc.Hashable, abc.Sized): + if args: + raise UnsupportedType(f'{label}: this ABC does not take type arguments') + return Contract('abc', label, python_type=origin) if origin is typing.IO: if len(args) != 1: raise UnsupportedType('IO requires one stream type') @@ -364,7 +381,7 @@ def generic(self,base,args,label,names): raise UnsupportedType('Qualifier requires one type') return Contract('qualifier',label,args,qualifier=base._name) if origin in CONTAINER_KINDS: - expected = 2 if origin in (dict,abc.Mapping,abc.MutableMapping,defaultdict,OrderedDict) else 1 + expected = 2 if origin in (dict,abc.Mapping,abc.MutableMapping,defaultdict,OrderedDict,ChainMap) else 1 if not args: args = (Contract('any', 'Any'),) * expected if len(args) != expected: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 7662681..94b826f 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1129,6 +1129,16 @@ def size(values: AbstractSet[str]) -> int: return len(values) bare_tuple: Tuple = (1, 'x') view: Sequence[int] = memoryview(b'A') print(count({1: 'one'}), extend(bytearray(b'A')), size(frozenset({'x'})), bare, bare_tuple, list(view)) +''', + 'hashable and sized annotations': '''from typing import Hashable, Sized +def describe(key: Hashable, values: Sized) -> tuple[int, int]: + return hash(key), len(values) +print(describe(3, [1, 2])) +''', + 'chain map annotation': '''from collections import ChainMap +from typing import ChainMap as TypedChainMap +layers: TypedChainMap[str, int] = ChainMap({'x': 1}, {'y': 2}) +print(layers['x'], layers['y'], len(layers.maps)) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 2d29ced..bfbe29f 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -1,5 +1,5 @@ import ast -from collections import Counter, OrderedDict, defaultdict, deque +from collections import ChainMap, Counter, OrderedDict, defaultdict, deque from dataclasses import dataclass import enum import io @@ -18,6 +18,30 @@ class ContractEdgeTests(unittest.TestCase): + def test_chainmap_checks_every_underlying_mapping(self): + namespace = {'ChainMap': ChainMap, 'typing': typing} + for annotation in ('ChainMap[str, int]', 'typing.ChainMap[str, int]'): + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, namespace) + self.assertEqual(contract.schema()['type'], 'object') + contract.validate(ChainMap({'x': 1}, {'y': 2})) + with self.assertRaisesRegex(tc.TypeViolation, r'maps\[1\]'): + contract.validate(ChainMap({'x': 1}, {'x': 'hidden wrong value'})) + tc.compile_contract('typing.ChainMap', namespace).validate( + ChainMap({'x': 1}, {2: 'other'})) + + def test_nominal_abstract_annotations_without_members(self): + for annotation, valid, invalid in ( + ('typing.Hashable', 3, []), + ('typing.Sized', [1], 3), + ): + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, {'typing': typing}) + self.assertIn('x-python-abc', contract.schema()) + contract.validate(valid) + with self.assertRaises(tc.TypeViolation): + contract.validate(invalid) + def test_abstract_collections_check_known_concrete_values(self): namespace = {'typing': typing} cases = ( From a20891a18e50752d9dfa55cc50191cf6c6236846 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:44:54 +0700 Subject: [PATCH 36/72] Validate typed mapping views --- docs/type-safety.md | 2 +- src/aiython/type_constraints.py | 27 +++++++++++++++++++++++++-- tests/test_python_compatibility.py | 9 +++++++++ tests/test_type_constraints_edges.py | 20 ++++++++++++++++++++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 4e45de2..85c6655 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -32,7 +32,7 @@ Expected types reach direct AI expressions in annotated assignments, later assig ## Type forms -Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, `ChainMap`, typed regular expression objects, standard text/binary `IO` streams, concrete Sequence/Mapping/Collection and mutable or set variants, `Hashable`, `Sized`, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. +Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, `ChainMap`, typed regular expression objects, standard text/binary `IO` streams, concrete Sequence/Mapping/Collection and mutable or set variants, mapping views, `Hashable`, `Sized`, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. Primitive checks are strict: no string-to-number conversion, and bool does not pass an int contract. `Any` is an explicit escape from value checking. Bare containers have unconstrained elements. `Final` bindings reject reassignment; `ClassVar` direct writes must target the class. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 3e7f08e..f5691dd 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -43,11 +43,19 @@ SAFE_COLLECTION_TYPES = (list, tuple, set, frozenset, dict, str, bytes, bytearray, memoryview, range, deque, defaultdict, OrderedDict, Counter) +SAFE_VIEW_TYPES = { + 'keys_view': (type({}.keys()), type(OrderedDict().keys())), + 'values_view': (type({}.values()), type(OrderedDict().values())), + 'items_view': (type({}.items()), type(OrderedDict().items())), +} +SAFE_VIEW_TYPES['mapping_view'] = tuple(kind for group in SAFE_VIEW_TYPES.values() for kind in group) CONTAINER_KINDS = {list:'list', set:'set', frozenset:'frozenset', dict:'dict', deque:'deque', defaultdict:'defaultdict', OrderedDict:'ordered_dict', Counter:'counter', ChainMap:'chainmap', abc.Sequence:'sequence', abc.Mapping:'mapping', abc.MutableMapping:'mutable_mapping', abc.MutableSequence:'mutable_sequence', - abc.Set:'abstract_set', abc.MutableSet:'mutable_set', abc.Collection:'collection'} + abc.Set:'abstract_set', abc.MutableSet:'mutable_set', abc.Collection:'collection', + abc.MappingView:'mapping_view', abc.KeysView:'keys_view', + abc.ValuesView:'values_view', abc.ItemsView:'items_view'} class TypeViolation(CapabilityError, TypeError): @@ -114,6 +122,11 @@ def schema(self, seen=None): result = {'type': 'array', 'items': self.args[0].schema(seen)} elif kind == 'collection': result = {'x-python-collection-items': self.args[0].schema(seen)} + elif kind in SAFE_VIEW_TYPES: + item = (self.args[0].schema(seen) if kind != 'items_view' else + {'type':'array', 'prefixItems':[arg.schema(seen) for arg in self.args], + 'minItems':2, 'maxItems':2}) + result = {'type':'array', 'items':item} elif kind == 'tuple': result = {'type': 'array', 'prefixItems': [a.schema(seen) for a in self.args], 'minItems': len(self.args), 'maxItems': len(self.args)} @@ -247,6 +260,15 @@ def child(contract, item, suffix): child(self.args[0], key, f'.maps[{map_index}].keys[{index}]') child(self.args[1], item, f'.maps[{map_index}][{key!r}]' if type(key) in (str,int) else f'.maps[{map_index}].values[{index}]') + elif kind in SAFE_VIEW_TYPES: + if type(value) not in SAFE_VIEW_TYPES[kind]: + fail('only concrete mapping views can be checked deeply') + for index, item in enumerate(value): + if kind == 'items_view': + child(self.args[0], item[0], f'[{index}][0]') + child(self.args[1], item[1], f'[{index}][1]') + else: + child(self.args[0], item, f'[{index}]') elif kind == 'typeddict': if type(value) is not dict: fail() missing = self.required - value.keys() @@ -381,7 +403,8 @@ def generic(self,base,args,label,names): raise UnsupportedType('Qualifier requires one type') return Contract('qualifier',label,args,qualifier=base._name) if origin in CONTAINER_KINDS: - expected = 2 if origin in (dict,abc.Mapping,abc.MutableMapping,defaultdict,OrderedDict,ChainMap) else 1 + expected = (2 if origin in (dict,abc.Mapping,abc.MutableMapping,defaultdict, + OrderedDict,ChainMap,abc.ItemsView) else 1) if not args: args = (Contract('any', 'Any'),) * expected if len(args) != expected: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 94b826f..2510a65 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1139,6 +1139,15 @@ def describe(key: Hashable, values: Sized) -> tuple[int, int]: from typing import ChainMap as TypedChainMap layers: TypedChainMap[str, int] = ChainMap({'x': 1}, {'y': 2}) print(layers['x'], layers['y'], len(layers.maps)) +''', + 'mapping view annotations': '''from collections import OrderedDict +from typing import ItemsView, KeysView, MappingView, ValuesView +mapping = OrderedDict(x=1, y=2) +keys: KeysView[str] = mapping.keys() +values: ValuesView[int] = mapping.values() +items: ItemsView[str, int] = mapping.items() +view: MappingView[tuple[str, int]] = items +print(list(keys), list(values), list(items), list(view)) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index bfbe29f..5977a25 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -18,6 +18,26 @@ class ContractEdgeTests(unittest.TestCase): + def test_mapping_views_check_live_members(self): + namespace = {'typing': typing} + cases = ( + ('typing.KeysView[str]', {'x': 1}.keys(), {1: 'x'}.keys()), + ('typing.ValuesView[int]', OrderedDict(x=1).values(), + OrderedDict(x='wrong').values()), + ('typing.ItemsView[str, int]', {'x': 1}.items(), + {'x': 'wrong'}.items()), + ('typing.MappingView[tuple[str, int]]', OrderedDict(x=1).items(), + OrderedDict(x='wrong').items()), + ) + for annotation, valid, invalid in cases: + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, namespace) + self.assertEqual(contract.schema()['type'], 'array') + contract.validate(valid) + with self.assertRaises(tc.TypeViolation): + contract.validate(invalid) + tc.compile_contract('typing.MappingView', namespace).validate({'x': 1}.keys()) + def test_chainmap_checks_every_underlying_mapping(self): namespace = {'ChainMap': ChainMap, 'typing': typing} for annotation in ('ChainMap[str, int]', 'typing.ChainMap[str, int]'): From 05dc829f72b4f0d7d87b3652b886681906c1e8e2 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:47:08 +0700 Subject: [PATCH 37/72] Validate Container and Reversible annotations --- docs/type-safety.md | 2 +- src/aiython/type_constraints.py | 14 ++++++++++---- tests/test_python_compatibility.py | 7 +++++++ tests/test_type_constraints_edges.py | 7 +++++++ 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 85c6655..6b5d7f2 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -32,7 +32,7 @@ Expected types reach direct AI expressions in annotated assignments, later assig ## Type forms -Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, `ChainMap`, typed regular expression objects, standard text/binary `IO` streams, concrete Sequence/Mapping/Collection and mutable or set variants, mapping views, `Hashable`, `Sized`, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. +Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, `ChainMap`, typed regular expression objects, standard text/binary `IO` streams, concrete Sequence/Mapping/Collection/Container/Reversible and mutable or set variants, mapping views, `Hashable`, `Sized`, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. Primitive checks are strict: no string-to-number conversion, and bool does not pass an int contract. `Any` is an explicit escape from value checking. Bare containers have unconstrained elements. `Final` bindings reject reassignment; `ClassVar` direct writes must target the class. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index f5691dd..6519042 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -43,6 +43,8 @@ SAFE_COLLECTION_TYPES = (list, tuple, set, frozenset, dict, str, bytes, bytearray, memoryview, range, deque, defaultdict, OrderedDict, Counter) +SAFE_REVERSIBLE_TYPES = (list, tuple, dict, str, bytes, bytearray, + memoryview, range, deque, defaultdict, OrderedDict, Counter) SAFE_VIEW_TYPES = { 'keys_view': (type({}.keys()), type(OrderedDict().keys())), 'values_view': (type({}.values()), type(OrderedDict().values())), @@ -54,6 +56,7 @@ Counter:'counter', ChainMap:'chainmap', abc.Sequence:'sequence', abc.Mapping:'mapping', abc.MutableMapping:'mutable_mapping', abc.MutableSequence:'mutable_sequence', abc.Set:'abstract_set', abc.MutableSet:'mutable_set', abc.Collection:'collection', + abc.Container:'container', abc.Reversible:'reversible', abc.MappingView:'mapping_view', abc.KeysView:'keys_view', abc.ValuesView:'values_view', abc.ItemsView:'items_view'} @@ -118,9 +121,9 @@ def schema(self, seen=None): elif kind in ('str', 'int', 'float', 'bool'): result = {'type': {'str':'string', 'int':'integer', 'float':'number', 'bool':'boolean'}[kind]} elif kind in ('list', 'set', 'frozenset', 'sequence', 'deque', - 'mutable_sequence', 'abstract_set', 'mutable_set'): + 'mutable_sequence', 'abstract_set', 'mutable_set', 'reversible'): result = {'type': 'array', 'items': self.args[0].schema(seen)} - elif kind == 'collection': + elif kind in ('collection', 'container'): result = {'x-python-collection-items': self.args[0].schema(seen)} elif kind in SAFE_VIEW_TYPES: item = (self.args[0].schema(seen) if kind != 'items_view' else @@ -210,15 +213,18 @@ def child(contract, item, suffix): elif kind in ('str','int','float','bool','bytes','complex'): if type(value) is not self.python_type: fail() elif kind in ('list','set','frozenset','sequence','deque','tuple_many','tuple', - 'mutable_sequence','abstract_set','mutable_set','collection'): + 'mutable_sequence','abstract_set','mutable_set','collection', + 'container','reversible'): expected = CONCRETE_SEQUENCE_TYPES.get(kind) if expected is not None and type(value) is not expected: fail() if kind == 'sequence' and type(value) not in (list,tuple,str,bytes,bytearray,memoryview,range): fail('only non-consuming concrete sequences can be checked') if kind == 'mutable_sequence' and type(value) not in (list,bytearray): fail() if kind == 'abstract_set' and type(value) not in (set,frozenset): fail() - if kind == 'collection' and type(value) not in SAFE_COLLECTION_TYPES: + if kind in ('collection','container') and type(value) not in SAFE_COLLECTION_TYPES: fail('only non-consuming concrete collections can be checked') + if kind == 'reversible' and type(value) not in SAFE_REVERSIBLE_TYPES: + fail('only non-consuming concrete reversible collections can be checked') if kind == 'tuple' and len(value) != len(self.args): fail('wrong tuple length') if kind != 'tuple' and self.args[0].kind in PRIMITIVE_KINDS: member = self.args[0] diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 2510a65..c40fe3e 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1148,6 +1148,13 @@ def describe(key: Hashable, values: Sized) -> tuple[int, int]: items: ItemsView[str, int] = mapping.items() view: MappingView[tuple[str, int]] = items print(list(keys), list(values), list(items), list(view)) +''', + 'container and reversible annotations': '''from collections import OrderedDict, deque +from typing import Container, Reversible +keys: Container[str] = {'x': 1, 'y': 2} +ordered: Reversible[str] = OrderedDict(x=1, y=2) +numbers: Reversible[int] = deque([1, 2, 3]) +print('x' in keys, list(reversed(ordered)), list(reversed(numbers))) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 5977a25..60d319c 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -69,6 +69,8 @@ def test_abstract_collections_check_known_concrete_values(self): ('typing.AbstractSet[str]', frozenset({'x'}), frozenset({1})), ('typing.MutableSet[int]', {1}, frozenset({1})), ('typing.Collection[int]', {1: 'value'}, ['wrong']), + ('typing.Container[int]', {1: 'value'}, {'wrong': 1}), + ('typing.Reversible[str]', OrderedDict(x=1), OrderedDict({1: 'x'})), ('typing.Sequence[int]', memoryview(b'abc'), ['wrong']), ) for annotation, valid, invalid in cases: @@ -78,9 +80,14 @@ def test_abstract_collections_check_known_concrete_values(self): with self.assertRaises(tc.TypeViolation): contract.validate(invalid) tc.compile_contract('typing.MutableSequence[int]', namespace).validate(bytearray(b'a')) + tc.compile_contract('typing.Reversible[int]', namespace).validate(deque([1, 2])) + with self.assertRaises(tc.TypeViolation): + tc.compile_contract('typing.Reversible[int]', namespace).validate({1, 2}) for annotation, value in (('typing.List', [1, 'x']), ('typing.Dict', {'x': 1}), ('typing.Collection', {1: 2}), + ('typing.Container', {1: 2}), + ('typing.Reversible', range(2)), ('typing.Sequence', memoryview(b'a')), ('typing.Tuple', (1, 'x'))): with self.subTest(annotation=annotation): From 912091e543ddb52d059e15862854fe24a8a488f7 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:49:18 +0700 Subject: [PATCH 38/72] Accept bare lazy and context manager annotations --- docs/type-safety.md | 2 ++ src/aiython/type_constraints.py | 9 ++++++ tests/test_python_compatibility.py | 10 +++++++ tests/test_type_constraints_edges.py | 41 ++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+) diff --git a/docs/type-safety.md b/docs/type-safety.md index 6b5d7f2..258782d 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -34,6 +34,8 @@ Expected types reach direct AI expressions in annotated assignments, later assig Contracts support primitives, `None`, `Any`, unions/Optional, Literal values, Annotated descriptions, nested list/dict/set/frozenset/tuple, `deque`, `defaultdict`, `OrderedDict`, `Counter`, `ChainMap`, typed regular expression objects, standard text/binary `IO` streams, concrete Sequence/Mapping/Collection/Container/Reversible and mutable or set variants, mapping views, `Hashable`, `Sized`, TypedDict with Required/NotRequired, dataclasses, nominal classes, Self, type parameters, TypeVar constraints, NewType's underlying runtime type, type[T], and recursive/generic type aliases on Python 3.12+. Generic class fields are checked after substituting supplied type arguments. `TypeGuard[T]` and `TypeIs[T]` returns are checked as booleans; their target type is for static narrowing. +Bare iterator, generator, awaitable, context manager, byte string, and `Type` annotations check the value's category without consuming it. Typed lazy values still require checks at yield, send, await, or context entry boundaries. + Primitive checks are strict: no string-to-number conversion, and bool does not pass an int contract. `Any` is an explicit escape from value checking. Bare containers have unconstrained elements. `Final` bindings reject reassignment; `ClassVar` direct writes must target the class. `TypeAlias` marks an alias declaration and does not constrain the alias object itself. Values annotated with that alias are checked against its target type. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 6519042..ac079f8 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -9,6 +9,7 @@ import builtins import collections.abc as abc from collections import ChainMap, Counter, OrderedDict, defaultdict, deque +import contextlib from dataclasses import dataclass, field import dataclasses import enum @@ -384,6 +385,12 @@ def generic(self,base,args,label,names): return args[0] if origin is abc.Callable: return Contract('callable',label) + if not args and origin in (abc.ByteString, abc.Iterable, abc.Iterator, + abc.Generator, abc.AsyncIterable, abc.AsyncIterator, + abc.AsyncGenerator, abc.Awaitable, abc.Coroutine, + contextlib.AbstractContextManager, + contextlib.AbstractAsyncContextManager): + return Contract('abc', label, python_type=origin) if origin in (abc.Hashable, abc.Sized): if args: raise UnsupportedType(f'{label}: this ABC does not take type arguments') @@ -443,6 +450,8 @@ def generic(self,base,args,label,names): padded = args + (Contract('null','None'),)*(3-len(args)) return Contract('async_generator' if async_kind else 'generator',label,padded) if origin is type: + if not args: + args = (Contract('any', 'Any'),) if len(args) != 1: raise UnsupportedType('type requires one parameter') return Contract('type',label,args) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index c40fe3e..0ad3893 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1155,6 +1155,16 @@ def describe(key: Hashable, values: Sized) -> tuple[int, int]: ordered: Reversible[str] = OrderedDict(x=1, y=2) numbers: Reversible[int] = deque([1, 2, 3]) print('x' in keys, list(reversed(ordered)), list(reversed(numbers))) +''', + 'bare abstract annotations': '''from contextlib import nullcontext +from typing import ByteString, ContextManager, Generator, Iterable, Type +items: Iterable = range(2) +generator: Generator = (i for i in items) +context: ContextManager = nullcontext(3) +binary: ByteString = bytearray(b'xy') +kind: Type = int +with context as value: + print(list(generator), value, list(binary), kind('4')) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 60d319c..3e3c51a 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -1,5 +1,7 @@ import ast +import asyncio from collections import ChainMap, Counter, OrderedDict, defaultdict, deque +import contextlib from dataclasses import dataclass import enum import io @@ -18,6 +20,45 @@ class ContractEdgeTests(unittest.TestCase): + def test_bare_abstract_annotations_do_not_consume_values(self): + namespace = {'typing': typing} + iterator = (item for item in range(2)) + for annotation in ('typing.Iterable', 'typing.Iterator', 'typing.Generator'): + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, namespace) + contract.validate(iterator) + with self.assertRaises(tc.TypeViolation): + contract.validate(1) + self.assertEqual(next(iterator), 0) + iterator.close() + + async def source(): + yield 1 + + async def value(): + return 1 + + async_iterator = source() + for annotation in ('typing.AsyncIterable', 'typing.AsyncIterator', + 'typing.AsyncGenerator'): + tc.compile_contract(annotation, namespace).validate(async_iterator) + coroutine = value() + for annotation in ('typing.Awaitable', 'typing.Coroutine'): + tc.compile_contract(annotation, namespace).validate(coroutine) + coroutine.close() + asyncio.run(async_iterator.aclose()) + + tc.compile_contract('typing.ContextManager', namespace).validate( + contextlib.nullcontext()) + tc.compile_contract('typing.AsyncContextManager', namespace).validate( + contextlib.AsyncExitStack()) + tc.compile_contract('typing.ByteString', namespace).validate(bytearray(b'abc')) + with self.assertRaises(tc.TypeViolation): + tc.compile_contract('typing.ByteString', namespace).validate(memoryview(b'abc')) + tc.compile_contract('typing.Type', namespace).validate(int) + with self.assertRaises(tc.TypeViolation): + tc.compile_contract('typing.Type', namespace).validate(1) + def test_mapping_views_check_live_members(self): namespace = {'typing': typing} cases = ( From 721455af447d557bdcaad7b1f3da864f99a89e68 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 16:56:09 +0700 Subject: [PATCH 39/72] Validate union and TypeVar class annotations --- src/aiython/type_constraints.py | 42 +++++++++++++++++++++++++++- tests/test_python_compatibility.py | 9 ++++++ tests/test_type_constraints_edges.py | 40 ++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index ac079f8..22232e0 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -91,6 +91,46 @@ class Contract: description: str = '' qualifier: str | None = None + def accepts_class(self, candidate, bindings, seen=None): + """Check a class supplied to type[T] without instantiating it.""" + seen = set() if seen is None else seen + if id(self) in seen: + return False + seen = seen | {id(self)} + if self.kind == 'any': + return True + if self.kind == 'null': + return candidate is type(None) + if self.kind in ('alias', 'annotated', 'qualifier'): + return self.args[0].accepts_class(candidate, bindings, seen) + if self.kind == 'union': + for option in self.args: + branch = dict(bindings) + if option.accepts_class(candidate, branch, seen): + bindings.update(branch) + return True + return False + if self.kind == 'typevar': + previous = bindings.get(self.python_type) + if previous is not None: + return candidate is previous + if self.args: + for option in self.args: + branch = dict(bindings) + if option.accepts_class(candidate, branch, seen): + bindings.update(branch) + break + else: + return False + bindings[self.python_type] = candidate + return True + if not isinstance(self.python_type, type): + return False + try: + return issubclass(candidate, self.python_type) + except TypeError: + return False + @property def marker(self): if self.qualifier: return self.qualifier @@ -297,7 +337,7 @@ def child(contract, item, suffix): elif kind == 'type': if not isinstance(value,type): fail() target = self.args[0] - if target.kind != 'any' and (target.python_type is None or not issubclass(value,target.python_type)): + if not target.accepts_class(value, bindings): fail() elif kind in ('generator','async_generator'): raise UnsupportedType('Lazy iterable contracts must be checked at yield/send boundaries, not by consuming the object') diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 0ad3893..242cbaf 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1165,6 +1165,15 @@ def describe(key: Hashable, values: Sized) -> tuple[int, int]: kind: Type = int with context as value: print(list(generator), value, list(binary), kind('4')) +''', + 'union class annotations': '''from typing import Annotated, TypeVar +T = TypeVar('T', int, str) +def same(kind: type[T], value: T) -> T: + return value +one: type[int | str] = int +none_type: type[None] = type(None) +annotated: type[Annotated[int, 'number']] = int +print(one('2'), none_type(), annotated('3'), same(str, 'four')) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 3e3c51a..5eaff07 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -471,6 +471,46 @@ def test_validation_rejects_invalid_scalar_container_and_typevar_values(self): with self.assertRaises(tc.TypeViolation): contract.validate(1.5) + def test_type_contracts_accept_unions_aliases_and_typevars(self): + namespace = {'typing': typing} + for annotation, valid, invalid in ( + ('type[int | str]', int, float), + ('typing.Type[typing.Union[int, str]]', str, float), + ('type[None]', type(None), int), + ('type[typing.Annotated[int, "metadata"]]', int, str), + ): + with self.subTest(annotation=annotation): + contract = tc.compile_contract(annotation, namespace) + contract.validate(valid) + with self.assertRaises(tc.TypeViolation): + contract.validate(invalid) + + alias = tc.TypeAliasType('ClassAlias', int | str) + tc.compile_contract('type[ClassAlias]', {'ClassAlias': alias}).validate(str) + recursive = tc.TypeAliasType('RecursiveClass', 'int | RecursiveClass') + recursive_contract = tc.compile_contract('type[RecursiveClass]', + {'RecursiveClass': recursive}) + recursive_contract.validate(int) + with self.assertRaises(tc.TypeViolation): + recursive_contract.validate(str) + + variable = TypeVar('ClassVariable', int, str) + namespace['ClassVariable'] = variable + class_contract = tc.compile_contract('type[ClassVariable]', namespace) + value_contract = tc.compile_contract('ClassVariable', namespace) + bindings = {} + class_contract.validate(int, bindings=bindings) + value_contract.validate(3, bindings=bindings) + with self.assertRaises(tc.TypeViolation): + value_contract.validate('wrong', bindings=bindings) + with self.assertRaises(tc.TypeViolation): + class_contract.validate(str, bindings=bindings) + with self.assertRaises(tc.TypeViolation): + class_contract.validate(float, bindings={}) + reverse_bindings = {} + value_contract.validate('first', bindings=reverse_bindings) + class_contract.validate(str, bindings=reverse_bindings) + def test_class_custom_validator_and_missing_field(self): class Choice: value: int From 3d1e0afa6dc708b2fa58ad2c1255de92bc787b9f Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 17:00:12 +0700 Subject: [PATCH 40/72] Validate class objects for generic annotations --- src/aiython/type_constraints.py | 30 ++++++++++++++++------------ tests/test_python_compatibility.py | 7 +++++-- tests/test_type_constraints_edges.py | 11 +++++++++- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 22232e0..6dbcd10 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -394,7 +394,7 @@ def node(self,node,names): if base in (typing.Callable, abc.Callable): if len(nodes) != 2: raise UnsupportedType('Callable requires parameters and a return type') - return Contract('callable', ast.unparse(node)) + return Contract('callable', ast.unparse(node), python_type=abc.Callable) if (base is typing.Unpack and len(nodes) == 1 and isinstance(nodes[0], ast.Name) and isinstance(names.get(nodes[0].id), typing.TypeVarTuple)): return Contract('unpack_any', ast.unparse(node), python_type=names[nodes[0].id]) @@ -424,7 +424,7 @@ def generic(self,base,args,label,names): if origin is typing.Unpack and len(args) == 1 and args[0].kind in ('unpack_any', 'unpack_fixed'): return args[0] if origin is abc.Callable: - return Contract('callable',label) + return Contract('callable',label,python_type=abc.Callable) if not args and origin in (abc.ByteString, abc.Iterable, abc.Iterator, abc.Generator, abc.AsyncIterable, abc.AsyncIterator, abc.AsyncGenerator, abc.Awaitable, abc.Coroutine, @@ -443,7 +443,7 @@ def generic(self,base,args,label,names): if len(args) > 1: raise UnsupportedType('Regex type requires one input type') return Contract('pattern' if origin is re.Pattern else 'match', label, - args or (Contract('any', 'Any'),)) + args or (Contract('any', 'Any'),), python_type=origin) if origin in (typing.Union,types.UnionType): return Contract('union',label,args) if base is typing.Optional: @@ -464,9 +464,10 @@ def generic(self,base,args,label,names): raise UnsupportedType(f'{label}: wrong number of type parameters') if origin is Counter: args += (Contract('int','int',python_type=int),) - return Contract(CONTAINER_KINDS[origin],label,args) + return Contract(CONTAINER_KINDS[origin],label,args,python_type=origin) if origin is tuple: - if len(args) == 2 and args[1] is Ellipsis: return Contract('tuple_many',label,args[:1]) + if len(args) == 2 and args[1] is Ellipsis: + return Contract('tuple_many',label,args[:1],python_type=tuple) expanded = [] for arg in args: if isinstance(arg, Contract) and arg.kind == 'unpack_fixed': @@ -477,24 +478,26 @@ def generic(self,base,args,label,names): unpacked = [index for index, arg in enumerate(args) if isinstance(arg, Contract) and arg.kind == 'unpack_any'] if len(unpacked) == 1: - if len(args) == 1: return Contract('tuple_many',label,(Contract('any','Any'),)) - return Contract('tuple_unpacked',label,args) + if len(args) == 1: + return Contract('tuple_many',label,(Contract('any','Any'),),python_type=tuple) + return Contract('tuple_unpacked',label,args,python_type=tuple) if unpacked: raise UnsupportedType('Only one variadic tuple parameter can be checked') - return Contract('tuple',label,args) + return Contract('tuple',label,args,python_type=tuple) if origin in (abc.Generator,abc.Iterator,abc.Iterable,abc.AsyncGenerator,abc.AsyncIterator,abc.AsyncIterable): async_kind = origin in (abc.AsyncGenerator,abc.AsyncIterator,abc.AsyncIterable) expected = 3 if origin is abc.Generator else 2 if origin is abc.AsyncGenerator else 1 if len(args) != expected: raise UnsupportedType('Wrong iterator type argument count') padded = args + (Contract('null','None'),)*(3-len(args)) - return Contract('async_generator' if async_kind else 'generator',label,padded) + return Contract('async_generator' if async_kind else 'generator',label,padded, + python_type=origin) if origin is type: if not args: args = (Contract('any', 'Any'),) if len(args) != 1: raise UnsupportedType('type requires one parameter') - return Contract('type',label,args) + return Contract('type',label,args,python_type=type) if isinstance(base,TYPE_ALIAS_TYPES): return self.alias(base,names,args,label) if isinstance(base,type) and (getattr(base,'__type_params__',()) or getattr(base,'__parameters__',())): parameters = getattr(base,'__type_params__',()) or base.__parameters__ @@ -641,7 +644,8 @@ def value(self,target,names): if target is None or target is type(None): return Contract('null','None') if target is typing.Any: return Contract('any','Any') if target is typing.Tuple: - return Contract('tuple_many', 'typing.Tuple', (Contract('any', 'Any'),)) + return Contract('tuple_many', 'typing.Tuple', (Contract('any', 'Any'),), + python_type=tuple) if target is typing.IO or target is typing.TextIO or target is typing.BinaryIO: stream_type = ('str' if target is typing.TextIO else 'bytes' if target is typing.BinaryIO else 'any') @@ -656,7 +660,7 @@ def value(self,target,names): if isinstance(bound, tuple) and all(isinstance(item, Contract) for item in bound) else Contract('unpack_any', target.__name__, python_type=target)) if target in (typing.Callable, abc.Callable): - return Contract('callable',str(target)) + return Contract('callable',str(target),python_type=abc.Callable) if target in (typing.Final,typing.ClassVar): return Contract('qualifier',str(target),(Contract('any','Any'),),qualifier=target._name) if target in (typing.Never,typing.NoReturn): return Contract('never',str(target)) @@ -688,7 +692,7 @@ def value(self,target,names): if target in (int,str,float,bool,bytes,complex): return Contract(target.__name__,target.__name__,python_type=target) if target in (list,set,frozenset,dict,tuple): any_type = Contract('any','Any') - if target is tuple: return Contract('tuple_many','tuple',(any_type,)) + if target is tuple: return Contract('tuple_many','tuple',(any_type,),python_type=tuple) return self.generic(target,(any_type,any_type) if target is dict else (any_type,),target.__name__,names) if not isinstance(target,type): raise UnsupportedType('Annotation is not a supported Python type') diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 242cbaf..66e2b42 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1166,14 +1166,17 @@ def describe(key: Hashable, values: Sized) -> tuple[int, int]: with context as value: print(list(generator), value, list(binary), kind('4')) ''', - 'union class annotations': '''from typing import Annotated, TypeVar + 'union class annotations': '''from typing import Annotated, Sequence, TypeVar T = TypeVar('T', int, str) def same(kind: type[T], value: T) -> T: return value one: type[int | str] = int none_type: type[None] = type(None) annotated: type[Annotated[int, 'number']] = int -print(one('2'), none_type(), annotated('3'), same(str, 'four')) +concrete: type[list[int]] = list +abstract: type[Sequence[int]] = list +print(one('2'), none_type(), annotated('3'), same(str, 'four'), + concrete([1, 2]), abstract([3])) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 5eaff07..ee9462d 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -478,9 +478,18 @@ def test_type_contracts_accept_unions_aliases_and_typevars(self): ('typing.Type[typing.Union[int, str]]', str, float), ('type[None]', type(None), int), ('type[typing.Annotated[int, "metadata"]]', int, str), + ('type[list[int]]', list, dict), + ('type[dict[str, int]]', dict, list), + ('type[tuple[int, ...]]', tuple, list), + ('type[typing.Sequence[int]]', list, dict), + ('type[typing.Mapping[str, int]]', OrderedDict, list), + ('type[typing.Iterator[int]]', type(iter([])), list), + ('type[typing.Callable[[int], str]]', type(lambda: None), list), + ('type[re.Pattern[str]]', re.Pattern, str), + ('type[type[int]]', type, int), ): with self.subTest(annotation=annotation): - contract = tc.compile_contract(annotation, namespace) + contract = tc.compile_contract(annotation, namespace | {'re': re}) contract.validate(valid) with self.assertRaises(tc.TypeViolation): contract.validate(invalid) From 362fed76e5c7a9ea138d43ed44027ce7c528bd4b Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 17:05:46 +0700 Subject: [PATCH 41/72] Validate unpacked TypedDict keyword arguments --- docs/type-safety.md | 1 + src/aiython/type_constraints.py | 14 ++++++++++--- src/aiython/typed_runtime.py | 4 +++- tests/test_python_compatibility.py | 8 ++++++++ tests/test_type_constraints_edges.py | 30 ++++++++++++++++++++++++++++ tests/test_type_safety.py | 15 ++++++++++++++ 6 files changed, 68 insertions(+), 4 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 258782d..95d9f52 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -21,6 +21,7 @@ analysis: TicketAnalysis = analyze the current ticket - Annotated assignments and later assignments to the same binding are checked. Invalid simple assignments are rejected before replacing the value. - Function arguments, positional-only/keyword-only arguments, `*args`, `**kwargs`, explicit returns and implicit `None` returns are checked. Async functions use the same rules. +- `**kwargs: Unpack[TypedDict]` checks required and optional keyword fields using the declared `TypedDict` contract. - The return value is checked again after `finally`, so cleanup cannot silently invalidate a return that was already checked. - Typed class fields are checked on direct attribute assignment. Dataclass values are checked without conversion to dictionaries or reconstruction. Nested registered project instances are checked at execution boundaries. - Mutable containers are checked deeply. Mutations through aliases are detected at statement boundaries. Enclosing scopes and globals are included; closures preserve referenced local annotation names. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 6dbcd10..6592d3a 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -187,10 +187,10 @@ def schema(self, seen=None): if any(type(v) not in (str,int,bool,float,type(None)) for v in values): raise UnsupportedType('Enum schema requires scalar member values') result = {'enum': values, 'x-python-enum-members': list(self.python_type.__members__)} - elif kind in ('typeddict', 'class') and self.fields: + elif kind == 'typeddict' or (kind == 'class' and self.fields): result = {'type':'object', 'properties':{k:v.schema(seen) for k,v in self.fields.items()}, 'required': sorted(self.required)} - elif kind in ('annotated', 'qualifier', 'alias'): + elif kind in ('annotated', 'qualifier', 'alias', 'unpack_typeddict'): result = self.args[0].schema(seen) elif kind == 'typevar': result = {'anyOf':[a.schema(seen) for a in self.args]} if self.args else {} @@ -220,7 +220,7 @@ def fail(detail=None): raise TypeViolation(f'{path}: expected {self.name}, got {type(value).__name__}' + (f' ({detail})' if detail else '')) def child(contract, item, suffix): contract.validate(item, path + suffix, bindings=bindings, seen=seen) - if kind in ('alias', 'annotated', 'qualifier'): + if kind in ('alias', 'annotated', 'qualifier', 'unpack_typeddict'): child(self.args[0], value, '') elif kind == 'never': fail('this boundary must not return') elif kind == 'union': @@ -423,6 +423,14 @@ def generic(self,base,args,label,names): origin = typing.get_origin(base) or base if origin is typing.Unpack and len(args) == 1 and args[0].kind in ('unpack_any', 'unpack_fixed'): return args[0] + if origin is typing.Unpack and len(args) == 1: + unpacked = args[0] + visited = set() + while unpacked.kind in ('alias', 'annotated') and id(unpacked) not in visited: + visited.add(id(unpacked)) + unpacked = unpacked.args[0] + if unpacked.kind == 'typeddict': + return Contract('unpack_typeddict', label, (unpacked,)) if origin is abc.Callable: return Contract('callable',label,python_type=abc.Callable) if not args and origin in (abc.ByteString, abc.Iterable, abc.Iterator, diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index b7d7f56..ebeb8d5 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -109,7 +109,9 @@ def _initialize(self, frame, declarations, parameters, returns): contract = (Contract('any', 'Any') if unconstrained_variadic(source, mode, namespace) else self.contract(source,namespace)) if mode == 'args': contract = Contract('tuple_many',source,(contract,)) - elif mode == 'kwargs': contract = Contract('dict',source,(compile_contract('str',namespace),contract)) + elif mode == 'kwargs': + contract = (contract.args[0] if contract.kind == 'unpack_typeddict' else + Contract('dict',source,(compile_contract('str',namespace),contract))) contract.validate(frame.f_locals[name],name,bindings=scope.bindings) scope.contracts[name] = contract if returns: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 66e2b42..5bc82ed 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1177,6 +1177,14 @@ def same(kind: type[T], value: T) -> T: abstract: type[Sequence[int]] = list print(one('2'), none_type(), annotated('3'), same(str, 'four'), concrete([1, 2]), abstract([3])) +''', + 'unpacked typed dict keyword arguments': '''from typing import NotRequired, TypedDict, Unpack +class Options(TypedDict): + count: int + label: NotRequired[str] +def describe(**kwargs: Unpack[Options]) -> tuple[int, str | None]: + return kwargs['count'], kwargs.get('label') +print(describe(count=2), describe(count=3, label='ready')) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index ee9462d..2bbf2b5 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -20,6 +20,36 @@ class ContractEdgeTests(unittest.TestCase): + def test_unpack_typed_dict_checks_keyword_mapping(self): + class Options(TypedDict): + count: int + label: typing.NotRequired[str] + + namespace = {'Unpack': Unpack, 'Options': Options} + contract = tc.compile_contract('Unpack[Options]', namespace) + self.assertEqual(contract.kind, 'unpack_typeddict') + self.assertEqual(contract.schema()['required'], ['count']) + class EmptyOptions(TypedDict): + pass + empty_schema = tc.compile_contract('Unpack[EmptyOptions]', + {'Unpack': Unpack, 'EmptyOptions': EmptyOptions}).schema() + self.assertEqual(empty_schema['type'], 'object') + self.assertEqual(empty_schema['properties'], {}) + alias = tc.TypeAliasType('OptionsAlias', Options) + tc.compile_contract('Unpack[OptionsAlias]', + {'Unpack': Unpack, 'OptionsAlias': alias}).validate({'count': 2}) + for annotation in ('Unpack[Options]', Unpack[Options]): + with self.subTest(annotation=annotation): + checked = tc.compile_contract(annotation, namespace) + checked.validate({'count': 2}) + checked.validate({'count': 2, 'label': 'ready'}) + with self.assertRaises(tc.TypeViolation): + checked.validate({'count': 'wrong'}) + with self.assertRaises(tc.TypeViolation): + checked.validate({'label': 'missing count'}) + with self.assertRaises(tc.TypeViolation): + checked.validate({'count': 2, 'label': 3}) + def test_bare_abstract_annotations_do_not_consume_values(self): namespace = {'typing': typing} iterator = (item for item in range(2)) diff --git a/tests/test_type_safety.py b/tests/test_type_safety.py index a6bf2d1..aab0247 100644 --- a/tests/test_type_safety.py +++ b/tests/test_type_safety.py @@ -72,6 +72,21 @@ def broken(x: T) -> T: broken(1) ''') + def test_unpack_typed_dict_keyword_arguments(self): + prelude = '''from typing import NotRequired, TypedDict, Unpack +class Options(TypedDict): + count: int + label: NotRequired[str] +def describe(**kwargs: Unpack[Options]) -> tuple[int, str | None]: + return kwargs['count'], kwargs.get('label') +''' + result = self.run_source(prelude + "answer = describe(count=2, label='ready')\n") + self.assertEqual(result['answer'], (2, 'ready')) + for call in ('describe(count="bad")', 'describe(label="missing")', + 'describe(count=2, label=3)'): + with self.subTest(call=call), self.assertRaises(TypeViolation): + self.run_source(prelude + call + '\n') + @unittest.skipIf(sys.version_info < (3, 12), "The type statement requires Python 3.12") def test_forward_local_alias_is_captured(self): self.assertEqual(self.run_source('''def factory(): From 38ad82fca6d2cffbd00e96361edb1c9bf1fa9307 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 17:07:57 +0700 Subject: [PATCH 42/72] Resolve Self in inherited class fields --- src/aiython/type_constraints.py | 5 +++-- src/aiython/typed_runtime.py | 4 +++- tests/test_python_compatibility.py | 9 +++++++++ tests/test_type_constraints_edges.py | 19 +++++++++++++++++++ tests/test_type_safety.py | 15 +++++++++++++++ 5 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 6592d3a..3f424fe 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -30,6 +30,7 @@ from .capabilities import CapabilityError VALIDATORS = {} +SELF_OWNER = object() PRIMITIVE_KINDS = frozenset(('str', 'int', 'float', 'bool', 'bytes', 'complex')) TYPE_ALIAS_TYPES = tuple({TypeAliasType, getattr(typing, "TypeAliasType", TypeAliasType)}) READ_ONLY_TYPES = tuple({ReadOnly, getattr(typing, "ReadOnly", ReadOnly)}) @@ -587,7 +588,7 @@ def class_contract(self,target,names,label=None): result = Contract('typeddict' if record else 'class',label or target.__qualname__,python_type=target) self.cache[key] = result if target in VALIDATORS: return result - scope = self.module_names(target,names) | {target.__name__:target} + scope = self.module_names(target,names) | {target.__name__:target, SELF_OWNER:target} scope.update({p.__name__:p for p in (getattr(target,'__type_params__',()) or getattr(target,'__parameters__',()))}) # isinstance can call a user's __getattribute__('__class__') here. scope.update({k:v for k,v in names.items() @@ -673,7 +674,7 @@ def value(self,target,names): return Contract('qualifier',str(target),(Contract('any','Any'),),qualifier=target._name) if target in (typing.Never,typing.NoReturn): return Contract('never',str(target)) if target is typing.Self: - owner = names.get('self',names.get('cls')) + owner = names.get(SELF_OWNER, names.get('self',names.get('cls'))) if owner is None: raise UnsupportedType('Self requires an instance or class scope') return self.value(owner if isinstance(owner,type) else type(owner),names) diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index ebeb8d5..4ffb6e6 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -10,7 +10,8 @@ import weakref from .frontend import RUNTIME_NAME -from .type_constraints import Contract, ContractCache, TypeViolation, compile_contract, annotations_of, Compiler +from .type_constraints import (Contract, ContractCache, TypeViolation, compile_contract, + annotations_of, Compiler, SELF_OWNER) SCOPE = '__aiython_type_scope__' _FRAME_SCOPES = ContextVar('aiython_frame_scopes', default=()) @@ -348,6 +349,7 @@ def assign_attribute(self,owner,name,value,annotation=None): fields.update(annotations_of(base)) source = fields.get(name) namespace = Compiler.module_names(target,self.namespace(frame)) + namespace[SELF_OWNER] = target namespace.update({p.__name__:p for p in (getattr(target,'__type_params__',()) or getattr(target,'__parameters__',()))}) contract = compile_contract(source,namespace) if source else None if contract: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 5bc82ed..f733742 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1185,6 +1185,15 @@ class Options(TypedDict): def describe(**kwargs: Unpack[Options]) -> tuple[int, str | None]: return kwargs['count'], kwargs.get('label') print(describe(count=2), describe(count=3, label='ready')) +''', + 'self annotated class fields': '''from typing import Self +class Node: + next: Self | None + def __init__(self): self.next = None +class Child(Node): pass +head = Child() +head.next = Child() +print(type(head.next).__name__, head.next.next) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 2bbf2b5..2ca9e35 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -20,6 +20,25 @@ class ContractEdgeTests(unittest.TestCase): + def test_self_field_uses_class_being_validated(self): + class Node: + next: Self | None + + class Child(Node): + pass + + parent = Node() + parent.next = Node() + child = Child() + child.next = Child() + tc.compile_contract(Node, {'Node': Node, 'Self': Self}).validate(parent) + child_contract = tc.compile_contract(Child, {'Node': Node, 'Child': Child, + 'Self': Self}) + child_contract.validate(child) + child.next = Node() + with self.assertRaises(tc.TypeViolation): + child_contract.validate(child) + def test_unpack_typed_dict_checks_keyword_mapping(self): class Options(TypedDict): count: int diff --git a/tests/test_type_safety.py b/tests/test_type_safety.py index aab0247..bf10214 100644 --- a/tests/test_type_safety.py +++ b/tests/test_type_safety.py @@ -87,6 +87,21 @@ def describe(**kwargs: Unpack[Options]) -> tuple[int, str | None]: with self.subTest(call=call), self.assertRaises(TypeViolation): self.run_source(prelude + call + '\n') + def test_self_annotated_field_uses_instance_class(self): + prelude = '''from typing import Self +class Node: + next: Self | None + def __init__(self): + self.next = None +class Child(Node): + pass +child = Child() +''' + result = self.run_source(prelude + 'child.next = Child()\nanswer = isinstance(child.next, Child)\n') + self.assertTrue(result['answer']) + with self.assertRaises(TypeViolation): + self.run_source(prelude + 'child.next = Node()\n') + @unittest.skipIf(sys.version_info < (3, 12), "The type statement requires Python 3.12") def test_forward_local_alias_is_captured(self): self.assertEqual(self.run_source('''def factory(): From d7668dc6a912351f58bab13c83cab5262324350e Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 17:13:36 +0700 Subject: [PATCH 43/72] Resolve Self for renamed method receivers --- docs/type-safety.md | 2 ++ src/aiython/typed_runtime.py | 38 +++++++++++++++++++++++++++++- tests/test_python_compatibility.py | 11 +++++++++ tests/test_type_safety.py | 30 +++++++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 95d9f52..5fba2fa 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -45,6 +45,8 @@ Primitive checks are strict: no string-to-number conversion, and bool does not p `Mapping[K, V]` and `MutableMapping[K, V]` check concrete `dict`, `defaultdict`, `OrderedDict`, and `Counter` values deeply. +`Self` follows the class of the actual method receiver, including subclass calls and methods whose receiver is not named `self` or `cls`. + Annotations are interpreted rather than passed to `eval`. On Python 3.14+, Aiython uses string-format annotation introspection; on 3.11–3.13, it reads stored annotations without evaluating strings. Type aliases on 3.12–3.13 use Python's lazy alias value machinery, which can evaluate code supplied by the alias author. Custom annotation machinery and custom validators are trusted Python code, not sandboxed code. `aiython.type_constraints.register_validator(Class, validator)` supplies a custom runtime predicate. A class validator returns true for a valid value. It can also provide the structural check for a Protocol that Aiython cannot prove automatically. diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 4ffb6e6..15434a9 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -6,6 +6,7 @@ from dataclasses import dataclass, field import inspect import threading +import types import typing import weakref @@ -57,6 +58,7 @@ def unconstrained_variadic(source, mode, namespace): @dataclass class Scope: declarations: dict = field(default_factory=dict) + self_owner: type | None = None contracts: dict = field(default_factory=dict) bindings: dict = field(default_factory=dict) final_names: set = field(default_factory=set) @@ -71,6 +73,7 @@ class TypeRuntime: def __init__(self, manager=None): self.manager = manager self.classes = weakref.WeakSet() + self.method_owners = weakref.WeakKeyDictionary() self._classes_lock = threading.RLock() self._contract_cache = threading.local() @@ -91,19 +94,52 @@ def register_class(self, cls): if isinstance(cls, type): with self._classes_lock: self.classes.add(cls) + for member in type.__getattribute__(cls, '__dict__').values(): + if type(member) is types.FunctionType: + methods = (member,) + elif type(member) is classmethod: + methods = (member.__func__,) + elif type(member) is property: + methods = (member.fget, member.fset, member.fdel) + else: + continue + for method in methods: + visited = set() + while type(method) is types.FunctionType and id(method) not in visited: + visited.add(id(method)) + owners = self.method_owners.setdefault(method.__code__, weakref.WeakSet()) + owners.add(cls) + method = vars(method).get('__wrapped__') return cls + def method_self_owner(self, frame): + with self._classes_lock: + owners = tuple(self.method_owners.get(frame.f_code, ())) + if not owners or not frame.f_code.co_argcount: + return None + receiver = frame.f_locals.get(frame.f_code.co_varnames[0]) + receiver_type = type(receiver) + candidate = receiver if issubclass(receiver_type, type) else receiver_type + mro = type.__getattribute__(candidate, '__mro__') + if any(owner in mro for owner in owners): + return candidate + return owners[0] if len(owners) == 1 else None + @staticmethod def namespace(frame): namespace = dict(frame.f_globals) | dict(frame.f_locals) scope = frame_scope(frame) or frame.f_locals.get(SCOPE) if isinstance(scope,Scope): namespace.update({parameter.__name__: bound for parameter,bound in scope.bindings.items()}) + if scope.self_owner is not None: + namespace[SELF_OWNER] = scope.self_owner return namespace def _initialize(self, frame, declarations, parameters, returns): - scope = Scope(declarations=declarations) + scope = Scope(declarations=declarations, self_owner=self.method_self_owner(frame)) namespace = self.namespace(frame) + if scope.self_owner is not None: + namespace[SELF_OWNER] = scope.self_owner for name, (source, mode) in (parameters or {}).items(): # Unbound variadic type parameters have no concrete element types # to check at this call boundary. diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index f733742..6ff68c2 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1194,6 +1194,17 @@ class Child(Node): pass head = Child() head.next = Child() print(type(head.next).__name__, head.next.next) +''', + 'self with renamed method receivers': '''from typing import Self +class Base: + def clone(this) -> Self: return type(this)() + @classmethod + def create(klass) -> Self: return klass() + @property + def same(this) -> Self: return this +class Child(Base): pass +child = Child() +print(type(child.clone()).__name__, type(Child.create()).__name__, type(child.same).__name__) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_safety.py b/tests/test_type_safety.py index bf10214..bc83f5d 100644 --- a/tests/test_type_safety.py +++ b/tests/test_type_safety.py @@ -102,6 +102,36 @@ class Child(Node): with self.assertRaises(TypeViolation): self.run_source(prelude + 'child.next = Node()\n') + def test_self_in_methods_with_renamed_receiver(self): + source = '''from typing import Self +class Base: + def clone(this) -> Self: + return type(this)() + @classmethod + def create(klass) -> Self: + return klass() + @property + def same(this) -> Self: + return this + def keep(this): + local: Self = this + return local +class Child(Base): + pass +child = Child() +answer = (type(child.clone()), type(Child.create()), type(child.same), type(child.keep())) +''' + result = self.run_source(source) + self.assertEqual(result['answer'], (result['Child'],) * 4) + for method in ('''def clone(this) -> Self: + return Base()''', '''@classmethod + def create(klass) -> Self: + return Base()'''): + with self.subTest(method=method), self.assertRaises(TypeViolation): + self.run_source('from typing import Self\nclass Base:\n ' + method + + '\nclass Child(Base): pass\n' + + ('Child.create()\n' if 'classmethod' in method else 'Child().clone()\n')) + @unittest.skipIf(sys.version_info < (3, 12), "The type statement requires Python 3.12") def test_forward_local_alias_is_captured(self): self.assertEqual(self.run_source('''def factory(): From cb2a355c79f2bce8776d3b75968d1519dff49fbd Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 17:19:22 +0700 Subject: [PATCH 44/72] Resolve Self in dynamically assigned methods --- src/aiython/typed_runtime.py | 56 ++++++++++++++++++++---------- tests/test_python_compatibility.py | 9 +++++ tests/test_type_safety.py | 43 +++++++++++++++++++++++ 3 files changed, 90 insertions(+), 18 deletions(-) diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 15434a9..a2d5e27 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -95,32 +95,41 @@ def register_class(self, cls): with self._classes_lock: self.classes.add(cls) for member in type.__getattribute__(cls, '__dict__').values(): - if type(member) is types.FunctionType: - methods = (member,) - elif type(member) is classmethod: - methods = (member.__func__,) - elif type(member) is property: - methods = (member.fget, member.fset, member.fdel) - else: - continue - for method in methods: - visited = set() - while type(method) is types.FunctionType and id(method) not in visited: - visited.add(id(method)) - owners = self.method_owners.setdefault(method.__code__, weakref.WeakSet()) - owners.add(cls) - method = vars(method).get('__wrapped__') + self._track_method(cls, member) return cls - def method_self_owner(self, frame): + def _track_method(self, cls, member): + if type(member) is types.FunctionType: + methods = (member,) + elif type(member) is classmethod: + methods = (member.__func__,) + elif type(member) is property: + methods = (member.fget, member.fset, member.fdel) + else: + return + for method in methods: + visited = set() + while type(method) is types.FunctionType and id(method) not in visited: + visited.add(id(method)) + owners = self.method_owners.setdefault(method.__code__, weakref.WeakSet()) + owners.add(cls) + method = vars(method).get('__wrapped__') + + def method_self_owner(self, frame, *, discover=False): with self._classes_lock: owners = tuple(self.method_owners.get(frame.f_code, ())) - if not owners or not frame.f_code.co_argcount: + if (not owners and not discover) or not frame.f_code.co_argcount: return None receiver = frame.f_locals.get(frame.f_code.co_varnames[0]) receiver_type = type(receiver) candidate = receiver if issubclass(receiver_type, type) else receiver_type mro = type.__getattribute__(candidate, '__mro__') + if not owners: + with self._classes_lock: + for base in mro: + for member in type.__getattribute__(base, '__dict__').values(): + self._track_method(base, member) + owners = tuple(self.method_owners.get(frame.f_code, ())) if any(owner in mro for owner in owners): return candidate return owners[0] if len(owners) == 1 else None @@ -136,8 +145,15 @@ def namespace(frame): return namespace def _initialize(self, frame, declarations, parameters, returns): - scope = Scope(declarations=declarations, self_owner=self.method_self_owner(frame)) namespace = self.namespace(frame) + owner = self.method_self_owner(frame) + if owner is None: + sources = (*declarations.values(), + *(source for source, _ in (parameters or {}).values()), returns) + if any(source and ('Self' in source or namespace.get(source) is typing.Self) + for source in sources): + owner = self.method_self_owner(frame, discover=True) + scope = Scope(declarations=declarations, self_owner=owner) if scope.self_owner is not None: namespace[SELF_OWNER] = scope.self_owner for name, (source, mode) in (parameters or {}).items(): @@ -397,6 +413,10 @@ def assign_attribute(self,owner,name,value,annotation=None): else: raise TypeViolation(f'{name}: Final attribute cannot be reassigned') contract.validate(value,f'{type(owner).__name__}.{name}') setattr(owner,name,value) + if issubclass(type(owner), type): + member = type.__getattribute__(owner, '__dict__').get(name) + with self._classes_lock: + self._track_method(owner, member) finally: del frame diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 6ff68c2..710ba58 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1205,6 +1205,15 @@ def same(this) -> Self: return this class Child(Base): pass child = Child() print(type(child.clone()).__name__, type(Child.create()).__name__, type(child.same).__name__) +''', + 'self in assigned methods': '''from typing import Self +class Base: pass +class Child(Base): pass +def clone(this) -> Self: return type(this)() +def create(klass) -> Self: return klass() +Base.clone = clone +setattr(Base, 'create', classmethod(create)) +print(type(Child().clone()).__name__, type(Child.create()).__name__) ''', } if sys.version_info >= (3, 12): diff --git a/tests/test_type_safety.py b/tests/test_type_safety.py index bc83f5d..a3028b0 100644 --- a/tests/test_type_safety.py +++ b/tests/test_type_safety.py @@ -132,6 +132,49 @@ def create(klass) -> Self: '\nclass Child(Base): pass\n' + ('Child.create()\n' if 'classmethod' in method else 'Child().clone()\n')) + def test_self_in_methods_assigned_after_class_creation(self): + prelude = '''from typing import Self +class Base: pass +class Child(Base): pass +def clone(this) -> Self: + return type(this)() +def create(klass) -> Self: + return klass() +Base.clone = clone +Base.create = classmethod(create) +''' + result = self.run_source(prelude + '''child = Child() +answer = (type(child.clone()), type(Child.create())) +''') + self.assertEqual(result['answer'], (result['Child'], result['Child'])) + late = self.run_source('''from typing import Self as S +class Base: pass +class Child(Base): pass +def clone(this) -> S: + return type(this)() +setattr(Base, 'clone', clone) +answer = type(Child().clone()) +''') + self.assertIs(late['answer'], late['Child']) + with self.assertRaises(TypeViolation): + self.run_source('''from typing import Self +class Base: pass +class Child(Base): pass +def wrong(this) -> Self: + return Base() +Base.clone = wrong +Child().clone() +''') + with self.assertRaises(TypeViolation): + self.run_source('''from typing import Self +class Base: pass +class Child(Base): pass +def wrong(this) -> Self: + return Base() +setattr(Base, 'clone', wrong) +Child().clone() +''') + @unittest.skipIf(sys.version_info < (3, 12), "The type statement requires Python 3.12") def test_forward_local_alias_is_captured(self): self.assertEqual(self.run_source('''def factory(): From 509dc50f59b96120ef0ab1eb85eea84fffbdedda Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 17:24:05 +0700 Subject: [PATCH 45/72] Run piped stdin without explicit dash --- src/aiython/cli.py | 22 ++++++++++++++++++---- tests/test_cli.py | 14 ++++++++++---- tests/test_python_compatibility.py | 4 +++- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/aiython/cli.py b/src/aiython/cli.py index 8e0a157..dfa8762 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -200,8 +200,16 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta entry_kind="file", entry_argument=None, runtime=None, preparation_started=None, resolved_seconds=None): started = preparation_started if preparation_started is not None else perf_counter() - argv0 = ("-c" if entry_kind == "command" else "-" if entry_kind == "stdin" else - module_spec.origin if entry_kind == "module" else entry_argument if entry_kind == "path" else str(path)) + if entry_kind == "command": + argv0 = "-c" + elif entry_kind == "stdin": + argv0 = "-" if entry_argument is None else entry_argument + elif entry_kind == "module": + argv0 = module_spec.origin + elif entry_kind == "path": + argv0 = entry_argument + else: + argv0 = str(path) display_path = ("" if entry_kind == "command" else "" if entry_kind == "stdin" else path if module_spec else path.absolute()) path = path.resolve() @@ -256,6 +264,7 @@ def finish(): sys.argv = [argv0, *arguments] sys.orig_argv = ([*interpreter_args, "-m", module_invocation, *arguments] if entry_kind == "module" else [*interpreter_args, "-c", source, *arguments] if entry_kind == "command" else + [*interpreter_args] if entry_kind == "stdin" and entry_argument == "" else [*interpreter_args, argv0, *arguments]) if entry_kind == "path": entry_path = str(Path(entry_argument).absolute()) @@ -324,9 +333,12 @@ def main(argv=None): arguments = list(sys.argv[1:] if argv is None else argv) original_path = sys.path[:] if argv is not None else None try: + implicit_stdin = not arguments and not sys.stdin.isatty() if not arguments: - parser().print_help() - return + if not implicit_stdin: + parser().print_help() + return + arguments = ["-"] if arguments[:1] == ["setup"]: from .setup import setup setup(arguments[1:]) @@ -416,6 +428,8 @@ def main(argv=None): source = sys.stdin.read() path = Path.cwd() / "__main__.py" entry_kind = "stdin" + if implicit_stdin: + entry_argument = "" elif path.suffix == ".pyc" and path.is_file(): compiled_code = importlib.machinery.SourcelessFileLoader( "__main__", str(path.absolute())).get_code("__main__") diff --git a/tests/test_cli.py b/tests/test_cli.py index 0b8eedd..5d4819d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,15 +15,21 @@ from prompt_toolkit.document import Document from aiython.config import CAPABILITIES, credential, resolve +from aiython.cli import main from aiython.models import ConfigError from aiython.setup import ModelChoice, _catalog, _choose_model, setup class CLITests(unittest.TestCase): - def test_cli_without_arguments_shows_first_run_help(self): - result = subprocess.run([sys.executable, "-m", "aiython"], capture_output=True, text=True) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("aiython setup", result.stdout) + def test_cli_without_arguments_shows_first_run_help_on_terminal(self): + class Terminal(io.StringIO): + def isatty(self): + return True + + output = io.StringIO() + with patch.object(sys, 'stdin', Terminal()), contextlib.redirect_stdout(output): + main([]) + self.assertIn("aiython setup", output.getvalue()) def test_cli_version_uses_distribution_metadata(self): result = subprocess.run([sys.executable, "-m", "aiython", "--version"], capture_output=True, text=True) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 710ba58..a7f1f75 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -90,7 +90,8 @@ def report(stage): ''' with tempfile.TemporaryDirectory() as directory: for name, arguments, input_source in (('command', ['-c', source, 'one', '-x'], None), - ('stdin', ['-', 'one', '-x'], source)): + ('stdin', ['-', 'one', '-x'], source), + ('implicit stdin', [], source)): with self.subTest(name=name): python = subprocess.run([sys.executable, *arguments], input=input_source, cwd=directory, capture_output=True, text=True) @@ -339,6 +340,7 @@ def test_safe_path_modes_for_new_entries_match_cpython(self): for flag in ('-I', '-P'): for name, arguments, input_source in (('command', ['-c', source], None), ('stdin', ['-'], source), + ('implicit stdin', [], source), ('directory', ['app'], None), ('zipapp', ['app.pyz'], None)): with self.subTest(flag=flag, name=name): From e607c124c0dbdbdd22719068ab9cf0605d2bfc82 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 17:36:56 +0700 Subject: [PATCH 46/72] Add persistent interactive Aiython console --- docs/getting-started.md | 2 + src/aiython/cli.py | 43 ++++++++++++++++++- src/aiython/repl.py | 61 +++++++++++++++++++++++++++ src/aiython/runtime.py | 8 ++-- src/aiython/typed_runtime.py | 14 ++++++- tests/test_cli.py | 36 ++++++++++++++-- tests/test_repl.py | 80 ++++++++++++++++++++++++++++++++++++ 7 files changed, 234 insertions(+), 10 deletions(-) create mode 100644 src/aiython/repl.py create mode 100644 tests/test_repl.py diff --git a/docs/getting-started.md b/docs/getting-started.md index a8db5b3..c5fd4c9 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -54,6 +54,8 @@ aiython tickets.py `--explain` shows the AI boundary without executing the script or contacting a provider. Running the script uses your chosen provider and may incur charges. For a uv project, prefix both commands with `uv run`. +Run `aiython` without arguments in a terminal for an interactive console. It keeps variables, type annotations, and future imports between commands, and accepts AI expressions. Piped input runs as a Python script, with or without an explicit `-` argument. + ## Next steps - Try the [small examples](examples.md) to see one behavior at a time. diff --git a/src/aiython/cli.py b/src/aiython/cli.py index dfa8762..61690c5 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -306,6 +306,47 @@ def finish(): module.__dict__.pop("__cached__", None) +def run_repl(*, restore_state=False): + from .repl import AiythonConsole + from .typed_runtime import Scope + + config = resolve(Path.cwd() / '__main__.py') + runtime = Runtime(config) + module = types.ModuleType('__main__') + module.__dict__.update({'__package__': None, '__spec__': None, + '__loader__': importlib.machinery.BuiltinImporter, + '__builtins__': builtins}) + runtime.types.interactive_globals = module.__dict__ + runtime.types.interactive_scope = Scope() + console = AiythonConsole(runtime, module.__dict__) + finder = ProjectFinder(runtime) + old_main = sys.modules.get('__main__') + old_argv, old_orig_argv, old_path = sys.argv, sys.orig_argv, sys.path[:] + if not restore_state: + atexit.register(runtime.capabilities.close) + try: + sys.modules['__main__'] = module + sys.argv = [''] + sys.orig_argv = interpreter_arguments() + if not sys.flags.safe_path: + sys.path[:1] = [''] + sys.meta_path.insert(0, finder) + banner = ('' if sys.flags.quiet else + f'Python {sys.version} on {sys.platform}\n' + 'Type "help", "copyright", "credits" or "license" for more information.') + console.interact(banner=banner, exitmsg='') + return module.__dict__ + finally: + if restore_state: + sys.meta_path.remove(finder) + sys.argv, sys.orig_argv, sys.path[:] = old_argv, old_orig_argv, old_path + if old_main is None: + sys.modules.pop('__main__', None) + else: + sys.modules['__main__'] = old_main + runtime.capabilities.close() + + def parser(): result = argparse.ArgumentParser( prog="aiython", description="Python with project-scoped AI execution", @@ -336,7 +377,7 @@ def main(argv=None): implicit_stdin = not arguments and not sys.stdin.isatty() if not arguments: if not implicit_stdin: - parser().print_help() + run_repl(restore_state=argv is not None) return arguments = ["-"] if arguments[:1] == ["setup"]: diff --git a/src/aiython/repl.py b/src/aiython/repl.py new file mode 100644 index 0000000..73b8282 --- /dev/null +++ b/src/aiython/repl.py @@ -0,0 +1,61 @@ +"""Interactive Python cells compiled through the Aiython runtime.""" + +import __future__ +import ast +import code + +from .frontend import parse +from .runtime import bind_runtime +from .typed_runtime import helper + +FUTURE_FLAGS = sum(getattr(__future__, name).compiler_flag + for name in __future__.all_feature_names) + + +class AiythonConsole(code.InteractiveConsole): + def __init__(self, runtime, namespace): + super().__init__(locals=namespace, filename='') + self.runtime = runtime + self.cell_number = 0 + + def runsource(self, source, filename='', symbol='single'): + cell_filename = f'' + unit = None + try: + complete = self.compile(source, cell_filename, symbol) + except (OverflowError, SyntaxError, ValueError): + try: + unit = parse(source, cell_filename) + if not unit.blocks: + raise + complete = self.compile(unit.transformed, cell_filename, symbol) + except (OverflowError, SyntaxError, ValueError): + self.showsyntaxerror(cell_filename) + return False + except SystemExit: + raise + except BaseException: + self.showtraceback() + return False + if complete is None: + return True + try: + unit = unit or parse(source, cell_filename) + if unit.tree.body and isinstance(unit.tree.body[-1], ast.Expr): + expression = unit.tree.body[-1] + expression.value = helper('display', expression.value, + runtime_name=unit.runtime_name) + flags = self.compile.compiler.flags & FUTURE_FLAGS + compiled = bind_runtime(self.runtime.prepare(unit, entry=True, flags=flags), + unit.runtime_name, self.runtime) + except (OverflowError, SyntaxError, ValueError): + self.showsyntaxerror(cell_filename) + return False + except SystemExit: + raise + except BaseException: + self.showtraceback() + return False + self.cell_number += 1 + self.runcode(compiled) + return False diff --git a/src/aiython/runtime.py b/src/aiython/runtime.py index 36c23e7..261c189 100644 --- a/src/aiython/runtime.py +++ b/src/aiython/runtime.py @@ -825,16 +825,16 @@ def frame_source(self, frame): return self.frame_sources.get(key, self.frame_sources.get( (code.co_filename, 1, ""), {"available": False, "filename": code.co_filename})) - def prepare(self, unit: Unit, *, entry: bool = False): + def prepare(self, unit: Unit, *, entry: bool = False, flags: int = 0): started = perf_counter() try: with self._lock: - return self._prepare(unit, entry=entry) + return self._prepare(unit, entry=entry, flags=flags) finally: if self.stats.enabled: self.stats.prepare_seconds += perf_counter() - started - def _prepare(self, unit: Unit, *, entry: bool = False): + def _prepare(self, unit: Unit, *, entry: bool = False, flags: int = 0): self.register(unit) self.blocks.update({key: (unit, block) for key, block in unit.blocks.items()}) linecache.cache[unit.filename] = (len(unit.source), None, unit.source.splitlines(True), unit.filename) @@ -878,7 +878,7 @@ def _prepare(self, unit: Unit, *, entry: bool = False): ast.fix_missing_locations(tree) tree = RuntimeReferences(unit.runtime_name).visit(tree) ast.fix_missing_locations(tree) - return compile(tree, unit.filename, "exec", dont_inherit=True) + return compile(tree, unit.filename, "exec", flags=flags, dont_inherit=True) def lookup(self, name: str): frame = inspect.currentframe().f_back diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index a2d5e27..dc200ce 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -5,6 +5,7 @@ from contextvars import ContextVar from dataclasses import dataclass, field import inspect +import sys import threading import types import typing @@ -76,6 +77,12 @@ def __init__(self, manager=None): self.method_owners = weakref.WeakKeyDictionary() self._classes_lock = threading.RLock() self._contract_cache = threading.local() + self.interactive_globals = None + self.interactive_scope = None + + @staticmethod + def display(value): + sys.displayhook(value) def contract(self, annotation, namespace): cache = getattr(self._contract_cache, 'value', None) @@ -153,7 +160,12 @@ def _initialize(self, frame, declarations, parameters, returns): if any(source and ('Self' in source or namespace.get(source) is typing.Self) for source in sources): owner = self.method_self_owner(frame, discover=True) - scope = Scope(declarations=declarations, self_owner=owner) + if (frame.f_code.co_name == '' and frame.f_globals is self.interactive_globals + and self.interactive_scope is not None): + scope = self.interactive_scope + scope.declarations.update(declarations) + else: + scope = Scope(declarations=declarations, self_owner=owner) if scope.self_owner is not None: namespace[SELF_OWNER] = scope.self_owner for name, (source, mode) in (parameters or {}).items(): diff --git a/tests/test_cli.py b/tests/test_cli.py index 5d4819d..125477c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,21 +15,49 @@ from prompt_toolkit.document import Document from aiython.config import CAPABILITIES, credential, resolve -from aiython.cli import main +from aiython.cli import main, run_repl from aiython.models import ConfigError from aiython.setup import ModelChoice, _catalog, _choose_model, setup class CLITests(unittest.TestCase): - def test_cli_without_arguments_shows_first_run_help_on_terminal(self): + def test_cli_without_arguments_opens_repl_on_terminal(self): class Terminal(io.StringIO): def isatty(self): return True output = io.StringIO() - with patch.object(sys, 'stdin', Terminal()), contextlib.redirect_stdout(output): + errors = io.StringIO() + original = (sys.argv, sys.orig_argv, sys.path[:], sys.modules['__main__']) + with patch.object(sys, 'stdin', Terminal('value: int = 2\nprint(value)\nvalue = 3\nprint(value)\n')), \ + contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors): main([]) - self.assertIn("aiython setup", output.getvalue()) + self.assertIn('2\n', output.getvalue()) + self.assertIn('3\n', output.getvalue()) + self.assertIn('Python ', errors.getvalue()) + self.assertIs(sys.argv, original[0]) + self.assertIs(sys.orig_argv, original[1]) + self.assertEqual(sys.path, original[2]) + self.assertIs(sys.modules['__main__'], original[3]) + + def test_repl_main_module_metadata_matches_python(self): + class Terminal(io.StringIO): + def isatty(self): + return True + + source = ('import sys\n' + "snapshot = (__name__, __package__, __spec__, sys.argv[:], " + "sys.orig_argv[:], '__file__' in globals(), sys.path[0])\n") + with patch.object(sys, 'stdin', Terminal(source)), \ + contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(io.StringIO()): + namespace = run_repl(restore_state=True) + name, package, spec, argv, orig_argv, has_file, path = namespace['snapshot'] + self.assertEqual((name, package, spec, argv, has_file), + ('__main__', None, None, [''], False)) + self.assertEqual(orig_argv[0], sys.executable) + if not sys.flags.safe_path: + self.assertEqual(path, '') def test_cli_version_uses_distribution_metadata(self): result = subprocess.run([sys.executable, "-m", "aiython", "--version"], capture_output=True, text=True) diff --git a/tests/test_repl.py b/tests/test_repl.py new file mode 100644 index 0000000..a12783f --- /dev/null +++ b/tests/test_repl.py @@ -0,0 +1,80 @@ +"""Interactive cells preserve Python state and Aiython boundaries.""" + +import builtins +import contextlib +import io +from pathlib import Path +import unittest + +from aiython.models import ProfileConfig, ResolvedConfig +from aiython.repl import AiythonConsole +from aiython.runtime import Runtime +from aiython.typed_runtime import Scope + + +class InteractiveConsoleTests(unittest.TestCase): + def setUp(self): + self.had_underscore = '_' in vars(builtins) + self.original_underscore = vars(builtins).get('_') + class Agent: + def execute(self, request, bridge): + return 7 + + root = Path.cwd() + config = ResolvedConfig(None, root, 'default', + {'default': ProfileConfig('default', 'fake', 'model')}) + self.runtime = Runtime(config, agent_factory=lambda _: Agent()) + self.namespace = {'__name__': '__main__', '__builtins__': builtins} + self.runtime.types.interactive_globals = self.namespace + self.runtime.types.interactive_scope = Scope() + self.console = AiythonConsole(self.runtime, self.namespace) + + def tearDown(self): + self.runtime.capabilities.close() + if self.had_underscore: + builtins._ = self.original_underscore + else: + vars(builtins).pop('_', None) + + def test_compound_input_waits_and_displays_last_expression(self): + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.assertTrue(self.console.runsource('for number in range(2):')) + self.assertTrue(self.console.runsource('for number in range(2):\n print(number)')) + self.assertFalse(self.console.runsource('for number in range(2):\n print(number)\n')) + self.assertFalse(self.console.runsource('number')) + self.assertFalse(self.console.runsource('_ + 1')) + self.assertEqual(output.getvalue(), '0\n1\n1\n2\n') + + def test_ai_syntax_and_type_contract_persist_across_cells(self): + output, errors = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors): + self.assertFalse(self.console.runsource('answer: int = choose seven')) + self.assertFalse(self.console.runsource('answer')) + self.assertFalse(self.console.runsource('answer = "wrong"')) + self.assertFalse(self.console.runsource('answer')) + self.assertEqual(output.getvalue(), '7\n7\n') + self.assertIn('TypeViolation', errors.getvalue()) + + def test_ai_syntax_in_compound_input_waits_for_blank_line(self): + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.assertTrue(self.console.runsource('for number in range(2):\n' + ' answer: int = choose seven')) + self.assertFalse(self.console.runsource('for number in range(2):\n' + ' answer: int = choose seven\n')) + self.assertFalse(self.console.runsource('answer')) + self.assertEqual(output.getvalue(), '7\n') + + def test_future_annotations_apply_to_later_cells(self): + errors = io.StringIO() + with contextlib.redirect_stderr(errors): + self.assertFalse(self.console.runsource('from __future__ import annotations')) + self.assertFalse(self.console.runsource('def later(value: Missing):\n' + ' return value\n')) + self.assertEqual(self.namespace['later'].__annotations__['value'], 'Missing') + self.assertEqual(errors.getvalue(), '') + + +if __name__ == '__main__': + unittest.main() From 1004ea7e98ca4c167f52a014fe526d4fb84e4e69 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 17:41:39 +0700 Subject: [PATCH 47/72] Apply CLI options to interactive and piped execution --- docs/getting-started.md | 2 +- src/aiython/cli.py | 35 +++++++++++++++++++++++++----- tests/test_cli.py | 24 ++++++++++++++++++++ tests/test_python_compatibility.py | 3 +-- 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index c5fd4c9..788ca3e 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -54,7 +54,7 @@ aiython tickets.py `--explain` shows the AI boundary without executing the script or contacting a provider. Running the script uses your chosen provider and may incur charges. For a uv project, prefix both commands with `uv run`. -Run `aiython` without arguments in a terminal for an interactive console. It keeps variables, type annotations, and future imports between commands, and accepts AI expressions. Piped input runs as a Python script, with or without an explicit `-` argument. +Run `aiython` without a script in a terminal for an interactive console. It keeps variables, type annotations, and future imports between commands, and accepts AI expressions. Pass `--profile NAME` or `--config PATH` to select its model configuration. Piped input runs as a Python script, with or without an explicit `-` argument. ## Next steps diff --git a/src/aiython/cli.py b/src/aiython/cli.py index 61690c5..102e490 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -306,12 +306,16 @@ def finish(): module.__dict__.pop("__cached__", None) -def run_repl(*, restore_state=False): +def run_repl(*, config_path=None, profile=None, force_profile=None, + stats=False, trace_plan=False, restore_state=False): from .repl import AiythonConsole from .typed_runtime import Scope - config = resolve(Path.cwd() / '__main__.py') - runtime = Runtime(config) + started = perf_counter() + config = resolve(Path.cwd() / '__main__.py', config_path=config_path, + profile=profile, force_profile=force_profile) + config_seconds = perf_counter() - started + runtime = Runtime(config, stats=stats, trace_plan=trace_plan) module = types.ModuleType('__main__') module.__dict__.update({'__package__': None, '__spec__': None, '__loader__': importlib.machinery.BuiltinImporter, @@ -322,8 +326,18 @@ def run_repl(*, restore_state=False): finder = ProjectFinder(runtime) old_main = sys.modules.get('__main__') old_argv, old_orig_argv, old_path = sys.argv, sys.orig_argv, sys.path[:] + execution_started = None + def finish(): + if stats: + runtime.stats.run = {'total_seconds': perf_counter() - started, + 'config_seconds': config_seconds, + 'execution_seconds': (perf_counter() - execution_started + if execution_started is not None else 0)} + runtime.stats.report() + runtime.capabilities.close() + if not restore_state: - atexit.register(runtime.capabilities.close) + atexit.register(finish) try: sys.modules['__main__'] = module sys.argv = [''] @@ -334,6 +348,7 @@ def run_repl(*, restore_state=False): banner = ('' if sys.flags.quiet else f'Python {sys.version} on {sys.platform}\n' 'Type "help", "copyright", "credits" or "license" for more information.') + execution_started = perf_counter() console.interact(banner=banner, exitmsg='') return module.__dict__ finally: @@ -344,7 +359,7 @@ def run_repl(*, restore_state=False): sys.modules.pop('__main__', None) else: sys.modules['__main__'] = old_main - runtime.capabilities.close() + finish() def parser(): @@ -462,7 +477,15 @@ def main(argv=None): entry_kind = "command" else: if args.script is None: - argument_parser.error("a script path or -m module is required") + if sys.stdin.isatty(): + if args.explain: + argument_parser.error("--explain requires a script, -c, -m, or piped source") + run_repl(config_path=args.config, profile=args.profile, + force_profile=args.force_profile, stats=args.stats, + trace_plan=args.trace_plan, restore_state=argv is not None) + return + args.script = "-" + implicit_stdin = True path = Path(args.script) script_args = args.args if args.script == "-": diff --git a/tests/test_cli.py b/tests/test_cli.py index 125477c..8ba05f9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -59,6 +59,30 @@ def isatty(self): if not sys.flags.safe_path: self.assertEqual(path, '') + def test_repl_options_and_piped_stdin_without_script(self): + class Terminal(io.StringIO): + def isatty(self): + return True + + with patch.object(sys, 'stdin', Terminal()), patch('aiython.cli.run_repl') as start: + main(['--config', 'settings.toml', '--profile', 'chosen', '--stats']) + start.assert_called_once_with(config_path='settings.toml', profile='chosen', + force_profile=None, stats=True, trace_plan=False, + restore_state=True) + + errors = io.StringIO() + with patch.object(sys, 'stdin', Terminal('print("ready")\n')), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(errors): + run_repl(stats=True, restore_state=True) + self.assertIn('aiython run stats:', errors.getvalue()) + + result = subprocess.run([sys.executable, '-m', 'aiython', '--stats'], + input='import sys\nprint(sys.argv)\n', + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "['']\n") + self.assertIn('aiython run stats:', result.stderr) + def test_cli_version_uses_distribution_metadata(self): result = subprocess.run([sys.executable, "-m", "aiython", "--version"], capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index a7f1f75..6ad3870 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -615,8 +615,7 @@ def test_zip_and_sourceless_module_execution_match_cpython(self): def test_module_cli_reports_invalid_requests(self): for arguments, expected in ((['-m'], '-m requires a module name'), (['-m', 'module_that_does_not_exist'], 'No module named'), - (['--explain', '-m', 'module_that_does_not_exist'], 'No module named'), - (['--stats'], 'a script path or -m module is required')): + (['--explain', '-m', 'module_that_does_not_exist'], 'No module named')): with self.subTest(arguments=arguments): result = subprocess.run([sys.executable, '-m', 'aiython', *arguments], capture_output=True, text=True) From ef37ddfea1f440d2ed85cc386929f6409766985b Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 17:50:36 +0700 Subject: [PATCH 48/72] Support dataclass InitVar keyword markers and field factories --- src/aiython/type_constraints.py | 17 ++++++++++--- src/aiython/typed_runtime.py | 39 ++++++++++++++++++++++++++++-- tests/test_python_compatibility.py | 26 ++++++++++++++++++++ tests/test_type_safety.py | 15 ++++++++++++ 4 files changed, 92 insertions(+), 5 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 3f424fe..011ab66 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -191,7 +191,7 @@ def schema(self, seen=None): elif kind == 'typeddict' or (kind == 'class' and self.fields): result = {'type':'object', 'properties':{k:v.schema(seen) for k,v in self.fields.items()}, 'required': sorted(self.required)} - elif kind in ('annotated', 'qualifier', 'alias', 'unpack_typeddict'): + elif kind in ('annotated', 'qualifier', 'alias', 'unpack_typeddict', 'initvar'): result = self.args[0].schema(seen) elif kind == 'typevar': result = {'anyOf':[a.schema(seen) for a in self.args]} if self.args else {} @@ -221,8 +221,10 @@ def fail(detail=None): raise TypeViolation(f'{path}: expected {self.name}, got {type(value).__name__}' + (f' ({detail})' if detail else '')) def child(contract, item, suffix): contract.validate(item, path + suffix, bindings=bindings, seen=seen) - if kind in ('alias', 'annotated', 'qualifier', 'unpack_typeddict'): + if kind in ('alias', 'annotated', 'qualifier', 'unpack_typeddict', 'initvar'): child(self.args[0], value, '') + elif kind == 'kw_only': + raise UnsupportedType('KW_ONLY marks dataclass parameters; it is not a value type') elif kind == 'never': fail('this boundary must not return') elif kind == 'union': for contract in self.args: @@ -422,6 +424,10 @@ def node(self,node,names): def generic(self,base,args,label,names): origin = typing.get_origin(base) or base + if origin is dataclasses.InitVar: + if len(args) != 1: + raise UnsupportedType('InitVar requires one type') + return Contract('initvar', label, args) if origin is typing.Unpack and len(args) == 1 and args[0].kind in ('unpack_any', 'unpack_fixed'): return args[0] if origin is typing.Unpack and len(args) == 1: @@ -634,7 +640,7 @@ def class_contract(self,target,names,label=None): fields.update({name: (source, base) for name, source in annotations_of(base).items()}) for name, (source, owner) in fields.items(): contract = self.compile(source, scopes.get(owner, scope)) - if contract.marker == 'ClassVar': + if contract.marker == 'ClassVar' or contract.kind in ('initvar', 'kw_only'): continue result.fields[name] = contract if record: @@ -663,6 +669,11 @@ def value(self,target,names): python_type={'str':str, 'bytes':bytes}.get(stream_type)),)) if target is typing.TypeAlias or target is TypeAlias: return Contract('any', 'TypeAlias') + if target is dataclasses.KW_ONLY: + return Contract('kw_only', 'KW_ONLY') + if isinstance(target, dataclasses.InitVar): + return Contract('initvar', str(target), + (self.compile(target.type, names),)) if isinstance(target, typing.TypeVarTuple): bound = names.get(target, names.get(target.__name__)) return (Contract('unpack_fixed', target.__name__, bound) diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index dc200ce..dbfc994 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -3,7 +3,9 @@ import ast from contextvars import ContextVar +import dataclasses from dataclasses import dataclass, field +import functools import inspect import sys import threading @@ -99,12 +101,44 @@ def validate_output(self, value, annotation, frame): def register_class(self, cls): if isinstance(cls, type): + self._check_dataclass_initvars(cls) with self._classes_lock: self.classes.add(cls) for member in type.__getattribute__(cls, '__dict__').values(): self._track_method(cls, member) return cls + def _check_dataclass_initvars(self, cls): + # A subclass inherits this attribute even when it was not decorated. + if '__dataclass_fields__' not in vars(cls): + return + initvars = [entry for entry in cls.__dataclass_fields__.values() + if entry._field_type is dataclasses._FIELD_INITVAR and entry.init] + original = vars(cls).get('__init__') + if (not initvars or type(original) is not types.FunctionType + or getattr(original, '__aiython_initvars__', False)): + return + namespace = Compiler.module_names(cls, {}) | {cls.__name__: cls, SELF_OWNER: cls} + contracts = {entry.name: self.contract(entry.type, namespace) for entry in initvars} + signature = inspect.signature(original) + + @functools.wraps(original) + def checked_init(instance, *args, **kwargs): + bound = signature.bind(instance, *args, **kwargs) + bound.apply_defaults() + for name, contract in contracts.items(): + if name in bound.arguments: + contract.validate(bound.arguments[name], name) + return original(instance, *args, **kwargs) + + checked_init.__aiython_initvars__ = True + type.__setattr__(cls, '__init__', checked_init) + + @staticmethod + def _dataclass_field_placeholder(frame, value): + return (type(value) is dataclasses.Field and frame.f_code.co_name != '' + and not frame.f_code.co_flags & inspect.CO_OPTIMIZED) + def _track_method(self, cls, member): if type(member) is types.FunctionType: methods = (member,) @@ -239,7 +273,8 @@ def assignment_in(self, frame, value, name, annotation=None): if contract: if name in scope.final_names: raise TypeViolation(f'{name}: Final binding cannot be reassigned') - contract.validate(value,name,bindings=scope.bindings) + if not self._dataclass_field_placeholder(frame, value): + contract.validate(value,name,bindings=scope.bindings) if contract.marker == 'Final': scope.final_names.add(name) with self._classes_lock: registered = type(value) in self.classes @@ -297,7 +332,7 @@ def check_instances(candidate): if name in values and name not in scope.contracts: scope.contracts[name] = self.contract(source,self.namespace(frame)) for name, contract in scope.contracts.items(): - if name in values: + if name in values and not self._dataclass_field_placeholder(frame, values[name]): contract.validate(values[name],name,bindings=scope.bindings) def checkpoint(self): diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 6ad3870..485fb53 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -943,6 +943,32 @@ def value(self): return super().value() + 1 class Point: x: int print(Point(2).x, hasattr(Point(2), '__dict__')) +''', + 'dataclass InitVar and post init': '''from dataclasses import dataclass, InitVar +@dataclass +class Point: + x: int + offset: InitVar[int] + def __post_init__(self, offset): + self.x += offset +point = Point(1, 2) +print(point.x, hasattr(point, 'offset')) +''', + 'dataclass keyword only marker': '''from dataclasses import dataclass, KW_ONLY +@dataclass +class Point: + x: int + _: KW_ONLY + label: str = 'x' +print(Point(1, label='ready').x, Point(1).label) +''', + 'dataclass default factory': '''from dataclasses import dataclass, field +@dataclass +class Point: + values: list[int] = field(default_factory=list) +point = Point() +point.values.append(1) +print(point.values) ''', 'enum': '''from enum import Enum class Color(Enum): diff --git a/tests/test_type_safety.py b/tests/test_type_safety.py index a3028b0..1ad1ae2 100644 --- a/tests/test_type_safety.py +++ b/tests/test_type_safety.py @@ -348,6 +348,21 @@ class Item: item = Item('bad') ''') + def test_dataclass_initvar_input_and_factory_field_are_checked(self): + prelude = '''from dataclasses import InitVar, dataclass, field +@dataclass +class Item: + offset: InitVar[int] + values: list[int] = field(default_factory=list) + def __post_init__(self, offset): + self.values.append(offset) +''' + self.assertEqual(self.run_source(prelude + 'item = Item(2)\nanswer = item.values\n')['answer'], [2]) + with self.assertRaises(TypeViolation): + self.run_source(prelude + "Item('bad')\n") + with self.assertRaises(TypeViolation): + self.run_source(prelude + "item = Item(2)\nitem.values.append('bad')\n") + def test_typed_natural_language_keeps_subscript_inside_statement(self): from aiython.frontend import parse unit = parse('analysis: TicketAnalysis = analyze the ticket from ticket["message"]\n','test.py') From a7041a0bc5e31585f54cd73f269c840026a8956f Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 17:55:00 +0700 Subject: [PATCH 49/72] Run PYTHONSTARTUP before interactive Aiython sessions --- src/aiython/cli.py | 26 ++++++++++++++++++++++++++ tests/test_cli.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/src/aiython/cli.py b/src/aiython/cli.py index 102e490..baf5178 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -11,6 +11,7 @@ import os import sys import tokenize +import traceback import types import warnings import zipfile @@ -349,6 +350,31 @@ def finish(): f'Python {sys.version} on {sys.platform}\n' 'Type "help", "copyright", "credits" or "license" for more information.') execution_started = perf_counter() + startup = os.environ.get('PYTHONSTARTUP') if not sys.flags.ignore_environment else None + if startup: + try: + with tokenize.open(startup) as source_file: + source = source_file.read() + except OSError as exc: + print('Could not open PYTHONSTARTUP', file=sys.stderr) + print(f'{type(exc).__name__}: {exc}', file=sys.stderr) + except (SyntaxError, UnicodeError) as exc: + traceback.print_exception(type(exc), exc, None) + else: + module.__file__ = startup + try: + try: + code = runtime.compile_source(source, startup, entry=True) + except (SyntaxError, OverflowError, ValueError) as exc: + traceback.print_exception(type(exc), exc, None) + else: + console.runcode(code) + except SystemExit: + raise + except BaseException: + traceback.print_exc() + finally: + module.__dict__.pop('__file__', None) console.interact(banner=banner, exitmsg='') return module.__dict__ finally: diff --git a/tests/test_cli.py b/tests/test_cli.py index 8ba05f9..678268d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -59,6 +59,46 @@ def isatty(self): if not sys.flags.safe_path: self.assertEqual(path, '') + def test_repl_runs_python_startup_in_interactive_namespace(self): + class Terminal(io.StringIO): + def isatty(self): + return True + + with tempfile.TemporaryDirectory() as directory: + startup = Path(directory) / 'startup.py' + startup.write_text('startup_value: int = 4\n' + "startup_file = __file__\n") + terminal = Terminal('answer = startup_value + 1\n' + "startup_value = 'bad'\n") + errors = io.StringIO() + with patch.dict(os.environ, {'PYTHONSTARTUP': str(startup)}), \ + patch.object(sys, 'stdin', terminal), \ + contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(errors): + namespace = run_repl(restore_state=True) + self.assertEqual(namespace['answer'], 5) + self.assertEqual(namespace['startup_file'], str(startup)) + self.assertNotIn('__file__', namespace) + self.assertIn('TypeViolation', errors.getvalue()) + + def test_repl_continues_after_python_startup_encoding_error(self): + class Terminal(io.StringIO): + def isatty(self): + return True + + with tempfile.TemporaryDirectory() as directory: + startup = Path(directory) / 'startup.py' + startup.write_text('# coding: does-not-exist\n') + errors = io.StringIO() + with patch.dict(os.environ, {'PYTHONSTARTUP': str(startup)}), \ + patch.object(sys, 'stdin', Terminal('answer = 3\n')), \ + contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(errors): + namespace = run_repl(restore_state=True) + self.assertEqual(namespace['answer'], 3) + self.assertIn('SyntaxError', errors.getvalue()) + self.assertNotIn('run_repl', errors.getvalue()) + def test_repl_options_and_piped_stdin_without_script(self): class Terminal(io.StringIO): def isatty(self): From 604a380e7fc51686dfff04c1947f4866fc8f4e94 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 18:07:04 +0700 Subject: [PATCH 50/72] Support interactive sessions after script execution --- docs/getting-started.md | 2 ++ src/aiython/cli.py | 54 +++++++++++++++++++++++++----- src/aiython/repl.py | 6 ++++ tests/test_cli.py | 38 ++++++++++++++++++++- tests/test_python_compatibility.py | 25 ++++++++++++++ 5 files changed, 116 insertions(+), 9 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 788ca3e..01d6566 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -56,6 +56,8 @@ aiython tickets.py Run `aiython` without a script in a terminal for an interactive console. It keeps variables, type annotations, and future imports between commands, and accepts AI expressions. Pass `--profile NAME` or `--config PATH` to select its model configuration. Piped input runs as a Python script, with or without an explicit `-` argument. +Use `aiython -i tickets.py` to enter the console after the script finishes. Its variables and type annotations remain available there. + ## Next steps - Try the [small examples](examples.md) to see one behavior at a time. diff --git a/src/aiython/cli.py b/src/aiython/cli.py index baf5178..7f9268c 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -146,10 +146,10 @@ def module_details(name): return spec, source, code -def module_source(name, arguments=(), *, runtime=None, explain=False): +def module_source(name, arguments=(), *, runtime=None, explain=False, interactive=False): old_main = sys.modules.get("__main__") old_argv, old_orig_argv = sys.argv, sys.orig_argv - interpreter_args = interpreter_arguments() + interpreter_args = [*interpreter_arguments(), *(['-i'] if interactive else [])] initial_main = types.ModuleType("__main__") initial_main.__loader__ = importlib.machinery.BuiltinImporter initial_main.__builtins__ = builtins @@ -199,7 +199,7 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta config_path=None, profile=None, force_profile=None, restore_state=True, source=None, module_spec=None, module_invocation=None, compiled_code=None, initial_main=None, entry_kind="file", entry_argument=None, runtime=None, preparation_started=None, - resolved_seconds=None): + resolved_seconds=None, interactive=False): started = preparation_started if preparation_started is not None else perf_counter() if entry_kind == "command": argv0 = "-c" @@ -242,9 +242,17 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta "__cached__": module_spec.cached if module_spec else None}) if sys.version_info < (3, 14): module.__dict__.setdefault("__annotations__", {}) + if interactive: + from .typed_runtime import Scope + runtime.types.interactive_globals = module.__dict__ + runtime.types.interactive_scope = Scope() old_main = sys.modules.get("__main__") old_argv, old_orig_argv, old_path = sys.argv, sys.orig_argv, sys.path[:] - interpreter_args = interpreter_arguments() + missing_last = object() + old_last = ({name: vars(sys).get(name, missing_last) + for name in ('last_type', 'last_value', 'last_exc', 'last_traceback')} + if interactive and restore_state else {}) + interpreter_args = [*interpreter_arguments(), *(['-i'] if interactive else [])] old_spawn_entry = os.environ.get("AIYTHON_SPAWN_ENTRY") finder = ProjectFinder(runtime) execution_started = None @@ -282,10 +290,37 @@ def finish(): if entry_kind in ("file", "module"): os.environ["AIYTHON_SPAWN_ENTRY"] = str(display_path) execution_started = perf_counter() - exec(code, module.__dict__) + if interactive: + try: + exec(code, module.__dict__) + except BaseException as exc: + trace = exc.__traceback__ + while trace is not None and trace.tb_frame.f_code.co_filename != str(display_path): + trace = trace.tb_next + trace = trace or exc.__traceback__ + sys.last_type = type(exc) + sys.last_value = sys.last_exc = exc.with_traceback(trace) + sys.last_traceback = trace + runtime.types.interactive_scope.failed = False + sys.excepthook(type(exc), exc, trace) + if entry_kind in ('file', 'bytecode', 'stdin'): + module.__dict__.pop('__file__', None) + module.__dict__.pop('__cached__', None) + from .repl import AiythonConsole + banner = ('' if sys.flags.quiet else + f'Python {sys.version} on {sys.platform}\n' + 'Type "help", "copyright", "credits" or "license" for more information.') + AiythonConsole(runtime, module.__dict__).interact(banner=banner, exitmsg='') + else: + exec(code, module.__dict__) return module.__dict__ finally: if restore_state: + for name, value in old_last.items(): + if value is missing_last: + vars(sys).pop(name, None) + else: + setattr(sys, name, value) sys.argv = old_argv sys.orig_argv = old_orig_argv sys.path[:] = old_path @@ -402,6 +437,7 @@ def parser(): result.add_argument("--explain", action="store_true", help="Show blocks/checkpoints without executing code") result.add_argument("--trace-plan", action="store_true", help="Trace actual capability routes, cache and timings") result.add_argument("--stats", action="store_true", help="Report model calls, tools, request bytes and timings on stderr") + result.add_argument("-i", "--interactive", action="store_true", help="Enter an interactive console after the program") result.add_argument("-m", "--module", dest="module_args", nargs=argparse.REMAINDER, help="Run a Python module as __main__") result.add_argument("-c", dest="command_args", nargs=argparse.REMAINDER, @@ -464,7 +500,8 @@ def main(argv=None): argument_parser.error("-m requires a module name") module_invocation, *script_args = args.module_args try: - module_spec, source, original_code, initial_main = module_source(module_invocation, script_args) + module_spec, source, original_code, initial_main = module_source( + module_invocation, script_args, interactive=args.interactive) except SyntaxError as exc: # A source file with AI syntax cannot be imported as an ordinary # parent package. A SyntaxError raised *by* package code is a @@ -484,7 +521,7 @@ def main(argv=None): try: module_spec, source, original_code, initial_main = module_source( module_invocation, script_args, runtime=module_runtime, - explain=args.explain) + explain=args.explain, interactive=args.interactive) except BaseException: module_runtime.capabilities.close() raise @@ -555,7 +592,8 @@ def main(argv=None): module_spec=module_spec, module_invocation=module_invocation, compiled_code=compiled_code, initial_main=initial_main, entry_kind=entry_kind, entry_argument=entry_argument, runtime=module_runtime, - preparation_started=module_started, resolved_seconds=module_config_seconds) + preparation_started=module_started, resolved_seconds=module_config_seconds, + interactive=args.interactive) except AiythonError as exc: # Keep the original runtime cause visible without leaking provider internals. if exc.__cause__: diff --git a/src/aiython/repl.py b/src/aiython/repl.py index 73b8282..a6e0391 100644 --- a/src/aiython/repl.py +++ b/src/aiython/repl.py @@ -3,6 +3,7 @@ import __future__ import ast import code +import sys from .frontend import parse from .runtime import bind_runtime @@ -18,6 +19,11 @@ def __init__(self, runtime, namespace): self.runtime = runtime self.cell_number = 0 + def raw_input(self, prompt=''): + sys.stderr.write(prompt) + sys.stderr.flush() + return input() + def runsource(self, source, filename='', symbol='single'): cell_filename = f'' unit = None diff --git a/tests/test_cli.py b/tests/test_cli.py index 678268d..08165d8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,7 +15,7 @@ from prompt_toolkit.document import Document from aiython.config import CAPABILITIES, credential, resolve -from aiython.cli import main, run_repl +from aiython.cli import main, run_repl, run_script from aiython.models import ConfigError from aiython.setup import ModelChoice, _catalog, _choose_model, setup @@ -34,7 +34,9 @@ def isatty(self): main([]) self.assertIn('2\n', output.getvalue()) self.assertIn('3\n', output.getvalue()) + self.assertNotIn('>>>', output.getvalue()) self.assertIn('Python ', errors.getvalue()) + self.assertIn('>>>', errors.getvalue()) self.assertIs(sys.argv, original[0]) self.assertIs(sys.orig_argv, original[1]) self.assertEqual(sys.path, original[2]) @@ -123,6 +125,40 @@ def isatty(self): self.assertEqual(result.stdout, "['']\n") self.assertIn('aiython run stats:', result.stderr) + def test_interactive_script_keeps_values_and_type_scope(self): + class Terminal(io.StringIO): + def isatty(self): + return True + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'main.py' + path.write_text('value: int = 2\n') + output, errors = io.StringIO(), io.StringIO() + previous_last = {name: (name in vars(sys), vars(sys).get(name)) + for name in ('last_type', 'last_value', 'last_exc', 'last_traceback')} + with patch.object(sys, 'stdin', Terminal('answer = value + 1\n' + "value = 'bad'\n" + "print(answer, value, '__file__' in globals())\n")), \ + contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors): + namespace = run_script(path, interactive=True) + self.assertEqual(namespace['answer'], 3) + self.assertIn('3 2 False', output.getvalue()) + self.assertIn('TypeViolation', errors.getvalue()) + self.assertEqual({name: (name in vars(sys), vars(sys).get(name)) + for name in previous_last}, previous_last) + + def test_interactive_flag_enters_console_after_script_error(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'main.py' + path.write_text('value = 3\nraise SystemExit(2)\n') + result = subprocess.run([sys.executable, '-m', 'aiython', '-i', str(path)], + input='import sys\nprint(value, type(sys.last_exc).__name__)\n', + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('3 SystemExit\n', result.stdout) + self.assertIn('SystemExit: 2', result.stderr) + self.assertNotIn('in run_script', result.stderr) + def test_cli_version_uses_distribution_metadata(self): result = subprocess.run([sys.executable, "-m", "aiython", "--version"], capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 485fb53..71a721a 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -758,6 +758,31 @@ def target(): return 1 self.assertEqual((aiython.returncode, aiython.stdout, aiython.stderr), (python.returncode, python.stdout, python.stderr)) + def test_interactive_entries_preserve_main_module_state(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + script = root / 'main.py' + script.write_text('value = 3\n') + module = root / 'samplemod.py' + module.write_text('value = 3\n') + app = root / 'app' + app.mkdir() + (app / '__main__.py').write_text('value = 3\n') + entries = ((str(script),), ('-m', 'samplemod'), (str(app),), + ('-c', 'value = 3')) + prompt = ('import sys\n' + 'print(value, __name__, "__file__" in globals(), ' + 'sys.orig_argv[1:])\n') + for entry in entries: + with self.subTest(entry=entry): + python = subprocess.run([sys.executable, '-i', *entry], input=prompt, + cwd=root, capture_output=True, text=True, timeout=10) + aiython = subprocess.run([sys.executable, '-m', 'aiython', '-i', *entry], + input=prompt, cwd=root, capture_output=True, + text=True, timeout=10) + self.assertEqual((aiython.returncode, aiython.stdout), + (python.returncode, python.stdout)) + def test_language_constructs_match_cpython(self): cases = { 'empty script': '', From 6b67ad2148f5ec9bbe1b05ba6b5f8b16737ebe65 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 18:11:30 +0700 Subject: [PATCH 51/72] Support Pydantic and attrs class field descriptors --- docs/type-safety.md | 2 +- src/aiython/type_constraints.py | 6 ++++++ src/aiython/typed_runtime.py | 20 +++++++++++++++----- tests/test_python_compatibility.py | 21 +++++++++++++++++++++ tests/test_type_safety.py | 22 ++++++++++++++++++++++ 5 files changed, 65 insertions(+), 6 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 5fba2fa..c137655 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -23,7 +23,7 @@ analysis: TicketAnalysis = analyze the current ticket - Function arguments, positional-only/keyword-only arguments, `*args`, `**kwargs`, explicit returns and implicit `None` returns are checked. Async functions use the same rules. - `**kwargs: Unpack[TypedDict]` checks required and optional keyword fields using the declared `TypedDict` contract. - The return value is checked again after `finally`, so cleanup cannot silently invalidate a return that was already checked. -- Typed class fields are checked on direct attribute assignment. Dataclass values are checked without conversion to dictionaries or reconstruction. Nested registered project instances are checked at execution boundaries. +- Typed class fields are checked on direct attribute assignment. Dataclass values are checked without conversion to dictionaries or reconstruction. `dataclasses.field`, Pydantic `Field`/`PrivateAttr`, and `attrs.field` descriptors are accepted while the class is built; the resulting instance values remain checked. Nested registered project instances are checked at execution boundaries. - Mutable containers are checked deeply. Mutations through aliases are detected at statement boundaries. Enclosing scopes and globals are included; closures preserve referenced local annotation names. - Generator functions check each yielded value, sent value and final return. Async generators and `yield from` retain their control-flow protocols. - AI `evaluate`, `execute`, binding writes, terminal results and recovery replacement values use the same checker. Invalid AI output can be repaired within the bounded agent loop before assignment; completed capability side effects are retained. diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 011ab66..e8697ff 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -334,6 +334,12 @@ def child(contract, item, suffix): for name, contract in self.fields.items(): try: item = object.__getattribute__(value,name) except AttributeError: + if name.startswith('_'): + try: private = object.__getattribute__(value, '__pydantic_private__') + except AttributeError: private = None + if type(private) is dict and name in private: + child(contract, private[name], '.' + name) + continue if dataclasses.is_dataclass(self.python_type): fail(f'missing attribute {name}') continue child(contract,item,'.'+name) diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index dbfc994..578a5e7 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -135,9 +135,19 @@ def checked_init(instance, *args, **kwargs): type.__setattr__(cls, '__init__', checked_init) @staticmethod - def _dataclass_field_placeholder(frame, value): - return (type(value) is dataclasses.Field and frame.f_code.co_name != '' - and not frame.f_code.co_flags & inspect.CO_OPTIMIZED) + def _class_field_placeholder(frame, value): + if frame.f_code.co_name == '' or frame.f_code.co_flags & inspect.CO_OPTIMIZED: + return False + kind = type(value) + if kind is dataclasses.Field: + return True + for module_name, class_name in (('pydantic.fields', 'FieldInfo'), + ('pydantic.fields', 'ModelPrivateAttr'), + ('attr._make', '_CountingAttr')): + module = sys.modules.get(module_name) + if module is not None and kind is vars(module).get(class_name): + return True + return False def _track_method(self, cls, member): if type(member) is types.FunctionType: @@ -273,7 +283,7 @@ def assignment_in(self, frame, value, name, annotation=None): if contract: if name in scope.final_names: raise TypeViolation(f'{name}: Final binding cannot be reassigned') - if not self._dataclass_field_placeholder(frame, value): + if not self._class_field_placeholder(frame, value): contract.validate(value,name,bindings=scope.bindings) if contract.marker == 'Final': scope.final_names.add(name) with self._classes_lock: @@ -332,7 +342,7 @@ def check_instances(candidate): if name in values and name not in scope.contracts: scope.contracts[name] = self.contract(source,self.namespace(frame)) for name, contract in scope.contracts.items(): - if name in values and not self._dataclass_field_placeholder(frame, values[name]): + if name in values and not self._class_field_placeholder(frame, values[name]): contract.validate(values[name],name,bindings=scope.bindings) def checkpoint(self): diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 71a721a..57fa239 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -994,6 +994,27 @@ class Point: point = Point() point.values.append(1) print(point.values) +''', + 'pydantic field descriptors': '''from pydantic import BaseModel, Field +class Point(BaseModel): + x: int = Field(default=1, ge=0) + values: list[int] = Field(default_factory=list) +point = Point() +point.values.append(2) +print(point.x, point.values) +''', + 'pydantic private field': '''from pydantic import BaseModel, PrivateAttr +class Point(BaseModel): + _cache: list[int] = PrivateAttr(default_factory=list) +point = Point() +point._cache.append(2) +print(point._cache) +''', + 'attrs field descriptors': '''from attrs import define, field +@define +class Point: + x: int = field(default=1) +print(Point().x) ''', 'enum': '''from enum import Enum class Color(Enum): diff --git a/tests/test_type_safety.py b/tests/test_type_safety.py index 1ad1ae2..618917d 100644 --- a/tests/test_type_safety.py +++ b/tests/test_type_safety.py @@ -363,6 +363,28 @@ def __post_init__(self, offset): with self.assertRaises(TypeViolation): self.run_source(prelude + "item = Item(2)\nitem.values.append('bad')\n") + def test_third_party_field_descriptors_keep_instance_checks(self): + for prelude in ('''from pydantic import BaseModel, Field +class Item(BaseModel): + count: int = Field(default=1) +''', '''from attrs import define, field +@define +class Item: + count: int = field(default=1) +'''): + with self.subTest(prelude=prelude): + self.assertEqual(self.run_source(prelude + 'answer = Item().count\n')['answer'], 1) + with self.assertRaises(TypeViolation): + self.run_source(prelude + "item = Item()\nitem.count = 'bad'\n") + + private = '''from pydantic import BaseModel, PrivateAttr +class Item(BaseModel): + _cache: list[int] = PrivateAttr(default_factory=list) +''' + self.assertEqual(self.run_source(private + 'item = Item()\nanswer = item._cache\n')['answer'], []) + with self.assertRaises(TypeViolation): + self.run_source(private + "item = Item()\nitem._cache.append('bad')\n") + def test_typed_natural_language_keeps_subscript_inside_statement(self): from aiython.frontend import parse unit = parse('analysis: TicketAnalysis = analyze the ticket from ticket["message"]\n','test.py') From 3cc9b25dffa5d64451ab0eff0c25e1808fb6ea68 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 18:14:40 +0700 Subject: [PATCH 52/72] Accept arbitrary Annotated metadata without reevaluation --- docs/type-safety.md | 2 +- src/aiython/type_constraints.py | 8 ++++++-- tests/test_python_compatibility.py | 14 ++++++++++++++ tests/test_type_constraints_edges.py | 14 ++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index c137655..9b47e39 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -15,7 +15,7 @@ class TicketAnalysis(TypedDict): analysis: TicketAnalysis = analyze the current ticket ``` -`analysis` is a real Python dict. The model receives its fields, required keys, literal choices and descriptions, even when the type definition is outside the nearby source window. `Annotated` text guides the AI; it is not an executable assertion about sentence length or language. Use a registered validator for additional enforceable conditions. +`analysis` is a real Python dict. The model receives its fields, required keys, literal choices and descriptions, even when the type definition is outside the nearby source window. Literal `Annotated` text guides the AI; it is not an executable assertion about sentence length or language. Other metadata, such as Pydantic `Field(...)`, is left to its framework and is not executed again by Aiython. Use a registered validator for additional enforceable conditions. ## Enforcement diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index e8697ff..7ccd7ee 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -417,9 +417,13 @@ def node(self,node,names): raise UnsupportedType('Unsupported Literal value') return Contract('literal',ast.unparse(node),values) if base is typing.Annotated: - metadata = [ast.literal_eval(n) for n in nodes[1:]] + # Metadata is arbitrary Python data, including calls and + # framework field objects. Only literal text is a prompt hint; + # checking the wrapped type must not evaluate metadata again. + descriptions = [n.value for n in nodes[1:] + if isinstance(n, ast.Constant) and isinstance(n.value, str)] return Contract('annotated',ast.unparse(node),(self.node(nodes[0],names),), - description='; '.join(v for v in metadata if isinstance(v,str))) + description='; '.join(descriptions)) if base in TYPE_NARROWING_TYPES: if len(nodes) != 1: raise UnsupportedType('Type narrowing requires one target type') diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 57fa239..b9565ba 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1002,6 +1002,20 @@ class Point(BaseModel): point = Point() point.values.append(2) print(point.x, point.values) +''', + 'pydantic Annotated field': '''from typing import Annotated +from pydantic import BaseModel, Field +class Point(BaseModel): + x: Annotated[int, Field(ge=0)] = 2 +print(Point().x) +''', + 'Annotated metadata evaluation': '''from typing import Annotated +calls = [] +def marker(): + calls.append('called') + return object() +value: Annotated[int, marker()] = 2 +print(value, calls) ''', 'pydantic private field': '''from pydantic import BaseModel, PrivateAttr class Point(BaseModel): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 2ca9e35..1bdda84 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -728,6 +728,20 @@ def test_compiler_literals_qualifiers_and_plain_types(self): self.assertIsNone(tc.describe_output(None, {})) self.assertIsNone(tc.validate_output(1, None, {})) + def test_annotated_metadata_does_not_run_during_contract_compilation(self): + calls = [] + def metadata(): + calls.append('called') + return object() + + contract = tc.compile_contract('Annotated[int, metadata()]', + {'Annotated': Annotated, 'metadata': metadata}) + self.assertEqual(calls, []) + self.assertEqual(contract.schema()['type'], 'integer') + contract.validate(2) + with self.assertRaises(tc.TypeViolation): + contract.validate('bad') + def test_cache_evicts_oldest_and_avoids_invalid_annotation(self): cache = tc.ContractCache() self.assertEqual(cache.compile('int', {}).kind, 'int') From 049bce97f7d4fc83c405d94b9ac2f01ecc67b40e Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 18:22:23 +0700 Subject: [PATCH 53/72] Validate annotated Enum members after auto resolution --- src/aiython/type_constraints.py | 3 ++- src/aiython/typed_runtime.py | 38 ++++++++++++++++++++++++++---- tests/test_python_compatibility.py | 18 ++++++++++++++ tests/test_type_safety.py | 17 +++++++++++++ 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 7ccd7ee..64fd0aa 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -650,7 +650,8 @@ def class_contract(self,target,names,label=None): fields.update({name: (source, base) for name, source in annotations_of(base).items()}) for name, (source, owner) in fields.items(): contract = self.compile(source, scopes.get(owner, scope)) - if contract.marker == 'ClassVar' or contract.kind in ('initvar', 'kw_only'): + if (contract.marker == 'ClassVar' or contract.kind in ('initvar', 'kw_only') + or (issubclass(target, enum.Enum) and name in target.__members__)): continue result.fields[name] = contract if record: diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 578a5e7..39c20b1 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -5,6 +5,7 @@ from contextvars import ContextVar import dataclasses from dataclasses import dataclass, field +import enum import functools import inspect import sys @@ -102,12 +103,29 @@ def validate_output(self, value, annotation, frame): def register_class(self, cls): if isinstance(cls, type): self._check_dataclass_initvars(cls) + self._check_enum_members(cls) with self._classes_lock: self.classes.add(cls) for member in type.__getattribute__(cls, '__dict__').values(): self._track_method(cls, member) return cls + def _check_enum_members(self, cls): + if not issubclass(cls, enum.Enum): + return + members = vars(cls).get('_member_map_', {}) + annotations = annotations_of(cls) + namespace = Compiler.module_names(cls, {}) | {cls.__name__: cls, SELF_OWNER: cls} + for name, source in annotations.items(): + if name not in members: + continue + contract = self.contract(source, namespace) + member = members[name] + try: + contract.validate(object.__getattribute__(member, '_value_'), name) + except TypeViolation: + contract.validate(member, name) + def _check_dataclass_initvars(self, cls): # A subclass inherits this attribute even when it was not decorated. if '__dataclass_fields__' not in vars(cls): @@ -149,6 +167,12 @@ def _class_field_placeholder(frame, value): return True return False + @staticmethod + def _enum_class_body(frame, name=None): + namespace = frame.f_locals + return (isinstance(namespace, enum._EnumDict) + and (name is None or name in namespace._member_names)) + def _track_method(self, cls, member): if type(member) is types.FunctionType: methods = (member,) @@ -276,6 +300,10 @@ def assignment_in(self, frame, value, name, annotation=None): scope = next((s for s,values in candidates if name in s.declarations or name in s.contracts),candidates[0][0]) source = annotation or scope.declarations.get(name) if annotation: scope.declarations[name] = annotation + if self._class_field_placeholder(frame, value) or self._enum_class_body(frame): + if name in scope.final_names: + raise TypeViolation(f'{name}: Final binding cannot be reassigned') + return value contract = scope.contracts.get(name) if contract is None and source: contract = self.contract(source,self.namespace(frame)) @@ -283,8 +311,7 @@ def assignment_in(self, frame, value, name, annotation=None): if contract: if name in scope.final_names: raise TypeViolation(f'{name}: Final binding cannot be reassigned') - if not self._class_field_placeholder(frame, value): - contract.validate(value,name,bindings=scope.bindings) + contract.validate(value,name,bindings=scope.bindings) if contract.marker == 'Final': scope.final_names.add(name) with self._classes_lock: registered = type(value) in self.classes @@ -339,10 +366,13 @@ def check_instances(candidate): check_instances(candidate) for scope, values in self.scopes(frame): for name, source in scope.declarations.items(): - if name in values and name not in scope.contracts: + if (name in values and name not in scope.contracts + and not self._class_field_placeholder(frame, values[name]) + and not self._enum_class_body(frame, name)): scope.contracts[name] = self.contract(source,self.namespace(frame)) for name, contract in scope.contracts.items(): - if name in values and not self._class_field_placeholder(frame, values[name]): + if (name in values and not self._class_field_placeholder(frame, values[name]) + and not self._enum_class_body(frame, name)): contract.validate(values[name],name,bindings=scope.bindings) def checkpoint(self): diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index b9565ba..0eb4966 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1035,6 +1035,24 @@ class Color(Enum): RED = 1 BLUE = 2 print([item.name for item in Color]) +''', + 'annotated enum members and auto': '''from enum import Enum, IntEnum, StrEnum, Flag, auto +class Plain(Enum): + RED: int = 1 +class Number(IntEnum): + RED: int = auto() +class Text(StrEnum): + RED: str = auto() +class Bits(Flag): + RED: int = auto() +print(Plain.RED.value, Number.RED.value, Text.RED.value, Bits.RED.value) +''', + 'self annotated enum member': '''from __future__ import annotations +from enum import Enum, auto +class Color(Enum): + RED: Color = auto() +color = Color.RED +print(color.value) ''', 'pattern matching': '''value = ('ok', 3) match value: diff --git a/tests/test_type_safety.py b/tests/test_type_safety.py index 618917d..b7b70f0 100644 --- a/tests/test_type_safety.py +++ b/tests/test_type_safety.py @@ -403,6 +403,23 @@ class Level(Enum): self.assertEqual(describe_output('Level',namespace)['enum'],['high','low']) json.dumps(describe_output('Literal[Level.HIGH]',namespace)) + def test_annotated_enum_members_check_materialized_value(self): + self.assertEqual(self.run_source('''from enum import IntEnum, auto +class Number(IntEnum): + ONE: int = auto() +answer = Number.ONE.value +''')['answer'], 1) + with self.assertRaises(TypeViolation): + self.run_source('''from enum import Enum +class Item(Enum): + ONE: str = 1 +''') + with self.assertRaises(TypeViolation): + self.run_source('''from enum import auto +class Item: + one: int = auto() +''') + def test_unsupported_contract_fails_before_model_call(self): provider = Mock() with self.assertRaises(UnsupportedType): From de908d96be819283dc0eb5d8597bc4a05f28fecf Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 18:27:42 +0700 Subject: [PATCH 54/72] Defer descriptor field checks until instance access --- docs/type-safety.md | 2 +- src/aiython/typed_runtime.py | 19 ++++++++++++++++--- tests/test_python_compatibility.py | 13 +++++++++++++ tests/test_type_safety.py | 11 +++++++++++ 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 9b47e39..98e83f6 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -23,7 +23,7 @@ analysis: TicketAnalysis = analyze the current ticket - Function arguments, positional-only/keyword-only arguments, `*args`, `**kwargs`, explicit returns and implicit `None` returns are checked. Async functions use the same rules. - `**kwargs: Unpack[TypedDict]` checks required and optional keyword fields using the declared `TypedDict` contract. - The return value is checked again after `finally`, so cleanup cannot silently invalidate a return that was already checked. -- Typed class fields are checked on direct attribute assignment. Dataclass values are checked without conversion to dictionaries or reconstruction. `dataclasses.field`, Pydantic `Field`/`PrivateAttr`, and `attrs.field` descriptors are accepted while the class is built; the resulting instance values remain checked. Nested registered project instances are checked at execution boundaries. +- Typed class fields are checked on direct attribute assignment. Dataclass values are checked without conversion to dictionaries or reconstruction. `dataclasses.field`, Pydantic `Field`/`PrivateAttr`, `attrs.field`, and Python descriptors are accepted while the class is built; the resulting instance values remain checked. Nested registered project instances are checked at execution boundaries. - Mutable containers are checked deeply. Mutations through aliases are detected at statement boundaries. Enclosing scopes and globals are included; closures preserve referenced local annotation names. - Generator functions check each yielded value, sent value and final return. Async generators and `yield from` retain their control-flow protocols. - AI `evaluate`, `execute`, binding writes, terminal results and recovery replacement values use the same checker. Invalid AI output can be repaired within the bounded agent loop before assignment; completed capability side effects are retained. diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 39c20b1..c4cdee8 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -80,6 +80,7 @@ def __init__(self, manager=None): self.method_owners = weakref.WeakKeyDictionary() self._classes_lock = threading.RLock() self._contract_cache = threading.local() + self._active_instance_checks = ContextVar('aiython_active_instance_checks', default=frozenset()) self.interactive_globals = None self.interactive_scope = None @@ -159,6 +160,8 @@ def _class_field_placeholder(frame, value): kind = type(value) if kind is dataclasses.Field: return True + if inspect.getattr_static(kind, '__get__', None) is not None: + return True for module_name, class_name in (('pydantic.fields', 'FieldInfo'), ('pydantic.fields', 'ModelPrivateAttr'), ('attr._make', '_CountingAttr')): @@ -173,6 +176,16 @@ def _enum_class_body(frame, name=None): return (isinstance(namespace, enum._EnumDict) and (name is None or name in namespace._member_names)) + def _validate_instance(self, value, path, frame): + active = self._active_instance_checks.get() + if id(value) in active: + return + token = self._active_instance_checks.set(active | {id(value)}) + try: + compile_contract(original_class(value), self.namespace(frame)).validate(value, path) + finally: + self._active_instance_checks.reset(token) + def _track_method(self, cls, member): if type(member) is types.FunctionType: methods = (member,) @@ -316,7 +329,7 @@ def assignment_in(self, frame, value, name, annotation=None): with self._classes_lock: registered = type(value) in self.classes if registered: - compile_contract(original_class(value),self.namespace(frame)).validate(value,name) + self._validate_instance(value, name, frame) return value def assignment(self,value,name,annotation=None): @@ -348,11 +361,11 @@ def check_frame(self,frame): if frame.f_code.co_name == '__init__' and 'self' in frame.f_locals: seen.add(id(frame.f_locals['self'])) def check_instances(candidate): - if id(candidate) in seen: return + if id(candidate) in seen or id(candidate) in self._active_instance_checks.get(): return seen.add(id(candidate)) cls = type(candidate) if cls in classes: - compile_contract(original_class(candidate),self.namespace(frame)).validate(candidate,cls.__qualname__) + self._validate_instance(candidate, cls.__qualname__, frame) try: state = object.__getattribute__(candidate,'__dict__') except AttributeError: state = {} check_instances(state) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 0eb4966..ee11c4e 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1029,6 +1029,19 @@ class Point(BaseModel): class Point: x: int = field(default=1) print(Point().x) +''', + 'annotated Python descriptors': '''from functools import cached_property +class Field: + def __set_name__(self, owner, name): + self.name = name + def __get__(self, obj, owner=None): + return 2 +class Item: + first: int = Field() + second: int = property(lambda self: 3) + third: int = cached_property(lambda self: 4) +item = Item() +print(item.first, item.second, item.third) ''', 'enum': '''from enum import Enum class Color(Enum): diff --git a/tests/test_type_safety.py b/tests/test_type_safety.py index b7b70f0..793f97d 100644 --- a/tests/test_type_safety.py +++ b/tests/test_type_safety.py @@ -385,6 +385,17 @@ class Item(BaseModel): with self.assertRaises(TypeViolation): self.run_source(private + "item = Item()\nitem._cache.append('bad')\n") + def test_descriptor_annotation_checks_resolved_instance_value(self): + prelude = '''class Field: + def __get__(self, instance, owner=None): + return self.value +class Item: + amount: int = Field() +''' + self.assertEqual(self.run_source(prelude + 'Field.value = 2\nanswer = Item().amount\n')['answer'], 2) + with self.assertRaises(TypeViolation): + self.run_source(prelude + "Field.value = 'bad'\nitem = Item()\n") + def test_typed_natural_language_keeps_subscript_inside_statement(self): from aiython.frontend import parse unit = parse('analysis: TicketAnalysis = analyze the ticket from ticket["message"]\n','test.py') From 253c450b45cb8b69bfa5bf20055f6d629ba05d07 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 18:33:54 +0700 Subject: [PATCH 55/72] Restore embedded console exception and display state --- src/aiython/cli.py | 24 +++++++++++++++++++++++- tests/test_cli.py | 32 +++++++++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/src/aiython/cli.py b/src/aiython/cli.py index 7f9268c..6bd8590 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -252,6 +252,7 @@ def run_script(path: Path, arguments=(), *, config=None, agent_factory=None, sta old_last = ({name: vars(sys).get(name, missing_last) for name in ('last_type', 'last_value', 'last_exc', 'last_traceback')} if interactive and restore_state else {}) + old_underscore = vars(builtins).get('_', missing_last) if interactive and restore_state else missing_last interpreter_args = [*interpreter_arguments(), *(['-i'] if interactive else [])] old_spawn_entry = os.environ.get("AIYTHON_SPAWN_ENTRY") finder = ProjectFinder(runtime) @@ -299,7 +300,9 @@ def finish(): trace = trace.tb_next trace = trace or exc.__traceback__ sys.last_type = type(exc) - sys.last_value = sys.last_exc = exc.with_traceback(trace) + sys.last_value = exc.with_traceback(trace) + if sys.version_info >= (3, 12): + sys.last_exc = sys.last_value sys.last_traceback = trace runtime.types.interactive_scope.failed = False sys.excepthook(type(exc), exc, trace) @@ -316,6 +319,11 @@ def finish(): return module.__dict__ finally: if restore_state: + if interactive: + if old_underscore is missing_last: + vars(builtins).pop('_', None) + else: + builtins._ = old_underscore for name, value in old_last.items(): if value is missing_last: vars(sys).pop(name, None) @@ -362,6 +370,11 @@ def run_repl(*, config_path=None, profile=None, force_profile=None, finder = ProjectFinder(runtime) old_main = sys.modules.get('__main__') old_argv, old_orig_argv, old_path = sys.argv, sys.orig_argv, sys.path[:] + missing_last = object() + old_last = ({name: vars(sys).get(name, missing_last) + for name in ('last_type', 'last_value', 'last_exc', 'last_traceback')} + if restore_state else {}) + old_underscore = vars(builtins).get('_', missing_last) if restore_state else missing_last execution_started = None def finish(): if stats: @@ -414,6 +427,15 @@ def finish(): return module.__dict__ finally: if restore_state: + if old_underscore is missing_last: + vars(builtins).pop('_', None) + else: + builtins._ = old_underscore + for name, value in old_last.items(): + if value is missing_last: + vars(sys).pop(name, None) + else: + setattr(sys, name, value) sys.meta_path.remove(finder) sys.argv, sys.orig_argv, sys.path[:] = old_argv, old_orig_argv, old_path if old_main is None: diff --git a/tests/test_cli.py b/tests/test_cli.py index 08165d8..c1cf914 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,4 @@ +import builtins import contextlib import importlib.metadata import io @@ -61,6 +62,26 @@ def isatty(self): if not sys.flags.safe_path: self.assertEqual(path, '') + def test_embedded_repl_restores_host_exception_state(self): + class Terminal(io.StringIO): + def isatty(self): + return True + + names = ('last_type', 'last_value', 'last_exc', 'last_traceback') + previous = {name: (name in vars(sys), vars(sys).get(name)) for name in names} + previous_underscore = ('_' in vars(builtins), vars(builtins).get('_')) + output = io.StringIO() + with patch.object(sys, 'stdin', Terminal('value: int = "bad"\n' + 'import sys\n' + "print(type(getattr(sys, 'last_exc', sys.last_value)).__name__)\n" + '2\n')), \ + contextlib.redirect_stdout(output), contextlib.redirect_stderr(io.StringIO()): + run_repl(restore_state=True) + self.assertIn('TypeViolation\n', output.getvalue()) + self.assertIn('2\n', output.getvalue()) + self.assertEqual({name: (name in vars(sys), vars(sys).get(name)) for name in names}, previous) + self.assertEqual(('_' in vars(builtins), vars(builtins).get('_')), previous_underscore) + def test_repl_runs_python_startup_in_interactive_namespace(self): class Terminal(io.StringIO): def isatty(self): @@ -134,11 +155,13 @@ def isatty(self): path = Path(directory) / 'main.py' path.write_text('value: int = 2\n') output, errors = io.StringIO(), io.StringIO() + previous_underscore = ('_' in vars(builtins), vars(builtins).get('_')) previous_last = {name: (name in vars(sys), vars(sys).get(name)) for name in ('last_type', 'last_value', 'last_exc', 'last_traceback')} with patch.object(sys, 'stdin', Terminal('answer = value + 1\n' "value = 'bad'\n" - "print(answer, value, '__file__' in globals())\n")), \ + "print(answer, value, '__file__' in globals())\n" + 'answer\n')), \ contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors): namespace = run_script(path, interactive=True) self.assertEqual(namespace['answer'], 3) @@ -146,16 +169,19 @@ def isatty(self): self.assertIn('TypeViolation', errors.getvalue()) self.assertEqual({name: (name in vars(sys), vars(sys).get(name)) for name in previous_last}, previous_last) + self.assertEqual(('_' in vars(builtins), vars(builtins).get('_')), previous_underscore) def test_interactive_flag_enters_console_after_script_error(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / 'main.py' path.write_text('value = 3\nraise SystemExit(2)\n') result = subprocess.run([sys.executable, '-m', 'aiython', '-i', str(path)], - input='import sys\nprint(value, type(sys.last_exc).__name__)\n', + input=('import sys\n' + "print(value, type(getattr(sys, 'last_exc', sys.last_value)).__name__, " + "hasattr(sys, 'last_exc'))\n"), capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn('3 SystemExit\n', result.stdout) + self.assertIn(f'3 SystemExit {sys.version_info >= (3, 12)}\n', result.stdout) self.assertIn('SystemExit: 2', result.stderr) self.assertNotIn('in run_script', result.stderr) From 4658b8f1ef3942e26b95913d902f8b8a71036509 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 18:37:07 +0700 Subject: [PATCH 56/72] Accept attrs Factory defaults in annotated classes --- docs/type-safety.md | 2 +- src/aiython/typed_runtime.py | 3 ++- tests/test_python_compatibility.py | 8 ++++++++ tests/test_type_safety.py | 9 +++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/type-safety.md b/docs/type-safety.md index 98e83f6..e00b71f 100644 --- a/docs/type-safety.md +++ b/docs/type-safety.md @@ -23,7 +23,7 @@ analysis: TicketAnalysis = analyze the current ticket - Function arguments, positional-only/keyword-only arguments, `*args`, `**kwargs`, explicit returns and implicit `None` returns are checked. Async functions use the same rules. - `**kwargs: Unpack[TypedDict]` checks required and optional keyword fields using the declared `TypedDict` contract. - The return value is checked again after `finally`, so cleanup cannot silently invalidate a return that was already checked. -- Typed class fields are checked on direct attribute assignment. Dataclass values are checked without conversion to dictionaries or reconstruction. `dataclasses.field`, Pydantic `Field`/`PrivateAttr`, `attrs.field`, and Python descriptors are accepted while the class is built; the resulting instance values remain checked. Nested registered project instances are checked at execution boundaries. +- Typed class fields are checked on direct attribute assignment. Dataclass values are checked without conversion to dictionaries or reconstruction. `dataclasses.field`, Pydantic `Field`/`PrivateAttr`, `attrs.field`/`Factory`, and Python descriptors are accepted while the class is built; the resulting instance values remain checked. Nested registered project instances are checked at execution boundaries. - Mutable containers are checked deeply. Mutations through aliases are detected at statement boundaries. Enclosing scopes and globals are included; closures preserve referenced local annotation names. - Generator functions check each yielded value, sent value and final return. Async generators and `yield from` retain their control-flow protocols. - AI `evaluate`, `execute`, binding writes, terminal results and recovery replacement values use the same checker. Invalid AI output can be repaired within the bounded agent loop before assignment; completed capability side effects are retained. diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index c4cdee8..47c0b63 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -164,7 +164,8 @@ def _class_field_placeholder(frame, value): return True for module_name, class_name in (('pydantic.fields', 'FieldInfo'), ('pydantic.fields', 'ModelPrivateAttr'), - ('attr._make', '_CountingAttr')): + ('attr._make', '_CountingAttr'), + ('attr._make', 'Factory')): module = sys.modules.get(module_name) if module is not None and kind is vars(module).get(class_name): return True diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index ee11c4e..a79a08f 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1029,6 +1029,14 @@ class Point(BaseModel): class Point: x: int = field(default=1) print(Point().x) +''', + 'attrs Factory default': '''from attrs import define, Factory +@define +class Point: + values: list[int] = Factory(list) +point = Point() +point.values.append(2) +print(point.values) ''', 'annotated Python descriptors': '''from functools import cached_property class Field: diff --git a/tests/test_type_safety.py b/tests/test_type_safety.py index 793f97d..bb9a0ef 100644 --- a/tests/test_type_safety.py +++ b/tests/test_type_safety.py @@ -385,6 +385,15 @@ class Item(BaseModel): with self.assertRaises(TypeViolation): self.run_source(private + "item = Item()\nitem._cache.append('bad')\n") + factory = '''from attrs import define, Factory +@define +class Item: + values: list[int] = Factory(list) +''' + self.assertEqual(self.run_source(factory + 'answer = Item().values\n')['answer'], []) + with self.assertRaises(TypeViolation): + self.run_source(factory + "item = Item()\nitem.values.append('bad')\n") + def test_descriptor_annotation_checks_resolved_instance_value(self): prelude = '''class Field: def __get__(self, instance, owner=None): From 420daba1d19a317d5d53a0dcc2d81f7a4ffa39f2 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 18:43:19 +0700 Subject: [PATCH 57/72] Preserve evaluation order for non-name annotations --- src/aiython/typed_runtime.py | 16 ++++++++++++++-- tests/test_python_compatibility.py | 24 ++++++++++++++++++++++++ tests/test_type_safety.py | 9 +++++++++ tests/test_typed_runtime_edges.py | 10 ---------- 4 files changed, 47 insertions(+), 12 deletions(-) diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 47c0b63..2cb50d7 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -700,8 +700,20 @@ def visit_AnnAssign(self,node): if isinstance(node.target,ast.Name): node.value = self.helper('assignment',self.visit(node.value),ast.Constant(node.target.id),ast.Constant(annotation)) return node - node.value = self.helper('expression',self.visit(node.value),ast.Constant(annotation)) - return node + if self.function and isinstance(node.target, ast.Attribute): + assignment = ast.Call( + ast.Attribute(ast.Attribute(ast.Name(self.runtime_name, ast.Load()), 'types', ast.Load()), + 'assign_attribute', ast.Load()), [], [ + ast.keyword(arg='value', value=self.visit(node.value)), + ast.keyword(arg='owner', value=self.visit(node.target.value)), + ast.keyword(arg='name', value=ast.Constant(node.target.attr)), + ast.keyword(arg='annotation', value=ast.Constant(annotation)), + ]) + return ast.copy_location(ast.Expr(assignment), node) + # Python evaluates a non-name target before its annotation. Function + # scopes do not evaluate these annotations at all. Let CPython keep + # those rules for attribute and subscript assignments. + return self.generic_visit(node) def visit_Assign(self,node): node.value = self.visit(node.value) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index a79a08f..0e589e5 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -845,6 +845,30 @@ def worker(): ''', 'module annotation metadata': '''value: int = 2 print('__annotations__' in globals(), '__annotate__' in globals()) +''', + 'non-name annotation evaluation order': '''events = [] +class Target: + def __setitem__(self, key, value): + events.append('set') +target = Target() +def key(): + events.append('key') + return 0 +def annotation(): + events.append('annotation') + return int +target[key()]: annotation() = 2 +def fail(): + try: + missing[also_missing]: 1/0 = 0 + except NameError as error: + print(type(error).__name__) + try: + missing.attr: 1/0 = 0 + except NameError as error: + print(type(error).__name__) +fail() +print(events) ''', 'metaclass namespace': '''class Meta(type): def __new__(meta, name, bases, namespace): diff --git a/tests/test_type_safety.py b/tests/test_type_safety.py index bb9a0ef..a302360 100644 --- a/tests/test_type_safety.py +++ b/tests/test_type_safety.py @@ -54,6 +54,15 @@ def __init__(self): ''') self.assertEqual(result['answer'],10) + def test_function_attribute_annotation_checks_after_target_lookup(self): + with self.assertRaises(TypeViolation): + self.run_source('''class Box: + pass +def set_value(box): + box.value: int = 'bad' +set_value(Box()) +''') + def test_parameter_variants_async_and_typevar(self): self.assertEqual(self.run_source('''import asyncio from typing import TypeVar diff --git a/tests/test_typed_runtime_edges.py b/tests/test_typed_runtime_edges.py index 53cb357..30dc2ea 100644 --- a/tests/test_typed_runtime_edges.py +++ b/tests/test_typed_runtime_edges.py @@ -153,16 +153,6 @@ def test_bare_return_annotation_only_and_named_expression(self): self.assertIn('returned', ast.unparse(transformed)) self.assertIn('assignment', ast.unparse(transformed)) - def test_non_name_annotated_target_uses_expression_validation(self): - tree = ast.parse('''class Sample: - value: int -sample = Sample() -sample.value: int = 2 -''') - transformed = TypedTransformer().visit(tree) - ast.fix_missing_locations(transformed) - self.assertIn('expression', ast.unparse(transformed)) - def test_expected_type_flows_through_conditional_expression(self): blocks = {'first': SimpleNamespace(output_type=None), 'second': SimpleNamespace(output_type=None)} From 0f6b93d8d7b0704f1b5e887e06009e5f60512274 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 18:53:01 +0700 Subject: [PATCH 58/72] Release generator frames promptly and preserve function name updates --- src/aiython/typed_runtime.py | 44 ++++++++++++++++++------------ tests/test_python_compatibility.py | 28 +++++++++++++++++++ 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 2cb50d7..08a1299 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -354,6 +354,25 @@ def expression(self,value,annotation): return value finally: del frame + def _check_instances(self, candidate, classes, seen, frame): + if id(candidate) in seen or id(candidate) in self._active_instance_checks.get(): + return + seen.add(id(candidate)) + cls = type(candidate) + if cls in classes: + self._validate_instance(candidate, cls.__qualname__, frame) + try: + state = object.__getattribute__(candidate, '__dict__') + except AttributeError: + state = {} + self._check_instances(state, classes, seen, frame) + elif cls in (list, tuple, set, frozenset): + for item in candidate: + self._check_instances(item, classes, seen, frame) + elif cls is dict: + for item in candidate.values(): + self._check_instances(item, classes, seen, frame) + def check_frame(self,frame): with self._classes_lock: classes = frozenset(self.classes) @@ -361,23 +380,10 @@ def check_frame(self,frame): seen = set() if frame.f_code.co_name == '__init__' and 'self' in frame.f_locals: seen.add(id(frame.f_locals['self'])) - def check_instances(candidate): - if id(candidate) in seen or id(candidate) in self._active_instance_checks.get(): return - seen.add(id(candidate)) - cls = type(candidate) - if cls in classes: - self._validate_instance(candidate, cls.__qualname__, frame) - try: state = object.__getattribute__(candidate,'__dict__') - except AttributeError: state = {} - check_instances(state) - elif cls in (list,tuple,set,frozenset): - for item in candidate: check_instances(item) - elif cls is dict: - for item in candidate.values(): check_instances(item) for namespace in (frame.f_locals,frame.f_globals): for name,candidate in namespace.items(): if not name.startswith('__'): - check_instances(candidate) + self._check_instances(candidate, classes, seen, frame) for scope, values in self.scopes(frame): for name, source in scope.declarations.items(): if (name in values and name not in scope.contracts @@ -501,10 +507,12 @@ def assign_attribute(self,owner,name,value,annotation=None): for base in reversed(target.__mro__): fields.update(annotations_of(base)) source = fields.get(name) - namespace = Compiler.module_names(target,self.namespace(frame)) - namespace[SELF_OWNER] = target - namespace.update({p.__name__:p for p in (getattr(target,'__type_params__',()) or getattr(target,'__parameters__',()))}) - contract = compile_contract(source,namespace) if source else None + contract = None + if source: + namespace = Compiler.module_names(target,self.namespace(frame)) + namespace[SELF_OWNER] = target + namespace.update({p.__name__:p for p in (getattr(target,'__type_params__',()) or getattr(target,'__parameters__',()))}) + contract = compile_contract(source,namespace) if contract: if contract.marker == 'ClassVar' and not issubclass(type(owner), type): raise TypeViolation(f'{name}: ClassVar must be assigned on the class') diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 0e589e5..7e1fe84 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1121,6 +1121,34 @@ def outer(): print(next(item)) try: item.send(7) except StopIteration: pass +''', + 'generator function name changes': '''def generate(): + yield 1 +item = generate() +item.__name__ = 'item_name' +item.__qualname__ = 'item_qualname' +generate.__name__ = 'generate_name' +generate.__qualname__ = 'generate_qualname' +new_item = generate() +print(item.__name__, item.__qualname__, new_item.__name__, new_item.__qualname__) +''', + 'generator close releases arguments': '''class DetectDelete: + def __init__(self): + DetectDelete.deleted = False + def __del__(self): + DetectDelete.deleted = True +def generate(arg): + yield +item = generate(DetectDelete()) +item.close() +print(DetectDelete.deleted, item.gi_frame is None) +item = generate(DetectDelete()) +next(item) +item.close() +print(DetectDelete.deleted, item.gi_frame is None) +item = generate(DetectDelete()) +item.gi_frame.clear() +print(DetectDelete.deleted, item.gi_frame is None) ''', 'generator delegation identity': '''from collections.abc import Generator def child(): From 7403eb2b53e8489d6a3e35866d264ffb9e26249c Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 19:08:58 +0700 Subject: [PATCH 59/72] Respect descriptor-backed dataclass fields during type checks --- src/aiython/type_constraints.py | 31 ++++++++++++++++++++++++++++++ src/aiython/typed_runtime.py | 15 +++++++++------ tests/test_python_compatibility.py | 24 +++++++++++++++++++++++ tests/test_typed_runtime_edges.py | 15 +++++++++++++++ 4 files changed, 79 insertions(+), 6 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 64fd0aa..caa72a0 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -81,6 +81,35 @@ def annotations_of(target): return inspect.get_annotations(target, eval_str=False) +def descriptor_field(target, name, source, namespace=None): + """Whether an annotation describes the descriptor stored on the class.""" + try: + descriptor = inspect.getattr_static(target, name) + except AttributeError: + return False + kind = type(descriptor) + if not any(inspect.getattr_static(kind, method, None) is not None + for method in ('__get__', '__set__', '__delete__')): + return False + if source is kind: + return True + if not isinstance(source, str): + return False + try: + annotation = ast.parse(source, mode='eval').body + if isinstance(annotation, ast.Constant) and isinstance(annotation.value, str): + annotation = ast.parse(annotation.value, mode='eval').body + except SyntaxError: + return False + if isinstance(annotation, ast.Name): + if namespace is not None and annotation.id in namespace: + return namespace[annotation.id] is kind + return annotation.id == kind.__name__ + if isinstance(annotation, ast.Attribute): + return ast.unparse(annotation) == f'{kind.__module__}.{kind.__name__}' + return False + + @dataclass class Contract: kind: str @@ -649,6 +678,8 @@ def class_contract(self,target,names,label=None): if base in (object,dict): continue fields.update({name: (source, base) for name, source in annotations_of(base).items()}) for name, (source, owner) in fields.items(): + if descriptor_field(target, name, source, scopes.get(owner, scope)): + continue contract = self.compile(source, scopes.get(owner, scope)) if (contract.marker == 'ClassVar' or contract.kind in ('initvar', 'kw_only') or (issubclass(target, enum.Enum) and name in target.__members__)): diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 08a1299..93f0e48 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -16,7 +16,7 @@ from .frontend import RUNTIME_NAME from .type_constraints import (Contract, ContractCache, TypeViolation, compile_contract, - annotations_of, Compiler, SELF_OWNER) + annotations_of, descriptor_field, Compiler, SELF_OWNER) SCOPE = '__aiython_type_scope__' _FRAME_SCOPES = ContextVar('aiython_frame_scopes', default=()) @@ -160,7 +160,8 @@ def _class_field_placeholder(frame, value): kind = type(value) if kind is dataclasses.Field: return True - if inspect.getattr_static(kind, '__get__', None) is not None: + if any(inspect.getattr_static(kind, method, None) is not None + for method in ('__get__', '__set__', '__delete__')): return True for module_name, class_name in (('pydantic.fields', 'FieldInfo'), ('pydantic.fields', 'ModelPrivateAttr'), @@ -509,10 +510,12 @@ def assign_attribute(self,owner,name,value,annotation=None): source = fields.get(name) contract = None if source: - namespace = Compiler.module_names(target,self.namespace(frame)) - namespace[SELF_OWNER] = target - namespace.update({p.__name__:p for p in (getattr(target,'__type_params__',()) or getattr(target,'__parameters__',()))}) - contract = compile_contract(source,namespace) + scope = self.namespace(frame) + if not descriptor_field(target, name, source, scope): + namespace = Compiler.module_names(target, scope) + namespace[SELF_OWNER] = target + namespace.update({p.__name__:p for p in (getattr(target,'__type_params__',()) or getattr(target,'__parameters__',()))}) + contract = compile_contract(source,namespace) if contract: if contract.marker == 'ClassVar' and not issubclass(type(owner), type): raise TypeViolation(f'{name}: ClassVar must be assigned on the class') diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 7e1fe84..a84e9a8 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1018,6 +1018,30 @@ class Point: point = Point() point.values.append(1) print(point.values) +''', + 'dataclass descriptor with local annotation': '''from dataclasses import dataclass +def check(): + class Descriptor: + def __get__(self, instance, owner=None): + return 100 if instance is None else instance._value + def __set__(self, instance, value): + instance._value = value + @dataclass + class Item: + value: Descriptor = Descriptor() + first, second = Item(), Item(5) + second.value = 7 + print(first.value, second.value) + class Setter: + calls = [] + def __set__(self, instance, value): + self.calls.append(value) + @dataclass + class SetOnly: + value: Setter = Setter() + SetOnly(3) + print(SetOnly.value.calls) +check() ''', 'pydantic field descriptors': '''from pydantic import BaseModel, Field class Point(BaseModel): diff --git a/tests/test_typed_runtime_edges.py b/tests/test_typed_runtime_edges.py index 30dc2ea..6661f92 100644 --- a/tests/test_typed_runtime_edges.py +++ b/tests/test_typed_runtime_edges.py @@ -61,6 +61,21 @@ def __init__(self): self.value = 2 self.runtime.check_frame(inspect.currentframe()) self.assertEqual(items[0].value, 2) + def test_annotation_without_class_descriptor_still_checks_instance(self): + class Descriptor: + def __set__(self, instance, value): + pass + + class Item: + value: Descriptor + + item = Item() + with self.assertRaises(TypeViolation): + self.runtime.assign_attribute(item, 'value', 5) + value = Descriptor() + self.runtime.assign_attribute(item, 'value', value) + self.assertIs(item.value, value) + def test_return_yield_abort_leave_and_send(self): self.assertEqual(self.runtime.returned(3), 3) self.assertEqual(self.runtime.yielded(3), 3) From bb8fff231ea26d0b2ce73de52752b0b1847e2aff Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 19:13:15 +0700 Subject: [PATCH 60/72] Preserve dataclass construction and decorated class source lines --- src/aiython/type_constraints.py | 8 +++++++- src/aiython/typed_runtime.py | 7 +++++-- tests/test_python_compatibility.py | 32 ++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index caa72a0..f9106cc 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -369,7 +369,13 @@ def child(contract, item, suffix): if type(private) is dict and name in private: child(contract, private[name], '.' + name) continue - if dataclasses.is_dataclass(self.python_type): fail(f'missing attribute {name}') + if dataclasses.is_dataclass(self.python_type): + params = getattr(self.python_type, '__dataclass_params__', None) + field = getattr(self.python_type, '__dataclass_fields__', {}).get(name) + if (params is None or params.init) and (field is None or + (field.init and field.default is dataclasses.MISSING and + field.default_factory is dataclasses.MISSING)): + fail(f'missing attribute {name}') continue child(contract,item,'.'+name) elif kind == 'type': diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 93f0e48..ad3dc8c 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -379,7 +379,7 @@ def check_frame(self,frame): classes = frozenset(self.classes) if classes: seen = set() - if frame.f_code.co_name == '__init__' and 'self' in frame.f_locals: + if frame.f_code.co_name in ('__init__', '__setstate__') and 'self' in frame.f_locals: seen.add(id(frame.f_locals['self'])) for namespace in (frame.f_locals,frame.f_globals): for name,candidate in namespace.items(): @@ -662,7 +662,10 @@ def function_nodes(node): yield from TypedTransformer.function_nodes(child) def visit_ClassDef(self,node): - node.decorator_list.insert(0, ast.Attribute(ast.Attribute(ast.Name(self.runtime_name,ast.Load()),'types',ast.Load()),'register_class',ast.Load())) + first = node.decorator_list[0] if node.decorator_list else node + register = ast.Attribute(ast.Attribute(ast.Name(self.runtime_name,ast.Load()),'types',ast.Load()), + 'register_class',ast.Load()) + node.decorator_list.insert(0, ast.copy_location(register, first)) previous = self.function previous_contract = self.delegation_contract self.function = False diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index a84e9a8..4f4380f 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1018,6 +1018,30 @@ class Point: point = Point() point.values.append(1) print(point.values) +''', + 'dataclass without generated init': '''from dataclasses import dataclass +@dataclass(slots=True, init=False) +class Point: + x: int +point = Point() +print(hasattr(point, 'x')) +point.x = 3 +print(point.x) +''', + 'slotted dataclass custom pickle state': '''from dataclasses import dataclass, field +import pickle +@dataclass(frozen=True, slots=True) +class Point: + x: int + cached: bool = field(default=False, compare=False) + restored: bool = field(default=False, compare=False) + def __getstate__(self): + return [self.x] + def __setstate__(self, state): + object.__setattr__(self, 'x', state[0]) + object.__setattr__(self, 'restored', True) +point = pickle.loads(pickle.dumps(Point(2))) +print(point.x, point.restored, hasattr(point, 'cached')) ''', 'dataclass descriptor with local annotation': '''from dataclasses import dataclass def check(): @@ -1469,6 +1493,14 @@ def __init__(self, left: T, right: U): print(pair.left, pair.right) ''' if sys.version_info >= (3, 14): + cases['decorated class source line'] = '''from dataclasses import dataclass +def make(): + @dataclass(slots=True) + class Item: + value: int + return Item +print(make().__firstlineno__ - make.__code__.co_firstlineno) +''' cases['template string interpolation'] = '''events = [] def pick(): events.append('called') From 21041338b752618b8ba157f54a3ac43cb69d3664 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 19:17:31 +0700 Subject: [PATCH 61/72] Resolve type(None) safely in annotations --- src/aiython/type_constraints.py | 5 +++++ tests/test_python_compatibility.py | 7 +++++++ tests/test_type_constraints_edges.py | 9 +++++++++ 3 files changed, 21 insertions(+) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index f9106cc..862cd54 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -401,6 +401,11 @@ def __init__(self, namespace): self.cache = {} def lookup(self, node, names): + if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name) + and node.func.id == 'type' and names.get('type') is type + and len(node.args) == 1 and isinstance(node.args[0], ast.Constant) + and node.args[0].value is None and not node.keywords): + return type(None) if isinstance(node,ast.Name): if node.id not in names: raise UnsupportedType(f'Unresolved output type: {node.id}') diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 4f4380f..db12959 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1018,6 +1018,13 @@ class Point: point = Point() point.values.append(1) print(point.values) +''', + 'dataclass union with NoneType': '''from dataclasses import dataclass +from typing import Union +@dataclass +class Item: + value: Union[int, type(None)] = None +print(Item.__doc__, Item().value) ''', 'dataclass without generated init': '''from dataclasses import dataclass @dataclass(slots=True, init=False) diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 1bdda84..a376671 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -569,6 +569,15 @@ def test_type_contracts_accept_unions_aliases_and_typevars(self): value_contract.validate('first', bindings=reverse_bindings) class_contract.validate(str, bindings=reverse_bindings) + def test_type_none_annotation_call_is_safe_and_respects_shadowing(self): + contract = tc.compile_contract('typing.Union[int, type(None)]', {'typing': typing}) + contract.validate(None) + contract.validate(2) + with self.assertRaises(tc.TypeViolation): + contract.validate('wrong') + with self.assertRaisesRegex(tc.UnsupportedType, 'annotation calls are not executed'): + tc.compile_contract('type(None)', {'type': lambda value: value}) + def test_class_custom_validator_and_missing_field(self): class Choice: value: int From a749a7776af6a3cf61cb8d4a97fbe83996577a8e Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 19:22:28 +0700 Subject: [PATCH 62/72] Resolve class local annotation aliases and bare InitVar --- src/aiython/type_constraints.py | 10 ++++++++-- tests/test_python_compatibility.py | 14 ++++++++++++++ tests/test_type_constraints_edges.py | 20 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 862cd54..53f75fa 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -644,7 +644,9 @@ def class_contract(self,target,names,label=None): result = Contract('typeddict' if record else 'class',label or target.__qualname__,python_type=target) self.cache[key] = result if target in VALIDATORS: return result - scope = self.module_names(target,names) | {target.__name__:target, SELF_OWNER:target} + scope = (self.module_names(target,names) | + dict(type.__getattribute__(target, '__dict__')) | + {target.__name__:target, SELF_OWNER:target}) scope.update({p.__name__:p for p in (getattr(target,'__type_params__',()) or getattr(target,'__parameters__',()))}) # isinstance can call a user's __getattribute__('__class__') here. scope.update({k:v for k,v in names.items() @@ -662,7 +664,9 @@ def class_contract(self,target,names,label=None): base = typing.get_origin(original) or original if not isinstance(base, type) or base in scopes or base not in current.__bases__: continue - base_scope = self.module_names(base, current_scope) | {base.__name__: base} + base_scope = (self.module_names(base, current_scope) | + dict(type.__getattribute__(base, '__dict__')) | + {base.__name__: base}) parameters = getattr(base, '__type_params__', ()) or getattr(base, '__parameters__', ()) arguments = typing.get_args(original) if parameters and arguments: @@ -724,6 +728,8 @@ def value(self,target,names): return Contract('any', 'TypeAlias') if target is dataclasses.KW_ONLY: return Contract('kw_only', 'KW_ONLY') + if target is dataclasses.InitVar: + return Contract('initvar', 'InitVar', (Contract('any', 'Any'),)) if isinstance(target, dataclasses.InitVar): return Contract('initvar', str(target), (self.compile(target.type, names),)) diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index db12959..1bedb72 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1025,6 +1025,20 @@ class Point: class Item: value: Union[int, type(None)] = None print(Item.__doc__, Item().value) +''', + 'dataclass class local markers': '''from dataclasses import dataclass, InitVar +from typing import ClassVar +@dataclass +class Item: + ClassMarker = ClassVar + InitMarker = InitVar + marker: ClassMarker = 1 + input: InitMarker + value: int = 0 + def __post_init__(self, input): + self.value = input +item = Item(2) +print(item.marker, item.value, 'input' in vars(item)) ''', 'dataclass without generated init': '''from dataclasses import dataclass @dataclass(slots=True, init=False) diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index a376671..1b858fa 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -578,6 +578,26 @@ def test_type_none_annotation_call_is_safe_and_respects_shadowing(self): with self.assertRaisesRegex(tc.UnsupportedType, 'annotation calls are not executed'): tc.compile_contract('type(None)', {'type': lambda value: value}) + def test_class_local_annotation_aliases_override_caller_names(self): + class Item: + Element = int + value: 'Element' + + class Child(Item): + pass + + for cls in (Item, Child): + with self.subTest(cls=cls): + contract = tc.compile_contract(cls, {'Element': str}) + good = cls() + good.value = 2 + contract.validate(good) + bad = cls() + bad.value = 'wrong' + with self.assertRaises(tc.TypeViolation): + contract.validate(bad) + self.assertEqual(tc.compile_contract(tc.dataclasses.InitVar, {}).kind, 'initvar') + def test_class_custom_validator_and_missing_field(self): class Choice: value: int From 549fd435692f5989835a042306feae63f85dbccc Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 19:25:31 +0700 Subject: [PATCH 63/72] Accept whitespace in string type annotations --- src/aiython/type_constraints.py | 6 +++--- tests/test_python_compatibility.py | 12 +++++++++++- tests/test_type_constraints_edges.py | 5 +++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 53f75fa..8e49ca0 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -96,9 +96,9 @@ def descriptor_field(target, name, source, namespace=None): if not isinstance(source, str): return False try: - annotation = ast.parse(source, mode='eval').body + annotation = ast.parse(source.strip(), mode='eval').body if isinstance(annotation, ast.Constant) and isinstance(annotation.value, str): - annotation = ast.parse(annotation.value, mode='eval').body + annotation = ast.parse(annotation.value.strip(), mode='eval').body except SyntaxError: return False if isinstance(annotation, ast.Name): @@ -782,7 +782,7 @@ def value(self,target,names): def annotation_node(annotation): # Compiler.node only reads this tree. Namespace resolution still happens on # every compile, including forward references and mutable class annotations. - return ast.parse(annotation, mode='eval').body + return ast.parse(annotation.strip(), mode='eval').body class ContractCache: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 1bedb72..1ad622f 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1039,6 +1039,16 @@ def __post_init__(self, input): self.value = input item = Item(2) print(item.marker, item.value, 'input' in vars(item)) +''', + 'dataclass string annotation whitespace': '''from dataclasses import dataclass, InitVar +@dataclass +class Item: + value: ' int ' = 1 + extra: ' InitVar ' = 2 + def __post_init__(self, extra): + self.value += extra +item = Item() +print(item.value, hasattr(item, 'extra')) ''', 'dataclass without generated init': '''from dataclasses import dataclass @dataclass(slots=True, init=False) @@ -1073,7 +1083,7 @@ def __set__(self, instance, value): instance._value = value @dataclass class Item: - value: Descriptor = Descriptor() + value: ' Descriptor ' = Descriptor() first, second = Item(), Item(5) second.value = 7 print(first.value, second.value) diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 1b858fa..0abaa16 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -578,6 +578,11 @@ def test_type_none_annotation_call_is_safe_and_respects_shadowing(self): with self.assertRaisesRegex(tc.UnsupportedType, 'annotation calls are not executed'): tc.compile_contract('type(None)', {'type': lambda value: value}) + def test_string_annotations_allow_surrounding_whitespace(self): + tc.compile_contract(' int ', {}).validate(2) + self.assertEqual(tc.compile_contract(' dataclasses.InitVar [ int ] ', + {'dataclasses': tc.dataclasses}).kind, 'initvar') + def test_class_local_annotation_aliases_override_caller_names(self): class Item: Element = int From 3b10fb6856e22a6fee903bfe59dbd8aff4153255 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 19:36:17 +0700 Subject: [PATCH 64/72] Resolve lexical class annotations and validate completed dataclasses --- src/aiython/type_constraints.py | 47 ++++++++++++++++--- src/aiython/typed_runtime.py | 28 +++++++++-- tests/test_python_compatibility.py | 69 ++++++++++++++++++++++++++++ tests/test_type_constraints_edges.py | 18 ++++++++ 4 files changed, 151 insertions(+), 11 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index 8e49ca0..a606734 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -81,6 +81,39 @@ def annotations_of(target): return inspect.get_annotations(target, eval_str=False) +def class_parameters(target): + if not isinstance(target, type): + return () + for name in ('__type_params__', '__parameters__'): + try: + parameters = type.__getattribute__(target, name) + except AttributeError: + continue + if isinstance(parameters, tuple) and parameters: + return parameters + return () + + +def annotation_locals(target): + if not isinstance(target, type): + return {} + try: + annotator = type.__getattribute__(target, '__annotate__') + except AttributeError: + return {} + if not isinstance(annotator, types.FunctionType): + return {} + names = {} + for name, cell in zip(annotator.__code__.co_freevars, annotator.__closure__ or ()): + if name == '__classdict__': + continue + try: + names[name] = cell.cell_contents + except ValueError: + pass + return names + + def descriptor_field(target, name, source, namespace=None): """Whether an annotation describes the descriptor stored on the class.""" try: @@ -564,8 +597,7 @@ def generic(self,base,args,label,names): raise UnsupportedType('type requires one parameter') return Contract('type',label,args,python_type=type) if isinstance(base,TYPE_ALIAS_TYPES): return self.alias(base,names,args,label) - if isinstance(base,type) and (getattr(base,'__type_params__',()) or getattr(base,'__parameters__',())): - parameters = getattr(base,'__type_params__',()) or base.__parameters__ + if isinstance(base,type) and (parameters := class_parameters(base)): scope = self.module_names(base,names) bindings = self.parameter_bindings(parameters, args, scope, 'Generic type argument count mismatch') @@ -631,8 +663,8 @@ def alias(self,alias,names,args=(),label=None): @staticmethod def module_names(target,names): module = sys.modules.get(getattr(target,'__module__','')) - # Defining module names win over unrelated caller aliases. - return names | (vars(module) if module else {}) + # Defining module and annotation closure names win over caller aliases. + return names | (vars(module) if module else {}) | annotation_locals(target) def class_contract(self,target,names,label=None): key = (id(target),label or target.__qualname__) @@ -647,7 +679,7 @@ def class_contract(self,target,names,label=None): scope = (self.module_names(target,names) | dict(type.__getattribute__(target, '__dict__')) | {target.__name__:target, SELF_OWNER:target}) - scope.update({p.__name__:p for p in (getattr(target,'__type_params__',()) or getattr(target,'__parameters__',()))}) + scope.update({p.__name__:p for p in class_parameters(target)}) # isinstance can call a user's __getattribute__('__class__') here. scope.update({k:v for k,v in names.items() if (issubclass(type(v), Contract) or @@ -667,7 +699,7 @@ def class_contract(self,target,names,label=None): base_scope = (self.module_names(base, current_scope) | dict(type.__getattribute__(base, '__dict__')) | {base.__name__: base}) - parameters = getattr(base, '__type_params__', ()) or getattr(base, '__parameters__', ()) + parameters = class_parameters(base) arguments = typing.get_args(original) if parameters and arguments: compiled = [] @@ -692,7 +724,10 @@ def class_contract(self,target,names,label=None): for base in reversed(target.__mro__): if base in (object,dict): continue fields.update({name: (source, base) for name, source in annotations_of(base).items()}) + dataclass_fields = getattr(target, '__dataclass_fields__', None) if dataclasses.is_dataclass(target) else None for name, (source, owner) in fields.items(): + if dataclass_fields is not None and name not in dataclass_fields: + continue if descriptor_field(target, name, source, scopes.get(owner, scope)): continue contract = self.compile(source, scopes.get(owner, scope)) diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index ad3dc8c..c4a76f9 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -16,7 +16,7 @@ from .frontend import RUNTIME_NAME from .type_constraints import (Contract, ContractCache, TypeViolation, compile_contract, - annotations_of, descriptor_field, Compiler, SELF_OWNER) + annotations_of, class_parameters, descriptor_field, Compiler, SELF_OWNER) SCOPE = '__aiython_type_scope__' _FRAME_SCOPES = ContextVar('aiython_frame_scopes', default=()) @@ -226,7 +226,18 @@ def method_self_owner(self, frame, *, discover=False): @staticmethod def namespace(frame): - namespace = dict(frame.f_globals) | dict(frame.f_locals) + namespace = dict(frame.f_globals) + if frame.f_code.co_name != '' and not frame.f_code.co_flags & inspect.CO_OPTIMIZED: + parents = [] + parent = frame.f_back + while parent is not None and parent.f_code.co_filename == frame.f_code.co_filename: + if (parent.f_code.co_flags & inspect.CO_OPTIMIZED + and frame.f_code.co_qualname.startswith(parent.f_code.co_qualname + '..')): + parents.append(dict(parent.f_locals)) + parent = parent.f_back + for local in reversed(parents): + namespace.update(local) + namespace.update(frame.f_locals) scope = frame_scope(frame) or frame.f_locals.get(SCOPE) if isinstance(scope,Scope): namespace.update({parameter.__name__: bound for parameter,bound in scope.bindings.items()}) @@ -379,8 +390,15 @@ def check_frame(self,frame): classes = frozenset(self.classes) if classes: seen = set() - if frame.f_code.co_name in ('__init__', '__setstate__') and 'self' in frame.f_locals: - seen.add(id(frame.f_locals['self'])) + if 'self' in frame.f_locals: + receiver = frame.f_locals['self'] + parent = frame + while parent is not None: + if (parent.f_code.co_name in ('__init__', '__setstate__') + and parent.f_locals.get('self') is receiver): + seen.add(id(receiver)) + break + parent = parent.f_back for namespace in (frame.f_locals,frame.f_globals): for name,candidate in namespace.items(): if not name.startswith('__'): @@ -514,7 +532,7 @@ def assign_attribute(self,owner,name,value,annotation=None): if not descriptor_field(target, name, source, scope): namespace = Compiler.module_names(target, scope) namespace[SELF_OWNER] = target - namespace.update({p.__name__:p for p in (getattr(target,'__type_params__',()) or getattr(target,'__parameters__',()))}) + namespace.update({p.__name__:p for p in class_parameters(target)}) contract = compile_contract(source,namespace) if contract: if contract.marker == 'ClassVar' and not issubclass(type(owner), type): diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 1ad622f..4b81aa0 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1097,6 +1097,75 @@ class SetOnly: SetOnly(3) print(SetOnly.value.calls) check() +''', + 'dataclass local type alias before decoration': '''from dataclasses import dataclass +def check(): + alias = list + try: + @dataclass + class Item: + values: alias = [] + except ValueError as error: + print(type(error).__name__, 'mutable default' in str(error)) +check() +''', + 'class annotation ignores dynamic caller alias': '''from dataclasses import dataclass +class Expected: + pass +def make(): + @dataclass + class Item: + value: Expected = Expected() + return Item() +def caller(): + Expected = str + return make() +print(type(caller().value).__name__) +''', + 'dataclass escaped local type': '''from dataclasses import dataclass +def make(): + class Local: + pass + @dataclass + class Box: + value: Local + return Box(Local()) +box = make() +box.value = type(box.value)() +print(type(box.value).__name__) +''', + 'dataclass intermediate non-dataclass base': '''from dataclasses import dataclass +@dataclass +class Base: + x: int +class Middle(Base): + y: int +@dataclass +class Child(Middle): + z: int +child = Child(1, 2) +print(child.x, child.z, hasattr(child, 'y')) +''', + 'dataclass generated init calls custom setter': '''from dataclasses import dataclass +calls = [] +@dataclass +class Pair: + left: int + right: int + def __setattr__(self, name, value): + calls.append((name, value)) + object.__setattr__(self, name, value) +pair = Pair(1, 2) +print(calls, pair.left, pair.right) +''', + 'dataclass subclass of GenericAlias': '''from dataclasses import dataclass +import types +@dataclass +class Alias(types.GenericAlias): + origin: type + args: type +alias = Alias(list, int) +print(alias.__origin__.__name__, alias.__args__[0].__name__) ''', 'pydantic field descriptors': '''from pydantic import BaseModel, Field class Point(BaseModel): diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index 0abaa16..ab422e0 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -603,6 +603,24 @@ class Child(Item): contract.validate(bad) self.assertEqual(tc.compile_contract(tc.dataclasses.InitVar, {}).kind, 'initvar') + def test_escaped_class_keeps_lexical_annotation_type(self): + def make(): + class Local: + pass + + class Holder: + value: Local + + return Holder, Local + + holder_type, value_type = make() + holder = holder_type() + holder.value = value_type() + tc.compile_contract(holder_type, {}).validate(holder) + holder.value = object() + with self.assertRaises(tc.TypeViolation): + tc.compile_contract(holder_type, {}).validate(holder) + def test_class_custom_validator_and_missing_field(self): class Choice: value: int From 5ee1aaed474ad58d79b22c45051b45b4ed30be1c Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 19:58:00 +0700 Subject: [PATCH 65/72] Resolve generic parameter scopes and skip empty class scans --- src/aiython/type_constraints.py | 23 +++++++++++-- src/aiython/typed_runtime.py | 53 ++++++++++++++++++++++++++++-- tests/test_python_compatibility.py | 18 ++++++++++ tests/test_type_safety.py | 20 +++++++++++ 4 files changed, 109 insertions(+), 5 deletions(-) diff --git a/src/aiython/type_constraints.py b/src/aiython/type_constraints.py index a606734..1633dde 100644 --- a/src/aiython/type_constraints.py +++ b/src/aiython/type_constraints.py @@ -501,7 +501,16 @@ def node(self,node,names): if len(nodes) != 1: raise UnsupportedType('Type narrowing requires one target type') return Contract('bool',ast.unparse(node),python_type=bool) - args = tuple(Ellipsis if isinstance(n,ast.Constant) and n.value is Ellipsis else self.node(n,names) for n in nodes) + accepts_parameter_list = (typing.get_origin(base) or base) is abc.Callable or ( + isinstance(typing.get_origin(base) or base, type) and + any(isinstance(parameter, typing.ParamSpec) + for parameter in class_parameters(typing.get_origin(base) or base))) + args = tuple( + Ellipsis if isinstance(n,ast.Constant) and n.value is Ellipsis else + Contract('param_spec_args', ast.unparse(n), + tuple(self.node(item,names) for item in n.elts)) + if isinstance(n, ast.List) and accepts_parameter_list else self.node(n,names) + for n in nodes) return self.generic(base,args,ast.unparse(node),names) return self.value(self.lookup(node,names),names) @@ -802,7 +811,17 @@ def value(self,target,names): if origin in TYPE_NARROWING_TYPES: return Contract('bool',str(target),python_type=bool) if origin is not None: - return self.generic(origin,tuple(Ellipsis if a is Ellipsis else self.compile(a,names) for a in args),str(target),names) + accepts_parameter_list = origin is abc.Callable or ( + isinstance(origin, type) and + any(isinstance(parameter, typing.ParamSpec) + for parameter in class_parameters(origin))) + compiled = tuple( + Ellipsis if arg is Ellipsis else + Contract('param_spec_args', str(arg), + tuple(self.compile(item,names) for item in arg)) + if isinstance(arg, (list, tuple)) and accepts_parameter_list else self.compile(arg,names) + for arg in args) + return self.generic(origin,compiled,str(target),names) if target in (int,str,float,bool,bytes,complex): return Contract(target.__name__,target.__name__,python_type=target) if target in (list,set,frozenset,dict,tuple): any_type = Contract('any','Any') diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index c4a76f9..f70c638 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -78,6 +78,7 @@ def __init__(self, manager=None): self.manager = manager self.classes = weakref.WeakSet() self.method_owners = weakref.WeakKeyDictionary() + self.function_type_params = weakref.WeakKeyDictionary() self._classes_lock = threading.RLock() self._contract_cache = threading.local() self._active_instance_checks = ContextVar('aiython_active_instance_checks', default=frozenset()) @@ -203,8 +204,26 @@ def _track_method(self, cls, member): visited.add(id(method)) owners = self.method_owners.setdefault(method.__code__, weakref.WeakSet()) owners.add(cls) + parameters = getattr(method, '__type_params__', ()) + if parameters: + self.function_type_params[method.__code__] = parameters method = vars(method).get('__wrapped__') + def register_function(self, function): + if isinstance(function, (classmethod, staticmethod)): + method = function.__func__ + else: + method = function + visited = set() + with self._classes_lock: + while type(method) is types.FunctionType and id(method) not in visited: + visited.add(id(method)) + parameters = getattr(method, '__type_params__', ()) + if parameters: + self.function_type_params[method.__code__] = parameters + method = vars(method).get('__wrapped__') + return function + def method_self_owner(self, frame, *, discover=False): with self._classes_lock: owners = tuple(self.method_owners.get(frame.f_code, ())) @@ -224,8 +243,7 @@ def method_self_owner(self, frame, *, discover=False): return candidate return owners[0] if len(owners) == 1 else None - @staticmethod - def namespace(frame): + def namespace(self, frame): namespace = dict(frame.f_globals) if frame.f_code.co_name != '' and not frame.f_code.co_flags & inspect.CO_OPTIMIZED: parents = [] @@ -238,6 +256,19 @@ def namespace(frame): for local in reversed(parents): namespace.update(local) namespace.update(frame.f_locals) + with self._classes_lock: + parameters = self.function_type_params.get(frame.f_code, ()) + owners = tuple(self.method_owners.get(frame.f_code, ())) + namespace.update({parameter.__name__: parameter for parameter in parameters}) + generic_owners = tuple(owner for owner in owners if class_parameters(owner)) + if generic_owners: + owner = self.method_self_owner(frame) or ( + generic_owners[0] if len(generic_owners) == 1 else None) + if owner is not None: + for parameter in class_parameters(owner): + namespace[parameter.__name__] = parameter + if parameter.__name__.startswith('__') and not parameter.__name__.endswith('__'): + namespace[f'_{owner.__name__.lstrip("_")}{parameter.__name__}'] = parameter scope = frame_scope(frame) or frame.f_locals.get(SCOPE) if isinstance(scope,Scope): namespace.update({parameter.__name__: bound for parameter,bound in scope.bindings.items()}) @@ -385,9 +416,20 @@ def _check_instances(self, candidate, classes, seen, frame): for item in candidate.values(): self._check_instances(item, classes, seen, frame) + @staticmethod + def _has_class_annotations(cls): + # Keep unannotated project classes out of the per-statement scan. + # Inspect this at each checkpoint so annotations added later still count. + for base in cls.__mro__: + members = vars(base) + if (members.get('__annotations__') or members.get('__annotations_cache__') + or members.get('__annotate_func__') or '__annotate__' in members): + return True + return False + def check_frame(self,frame): with self._classes_lock: - classes = frozenset(self.classes) + classes = frozenset(cls for cls in self.classes if self._has_class_annotations(cls)) if classes: seen = set() if 'self' in frame.f_locals: @@ -634,6 +676,11 @@ def visit_Module(self,node): return node def visit_FunctionDef(self,node): + if getattr(node, 'type_params', ()): + last = node.decorator_list[-1] if node.decorator_list else node + register = ast.Attribute(ast.Attribute(ast.Name(self.runtime_name,ast.Load()),'types',ast.Load()), + 'register_function',ast.Load()) + node.decorator_list.append(ast.copy_location(register, last)) previous = self.function previous_contract = self.delegation_contract parent_declarations = dict(self.declarations) if previous else {} diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index 4b81aa0..adb147e 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -1543,6 +1543,24 @@ def create(klass) -> Self: return klass() ''', } if sys.version_info >= (3, 12): + cases['generic function and mangled method parameters'] = '''def identity[T](value: T) -> T: return value +def decorate(function): + def wrapper(*args, **kwargs): return function(*args, **kwargs) + return wrapper +@decorate +def wrapped[T](value: T) -> T: return value +class Box[__T]: + def pair[__U](self, left: __T, right: __U): + return (__T, __U, left, right) +box = Box() +first, second, left, right = box.pair(1, 'x') +print(identity(3), wrapped(4), first is Box.__type_params__[0], + second is Box.pair.__type_params__[0], left, right) +''' + cases['variadic generic class with parameter list'] = '''class Shape[X, *Y, **Z]: pass +shape = Shape[int, str, bytes, [float, object]]() +print(type(shape).__name__, shape.__orig_class__.__args__[-1]) +''' cases['generic variadic parameters'] = '''def collect[*Ts](*args: *Ts) -> tuple[*Ts]: return args def mixed[*Ts](first: int, *args: *Ts) -> tuple[int, *Ts, str]: return (first, *args, 'done') diff --git a/tests/test_type_safety.py b/tests/test_type_safety.py index a302360..767a530 100644 --- a/tests/test_type_safety.py +++ b/tests/test_type_safety.py @@ -54,6 +54,15 @@ def __init__(self): ''') self.assertEqual(result['answer'],10) + def test_annotations_added_after_class_creation_are_checked(self): + with self.assertRaises(TypeViolation): + self.run_source('''class Box: pass +box = Box() +box.value = 'wrong' +Box.__annotations__ = {'value': int} +answer = 1 +''') + def test_function_attribute_annotation_checks_after_target_lookup(self): with self.assertRaises(TypeViolation): self.run_source('''class Box: @@ -493,3 +502,14 @@ def __init__(self,value): answer = identity(3) ''') self.assertEqual(result['answer'],3) + with self.assertRaises(TypeViolation): + self.run_source('''def broken[T](value: T) -> T: + return 'wrong' +broken(1) +''') + with self.assertRaises(TypeViolation): + self.run_source('''class Box[__T]: + def broken[__U](self, left: __T, right: __U) -> __U: + return left +Box[int]().broken(1, 'expected a string') +''') From 558e0c91b4775717b796a305d56c1838cdaede4a Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 20:08:23 +0700 Subject: [PATCH 66/72] Handle methods invoked before metaclass MRO is ready --- src/aiython/typed_runtime.py | 4 ++++ tests/test_python_compatibility.py | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index f70c638..5f18b58 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -233,6 +233,10 @@ def method_self_owner(self, frame, *, discover=False): receiver_type = type(receiver) candidate = receiver if issubclass(receiver_type, type) else receiver_type mro = type.__getattribute__(candidate, '__mro__') + # A metaclass may call a method while building its MRO, before the + # new class has a finalized __mro__ tuple. + if not isinstance(mro, tuple): + return owners[0] if len(owners) == 1 else None if not owners: with self._classes_lock: for base in mro: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index adb147e..fa0f7cc 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -891,6 +891,17 @@ class Example(metaclass=Meta): for index in range(2): value: int = index print(Example.value) +''', + 'method called while metaclass builds mro': '''captured = None +class Meta(type): + def mro(cls): + cls.__dict__['capture']() + return super().mro() +class Example(metaclass=Meta): + def capture(): + global captured + captured = __class__ +print(captured is Example) ''', 'class locals': '''class Example: snapshot = sorted(locals()) From 112fcd39e7b0ea0a518d9815d8b82e5299645937 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Sun, 27 Sep 2026 21:13:46 +0700 Subject: [PATCH 67/72] Preserve private attribute mangling and safe instance scans --- src/aiython/typed_runtime.py | 37 ++++++++++++++++++++++-------- tests/test_python_compatibility.py | 27 ++++++++++++++++++++++ tests/test_type_safety.py | 12 ++++++++++ 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 5f18b58..5f80acd 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -408,11 +408,16 @@ def _check_instances(self, candidate, classes, seen, frame): cls = type(candidate) if cls in classes: self._validate_instance(candidate, cls.__qualname__, frame) - try: - state = object.__getattribute__(candidate, '__dict__') - except AttributeError: - state = {} - self._check_instances(state, classes, seen, frame) + for base in cls.__mro__: + descriptor = vars(base).get('__dict__') + if type(descriptor) in (types.GetSetDescriptorType, types.MemberDescriptorType): + try: + state = descriptor.__get__(candidate, cls) + except AttributeError: + break + if type(state) is dict: + self._check_instances(state, classes, seen, frame) + break elif cls in (list, tuple, set, frozenset): for item in candidate: self._check_instances(item, classes, seen, frame) @@ -426,8 +431,11 @@ def _has_class_annotations(cls): # Inspect this at each checkpoint so annotations added later still count. for base in cls.__mro__: members = vars(base) - if (members.get('__annotations__') or members.get('__annotations_cache__') - or members.get('__annotate_func__') or '__annotate__' in members): + annotation = members.get('__annotations__') + cache = members.get('__annotations_cache__') + if ((type(annotation) is dict and annotation) or + (type(cache) is dict and cache) or + callable(members.get('__annotate_func__'))): return True return False @@ -611,10 +619,18 @@ def __init__(self, *, snippet=False, runtime_name=RUNTIME_NAME): self.declarations = {} self.function = False self.delegation_contract = False + self.class_name = None def helper(self, name, *args): return helper(name, *args, runtime_name=self.runtime_name) + def attribute_name(self, name): + if self.class_name is not None and name.startswith('__') and not name.endswith('__'): + prefix = self.class_name.lstrip('_') + if prefix: + return f'_{prefix}{name}' + return name + @staticmethod def declarations_in(body): result = {} @@ -737,8 +753,10 @@ def visit_ClassDef(self,node): node.decorator_list.insert(0, ast.copy_location(register, first)) previous = self.function previous_contract = self.delegation_contract + previous_class = self.class_name self.function = False self.delegation_contract = False + self.class_name = node.name body = self.body(node.body) header = body[:1] if isinstance(body[0],ast.Expr) and isinstance(body[0].value,ast.Constant) and isinstance(body[0].value.value,str) else [] enter, *statements = body[len(header):] @@ -751,6 +769,7 @@ def visit_ClassDef(self,node): node.body = header + [enter, exit_call] self.function = previous self.delegation_contract = previous_contract + self.class_name = previous_class return node def visit_Lambda(self,node): @@ -789,7 +808,7 @@ def visit_AnnAssign(self,node): 'assign_attribute', ast.Load()), [], [ ast.keyword(arg='value', value=self.visit(node.value)), ast.keyword(arg='owner', value=self.visit(node.target.value)), - ast.keyword(arg='name', value=ast.Constant(node.target.attr)), + ast.keyword(arg='name', value=ast.Constant(self.attribute_name(node.target.attr))), ast.keyword(arg='annotation', value=ast.Constant(annotation)), ]) return ast.copy_location(ast.Expr(assignment), node) @@ -807,7 +826,7 @@ def visit_Assign(self,node): 'assign_attribute',ast.Load()), [], [ ast.keyword(arg='value',value=node.value), ast.keyword(arg='owner',value=self.visit(target.value)), - ast.keyword(arg='name',value=ast.Constant(target.attr)), + ast.keyword(arg='name',value=ast.Constant(self.attribute_name(target.attr))), ]) return ast.copy_location(ast.Expr(assign),node) for target in node.targets: diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index fa0f7cc..a8fdce9 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -902,6 +902,33 @@ def capture(): global captured captured = __class__ print(captured is Example) +''', + 'private attribute writes in nested classes': '''class Box: + class Descriptor: + def __init__(self, getter): self.__getter = getter + def __get__(self, owner, kind=None): return self.__getter(owner) + def __init__(self): self.__value: int = 3 + def read(self): return self.__value + value = Descriptor(read) +box = Box() +print(box.value, box.read(), sorted(vars(box))) +''', + 'custom instance dict descriptors': '''import sys +calls = [] +class Box: + value: int + def __init__(self): self.value = 3 + @property + def __dict__(self): + calls.append('dict property') + return 'not a dict' +box = Box() +print(box.value, calls) +class Module(type(sys)): + __dict__ = property(lambda self: 'not a dict') +module = Module('example') +try: dir(module) +except TypeError: print('TypeError') ''', 'class locals': '''class Example: snapshot = sorted(locals()) diff --git a/tests/test_type_safety.py b/tests/test_type_safety.py index 767a530..460d11d 100644 --- a/tests/test_type_safety.py +++ b/tests/test_type_safety.py @@ -61,6 +61,18 @@ def test_annotations_added_after_class_creation_are_checked(self): box.value = 'wrong' Box.__annotations__ = {'value': int} answer = 1 +''') + + def test_custom_dict_descriptor_does_not_hide_invalid_fields(self): + with self.assertRaises(TypeViolation): + self.run_source('''class Box: + value: int + def __init__(self): self.value = 3 + @property + def __dict__(self): return 'not a dict' +box = Box() +object.__setattr__(box, 'value', 'wrong') +answer = 1 ''') def test_function_attribute_annotation_checks_after_target_lookup(self): From bdb33588b003966224862c1d2b58aa3e101f7942 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Mon, 28 Sep 2026 12:32:26 +0700 Subject: [PATCH 68/72] Route execution through native bridge and preserve recovery and yield checks --- .github/workflows/docs.yml | 6 +- .github/workflows/release.yml | 12 +- .github/workflows/tests.yml | 32 +- benchmarks/stdlib_syntax.py | 19 +- native/README.md | 75 + native/cpython.lock.toml | 6 + native/patches/cpython-3.11.16-hooks.patch | 301 ++ native/patches/cpython-3.12.14-hooks.patch | 3960 ++++++++++++++++++++ native/patches/cpython-3.13.15-hooks.patch | 678 ++++ native/patches/cpython-3.14.4-hooks.patch | 764 ++++ native/tests/store_hook.py | 233 ++ native/tests/typed_bridge.py | 140 + scripts/build_cpython_baseline.py | 91 + scripts/build_cpython_hooks.py | 61 + src/aiython/cli.py | 15 +- src/aiython/native_bridge.py | 246 ++ src/aiython/repl.py | 12 +- src/aiython/runtime.py | 95 +- src/aiython/typed_runtime.py | 69 +- tests/test_overhead.py | 92 + tests/test_python_compatibility.py | 72 + tests/test_repl.py | 10 + tests/test_runtime.py | 62 + tests/test_runtime_edges.py | 5 +- tests/test_typed_runtime_edges.py | 2 +- 25 files changed, 6990 insertions(+), 68 deletions(-) create mode 100644 native/README.md create mode 100644 native/cpython.lock.toml create mode 100644 native/patches/cpython-3.11.16-hooks.patch create mode 100644 native/patches/cpython-3.12.14-hooks.patch create mode 100644 native/patches/cpython-3.13.15-hooks.patch create mode 100644 native/patches/cpython-3.14.4-hooks.patch create mode 100644 native/tests/store_hook.py create mode 100644 native/tests/typed_bridge.py create mode 100644 scripts/build_cpython_baseline.py create mode 100644 scripts/build_cpython_hooks.py create mode 100644 src/aiython/native_bridge.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index fd4fe6d..174123c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -38,8 +38,8 @@ jobs: persist-credentials: false - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: - python-version: "3.11" + python-version: "3.11.16" version: latest-known enable-cache: false - - run: uv run --locked zensical build --clean --strict - - run: uv run --locked python scripts/check_docs_site.py + - run: uv run --locked --python 3.11.16 zensical build --clean --strict + - run: uv run --locked --python 3.11.16 python scripts/check_docs_site.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 68bc536..31c6d21 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] + python-version: ["3.11.16", "3.12.14", "3.13.15", "3.14.4"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -25,6 +25,8 @@ jobs: python-version: ${{ matrix.python-version }} version: latest-known enable-cache: false + - name: Check CPython frontend code objects + run: uv run --locked --python ${{ matrix.python-version }} python benchmarks/stdlib_syntax.py - name: Test with line and branch coverage env: COVERAGE_PROCESS_START: ${{ github.workspace }}/.coveragerc @@ -45,7 +47,7 @@ jobs: persist-credentials: false - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: - python-version: "3.11" + python-version: "3.11.16" version: latest-known enable-cache: false - name: Check tag matches package version @@ -84,11 +86,11 @@ jobs: persist-credentials: false - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: - python-version: "3.11" + python-version: "3.11.16" version: latest-known enable-cache: false - - run: uv run --locked zensical build --clean --strict - - run: uv run --locked python scripts/check_docs_site.py + - run: uv run --locked --python 3.11.16 zensical build --clean --strict + - run: uv run --locked --python 3.11.16 python scripts/check_docs_site.py - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: docs-site diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2cae1e4..f4ed8e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,10 +14,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: - python-version: "3.11" + python-version: "3.11.16" version: latest-known - run: uv build --no-sources - - run: uv venv --python 3.11 /tmp/aiython-wheel-smoke + - run: uv venv --python 3.11.16 /tmp/aiython-wheel-smoke - run: uv pip install --python /tmp/aiython-wheel-smoke/bin/python dist/*.whl - run: /tmp/aiython-wheel-smoke/bin/aiython --version - run: | @@ -29,7 +29,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] + python-version: ["3.11.16", "3.12.14", "3.13.15", "3.14.4"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 @@ -37,6 +37,8 @@ jobs: python-version: ${{ matrix.python-version }} version: latest-known - run: uv python install ${{ matrix.python-version }} + - name: Check CPython frontend code objects + run: uv run --locked --python ${{ matrix.python-version }} python benchmarks/stdlib_syntax.py - name: Test with line and branch coverage env: COVERAGE_PROCESS_START: ${{ github.workspace }}/.coveragerc @@ -47,10 +49,30 @@ jobs: uv run --locked --python ${{ matrix.python-version }} --extra a2a python -m coverage report uv run --locked --python ${{ matrix.python-version }} --extra a2a python -m coverage json -o coverage.json uv run --locked --python ${{ matrix.python-version }} python scripts/coverage_summary.py - - name: Compile standard library syntax - run: uv run --locked --python ${{ matrix.python-version }} python benchmarks/stdlib_syntax.py - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-${{ matrix.python-version }} path: coverage.json if-no-files-found: error + + native-hooks: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11.16", "3.12.14", "3.13.15", "3.14.4"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 + with: + python-version: ${{ matrix.python-version }} + version: latest-known + - name: Install CPython build headers + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq build-essential pkg-config libbz2-dev libffi-dev \ + libgdbm-dev libgdbm-compat-dev liblzma-dev libncurses-dev \ + libreadline-dev libsqlite3-dev libssl-dev libzstd-dev tk-dev \ + uuid-dev zlib1g-dev libnsl-dev libtirpc-dev + - name: Build and test patched CPython + run: uv run --locked --python ${{ matrix.python-version }} python scripts/build_cpython_hooks.py ${{ matrix.python-version }} diff --git a/benchmarks/stdlib_syntax.py b/benchmarks/stdlib_syntax.py index bcc1100..6627f9d 100644 --- a/benchmarks/stdlib_syntax.py +++ b/benchmarks/stdlib_syntax.py @@ -1,7 +1,7 @@ -"""Check that Aiython compiles the current interpreter's standard library. +"""Check the frontend against CPython and compile the standard library. Run: uv run python benchmarks/stdlib_syntax.py -This checks syntax transformation only; it does not execute the modules. +This checks compilation only; it does not execute the modules. """ import argparse @@ -10,6 +10,7 @@ import sysconfig import tokenize +from aiython.frontend import parse from aiython.models import ProfileConfig, ResolvedConfig from aiython.runtime import Runtime @@ -39,8 +40,20 @@ def main(): try: with tokenize.open(path) as file: source = file.read() - compile(source, str(path), 'exec', dont_inherit=True) + native = compile(source, str(path), 'exec', dont_inherit=True) + unit = parse(source, str(path)) + if unit.blocks: + raise AssertionError('valid Python source was classified as AI syntax') + frontend_code = compile(unit.tree, str(path), 'exec', dont_inherit=True) + # CPython compares code contents, including nested code objects. + # marshal byte streams can differ in reference encoding even when + # the code objects are equal; both compiles use the same filename. + if frontend_code != native: + raise AssertionError('frontend changed the native CPython code object') runtime.compile_source(source, str(path)) + source_tree = runtime.units[str(path)].tree + if compile(source_tree, str(path), 'exec', dont_inherit=True) != native: + raise AssertionError('runtime preparation changed the native source tree') except Exception as error: failures.append((path.relative_to(root), error)) if (index + 1) % 100 == 0: diff --git a/native/README.md b/native/README.md new file mode 100644 index 0000000..1491698 --- /dev/null +++ b/native/README.md @@ -0,0 +1,75 @@ +# Native CPython experiment + +`cpython.lock.toml` pins the exact CPython release commits used by the +compatibility work. `scripts/build_cpython_baseline.py` builds and checks an +unmodified interpreter before any VM changes are applied. The current patches +have been built and tested on Linux. + +The 3.11.16–3.14.4 patches are **experimental VM hooks**, not Aiython +interpreters. +When `sys._aiython_before_store` exists, it calls that function with +`(frame, name, value)` before local, global, namespace, or closure stores. +The callback may raise, and Python's normal exception handlers receive that +exception. It skips empty stack references used by `STORE_FAST_MAYBE_NULL`, +and a thread-local guard stops recursive callbacks. With the attribute absent, +the interpreter follows CPython's normal store behavior. + +When `sys._aiython_before_mutation` exists, it calls that function with +`(frame, kind, owner, key, value)` before attribute (`kind == "attr"`) and item +(`kind == "item"`) assignment, including slice assignment. Specialized +attribute, dict, and list store opcodes return to the generic opcode while +this hook is active, so enabling it after specialization still works. +The optimized string `+=` instruction returns to ordinary bytecode execution +while the store hook is active. + +`sys._aiython_on_call(frame)` runs at the first `RESUME` of a function or +generator. `sys._aiython_on_return(frame, value)` runs at a return opcode. +These callbacks can raise. A return callback error raised at the return opcode +is outside the function's source-level `try` range in some cases, so an inner +`except` will not necessarily catch it. This remains an unresolved semantic +gap. + +The automated builder verifies the unmodified CPython baseline, applies the +matching patch, builds again, runs the VM hook smoke test, and runs eight +CPython test modules. It also runs `native/tests/typed_bridge.py`, which +checks type enforcement on original CPython code objects. Replace `VERSION` +with one of the exact releases in +`cpython.lock.toml`: + +```sh +VERSION=3.13.15 +uv run --locked --python "$VERSION" python scripts/build_cpython_hooks.py "$VERSION" +``` + +To apply a patch manually to a clean checkout: + +```sh +VERSION=3.13.15 +uv run --locked --python "$VERSION" python scripts/build_cpython_baseline.py "$VERSION" \ + --source-dir "/path/to/cpython-$VERSION" +git -C "/path/to/cpython-$VERSION" apply \ + "$PWD/native/patches/cpython-$VERSION-hooks.patch" +make -C "/path/to/cpython-$VERSION" regen-generated-cases +make -C "/path/to/cpython-$VERSION" -j4 +"/path/to/cpython-$VERSION/python" -I native/tests/store_hook.py +``` + +For 3.12, use `make regen-cases` instead of `regen-generated-cases`. For 3.11, +skip the regeneration command: its interpreter is maintained directly in +`Python/ceval.c`. + +`aiython.native_bridge.NativeTypeBridge.prepare_unit` is the normal compiler +entry point called by `Runtime.compile_source`. It selects the exact CPython +code object for ordinary source without imports, annotations, AI directives, +blocks, or configured recovery. For annotated source and configured recovery, +it delegates to Aiython's boundary compiler; that path checks yielded values +and can resume failed statements. This routing works on stock CPython and the +four pinned patched interpreters. + +The separate experimental `NativeTypeBridge.compile_source` and `installed` +API parses annotations, compiles the original source, and validates annotated +parameters, generator returns, local/global/nonlocal assignments, `Final` +rebinding, and class attributes through the VM hooks. It does not yet check +yielded values, deletions, in-place mutation through method calls, generic +bindings, or provide AI recovery through VM callbacks. These results do not +establish full Python compatibility. diff --git a/native/cpython.lock.toml b/native/cpython.lock.toml new file mode 100644 index 0000000..ad6f26b --- /dev/null +++ b/native/cpython.lock.toml @@ -0,0 +1,6 @@ +# Peeled CPython release-tag commits. Keep these in sync with the CI matrix. +[versions] +"3.11.16" = "41388c9cb160d0886d5ca00d2e6c8782608a4549" +"3.12.14" = "2abcf904b8dac8c999d2b3aac76681abb333798a" +"3.13.15" = "4061bc4c35f7c26f25264666d4ba083b93d2f6f9" +"3.14.4" = "23116f998f6789d8c2fbe5ed5b8146854c8c2a4f" diff --git a/native/patches/cpython-3.11.16-hooks.patch b/native/patches/cpython-3.11.16-hooks.patch new file mode 100644 index 0000000..db0ae81 --- /dev/null +++ b/native/patches/cpython-3.11.16-hooks.patch @@ -0,0 +1,301 @@ +diff --git a/Python/ceval.c b/Python/ceval.c +index 1ce6bbc..41fdc4d 100644 +--- a/Python/ceval.c ++++ b/Python/ceval.c +@@ -1625,6 +1625,124 @@ typedef struct { + #define KWNAMES_LEN() \ + (call_shape.kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(call_shape.kwnames))) + ++/* Experimental Aiython hook. The Python callback can raise to reject a store. ++ The guard prevents assignments inside that callback from calling it again. */ ++static _Thread_local int aiython_in_hook = 0; ++ ++static int ++aiython_before_store(_PyInterpreterFrame *frame, PyObject *name, PyObject *value) ++{ ++ /* STORE_FAST_MAYBE_NULL may legally write an empty stack reference. */ ++ if (value == NULL || aiython_in_hook) { ++ return 0; ++ } ++ PyObject *callback = NULL; ++ callback = PySys_GetObject("_aiython_before_store"); ++ if (callback == NULL) { ++ return 0; ++ } ++ Py_INCREF(callback); ++ PyFrameObject *borrowed = _PyFrame_GetFrameObject(frame); ++ if (borrowed == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *frame_obj = Py_NewRef(borrowed); ++ aiython_in_hook = 1; ++ PyObject *result = PyObject_CallFunctionObjArgs(callback, frame_obj, name, value, NULL); ++ aiython_in_hook = 0; ++ Py_DECREF(frame_obj); ++ Py_DECREF(callback); ++ if (result == NULL) { ++ return -1; ++ } ++ Py_DECREF(result); ++ return 0; ++} ++ ++static int ++aiython_mutation_hook_enabled(void) ++{ ++ return PySys_GetObject("_aiython_before_mutation") != NULL; ++} ++ ++static int ++aiython_store_hook_enabled(void) ++{ ++ return PySys_GetObject("_aiython_before_store") != NULL; ++} ++ ++static int ++aiython_before_mutation(_PyInterpreterFrame *frame, const char *kind, ++ PyObject *owner, PyObject *key, PyObject *value) ++{ ++ if (aiython_in_hook) { ++ return 0; ++ } ++ PyObject *callback = NULL; ++ callback = PySys_GetObject("_aiython_before_mutation"); ++ if (callback == NULL) { ++ return 0; ++ } ++ Py_INCREF(callback); ++ PyFrameObject *borrowed = _PyFrame_GetFrameObject(frame); ++ if (borrowed == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *kind_obj = PyUnicode_FromString(kind); ++ if (kind_obj == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *frame_obj = Py_NewRef(borrowed); ++ aiython_in_hook = 1; ++ PyObject *result = PyObject_CallFunctionObjArgs( ++ callback, frame_obj, kind_obj, owner, key, value, NULL); ++ aiython_in_hook = 0; ++ Py_DECREF(frame_obj); ++ Py_DECREF(kind_obj); ++ Py_DECREF(callback); ++ if (result == NULL) { ++ return -1; ++ } ++ Py_DECREF(result); ++ return 0; ++} ++ ++static int ++aiython_function_event(_PyInterpreterFrame *frame, const char *attribute, ++ PyObject *value) ++{ ++ if (aiython_in_hook) { ++ return 0; ++ } ++ PyObject *callback = NULL; ++ callback = PySys_GetObject(attribute); ++ if (callback == NULL) { ++ return 0; ++ } ++ Py_INCREF(callback); ++ PyFrameObject *borrowed = _PyFrame_GetFrameObject(frame); ++ if (borrowed == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *frame_obj = Py_NewRef(borrowed); ++ aiython_in_hook = 1; ++ PyObject *result = value == NULL ++ ? PyObject_CallOneArg(callback, frame_obj) ++ : PyObject_CallFunctionObjArgs(callback, frame_obj, value, NULL); ++ aiython_in_hook = 0; ++ Py_DECREF(frame_obj); ++ Py_DECREF(callback); ++ if (result == NULL) { ++ return -1; ++ } ++ Py_DECREF(result); ++ return 0; ++} ++ + PyObject* _Py_HOT_FUNCTION + _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) + { +@@ -1779,6 +1897,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + goto handle_eval_breaker; + } ++ if (oparg == 0 && aiython_function_event(frame, "_aiython_on_call", NULL) < 0) { ++ goto error; ++ } + DISPATCH(); + } + +@@ -1814,6 +1935,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + TARGET(STORE_FAST) { + PREDICTED(STORE_FAST); + PyObject *value = POP(); ++ PyObject *name = GETITEM(frame->f_code->co_localsplusnames, oparg); ++ if (aiython_before_store(frame, name, value) < 0) { ++ Py_XDECREF(value); ++ goto error; ++ } + SETLOCAL(oparg, value); + DISPATCH(); + } +@@ -1853,6 +1979,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + + TARGET(STORE_FAST__LOAD_FAST) { + PyObject *value = POP(); ++ PyObject *name = GETITEM(frame->f_code->co_localsplusnames, oparg); ++ if (aiython_before_store(frame, name, value) < 0) { ++ Py_XDECREF(value); ++ goto error; ++ } + SETLOCAL(oparg, value); + NEXTOPARG(); + next_instr++; +@@ -1867,10 +1998,20 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + + TARGET(STORE_FAST__STORE_FAST) { + PyObject *value = POP(); ++ PyObject *name = GETITEM(frame->f_code->co_localsplusnames, oparg); ++ if (aiython_before_store(frame, name, value) < 0) { ++ Py_XDECREF(value); ++ goto error; ++ } + SETLOCAL(oparg, value); + NEXTOPARG(); + next_instr++; + value = POP(); ++ name = GETITEM(frame->f_code->co_localsplusnames, oparg); ++ if (aiython_before_store(frame, name, value) < 0) { ++ Py_XDECREF(value); ++ goto error; ++ } + SETLOCAL(oparg, value); + DISPATCH(); + } +@@ -2050,6 +2191,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + assert(cframe.use_tracing == 0); ++ DEOPT_IF(aiython_store_hook_enabled(), BINARY_OP); + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); +@@ -2294,7 +2436,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + int err; + STACK_SHRINK(3); + /* container[sub] = v */ +- err = PyObject_SetItem(container, sub, v); ++ err = aiython_before_mutation(frame, "item", container, sub, v); ++ if (err == 0) { ++ err = PyObject_SetItem(container, sub, v); ++ } + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); +@@ -2326,6 +2471,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + + TARGET(STORE_SUBSCR_LIST_INT) { + assert(cframe.use_tracing == 0); ++ DEOPT_IF(aiython_mutation_hook_enabled(), STORE_SUBSCR); + PyObject *sub = TOP(); + PyObject *list = SECOND(); + PyObject *value = THIRD(); +@@ -2352,6 +2498,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + + TARGET(STORE_SUBSCR_DICT) { + assert(cframe.use_tracing == 0); ++ DEOPT_IF(aiython_mutation_hook_enabled(), STORE_SUBSCR); + PyObject *sub = TOP(); + PyObject *dict = SECOND(); + PyObject *value = THIRD(); +@@ -2423,6 +2570,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + + TARGET(RETURN_VALUE) { + PyObject *retval = POP(); ++ if (aiython_function_event(frame, "_aiython_on_return", retval) < 0) { ++ Py_DECREF(retval); ++ goto error; ++ } + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + TRACE_FUNCTION_EXIT(); +@@ -2758,6 +2909,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + Py_DECREF(v); + goto error; + } ++ if (aiython_before_store(frame, name, v) < 0) { ++ Py_DECREF(v); ++ goto error; ++ } + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else +@@ -2879,7 +3034,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + PyObject *v = SECOND(); + int err; + STACK_SHRINK(2); +- err = PyObject_SetAttr(owner, name, v); ++ err = aiython_before_mutation(frame, "attr", owner, name, v); ++ if (err == 0) { ++ err = PyObject_SetAttr(owner, name, v); ++ } + Py_DECREF(v); + Py_DECREF(owner); + if (err != 0) { +@@ -2904,6 +3062,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + PyObject *name = GETITEM(names, oparg); + PyObject *v = POP(); + int err; ++ if (aiython_before_store(frame, name, v) < 0) { ++ Py_DECREF(v); ++ goto error; ++ } + err = PyDict_SetItem(GLOBALS(), name, v); + Py_DECREF(v); + if (err != 0) +@@ -3190,6 +3352,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + + TARGET(STORE_DEREF) { + PyObject *v = POP(); ++ PyObject *name = GETITEM(frame->f_code->co_localsplusnames, oparg); ++ if (aiython_before_store(frame, name, v) < 0) { ++ Py_DECREF(v); ++ goto error; ++ } + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + PyCell_SET(cell, v); +@@ -3600,6 +3767,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + + TARGET(STORE_ATTR_INSTANCE_VALUE) { + assert(cframe.use_tracing == 0); ++ DEOPT_IF(aiython_mutation_hook_enabled(), STORE_ATTR); + PyObject *owner = TOP(); + PyTypeObject *tp = Py_TYPE(owner); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; +@@ -3628,6 +3796,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + + TARGET(STORE_ATTR_WITH_HINT) { + assert(cframe.use_tracing == 0); ++ DEOPT_IF(aiython_mutation_hook_enabled(), STORE_ATTR); + PyObject *owner = TOP(); + PyTypeObject *tp = Py_TYPE(owner); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; +@@ -3675,6 +3844,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int + + TARGET(STORE_ATTR_SLOT) { + assert(cframe.use_tracing == 0); ++ DEOPT_IF(aiython_mutation_hook_enabled(), STORE_ATTR); + PyObject *owner = TOP(); + PyTypeObject *tp = Py_TYPE(owner); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; diff --git a/native/patches/cpython-3.12.14-hooks.patch b/native/patches/cpython-3.12.14-hooks.patch new file mode 100644 index 0000000..b3f1a70 --- /dev/null +++ b/native/patches/cpython-3.12.14-hooks.patch @@ -0,0 +1,3960 @@ +diff --git a/Python/bytecodes.c b/Python/bytecodes.c +index 6ee3bda..017a396 100644 +--- a/Python/bytecodes.c ++++ b/Python/bytecodes.c +@@ -142,8 +142,14 @@ dummy_func( + ERROR_IF(err, error); + next_instr--; + } +- else if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker) && oparg < 2) { +- goto handle_eval_breaker; ++ else { ++ if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker) && oparg < 2) { ++ goto handle_eval_breaker; ++ } ++ if (oparg == 0) { ++ int hook_err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ ERROR_IF(hook_err, error); ++ } + } + } + +@@ -172,6 +178,10 @@ dummy_func( + if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker) && oparg < 2) { + goto handle_eval_breaker; + } ++ if (oparg == 0) { ++ int hook_err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ ERROR_IF(hook_err, error); ++ } + } + } + +@@ -206,6 +216,12 @@ dummy_func( + } + + inst(STORE_FAST, (value --)) { ++ PyObject *name = GETITEM(frame->f_code->co_localsplusnames, oparg); ++ int err = aiython_before_store(frame, name, value); ++ if (err < 0) { ++ DECREF_INPUTS(); ++ ERROR_IF(true, error); ++ } + SETLOCAL(oparg, value); + } + +@@ -344,6 +360,8 @@ dummy_func( + // specializations, but there is no output. + // At the end we just skip over the STORE_FAST. + inst(BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { ++ int store_hook_enabled = aiython_store_hook_enabled(); ++ DEOPT_IF(store_hook_enabled, BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); + _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; +@@ -437,7 +455,10 @@ dummy_func( + err = 1; + } + else { +- err = PyObject_SetItem(container, slice, v); ++ err = aiython_before_mutation(frame, "item", container, slice, v); ++ if (err == 0) { ++ err = PyObject_SetItem(container, slice, v); ++ } + Py_DECREF(slice); + } + Py_DECREF(v); +@@ -549,12 +570,17 @@ dummy_func( + (void)counter; // Unused. + #endif /* ENABLE_SPECIALIZATION */ + /* container[sub] = v */ +- int err = PyObject_SetItem(container, sub, v); ++ int err = aiython_before_mutation(frame, "item", container, sub, v); ++ if (err == 0) { ++ err = PyObject_SetItem(container, sub, v); ++ } + DECREF_INPUTS(); + ERROR_IF(err, error); + } + + inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) { ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + +@@ -574,6 +600,8 @@ dummy_func( + } + + inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) { ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); +@@ -640,6 +668,8 @@ dummy_func( + } + + inst(RETURN_VALUE, (retval --)) { ++ int hook_err = aiython_function_event(frame, "_aiython_on_return", retval); ++ ERROR_IF(hook_err, error); + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); +@@ -659,6 +689,8 @@ dummy_func( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, retval); + if (err) goto error; ++ err = aiython_function_event(frame, "_aiython_on_return", retval); ++ if (err) goto error; + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); +@@ -675,6 +707,8 @@ dummy_func( + + inst(RETURN_CONST, (--)) { + PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); ++ int hook_err = aiython_function_event(frame, "_aiython_on_return", retval); ++ ERROR_IF(hook_err, error); + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); +@@ -695,6 +729,8 @@ dummy_func( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, retval); + if (err) goto error; ++ err = aiython_function_event(frame, "_aiython_on_return", retval); ++ if (err) goto error; + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); +@@ -1019,6 +1055,11 @@ dummy_func( + DECREF_INPUTS(); + ERROR_IF(true, error); + } ++ err = aiython_before_store(frame, name, v); ++ if (err < 0) { ++ DECREF_INPUTS(); ++ ERROR_IF(true, error); ++ } + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else +@@ -1132,7 +1173,10 @@ dummy_func( + (void)counter; // Unused. + #endif /* ENABLE_SPECIALIZATION */ + PyObject *name = GETITEM(frame->f_code->co_names, oparg); +- int err = PyObject_SetAttr(owner, name, v); ++ int err = aiython_before_mutation(frame, "attr", owner, name, v); ++ if (err == 0) { ++ err = PyObject_SetAttr(owner, name, v); ++ } + DECREF_INPUTS(); + ERROR_IF(err, error); + } +@@ -1146,6 +1190,11 @@ dummy_func( + + inst(STORE_GLOBAL, (v --)) { + PyObject *name = GETITEM(frame->f_code->co_names, oparg); ++ int hook_err = aiython_before_store(frame, name, v); ++ if (hook_err < 0) { ++ DECREF_INPUTS(); ++ ERROR_IF(true, error); ++ } + int err = PyDict_SetItem(GLOBALS(), name, v); + DECREF_INPUTS(); + ERROR_IF(err, error); +@@ -1468,6 +1517,12 @@ dummy_func( + } + + inst(STORE_DEREF, (v --)) { ++ PyObject *name = GETITEM(frame->f_code->co_localsplusnames, oparg); ++ int err = aiython_before_store(frame, name, v); ++ if (err < 0) { ++ DECREF_INPUTS(); ++ ERROR_IF(true, error); ++ } + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + PyCell_SET(cell, v); +@@ -1950,6 +2005,8 @@ dummy_func( + } + + inst(STORE_ATTR_INSTANCE_VALUE, (unused/1, type_version/2, index/1, value, owner --)) { ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); +@@ -1970,6 +2027,8 @@ dummy_func( + } + + inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) { ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); +@@ -2012,6 +2071,8 @@ dummy_func( + } + + inst(STORE_ATTR_SLOT, (unused/1, type_version/2, index/1, value, owner --)) { ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); +diff --git a/Python/ceval.c b/Python/ceval.c +index 3985b52..d4ffda2 100644 +--- a/Python/ceval.c ++++ b/Python/ceval.c +@@ -652,6 +652,122 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { + * so consume 3 units of C stack */ + #define PY_EVAL_C_STACK_UNITS 2 + ++/* Experimental Aiython hook. The Python callback can raise to reject a store. ++ The guard prevents assignments inside that callback from calling it again. */ ++static _Thread_local int aiython_in_hook = 0; ++ ++static int ++aiython_before_store(_PyInterpreterFrame *frame, PyObject *name, PyObject *value) ++{ ++ /* STORE_FAST_MAYBE_NULL may legally write an empty stack reference. */ ++ if (value == NULL || aiython_in_hook) { ++ return 0; ++ } ++ PyObject *callback = NULL; ++ int found = _PySys_GetOptionalAttrString("_aiython_before_store", &callback); ++ if (found <= 0) { ++ return found < 0 ? -1 : 0; ++ } ++ PyFrameObject *borrowed = _PyFrame_GetFrameObject(frame); ++ if (borrowed == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *frame_obj = Py_NewRef(borrowed); ++ aiython_in_hook = 1; ++ PyObject *result = PyObject_CallFunctionObjArgs(callback, frame_obj, name, value, NULL); ++ aiython_in_hook = 0; ++ Py_DECREF(frame_obj); ++ Py_DECREF(callback); ++ if (result == NULL) { ++ return -1; ++ } ++ Py_DECREF(result); ++ return 0; ++} ++ ++static int ++aiython_mutation_hook_enabled(void) ++{ ++ return PySys_GetObject("_aiython_before_mutation") != NULL; ++} ++ ++static int ++aiython_store_hook_enabled(void) ++{ ++ return PySys_GetObject("_aiython_before_store") != NULL; ++} ++ ++static int ++aiython_before_mutation(_PyInterpreterFrame *frame, const char *kind, ++ PyObject *owner, PyObject *key, PyObject *value) ++{ ++ if (aiython_in_hook) { ++ return 0; ++ } ++ PyObject *callback = NULL; ++ int found = _PySys_GetOptionalAttrString("_aiython_before_mutation", &callback); ++ if (found <= 0) { ++ return found < 0 ? -1 : 0; ++ } ++ PyFrameObject *borrowed = _PyFrame_GetFrameObject(frame); ++ if (borrowed == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *kind_obj = PyUnicode_FromString(kind); ++ if (kind_obj == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *frame_obj = Py_NewRef(borrowed); ++ aiython_in_hook = 1; ++ PyObject *result = PyObject_CallFunctionObjArgs( ++ callback, frame_obj, kind_obj, owner, key, value, NULL); ++ aiython_in_hook = 0; ++ Py_DECREF(frame_obj); ++ Py_DECREF(kind_obj); ++ Py_DECREF(callback); ++ if (result == NULL) { ++ return -1; ++ } ++ Py_DECREF(result); ++ return 0; ++} ++ ++static int ++aiython_function_event(_PyInterpreterFrame *frame, const char *attribute, ++ PyObject *value) ++{ ++ if (aiython_in_hook) { ++ return 0; ++ } ++ PyObject *callback = NULL; ++ int found = _PySys_GetOptionalAttrString(attribute, &callback); ++ if (found <= 0) { ++ return found < 0 ? -1 : 0; ++ } ++ PyFrameObject *borrowed = _PyFrame_GetFrameObject(frame); ++ if (borrowed == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *frame_obj = Py_NewRef(borrowed); ++ aiython_in_hook = 1; ++ PyObject *result = value == NULL ++ ? PyObject_CallOneArg(callback, frame_obj) ++ : PyObject_CallFunctionObjArgs(callback, frame_obj, value, NULL); ++ aiython_in_hook = 0; ++ Py_DECREF(frame_obj); ++ Py_DECREF(callback); ++ if (result == NULL) { ++ return -1; ++ } ++ Py_DECREF(result); ++ return 0; ++} ++ ++ + PyObject* _Py_HOT_FUNCTION + _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) + { +diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h +index 246d37e..550964b 100644 +--- a/Python/generated_cases.c.h ++++ b/Python/generated_cases.c.h +@@ -17,15 +17,21 @@ + if (err) goto error; + next_instr--; + } +- else if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker) && oparg < 2) { +- goto handle_eval_breaker; ++ else { ++ if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker) && oparg < 2) { ++ goto handle_eval_breaker; ++ } ++ if (oparg == 0) { ++ int hook_err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ if (hook_err) goto error; ++ } + } +- #line 24 "Python/generated_cases.c.h" ++ #line 30 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(INSTRUMENTED_RESUME) { +- #line 151 "Python/bytecodes.c" ++ #line 157 "Python/bytecodes.c" + /* Possible performance enhancement: + * We need to check the eval breaker anyway, can we + * combine the instrument verison check and the eval breaker test? +@@ -50,19 +56,23 @@ + if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker) && oparg < 2) { + goto handle_eval_breaker; + } ++ if (oparg == 0) { ++ int hook_err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ if (hook_err) goto error; ++ } + } +- #line 55 "Python/generated_cases.c.h" ++ #line 65 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(LOAD_CLOSURE) { + PyObject *value; +- #line 179 "Python/bytecodes.c" ++ #line 189 "Python/bytecodes.c" + /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ + value = GETLOCAL(oparg); + if (value == NULL) goto unbound_local_error; + Py_INCREF(value); +- #line 66 "Python/generated_cases.c.h" ++ #line 76 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = value; + DISPATCH(); +@@ -70,11 +80,11 @@ + + TARGET(LOAD_FAST_CHECK) { + PyObject *value; +- #line 186 "Python/bytecodes.c" ++ #line 196 "Python/bytecodes.c" + value = GETLOCAL(oparg); + if (value == NULL) goto unbound_local_error; + Py_INCREF(value); +- #line 78 "Python/generated_cases.c.h" ++ #line 88 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = value; + DISPATCH(); +@@ -82,11 +92,11 @@ + + TARGET(LOAD_FAST) { + PyObject *value; +- #line 192 "Python/bytecodes.c" ++ #line 202 "Python/bytecodes.c" + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); +- #line 90 "Python/generated_cases.c.h" ++ #line 100 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = value; + DISPATCH(); +@@ -94,11 +104,11 @@ + + TARGET(LOAD_FAST_AND_CLEAR) { + PyObject *value; +- #line 198 "Python/bytecodes.c" ++ #line 208 "Python/bytecodes.c" + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = NULL; +- #line 102 "Python/generated_cases.c.h" ++ #line 112 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = value; + DISPATCH(); +@@ -107,10 +117,10 @@ + TARGET(LOAD_CONST) { + PREDICTED(LOAD_CONST); + PyObject *value; +- #line 204 "Python/bytecodes.c" ++ #line 214 "Python/bytecodes.c" + value = GETITEM(frame->f_code->co_consts, oparg); + Py_INCREF(value); +- #line 114 "Python/generated_cases.c.h" ++ #line 124 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = value; + DISPATCH(); +@@ -118,9 +128,17 @@ + + TARGET(STORE_FAST) { + PyObject *value = stack_pointer[-1]; +- #line 209 "Python/bytecodes.c" ++ #line 219 "Python/bytecodes.c" ++ PyObject *name = GETITEM(frame->f_code->co_localsplusnames, oparg); ++ int err = aiython_before_store(frame, name, value); ++ if (err < 0) { ++ #line 136 "Python/generated_cases.c.h" ++ Py_DECREF(value); ++ #line 223 "Python/bytecodes.c" ++ if (true) goto pop_1_error; ++ } + SETLOCAL(oparg, value); +- #line 124 "Python/generated_cases.c.h" ++ #line 142 "Python/generated_cases.c.h" + STACK_SHRINK(1); + DISPATCH(); + } +@@ -130,21 +148,21 @@ + PyObject *_tmp_2; + { + PyObject *value; +- #line 192 "Python/bytecodes.c" ++ #line 202 "Python/bytecodes.c" + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); +- #line 138 "Python/generated_cases.c.h" ++ #line 156 "Python/generated_cases.c.h" + _tmp_2 = value; + } + oparg = (next_instr++)->op.arg; + { + PyObject *value; +- #line 192 "Python/bytecodes.c" ++ #line 202 "Python/bytecodes.c" + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); +- #line 148 "Python/generated_cases.c.h" ++ #line 166 "Python/generated_cases.c.h" + _tmp_1 = value; + } + STACK_GROW(2); +@@ -158,20 +176,20 @@ + PyObject *_tmp_2; + { + PyObject *value; +- #line 192 "Python/bytecodes.c" ++ #line 202 "Python/bytecodes.c" + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); +- #line 166 "Python/generated_cases.c.h" ++ #line 184 "Python/generated_cases.c.h" + _tmp_2 = value; + } + oparg = (next_instr++)->op.arg; + { + PyObject *value; +- #line 204 "Python/bytecodes.c" ++ #line 214 "Python/bytecodes.c" + value = GETITEM(frame->f_code->co_consts, oparg); + Py_INCREF(value); +- #line 175 "Python/generated_cases.c.h" ++ #line 193 "Python/generated_cases.c.h" + _tmp_1 = value; + } + STACK_GROW(2); +@@ -184,18 +202,26 @@ + PyObject *_tmp_1 = stack_pointer[-1]; + { + PyObject *value = _tmp_1; +- #line 209 "Python/bytecodes.c" ++ #line 219 "Python/bytecodes.c" ++ PyObject *name = GETITEM(frame->f_code->co_localsplusnames, oparg); ++ int err = aiython_before_store(frame, name, value); ++ if (err < 0) { ++ #line 210 "Python/generated_cases.c.h" ++ Py_DECREF(value); ++ #line 223 "Python/bytecodes.c" ++ if (true) goto pop_1_error; ++ } + SETLOCAL(oparg, value); +- #line 190 "Python/generated_cases.c.h" ++ #line 216 "Python/generated_cases.c.h" + } + oparg = (next_instr++)->op.arg; + { + PyObject *value; +- #line 192 "Python/bytecodes.c" ++ #line 202 "Python/bytecodes.c" + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); +- #line 199 "Python/generated_cases.c.h" ++ #line 225 "Python/generated_cases.c.h" + _tmp_1 = value; + } + stack_pointer[-1] = _tmp_1; +@@ -207,16 +233,32 @@ + PyObject *_tmp_2 = stack_pointer[-2]; + { + PyObject *value = _tmp_1; +- #line 209 "Python/bytecodes.c" ++ #line 219 "Python/bytecodes.c" ++ PyObject *name = GETITEM(frame->f_code->co_localsplusnames, oparg); ++ int err = aiython_before_store(frame, name, value); ++ if (err < 0) { ++ #line 241 "Python/generated_cases.c.h" ++ Py_DECREF(value); ++ #line 223 "Python/bytecodes.c" ++ if (true) goto pop_1_error; ++ } + SETLOCAL(oparg, value); +- #line 213 "Python/generated_cases.c.h" ++ #line 247 "Python/generated_cases.c.h" + } + oparg = (next_instr++)->op.arg; + { + PyObject *value = _tmp_2; +- #line 209 "Python/bytecodes.c" ++ #line 219 "Python/bytecodes.c" ++ PyObject *name = GETITEM(frame->f_code->co_localsplusnames, oparg); ++ int err = aiython_before_store(frame, name, value); ++ if (err < 0) { ++ #line 256 "Python/generated_cases.c.h" ++ Py_DECREF(value); ++ #line 223 "Python/bytecodes.c" ++ if (true) goto pop_1_error; ++ } + SETLOCAL(oparg, value); +- #line 220 "Python/generated_cases.c.h" ++ #line 262 "Python/generated_cases.c.h" + } + STACK_SHRINK(2); + DISPATCH(); +@@ -227,20 +269,20 @@ + PyObject *_tmp_2; + { + PyObject *value; +- #line 204 "Python/bytecodes.c" ++ #line 214 "Python/bytecodes.c" + value = GETITEM(frame->f_code->co_consts, oparg); + Py_INCREF(value); +- #line 234 "Python/generated_cases.c.h" ++ #line 276 "Python/generated_cases.c.h" + _tmp_2 = value; + } + oparg = (next_instr++)->op.arg; + { + PyObject *value; +- #line 192 "Python/bytecodes.c" ++ #line 202 "Python/bytecodes.c" + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); +- #line 244 "Python/generated_cases.c.h" ++ #line 286 "Python/generated_cases.c.h" + _tmp_1 = value; + } + STACK_GROW(2); +@@ -251,8 +293,8 @@ + + TARGET(POP_TOP) { + PyObject *value = stack_pointer[-1]; +- #line 219 "Python/bytecodes.c" +- #line 256 "Python/generated_cases.c.h" ++ #line 235 "Python/bytecodes.c" ++ #line 298 "Python/generated_cases.c.h" + Py_DECREF(value); + STACK_SHRINK(1); + DISPATCH(); +@@ -260,9 +302,9 @@ + + TARGET(PUSH_NULL) { + PyObject *res; +- #line 223 "Python/bytecodes.c" ++ #line 239 "Python/bytecodes.c" + res = NULL; +- #line 266 "Python/generated_cases.c.h" ++ #line 308 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = res; + DISPATCH(); +@@ -273,14 +315,14 @@ + PyObject *_tmp_2 = stack_pointer[-2]; + { + PyObject *value = _tmp_1; +- #line 219 "Python/bytecodes.c" +- #line 278 "Python/generated_cases.c.h" ++ #line 235 "Python/bytecodes.c" ++ #line 320 "Python/generated_cases.c.h" + Py_DECREF(value); + } + { + PyObject *value = _tmp_2; +- #line 219 "Python/bytecodes.c" +- #line 284 "Python/generated_cases.c.h" ++ #line 235 "Python/bytecodes.c" ++ #line 326 "Python/generated_cases.c.h" + Py_DECREF(value); + } + STACK_SHRINK(2); +@@ -290,7 +332,7 @@ + TARGET(INSTRUMENTED_END_FOR) { + PyObject *value = stack_pointer[-1]; + PyObject *receiver = stack_pointer[-2]; +- #line 229 "Python/bytecodes.c" ++ #line 245 "Python/bytecodes.c" + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyGen_Check(receiver)) { +@@ -300,7 +342,7 @@ + } + PyErr_SetRaisedException(NULL); + } +- #line 304 "Python/generated_cases.c.h" ++ #line 346 "Python/generated_cases.c.h" + Py_DECREF(receiver); + Py_DECREF(value); + STACK_SHRINK(2); +@@ -310,9 +352,9 @@ + TARGET(END_SEND) { + PyObject *value = stack_pointer[-1]; + PyObject *receiver = stack_pointer[-2]; +- #line 242 "Python/bytecodes.c" ++ #line 258 "Python/bytecodes.c" + Py_DECREF(receiver); +- #line 316 "Python/generated_cases.c.h" ++ #line 358 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = value; + DISPATCH(); +@@ -321,7 +363,7 @@ + TARGET(INSTRUMENTED_END_SEND) { + PyObject *value = stack_pointer[-1]; + PyObject *receiver = stack_pointer[-2]; +- #line 246 "Python/bytecodes.c" ++ #line 262 "Python/bytecodes.c" + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) { +@@ -330,7 +372,7 @@ + PyErr_SetRaisedException(NULL); + } + Py_DECREF(receiver); +- #line 334 "Python/generated_cases.c.h" ++ #line 376 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = value; + DISPATCH(); +@@ -339,13 +381,13 @@ + TARGET(UNARY_NEGATIVE) { + PyObject *value = stack_pointer[-1]; + PyObject *res; +- #line 257 "Python/bytecodes.c" ++ #line 273 "Python/bytecodes.c" + res = PyNumber_Negative(value); +- #line 345 "Python/generated_cases.c.h" ++ #line 387 "Python/generated_cases.c.h" + Py_DECREF(value); +- #line 259 "Python/bytecodes.c" ++ #line 275 "Python/bytecodes.c" + if (res == NULL) goto pop_1_error; +- #line 349 "Python/generated_cases.c.h" ++ #line 391 "Python/generated_cases.c.h" + stack_pointer[-1] = res; + DISPATCH(); + } +@@ -353,11 +395,11 @@ + TARGET(UNARY_NOT) { + PyObject *value = stack_pointer[-1]; + PyObject *res; +- #line 263 "Python/bytecodes.c" ++ #line 279 "Python/bytecodes.c" + int err = PyObject_IsTrue(value); +- #line 359 "Python/generated_cases.c.h" ++ #line 401 "Python/generated_cases.c.h" + Py_DECREF(value); +- #line 265 "Python/bytecodes.c" ++ #line 281 "Python/bytecodes.c" + if (err < 0) goto pop_1_error; + if (err == 0) { + res = Py_True; +@@ -365,7 +407,7 @@ + else { + res = Py_False; + } +- #line 369 "Python/generated_cases.c.h" ++ #line 411 "Python/generated_cases.c.h" + stack_pointer[-1] = res; + DISPATCH(); + } +@@ -373,13 +415,13 @@ + TARGET(UNARY_INVERT) { + PyObject *value = stack_pointer[-1]; + PyObject *res; +- #line 275 "Python/bytecodes.c" ++ #line 291 "Python/bytecodes.c" + res = PyNumber_Invert(value); +- #line 379 "Python/generated_cases.c.h" ++ #line 421 "Python/generated_cases.c.h" + Py_DECREF(value); +- #line 277 "Python/bytecodes.c" ++ #line 293 "Python/bytecodes.c" + if (res == NULL) goto pop_1_error; +- #line 383 "Python/generated_cases.c.h" ++ #line 425 "Python/generated_cases.c.h" + stack_pointer[-1] = res; + DISPATCH(); + } +@@ -388,7 +430,7 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *prod; +- #line 294 "Python/bytecodes.c" ++ #line 310 "Python/bytecodes.c" + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + STAT_INC(BINARY_OP, hit); +@@ -396,7 +438,7 @@ + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (prod == NULL) goto pop_2_error; +- #line 400 "Python/generated_cases.c.h" ++ #line 442 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = prod; + next_instr += 1; +@@ -407,14 +449,14 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *prod; +- #line 304 "Python/bytecodes.c" ++ #line 320 "Python/bytecodes.c" + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + STAT_INC(BINARY_OP, hit); + double dprod = ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); +- #line 418 "Python/generated_cases.c.h" ++ #line 460 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = prod; + next_instr += 1; +@@ -425,7 +467,7 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *sub; +- #line 313 "Python/bytecodes.c" ++ #line 329 "Python/bytecodes.c" + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + STAT_INC(BINARY_OP, hit); +@@ -433,7 +475,7 @@ + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (sub == NULL) goto pop_2_error; +- #line 437 "Python/generated_cases.c.h" ++ #line 479 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = sub; + next_instr += 1; +@@ -444,13 +486,13 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *sub; +- #line 323 "Python/bytecodes.c" ++ #line 339 "Python/bytecodes.c" + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + STAT_INC(BINARY_OP, hit); + double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); +- #line 454 "Python/generated_cases.c.h" ++ #line 496 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = sub; + next_instr += 1; +@@ -461,7 +503,7 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *res; +- #line 331 "Python/bytecodes.c" ++ #line 347 "Python/bytecodes.c" + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); + STAT_INC(BINARY_OP, hit); +@@ -469,7 +511,7 @@ + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (res == NULL) goto pop_2_error; +- #line 473 "Python/generated_cases.c.h" ++ #line 515 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = res; + next_instr += 1; +@@ -479,7 +521,9 @@ + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; +- #line 347 "Python/bytecodes.c" ++ #line 363 "Python/bytecodes.c" ++ int store_hook_enabled = aiython_store_hook_enabled(); ++ DEOPT_IF(store_hook_enabled, BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); + _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; +@@ -506,7 +550,7 @@ + if (*target_local == NULL) goto pop_2_error; + // The STORE_FAST is already done. + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); +- #line 510 "Python/generated_cases.c.h" ++ #line 554 "Python/generated_cases.c.h" + STACK_SHRINK(2); + DISPATCH(); + } +@@ -515,14 +559,14 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *sum; +- #line 376 "Python/bytecodes.c" ++ #line 394 "Python/bytecodes.c" + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); + STAT_INC(BINARY_OP, hit); + double dsum = ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); +- #line 526 "Python/generated_cases.c.h" ++ #line 570 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = sum; + next_instr += 1; +@@ -533,7 +577,7 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *sum; +- #line 385 "Python/bytecodes.c" ++ #line 403 "Python/bytecodes.c" + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); + STAT_INC(BINARY_OP, hit); +@@ -541,7 +585,7 @@ + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (sum == NULL) goto pop_2_error; +- #line 545 "Python/generated_cases.c.h" ++ #line 589 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = sum; + next_instr += 1; +@@ -554,7 +598,7 @@ + PyObject *sub = stack_pointer[-1]; + PyObject *container = stack_pointer[-2]; + PyObject *res; +- #line 403 "Python/bytecodes.c" ++ #line 421 "Python/bytecodes.c" + #if ENABLE_SPECIALIZATION + _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { +@@ -566,12 +610,12 @@ + DECREMENT_ADAPTIVE_COUNTER(cache->counter); + #endif /* ENABLE_SPECIALIZATION */ + res = PyObject_GetItem(container, sub); +- #line 570 "Python/generated_cases.c.h" ++ #line 614 "Python/generated_cases.c.h" + Py_DECREF(container); + Py_DECREF(sub); +- #line 415 "Python/bytecodes.c" ++ #line 433 "Python/bytecodes.c" + if (res == NULL) goto pop_2_error; +- #line 575 "Python/generated_cases.c.h" ++ #line 619 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = res; + next_instr += 1; +@@ -583,7 +627,7 @@ + PyObject *start = stack_pointer[-2]; + PyObject *container = stack_pointer[-3]; + PyObject *res; +- #line 419 "Python/bytecodes.c" ++ #line 437 "Python/bytecodes.c" + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. +@@ -596,7 +640,7 @@ + } + Py_DECREF(container); + if (res == NULL) goto pop_3_error; +- #line 600 "Python/generated_cases.c.h" ++ #line 644 "Python/generated_cases.c.h" + STACK_SHRINK(2); + stack_pointer[-1] = res; + DISPATCH(); +@@ -607,20 +651,23 @@ + PyObject *start = stack_pointer[-2]; + PyObject *container = stack_pointer[-3]; + PyObject *v = stack_pointer[-4]; +- #line 434 "Python/bytecodes.c" ++ #line 452 "Python/bytecodes.c" + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + int err; + if (slice == NULL) { + err = 1; + } + else { +- err = PyObject_SetItem(container, slice, v); ++ err = aiython_before_mutation(frame, "item", container, slice, v); ++ if (err == 0) { ++ err = PyObject_SetItem(container, slice, v); ++ } + Py_DECREF(slice); + } + Py_DECREF(v); + Py_DECREF(container); + if (err) goto pop_4_error; +- #line 624 "Python/generated_cases.c.h" ++ #line 671 "Python/generated_cases.c.h" + STACK_SHRINK(4); + DISPATCH(); + } +@@ -629,7 +676,7 @@ + PyObject *sub = stack_pointer[-1]; + PyObject *list = stack_pointer[-2]; + PyObject *res; +- #line 449 "Python/bytecodes.c" ++ #line 470 "Python/bytecodes.c" + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + +@@ -643,7 +690,7 @@ + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); +- #line 647 "Python/generated_cases.c.h" ++ #line 694 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = res; + next_instr += 1; +@@ -654,7 +701,7 @@ + PyObject *sub = stack_pointer[-1]; + PyObject *tuple = stack_pointer[-2]; + PyObject *res; +- #line 465 "Python/bytecodes.c" ++ #line 486 "Python/bytecodes.c" + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + +@@ -668,7 +715,7 @@ + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(tuple); +- #line 672 "Python/generated_cases.c.h" ++ #line 719 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = res; + next_instr += 1; +@@ -679,7 +726,7 @@ + PyObject *sub = stack_pointer[-1]; + PyObject *dict = stack_pointer[-2]; + PyObject *res; +- #line 481 "Python/bytecodes.c" ++ #line 502 "Python/bytecodes.c" + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = PyDict_GetItemWithError(dict, sub); +@@ -687,14 +734,14 @@ + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetKeyError(sub); + } +- #line 691 "Python/generated_cases.c.h" ++ #line 738 "Python/generated_cases.c.h" + Py_DECREF(dict); + Py_DECREF(sub); +- #line 489 "Python/bytecodes.c" ++ #line 510 "Python/bytecodes.c" + if (true) goto pop_2_error; + } + Py_INCREF(res); // Do this before DECREF'ing dict, sub +- #line 698 "Python/generated_cases.c.h" ++ #line 745 "Python/generated_cases.c.h" + Py_DECREF(dict); + Py_DECREF(sub); + STACK_SHRINK(1); +@@ -706,7 +753,7 @@ + TARGET(BINARY_SUBSCR_GETITEM) { + PyObject *sub = stack_pointer[-1]; + PyObject *container = stack_pointer[-2]; +- #line 496 "Python/bytecodes.c" ++ #line 517 "Python/bytecodes.c" + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + PyTypeObject *tp = Py_TYPE(container); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); +@@ -729,15 +776,15 @@ + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + frame->return_offset = 0; + DISPATCH_INLINED(new_frame); +- #line 733 "Python/generated_cases.c.h" ++ #line 780 "Python/generated_cases.c.h" + } + + TARGET(LIST_APPEND) { + PyObject *v = stack_pointer[-1]; + PyObject *list = stack_pointer[-(2 + (oparg-1))]; +- #line 521 "Python/bytecodes.c" ++ #line 542 "Python/bytecodes.c" + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; +- #line 741 "Python/generated_cases.c.h" ++ #line 788 "Python/generated_cases.c.h" + STACK_SHRINK(1); + PREDICT(JUMP_BACKWARD); + DISPATCH(); +@@ -746,13 +793,13 @@ + TARGET(SET_ADD) { + PyObject *v = stack_pointer[-1]; + PyObject *set = stack_pointer[-(2 + (oparg-1))]; +- #line 526 "Python/bytecodes.c" ++ #line 547 "Python/bytecodes.c" + int err = PySet_Add(set, v); +- #line 752 "Python/generated_cases.c.h" ++ #line 799 "Python/generated_cases.c.h" + Py_DECREF(v); +- #line 528 "Python/bytecodes.c" ++ #line 549 "Python/bytecodes.c" + if (err) goto pop_1_error; +- #line 756 "Python/generated_cases.c.h" ++ #line 803 "Python/generated_cases.c.h" + STACK_SHRINK(1); + PREDICT(JUMP_BACKWARD); + DISPATCH(); +@@ -765,7 +812,7 @@ + PyObject *container = stack_pointer[-2]; + PyObject *v = stack_pointer[-3]; + uint16_t counter = read_u16(&next_instr[0].cache); +- #line 539 "Python/bytecodes.c" ++ #line 560 "Python/bytecodes.c" + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr--; +@@ -779,14 +826,17 @@ + (void)counter; // Unused. + #endif /* ENABLE_SPECIALIZATION */ + /* container[sub] = v */ +- int err = PyObject_SetItem(container, sub, v); +- #line 784 "Python/generated_cases.c.h" ++ int err = aiython_before_mutation(frame, "item", container, sub, v); ++ if (err == 0) { ++ err = PyObject_SetItem(container, sub, v); ++ } ++ #line 834 "Python/generated_cases.c.h" + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); +- #line 554 "Python/bytecodes.c" ++ #line 578 "Python/bytecodes.c" + if (err) goto pop_3_error; +- #line 790 "Python/generated_cases.c.h" ++ #line 840 "Python/generated_cases.c.h" + STACK_SHRINK(3); + next_instr += 1; + DISPATCH(); +@@ -796,7 +846,9 @@ + PyObject *sub = stack_pointer[-1]; + PyObject *list = stack_pointer[-2]; + PyObject *value = stack_pointer[-3]; +- #line 558 "Python/bytecodes.c" ++ #line 582 "Python/bytecodes.c" ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + +@@ -813,7 +865,7 @@ + Py_DECREF(old_value); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); +- #line 817 "Python/generated_cases.c.h" ++ #line 869 "Python/generated_cases.c.h" + STACK_SHRINK(3); + next_instr += 1; + DISPATCH(); +@@ -823,13 +875,15 @@ + PyObject *sub = stack_pointer[-1]; + PyObject *dict = stack_pointer[-2]; + PyObject *value = stack_pointer[-3]; +- #line 577 "Python/bytecodes.c" ++ #line 603 "Python/bytecodes.c" ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); + Py_DECREF(dict); + if (err) goto pop_3_error; +- #line 833 "Python/generated_cases.c.h" ++ #line 887 "Python/generated_cases.c.h" + STACK_SHRINK(3); + next_instr += 1; + DISPATCH(); +@@ -838,15 +892,15 @@ + TARGET(DELETE_SUBSCR) { + PyObject *sub = stack_pointer[-1]; + PyObject *container = stack_pointer[-2]; +- #line 585 "Python/bytecodes.c" ++ #line 613 "Python/bytecodes.c" + /* del container[sub] */ + int err = PyObject_DelItem(container, sub); +- #line 845 "Python/generated_cases.c.h" ++ #line 899 "Python/generated_cases.c.h" + Py_DECREF(container); + Py_DECREF(sub); +- #line 588 "Python/bytecodes.c" ++ #line 616 "Python/bytecodes.c" + if (err) goto pop_2_error; +- #line 850 "Python/generated_cases.c.h" ++ #line 904 "Python/generated_cases.c.h" + STACK_SHRINK(2); + DISPATCH(); + } +@@ -854,14 +908,14 @@ + TARGET(CALL_INTRINSIC_1) { + PyObject *value = stack_pointer[-1]; + PyObject *res; +- #line 592 "Python/bytecodes.c" ++ #line 620 "Python/bytecodes.c" + assert(oparg <= MAX_INTRINSIC_1); + res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); +- #line 861 "Python/generated_cases.c.h" ++ #line 915 "Python/generated_cases.c.h" + Py_DECREF(value); +- #line 595 "Python/bytecodes.c" ++ #line 623 "Python/bytecodes.c" + if (res == NULL) goto pop_1_error; +- #line 865 "Python/generated_cases.c.h" ++ #line 919 "Python/generated_cases.c.h" + stack_pointer[-1] = res; + DISPATCH(); + } +@@ -870,15 +924,15 @@ + PyObject *value1 = stack_pointer[-1]; + PyObject *value2 = stack_pointer[-2]; + PyObject *res; +- #line 599 "Python/bytecodes.c" ++ #line 627 "Python/bytecodes.c" + assert(oparg <= MAX_INTRINSIC_2); + res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); +- #line 877 "Python/generated_cases.c.h" ++ #line 931 "Python/generated_cases.c.h" + Py_DECREF(value2); + Py_DECREF(value1); +- #line 602 "Python/bytecodes.c" ++ #line 630 "Python/bytecodes.c" + if (res == NULL) goto pop_2_error; +- #line 882 "Python/generated_cases.c.h" ++ #line 936 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = res; + DISPATCH(); +@@ -886,7 +940,7 @@ + + TARGET(RAISE_VARARGS) { + PyObject **args = (stack_pointer - oparg); +- #line 606 "Python/bytecodes.c" ++ #line 634 "Python/bytecodes.c" + PyObject *cause = NULL, *exc = NULL; + switch (oparg) { + case 2: +@@ -908,12 +962,12 @@ + break; + } + if (true) { STACK_SHRINK(oparg); goto error; } +- #line 912 "Python/generated_cases.c.h" ++ #line 966 "Python/generated_cases.c.h" + } + + TARGET(INTERPRETER_EXIT) { + PyObject *retval = stack_pointer[-1]; +- #line 630 "Python/bytecodes.c" ++ #line 658 "Python/bytecodes.c" + assert(frame == &entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + STACK_SHRINK(1); // Since we're not going to DISPATCH() +@@ -924,12 +978,14 @@ + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return retval; +- #line 928 "Python/generated_cases.c.h" ++ #line 982 "Python/generated_cases.c.h" + } + + TARGET(RETURN_VALUE) { + PyObject *retval = stack_pointer[-1]; +- #line 643 "Python/bytecodes.c" ++ #line 671 "Python/bytecodes.c" ++ int hook_err = aiython_function_event(frame, "_aiython_on_return", retval); ++ if (hook_err) goto pop_1_error; + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); +@@ -942,16 +998,18 @@ + frame->prev_instr += frame->return_offset; + _PyFrame_StackPush(frame, retval); + goto resume_frame; +- #line 946 "Python/generated_cases.c.h" ++ #line 1002 "Python/generated_cases.c.h" + } + + TARGET(INSTRUMENTED_RETURN_VALUE) { + PyObject *retval = stack_pointer[-1]; +- #line 658 "Python/bytecodes.c" ++ #line 688 "Python/bytecodes.c" + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, retval); + if (err) goto error; ++ err = aiython_function_event(frame, "_aiython_on_return", retval); ++ if (err) goto error; + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); +@@ -964,12 +1022,14 @@ + frame->prev_instr += frame->return_offset; + _PyFrame_StackPush(frame, retval); + goto resume_frame; +- #line 968 "Python/generated_cases.c.h" ++ #line 1026 "Python/generated_cases.c.h" + } + + TARGET(RETURN_CONST) { +- #line 677 "Python/bytecodes.c" ++ #line 709 "Python/bytecodes.c" + PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); ++ int hook_err = aiython_function_event(frame, "_aiython_on_return", retval); ++ if (hook_err) goto error; + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); +@@ -982,16 +1042,18 @@ + frame->prev_instr += frame->return_offset; + _PyFrame_StackPush(frame, retval); + goto resume_frame; +- #line 986 "Python/generated_cases.c.h" ++ #line 1046 "Python/generated_cases.c.h" + } + + TARGET(INSTRUMENTED_RETURN_CONST) { +- #line 693 "Python/bytecodes.c" ++ #line 727 "Python/bytecodes.c" + PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, retval); + if (err) goto error; ++ err = aiython_function_event(frame, "_aiython_on_return", retval); ++ if (err) goto error; + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); +@@ -1004,13 +1066,13 @@ + frame->prev_instr += frame->return_offset; + _PyFrame_StackPush(frame, retval); + goto resume_frame; +- #line 1008 "Python/generated_cases.c.h" ++ #line 1070 "Python/generated_cases.c.h" + } + + TARGET(GET_AITER) { + PyObject *obj = stack_pointer[-1]; + PyObject *iter; +- #line 713 "Python/bytecodes.c" ++ #line 749 "Python/bytecodes.c" + unaryfunc getter = NULL; + PyTypeObject *type = Py_TYPE(obj); + +@@ -1023,16 +1085,16 @@ + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); +- #line 1027 "Python/generated_cases.c.h" ++ #line 1089 "Python/generated_cases.c.h" + Py_DECREF(obj); +- #line 726 "Python/bytecodes.c" ++ #line 762 "Python/bytecodes.c" + if (true) goto pop_1_error; + } + + iter = (*getter)(obj); +- #line 1034 "Python/generated_cases.c.h" ++ #line 1096 "Python/generated_cases.c.h" + Py_DECREF(obj); +- #line 731 "Python/bytecodes.c" ++ #line 767 "Python/bytecodes.c" + if (iter == NULL) goto pop_1_error; + + if (Py_TYPE(iter)->tp_as_async == NULL || +@@ -1045,7 +1107,7 @@ + Py_DECREF(iter); + if (true) goto pop_1_error; + } +- #line 1049 "Python/generated_cases.c.h" ++ #line 1111 "Python/generated_cases.c.h" + stack_pointer[-1] = iter; + DISPATCH(); + } +@@ -1053,7 +1115,7 @@ + TARGET(GET_ANEXT) { + PyObject *aiter = stack_pointer[-1]; + PyObject *awaitable; +- #line 746 "Python/bytecodes.c" ++ #line 782 "Python/bytecodes.c" + unaryfunc getter = NULL; + PyObject *next_iter = NULL; + PyTypeObject *type = Py_TYPE(aiter); +@@ -1097,7 +1159,7 @@ + } + } + +- #line 1101 "Python/generated_cases.c.h" ++ #line 1163 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = awaitable; + PREDICT(LOAD_CONST); +@@ -1108,16 +1170,16 @@ + PREDICTED(GET_AWAITABLE); + PyObject *iterable = stack_pointer[-1]; + PyObject *iter; +- #line 793 "Python/bytecodes.c" ++ #line 829 "Python/bytecodes.c" + iter = _PyCoro_GetAwaitableIter(iterable); + + if (iter == NULL) { + format_awaitable_error(tstate, Py_TYPE(iterable), oparg); + } + +- #line 1119 "Python/generated_cases.c.h" ++ #line 1181 "Python/generated_cases.c.h" + Py_DECREF(iterable); +- #line 800 "Python/bytecodes.c" ++ #line 836 "Python/bytecodes.c" + + if (iter != NULL && PyCoro_CheckExact(iter)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter); +@@ -1135,7 +1197,7 @@ + + if (iter == NULL) goto pop_1_error; + +- #line 1139 "Python/generated_cases.c.h" ++ #line 1201 "Python/generated_cases.c.h" + stack_pointer[-1] = iter; + PREDICT(LOAD_CONST); + DISPATCH(); +@@ -1147,7 +1209,7 @@ + PyObject *v = stack_pointer[-1]; + PyObject *receiver = stack_pointer[-2]; + PyObject *retval; +- #line 826 "Python/bytecodes.c" ++ #line 862 "Python/bytecodes.c" + #if ENABLE_SPECIALIZATION + _PySendCache *cache = (_PySendCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { +@@ -1194,7 +1256,7 @@ + } + } + Py_DECREF(v); +- #line 1198 "Python/generated_cases.c.h" ++ #line 1260 "Python/generated_cases.c.h" + stack_pointer[-1] = retval; + next_instr += 1; + DISPATCH(); +@@ -1203,7 +1265,7 @@ + TARGET(SEND_GEN) { + PyObject *v = stack_pointer[-1]; + PyObject *receiver = stack_pointer[-2]; +- #line 875 "Python/bytecodes.c" ++ #line 911 "Python/bytecodes.c" + DEOPT_IF(tstate->interp->eval_frame, SEND); + PyGenObject *gen = (PyGenObject *)receiver; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && +@@ -1219,12 +1281,12 @@ + tstate->exc_info = &gen->gi_exc_state; + JUMPBY(INLINE_CACHE_ENTRIES_SEND); + DISPATCH_INLINED(gen_frame); +- #line 1223 "Python/generated_cases.c.h" ++ #line 1285 "Python/generated_cases.c.h" + } + + TARGET(INSTRUMENTED_YIELD_VALUE) { + PyObject *retval = stack_pointer[-1]; +- #line 893 "Python/bytecodes.c" ++ #line 929 "Python/bytecodes.c" + assert(frame != &entry_frame); + PyGenObject *gen = _PyFrame_GetGenerator(frame); + gen->gi_frame_state = FRAME_SUSPENDED; +@@ -1241,12 +1303,12 @@ + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + goto resume_frame; +- #line 1245 "Python/generated_cases.c.h" ++ #line 1307 "Python/generated_cases.c.h" + } + + TARGET(YIELD_VALUE) { + PyObject *retval = stack_pointer[-1]; +- #line 912 "Python/bytecodes.c" ++ #line 948 "Python/bytecodes.c" + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. +@@ -1262,15 +1324,15 @@ + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + goto resume_frame; +- #line 1266 "Python/generated_cases.c.h" ++ #line 1328 "Python/generated_cases.c.h" + } + + TARGET(POP_EXCEPT) { + PyObject *exc_value = stack_pointer[-1]; +- #line 930 "Python/bytecodes.c" ++ #line 966 "Python/bytecodes.c" + _PyErr_StackItem *exc_info = tstate->exc_info; + Py_XSETREF(exc_info->exc_value, exc_value); +- #line 1274 "Python/generated_cases.c.h" ++ #line 1336 "Python/generated_cases.c.h" + STACK_SHRINK(1); + DISPATCH(); + } +@@ -1278,7 +1340,7 @@ + TARGET(RERAISE) { + PyObject *exc = stack_pointer[-1]; + PyObject **values = (stack_pointer - (1 + oparg)); +- #line 935 "Python/bytecodes.c" ++ #line 971 "Python/bytecodes.c" + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + PyObject *lasti = values[0]; +@@ -1297,19 +1359,19 @@ + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, next_instr-1); + goto exception_unwind; +- #line 1301 "Python/generated_cases.c.h" ++ #line 1363 "Python/generated_cases.c.h" + } + + TARGET(END_ASYNC_FOR) { + PyObject *exc = stack_pointer[-1]; + PyObject *awaitable = stack_pointer[-2]; +- #line 956 "Python/bytecodes.c" ++ #line 992 "Python/bytecodes.c" + assert(exc && PyExceptionInstance_Check(exc)); + if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { +- #line 1310 "Python/generated_cases.c.h" ++ #line 1372 "Python/generated_cases.c.h" + Py_DECREF(awaitable); + Py_DECREF(exc); +- #line 959 "Python/bytecodes.c" ++ #line 995 "Python/bytecodes.c" + } + else { + Py_INCREF(exc); +@@ -1317,7 +1379,7 @@ + monitor_reraise(tstate, frame, next_instr-1); + goto exception_unwind; + } +- #line 1321 "Python/generated_cases.c.h" ++ #line 1383 "Python/generated_cases.c.h" + STACK_SHRINK(2); + DISPATCH(); + } +@@ -1328,16 +1390,16 @@ + PyObject *sub_iter = stack_pointer[-3]; + PyObject *none; + PyObject *value; +- #line 969 "Python/bytecodes.c" ++ #line 1005 "Python/bytecodes.c" + assert(throwflag); + assert(exc_value && PyExceptionInstance_Check(exc_value)); + if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { + value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); +- #line 1337 "Python/generated_cases.c.h" ++ #line 1399 "Python/generated_cases.c.h" + Py_DECREF(sub_iter); + Py_DECREF(last_sent_val); + Py_DECREF(exc_value); +- #line 974 "Python/bytecodes.c" ++ #line 1010 "Python/bytecodes.c" + none = Py_None; + } + else { +@@ -1345,7 +1407,7 @@ + monitor_reraise(tstate, frame, next_instr-1); + goto exception_unwind; + } +- #line 1349 "Python/generated_cases.c.h" ++ #line 1411 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = value; + stack_pointer[-2] = none; +@@ -1354,9 +1416,9 @@ + + TARGET(LOAD_ASSERTION_ERROR) { + PyObject *value; +- #line 984 "Python/bytecodes.c" ++ #line 1020 "Python/bytecodes.c" + value = Py_NewRef(PyExc_AssertionError); +- #line 1360 "Python/generated_cases.c.h" ++ #line 1422 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = value; + DISPATCH(); +@@ -1364,7 +1426,7 @@ + + TARGET(LOAD_BUILD_CLASS) { + PyObject *bc; +- #line 988 "Python/bytecodes.c" ++ #line 1024 "Python/bytecodes.c" + if (PyDict_CheckExact(BUILTINS())) { + bc = _PyDict_GetItemWithError(BUILTINS(), + &_Py_ID(__build_class__)); +@@ -1386,7 +1448,7 @@ + if (true) goto error; + } + } +- #line 1390 "Python/generated_cases.c.h" ++ #line 1452 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = bc; + DISPATCH(); +@@ -1394,33 +1456,40 @@ + + TARGET(STORE_NAME) { + PyObject *v = stack_pointer[-1]; +- #line 1013 "Python/bytecodes.c" ++ #line 1049 "Python/bytecodes.c" + PyObject *name = GETITEM(frame->f_code->co_names, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); +- #line 1405 "Python/generated_cases.c.h" ++ #line 1467 "Python/generated_cases.c.h" + Py_DECREF(v); +- #line 1020 "Python/bytecodes.c" ++ #line 1056 "Python/bytecodes.c" ++ if (true) goto pop_1_error; ++ } ++ err = aiython_before_store(frame, name, v); ++ if (err < 0) { ++ #line 1474 "Python/generated_cases.c.h" ++ Py_DECREF(v); ++ #line 1061 "Python/bytecodes.c" + if (true) goto pop_1_error; + } + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else + err = PyObject_SetItem(ns, name, v); +- #line 1414 "Python/generated_cases.c.h" ++ #line 1483 "Python/generated_cases.c.h" + Py_DECREF(v); +- #line 1027 "Python/bytecodes.c" ++ #line 1068 "Python/bytecodes.c" + if (err) goto pop_1_error; +- #line 1418 "Python/generated_cases.c.h" ++ #line 1487 "Python/generated_cases.c.h" + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(DELETE_NAME) { +- #line 1031 "Python/bytecodes.c" ++ #line 1072 "Python/bytecodes.c" + PyObject *name = GETITEM(frame->f_code->co_names, oparg); + PyObject *ns = LOCALS(); + int err; +@@ -1437,7 +1506,7 @@ + name); + goto error; + } +- #line 1441 "Python/generated_cases.c.h" ++ #line 1510 "Python/generated_cases.c.h" + DISPATCH(); + } + +@@ -1445,7 +1514,7 @@ + PREDICTED(UNPACK_SEQUENCE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq = stack_pointer[-1]; +- #line 1057 "Python/bytecodes.c" ++ #line 1098 "Python/bytecodes.c" + #if ENABLE_SPECIALIZATION + _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { +@@ -1458,11 +1527,11 @@ + #endif /* ENABLE_SPECIALIZATION */ + PyObject **top = stack_pointer + oparg - 1; + int res = unpack_iterable(tstate, seq, oparg, -1, top); +- #line 1462 "Python/generated_cases.c.h" ++ #line 1531 "Python/generated_cases.c.h" + Py_DECREF(seq); +- #line 1070 "Python/bytecodes.c" ++ #line 1111 "Python/bytecodes.c" + if (res == 0) goto pop_1_error; +- #line 1466 "Python/generated_cases.c.h" ++ #line 1535 "Python/generated_cases.c.h" + STACK_SHRINK(1); + STACK_GROW(oparg); + next_instr += 1; +@@ -1472,14 +1541,14 @@ + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + PyObject *seq = stack_pointer[-1]; + PyObject **values = stack_pointer - (1); +- #line 1074 "Python/bytecodes.c" ++ #line 1115 "Python/bytecodes.c" + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + assert(oparg == 2); + STAT_INC(UNPACK_SEQUENCE, hit); + values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); + values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); +- #line 1483 "Python/generated_cases.c.h" ++ #line 1552 "Python/generated_cases.c.h" + Py_DECREF(seq); + STACK_SHRINK(1); + STACK_GROW(oparg); +@@ -1490,7 +1559,7 @@ + TARGET(UNPACK_SEQUENCE_TUPLE) { + PyObject *seq = stack_pointer[-1]; + PyObject **values = stack_pointer - (1); +- #line 1084 "Python/bytecodes.c" ++ #line 1125 "Python/bytecodes.c" + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); +@@ -1498,7 +1567,7 @@ + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } +- #line 1502 "Python/generated_cases.c.h" ++ #line 1571 "Python/generated_cases.c.h" + Py_DECREF(seq); + STACK_SHRINK(1); + STACK_GROW(oparg); +@@ -1509,7 +1578,7 @@ + TARGET(UNPACK_SEQUENCE_LIST) { + PyObject *seq = stack_pointer[-1]; + PyObject **values = stack_pointer - (1); +- #line 1095 "Python/bytecodes.c" ++ #line 1136 "Python/bytecodes.c" + DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); +@@ -1517,7 +1586,7 @@ + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } +- #line 1521 "Python/generated_cases.c.h" ++ #line 1590 "Python/generated_cases.c.h" + Py_DECREF(seq); + STACK_SHRINK(1); + STACK_GROW(oparg); +@@ -1527,15 +1596,15 @@ + + TARGET(UNPACK_EX) { + PyObject *seq = stack_pointer[-1]; +- #line 1106 "Python/bytecodes.c" ++ #line 1147 "Python/bytecodes.c" + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); + PyObject **top = stack_pointer + totalargs - 1; + int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); +- #line 1535 "Python/generated_cases.c.h" ++ #line 1604 "Python/generated_cases.c.h" + Py_DECREF(seq); +- #line 1110 "Python/bytecodes.c" ++ #line 1151 "Python/bytecodes.c" + if (res == 0) goto pop_1_error; +- #line 1539 "Python/generated_cases.c.h" ++ #line 1608 "Python/generated_cases.c.h" + STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + DISPATCH(); + } +@@ -1546,7 +1615,7 @@ + PyObject *owner = stack_pointer[-1]; + PyObject *v = stack_pointer[-2]; + uint16_t counter = read_u16(&next_instr[0].cache); +- #line 1121 "Python/bytecodes.c" ++ #line 1162 "Python/bytecodes.c" + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(frame->f_code->co_names, oparg); +@@ -1561,13 +1630,16 @@ + (void)counter; // Unused. + #endif /* ENABLE_SPECIALIZATION */ + PyObject *name = GETITEM(frame->f_code->co_names, oparg); +- int err = PyObject_SetAttr(owner, name, v); +- #line 1566 "Python/generated_cases.c.h" ++ int err = aiython_before_mutation(frame, "attr", owner, name, v); ++ if (err == 0) { ++ err = PyObject_SetAttr(owner, name, v); ++ } ++ #line 1638 "Python/generated_cases.c.h" + Py_DECREF(v); + Py_DECREF(owner); +- #line 1137 "Python/bytecodes.c" ++ #line 1181 "Python/bytecodes.c" + if (err) goto pop_2_error; +- #line 1571 "Python/generated_cases.c.h" ++ #line 1643 "Python/generated_cases.c.h" + STACK_SHRINK(2); + next_instr += 4; + DISPATCH(); +@@ -1575,34 +1647,41 @@ + + TARGET(DELETE_ATTR) { + PyObject *owner = stack_pointer[-1]; +- #line 1141 "Python/bytecodes.c" ++ #line 1185 "Python/bytecodes.c" + PyObject *name = GETITEM(frame->f_code->co_names, oparg); + int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); +- #line 1582 "Python/generated_cases.c.h" ++ #line 1654 "Python/generated_cases.c.h" + Py_DECREF(owner); +- #line 1144 "Python/bytecodes.c" ++ #line 1188 "Python/bytecodes.c" + if (err) goto pop_1_error; +- #line 1586 "Python/generated_cases.c.h" ++ #line 1658 "Python/generated_cases.c.h" + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(STORE_GLOBAL) { + PyObject *v = stack_pointer[-1]; +- #line 1148 "Python/bytecodes.c" ++ #line 1192 "Python/bytecodes.c" + PyObject *name = GETITEM(frame->f_code->co_names, oparg); ++ int hook_err = aiython_before_store(frame, name, v); ++ if (hook_err < 0) { ++ #line 1669 "Python/generated_cases.c.h" ++ Py_DECREF(v); ++ #line 1196 "Python/bytecodes.c" ++ if (true) goto pop_1_error; ++ } + int err = PyDict_SetItem(GLOBALS(), name, v); +- #line 1596 "Python/generated_cases.c.h" ++ #line 1675 "Python/generated_cases.c.h" + Py_DECREF(v); +- #line 1151 "Python/bytecodes.c" ++ #line 1200 "Python/bytecodes.c" + if (err) goto pop_1_error; +- #line 1600 "Python/generated_cases.c.h" ++ #line 1679 "Python/generated_cases.c.h" + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(DELETE_GLOBAL) { +- #line 1155 "Python/bytecodes.c" ++ #line 1204 "Python/bytecodes.c" + PyObject *name = GETITEM(frame->f_code->co_names, oparg); + int err; + err = PyDict_DelItem(GLOBALS(), name); +@@ -1614,13 +1693,13 @@ + } + goto error; + } +- #line 1618 "Python/generated_cases.c.h" ++ #line 1697 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(LOAD_LOCALS) { + PyObject *locals; +- #line 1169 "Python/bytecodes.c" ++ #line 1218 "Python/bytecodes.c" + locals = LOCALS(); + if (locals == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, +@@ -1628,7 +1707,7 @@ + if (true) goto error; + } + Py_INCREF(locals); +- #line 1632 "Python/generated_cases.c.h" ++ #line 1711 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = locals; + DISPATCH(); +@@ -1637,7 +1716,7 @@ + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + PyObject *mod_or_class_dict = stack_pointer[-1]; + PyObject *v; +- #line 1179 "Python/bytecodes.c" ++ #line 1228 "Python/bytecodes.c" + PyObject *name = GETITEM(frame->f_code->co_names, oparg); + if (PyDict_CheckExact(mod_or_class_dict)) { + v = PyDict_GetItemWithError(mod_or_class_dict, name); +@@ -1699,7 +1778,7 @@ + } + } + } +- #line 1703 "Python/generated_cases.c.h" ++ #line 1782 "Python/generated_cases.c.h" + Py_DECREF(mod_or_class_dict); + stack_pointer[-1] = v; + DISPATCH(); +@@ -1707,7 +1786,7 @@ + + TARGET(LOAD_NAME) { + PyObject *v; +- #line 1244 "Python/bytecodes.c" ++ #line 1293 "Python/bytecodes.c" + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, +@@ -1767,7 +1846,7 @@ + } + } + } +- #line 1771 "Python/generated_cases.c.h" ++ #line 1850 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = v; + DISPATCH(); +@@ -1778,7 +1857,7 @@ + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyObject *null = NULL; + PyObject *v; +- #line 1312 "Python/bytecodes.c" ++ #line 1361 "Python/bytecodes.c" + #if ENABLE_SPECIALIZATION + _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { +@@ -1830,7 +1909,7 @@ + } + } + null = NULL; +- #line 1834 "Python/generated_cases.c.h" ++ #line 1913 "Python/generated_cases.c.h" + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = v; +@@ -1844,7 +1923,7 @@ + PyObject *res; + uint16_t index = read_u16(&next_instr[1].cache); + uint16_t version = read_u16(&next_instr[2].cache); +- #line 1366 "Python/bytecodes.c" ++ #line 1415 "Python/bytecodes.c" + DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); +@@ -1855,7 +1934,7 @@ + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; +- #line 1859 "Python/generated_cases.c.h" ++ #line 1938 "Python/generated_cases.c.h" + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; +@@ -1870,7 +1949,7 @@ + uint16_t index = read_u16(&next_instr[1].cache); + uint16_t mod_version = read_u16(&next_instr[2].cache); + uint16_t bltn_version = read_u16(&next_instr[3].cache); +- #line 1379 "Python/bytecodes.c" ++ #line 1428 "Python/bytecodes.c" + DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); + PyDictObject *mdict = (PyDictObject *)GLOBALS(); +@@ -1885,7 +1964,7 @@ + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; +- #line 1889 "Python/generated_cases.c.h" ++ #line 1968 "Python/generated_cases.c.h" + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; +@@ -1895,16 +1974,16 @@ + } + + TARGET(DELETE_FAST) { +- #line 1396 "Python/bytecodes.c" ++ #line 1445 "Python/bytecodes.c" + PyObject *v = GETLOCAL(oparg); + if (v == NULL) goto unbound_local_error; + SETLOCAL(oparg, NULL); +- #line 1903 "Python/generated_cases.c.h" ++ #line 1982 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(MAKE_CELL) { +- #line 1402 "Python/bytecodes.c" ++ #line 1451 "Python/bytecodes.c" + // "initial" is probably NULL but not if it's an arg (or set + // via PyFrame_LocalsToFast() before MAKE_CELL has run). + PyObject *initial = GETLOCAL(oparg); +@@ -1913,12 +1992,12 @@ + goto resume_with_error; + } + SETLOCAL(oparg, cell); +- #line 1917 "Python/generated_cases.c.h" ++ #line 1996 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(DELETE_DEREF) { +- #line 1413 "Python/bytecodes.c" ++ #line 1462 "Python/bytecodes.c" + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + // Can't use ERROR_IF here. +@@ -1929,14 +2008,14 @@ + } + PyCell_SET(cell, NULL); + Py_DECREF(oldobj); +- #line 1933 "Python/generated_cases.c.h" ++ #line 2012 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(LOAD_FROM_DICT_OR_DEREF) { + PyObject *class_dict = stack_pointer[-1]; + PyObject *value; +- #line 1426 "Python/bytecodes.c" ++ #line 1475 "Python/bytecodes.c" + PyObject *name; + assert(class_dict); + assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); +@@ -1969,14 +2048,14 @@ + Py_INCREF(value); + } + Py_DECREF(class_dict); +- #line 1973 "Python/generated_cases.c.h" ++ #line 2052 "Python/generated_cases.c.h" + stack_pointer[-1] = value; + DISPATCH(); + } + + TARGET(LOAD_DEREF) { + PyObject *value; +- #line 1461 "Python/bytecodes.c" ++ #line 1510 "Python/bytecodes.c" + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { +@@ -1984,7 +2063,7 @@ + if (true) goto error; + } + Py_INCREF(value); +- #line 1988 "Python/generated_cases.c.h" ++ #line 2067 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = value; + DISPATCH(); +@@ -1992,18 +2071,26 @@ + + TARGET(STORE_DEREF) { + PyObject *v = stack_pointer[-1]; +- #line 1471 "Python/bytecodes.c" ++ #line 1520 "Python/bytecodes.c" ++ PyObject *name = GETITEM(frame->f_code->co_localsplusnames, oparg); ++ int err = aiython_before_store(frame, name, v); ++ if (err < 0) { ++ #line 2079 "Python/generated_cases.c.h" ++ Py_DECREF(v); ++ #line 1524 "Python/bytecodes.c" ++ if (true) goto pop_1_error; ++ } + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + PyCell_SET(cell, v); + Py_XDECREF(oldobj); +- #line 2001 "Python/generated_cases.c.h" ++ #line 2088 "Python/generated_cases.c.h" + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(COPY_FREE_VARS) { +- #line 1478 "Python/bytecodes.c" ++ #line 1533 "Python/bytecodes.c" + /* Copy closure variables to free variables */ + PyCodeObject *co = frame->f_code; + assert(PyFunction_Check(frame->f_funcobj)); +@@ -2014,22 +2101,22 @@ + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = Py_NewRef(o); + } +- #line 2018 "Python/generated_cases.c.h" ++ #line 2105 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(BUILD_STRING) { + PyObject **pieces = (stack_pointer - oparg); + PyObject *str; +- #line 1491 "Python/bytecodes.c" ++ #line 1546 "Python/bytecodes.c" + str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); +- #line 2027 "Python/generated_cases.c.h" ++ #line 2114 "Python/generated_cases.c.h" + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(pieces[_i]); + } +- #line 1493 "Python/bytecodes.c" ++ #line 1548 "Python/bytecodes.c" + if (str == NULL) { STACK_SHRINK(oparg); goto error; } +- #line 2033 "Python/generated_cases.c.h" ++ #line 2120 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_GROW(1); + stack_pointer[-1] = str; +@@ -2039,10 +2126,10 @@ + TARGET(BUILD_TUPLE) { + PyObject **values = (stack_pointer - oparg); + PyObject *tup; +- #line 1497 "Python/bytecodes.c" ++ #line 1552 "Python/bytecodes.c" + tup = _PyTuple_FromArraySteal(values, oparg); + if (tup == NULL) { STACK_SHRINK(oparg); goto error; } +- #line 2046 "Python/generated_cases.c.h" ++ #line 2133 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_GROW(1); + stack_pointer[-1] = tup; +@@ -2052,10 +2139,10 @@ + TARGET(BUILD_LIST) { + PyObject **values = (stack_pointer - oparg); + PyObject *list; +- #line 1502 "Python/bytecodes.c" ++ #line 1557 "Python/bytecodes.c" + list = _PyList_FromArraySteal(values, oparg); + if (list == NULL) { STACK_SHRINK(oparg); goto error; } +- #line 2059 "Python/generated_cases.c.h" ++ #line 2146 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_GROW(1); + stack_pointer[-1] = list; +@@ -2065,7 +2152,7 @@ + TARGET(LIST_EXTEND) { + PyObject *iterable = stack_pointer[-1]; + PyObject *list = stack_pointer[-(2 + (oparg-1))]; +- #line 1507 "Python/bytecodes.c" ++ #line 1562 "Python/bytecodes.c" + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + if (none_val == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && +@@ -2076,13 +2163,13 @@ + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + } +- #line 2080 "Python/generated_cases.c.h" ++ #line 2167 "Python/generated_cases.c.h" + Py_DECREF(iterable); +- #line 1518 "Python/bytecodes.c" ++ #line 1573 "Python/bytecodes.c" + if (true) goto pop_1_error; + } + assert(Py_IsNone(none_val)); +- #line 2086 "Python/generated_cases.c.h" ++ #line 2173 "Python/generated_cases.c.h" + Py_DECREF(iterable); + STACK_SHRINK(1); + DISPATCH(); +@@ -2091,13 +2178,13 @@ + TARGET(SET_UPDATE) { + PyObject *iterable = stack_pointer[-1]; + PyObject *set = stack_pointer[-(2 + (oparg-1))]; +- #line 1525 "Python/bytecodes.c" ++ #line 1580 "Python/bytecodes.c" + int err = _PySet_Update(set, iterable); +- #line 2097 "Python/generated_cases.c.h" ++ #line 2184 "Python/generated_cases.c.h" + Py_DECREF(iterable); +- #line 1527 "Python/bytecodes.c" ++ #line 1582 "Python/bytecodes.c" + if (err < 0) goto pop_1_error; +- #line 2101 "Python/generated_cases.c.h" ++ #line 2188 "Python/generated_cases.c.h" + STACK_SHRINK(1); + DISPATCH(); + } +@@ -2105,7 +2192,7 @@ + TARGET(BUILD_SET) { + PyObject **values = (stack_pointer - oparg); + PyObject *set; +- #line 1531 "Python/bytecodes.c" ++ #line 1586 "Python/bytecodes.c" + set = PySet_New(NULL); + if (set == NULL) + goto error; +@@ -2120,7 +2207,7 @@ + Py_DECREF(set); + if (true) { STACK_SHRINK(oparg); goto error; } + } +- #line 2124 "Python/generated_cases.c.h" ++ #line 2211 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_GROW(1); + stack_pointer[-1] = set; +@@ -2130,18 +2217,18 @@ + TARGET(BUILD_MAP) { + PyObject **values = (stack_pointer - oparg*2); + PyObject *map; +- #line 1548 "Python/bytecodes.c" ++ #line 1603 "Python/bytecodes.c" + map = _PyDict_FromItems( + values, 2, + values+1, 2, + oparg); +- #line 2139 "Python/generated_cases.c.h" ++ #line 2226 "Python/generated_cases.c.h" + for (int _i = oparg*2; --_i >= 0;) { + Py_DECREF(values[_i]); + } +- #line 1553 "Python/bytecodes.c" ++ #line 1608 "Python/bytecodes.c" + if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } +- #line 2145 "Python/generated_cases.c.h" ++ #line 2232 "Python/generated_cases.c.h" + STACK_SHRINK(oparg*2); + STACK_GROW(1); + stack_pointer[-1] = map; +@@ -2149,7 +2236,7 @@ + } + + TARGET(SETUP_ANNOTATIONS) { +- #line 1557 "Python/bytecodes.c" ++ #line 1612 "Python/bytecodes.c" + int err; + PyObject *ann_dict; + if (LOCALS() == NULL) { +@@ -2189,7 +2276,7 @@ + Py_DECREF(ann_dict); + } + } +- #line 2193 "Python/generated_cases.c.h" ++ #line 2280 "Python/generated_cases.c.h" + DISPATCH(); + } + +@@ -2197,7 +2284,7 @@ + PyObject *keys = stack_pointer[-1]; + PyObject **values = (stack_pointer - (1 + oparg)); + PyObject *map; +- #line 1599 "Python/bytecodes.c" ++ #line 1654 "Python/bytecodes.c" + if (!PyTuple_CheckExact(keys) || + PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { + _PyErr_SetString(tstate, PyExc_SystemError, +@@ -2207,14 +2294,14 @@ + map = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); +- #line 2211 "Python/generated_cases.c.h" ++ #line 2298 "Python/generated_cases.c.h" + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(values[_i]); + } + Py_DECREF(keys); +- #line 1609 "Python/bytecodes.c" ++ #line 1664 "Python/bytecodes.c" + if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } +- #line 2218 "Python/generated_cases.c.h" ++ #line 2305 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + stack_pointer[-1] = map; + DISPATCH(); +@@ -2222,7 +2309,7 @@ + + TARGET(DICT_UPDATE) { + PyObject *update = stack_pointer[-1]; +- #line 1613 "Python/bytecodes.c" ++ #line 1668 "Python/bytecodes.c" + PyObject *dict = PEEK(oparg + 1); // update is still on the stack + if (PyDict_Update(dict, update) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { +@@ -2230,12 +2317,12 @@ + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); + } +- #line 2234 "Python/generated_cases.c.h" ++ #line 2321 "Python/generated_cases.c.h" + Py_DECREF(update); +- #line 1621 "Python/bytecodes.c" ++ #line 1676 "Python/bytecodes.c" + if (true) goto pop_1_error; + } +- #line 2239 "Python/generated_cases.c.h" ++ #line 2326 "Python/generated_cases.c.h" + Py_DECREF(update); + STACK_SHRINK(1); + DISPATCH(); +@@ -2243,17 +2330,17 @@ + + TARGET(DICT_MERGE) { + PyObject *update = stack_pointer[-1]; +- #line 1627 "Python/bytecodes.c" ++ #line 1682 "Python/bytecodes.c" + PyObject *dict = PEEK(oparg + 1); // update is still on the stack + + if (_PyDict_MergeEx(dict, update, 2) < 0) { + format_kwargs_error(tstate, PEEK(3 + oparg), update); +- #line 2252 "Python/generated_cases.c.h" ++ #line 2339 "Python/generated_cases.c.h" + Py_DECREF(update); +- #line 1632 "Python/bytecodes.c" ++ #line 1687 "Python/bytecodes.c" + if (true) goto pop_1_error; + } +- #line 2257 "Python/generated_cases.c.h" ++ #line 2344 "Python/generated_cases.c.h" + Py_DECREF(update); + STACK_SHRINK(1); + PREDICT(CALL_FUNCTION_EX); +@@ -2263,26 +2350,26 @@ + TARGET(MAP_ADD) { + PyObject *value = stack_pointer[-1]; + PyObject *key = stack_pointer[-2]; +- #line 1639 "Python/bytecodes.c" ++ #line 1694 "Python/bytecodes.c" + PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; +- #line 2273 "Python/generated_cases.c.h" ++ #line 2360 "Python/generated_cases.c.h" + STACK_SHRINK(2); + PREDICT(JUMP_BACKWARD); + DISPATCH(); + } + + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { +- #line 1648 "Python/bytecodes.c" ++ #line 1703 "Python/bytecodes.c" + _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); +- #line 2286 "Python/generated_cases.c.h" ++ #line 2373 "Python/generated_cases.c.h" + } + + TARGET(LOAD_SUPER_ATTR) { +@@ -2293,7 +2380,7 @@ + PyObject *global_super = stack_pointer[-3]; + PyObject *res2 = NULL; + PyObject *res; +- #line 1662 "Python/bytecodes.c" ++ #line 1717 "Python/bytecodes.c" + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); + int load_method = oparg & 1; + #if ENABLE_SPECIALIZATION +@@ -2335,16 +2422,16 @@ + } + } + } +- #line 2339 "Python/generated_cases.c.h" ++ #line 2426 "Python/generated_cases.c.h" + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); +- #line 1704 "Python/bytecodes.c" ++ #line 1759 "Python/bytecodes.c" + if (super == NULL) goto pop_3_error; + res = PyObject_GetAttr(super, name); + Py_DECREF(super); + if (res == NULL) goto pop_3_error; +- #line 2348 "Python/generated_cases.c.h" ++ #line 2435 "Python/generated_cases.c.h" + STACK_SHRINK(2); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; +@@ -2359,20 +2446,20 @@ + PyObject *global_super = stack_pointer[-3]; + PyObject *res2 = NULL; + PyObject *res; +- #line 1711 "Python/bytecodes.c" ++ #line 1766 "Python/bytecodes.c" + assert(!(oparg & 1)); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); + res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); +- #line 2370 "Python/generated_cases.c.h" ++ #line 2457 "Python/generated_cases.c.h" + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); +- #line 1718 "Python/bytecodes.c" ++ #line 1773 "Python/bytecodes.c" + if (res == NULL) goto pop_3_error; +- #line 2376 "Python/generated_cases.c.h" ++ #line 2463 "Python/generated_cases.c.h" + STACK_SHRINK(2); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; +@@ -2387,7 +2474,7 @@ + PyObject *global_super = stack_pointer[-3]; + PyObject *res2; + PyObject *res; +- #line 1722 "Python/bytecodes.c" ++ #line 1777 "Python/bytecodes.c" + assert(oparg & 1); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); +@@ -2410,7 +2497,7 @@ + res = res2; + res2 = NULL; + } +- #line 2414 "Python/generated_cases.c.h" ++ #line 2501 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = res; + stack_pointer[-2] = res2; +@@ -2424,7 +2511,7 @@ + PyObject *owner = stack_pointer[-1]; + PyObject *res2 = NULL; + PyObject *res; +- #line 1761 "Python/bytecodes.c" ++ #line 1816 "Python/bytecodes.c" + #if ENABLE_SPECIALIZATION + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { +@@ -2458,9 +2545,9 @@ + + NULL | meth | arg1 | ... | argN + */ +- #line 2462 "Python/generated_cases.c.h" ++ #line 2549 "Python/generated_cases.c.h" + Py_DECREF(owner); +- #line 1795 "Python/bytecodes.c" ++ #line 1850 "Python/bytecodes.c" + if (meth == NULL) goto pop_1_error; + res2 = NULL; + res = meth; +@@ -2469,12 +2556,12 @@ + else { + /* Classic, pushes one value. */ + res = PyObject_GetAttr(owner, name); +- #line 2473 "Python/generated_cases.c.h" ++ #line 2560 "Python/generated_cases.c.h" + Py_DECREF(owner); +- #line 1804 "Python/bytecodes.c" ++ #line 1859 "Python/bytecodes.c" + if (res == NULL) goto pop_1_error; + } +- #line 2478 "Python/generated_cases.c.h" ++ #line 2565 "Python/generated_cases.c.h" + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } +@@ -2488,7 +2575,7 @@ + PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); +- #line 1809 "Python/bytecodes.c" ++ #line 1864 "Python/bytecodes.c" + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); +@@ -2501,7 +2588,7 @@ + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(res); + res2 = NULL; +- #line 2505 "Python/generated_cases.c.h" ++ #line 2592 "Python/generated_cases.c.h" + Py_DECREF(owner); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; +@@ -2516,7 +2603,7 @@ + PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); +- #line 1825 "Python/bytecodes.c" ++ #line 1880 "Python/bytecodes.c" + DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict != NULL); +@@ -2529,7 +2616,7 @@ + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(res); + res2 = NULL; +- #line 2533 "Python/generated_cases.c.h" ++ #line 2620 "Python/generated_cases.c.h" + Py_DECREF(owner); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; +@@ -2544,7 +2631,7 @@ + PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); +- #line 1841 "Python/bytecodes.c" ++ #line 1896 "Python/bytecodes.c" + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); +@@ -2571,7 +2658,7 @@ + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(res); + res2 = NULL; +- #line 2575 "Python/generated_cases.c.h" ++ #line 2662 "Python/generated_cases.c.h" + Py_DECREF(owner); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; +@@ -2586,7 +2673,7 @@ + PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); +- #line 1871 "Python/bytecodes.c" ++ #line 1926 "Python/bytecodes.c" + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); +@@ -2596,7 +2683,7 @@ + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(res); + res2 = NULL; +- #line 2600 "Python/generated_cases.c.h" ++ #line 2687 "Python/generated_cases.c.h" + Py_DECREF(owner); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; +@@ -2611,7 +2698,7 @@ + PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + PyObject *descr = read_obj(&next_instr[5].cache); +- #line 1884 "Python/bytecodes.c" ++ #line 1939 "Python/bytecodes.c" + + DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, +@@ -2623,7 +2710,7 @@ + res = descr; + assert(res != NULL); + Py_INCREF(res); +- #line 2627 "Python/generated_cases.c.h" ++ #line 2714 "Python/generated_cases.c.h" + Py_DECREF(cls); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; +@@ -2637,7 +2724,7 @@ + uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t func_version = read_u32(&next_instr[3].cache); + PyObject *fget = read_obj(&next_instr[5].cache); +- #line 1899 "Python/bytecodes.c" ++ #line 1954 "Python/bytecodes.c" + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + + PyTypeObject *cls = Py_TYPE(owner); +@@ -2661,7 +2748,7 @@ + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + frame->return_offset = 0; + DISPATCH_INLINED(new_frame); +- #line 2665 "Python/generated_cases.c.h" ++ #line 2752 "Python/generated_cases.c.h" + } + + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { +@@ -2669,7 +2756,7 @@ + uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t func_version = read_u32(&next_instr[3].cache); + PyObject *getattribute = read_obj(&next_instr[5].cache); +- #line 1925 "Python/bytecodes.c" ++ #line 1980 "Python/bytecodes.c" + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); +@@ -2695,7 +2782,7 @@ + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + frame->return_offset = 0; + DISPATCH_INLINED(new_frame); +- #line 2699 "Python/generated_cases.c.h" ++ #line 2786 "Python/generated_cases.c.h" + } + + TARGET(STORE_ATTR_INSTANCE_VALUE) { +@@ -2703,7 +2790,9 @@ + PyObject *value = stack_pointer[-2]; + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); +- #line 1953 "Python/bytecodes.c" ++ #line 2008 "Python/bytecodes.c" ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); +@@ -2721,7 +2810,7 @@ + Py_DECREF(old_value); + } + Py_DECREF(owner); +- #line 2725 "Python/generated_cases.c.h" ++ #line 2814 "Python/generated_cases.c.h" + STACK_SHRINK(2); + next_instr += 4; + DISPATCH(); +@@ -2732,7 +2821,9 @@ + PyObject *value = stack_pointer[-2]; + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t hint = read_u16(&next_instr[3].cache); +- #line 1973 "Python/bytecodes.c" ++ #line 2030 "Python/bytecodes.c" ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); +@@ -2772,7 +2863,7 @@ + Py_DECREF(old_value); + STAT_INC(STORE_ATTR, hit); + Py_DECREF(owner); +- #line 2776 "Python/generated_cases.c.h" ++ #line 2867 "Python/generated_cases.c.h" + STACK_SHRINK(2); + next_instr += 4; + DISPATCH(); +@@ -2783,7 +2874,9 @@ + PyObject *value = stack_pointer[-2]; + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); +- #line 2015 "Python/bytecodes.c" ++ #line 2074 "Python/bytecodes.c" ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); +@@ -2793,7 +2886,7 @@ + *(PyObject **)addr = value; + Py_XDECREF(old_value); + Py_DECREF(owner); +- #line 2797 "Python/generated_cases.c.h" ++ #line 2890 "Python/generated_cases.c.h" + STACK_SHRINK(2); + next_instr += 4; + DISPATCH(); +@@ -2805,7 +2898,7 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *res; +- #line 2034 "Python/bytecodes.c" ++ #line 2095 "Python/bytecodes.c" + #if ENABLE_SPECIALIZATION + _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { +@@ -2818,12 +2911,12 @@ + #endif /* ENABLE_SPECIALIZATION */ + assert((oparg >> 4) <= Py_GE); + res = PyObject_RichCompare(left, right, oparg>>4); +- #line 2822 "Python/generated_cases.c.h" ++ #line 2915 "Python/generated_cases.c.h" + Py_DECREF(left); + Py_DECREF(right); +- #line 2047 "Python/bytecodes.c" ++ #line 2108 "Python/bytecodes.c" + if (res == NULL) goto pop_2_error; +- #line 2827 "Python/generated_cases.c.h" ++ #line 2920 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = res; + next_instr += 1; +@@ -2834,7 +2927,7 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *res; +- #line 2051 "Python/bytecodes.c" ++ #line 2112 "Python/bytecodes.c" + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); +@@ -2845,7 +2938,7 @@ + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? Py_True : Py_False; +- #line 2849 "Python/generated_cases.c.h" ++ #line 2942 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = res; + next_instr += 1; +@@ -2856,7 +2949,7 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *res; +- #line 2065 "Python/bytecodes.c" ++ #line 2126 "Python/bytecodes.c" + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); +@@ -2871,7 +2964,7 @@ + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? Py_True : Py_False; +- #line 2875 "Python/generated_cases.c.h" ++ #line 2968 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = res; + next_instr += 1; +@@ -2882,7 +2975,7 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *res; +- #line 2083 "Python/bytecodes.c" ++ #line 2144 "Python/bytecodes.c" + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); +@@ -2894,7 +2987,7 @@ + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; +- #line 2898 "Python/generated_cases.c.h" ++ #line 2991 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = res; + next_instr += 1; +@@ -2905,14 +2998,14 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *b; +- #line 2097 "Python/bytecodes.c" ++ #line 2158 "Python/bytecodes.c" + int res = Py_Is(left, right) ^ oparg; +- #line 2911 "Python/generated_cases.c.h" ++ #line 3004 "Python/generated_cases.c.h" + Py_DECREF(left); + Py_DECREF(right); +- #line 2099 "Python/bytecodes.c" ++ #line 2160 "Python/bytecodes.c" + b = res ? Py_True : Py_False; +- #line 2916 "Python/generated_cases.c.h" ++ #line 3009 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = b; + DISPATCH(); +@@ -2922,15 +3015,15 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *b; +- #line 2103 "Python/bytecodes.c" ++ #line 2164 "Python/bytecodes.c" + int res = PySequence_Contains(right, left); +- #line 2928 "Python/generated_cases.c.h" ++ #line 3021 "Python/generated_cases.c.h" + Py_DECREF(left); + Py_DECREF(right); +- #line 2105 "Python/bytecodes.c" ++ #line 2166 "Python/bytecodes.c" + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? Py_True : Py_False; +- #line 2934 "Python/generated_cases.c.h" ++ #line 3027 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = b; + DISPATCH(); +@@ -2941,12 +3034,12 @@ + PyObject *exc_value = stack_pointer[-2]; + PyObject *rest; + PyObject *match; +- #line 2110 "Python/bytecodes.c" ++ #line 2171 "Python/bytecodes.c" + if (check_except_star_type_valid(tstate, match_type) < 0) { +- #line 2947 "Python/generated_cases.c.h" ++ #line 3040 "Python/generated_cases.c.h" + Py_DECREF(exc_value); + Py_DECREF(match_type); +- #line 2112 "Python/bytecodes.c" ++ #line 2173 "Python/bytecodes.c" + if (true) goto pop_2_error; + } + +@@ -2954,10 +3047,10 @@ + rest = NULL; + int res = exception_group_match(frame, exc_value, match_type, + &match, &rest); +- #line 2958 "Python/generated_cases.c.h" ++ #line 3051 "Python/generated_cases.c.h" + Py_DECREF(exc_value); + Py_DECREF(match_type); +- #line 2120 "Python/bytecodes.c" ++ #line 2181 "Python/bytecodes.c" + if (res < 0) goto pop_2_error; + + assert((match == NULL) == (rest == NULL)); +@@ -2966,7 +3059,7 @@ + if (!Py_IsNone(match)) { + PyErr_SetHandledException(match); + } +- #line 2970 "Python/generated_cases.c.h" ++ #line 3063 "Python/generated_cases.c.h" + stack_pointer[-1] = match; + stack_pointer[-2] = rest; + DISPATCH(); +@@ -2976,21 +3069,21 @@ + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; + PyObject *b; +- #line 2131 "Python/bytecodes.c" ++ #line 2192 "Python/bytecodes.c" + assert(PyExceptionInstance_Check(left)); + if (check_except_type_valid(tstate, right) < 0) { +- #line 2983 "Python/generated_cases.c.h" ++ #line 3076 "Python/generated_cases.c.h" + Py_DECREF(right); +- #line 2134 "Python/bytecodes.c" ++ #line 2195 "Python/bytecodes.c" + if (true) goto pop_1_error; + } + + int res = PyErr_GivenExceptionMatches(left, right); +- #line 2990 "Python/generated_cases.c.h" ++ #line 3083 "Python/generated_cases.c.h" + Py_DECREF(right); +- #line 2139 "Python/bytecodes.c" ++ #line 2200 "Python/bytecodes.c" + b = res ? Py_True : Py_False; +- #line 2994 "Python/generated_cases.c.h" ++ #line 3087 "Python/generated_cases.c.h" + stack_pointer[-1] = b; + DISPATCH(); + } +@@ -2999,15 +3092,15 @@ + PyObject *fromlist = stack_pointer[-1]; + PyObject *level = stack_pointer[-2]; + PyObject *res; +- #line 2143 "Python/bytecodes.c" ++ #line 2204 "Python/bytecodes.c" + PyObject *name = GETITEM(frame->f_code->co_names, oparg); + res = import_name(tstate, frame, name, fromlist, level); +- #line 3006 "Python/generated_cases.c.h" ++ #line 3099 "Python/generated_cases.c.h" + Py_DECREF(level); + Py_DECREF(fromlist); +- #line 2146 "Python/bytecodes.c" ++ #line 2207 "Python/bytecodes.c" + if (res == NULL) goto pop_2_error; +- #line 3011 "Python/generated_cases.c.h" ++ #line 3104 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = res; + DISPATCH(); +@@ -3016,29 +3109,29 @@ + TARGET(IMPORT_FROM) { + PyObject *from = stack_pointer[-1]; + PyObject *res; +- #line 2150 "Python/bytecodes.c" ++ #line 2211 "Python/bytecodes.c" + PyObject *name = GETITEM(frame->f_code->co_names, oparg); + res = import_from(tstate, from, name); + if (res == NULL) goto error; +- #line 3024 "Python/generated_cases.c.h" ++ #line 3117 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(JUMP_FORWARD) { +- #line 2156 "Python/bytecodes.c" ++ #line 2217 "Python/bytecodes.c" + JUMPBY(oparg); +- #line 3033 "Python/generated_cases.c.h" ++ #line 3126 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(JUMP_BACKWARD) { + PREDICTED(JUMP_BACKWARD); +- #line 2160 "Python/bytecodes.c" ++ #line 2221 "Python/bytecodes.c" + assert(oparg < INSTR_OFFSET()); + JUMPBY(-oparg); +- #line 3042 "Python/generated_cases.c.h" ++ #line 3135 "Python/generated_cases.c.h" + CHECK_EVAL_BREAKER(); + DISPATCH(); + } +@@ -3046,15 +3139,15 @@ + TARGET(POP_JUMP_IF_FALSE) { + PREDICTED(POP_JUMP_IF_FALSE); + PyObject *cond = stack_pointer[-1]; +- #line 2166 "Python/bytecodes.c" ++ #line 2227 "Python/bytecodes.c" + if (Py_IsFalse(cond)) { + JUMPBY(oparg); + } + else if (!Py_IsTrue(cond)) { + int err = PyObject_IsTrue(cond); +- #line 3056 "Python/generated_cases.c.h" ++ #line 3149 "Python/generated_cases.c.h" + Py_DECREF(cond); +- #line 2172 "Python/bytecodes.c" ++ #line 2233 "Python/bytecodes.c" + if (err == 0) { + JUMPBY(oparg); + } +@@ -3062,22 +3155,22 @@ + if (err < 0) goto pop_1_error; + } + } +- #line 3066 "Python/generated_cases.c.h" ++ #line 3159 "Python/generated_cases.c.h" + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(POP_JUMP_IF_TRUE) { + PyObject *cond = stack_pointer[-1]; +- #line 2182 "Python/bytecodes.c" ++ #line 2243 "Python/bytecodes.c" + if (Py_IsTrue(cond)) { + JUMPBY(oparg); + } + else if (!Py_IsFalse(cond)) { + int err = PyObject_IsTrue(cond); +- #line 3079 "Python/generated_cases.c.h" ++ #line 3172 "Python/generated_cases.c.h" + Py_DECREF(cond); +- #line 2188 "Python/bytecodes.c" ++ #line 2249 "Python/bytecodes.c" + if (err > 0) { + JUMPBY(oparg); + } +@@ -3085,63 +3178,63 @@ + if (err < 0) goto pop_1_error; + } + } +- #line 3089 "Python/generated_cases.c.h" ++ #line 3182 "Python/generated_cases.c.h" + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NOT_NONE) { + PyObject *value = stack_pointer[-1]; +- #line 2198 "Python/bytecodes.c" ++ #line 2259 "Python/bytecodes.c" + if (!Py_IsNone(value)) { +- #line 3098 "Python/generated_cases.c.h" ++ #line 3191 "Python/generated_cases.c.h" + Py_DECREF(value); +- #line 2200 "Python/bytecodes.c" ++ #line 2261 "Python/bytecodes.c" + JUMPBY(oparg); + } +- #line 3103 "Python/generated_cases.c.h" ++ #line 3196 "Python/generated_cases.c.h" + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NONE) { + PyObject *value = stack_pointer[-1]; +- #line 2205 "Python/bytecodes.c" ++ #line 2266 "Python/bytecodes.c" + if (Py_IsNone(value)) { + JUMPBY(oparg); + } + else { +- #line 3115 "Python/generated_cases.c.h" ++ #line 3208 "Python/generated_cases.c.h" + Py_DECREF(value); +- #line 2210 "Python/bytecodes.c" ++ #line 2271 "Python/bytecodes.c" + } +- #line 3119 "Python/generated_cases.c.h" ++ #line 3212 "Python/generated_cases.c.h" + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { +- #line 2214 "Python/bytecodes.c" ++ #line 2275 "Python/bytecodes.c" + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMPBY(-oparg); +- #line 3132 "Python/generated_cases.c.h" ++ #line 3225 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(GET_LEN) { + PyObject *obj = stack_pointer[-1]; + PyObject *len_o; +- #line 2223 "Python/bytecodes.c" ++ #line 2284 "Python/bytecodes.c" + // PUSH(len(TOS)) + Py_ssize_t len_i = PyObject_Length(obj); + if (len_i < 0) goto error; + len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) goto error; +- #line 3145 "Python/generated_cases.c.h" ++ #line 3238 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = len_o; + DISPATCH(); +@@ -3152,16 +3245,16 @@ + PyObject *type = stack_pointer[-2]; + PyObject *subject = stack_pointer[-3]; + PyObject *attrs; +- #line 2231 "Python/bytecodes.c" ++ #line 2292 "Python/bytecodes.c" + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(names)); + attrs = match_class(tstate, subject, type, oparg, names); +- #line 3161 "Python/generated_cases.c.h" ++ #line 3254 "Python/generated_cases.c.h" + Py_DECREF(subject); + Py_DECREF(type); + Py_DECREF(names); +- #line 2236 "Python/bytecodes.c" ++ #line 2297 "Python/bytecodes.c" + if (attrs) { + assert(PyTuple_CheckExact(attrs)); // Success! + } +@@ -3169,7 +3262,7 @@ + if (_PyErr_Occurred(tstate)) goto pop_3_error; + attrs = Py_None; // Failure! + } +- #line 3173 "Python/generated_cases.c.h" ++ #line 3266 "Python/generated_cases.c.h" + STACK_SHRINK(2); + stack_pointer[-1] = attrs; + DISPATCH(); +@@ -3178,10 +3271,10 @@ + TARGET(MATCH_MAPPING) { + PyObject *subject = stack_pointer[-1]; + PyObject *res; +- #line 2246 "Python/bytecodes.c" ++ #line 2307 "Python/bytecodes.c" + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? Py_True : Py_False; +- #line 3185 "Python/generated_cases.c.h" ++ #line 3278 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = res; + PREDICT(POP_JUMP_IF_FALSE); +@@ -3191,10 +3284,10 @@ + TARGET(MATCH_SEQUENCE) { + PyObject *subject = stack_pointer[-1]; + PyObject *res; +- #line 2252 "Python/bytecodes.c" ++ #line 2313 "Python/bytecodes.c" + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? Py_True : Py_False; +- #line 3198 "Python/generated_cases.c.h" ++ #line 3291 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = res; + PREDICT(POP_JUMP_IF_FALSE); +@@ -3205,11 +3298,11 @@ + PyObject *keys = stack_pointer[-1]; + PyObject *subject = stack_pointer[-2]; + PyObject *values_or_none; +- #line 2258 "Python/bytecodes.c" ++ #line 2319 "Python/bytecodes.c" + // On successful match, PUSH(values). Otherwise, PUSH(None). + values_or_none = match_keys(tstate, subject, keys); + if (values_or_none == NULL) goto error; +- #line 3213 "Python/generated_cases.c.h" ++ #line 3306 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = values_or_none; + DISPATCH(); +@@ -3218,14 +3311,14 @@ + TARGET(GET_ITER) { + PyObject *iterable = stack_pointer[-1]; + PyObject *iter; +- #line 2264 "Python/bytecodes.c" ++ #line 2325 "Python/bytecodes.c" + /* before: [obj]; after [getiter(obj)] */ + iter = PyObject_GetIter(iterable); +- #line 3225 "Python/generated_cases.c.h" ++ #line 3318 "Python/generated_cases.c.h" + Py_DECREF(iterable); +- #line 2267 "Python/bytecodes.c" ++ #line 2328 "Python/bytecodes.c" + if (iter == NULL) goto pop_1_error; +- #line 3229 "Python/generated_cases.c.h" ++ #line 3322 "Python/generated_cases.c.h" + stack_pointer[-1] = iter; + DISPATCH(); + } +@@ -3233,7 +3326,7 @@ + TARGET(GET_YIELD_FROM_ITER) { + PyObject *iterable = stack_pointer[-1]; + PyObject *iter; +- #line 2271 "Python/bytecodes.c" ++ #line 2332 "Python/bytecodes.c" + /* before: [obj]; after [getiter(obj)] */ + if (PyCoro_CheckExact(iterable)) { + /* `iterable` is a coroutine */ +@@ -3256,11 +3349,11 @@ + if (iter == NULL) { + goto error; + } +- #line 3260 "Python/generated_cases.c.h" ++ #line 3353 "Python/generated_cases.c.h" + Py_DECREF(iterable); +- #line 2294 "Python/bytecodes.c" ++ #line 2355 "Python/bytecodes.c" + } +- #line 3264 "Python/generated_cases.c.h" ++ #line 3357 "Python/generated_cases.c.h" + stack_pointer[-1] = iter; + PREDICT(LOAD_CONST); + DISPATCH(); +@@ -3271,7 +3364,7 @@ + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter = stack_pointer[-1]; + PyObject *next; +- #line 2313 "Python/bytecodes.c" ++ #line 2374 "Python/bytecodes.c" + #if ENABLE_SPECIALIZATION + _PyForIterCache *cache = (_PyForIterCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { +@@ -3302,7 +3395,7 @@ + DISPATCH(); + } + // Common case: no jump, leave it to the code generator +- #line 3306 "Python/generated_cases.c.h" ++ #line 3399 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = next; + next_instr += 1; +@@ -3310,7 +3403,7 @@ + } + + TARGET(INSTRUMENTED_FOR_ITER) { +- #line 2346 "Python/bytecodes.c" ++ #line 2407 "Python/bytecodes.c" + _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *target; + PyObject *iter = TOP(); +@@ -3336,14 +3429,14 @@ + target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + } + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); +- #line 3340 "Python/generated_cases.c.h" ++ #line 3433 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(FOR_ITER_LIST) { + PyObject *iter = stack_pointer[-1]; + PyObject *next; +- #line 2374 "Python/bytecodes.c" ++ #line 2435 "Python/bytecodes.c" + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + _PyListIterObject *it = (_PyListIterObject *)iter; + STAT_INC(FOR_ITER, hit); +@@ -3363,7 +3456,7 @@ + DISPATCH(); + end_for_iter_list: + // Common case: no jump, leave it to the code generator +- #line 3367 "Python/generated_cases.c.h" ++ #line 3460 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = next; + next_instr += 1; +@@ -3373,7 +3466,7 @@ + TARGET(FOR_ITER_TUPLE) { + PyObject *iter = stack_pointer[-1]; + PyObject *next; +- #line 2396 "Python/bytecodes.c" ++ #line 2457 "Python/bytecodes.c" + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); + STAT_INC(FOR_ITER, hit); +@@ -3393,7 +3486,7 @@ + DISPATCH(); + end_for_iter_tuple: + // Common case: no jump, leave it to the code generator +- #line 3397 "Python/generated_cases.c.h" ++ #line 3490 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = next; + next_instr += 1; +@@ -3403,7 +3496,7 @@ + TARGET(FOR_ITER_RANGE) { + PyObject *iter = stack_pointer[-1]; + PyObject *next; +- #line 2418 "Python/bytecodes.c" ++ #line 2479 "Python/bytecodes.c" + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + STAT_INC(FOR_ITER, hit); +@@ -3421,7 +3514,7 @@ + if (next == NULL) { + goto error; + } +- #line 3425 "Python/generated_cases.c.h" ++ #line 3518 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = next; + next_instr += 1; +@@ -3430,7 +3523,7 @@ + + TARGET(FOR_ITER_GEN) { + PyObject *iter = stack_pointer[-1]; +- #line 2438 "Python/bytecodes.c" ++ #line 2499 "Python/bytecodes.c" + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + PyGenObject *gen = (PyGenObject *)iter; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); +@@ -3446,14 +3539,14 @@ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + DISPATCH_INLINED(gen_frame); +- #line 3450 "Python/generated_cases.c.h" ++ #line 3543 "Python/generated_cases.c.h" + } + + TARGET(BEFORE_ASYNC_WITH) { + PyObject *mgr = stack_pointer[-1]; + PyObject *exit; + PyObject *res; +- #line 2456 "Python/bytecodes.c" ++ #line 2517 "Python/bytecodes.c" + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { +@@ -3476,16 +3569,16 @@ + Py_DECREF(enter); + goto error; + } +- #line 3480 "Python/generated_cases.c.h" ++ #line 3573 "Python/generated_cases.c.h" + Py_DECREF(mgr); +- #line 2479 "Python/bytecodes.c" ++ #line 2540 "Python/bytecodes.c" + res = _PyObject_CallNoArgs(enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; + } +- #line 3489 "Python/generated_cases.c.h" ++ #line 3582 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = res; + stack_pointer[-2] = exit; +@@ -3497,7 +3590,7 @@ + PyObject *mgr = stack_pointer[-1]; + PyObject *exit; + PyObject *res; +- #line 2489 "Python/bytecodes.c" ++ #line 2550 "Python/bytecodes.c" + /* pop the context manager, push its __exit__ and the + * value returned from calling its __enter__ + */ +@@ -3523,16 +3616,16 @@ + Py_DECREF(enter); + goto error; + } +- #line 3527 "Python/generated_cases.c.h" ++ #line 3620 "Python/generated_cases.c.h" + Py_DECREF(mgr); +- #line 2515 "Python/bytecodes.c" ++ #line 2576 "Python/bytecodes.c" + res = _PyObject_CallNoArgs(enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; + } +- #line 3536 "Python/generated_cases.c.h" ++ #line 3629 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = res; + stack_pointer[-2] = exit; +@@ -3544,7 +3637,7 @@ + PyObject *lasti = stack_pointer[-3]; + PyObject *exit_func = stack_pointer[-4]; + PyObject *res; +- #line 2524 "Python/bytecodes.c" ++ #line 2585 "Python/bytecodes.c" + /* At the top of the stack are 4 values: + - val: TOP = exc_info() + - unused: SECOND = previous exception +@@ -3570,7 +3663,7 @@ + res = PyObject_Vectorcall(exit_func, stack + 1, + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + if (res == NULL) goto error; +- #line 3574 "Python/generated_cases.c.h" ++ #line 3667 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = res; + DISPATCH(); +@@ -3579,7 +3672,7 @@ + TARGET(PUSH_EXC_INFO) { + PyObject *new_exc = stack_pointer[-1]; + PyObject *prev_exc; +- #line 2552 "Python/bytecodes.c" ++ #line 2613 "Python/bytecodes.c" + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = exc_info->exc_value; +@@ -3589,7 +3682,7 @@ + } + assert(PyExceptionInstance_Check(new_exc)); + exc_info->exc_value = Py_NewRef(new_exc); +- #line 3593 "Python/generated_cases.c.h" ++ #line 3686 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = new_exc; + stack_pointer[-2] = prev_exc; +@@ -3603,7 +3696,7 @@ + uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t keys_version = read_u32(&next_instr[3].cache); + PyObject *descr = read_obj(&next_instr[5].cache); +- #line 2564 "Python/bytecodes.c" ++ #line 2625 "Python/bytecodes.c" + /* Cached method object */ + PyTypeObject *self_cls = Py_TYPE(self); + assert(type_version != 0); +@@ -3620,7 +3713,7 @@ + assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); + res = self; + assert(oparg & 1); +- #line 3624 "Python/generated_cases.c.h" ++ #line 3717 "Python/generated_cases.c.h" + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } +@@ -3634,7 +3727,7 @@ + PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + PyObject *descr = read_obj(&next_instr[5].cache); +- #line 2583 "Python/bytecodes.c" ++ #line 2644 "Python/bytecodes.c" + PyTypeObject *self_cls = Py_TYPE(self); + DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(self_cls->tp_dictoffset == 0); +@@ -3644,7 +3737,7 @@ + res2 = Py_NewRef(descr); + res = self; + assert(oparg & 1); +- #line 3648 "Python/generated_cases.c.h" ++ #line 3741 "Python/generated_cases.c.h" + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } +@@ -3658,7 +3751,7 @@ + PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + PyObject *descr = read_obj(&next_instr[5].cache); +- #line 2595 "Python/bytecodes.c" ++ #line 2656 "Python/bytecodes.c" + PyTypeObject *self_cls = Py_TYPE(self); + DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); + Py_ssize_t dictoffset = self_cls->tp_dictoffset; +@@ -3672,7 +3765,7 @@ + res2 = Py_NewRef(descr); + res = self; + assert(oparg & 1); +- #line 3676 "Python/generated_cases.c.h" ++ #line 3769 "Python/generated_cases.c.h" + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } +@@ -3681,16 +3774,16 @@ + } + + TARGET(KW_NAMES) { +- #line 2611 "Python/bytecodes.c" ++ #line 2672 "Python/bytecodes.c" + assert(kwnames == NULL); + assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); + kwnames = GETITEM(frame->f_code->co_consts, oparg); +- #line 3689 "Python/generated_cases.c.h" ++ #line 3782 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(INSTRUMENTED_CALL) { +- #line 2617 "Python/bytecodes.c" ++ #line 2678 "Python/bytecodes.c" + int is_meth = PEEK(oparg+2) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(total_args + 1); +@@ -3703,7 +3796,7 @@ + _PyCallCache *cache = (_PyCallCache *)next_instr; + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + GO_TO_INSTRUCTION(CALL); +- #line 3707 "Python/generated_cases.c.h" ++ #line 3800 "Python/generated_cases.c.h" + } + + TARGET(CALL) { +@@ -3713,7 +3806,7 @@ + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 2662 "Python/bytecodes.c" ++ #line 2723 "Python/bytecodes.c" + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { +@@ -3795,7 +3888,7 @@ + Py_DECREF(args[i]); + } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } +- #line 3799 "Python/generated_cases.c.h" ++ #line 3892 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -3807,7 +3900,7 @@ + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; +- #line 2750 "Python/bytecodes.c" ++ #line 2811 "Python/bytecodes.c" + DEOPT_IF(method != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + STAT_INC(CALL, hit); +@@ -3817,7 +3910,7 @@ + PEEK(oparg + 2) = Py_NewRef(meth); // method + Py_DECREF(callable); + GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); +- #line 3821 "Python/generated_cases.c.h" ++ #line 3914 "Python/generated_cases.c.h" + } + + TARGET(CALL_PY_EXACT_ARGS) { +@@ -3826,7 +3919,7 @@ + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; + uint32_t func_version = read_u32(&next_instr[1].cache); +- #line 2762 "Python/bytecodes.c" ++ #line 2823 "Python/bytecodes.c" + assert(kwnames == NULL); + DEOPT_IF(tstate->interp->eval_frame, CALL); + int is_meth = method != NULL; +@@ -3852,7 +3945,7 @@ + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->return_offset = 0; + DISPATCH_INLINED(new_frame); +- #line 3856 "Python/generated_cases.c.h" ++ #line 3949 "Python/generated_cases.c.h" + } + + TARGET(CALL_PY_WITH_DEFAULTS) { +@@ -3860,7 +3953,7 @@ + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; + uint32_t func_version = read_u32(&next_instr[1].cache); +- #line 2790 "Python/bytecodes.c" ++ #line 2851 "Python/bytecodes.c" + assert(kwnames == NULL); + DEOPT_IF(tstate->interp->eval_frame, CALL); + int is_meth = method != NULL; +@@ -3896,7 +3989,7 @@ + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->return_offset = 0; + DISPATCH_INLINED(new_frame); +- #line 3900 "Python/generated_cases.c.h" ++ #line 3993 "Python/generated_cases.c.h" + } + + TARGET(CALL_NO_KW_TYPE_1) { +@@ -3904,7 +3997,7 @@ + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 2828 "Python/bytecodes.c" ++ #line 2889 "Python/bytecodes.c" + assert(kwnames == NULL); + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); +@@ -3914,7 +4007,7 @@ + res = Py_NewRef(Py_TYPE(obj)); + Py_DECREF(obj); + Py_DECREF(&PyType_Type); // I.e., callable +- #line 3918 "Python/generated_cases.c.h" ++ #line 4011 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -3927,7 +4020,7 @@ + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 2840 "Python/bytecodes.c" ++ #line 2901 "Python/bytecodes.c" + assert(kwnames == NULL); + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); +@@ -3938,7 +4031,7 @@ + Py_DECREF(arg); + Py_DECREF(&PyUnicode_Type); // I.e., callable + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } +- #line 3942 "Python/generated_cases.c.h" ++ #line 4035 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -3952,7 +4045,7 @@ + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 2854 "Python/bytecodes.c" ++ #line 2915 "Python/bytecodes.c" + assert(kwnames == NULL); + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); +@@ -3963,7 +4056,7 @@ + Py_DECREF(arg); + Py_DECREF(&PyTuple_Type); // I.e., tuple + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } +- #line 3967 "Python/generated_cases.c.h" ++ #line 4060 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -3977,7 +4070,7 @@ + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 2868 "Python/bytecodes.c" ++ #line 2929 "Python/bytecodes.c" + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { +@@ -3999,7 +4092,7 @@ + } + Py_DECREF(tp); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } +- #line 4003 "Python/generated_cases.c.h" ++ #line 4096 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -4013,7 +4106,7 @@ + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 2893 "Python/bytecodes.c" ++ #line 2954 "Python/bytecodes.c" + /* Builtin METH_O functions */ + assert(kwnames == NULL); + int is_meth = method != NULL; +@@ -4041,7 +4134,7 @@ + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } +- #line 4045 "Python/generated_cases.c.h" ++ #line 4138 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -4055,7 +4148,7 @@ + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 2924 "Python/bytecodes.c" ++ #line 2985 "Python/bytecodes.c" + /* Builtin METH_FASTCALL functions, without keywords */ + assert(kwnames == NULL); + int is_meth = method != NULL; +@@ -4087,7 +4180,7 @@ + 'invalid'). In those cases an exception is set, so we must + handle it. + */ +- #line 4091 "Python/generated_cases.c.h" ++ #line 4184 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -4101,7 +4194,7 @@ + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 2959 "Python/bytecodes.c" ++ #line 3020 "Python/bytecodes.c" + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + int is_meth = method != NULL; + int total_args = oparg; +@@ -4133,7 +4226,7 @@ + } + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } +- #line 4137 "Python/generated_cases.c.h" ++ #line 4230 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -4147,7 +4240,7 @@ + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 2994 "Python/bytecodes.c" ++ #line 3055 "Python/bytecodes.c" + assert(kwnames == NULL); + /* len(o) */ + int is_meth = method != NULL; +@@ -4172,7 +4265,7 @@ + Py_DECREF(callable); + Py_DECREF(arg); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } +- #line 4176 "Python/generated_cases.c.h" ++ #line 4269 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -4185,7 +4278,7 @@ + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 3021 "Python/bytecodes.c" ++ #line 3082 "Python/bytecodes.c" + assert(kwnames == NULL); + /* isinstance(o, o2) */ + int is_meth = method != NULL; +@@ -4212,7 +4305,7 @@ + Py_DECREF(cls); + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } +- #line 4216 "Python/generated_cases.c.h" ++ #line 4309 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -4224,7 +4317,7 @@ + PyObject **args = (stack_pointer - oparg); + PyObject *self = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; +- #line 3051 "Python/bytecodes.c" ++ #line 3112 "Python/bytecodes.c" + assert(kwnames == NULL); + assert(oparg == 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); +@@ -4242,14 +4335,14 @@ + JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); + assert(next_instr[-1].op.code == POP_TOP); + DISPATCH(); +- #line 4246 "Python/generated_cases.c.h" ++ #line 4339 "Python/generated_cases.c.h" + } + + TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { + PyObject **args = (stack_pointer - oparg); + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 3071 "Python/bytecodes.c" ++ #line 3132 "Python/bytecodes.c" + assert(kwnames == NULL); + int is_meth = method != NULL; + int total_args = oparg; +@@ -4280,7 +4373,7 @@ + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } +- #line 4284 "Python/generated_cases.c.h" ++ #line 4377 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -4293,7 +4386,7 @@ + PyObject **args = (stack_pointer - oparg); + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 3105 "Python/bytecodes.c" ++ #line 3166 "Python/bytecodes.c" + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { +@@ -4322,7 +4415,7 @@ + } + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } +- #line 4326 "Python/generated_cases.c.h" ++ #line 4419 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -4335,7 +4428,7 @@ + PyObject **args = (stack_pointer - oparg); + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 3137 "Python/bytecodes.c" ++ #line 3198 "Python/bytecodes.c" + assert(kwnames == NULL); + assert(oparg == 0 || oparg == 1); + int is_meth = method != NULL; +@@ -4364,7 +4457,7 @@ + Py_DECREF(self); + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } +- #line 4368 "Python/generated_cases.c.h" ++ #line 4461 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -4377,7 +4470,7 @@ + PyObject **args = (stack_pointer - oparg); + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; +- #line 3169 "Python/bytecodes.c" ++ #line 3230 "Python/bytecodes.c" + assert(kwnames == NULL); + int is_meth = method != NULL; + int total_args = oparg; +@@ -4405,7 +4498,7 @@ + } + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } +- #line 4409 "Python/generated_cases.c.h" ++ #line 4502 "Python/generated_cases.c.h" + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; +@@ -4415,9 +4508,9 @@ + } + + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { +- #line 3200 "Python/bytecodes.c" ++ #line 3261 "Python/bytecodes.c" + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); +- #line 4421 "Python/generated_cases.c.h" ++ #line 4514 "Python/generated_cases.c.h" + } + + TARGET(CALL_FUNCTION_EX) { +@@ -4426,7 +4519,7 @@ + PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; + PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; + PyObject *result; +- #line 3204 "Python/bytecodes.c" ++ #line 3265 "Python/bytecodes.c" + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); +@@ -4488,14 +4581,14 @@ + } + result = PyObject_Call(func, callargs, kwargs); + } +- #line 4492 "Python/generated_cases.c.h" ++ #line 4585 "Python/generated_cases.c.h" + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); +- #line 3266 "Python/bytecodes.c" ++ #line 3327 "Python/bytecodes.c" + assert(PEEK(3 + (oparg & 1)) == NULL); + if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } +- #line 4499 "Python/generated_cases.c.h" ++ #line 4592 "Python/generated_cases.c.h" + STACK_SHRINK(((oparg & 1) ? 1 : 0)); + STACK_SHRINK(2); + stack_pointer[-1] = result; +@@ -4510,7 +4603,7 @@ + PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; + PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; + PyObject *func; +- #line 3276 "Python/bytecodes.c" ++ #line 3337 "Python/bytecodes.c" + + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); +@@ -4539,14 +4632,14 @@ + + func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; + func = (PyObject *)func_obj; +- #line 4543 "Python/generated_cases.c.h" ++ #line 4636 "Python/generated_cases.c.h" + STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); + stack_pointer[-1] = func; + DISPATCH(); + } + + TARGET(RETURN_GENERATOR) { +- #line 3307 "Python/bytecodes.c" ++ #line 3368 "Python/bytecodes.c" + assert(PyFunction_Check(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); +@@ -4567,7 +4660,7 @@ + frame = cframe.current_frame = prev; + _PyFrame_StackPush(frame, (PyObject *)gen); + goto resume_frame; +- #line 4571 "Python/generated_cases.c.h" ++ #line 4664 "Python/generated_cases.c.h" + } + + TARGET(BUILD_SLICE) { +@@ -4575,15 +4668,15 @@ + PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; + PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; + PyObject *slice; +- #line 3330 "Python/bytecodes.c" ++ #line 3391 "Python/bytecodes.c" + slice = PySlice_New(start, stop, step); +- #line 4581 "Python/generated_cases.c.h" ++ #line 4674 "Python/generated_cases.c.h" + Py_DECREF(start); + Py_DECREF(stop); + Py_XDECREF(step); +- #line 3332 "Python/bytecodes.c" ++ #line 3393 "Python/bytecodes.c" + if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } +- #line 4587 "Python/generated_cases.c.h" ++ #line 4680 "Python/generated_cases.c.h" + STACK_SHRINK(((oparg == 3) ? 1 : 0)); + STACK_SHRINK(1); + stack_pointer[-1] = slice; +@@ -4594,7 +4687,7 @@ + PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; + PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; + PyObject *result; +- #line 3336 "Python/bytecodes.c" ++ #line 3397 "Python/bytecodes.c" + /* Handles f-string value formatting. */ + PyObject *(*conv_fn)(PyObject *); + int which_conversion = oparg & FVC_MASK; +@@ -4629,7 +4722,7 @@ + Py_DECREF(value); + Py_XDECREF(fmt_spec); + if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } +- #line 4633 "Python/generated_cases.c.h" ++ #line 4726 "Python/generated_cases.c.h" + STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); + stack_pointer[-1] = result; + DISPATCH(); +@@ -4638,10 +4731,10 @@ + TARGET(COPY) { + PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; + PyObject *top; +- #line 3373 "Python/bytecodes.c" ++ #line 3434 "Python/bytecodes.c" + assert(oparg > 0); + top = Py_NewRef(bottom); +- #line 4645 "Python/generated_cases.c.h" ++ #line 4738 "Python/generated_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = top; + DISPATCH(); +@@ -4653,7 +4746,7 @@ + PyObject *rhs = stack_pointer[-1]; + PyObject *lhs = stack_pointer[-2]; + PyObject *res; +- #line 3378 "Python/bytecodes.c" ++ #line 3439 "Python/bytecodes.c" + #if ENABLE_SPECIALIZATION + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { +@@ -4668,12 +4761,12 @@ + assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); + assert(binary_ops[oparg]); + res = binary_ops[oparg](lhs, rhs); +- #line 4672 "Python/generated_cases.c.h" ++ #line 4765 "Python/generated_cases.c.h" + Py_DECREF(lhs); + Py_DECREF(rhs); +- #line 3393 "Python/bytecodes.c" ++ #line 3454 "Python/bytecodes.c" + if (res == NULL) goto pop_2_error; +- #line 4677 "Python/generated_cases.c.h" ++ #line 4770 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = res; + next_instr += 1; +@@ -4683,16 +4776,16 @@ + TARGET(SWAP) { + PyObject *top = stack_pointer[-1]; + PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; +- #line 3398 "Python/bytecodes.c" ++ #line 3459 "Python/bytecodes.c" + assert(oparg >= 2); +- #line 4689 "Python/generated_cases.c.h" ++ #line 4782 "Python/generated_cases.c.h" + stack_pointer[-1] = bottom; + stack_pointer[-(2 + (oparg-2))] = top; + DISPATCH(); + } + + TARGET(INSTRUMENTED_INSTRUCTION) { +- #line 3402 "Python/bytecodes.c" ++ #line 3463 "Python/bytecodes.c" + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, next_instr-1); + if (next_opcode < 0) goto error; +@@ -4704,26 +4797,26 @@ + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); +- #line 4708 "Python/generated_cases.c.h" ++ #line 4801 "Python/generated_cases.c.h" + } + + TARGET(INSTRUMENTED_JUMP_FORWARD) { +- #line 3416 "Python/bytecodes.c" ++ #line 3477 "Python/bytecodes.c" + INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); +- #line 4714 "Python/generated_cases.c.h" ++ #line 4807 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(INSTRUMENTED_JUMP_BACKWARD) { +- #line 3420 "Python/bytecodes.c" ++ #line 3481 "Python/bytecodes.c" + INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); +- #line 4721 "Python/generated_cases.c.h" ++ #line 4814 "Python/generated_cases.c.h" + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { +- #line 3425 "Python/bytecodes.c" ++ #line 3486 "Python/bytecodes.c" + PyObject *cond = POP(); + int err = PyObject_IsTrue(cond); + Py_DECREF(cond); +@@ -4732,12 +4825,12 @@ + assert(err == 0 || err == 1); + int offset = err*oparg; + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); +- #line 4736 "Python/generated_cases.c.h" ++ #line 4829 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { +- #line 3436 "Python/bytecodes.c" ++ #line 3497 "Python/bytecodes.c" + PyObject *cond = POP(); + int err = PyObject_IsTrue(cond); + Py_DECREF(cond); +@@ -4746,12 +4839,12 @@ + assert(err == 0 || err == 1); + int offset = (1-err)*oparg; + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); +- #line 4750 "Python/generated_cases.c.h" ++ #line 4843 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { +- #line 3447 "Python/bytecodes.c" ++ #line 3508 "Python/bytecodes.c" + PyObject *value = POP(); + _Py_CODEUNIT *here = next_instr-1; + int offset; +@@ -4763,12 +4856,12 @@ + offset = 0; + } + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); +- #line 4767 "Python/generated_cases.c.h" ++ #line 4860 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { +- #line 3461 "Python/bytecodes.c" ++ #line 3522 "Python/bytecodes.c" + PyObject *value = POP(); + _Py_CODEUNIT *here = next_instr-1; + int offset; +@@ -4780,30 +4873,30 @@ + offset = oparg; + } + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); +- #line 4784 "Python/generated_cases.c.h" ++ #line 4877 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(EXTENDED_ARG) { +- #line 3475 "Python/bytecodes.c" ++ #line 3536 "Python/bytecodes.c" + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); +- #line 4795 "Python/generated_cases.c.h" ++ #line 4888 "Python/generated_cases.c.h" + } + + TARGET(CACHE) { +- #line 3483 "Python/bytecodes.c" ++ #line 3544 "Python/bytecodes.c" + assert(0 && "Executing a cache."); + Py_UNREACHABLE(); +- #line 4802 "Python/generated_cases.c.h" ++ #line 4895 "Python/generated_cases.c.h" + } + + TARGET(RESERVED) { +- #line 3488 "Python/bytecodes.c" ++ #line 3549 "Python/bytecodes.c" + assert(0 && "Executing RESERVED instruction."); + Py_UNREACHABLE(); +- #line 4809 "Python/generated_cases.c.h" ++ #line 4902 "Python/generated_cases.c.h" + } diff --git a/native/patches/cpython-3.13.15-hooks.patch b/native/patches/cpython-3.13.15-hooks.patch new file mode 100644 index 0000000..82b0832 --- /dev/null +++ b/native/patches/cpython-3.13.15-hooks.patch @@ -0,0 +1,678 @@ +diff --git a/Python/bytecodes.c b/Python/bytecodes.c +index 2456848..ab70701 100644 +--- a/Python/bytecodes.c ++++ b/Python/bytecodes.c +@@ -182,6 +182,10 @@ dummy_func( + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } ++ if ((oparg & RESUME_OPARG_LOCATION_MASK) == RESUME_AT_FUNC_START) { ++ int hook_err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ ERROR_IF(hook_err, error); ++ } + } + + inst(RESUME_CHECK, (--)) { +@@ -193,6 +197,10 @@ dummy_func( + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version); ++ if ((oparg & RESUME_OPARG_LOCATION_MASK) == RESUME_AT_FUNC_START) { ++ int hook_err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ ERROR_IF(hook_err, error); ++ } + } + + inst(INSTRUMENTED_RESUME, (--)) { +@@ -218,6 +226,10 @@ dummy_func( + next_instr = frame->instr_ptr; + DISPATCH(); + } ++ if ((oparg & RESUME_OPARG_LOCATION_MASK) == RESUME_AT_FUNC_START) { ++ int hook_err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ ERROR_IF(hook_err, error); ++ } + } + } + +@@ -264,6 +276,12 @@ dummy_func( + } + + replicate(8) inst(STORE_FAST, (value --)) { ++ PyObject *name = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); ++ int err = aiython_before_store(frame, name, value); ++ if (err < 0) { ++ DECREF_INPUTS(); ++ ERROR_IF(true, error); ++ } + SETLOCAL(oparg, value); + } + +@@ -274,6 +292,12 @@ dummy_func( + inst(STORE_FAST_LOAD_FAST, (value1 -- value2)) { + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; ++ PyObject *name = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg1); ++ int err = aiython_before_store(frame, name, value1); ++ if (err < 0) { ++ DECREF_INPUTS(); ++ ERROR_IF(true, error); ++ } + SETLOCAL(oparg1, value1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value2); +@@ -282,7 +306,19 @@ dummy_func( + inst(STORE_FAST_STORE_FAST, (value2, value1 --)) { + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; ++ PyObject *name1 = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg1); ++ PyObject *name2 = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg2); ++ int err = aiython_before_store(frame, name1, value1); ++ if (err < 0) { ++ DECREF_INPUTS(); ++ ERROR_IF(true, error); ++ } + SETLOCAL(oparg1, value1); ++ err = aiython_before_store(frame, name2, value2); ++ if (err < 0) { ++ Py_DECREF(value2); ++ ERROR_IF(true, error); ++ } + SETLOCAL(oparg2, value2); + } + +@@ -545,6 +581,8 @@ dummy_func( + // specializations, but there is no output. + // At the end we just skip over the STORE_FAST. + tier1 op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { ++ int store_hook_enabled = aiython_store_hook_enabled(); ++ DEOPT_IF(store_hook_enabled); + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(*target_local != left); +@@ -623,7 +661,10 @@ dummy_func( + err = 1; + } + else { +- err = PyObject_SetItem(container, slice, v); ++ err = aiython_before_mutation(frame, "item", container, slice, v); ++ if (err == 0) { ++ err = PyObject_SetItem(container, slice, v); ++ } + Py_DECREF(slice); + } + Py_DECREF(v); +@@ -742,14 +783,24 @@ dummy_func( + + op(_STORE_SUBSCR, (v, container, sub -- )) { + /* container[sub] = v */ +- int err = PyObject_SetItem(container, sub, v); ++ int err = aiython_before_mutation(frame, "item", container, sub, v); ++ if (err == 0) { ++ err = PyObject_SetItem(container, sub, v); ++ } + DECREF_INPUTS(); + ERROR_IF(err, error); + } + ++ op(_AIYTHON_DEOPT_MUTATION, (--)) { ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled); ++ } ++ + macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR; + + inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) { ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled); + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyList_CheckExact(list)); + +@@ -769,6 +820,8 @@ dummy_func( + } + + inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) { ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled); + DEOPT_IF(!PyDict_CheckExact(dict)); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); +@@ -853,14 +906,27 @@ dummy_func( + LLTRACE_RESUME_FRAME(); + } + ++ op(_AIYTHON_ON_RETURN, (retval -- retval)) { ++ int hook_err = aiython_function_event(frame, "_aiython_on_return", retval); ++ ERROR_IF(hook_err, error); ++ } ++ ++ op(_AIYTHON_ON_RETURN_CONST, (--)) { ++ PyObject *value = GETITEM(FRAME_CO_CONSTS, oparg); ++ int hook_err = aiython_function_event(frame, "_aiython_on_return", value); ++ ERROR_IF(hook_err, error); ++ } ++ + macro(RETURN_VALUE) = +- _POP_FRAME; ++ _AIYTHON_ON_RETURN + _POP_FRAME; + + inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) ERROR_NO_POP(); ++ err = aiython_function_event(frame, "_aiython_on_return", retval); ++ if (err) ERROR_NO_POP(); + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); +@@ -876,6 +942,7 @@ dummy_func( + } + + macro(RETURN_CONST) = ++ _AIYTHON_ON_RETURN_CONST + + LOAD_CONST + + _POP_FRAME; + +@@ -885,6 +952,8 @@ dummy_func( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) ERROR_NO_POP(); ++ err = aiython_function_event(frame, "_aiython_on_return", retval); ++ if (err) ERROR_NO_POP(); + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); +@@ -1215,6 +1284,11 @@ dummy_func( + DECREF_INPUTS(); + ERROR_IF(true, error); + } ++ err = aiython_before_store(frame, name, v); ++ if (err < 0) { ++ DECREF_INPUTS(); ++ ERROR_IF(true, error); ++ } + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else +@@ -1332,7 +1406,10 @@ dummy_func( + + op(_STORE_ATTR, (v, owner --)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); +- int err = PyObject_SetAttr(owner, name, v); ++ int err = aiython_before_mutation(frame, "attr", owner, name, v); ++ if (err == 0) { ++ err = PyObject_SetAttr(owner, name, v); ++ } + DECREF_INPUTS(); + ERROR_IF(err, error); + } +@@ -1348,6 +1425,11 @@ dummy_func( + + inst(STORE_GLOBAL, (v --)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); ++ int hook_err = aiython_before_store(frame, name, v); ++ if (hook_err < 0) { ++ DECREF_INPUTS(); ++ ERROR_IF(true, error); ++ } + int err = PyDict_SetItem(GLOBALS(), name, v); + DECREF_INPUTS(); + ERROR_IF(err, error); +@@ -1618,6 +1700,12 @@ dummy_func( + } + + inst(STORE_DEREF, (v --)) { ++ PyObject *name = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); ++ int err = aiython_before_store(frame, name, v); ++ if (err < 0) { ++ DECREF_INPUTS(); ++ ERROR_IF(true, error); ++ } + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + PyCell_SetTakeRef(cell, v); + } +@@ -2146,11 +2234,14 @@ dummy_func( + + macro(STORE_ATTR_INSTANCE_VALUE) = + unused/1 + ++ _AIYTHON_DEOPT_MUTATION + + _GUARD_TYPE_VERSION + + _GUARD_DORV_NO_DICT + + _STORE_ATTR_INSTANCE_VALUE; + + inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) { ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version); +@@ -2202,6 +2293,7 @@ dummy_func( + + macro(STORE_ATTR_SLOT) = + unused/1 + ++ _AIYTHON_DEOPT_MUTATION + + _GUARD_TYPE_VERSION + + _STORE_ATTR_SLOT; + +diff --git a/Python/ceval.c b/Python/ceval.c +index 7a4e704..29f18ea 100644 +--- a/Python/ceval.c ++++ b/Python/ceval.c +@@ -681,6 +681,122 @@ extern void _PyUOpPrint(const _PyUOpInstruction *uop); + * so consume 3 units of C stack */ + #define PY_EVAL_C_STACK_UNITS 2 + ++/* Experimental Aiython hook. The Python callback can raise to reject a store. ++ The guard prevents assignments inside that callback from calling it again. */ ++static _Thread_local int aiython_in_hook = 0; ++ ++static int ++aiython_before_store(_PyInterpreterFrame *frame, PyObject *name, PyObject *value) ++{ ++ /* STORE_FAST_MAYBE_NULL may legally write an empty stack reference. */ ++ if (value == NULL || aiython_in_hook) { ++ return 0; ++ } ++ PyObject *callback = NULL; ++ int found = _PySys_GetOptionalAttrString("_aiython_before_store", &callback); ++ if (found <= 0) { ++ return found < 0 ? -1 : 0; ++ } ++ PyFrameObject *borrowed = _PyFrame_GetFrameObject(frame); ++ if (borrowed == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *frame_obj = Py_NewRef(borrowed); ++ aiython_in_hook = 1; ++ PyObject *result = PyObject_CallFunctionObjArgs(callback, frame_obj, name, value, NULL); ++ aiython_in_hook = 0; ++ Py_DECREF(frame_obj); ++ Py_DECREF(callback); ++ if (result == NULL) { ++ return -1; ++ } ++ Py_DECREF(result); ++ return 0; ++} ++ ++static int ++aiython_mutation_hook_enabled(void) ++{ ++ return PySys_GetObject("_aiython_before_mutation") != NULL; ++} ++ ++static int ++aiython_store_hook_enabled(void) ++{ ++ return PySys_GetObject("_aiython_before_store") != NULL; ++} ++ ++static int ++aiython_before_mutation(_PyInterpreterFrame *frame, const char *kind, ++ PyObject *owner, PyObject *key, PyObject *value) ++{ ++ if (aiython_in_hook) { ++ return 0; ++ } ++ PyObject *callback = NULL; ++ int found = _PySys_GetOptionalAttrString("_aiython_before_mutation", &callback); ++ if (found <= 0) { ++ return found < 0 ? -1 : 0; ++ } ++ PyFrameObject *borrowed = _PyFrame_GetFrameObject(frame); ++ if (borrowed == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *kind_obj = PyUnicode_FromString(kind); ++ if (kind_obj == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *frame_obj = Py_NewRef(borrowed); ++ aiython_in_hook = 1; ++ PyObject *result = PyObject_CallFunctionObjArgs( ++ callback, frame_obj, kind_obj, owner, key, value, NULL); ++ aiython_in_hook = 0; ++ Py_DECREF(frame_obj); ++ Py_DECREF(kind_obj); ++ Py_DECREF(callback); ++ if (result == NULL) { ++ return -1; ++ } ++ Py_DECREF(result); ++ return 0; ++} ++ ++static int ++aiython_function_event(_PyInterpreterFrame *frame, const char *attribute, ++ PyObject *value) ++{ ++ if (aiython_in_hook) { ++ return 0; ++ } ++ PyObject *callback = NULL; ++ int found = _PySys_GetOptionalAttrString(attribute, &callback); ++ if (found <= 0) { ++ return found < 0 ? -1 : 0; ++ } ++ PyFrameObject *borrowed = _PyFrame_GetFrameObject(frame); ++ if (borrowed == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *frame_obj = Py_NewRef(borrowed); ++ aiython_in_hook = 1; ++ PyObject *result = value == NULL ++ ? PyObject_CallOneArg(callback, frame_obj) ++ : PyObject_CallFunctionObjArgs(callback, frame_obj, value, NULL); ++ aiython_in_hook = 0; ++ Py_DECREF(frame_obj); ++ Py_DECREF(callback); ++ if (result == NULL) { ++ return -1; ++ } ++ Py_DECREF(result); ++ return 0; ++} ++ ++ + PyObject* _Py_HOT_FUNCTION + _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) + { +diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h +index ad49bec..fd75239 100644 +--- a/Python/generated_cases.c.h ++++ b/Python/generated_cases.c.h +@@ -243,6 +243,8 @@ + /* Skip 1 cache entry */ + // _BINARY_OP_INPLACE_ADD_UNICODE + { ++ int store_hook_enabled = aiython_store_hook_enabled(); ++ DEOPT_IF(store_hook_enabled, BINARY_OP); + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(*target_local != left, BINARY_OP); +@@ -3498,6 +3500,10 @@ + next_instr = frame->instr_ptr; + DISPATCH(); + } ++ if ((oparg & RESUME_OPARG_LOCATION_MASK) == RESUME_AT_FUNC_START) { ++ int hook_err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ if (hook_err) goto error; ++ } + } + DISPATCH(); + } +@@ -3512,6 +3518,8 @@ + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) goto error; ++ err = aiython_function_event(frame, "_aiython_on_return", retval); ++ if (err) goto error; + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); +@@ -3537,6 +3545,8 @@ + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) goto error; ++ err = aiython_function_event(frame, "_aiython_on_return", retval); ++ if (err) goto error; + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); +@@ -5185,6 +5195,10 @@ + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } ++ if ((oparg & RESUME_OPARG_LOCATION_MASK) == RESUME_AT_FUNC_START) { ++ int hook_err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ if (hook_err) goto error; ++ } + DISPATCH(); + } + +@@ -5201,6 +5215,10 @@ + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); ++ if ((oparg & RESUME_OPARG_LOCATION_MASK) == RESUME_AT_FUNC_START) { ++ int hook_err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ if (hook_err) goto error; ++ } + DISPATCH(); + } + +@@ -5210,6 +5228,12 @@ + INSTRUCTION_STATS(RETURN_CONST); + PyObject *value; + PyObject *retval; ++ // _AIYTHON_ON_RETURN_CONST ++ { ++ PyObject *value = GETITEM(FRAME_CO_CONSTS, oparg); ++ int hook_err = aiython_function_event(frame, "_aiython_on_return", value); ++ if (hook_err) goto error; ++ } + // _LOAD_CONST + { + value = GETITEM(FRAME_CO_CONSTS, oparg); +@@ -5273,22 +5297,30 @@ + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + PyObject *retval; ++ // _AIYTHON_ON_RETURN + retval = stack_pointer[-1]; +- #if TIER_ONE +- assert(frame != &entry_frame); +- #endif +- stack_pointer += -1; +- _PyFrame_SetStackPointer(frame, stack_pointer); +- assert(EMPTY()); +- _Py_LeaveRecursiveCallPy(tstate); +- // GH-99729: We need to unlink the frame *before* clearing it: +- _PyInterpreterFrame *dying = frame; +- frame = tstate->current_frame = dying->previous; +- _PyEval_FrameClearAndPop(tstate, dying); +- _PyFrame_StackPush(frame, retval); +- LOAD_SP(); +- LOAD_IP(frame->return_offset); +- LLTRACE_RESUME_FRAME(); ++ { ++ int hook_err = aiython_function_event(frame, "_aiython_on_return", retval); ++ if (hook_err) goto error; ++ } ++ // _POP_FRAME ++ { ++ #if TIER_ONE ++ assert(frame != &entry_frame); ++ #endif ++ stack_pointer += -1; ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ assert(EMPTY()); ++ _Py_LeaveRecursiveCallPy(tstate); ++ // GH-99729: We need to unlink the frame *before* clearing it: ++ _PyInterpreterFrame *dying = frame; ++ frame = tstate->current_frame = dying->previous; ++ _PyEval_FrameClearAndPop(tstate, dying); ++ _PyFrame_StackPush(frame, retval); ++ LOAD_SP(); ++ LOAD_IP(frame->return_offset); ++ LLTRACE_RESUME_FRAME(); ++ } + DISPATCH(); + } + +@@ -5511,7 +5543,10 @@ + v = stack_pointer[-2]; + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); +- int err = PyObject_SetAttr(owner, name, v); ++ int err = aiython_before_mutation(frame, "attr", owner, name, v); ++ if (err == 0) { ++ err = PyObject_SetAttr(owner, name, v); ++ } + Py_DECREF(v); + Py_DECREF(owner); + if (err) goto pop_2_error; +@@ -5528,6 +5563,11 @@ + PyObject *owner; + PyObject *value; + /* Skip 1 cache entry */ ++ // _AIYTHON_DEOPT_MUTATION ++ { ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_ATTR); ++ } + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { +@@ -5572,6 +5612,11 @@ + PyObject *owner; + PyObject *value; + /* Skip 1 cache entry */ ++ // _AIYTHON_DEOPT_MUTATION ++ { ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_ATTR); ++ } + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { +@@ -5607,6 +5652,8 @@ + value = stack_pointer[-2]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint16_t hint = read_u16(&this_instr[4].cache); ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); +@@ -5654,6 +5701,12 @@ + INSTRUCTION_STATS(STORE_DEREF); + PyObject *v; + v = stack_pointer[-1]; ++ PyObject *name = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); ++ int err = aiython_before_store(frame, name, v); ++ if (err < 0) { ++ Py_DECREF(v); ++ if (true) goto pop_1_error; ++ } + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + PyCell_SetTakeRef(cell, v); + stack_pointer += -1; +@@ -5666,6 +5719,12 @@ + INSTRUCTION_STATS(STORE_FAST); + PyObject *value; + value = stack_pointer[-1]; ++ PyObject *name = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); ++ int err = aiython_before_store(frame, name, value); ++ if (err < 0) { ++ Py_DECREF(value); ++ if (true) goto pop_1_error; ++ } + SETLOCAL(oparg, value); + stack_pointer += -1; + DISPATCH(); +@@ -5680,6 +5739,12 @@ + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; ++ PyObject *name = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg1); ++ int err = aiython_before_store(frame, name, value1); ++ if (err < 0) { ++ Py_DECREF(value1); ++ if (true) goto pop_1_error; ++ } + SETLOCAL(oparg1, value1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value2); +@@ -5697,7 +5762,20 @@ + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; ++ PyObject *name1 = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg1); ++ PyObject *name2 = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg2); ++ int err = aiython_before_store(frame, name1, value1); ++ if (err < 0) { ++ Py_DECREF(value2); ++ Py_DECREF(value1); ++ if (true) goto pop_2_error; ++ } + SETLOCAL(oparg1, value1); ++ err = aiython_before_store(frame, name2, value2); ++ if (err < 0) { ++ Py_DECREF(value2); ++ if (true) goto pop_2_error; ++ } + SETLOCAL(oparg2, value2); + stack_pointer += -2; + DISPATCH(); +@@ -5710,6 +5788,11 @@ + PyObject *v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); ++ int hook_err = aiython_before_store(frame, name, v); ++ if (hook_err < 0) { ++ Py_DECREF(v); ++ if (true) goto pop_1_error; ++ } + int err = PyDict_SetItem(GLOBALS(), name, v); + Py_DECREF(v); + if (err) goto pop_1_error; +@@ -5732,6 +5815,11 @@ + Py_DECREF(v); + if (true) goto pop_1_error; + } ++ err = aiython_before_store(frame, name, v); ++ if (err < 0) { ++ Py_DECREF(v); ++ if (true) goto pop_1_error; ++ } + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else +@@ -5760,7 +5848,10 @@ + err = 1; + } + else { +- err = PyObject_SetItem(container, slice, v); ++ err = aiython_before_mutation(frame, "item", container, slice, v); ++ if (err == 0) { ++ err = PyObject_SetItem(container, slice, v); ++ } + Py_DECREF(slice); + } + Py_DECREF(v); +@@ -5800,7 +5891,10 @@ + v = stack_pointer[-3]; + { + /* container[sub] = v */ +- int err = PyObject_SetItem(container, sub, v); ++ int err = aiython_before_mutation(frame, "item", container, sub, v); ++ if (err == 0) { ++ err = PyObject_SetItem(container, sub, v); ++ } + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); +@@ -5822,6 +5916,8 @@ + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); +@@ -5843,6 +5939,8 @@ + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled, STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + // Ensure nonnegative, zero-or-one-digit ints. diff --git a/native/patches/cpython-3.14.4-hooks.patch b/native/patches/cpython-3.14.4-hooks.patch new file mode 100644 index 0000000..9a02aec --- /dev/null +++ b/native/patches/cpython-3.14.4-hooks.patch @@ -0,0 +1,764 @@ +diff --git a/Python/bytecodes.c b/Python/bytecodes.c +index a477fdd..08d6598 100644 +--- a/Python/bytecodes.c ++++ b/Python/bytecodes.c +@@ -221,11 +221,19 @@ dummy_func( + #endif + } + ++ op(_AIYTHON_FUNCTION_ENTRY, (--)) { ++ if (oparg == 0) { ++ int err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ ERROR_IF(err); ++ } ++ } ++ + macro(RESUME) = + _LOAD_BYTECODE + + _MAYBE_INSTRUMENT + + _QUICKEN_RESUME + +- _CHECK_PERIODIC_IF_NOT_YIELD_FROM; ++ _CHECK_PERIODIC_IF_NOT_YIELD_FROM + ++ _AIYTHON_FUNCTION_ENTRY; + + inst(RESUME_CHECK, (--)) { + #if defined(__EMSCRIPTEN__) +@@ -240,6 +248,10 @@ dummy_func( + DEOPT_IF(frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index); + #endif ++ if (oparg == 0) { ++ int err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ ERROR_IF(err); ++ } + } + + op(_MONITOR_RESUME, (--)) { +@@ -256,7 +268,8 @@ dummy_func( + _LOAD_BYTECODE + + _MAYBE_INSTRUMENT + + _CHECK_PERIODIC_IF_NOT_YIELD_FROM + +- _MONITOR_RESUME; ++ _MONITOR_RESUME + ++ _AIYTHON_FUNCTION_ENTRY; + + pseudo(LOAD_CLOSURE, (-- unused)) = { + LOAD_FAST, +@@ -352,6 +365,12 @@ dummy_func( + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value) + ); ++ PyObject *name = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); ++ int err = aiython_before_store(frame, name, PyStackRef_AsPyObjectBorrow(value)); ++ if (err < 0) { ++ PyStackRef_CLOSE(value); ++ ERROR_IF(true); ++ } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + DEAD(value); +@@ -369,6 +388,12 @@ dummy_func( + ); + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; ++ PyObject *name = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg1); ++ int err = aiython_before_store(frame, name, PyStackRef_AsPyObjectBorrow(value1)); ++ if (err < 0) { ++ PyStackRef_CLOSE(value1); ++ ERROR_IF(true); ++ } + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + DEAD(value1); +@@ -387,10 +412,23 @@ dummy_func( + ); + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; ++ PyObject *name1 = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg1); ++ PyObject *name2 = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg2); ++ int err = aiython_before_store(frame, name1, PyStackRef_AsPyObjectBorrow(value1)); ++ if (err < 0) { ++ PyStackRef_CLOSE(value1); ++ PyStackRef_CLOSE(value2); ++ ERROR_IF(true); ++ } + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + DEAD(value1); + PyStackRef_XCLOSE(tmp); ++ err = aiython_before_store(frame, name2, PyStackRef_AsPyObjectBorrow(value2)); ++ if (err < 0) { ++ PyStackRef_CLOSE(value2); ++ ERROR_IF(true); ++ } + tmp = GETLOCAL(oparg2); + GETLOCAL(oparg2) = value2; + DEAD(value2); +@@ -761,6 +799,8 @@ dummy_func( + // specializations, but there is no output. + // At the end we just skip over the STORE_FAST. + op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { ++ int store_hook_enabled = aiython_store_hook_enabled(); ++ DEOPT_IF(store_hook_enabled); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); +@@ -879,7 +919,13 @@ dummy_func( + err = 1; + } + else { +- err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); ++ err = aiython_before_mutation(frame, "item", ++ PyStackRef_AsPyObjectBorrow(container), slice, ++ PyStackRef_AsPyObjectBorrow(v)); ++ if (err == 0) { ++ err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, ++ PyStackRef_AsPyObjectBorrow(v)); ++ } + Py_DECREF(slice); + } + DECREF_INPUTS(); +@@ -1080,15 +1126,27 @@ dummy_func( + + op(_STORE_SUBSCR, (v, container, sub -- )) { + /* container[sub] = v */ +- int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); ++ int err = aiython_before_mutation(frame, "item", ++ PyStackRef_AsPyObjectBorrow(container), ++ PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); ++ if (err == 0) { ++ err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), ++ PyStackRef_AsPyObjectBorrow(sub), ++ PyStackRef_AsPyObjectBorrow(v)); ++ } + DECREF_INPUTS(); + ERROR_IF(err); + } + ++ op(_AIYTHON_DEOPT_MUTATION, (--)) { ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ DEOPT_IF(hook_enabled); ++ } ++ + macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR; + + macro(STORE_SUBSCR_LIST_INT) = +- _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/1 + _STORE_SUBSCR_LIST_INT; ++ _AIYTHON_DEOPT_MUTATION + _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/1 + _STORE_SUBSCR_LIST_INT; + + op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- )) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); +@@ -1120,7 +1178,7 @@ dummy_func( + } + + macro(STORE_SUBSCR_DICT) = +- _GUARD_NOS_DICT + unused/1 + _STORE_SUBSCR_DICT; ++ _AIYTHON_DEOPT_MUTATION + _GUARD_NOS_DICT + unused/1 + _STORE_SUBSCR_DICT; + + op(_STORE_SUBSCR_DICT, (value, dict_st, sub -- )) { + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); +@@ -1201,6 +1259,12 @@ dummy_func( + // is pushed to a different frame, the callers' frame. + inst(RETURN_VALUE, (retval -- res)) { + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); ++ int hook_err = aiython_function_event(frame, "_aiython_on_return", ++ PyStackRef_AsPyObjectBorrow(retval)); ++ if (hook_err < 0) { ++ PyStackRef_CLOSE(retval); ++ ERROR_IF(true); ++ } + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + DEAD(retval); + SAVE_STACK(); +@@ -1516,6 +1580,11 @@ dummy_func( + PyStackRef_CLOSE(v); + ERROR_IF(true); + } ++ err = aiython_before_store(frame, name, PyStackRef_AsPyObjectBorrow(v)); ++ if (err < 0) { ++ PyStackRef_CLOSE(v); ++ ERROR_IF(true); ++ } + if (PyDict_CheckExact(ns)) { + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + } +@@ -1651,8 +1720,13 @@ dummy_func( + + op(_STORE_ATTR, (v, owner --)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); +- int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), ++ int err = aiython_before_mutation(frame, "attr", ++ PyStackRef_AsPyObjectBorrow(owner), name, ++ PyStackRef_AsPyObjectBorrow(v)); ++ if (err == 0) { ++ err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); ++ } + DECREF_INPUTS(); + ERROR_IF(err); + } +@@ -1668,6 +1742,11 @@ dummy_func( + + inst(STORE_GLOBAL, (v --)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); ++ int hook_err = aiython_before_store(frame, name, PyStackRef_AsPyObjectBorrow(v)); ++ if (hook_err < 0) { ++ PyStackRef_CLOSE(v); ++ ERROR_IF(true); ++ } + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); + ERROR_IF(err); +@@ -1919,6 +1998,12 @@ dummy_func( + } + + inst(STORE_DEREF, (v --)) { ++ PyObject *name = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); ++ int err = aiython_before_store(frame, name, PyStackRef_AsPyObjectBorrow(v)); ++ if (err < 0) { ++ PyStackRef_CLOSE(v); ++ ERROR_IF(true); ++ } + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + } +@@ -2577,6 +2662,7 @@ dummy_func( + + macro(STORE_ATTR_INSTANCE_VALUE) = + unused/1 + ++ _AIYTHON_DEOPT_MUTATION + + _GUARD_TYPE_VERSION_AND_LOCK + + _GUARD_DORV_NO_DICT + + _STORE_ATTR_INSTANCE_VALUE; +@@ -2623,6 +2709,7 @@ dummy_func( + + macro(STORE_ATTR_WITH_HINT) = + unused/1 + ++ _AIYTHON_DEOPT_MUTATION + + _GUARD_TYPE_VERSION + + _STORE_ATTR_WITH_HINT; + +@@ -2641,6 +2728,7 @@ dummy_func( + + macro(STORE_ATTR_SLOT) = + unused/1 + ++ _AIYTHON_DEOPT_MUTATION + + _GUARD_TYPE_VERSION + + _STORE_ATTR_SLOT; + +diff --git a/Python/ceval.c b/Python/ceval.c +index 377b464..d459d64 100644 +--- a/Python/ceval.c ++++ b/Python/ceval.c +@@ -1141,6 +1141,121 @@ typedef struct { + _PyStackRef stack[1]; + } _PyEntryFrame; + ++/* Experimental Aiython hook. The Python callback can raise to reject a store. ++ The guard prevents assignments inside that callback from calling it again. */ ++static _Thread_local int aiython_in_hook = 0; ++ ++static int ++aiython_before_store(_PyInterpreterFrame *frame, PyObject *name, PyObject *value) ++{ ++ /* STORE_FAST_MAYBE_NULL may legally write an empty stack reference. */ ++ if (value == NULL || aiython_in_hook) { ++ return 0; ++ } ++ PyObject *callback = NULL; ++ int found = _PySys_GetOptionalAttrString("_aiython_before_store", &callback); ++ if (found <= 0) { ++ return found < 0 ? -1 : 0; ++ } ++ PyFrameObject *borrowed = _PyFrame_GetFrameObject(frame); ++ if (borrowed == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *frame_obj = Py_NewRef(borrowed); ++ aiython_in_hook = 1; ++ PyObject *result = PyObject_CallFunctionObjArgs(callback, frame_obj, name, value, NULL); ++ aiython_in_hook = 0; ++ Py_DECREF(frame_obj); ++ Py_DECREF(callback); ++ if (result == NULL) { ++ return -1; ++ } ++ Py_DECREF(result); ++ return 0; ++} ++ ++static int ++aiython_mutation_hook_enabled(void) ++{ ++ return PySys_GetObject("_aiython_before_mutation") != NULL; ++} ++ ++static int ++aiython_store_hook_enabled(void) ++{ ++ return PySys_GetObject("_aiython_before_store") != NULL; ++} ++ ++static int ++aiython_before_mutation(_PyInterpreterFrame *frame, const char *kind, ++ PyObject *owner, PyObject *key, PyObject *value) ++{ ++ if (aiython_in_hook) { ++ return 0; ++ } ++ PyObject *callback = NULL; ++ int found = _PySys_GetOptionalAttrString("_aiython_before_mutation", &callback); ++ if (found <= 0) { ++ return found < 0 ? -1 : 0; ++ } ++ PyFrameObject *borrowed = _PyFrame_GetFrameObject(frame); ++ if (borrowed == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *kind_obj = PyUnicode_FromString(kind); ++ if (kind_obj == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *frame_obj = Py_NewRef(borrowed); ++ aiython_in_hook = 1; ++ PyObject *result = PyObject_CallFunctionObjArgs( ++ callback, frame_obj, kind_obj, owner, key, value, NULL); ++ aiython_in_hook = 0; ++ Py_DECREF(frame_obj); ++ Py_DECREF(kind_obj); ++ Py_DECREF(callback); ++ if (result == NULL) { ++ return -1; ++ } ++ Py_DECREF(result); ++ return 0; ++} ++ ++static int ++aiython_function_event(_PyInterpreterFrame *frame, const char *attribute, ++ PyObject *value) ++{ ++ if (aiython_in_hook) { ++ return 0; ++ } ++ PyObject *callback = NULL; ++ int found = _PySys_GetOptionalAttrString(attribute, &callback); ++ if (found <= 0) { ++ return found < 0 ? -1 : 0; ++ } ++ PyFrameObject *borrowed = _PyFrame_GetFrameObject(frame); ++ if (borrowed == NULL) { ++ Py_DECREF(callback); ++ return -1; ++ } ++ PyObject *frame_obj = Py_NewRef(borrowed); ++ aiython_in_hook = 1; ++ PyObject *result = value == NULL ++ ? PyObject_CallOneArg(callback, frame_obj) ++ : PyObject_CallFunctionObjArgs(callback, frame_obj, value, NULL); ++ aiython_in_hook = 0; ++ Py_DECREF(frame_obj); ++ Py_DECREF(callback); ++ if (result == NULL) { ++ return -1; ++ } ++ Py_DECREF(result); ++ return 0; ++} ++ + PyObject* _Py_HOT_FUNCTION DONT_SLP_VECTORIZE + _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) + { +diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h +index 5cba25d..b9b7c86 100644 +--- a/Python/generated_cases.c.h ++++ b/Python/generated_cases.c.h +@@ -366,6 +366,14 @@ + { + right = value; + left = nos; ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int store_hook_enabled = aiython_store_hook_enabled(); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (store_hook_enabled) { ++ UPDATE_MISS_STATS(BINARY_OP); ++ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); ++ JUMP_TO_PREDICTED(BINARY_OP); ++ } + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); +@@ -7555,6 +7563,17 @@ + next_instr = frame->instr_ptr; + } + } ++ // _AIYTHON_FUNCTION_ENTRY ++ { ++ if (oparg == 0) { ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (err) { ++ JUMP_TO_LABEL(error); ++ } ++ } ++ } + DISPATCH(); + } + +@@ -7587,6 +7606,18 @@ + { + retval = val; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int hook_err = aiython_function_event(frame, "_aiython_on_return", ++ PyStackRef_AsPyObjectBorrow(retval)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (hook_err < 0) { ++ stack_pointer += -1; ++ assert(WITHIN_STACK_BOUNDS()); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ PyStackRef_CLOSE(retval); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ JUMP_TO_LABEL(error); ++ } + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); +@@ -10517,6 +10548,17 @@ + } + } + } ++ // _AIYTHON_FUNCTION_ENTRY ++ { ++ if (oparg == 0) { ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (err) { ++ JUMP_TO_LABEL(error); ++ } ++ } ++ } + DISPATCH(); + } + +@@ -10555,6 +10597,14 @@ + JUMP_TO_PREDICTED(RESUME); + } + #endif ++ if (oparg == 0) { ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int err = aiython_function_event(frame, "_aiython_on_call", NULL); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (err) { ++ JUMP_TO_LABEL(error); ++ } ++ } + DISPATCH(); + } + +@@ -10609,6 +10659,18 @@ + _PyStackRef res; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int hook_err = aiython_function_event(frame, "_aiython_on_return", ++ PyStackRef_AsPyObjectBorrow(retval)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (hook_err < 0) { ++ stack_pointer += -1; ++ assert(WITHIN_STACK_BOUNDS()); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ PyStackRef_CLOSE(retval); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ JUMP_TO_LABEL(error); ++ } + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); +@@ -10963,8 +11025,17 @@ + v = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); +- int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), ++ int err = aiython_before_mutation(frame, "attr", ++ PyStackRef_AsPyObjectBorrow(owner), name, ++ PyStackRef_AsPyObjectBorrow(v)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (err == 0) { ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ } ++ _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = PyStackRef_NULL; + stack_pointer[-1] = owner; +@@ -10997,6 +11068,17 @@ + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ ++ // _AIYTHON_DEOPT_MUTATION ++ { ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (hook_enabled) { ++ UPDATE_MISS_STATS(STORE_ATTR); ++ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); ++ JUMP_TO_PREDICTED(STORE_ATTR); ++ } ++ } + // _GUARD_TYPE_VERSION_AND_LOCK + { + owner = stack_pointer[-1]; +@@ -11073,6 +11155,17 @@ + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ ++ // _AIYTHON_DEOPT_MUTATION ++ { ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (hook_enabled) { ++ UPDATE_MISS_STATS(STORE_ATTR); ++ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); ++ JUMP_TO_PREDICTED(STORE_ATTR); ++ } ++ } + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; +@@ -11124,6 +11217,17 @@ + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ ++ // _AIYTHON_DEOPT_MUTATION ++ { ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (hook_enabled) { ++ UPDATE_MISS_STATS(STORE_ATTR); ++ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); ++ JUMP_TO_PREDICTED(STORE_ATTR); ++ } ++ } + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; +@@ -11218,6 +11322,18 @@ + INSTRUCTION_STATS(STORE_DEREF); + _PyStackRef v; + v = stack_pointer[-1]; ++ PyObject *name = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int err = aiython_before_store(frame, name, PyStackRef_AsPyObjectBorrow(v)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (err < 0) { ++ stack_pointer += -1; ++ assert(WITHIN_STACK_BOUNDS()); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ PyStackRef_CLOSE(v); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ JUMP_TO_LABEL(error); ++ } + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); +@@ -11241,6 +11357,18 @@ + ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || + PyStackRef_IsHeapSafe(value) + ); ++ PyObject *name = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int err = aiython_before_store(frame, name, PyStackRef_AsPyObjectBorrow(value)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (err < 0) { ++ stack_pointer += -1; ++ assert(WITHIN_STACK_BOUNDS()); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ PyStackRef_CLOSE(value); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ JUMP_TO_LABEL(error); ++ } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + stack_pointer += -1; +@@ -11268,6 +11396,18 @@ + ); + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; ++ PyObject *name = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg1); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int err = aiython_before_store(frame, name, PyStackRef_AsPyObjectBorrow(value1)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (err < 0) { ++ stack_pointer += -1; ++ assert(WITHIN_STACK_BOUNDS()); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ PyStackRef_CLOSE(value1); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ JUMP_TO_LABEL(error); ++ } + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); +@@ -11300,13 +11440,40 @@ + ); + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; ++ PyObject *name1 = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg1); ++ PyObject *name2 = GETITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg2); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int err = aiython_before_store(frame, name1, PyStackRef_AsPyObjectBorrow(value1)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (err < 0) { ++ stack_pointer += -1; ++ assert(WITHIN_STACK_BOUNDS()); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ PyStackRef_CLOSE(value1); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ stack_pointer += -1; ++ assert(WITHIN_STACK_BOUNDS()); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ PyStackRef_CLOSE(value2); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ JUMP_TO_LABEL(error); ++ } + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); ++ err = aiython_before_store(frame, name2, PyStackRef_AsPyObjectBorrow(value2)); + stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (err < 0) { ++ stack_pointer += -1; ++ assert(WITHIN_STACK_BOUNDS()); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ PyStackRef_CLOSE(value2); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ JUMP_TO_LABEL(error); ++ } + tmp = GETLOCAL(oparg2); + GETLOCAL(oparg2) = value2; + stack_pointer += -1; +@@ -11329,6 +11496,17 @@ + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); ++ int hook_err = aiython_before_store(frame, name, PyStackRef_AsPyObjectBorrow(v)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (hook_err < 0) { ++ stack_pointer += -1; ++ assert(WITHIN_STACK_BOUNDS()); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ PyStackRef_CLOSE(v); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ JUMP_TO_LABEL(error); ++ } ++ _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; +@@ -11367,6 +11545,17 @@ + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ err = aiython_before_store(frame, name, PyStackRef_AsPyObjectBorrow(v)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (err < 0) { ++ stack_pointer += -1; ++ assert(WITHIN_STACK_BOUNDS()); ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ PyStackRef_CLOSE(v); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ JUMP_TO_LABEL(error); ++ } + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); +@@ -11424,7 +11613,17 @@ + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); +- err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); ++ err = aiython_before_mutation(frame, "item", ++ PyStackRef_AsPyObjectBorrow(container), slice, ++ PyStackRef_AsPyObjectBorrow(v)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (err == 0) { ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, ++ PyStackRef_AsPyObjectBorrow(v)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ } ++ _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; +@@ -11484,7 +11683,18 @@ + { + v = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); +- int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); ++ int err = aiython_before_mutation(frame, "item", ++ PyStackRef_AsPyObjectBorrow(container), ++ PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (err == 0) { ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), ++ PyStackRef_AsPyObjectBorrow(sub), ++ PyStackRef_AsPyObjectBorrow(v)); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ } ++ _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; +@@ -11522,6 +11732,17 @@ + _PyStackRef value; + _PyStackRef dict_st; + _PyStackRef sub; ++ // _AIYTHON_DEOPT_MUTATION ++ { ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (hook_enabled) { ++ UPDATE_MISS_STATS(STORE_SUBSCR); ++ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); ++ JUMP_TO_PREDICTED(STORE_SUBSCR); ++ } ++ } + // _GUARD_NOS_DICT + { + nos = stack_pointer[-2]; +@@ -11573,6 +11794,17 @@ + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; ++ // _AIYTHON_DEOPT_MUTATION ++ { ++ _PyFrame_SetStackPointer(frame, stack_pointer); ++ int hook_enabled = aiython_mutation_hook_enabled(); ++ stack_pointer = _PyFrame_GetStackPointer(frame); ++ if (hook_enabled) { ++ UPDATE_MISS_STATS(STORE_SUBSCR); ++ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); ++ JUMP_TO_PREDICTED(STORE_SUBSCR); ++ } ++ } + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; diff --git a/native/tests/store_hook.py b/native/tests/store_hook.py new file mode 100644 index 0000000..374d12a --- /dev/null +++ b/native/tests/store_hook.py @@ -0,0 +1,233 @@ +"""Smoke test for the experimental CPython 3.13–3.14 VM hooks. + +Run with an interpreter built from a patch in native/patches. +""" + +import dis +import sys + + +events = [] +mutations = [] +function_events = [] +global_value = 0 +mapping = {} +sequence = [0, 0, 0] + + +class Box: + pass + + +class SlottedBox: + __slots__ = ("value",) + + +box = Box() +slotted_box = SlottedBox() + + +def before_store(frame, name, value): + internal = "the recursion guard must hide this" + if frame.f_code.co_filename in {__file__, ""}: + events.append((frame.f_code.co_name, name, value)) + if name == "blocked": + raise TypeError("blocked by the store hook") + + +def before_mutation(frame, kind, owner, key, value): + if frame.f_code.co_filename == __file__: + mutations.append((kind, owner, key, value)) + # An assignment made by a callback must not recursively invoke the hook. + owner_guard = Box() + owner_guard.internal = 1 + if key == "blocked": + raise TypeError("blocked by the mutation hook") + + +def on_call(frame): + if frame.f_code.co_name == "rejected_entry": + raise TypeError("entry rejected") + if frame.f_code.co_name == "checked_function": + function_events.append(("call", frame.f_locals["value"])) + + +def on_return(frame, value): + if frame.f_code.co_name == "rejected_return": + raise TypeError("return rejected") + if frame.f_code.co_name == "checked_function": + function_events.append(("return", value)) + + +def local_store(): + first = 1 + second = 2 + try: + blocked = 3 + except TypeError: + return first, second + raise AssertionError("store hook did not reject the assignment") + + +def global_store(): + global global_value + global_value = 4 + + +def closure_store(): + value = 5 + + def inner(): + nonlocal value + value = 6 + + inner() + return value + + +def specialized_mutations(value): + box.value = value + slotted_box.value = value + mapping["value"] = value + sequence[0] = value + + +def rejected_mutations(): + try: + box.blocked = 1 + except TypeError: + assert not hasattr(box, "blocked") + else: + raise AssertionError("attribute store was not rejected") + try: + mapping["blocked"] = 2 + except TypeError: + assert "blocked" not in mapping + else: + raise AssertionError("item store was not rejected") + + +def slice_mutation(): + sequence[1:3] = [7, 8] + + +def checked_function(value): + return value + 1 + + +def rejected_entry(): + raise AssertionError("entry hook should prevent execution") + + +def rejected_return(): + return 9 + + +def tracer(frame, event, argument): + return tracer + + +def unicode_inplace(): + text = "a" + text += "b" + return text + + +def main(): + for _ in range(1000): + assert unicode_inplace() == "ab" + sys._aiython_before_store = before_store + try: + assert unicode_inplace() == "ab" + assert local_store() == (1, 2) + global_store() + assert global_value == 4 + assert closure_store() == 6 + namespace = {} + exec("module_value = 7\nclass Example:\n class_value = 8", namespace) + assert namespace["module_value"] == 7 + assert namespace["Example"].class_value == 8 + try: + exec("blocked = 9", namespace) + except TypeError: + assert "blocked" not in namespace + else: + raise AssertionError("module store did not reject the assignment") + assert list(dis.get_instructions(local_store)) + for value in range(1000): + specialized_mutations(value) + names = {instruction.opname for instruction in + dis.get_instructions(specialized_mutations, adaptive=True)} + assert "STORE_ATTR_INSTANCE_VALUE" in names, names + assert "STORE_ATTR_SLOT" in names, names + assert "STORE_SUBSCR_DICT" in names, names + assert "STORE_SUBSCR_LIST_INT" in names, names + sys._aiython_before_mutation = before_mutation + specialized_mutations(1001) + rejected_mutations() + slice_mutation() + assert box.value == slotted_box.value == mapping["value"] == sequence[0] == 1001 + assert sequence[1:3] == [7, 8] + sys._aiython_on_call = on_call + sys._aiython_on_return = on_return + assert checked_function(3) == 4 + sys.settrace(tracer) + try: + assert checked_function(10) == 11 + finally: + sys.settrace(None) + try: + rejected_entry() + except TypeError as error: + assert str(error) == "entry rejected" + else: + raise AssertionError("entry hook did not reject the call") + try: + rejected_return() + except TypeError as error: + assert str(error) == "return rejected" + else: + raise AssertionError("return hook did not reject the value") + finally: + del sys._aiython_before_store + if hasattr(sys, "_aiython_before_mutation"): + del sys._aiython_before_mutation + if hasattr(sys, "_aiython_on_call"): + del sys._aiython_on_call + if hasattr(sys, "_aiython_on_return"): + del sys._aiython_on_return + + expected = { + ("local_store", "first", 1), + ("local_store", "second", 2), + ("local_store", "blocked", 3), + ("global_store", "global_value", 4), + ("inner", "value", 6), + ("", "module_value", 7), + ("Example", "class_value", 8), + ("", "blocked", 9), + ("unicode_inplace", "text", "ab"), + } + missing = [item for item in expected if item not in events] + assert not missing, missing + assert not any(name == "internal" for _, name, _ in events) + assert any(kind == "attr" and owner is box and key == "value" and value == 1001 + for kind, owner, key, value in mutations) + assert any(kind == "attr" and owner is slotted_box and key == "value" and value == 1001 + for kind, owner, key, value in mutations) + assert any(kind == "item" and owner is mapping and key == "value" and value == 1001 + for kind, owner, key, value in mutations) + assert any(kind == "item" and owner is sequence and key == 0 and value == 1001 + for kind, owner, key, value in mutations) + assert any(kind == "item" and owner is sequence and isinstance(key, slice) + and key == slice(1, 3) for kind, owner, key, value in mutations) + assert not any(key == "internal" for _, _, key, _ in mutations) + assert function_events == [ + ("call", 3), ("return", 4), ("call", 10), ("return", 11) + ], function_events + version = ".".join(map(str, sys.version_info[:3])) + print(f"CPython {version} VM hook smoke passed") + + +if __name__ == "__main__": + main() diff --git a/native/tests/typed_bridge.py b/native/tests/typed_bridge.py new file mode 100644 index 0000000..91c76f1 --- /dev/null +++ b/native/tests/typed_bridge.py @@ -0,0 +1,140 @@ +"""Exercise Aiython's experimental type bridge on patched CPython.""" +import sys + +from aiython.native_bridge import NativeTypeBridge +from aiython.type_constraints import TypeViolation + + +SOURCE = ''' +from typing import Final, Generator + +answer: int = 1 +fixed: Final[int] = 1 + +class Box: + value: int + + def __init__(self, value: int): + self.value = value + +def update(value: int) -> int: + local: int = value + return local + +def change_global(value: int): + global answer + answer = value + +def change_final(): + global fixed + fixed = 2 + +def change_box(box: Box, value): + box.value = value + +def local_error(): + local: int = "invalid" + +def parameter_reassignment(value: int): + value = "invalid" + +def local_final(): + value: Final[int] = 1 + value = 2 + +def return_error() -> int: + return "invalid" + +def native_feature(value): + match value: + case [first, *rest]: + return first, rest + +def identity(function): + return function + +@identity +def decorated(value: int) -> int: + return value + +if True: + def conditional(value: int) -> int: + return value + +def variadic(*values: int, **named: int): + return values, named + +def make_setter(): + current: int = 1 + def set_current(value): + nonlocal current + current = value + return current + return set_current + +def make_final_setter(): + current: Final[int] = 1 + def set_current(): + nonlocal current + current = 2 + return set_current + +def generator() -> Generator[int, None, None]: + yield 1 + +def generator_error() -> Generator[int, None, int]: + yield 1 + return "invalid" +''' + + +def rejected(action): + try: + action() + except TypeViolation: + return + raise AssertionError('A type violation was not raised') + + +def main(): + bridge = NativeTypeBridge() + code = bridge.compile_source(SOURCE, '') + native = compile(SOURCE, '', 'exec', dont_inherit=True) + assert (code.co_code, code.co_consts, code.co_linetable, code.co_flags) == ( + native.co_code, native.co_consts, native.co_linetable, native.co_flags) + namespace = {'__name__': '__main__'} + with bridge.installed(): + exec(code, namespace) + assert namespace['update'](3) == 3 + assert namespace['native_feature']([1, 2, 3]) == (1, [2, 3]) + rejected(lambda: namespace['update']('invalid')) + rejected(lambda: namespace['decorated']('invalid')) + rejected(lambda: namespace['conditional']('invalid')) + assert namespace['variadic'](1, extra=2) == ((1,), {'extra': 2}) + rejected(lambda: namespace['variadic']('invalid')) + rejected(lambda: namespace['variadic'](extra='invalid')) + setter = namespace['make_setter']() + assert setter(2) == 2 + rejected(lambda: setter('invalid')) + rejected(namespace['make_final_setter']()) + assert list(namespace['generator']()) == [1] + rejected(lambda: list(namespace['generator_error']())) + rejected(namespace['local_error']) + rejected(namespace['local_final']) + rejected(lambda: namespace['parameter_reassignment'](1)) + rejected(namespace['return_error']) + namespace['change_global'](4) + assert namespace['answer'] == 4 + rejected(lambda: namespace['change_global']('invalid')) + assert namespace['answer'] == 4 + rejected(namespace['change_final']) + assert namespace['fixed'] == 1 + box = namespace['Box'](5) + rejected(lambda: namespace['change_box'](box, 'invalid')) + assert box.value == 5 + assert all(not hasattr(sys, name) for name in NativeTypeBridge._HOOKS) + print('Native type bridge passed') + + +if __name__ == '__main__': + main() diff --git a/scripts/build_cpython_baseline.py b/scripts/build_cpython_baseline.py new file mode 100644 index 0000000..a556381 --- /dev/null +++ b/scripts/build_cpython_baseline.py @@ -0,0 +1,91 @@ +"""Build an exact CPython baseline before applying Aiython VM changes. + +This produces stock CPython; it does not enable Aiython's type or recovery +features. Example: python scripts/build_cpython_baseline.py 3.14.4 +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import tomllib + + +ROOT = Path(__file__).resolve().parents[1] +LOCK = tomllib.loads((ROOT / "native/cpython.lock.toml").read_text())["versions"] + + +def run(*command: str, cwd: Path | None = None) -> None: + subprocess.run(command, cwd=cwd, check=True) + + +def available_stdlib(python: Path) -> set[str]: + probe = ("import importlib.util, json, sys\n" + "def available(name):\n" + " try: return importlib.util.find_spec(name) is not None\n" + " except Exception: return False\n" + "print(json.dumps(sorted(name for name in sys.stdlib_module_names " + "if available(name))))\n") + output = subprocess.check_output([str(python), "-I", "-c", probe], text=True) + return set(json.loads(output)) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("version", choices=LOCK) + parser.add_argument("--build-root", type=Path, + default=Path(tempfile.gettempdir()) / "aiython-cpython-baselines") + parser.add_argument("--source-dir", type=Path, help="reuse an existing CPython checkout") + parser.add_argument("--reconfigure", action="store_true", + help="rerun configure after changing build dependencies or flags") + parser.add_argument("--jobs", type=int, default=max(1, min(os.cpu_count() or 1, 4))) + args = parser.parse_args() + if args.jobs < 1: + parser.error("--jobs must be positive") + + source = ((args.source_dir or args.build_root / f"cpython-{args.version}").resolve()) + if not source.exists(): + source.parent.mkdir(parents=True, exist_ok=True) + run("git", "clone", "--depth", "1", "--filter=blob:none", "--branch", + f"v{args.version}", "https://github.com/python/cpython.git", str(source)) + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=source, text=True).strip() + if commit != LOCK[args.version]: + raise SystemExit(f"CPython {args.version}: expected {LOCK[args.version]}, got {commit}") + if subprocess.check_output(["git", "status", "--porcelain", "--untracked-files=no"], + cwd=source): + raise SystemExit('CPython source has tracked changes; baseline must be unmodified') + if args.reconfigure or not (source / "Makefile").exists(): + run("./configure", f"--prefix={source / 'install'}", cwd=source) + run("make", "-s", f"-j{args.jobs}", cwd=source) + actual = subprocess.check_output([str(source / "python"), "-c", + "import json, sys, sysconfig; " + "print('.'.join(map(str, sys.version_info[:3]))); " + "print(sysconfig.get_config_var('CONFIG_ARGS')); " + "print(json.dumps(sorted(k for k, v in " + "sysconfig.get_config_vars().items() if " + "k.startswith('MODULE_') and k.endswith('_STATE') " + "and v == 'missing')))"], + cwd=source, text=True).strip() + version, config_args, missing_modules_json = actual.splitlines() + if version != args.version: + raise SystemExit(f"Built interpreter reported {version}, expected {args.version}") + if '--disable-test-modules' in config_args or '--without-ensurepip' in config_args: + raise SystemExit('Build omitted CPython modules or ensurepip; reconfigure the checkout') + missing_modules = json.loads(missing_modules_json) + if missing_modules: + raise SystemExit(f"Built interpreter has missing optional extensions: {', '.join(missing_modules)}") + if sys.version_info[:3] != tuple(map(int, args.version.split('.'))): + raise SystemExit('Run this script with the same CPython patch version as the build') + missing = available_stdlib(Path(sys.executable)) - available_stdlib(source / "python") + if missing: + raise SystemExit(f"Built interpreter lacks reference stdlib modules: {', '.join(sorted(missing))}") + print(f"Verified CPython {args.version} at {source / 'python'} ({commit})") + + +if __name__ == "__main__": + main() diff --git a/scripts/build_cpython_hooks.py b/scripts/build_cpython_hooks.py new file mode 100644 index 0000000..377d5c1 --- /dev/null +++ b/scripts/build_cpython_hooks.py @@ -0,0 +1,61 @@ +"""Build and test an exact CPython release with Aiython experimental hooks. + +The output is a patched CPython interpreter, not a complete Aiython runtime. +Run with the same Python patch version as the requested source release. +""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import tomllib + + +ROOT = Path(__file__).resolve().parents[1] +VERSIONS = tomllib.loads((ROOT / "native/cpython.lock.toml").read_text())["versions"] +CPYTHON_TESTS = ( + "test_compile", "test_dis", "test_scope", "test_sys", "test_frame", + "test_exceptions", "test_generators", "test_coroutines", +) + + +def run(*command: str, cwd: Path = ROOT) -> None: + subprocess.run(command, cwd=cwd, check=True) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("version", choices=VERSIONS) + parser.add_argument("--source-dir", type=Path, + help="clean CPython checkout; defaults to a temporary build directory") + parser.add_argument("--jobs", type=int, default=max(1, min(os.cpu_count() or 1, 4))) + args = parser.parse_args() + if args.jobs < 1: + parser.error("--jobs must be positive") + source = (args.source_dir or Path(tempfile.gettempdir()) / f"aiython-cpython-hooks-{args.version}").resolve() + patch = ROOT / "native/patches" / f"cpython-{args.version}-hooks.patch" + if not patch.is_file(): + raise SystemExit(f"Missing pinned CPython patch: {patch}") + + run(sys.executable, str(ROOT / "scripts/build_cpython_baseline.py"), args.version, + "--source-dir", str(source), "--jobs", str(args.jobs)) + run("git", "apply", "--check", str(patch), cwd=source) + run("git", "apply", str(patch), cwd=source) + if args.version.startswith("3.12."): + run("make", "regen-cases", cwd=source) + elif not args.version.startswith("3.11."): + run("make", "regen-generated-cases", cwd=source) + run("make", "-s", f"-j{args.jobs}", cwd=source) + run(str(source / "python"), "-I", str(ROOT / "native/tests/store_hook.py")) + run("uv", "run", "--locked", "--python", str(source / "python"), + "python", "native/tests/typed_bridge.py") + run(str(source / "python"), "-I", "-m", "test", "-q", *CPYTHON_TESTS, cwd=source) + print(f"Experimental CPython {args.version} hooks verified at {source / 'python'}") + + +if __name__ == "__main__": + main() diff --git a/src/aiython/cli.py b/src/aiython/cli.py index 6bd8590..72293d9 100644 --- a/src/aiython/cli.py +++ b/src/aiython/cli.py @@ -315,7 +315,18 @@ def finish(): 'Type "help", "copyright", "credits" or "license" for more information.') AiythonConsole(runtime, module.__dict__).interact(banner=banner, exitmsg='') else: - exec(code, module.__dict__) + try: + exec(code, module.__dict__) + except Exception as exc: + if not restore_state and not config.profiles and not isinstance(exc, AiythonError): + trace = exc.__traceback__ + while trace is not None and trace.tb_frame.f_code.co_filename != code.co_filename: + trace = trace.tb_next + if trace is not None: + exc.__traceback__ = trace + sys.excepthook(type(exc), exc, trace) + raise SystemExit(1) from None + raise return module.__dict__ finally: if restore_state: @@ -600,7 +611,7 @@ def main(argv=None): str(path if module_spec else path.resolve())) unit = parse(read_source(path.resolve()) if source is None else source, filename) runtime = Runtime(config) - runtime.prepare(unit, entry=True) + runtime.bridge.prepare_unit(unit, entry=True, recovery_metadata=True) print(json.dumps({"config": describe(config), "blocks": [ {"statement": b.statement, "span": vars(b.span), "expression": b.expression, "output_type": b.output_type, "plan": "requires runtime intent resolution", "cache": "unknown", "cost": "unknown", diff --git a/src/aiython/native_bridge.py b/src/aiython/native_bridge.py new file mode 100644 index 0000000..3385b2f --- /dev/null +++ b/src/aiython/native_bridge.py @@ -0,0 +1,246 @@ +"""Execution routing plus optional VM type hooks for original CPython code. + +The normal runtime uses this bridge to select native code or the existing +boundary compiler for contracts, typed yields and recovery. The VM hook API +needs the optional CPython patches in ``native/patches``. +""" +from __future__ import annotations + +import ast +from contextlib import contextmanager +from dataclasses import dataclass +import inspect +import linecache +import sys + +from .type_constraints import TypeViolation +from .typed_runtime import TypedTransformer, TypeRuntime + + +@dataclass(frozen=True) +class _FrameTypes: + declarations: dict[str, str] + parameters: dict[str, tuple[str, str]] + returns: str | None + global_names: frozenset[str] + outer: _FrameTypes | None = None + + +def _parameter_types(arguments): + result = {} + for argument in (*arguments.posonlyargs, *arguments.args, *arguments.kwonlyargs): + if argument.annotation is not None: + result[argument.arg] = (ast.unparse(argument.annotation), 'value') + for argument, mode in ((arguments.vararg, 'args'), (arguments.kwarg, 'kwargs')): + if argument is not None and argument.annotation is not None: + result[argument.arg] = (ast.unparse(argument.annotation), mode) + return result + + +def _global_names(body): + names = set() + + def visit(node): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)): + return + if isinstance(node, ast.Global): + names.update(node.names) + for child in ast.iter_child_nodes(node): + visit(child) + + for statement in body: + visit(statement) + return frozenset(names) + + +class NativeTypeBridge: + """Choose CPython bytecode or Aiython boundaries and validate VM hooks. + + ``prepare_unit`` is the normal runtime entry point. It compiles plain + source directly with CPython, while annotated source, typed yields and AI + recovery use the existing Aiython boundary compiler. ``compile_source`` + and ``installed`` expose the experimental patched-VM hooks separately. + """ + + _HOOKS = ('_aiython_before_store', '_aiython_before_mutation', + '_aiython_on_call', '_aiython_on_return') + + def __init__(self, types: TypeRuntime | None = None, *, manager=None): + self.types = types or TypeRuntime() + self.manager = manager + self.frames: dict[tuple[str, str, int], _FrameTypes] = {} + self.module_types: dict[str, _FrameTypes] = {} + + def native_source(self, unit, *, flags=0, display_last_expr=False, + recovery_metadata=False): + """Keep runtime checks when source can introduce typed objects.""" + manager = self.manager + if manager is None: + raise RuntimeError('Native source selection requires a Runtime') + if (flags or display_last_expr or recovery_metadata or manager.config.profiles + or unit.blocks or unit.directives.annotations + or self.types.classes or self.types._module_scopes): + return False + for node in ast.walk(unit.tree): + if isinstance(node, (ast.Import, ast.ImportFrom)): + return False + if isinstance(node, ast.Name) and node.id in ('__import__', '__builtins__'): + return False + if isinstance(node, ast.Call) and ( + isinstance(node.func, ast.Name) and node.func.id in + ('__import__', 'exec', 'eval', 'compile', 'getattr', 'vars', 'globals', 'locals') + or isinstance(node.func, ast.Attribute) and node.func.attr in + ('import_module', 'run_module', 'run_path')): + return False + if isinstance(node, ast.AnnAssign): + return False + if isinstance(node, ast.arg) and node.annotation is not None: + return False + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.returns is not None: + return False + if getattr(node, 'type_params', ()): + return False + if getattr(ast, 'TypeAlias', None) is not None and isinstance(node, ast.TypeAlias): + return False + return True + + def prepare_unit(self, unit, *, entry=False, flags=0, + display_last_expr=False, recovery_metadata=False): + """Return (code, native); recovery and typed yields remain executable.""" + manager = self.manager + if manager is None: + raise RuntimeError('Source preparation requires a Runtime') + if not self.native_source(unit, flags=flags, + display_last_expr=display_last_expr, + recovery_metadata=recovery_metadata): + return manager.prepare(unit, entry=entry, flags=flags, + display_last_expr=display_last_expr, + recovery_metadata=recovery_metadata), False + manager.register(unit) + linecache.cache[unit.filename] = ( + len(unit.source), None, unit.source.splitlines(True), unit.filename) + return compile(unit.source, unit.filename, 'exec', dont_inherit=True), True + + def compile_source(self, source: str, filename: str): + tree = ast.parse(source, filename) + module = _FrameTypes(TypedTransformer.declarations_in(tree.body), {}, None, + frozenset()) + self.module_types[filename] = module + self.frames[(filename, '', 1)] = module + + def collect(node, prefix, outer): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + name = prefix + node.name + info = _FrameTypes( + TypedTransformer.declarations_in(node.body), + _parameter_types(node.args), + ast.unparse(node.returns) if node.returns is not None else None, + _global_names(node.body), outer) + for line in {node.lineno, *(item.lineno for item in node.decorator_list)}: + self.frames[(filename, name, line)] = info + for statement in node.body: + collect(statement, name + '..', info) + elif isinstance(node, ast.ClassDef): + name = prefix + node.name + info = _FrameTypes(TypedTransformer.declarations_in(node.body), + {}, None, _global_names(node.body), outer) + for line in {node.lineno, *(item.lineno for item in node.decorator_list)}: + self.frames[(filename, name, line)] = info + for statement in node.body: + # Methods cannot capture names from a class namespace. + collect(statement, name + '.', outer) + else: + for child in ast.iter_child_nodes(node): + collect(child, prefix, outer) + + for statement in tree.body: + collect(statement, '', module) + # Compile the exact source. Compiling an AST can produce a different + # code object, even when the tree is not explicitly transformed. + return compile(source, filename, 'exec', dont_inherit=True) + + def _frame_types(self, frame): + code = frame.f_code + return self.frames.get((code.co_filename, code.co_qualname, + code.co_firstlineno)) + + def before_store(self, frame, name, value): + info = self._frame_types(frame) + if info is None: + return + source = info.declarations.get(name) + if source is None and name in info.parameters: + parameter_source, mode = info.parameters[name] + if mode == 'value': + source = parameter_source + if source is None and name in frame.f_code.co_freevars: + outer = info.outer + while outer is not None: + source = outer.declarations.get(name) + if source is None and name in outer.parameters: + source, mode = outer.parameters[name] + if mode != 'value': + source = None + if source is not None: + break + outer = outer.outer + if name in info.global_names: + source = self.module_types[frame.f_code.co_filename].declarations.get(name) + if source: + contract = self.types.contract(source, self.types.namespace(frame)) + if contract.marker == 'Final': + values = frame.f_globals if name in info.global_names else frame.f_locals + if name in values: + raise TypeViolation(f'{name}: Final binding cannot be reassigned') + contract.validate(value, name) + + def before_mutation(self, frame, kind, owner, key, value): + if kind == 'attr' and self._frame_types(frame) is not None: + self.types.validate_attribute(frame, owner, key, value) + + def on_call(self, frame): + info = self._frame_types(frame) + if info is None or not info.parameters: + return + self.types._initialize(frame, info.declarations, info.parameters, info.returns) + + def on_return(self, frame, value): + info = self._frame_types(frame) + if info is not None and info.returns: + contract = self.types.contract(info.returns, self.types.namespace(frame)) + if (frame.f_code.co_flags & (inspect.CO_GENERATOR | inspect.CO_ASYNC_GENERATOR) + and contract.kind in ('generator', 'async_generator')): + contract = contract.args[2] + contract.validate(value, 'return') + + @contextmanager + def installed(self): + """Install callbacks for a scoped experiment, restoring prior hooks.""" + missing = object() + previous = {name: getattr(sys, name, missing) for name in self._HOOKS} + callbacks = (self.before_store, self.before_mutation, + self.on_call, self.on_return) + try: + for name, callback in zip(self._HOOKS, callbacks): + setattr(sys, name, callback) + observed = False + + def probe(frame, name, value): + nonlocal observed + if frame.f_code.co_filename == '': + observed = True + + sys._aiython_before_store = probe + try: + exec(compile('probe_value = 1', '', 'exec'), {}) + finally: + sys._aiython_before_store = self.before_store + if not observed: + raise RuntimeError('NativeTypeBridge requires a CPython build with Aiython VM hooks') + yield self + finally: + for name, value in previous.items(): + if value is missing: + vars(sys).pop(name, None) + else: + setattr(sys, name, value) diff --git a/src/aiython/repl.py b/src/aiython/repl.py index a6e0391..9b9620d 100644 --- a/src/aiython/repl.py +++ b/src/aiython/repl.py @@ -1,13 +1,11 @@ """Interactive Python cells compiled through the Aiython runtime.""" import __future__ -import ast import code import sys from .frontend import parse from .runtime import bind_runtime -from .typed_runtime import helper FUTURE_FLAGS = sum(getattr(__future__, name).compiler_flag for name in __future__.all_feature_names) @@ -47,13 +45,11 @@ def runsource(self, source, filename='', symbol='single'): return True try: unit = unit or parse(source, cell_filename) - if unit.tree.body and isinstance(unit.tree.body[-1], ast.Expr): - expression = unit.tree.body[-1] - expression.value = helper('display', expression.value, - runtime_name=unit.runtime_name) flags = self.compile.compiler.flags & FUTURE_FLAGS - compiled = bind_runtime(self.runtime.prepare(unit, entry=True, flags=flags), - unit.runtime_name, self.runtime) + compiled, native = self.runtime.bridge.prepare_unit( + unit, entry=True, flags=flags, display_last_expr=True) + if not native: + compiled = bind_runtime(compiled, unit.runtime_name, self.runtime) except (OverflowError, SyntaxError, ValueError): self.showsyntaxerror(cell_filename) return False diff --git a/src/aiython/runtime.py b/src/aiython/runtime.py index 261c189..3f3d2c0 100644 --- a/src/aiython/runtime.py +++ b/src/aiython/runtime.py @@ -2,6 +2,7 @@ import ast from collections import OrderedDict +import copy import hashlib from functools import lru_cache import inspect @@ -708,6 +709,8 @@ def __init__(self, config: ResolvedConfig, *, agent_factory=None, stats=False, t self.config = config from .typed_runtime import TypeRuntime self.types = TypeRuntime(self) + from .native_bridge import NativeTypeBridge + self.bridge = NativeTypeBridge(self.types, manager=self) from .capabilities import CapabilityRuntime self.capabilities = CapabilityRuntime(config, trace=trace_plan) self.agent_factory = agent_factory @@ -735,14 +738,16 @@ def source_revision(self, source): def compile_source(self, source, filename, *, entry=False): """Reuse preparation across executions, always restoring fresh metadata.""" from .frontend import parse - key = (type(self), sys.implementation.cache_tag, filename, self.source_revision(source), entry) + key = (type(self), sys.implementation.cache_tag, filename, + self.source_revision(source), entry, bool(self.config.profiles), + bool(self.types.classes), bool(self.types._module_scopes)) started = perf_counter() with _PREPARED_LOCK: cached = _PREPARED.get(key) if cached is not None: _PREPARED.move_to_end(key) if cached is not None: - code, packet = cached + code, packet, native = cached unit, blocks, checkpoints, frames, nodes, names, hints = pickle.loads(packet) with self._lock: self.units[filename] = unit @@ -756,12 +761,16 @@ def compile_source(self, source, filename, *, entry=False): if self.stats.enabled: self.stats.preparation_cache_hits += 1 self.stats.prepare_seconds += perf_counter() - started - return self._bind_compiled(code, unit.runtime_name) + return code if native else self._bind_compiled(code, unit.runtime_name) unit = parse(source, filename) if self.stats.enabled: self.stats.parse_seconds += perf_counter() - started self.stats.preparation_cache_misses += 1 - code = self.prepare(unit, entry=entry) + prepared = perf_counter() + code, native = self.bridge.prepare_unit(unit, entry=entry) + if native: + if self.stats.enabled: + self.stats.prepare_seconds += perf_counter() - prepared # Large generated programs run normally without displacing the cache. if len(source) <= 256 * 1024: with self._lock: @@ -774,10 +783,11 @@ def compile_source(self, source, filename, *, entry=False): {node: self.source_hints[node] for node in ast.walk(unit.tree) if node in self.source_hints})) with _PREPARED_LOCK: - _PREPARED[key] = (code, packet) + _PREPARED[key] = (code, packet, native) while len(_PREPARED) > _PREPARED_LIMIT or sum(len(v[1]) for v in _PREPARED.values()) > _PREPARED_BYTES: _PREPARED.popitem(last=False) - return self._bind_compiled(code, unit.runtime_name, cache=len(source) <= 256 * 1024) + return (code if native else + self._bind_compiled(code, unit.runtime_name, cache=len(source) <= 256 * 1024)) def _bind_compiled(self, code, runtime_name, *, cache=True): if not cache: @@ -825,21 +835,35 @@ def frame_source(self, frame): return self.frame_sources.get(key, self.frame_sources.get( (code.co_filename, 1, ""), {"available": False, "filename": code.co_filename})) - def prepare(self, unit: Unit, *, entry: bool = False, flags: int = 0): + def prepare(self, unit: Unit, *, entry: bool = False, flags: int = 0, + display_last_expr: bool = False, recovery_metadata: bool = False): started = perf_counter() try: with self._lock: - return self._prepare(unit, entry=entry, flags=flags) + return self._prepare(unit, entry=entry, flags=flags, + display_last_expr=display_last_expr, + recovery_metadata=recovery_metadata) finally: if self.stats.enabled: self.stats.prepare_seconds += perf_counter() - started - def _prepare(self, unit: Unit, *, entry: bool = False, flags: int = 0): + def _prepare(self, unit: Unit, *, entry: bool = False, flags: int = 0, + display_last_expr: bool = False, recovery_metadata: bool = False): self.register(unit) self.blocks.update({key: (unit, block) for key, block in unit.blocks.items()}) linecache.cache[unit.filename] = (len(unit.source), None, unit.source.splitlines(True), unit.filename) - tree = unit.tree - from .typed_runtime import ExpectedTypes, TypedTransformer + # Keep the CPython-parsed tree as source metadata. Runtime checks and + # recovery transform a separate tree until they can move to VM hooks. + # Reparse valid source: CPython's parser is faster than deepcopy for + # large ASTs. AI blocks need their remapped locations copied intact. + tree = (copy.deepcopy(unit.tree) if unit.blocks else + ast.parse(unit.source, unit.filename)) + from .typed_runtime import ExpectedTypes, TypedTransformer, helper + if display_last_expr and tree.body and isinstance(tree.body[-1], ast.Expr): + expression = tree.body[-1] + expression.value = ast.copy_location( + helper('display', expression.value, runtime_name=unit.runtime_name), + expression.value) ExpectedTypes(unit.blocks, unit.runtime_name).visit(tree) tables = [symtable.symtable(unit.transformed, unit.filename, "exec")] definitions = {(n.name, n.lineno): min([n.lineno, *(d.lineno for d in n.decorator_list)]) @@ -856,9 +880,11 @@ def _prepare(self, unit: Unit, *, entry: bool = False, flags: int = 0): tree = TypedTransformer(runtime_name=unit.runtime_name).visit(tree) if unit.blocks and any(isinstance(node, ast.AsyncFunctionDef) for node in ast.walk(tree)): tree = AsyncCalls(unit.runtime_name).visit(tree) - tree = NestedCheckpoints(self, unit).visit(tree) + recovery_enabled = bool(self.config.profiles) or recovery_metadata + if recovery_enabled: + tree = NestedCheckpoints(self, unit).visit(tree) ast.fix_missing_locations(tree) - if entry: + if entry and recovery_enabled: guard = next((node for node in tree.body if getattr(node, '_aiython_module_guard', False)), None) if guard is not None: body = [] @@ -972,7 +998,48 @@ def recoverable(error: BaseException) -> bool: import concurrent.futures return not isinstance(error, concurrent.futures.CancelledError) + def caller_has_python_handler(self, frame) -> bool: + """Let an enclosing user try/with body receive the exception first.""" + scope_types = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda) + protected_types = (ast.Try, ast.TryStar, ast.With, ast.AsyncWith) + caller = frame.f_back + while caller is not None: + unit = self.units.get(caller.f_code.co_filename) + if unit is not None: + code = caller.f_code + scope = unit.tree if code.co_name == '' else None + if scope is None: + for candidate in ast.walk(unit.tree): + if not isinstance(candidate, scope_types): + continue + name = '' if isinstance(candidate, ast.Lambda) else candidate.name + first = min([candidate.lineno, + *(node.lineno for node in getattr(candidate, 'decorator_list', ()))]) + if (name == code.co_name and + code.co_firstlineno in (first, candidate.lineno)): + scope = candidate + break + if scope is not None: + pending = [scope] + while pending: + node = pending.pop() + if node is not scope and isinstance(node, scope_types): + continue + if isinstance(node, protected_types): + bodies = [node.body] + if isinstance(node, (ast.Try, ast.TryStar)) and node.finalbody: + bodies.extend(handler.body for handler in node.handlers) + bodies.append(node.orelse) + for body in bodies: + if body and body[0].lineno <= caller.f_lineno <= body[-1].end_lineno: + return True + pending.extend(ast.iter_child_nodes(node)) + caller = caller.f_back + return False + def recover(self, key: str, error: BaseException, attempt: int | None = None) -> bool: + if not self.config.profiles: + raise error if not self.recoverable(error): from .type_constraints import TypeViolation, UnsupportedType if isinstance(error, (TypeViolation, UnsupportedType)) and not getattr(error, '_aiython_location', False): @@ -988,6 +1055,8 @@ def recover(self, key: str, error: BaseException, attempt: int | None = None) -> raise error checkpoint = self.checkpoints[key] frame = inspect.currentframe().f_back + if self.caller_has_python_handler(frame): + raise error # Function/module boundaries pass their own counter. Class boundaries # keep theirs outside the metaclass's namespace. counts = None diff --git a/src/aiython/typed_runtime.py b/src/aiython/typed_runtime.py index 5f80acd..4c1632d 100644 --- a/src/aiython/typed_runtime.py +++ b/src/aiython/typed_runtime.py @@ -82,6 +82,7 @@ def __init__(self, manager=None): self._classes_lock = threading.RLock() self._contract_cache = threading.local() self._active_instance_checks = ContextVar('aiython_active_instance_checks', default=frozenset()) + self._module_scopes = {} self.interactive_globals = None self.interactive_scope = None @@ -273,6 +274,10 @@ def namespace(self, frame): namespace[parameter.__name__] = parameter if parameter.__name__.startswith('__') and not parameter.__name__.endswith('__'): namespace[f'_{owner.__name__.lstrip("_")}{parameter.__name__}'] = parameter + global_scope = self._module_scopes.get(id(frame.f_globals)) + if global_scope is not None and global_scope[0] is frame.f_globals: + namespace.update({parameter.__name__: bound + for parameter, bound in global_scope[1].bindings.items()}) scope = frame_scope(frame) or frame.f_locals.get(SCOPE) if isinstance(scope,Scope): namespace.update({parameter.__name__: bound for parameter,bound in scope.bindings.items()}) @@ -293,6 +298,11 @@ def _initialize(self, frame, declarations, parameters, returns): and self.interactive_scope is not None): scope = self.interactive_scope scope.declarations.update(declarations) + elif (frame.f_code.co_name == '' + and (retained := self._module_scopes.get(id(frame.f_globals))) is not None + and retained[0] is frame.f_globals): + scope = retained[1] + scope.declarations.update(declarations) else: scope = Scope(declarations=declarations, self_owner=owner) if scope.self_owner is not None: @@ -316,6 +326,8 @@ def enter_scope(self, declarations, parameters=None, returns=None): frame = inspect.currentframe().f_back try: scope = self._initialize(frame, declarations, parameters, returns) + if frame.f_code.co_name == '': + self._module_scopes[id(frame.f_globals)] = (frame.f_globals, scope) _FRAME_SCOPES.set(_FRAME_SCOPES.get() + ((frame, scope),)) finally: del frame @@ -341,10 +353,12 @@ def exit_generator_scope(self): finally: del frame - @staticmethod - def scopes(frame): + def scopes(self, frame): local = frame_scope(frame) or frame.f_locals.get(SCOPE) global_scope = frame.f_globals.get(SCOPE) + retained = self._module_scopes.get(id(frame.f_globals)) + if retained is not None and retained[0] is frame.f_globals: + global_scope = retained[1] scopes = [(local,frame.f_locals)] if isinstance(local,Scope) else [] if isinstance(global_scope,Scope) and global_scope is not local: scopes.append((global_scope,frame.f_globals)) @@ -569,33 +583,36 @@ def checked(): except StopIteration as stop: return stop.value return checked() + def validate_attribute(self, frame, owner, name, value, annotation=None): + if annotation: + contract = compile_contract(annotation,self.namespace(frame)) + else: + target = owner if issubclass(type(owner), type) else type(owner) + fields = {} + for base in reversed(target.__mro__): + fields.update(annotations_of(base)) + source = fields.get(name) + contract = None + if source: + scope = self.namespace(frame) + if not descriptor_field(target, name, source, scope): + namespace = Compiler.module_names(target, scope) + namespace[SELF_OWNER] = target + namespace.update({p.__name__:p for p in class_parameters(target)}) + contract = compile_contract(source,namespace) + if contract: + if contract.marker == 'ClassVar' and not issubclass(type(owner), type): + raise TypeViolation(f'{name}: ClassVar must be assigned on the class') + if contract.marker == 'Final': + try: inspect.getattr_static(owner,name) + except AttributeError: pass + else: raise TypeViolation(f'{name}: Final attribute cannot be reassigned') + contract.validate(value,f'{type(owner).__name__}.{name}') + def assign_attribute(self,owner,name,value,annotation=None): frame = inspect.currentframe().f_back try: - if annotation: - contract = compile_contract(annotation,self.namespace(frame)) - else: - target = owner if issubclass(type(owner), type) else type(owner) - fields = {} - for base in reversed(target.__mro__): - fields.update(annotations_of(base)) - source = fields.get(name) - contract = None - if source: - scope = self.namespace(frame) - if not descriptor_field(target, name, source, scope): - namespace = Compiler.module_names(target, scope) - namespace[SELF_OWNER] = target - namespace.update({p.__name__:p for p in class_parameters(target)}) - contract = compile_contract(source,namespace) - if contract: - if contract.marker == 'ClassVar' and not issubclass(type(owner), type): - raise TypeViolation(f'{name}: ClassVar must be assigned on the class') - if contract.marker == 'Final': - try: inspect.getattr_static(owner,name) - except AttributeError: pass - else: raise TypeViolation(f'{name}: Final attribute cannot be reassigned') - contract.validate(value,f'{type(owner).__name__}.{name}') + self.validate_attribute(frame, owner, name, value, annotation) setattr(owner,name,value) if issubclass(type(owner), type): member = type.__getattribute__(owner, '__dict__').get(name) diff --git a/tests/test_overhead.py b/tests/test_overhead.py index fbc24cf..8f8d515 100644 --- a/tests/test_overhead.py +++ b/tests/test_overhead.py @@ -23,6 +23,98 @@ class OverheadTests(unittest.TestCase): + def test_plain_source_uses_cpython_code_object(self): + source = ('def values():\n' + ' yield from (1, 2)\n' + 'result = list(values())\n') + filename = '' + runtime = Runtime(ResolvedConfig(None, Path.cwd()), stats=True) + try: + with patch.object(runtime.bridge, 'prepare_unit', + wraps=runtime.bridge.prepare_unit) as prepare: + code = runtime.compile_source(source, filename, entry=True) + prepare.assert_called_once() + self.assertEqual(code, compile(source, filename, 'exec', dont_inherit=True)) + self.assertIs(code, runtime.compile_source(source, filename, entry=True)) + self.assertEqual(runtime.stats.preparation_cache_hits, 1) + namespace = {} + exec(code, namespace) + self.assertEqual(namespace['result'], [1, 2]) + self.assertFalse(runtime.checkpoints) + finally: + runtime.capabilities.close() + + def test_native_cache_stops_after_typed_class_registration(self): + source = 'value = 1\n' + filename = '' + runtime = Runtime(ResolvedConfig(None, Path.cwd())) + try: + native = runtime.compile_source(source, filename, entry=True) + class Box: + value: int + runtime.types.register_class(Box) + checked = runtime.compile_source(source, filename, entry=True) + self.assertEqual(native, compile(source, filename, 'exec', dont_inherit=True)) + self.assertNotEqual(checked, native) + finally: + runtime.capabilities.close() + + def test_reused_module_namespace_retains_global_contract(self): + runtime = Runtime(ResolvedConfig(None, Path.cwd())) + namespace = {} + try: + exec(runtime.compile_source('value: int = 1\n', ''), namespace) + source = 'value = "invalid"\n' + code = runtime.compile_source(source, '') + self.assertNotEqual(code, compile(source, '', 'exec', dont_inherit=True)) + with self.assertRaises(TypeViolation): + exec(code, namespace) + self.assertEqual(namespace['value'], 1) + finally: + runtime.capabilities.close() + + def test_preparation_cache_separates_configured_recovery(self): + filename = '' + source = '1 / 0\n' + profile = ProfileConfig('default', 'fake', 'fake') + configured = Runtime(ResolvedConfig(None, Path.cwd(), 'default', + {'default': profile})) + unconfigured = Runtime(ResolvedConfig(None, Path.cwd())) + try: + configured.compile_source(source, filename, entry=True) + unconfigured.compile_source(source, filename, entry=True) + self.assertTrue(configured.checkpoints) + self.assertFalse(unconfigured.checkpoints) + finally: + configured.capabilities.close() + unconfigured.capabilities.close() + + def test_preparation_preserves_native_source_tree_and_type_checks(self): + source = 'value: int = 1\nvalue = "bad"\n' + filename = '' + config = ResolvedConfig(None, Path.cwd()) + runtime = Runtime(config) + try: + code = runtime.compile_source(source, filename, entry=True) + original = compile(source, filename, 'exec', dont_inherit=True) + unit = runtime.units[filename] + self.assertEqual(compile(unit.tree, filename, 'exec', dont_inherit=True), original) + self.assertNotEqual(code, original) + with self.assertRaises(TypeViolation): + exec(code, {}) + finally: + runtime.capabilities.close() + cached_runtime = Runtime(config, stats=True) + try: + cached_code = cached_runtime.compile_source(source, filename, entry=True) + self.assertEqual(cached_runtime.stats.preparation_cache_hits, 1) + self.assertEqual(compile(cached_runtime.units[filename].tree, filename, 'exec', + dont_inherit=True), original) + with self.assertRaises(TypeViolation): + exec(cached_code, {}) + finally: + cached_runtime.capabilities.close() + def test_primitive_containers_keep_strict_types_and_error_paths(self): for annotation, good, bad, path in [ ('list[int]', [1, 2], [1, True], 'value[1]'), diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index a8fdce9..d925699 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -20,9 +20,81 @@ from aiython.config import resolve from aiython.models import AiythonError, ProfileConfig, ResolvedConfig from aiython.runtime import Runtime +from aiython.type_constraints import TypeViolation class PythonCompatibilityTests(unittest.TestCase): + def test_plain_entry_preserves_annotated_project_imports(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + name = 'mixed_native_helper_for_aiython' + (root / 'main.py').write_text(f'import {name}\n{name}.change()\n') + (root / f'{name}.py').write_text('value: int = 1\n' + 'def change():\n' + ' global value\n' + ' value = "invalid"\n') + runtime = Runtime(ResolvedConfig(None, root)) + previous = sys.modules.pop(name, None) + try: + with self.assertRaises(TypeViolation): + run_script(root / 'main.py', config=runtime.config, runtime=runtime) + self.assertFalse(any(key.startswith(str(root / 'main.py') + ':') + for key in runtime.checkpoints)) + finally: + sys.modules.pop(name, None) + if previous is not None: + sys.modules[name] = previous + + def test_plain_entry_rechecks_mutated_imported_typed_instance(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + name = 'mixed_typed_instance_for_aiython' + (root / f'{name}.py').write_text('class Box:\n' + ' values: list[int]\n' + ' def __init__(self):\n' + ' self.values = [1]\n') + previous = sys.modules.pop(name, None) + try: + for import_source in (f'import {name}\nmodule = {name}\n', + f'module = __import__("{name}")\n', + f'importer = getattr(__builtins__, "__import__")\n' + f'module = importer("{name}")\n'): + with self.subTest(import_source=import_source): + sys.modules.pop(name, None) + (root / 'main.py').write_text(import_source + + 'box = module.Box()\n' + 'box.values.append("invalid")\n') + with self.assertRaises(TypeViolation): + run_script(root / 'main.py', config=ResolvedConfig(None, root)) + finally: + sys.modules.pop(name, None) + if previous is not None: + sys.modules[name] = previous + + def test_unconfigured_program_raises_original_error(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + script = root / 'main.py' + script.write_text('1 / 0\n') + config = ResolvedConfig(None, root) + runtime = Runtime(config) + with self.assertRaises(ZeroDivisionError): + run_script(script, config=config, runtime=runtime) + self.assertFalse(runtime.checkpoints) + result = subprocess.run([sys.executable, '-m', 'aiython', str(script)], + cwd=root, capture_output=True, text=True, timeout=10) + native = subprocess.run([sys.executable, str(script)], cwd=root, + capture_output=True, text=True, timeout=10) + self.assertEqual((result.returncode, result.stdout, result.stderr), + (native.returncode, native.stdout, native.stderr)) + command = subprocess.run([sys.executable, '-m', 'aiython', '-c', '1/0'], + cwd=root, capture_output=True, text=True, timeout=10) + native_command = subprocess.run([sys.executable, '-c', '1/0'], + cwd=root, capture_output=True, text=True, timeout=10) + self.assertEqual((command.returncode, command.stdout, command.stderr), + (native_command.returncode, native_command.stdout, + native_command.stderr)) + def test_module_resolution_errors_are_explicit(self): with self.assertRaisesRegex(AiythonError, 'Relative module names not supported'): module_details('.relative') diff --git a/tests/test_repl.py b/tests/test_repl.py index a12783f..d36e4a7 100644 --- a/tests/test_repl.py +++ b/tests/test_repl.py @@ -36,6 +36,16 @@ def tearDown(self): else: vars(builtins).pop('_', None) + def test_display_keeps_native_cell_tree(self): + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.assertFalse(self.console.runsource('1 + 2')) + self.assertEqual(output.getvalue(), '3\n') + filename = '' + source_tree = self.runtime.units[filename].tree + self.assertEqual(compile(source_tree, filename, 'exec', dont_inherit=True), + compile('1 + 2', filename, 'exec', dont_inherit=True)) + def test_compound_input_waits_and_displays_last_expression(self): output = io.StringIO() with contextlib.redirect_stdout(output): diff --git a/tests/test_runtime.py b/tests/test_runtime.py index be3594d..3adb15e 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -2,9 +2,11 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch from aiython.cli import run_script from aiython.models import AiythonError, ProfileConfig, RecoveryDecision, ResolvedConfig +from aiython.native_bridge import NativeTypeBridge class FakeAgent: @@ -38,6 +40,27 @@ def run_source(self, source, agent=None, extra=None, force=None): config = ResolvedConfig(None, root, "fast", profiles, force_profile=force) return run_script(path, config=config, agent_factory=lambda p: agent or FakeAgent()) + def test_configured_recovery_and_generator_yield_contract(self): + agent = FakeAgent(recover=lambda request, runtime: + RecoveryDecision('complete', 42, True)) + original = NativeTypeBridge.prepare_unit + with patch.object(NativeTypeBridge, 'prepare_unit', autospec=True, + side_effect=original) as prepare: + result = self.run_source('''from typing import Generator +def values() -> Generator[int, None, None]: + yield 1 + yield "invalid" +try: + list(values()) +except TypeError: + caught = True +answer = missing_value +''', agent) + prepare.assert_called_once() + self.assertTrue(result['caught']) + self.assertEqual(result['answer'], 42) + self.assertEqual(len(agent.errors), 1) + def test_plain_python_and_handled_error_never_call_agent(self): agent = FakeAgent() result = self.run_source('''"module doc" @@ -239,6 +262,45 @@ def test_user_try_handler_runs_before_ai_recovery(self): self.assertEqual(result['events'], ['handled', 'handled']) self.assertFalse(agent.errors) + def test_caller_try_and_with_handle_error_before_ai_recovery(self): + agent = FakeAgent() + result = self.run_source('''from contextlib import suppress +def fail(): + return 1 / 0 +def relay(): + return fail() +events = [] +try: + relay() +except ZeroDivisionError: + events.append('except') +with suppress(ZeroDivisionError): + relay() +events.append('after with') +''', agent) + self.assertEqual(result['events'], ['except', 'after with']) + self.assertFalse(agent.errors) + + def test_caller_finally_runs_before_ai_recovery(self): + observed = [] + def repair(request, bridge): + observed.extend(bridge.eval('events')) + return RecoveryDecision('complete') + agent = FakeAgent(recover=repair) + result = self.run_source('''events = [] +def fail(): + raise ValueError('bad') +try: + raise KeyError('first') +except KeyError: + fail() +finally: + events.append('finally') +''', agent) + self.assertEqual(result['events'], ['finally']) + self.assertEqual(observed, ['finally']) + self.assertEqual(len(agent.errors), 1) + def test_enclosing_loop_retry_is_rejected_without_replaying_effects(self): seen = [] def repair(request, bridge): diff --git a/tests/test_runtime_edges.py b/tests/test_runtime_edges.py index 8211c22..e691f66 100644 --- a/tests/test_runtime_edges.py +++ b/tests/test_runtime_edges.py @@ -174,14 +174,15 @@ def test_legacy_cpython_frame_support_import(self): class RuntimeRecoveryEdgeTests(unittest.TestCase): def setUp(self): - self.manager = rt.Runtime(ResolvedConfig(None, Path.cwd())) + profile = ProfileConfig('test', 'fake', 'model') + self.manager = rt.Runtime(ResolvedConfig(None, Path.cwd(), 'test', {'test': profile})) self.unit = parse('answer = 1 / 0', 'recovery-test.py') self.span = SourceSpan(self.unit.filename, 1, 0, 1, 14) self.manager.units[self.unit.filename] = self.unit self.manager.checkpoints['failure'] = rt.Checkpoint( self.unit, self.span, 'answer = 1 / 0', 'answer') self.request = AgentRequest('answer = 1 / 0', self.unit.source, {}, {}, - self.span, ProfileConfig('test', 'fake', 'model'), ()) + self.span, profile, ()) self.agent = SimpleNamespace(recover=Mock(return_value=RecoveryDecision('complete'))) def recover(self, error, decision, *, attempt=1, target='answer'): diff --git a/tests/test_typed_runtime_edges.py b/tests/test_typed_runtime_edges.py index 6661f92..98f48f7 100644 --- a/tests/test_typed_runtime_edges.py +++ b/tests/test_typed_runtime_edges.py @@ -30,7 +30,7 @@ def scoped(): __aiython_type_scope__ = Scope(declarations={'y': 'str'}) globals()[SCOPE] = nonlocal_scope try: - scopes = TypeRuntime.scopes(inspect.currentframe()) + scopes = self.runtime.scopes(inspect.currentframe()) self.assertEqual(len(scopes), 2) finally: globals().pop(SCOPE) From 54dabec17d06bedb97bc14463ce88e75b327ff7c Mon Sep 17 00:00:00 2001 From: sunmodza Date: Mon, 28 Sep 2026 12:38:43 +0700 Subject: [PATCH 69/72] Make offline CI independent of local credentials and launcher alias --- .github/workflows/release.yml | 4 ++++ .github/workflows/tests.yml | 4 ++++ tests/test_python_compatibility.py | 7 ++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 31c6d21..a7927a5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,6 +25,10 @@ jobs: python-version: ${{ matrix.python-version }} version: latest-known enable-cache: false + - name: Prepare offline project credentials + run: | + mkdir -p .aiython + printf 'OPENROUTER_API_KEY=offline-test\n' > .aiython/credentials.env - name: Check CPython frontend code objects run: uv run --locked --python ${{ matrix.python-version }} python benchmarks/stdlib_syntax.py - name: Test with line and branch coverage diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f4ed8e7..aa1127f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,6 +37,10 @@ jobs: python-version: ${{ matrix.python-version }} version: latest-known - run: uv python install ${{ matrix.python-version }} + - name: Prepare offline project credentials + run: | + mkdir -p .aiython + printf 'OPENROUTER_API_KEY=offline-test\n' > .aiython/credentials.env - name: Check CPython frontend code objects run: uv run --locked --python ${{ matrix.python-version }} python benchmarks/stdlib_syntax.py - name: Test with line and branch coverage diff --git a/tests/test_python_compatibility.py b/tests/test_python_compatibility.py index d925699..250137c 100644 --- a/tests/test_python_compatibility.py +++ b/tests/test_python_compatibility.py @@ -620,10 +620,15 @@ def test_parent_package_startup_matches_cpython(self): def test_console_entry_point_finds_current_directory_modules(self): console = Path(sys.executable).with_name('aiython') self.assertTrue(console.is_file()) + # A uv environment may invoke the test suite through python3 while + # the generated console script's shebang uses python. Compare two + # processes started with the same interpreter path. + with console.open('rb') as script: + console_python = script.readline().decode().removeprefix('#!').strip() with tempfile.TemporaryDirectory() as directory: root = Path(directory) (root / 'localmodule.py').write_text('import sys\nprint(sys.argv, sys.orig_argv, sys.path[0])\n') - python = subprocess.run([sys.executable, '-m', 'localmodule', 'arg'], cwd=root, + python = subprocess.run([console_python, '-m', 'localmodule', 'arg'], cwd=root, capture_output=True, text=True) aiython = subprocess.run([str(console), '-m', 'localmodule', 'arg'], cwd=root, capture_output=True, text=True) From 6aa6f12b4687438bad42e2560a4a4d3566cf4ff6 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Mon, 28 Sep 2026 12:44:01 +0700 Subject: [PATCH 70/72] Expose recovery and yield through managed bridge API --- native/README.md | 7 ++++--- src/aiython/native_bridge.py | 12 +++++++++--- tests/test_runtime.py | 25 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/native/README.md b/native/README.md index 1491698..357b6a8 100644 --- a/native/README.md +++ b/native/README.md @@ -63,10 +63,11 @@ entry point called by `Runtime.compile_source`. It selects the exact CPython code object for ordinary source without imports, annotations, AI directives, blocks, or configured recovery. For annotated source and configured recovery, it delegates to Aiython's boundary compiler; that path checks yielded values -and can resume failed statements. This routing works on stock CPython and the -four pinned patched interpreters. +and can resume failed statements. A bridge bound to a `Runtime` offers the +same behavior through `compile_source`. This routing works on stock CPython +and the four pinned patched interpreters. -The separate experimental `NativeTypeBridge.compile_source` and `installed` +The standalone experimental `NativeTypeBridge.compile_source` and `installed` API parses annotations, compiles the original source, and validates annotated parameters, generator returns, local/global/nonlocal assignments, `Final` rebinding, and class attributes through the VM hooks. It does not yet check diff --git a/src/aiython/native_bridge.py b/src/aiython/native_bridge.py index 3385b2f..2856ac9 100644 --- a/src/aiython/native_bridge.py +++ b/src/aiython/native_bridge.py @@ -58,8 +58,9 @@ class NativeTypeBridge: ``prepare_unit`` is the normal runtime entry point. It compiles plain source directly with CPython, while annotated source, typed yields and AI - recovery use the existing Aiython boundary compiler. ``compile_source`` - and ``installed`` expose the experimental patched-VM hooks separately. + recovery use the existing Aiython boundary compiler. A bridge bound to a + Runtime also exposes this behavior through ``compile_source``. Without a + Runtime, ``compile_source`` and ``installed`` expose patched-VM hooks. """ _HOOKS = ('_aiython_before_store', '_aiython_before_mutation', @@ -121,7 +122,12 @@ def prepare_unit(self, unit, *, entry=False, flags=0, len(unit.source), None, unit.source.splitlines(True), unit.filename) return compile(unit.source, unit.filename, 'exec', dont_inherit=True), True - def compile_source(self, source: str, filename: str): + def compile_source(self, source: str, filename: str, *, entry=False): + if self.manager is not None: + from .frontend import parse + unit = parse(source, filename) + code, native = self.prepare_unit(unit, entry=entry) + return code if native else self.manager._bind_compiled(code, unit.runtime_name) tree = ast.parse(source, filename) module = _FrameTypes(TypedTransformer.declarations_in(tree.body), {}, None, frozenset()) diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 3adb15e..ca49123 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -7,6 +7,7 @@ from aiython.cli import run_script from aiython.models import AiythonError, ProfileConfig, RecoveryDecision, ResolvedConfig from aiython.native_bridge import NativeTypeBridge +from aiython.runtime import Runtime class FakeAgent: @@ -61,6 +62,30 @@ def values() -> Generator[int, None, None]: self.assertEqual(result['answer'], 42) self.assertEqual(len(agent.errors), 1) + def test_managed_bridge_compiles_recovery_and_typed_yield_directly(self): + agent = FakeAgent(recover=lambda request, runtime: + RecoveryDecision('complete', 42, True)) + profile = ProfileConfig('fast', 'fake', 'fast') + runtime = Runtime(ResolvedConfig(None, Path.cwd(), 'fast', {'fast': profile}), + agent_factory=lambda _: agent) + try: + code = runtime.bridge.compile_source('''from typing import Generator +def values() -> Generator[int, None, None]: + yield "invalid" +try: + next(values()) +except TypeError: + caught = True +answer = missing_value +''', '', entry=True) + namespace = {} + exec(code, namespace) + self.assertTrue(namespace['caught']) + self.assertEqual(namespace['answer'], 42) + self.assertEqual(len(agent.errors), 1) + finally: + runtime.capabilities.close() + def test_plain_python_and_handled_error_never_call_agent(self): agent = FakeAgent() result = self.run_source('''"module doc" From cb9778e7e6f8e59ab79bf4cb7a82e9573354d2d3 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Mon, 28 Sep 2026 12:53:58 +0700 Subject: [PATCH 71/72] Validate native generator yields with Python monitoring --- .github/workflows/tests.yml | 3 ++- native/README.md | 16 +++++++++++---- native/tests/typed_bridge.py | 27 ++++++++++++++++++++++++- src/aiython/native_bridge.py | 39 +++++++++++++++++++++++++++++++++++- tests/test_native_bridge.py | 32 +++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 tests/test_native_bridge.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index aa1127f..9fe2fd7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,10 +54,11 @@ jobs: uv run --locked --python ${{ matrix.python-version }} --extra a2a python -m coverage json -o coverage.json uv run --locked --python ${{ matrix.python-version }} python scripts/coverage_summary.py - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() with: name: coverage-${{ matrix.python-version }} path: coverage.json - if-no-files-found: error + if-no-files-found: warn native-hooks: runs-on: ubuntu-latest diff --git a/native/README.md b/native/README.md index 357b6a8..d579cf5 100644 --- a/native/README.md +++ b/native/README.md @@ -29,6 +29,13 @@ is outside the function's source-level `try` range in some cases, so an inner `except` will not necessarily catch it. This remains an unresolved semantic gap. +On Python 3.12–3.14, the standalone bridge also installs a scoped +`sys.monitoring.PY_YIELD` callback. It validates values from `yield`, +`yield from`, and async generators before they reach the caller. A validation +error follows CPython's normal exception path, including generator `finally` +blocks. Python 3.11 has no `sys.monitoring`, so typed yields there continue +through the managed bridge's boundary compiler. + The automated builder verifies the unmodified CPython baseline, applies the matching patch, builds again, runs the VM hook smoke test, and runs eight CPython test modules. It also runs `native/tests/typed_bridge.py`, which @@ -70,7 +77,8 @@ and the four pinned patched interpreters. The standalone experimental `NativeTypeBridge.compile_source` and `installed` API parses annotations, compiles the original source, and validates annotated parameters, generator returns, local/global/nonlocal assignments, `Final` -rebinding, and class attributes through the VM hooks. It does not yet check -yielded values, deletions, in-place mutation through method calls, generic -bindings, or provide AI recovery through VM callbacks. These results do not -establish full Python compatibility. +rebinding, and class attributes through the VM hooks. On Python 3.12–3.14 it +also validates yielded values through `sys.monitoring`. It does not yet check +yielded values on Python 3.11, deletions, in-place mutation through method +calls, generic bindings, or provide AI recovery through VM callbacks. These +results do not establish full Python compatibility. diff --git a/native/tests/typed_bridge.py b/native/tests/typed_bridge.py index 91c76f1..58b8a3a 100644 --- a/native/tests/typed_bridge.py +++ b/native/tests/typed_bridge.py @@ -1,4 +1,5 @@ """Exercise Aiython's experimental type bridge on patched CPython.""" +import asyncio import sys from aiython.native_bridge import NativeTypeBridge @@ -6,7 +7,9 @@ SOURCE = ''' -from typing import Final, Generator +from typing import AsyncGenerator, Final, Generator + +yield_cleanup = [] answer: int = 1 fixed: Final[int] = 1 @@ -85,6 +88,18 @@ def generator() -> Generator[int, None, None]: def generator_error() -> Generator[int, None, int]: yield 1 return "invalid" + +def yield_error() -> Generator[int, None, None]: + try: + yield "invalid" + finally: + yield_cleanup.append("closed") + +def yield_from_error() -> Generator[int, None, None]: + yield from (1, "invalid") + +async def async_yield_error() -> AsyncGenerator[int, None]: + yield "invalid" ''' @@ -119,6 +134,16 @@ def main(): rejected(namespace['make_final_setter']()) assert list(namespace['generator']()) == [1] rejected(lambda: list(namespace['generator_error']())) + if hasattr(sys, 'monitoring'): + rejected(lambda: list(namespace['yield_error']())) + assert namespace['yield_cleanup'] == ['closed'] + rejected(lambda: list(namespace['yield_from_error']())) + + async def consume_async(): + async for _ in namespace['async_yield_error'](): + pass + + rejected(lambda: asyncio.run(consume_async())) rejected(namespace['local_error']) rejected(namespace['local_final']) rejected(lambda: namespace['parameter_reassignment'](1)) diff --git a/src/aiython/native_bridge.py b/src/aiython/native_bridge.py index 2856ac9..2e24e77 100644 --- a/src/aiython/native_bridge.py +++ b/src/aiython/native_bridge.py @@ -219,6 +219,42 @@ def on_return(self, frame, value): contract = contract.args[2] contract.validate(value, 'return') + def on_yield(self, code, offset, value): + """Validate the value before CPython exposes it to the generator caller.""" + frame = inspect.currentframe().f_back + try: + info = self._frame_types(frame) + if info is None or not info.returns: + return + contract = self.types.contract(info.returns, self.types.namespace(frame)) + if contract.kind not in ('generator', 'async_generator'): + raise TypeViolation('Generator return annotation must describe yielded values') + contract.args[0].validate(value, 'yield') + finally: + del frame + + @contextmanager + def _yield_monitor(self): + monitoring = getattr(sys, 'monitoring', None) + if monitoring is None or not hasattr(monitoring.events, 'PY_YIELD'): + yield + return + tool = next((number for number in reversed(range(6)) + if monitoring.get_tool(number) is None), None) + if tool is None: + raise RuntimeError('No free sys.monitoring tool ID for typed yields') + monitoring.use_tool_id(tool, 'aiython-native-yield') + try: + monitoring.register_callback(tool, monitoring.events.PY_YIELD, self.on_yield) + monitoring.set_events(tool, monitoring.events.PY_YIELD) + try: + yield + finally: + monitoring.set_events(tool, 0) + monitoring.register_callback(tool, monitoring.events.PY_YIELD, None) + finally: + monitoring.free_tool_id(tool) + @contextmanager def installed(self): """Install callbacks for a scoped experiment, restoring prior hooks.""" @@ -243,7 +279,8 @@ def probe(frame, name, value): sys._aiython_before_store = self.before_store if not observed: raise RuntimeError('NativeTypeBridge requires a CPython build with Aiython VM hooks') - yield self + with self._yield_monitor(): + yield self finally: for name, value in previous.items(): if value is missing: diff --git a/tests/test_native_bridge.py b/tests/test_native_bridge.py new file mode 100644 index 0000000..741727a --- /dev/null +++ b/tests/test_native_bridge.py @@ -0,0 +1,32 @@ +"""Bridge contracts that also run without a patched CPython interpreter.""" + +import sys +import unittest + +from aiython.native_bridge import NativeTypeBridge +from aiython.type_constraints import TypeViolation + + +class NativeBridgeTests(unittest.TestCase): + @unittest.skipUnless(hasattr(sys, 'monitoring'), 'requires Python 3.12+') + def test_yield_monitor_rejects_value_and_runs_generator_finally(self): + source = '''from typing import Generator +cleanup = [] +def values() -> Generator[int, None, None]: + try: + yield from (1, "invalid") + finally: + cleanup.append("closed") +''' + filename = '' + bridge = NativeTypeBridge() + code = bridge.compile_source(source, filename) + self.assertEqual(code, compile(source, filename, 'exec', dont_inherit=True)) + namespace = {} + before = [sys.monitoring.get_tool(number) for number in range(6)] + with bridge._yield_monitor(): + exec(code, namespace) + with self.assertRaisesRegex(TypeViolation, 'yield'): + list(namespace['values']()) + self.assertEqual(namespace['cleanup'], ['closed']) + self.assertEqual([sys.monitoring.get_tool(number) for number in range(6)], before) From 9f0bbead08bb53152e2e805da9260ca48b6f2c42 Mon Sep 17 00:00:00 2001 From: sunmodza Date: Mon, 28 Sep 2026 14:28:50 +0700 Subject: [PATCH 72/72] Use VM type callbacks with AI recovery and restore full coverage --- native/README.md | 12 +- src/aiython/native_bridge.py | 217 +++++++++- src/aiython/runtime.py | 18 +- tests/test_cli.py | 153 +++++++ tests/test_native_bridge.py | 601 ++++++++++++++++++++++++++- tests/test_overhead.py | 7 +- tests/test_repl.py | 43 ++ tests/test_runtime_edges.py | 52 +++ tests/test_type_constraints_edges.py | 180 ++++++++ tests/test_typed_runtime_edges.py | 162 ++++++++ 10 files changed, 1418 insertions(+), 27 deletions(-) diff --git a/native/README.md b/native/README.md index d579cf5..41d6308 100644 --- a/native/README.md +++ b/native/README.md @@ -68,11 +68,13 @@ skip the regeneration command: its interpreter is maintained directly in `aiython.native_bridge.NativeTypeBridge.prepare_unit` is the normal compiler entry point called by `Runtime.compile_source`. It selects the exact CPython code object for ordinary source without imports, annotations, AI directives, -blocks, or configured recovery. For annotated source and configured recovery, -it delegates to Aiython's boundary compiler; that path checks yielded values -and can resume failed statements. A bridge bound to a `Runtime` offers the -same behavior through `compile_source`. This routing works on stock CPython -and the four pinned patched interpreters. +blocks, or configured recovery. On patched CPython, simple annotated module +assignments use the VM store callback by default. When recovery is configured, +that subset adds statement checkpoints so AI can resume after an error while +the VM callback checks typed stores. Other annotated source and configured +recovery use Aiython's boundary compiler; that path also checks yielded values. +On stock CPython the boundary compiler handles typed source and recovery. +A bridge bound to a `Runtime` offers the same routing through `compile_source`. The standalone experimental `NativeTypeBridge.compile_source` and `installed` API parses annotations, compiles the original source, and validates annotated diff --git a/src/aiython/native_bridge.py b/src/aiython/native_bridge.py index 2e24e77..f0038ef 100644 --- a/src/aiython/native_bridge.py +++ b/src/aiython/native_bridge.py @@ -12,9 +12,93 @@ import inspect import linecache import sys +import threading +import weakref -from .type_constraints import TypeViolation -from .typed_runtime import TypedTransformer, TypeRuntime +from .type_constraints import Contract, TypeViolation, compile_contract +from .typed_runtime import TypedTransformer, TypeRuntime, unconstrained_variadic + + +_VM_LOCK = threading.RLock() +_VM_AVAILABLE = None +_VM_CODES = {} +_VM_HOOK_INSTALLED = False +_VM_PREVIOUS_STORE = None +_VM_PREVIOUS_CALL = None + + +def vm_available(): + """Probe the store callback once; stock CPython ignores the sys attribute.""" + global _VM_AVAILABLE + if _VM_AVAILABLE is not None: + return _VM_AVAILABLE + with _VM_LOCK: + if _VM_AVAILABLE is None: + missing = object() + previous = getattr(sys, '_aiython_before_store', missing) + observed = False + + def probe(frame, name, value): + nonlocal observed + if frame.f_code.co_filename == '': + observed = True + + try: + sys._aiython_before_store = probe + exec(compile('probe_value = 1', '', 'exec'), {}) + finally: + if previous is missing: + vars(sys).pop('_aiython_before_store', None) + else: + sys._aiython_before_store = previous + _VM_AVAILABLE = observed + return _VM_AVAILABLE + + +def _vm_entry(code): + with _VM_LOCK: + entry = _VM_CODES.get(id(code)) + return entry if entry is not None and entry[0]() is code else None + + +def _vm_before_store(frame, name, value): + if _VM_LOCK is None: + return + entry = _vm_entry(frame.f_code) + if entry is not None: + entry[1].before_store(frame, name, value) + if _VM_PREVIOUS_STORE is not None: + _VM_PREVIOUS_STORE(frame, name, value) + + +def _vm_on_call(frame): + if _VM_LOCK is None: + return + entry = _vm_entry(frame.f_code) + if entry is not None: + entry[1].on_call(frame) + if _VM_PREVIOUS_CALL is not None: + _VM_PREVIOUS_CALL(frame) + + +def _register_vm_code(code, bridge, info): + global _VM_HOOK_INSTALLED, _VM_PREVIOUS_STORE, _VM_PREVIOUS_CALL + with _VM_LOCK: + if not _VM_HOOK_INSTALLED: + _VM_PREVIOUS_STORE = getattr(sys, '_aiython_before_store', None) + _VM_PREVIOUS_CALL = getattr(sys, '_aiython_on_call', None) + sys._aiython_before_store = _vm_before_store + sys._aiython_on_call = _vm_on_call + _VM_HOOK_INSTALLED = True + key = id(code) + + def discard(reference): + with _VM_LOCK: + current = _VM_CODES.get(key) + if current is not None and current[0] is reference: + _VM_CODES.pop(key, None) + + _VM_CODES[key] = (weakref.ref(code, discard), bridge, info) @dataclass(frozen=True) @@ -56,11 +140,12 @@ def visit(node): class NativeTypeBridge: """Choose CPython bytecode or Aiython boundaries and validate VM hooks. - ``prepare_unit`` is the normal runtime entry point. It compiles plain - source directly with CPython, while annotated source, typed yields and AI - recovery use the existing Aiython boundary compiler. A bridge bound to a - Runtime also exposes this behavior through ``compile_source``. Without a - Runtime, ``compile_source`` and ``installed`` expose patched-VM hooks. + ``prepare_unit`` is the normal runtime entry point. Plain source keeps + CPython code objects. Patched CPython uses VM callbacks for simple typed + module assignments, with statement checkpoints when recovery is enabled. + Other typed source and recovery use Aiython's boundary compiler. A bridge + bound to a Runtime also exposes this behavior through ``compile_source``. + Without a Runtime, ``compile_source`` and ``installed`` expose VM hooks. """ _HOOKS = ('_aiython_before_store', '_aiython_before_mutation', @@ -105,12 +190,52 @@ def native_source(self, unit, *, flags=0, display_last_expr=False, return False return True + def vm_source(self, unit): + """Use VM stores for simple annotated module assignments.""" + manager = self.manager + if (manager is None or unit.blocks or unit.directives.annotations or self.types.classes + or self.types._module_scopes): + return False + found_annotation = False + for statement in unit.tree.body: + if isinstance(statement, ast.AnnAssign) and isinstance(statement.target, ast.Name): + found_annotation = True + if not isinstance(statement.annotation, ast.Name): + return False + if statement.annotation.id not in ('int', 'str', 'bool', 'float', 'bytes'): + return False + value = statement.value + elif isinstance(statement, ast.Assign) and all( + isinstance(target, ast.Name) for target in statement.targets): + value = statement.value + elif isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Constant): + continue + elif isinstance(statement, ast.Pass): + continue + else: + return False + if value is not None and not isinstance(value, ast.Name): + try: + ast.literal_eval(value) + except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError): + return False + return found_annotation and vm_available() + def prepare_unit(self, unit, *, entry=False, flags=0, display_last_expr=False, recovery_metadata=False): """Return (code, native); recovery and typed yields remain executable.""" manager = self.manager if manager is None: raise RuntimeError('Source preparation requires a Runtime') + if not (flags or display_last_expr or recovery_metadata) and self.vm_source(unit): + code = (self._compile_vm_recovery(unit) if manager.config.profiles else + self._compile_vm_source(unit.source, unit.filename)) + _register_vm_code(code, self, self.module_types[unit.filename]) + if not manager.config.profiles: + manager.register(unit) + linecache.cache[unit.filename] = ( + len(unit.source), None, unit.source.splitlines(True), unit.filename) + return code, True if not self.native_source(unit, flags=flags, display_last_expr=display_last_expr, recovery_metadata=recovery_metadata): @@ -122,12 +247,19 @@ def prepare_unit(self, unit, *, entry=False, flags=0, len(unit.source), None, unit.source.splitlines(True), unit.filename) return compile(unit.source, unit.filename, 'exec', dont_inherit=True), True + def uses_vm(self, code): + entry = _vm_entry(code) + return entry is not None and entry[1] is self + def compile_source(self, source: str, filename: str, *, entry=False): if self.manager is not None: from .frontend import parse unit = parse(source, filename) code, native = self.prepare_unit(unit, entry=entry) return code if native else self.manager._bind_compiled(code, unit.runtime_name) + return self._compile_vm_source(source, filename) + + def _index_vm_source(self, source, filename): tree = ast.parse(source, filename) module = _FrameTypes(TypedTransformer.declarations_in(tree.body), {}, None, frozenset()) @@ -161,12 +293,40 @@ def collect(node, prefix, outer): for statement in tree.body: collect(statement, '', module) + return tree + + def _compile_vm_source(self, source, filename): + self._index_vm_source(source, filename) # Compile the exact source. Compiling an AST can produce a different # code object, even when the tree is not explicitly transformed. return compile(source, filename, 'exec', dont_inherit=True) + def _compile_vm_recovery(self, unit): + """Keep VM type callbacks while adding statement retry boundaries.""" + from .runtime import RuntimeReferences, bind_runtime, install_checkpoint + + manager = self.manager + tree = self._index_vm_source(unit.source, unit.filename) + manager.register(unit) + body = [] + for index, statement in enumerate(tree.body): + if isinstance(statement, (ast.Assign, ast.AnnAssign)): + key = f'{unit.filename}:vm-checkpoint:{index}' + statement = install_checkpoint(manager, unit, statement, key, + scoped_retries=True) + body.append(statement) + tree.body = body + ast.fix_missing_locations(tree) + tree = RuntimeReferences(unit.runtime_name).visit(tree) + ast.fix_missing_locations(tree) + code = compile(tree, unit.filename, 'exec', dont_inherit=True) + return bind_runtime(code, unit.runtime_name, manager) + def _frame_types(self, frame): code = frame.f_code + entry = _vm_entry(code) + if entry is not None and entry[1] is self: + return entry[2] return self.frames.get((code.co_filename, code.co_qualname, code.co_firstlineno)) @@ -174,26 +334,49 @@ def before_store(self, frame, name, value): info = self._frame_types(frame) if info is None: return + if frame.f_code.co_name == '': + retained = self.types._module_scopes.get(id(frame.f_globals)) + if retained is None or retained[0] is not frame.f_globals: + scope = self.types._initialize(frame, info.declarations, {}, None) + self.types._module_scopes[id(frame.f_globals)] = (frame.f_globals, scope) source = info.declarations.get(name) + variadic = None if source is None and name in info.parameters: parameter_source, mode = info.parameters[name] if mode == 'value': source = parameter_source + else: + variadic = (parameter_source, mode) if source is None and name in frame.f_code.co_freevars: outer = info.outer while outer is not None: source = outer.declarations.get(name) if source is None and name in outer.parameters: - source, mode = outer.parameters[name] - if mode != 'value': - source = None - if source is not None: + parameter_source, mode = outer.parameters[name] + if mode == 'value': + source = parameter_source + else: + variadic = (parameter_source, mode) + if source is not None or variadic is not None: break outer = outer.outer if name in info.global_names: source = self.module_types[frame.f_code.co_filename].declarations.get(name) - if source: - contract = self.types.contract(source, self.types.namespace(frame)) + if source or variadic is not None: + namespace = self.types.namespace(frame) + if variadic is not None: + parameter_source, mode = variadic + contract = (Contract('any', 'Any') if unconstrained_variadic( + parameter_source, mode, namespace) else + self.types.contract(parameter_source, namespace)) + if mode == 'args': + contract = Contract('tuple_many', parameter_source, (contract,)) + else: + contract = (contract.args[0] if contract.kind == 'unpack_typeddict' else + Contract('dict', parameter_source, + (compile_contract('str', namespace), contract))) + else: + contract = self.types.contract(source, namespace) if contract.marker == 'Final': values = frame.f_globals if name in info.global_names else frame.f_locals if name in values: @@ -206,7 +389,13 @@ def before_mutation(self, frame, kind, owner, key, value): def on_call(self, frame): info = self._frame_types(frame) - if info is None or not info.parameters: + if info is None: + return + if frame.f_code.co_name == '': + scope = self.types._initialize(frame, info.declarations, {}, None) + self.types._module_scopes[id(frame.f_globals)] = (frame.f_globals, scope) + return + if not info.parameters: return self.types._initialize(frame, info.declarations, info.parameters, info.returns) diff --git a/src/aiython/runtime.py b/src/aiython/runtime.py index 3f3d2c0..497ca37 100644 --- a/src/aiython/runtime.py +++ b/src/aiython/runtime.py @@ -772,7 +772,7 @@ def compile_source(self, source, filename, *, entry=False): if self.stats.enabled: self.stats.prepare_seconds += perf_counter() - prepared # Large generated programs run normally without displacing the cache. - if len(source) <= 256 * 1024: + if len(source) <= 256 * 1024 and not self.bridge.uses_vm(code): with self._lock: packet = pickle.dumps((unit, {k: v for k, v in self.blocks.items() if v[0].filename == filename}, @@ -1061,8 +1061,8 @@ def recover(self, key: str, error: BaseException, attempt: int | None = None) -> # keep theirs outside the metaclass's namespace. counts = None if attempt is None: - from .typed_runtime import SCOPE, Scope, frame_scope - scope = frame_scope(frame) or frame.f_locals.get(SCOPE) + from .typed_runtime import Scope + scope = self.recovery_scope(frame) counts = (scope.recovery_counts if isinstance(scope, Scope) else frame.f_locals.setdefault(checkpoint.unit.runtime_name + 'recovery_counts', {})) attempt = counts.get(key, 0) + 1 @@ -1111,10 +1111,18 @@ def recover(self, key: str, error: BaseException, attempt: int | None = None) -> finally: del frame + def recovery_scope(self, frame): + from .typed_runtime import SCOPE, Scope, frame_scope + scope = frame_scope(frame) or frame.f_locals.get(SCOPE) + if not isinstance(scope, Scope): + retained = self.types._module_scopes.get(id(frame.f_globals)) + if retained is not None and retained[0] is frame.f_globals: + scope = retained[1] + return scope + def clear_recovery_count(self, key): - from .typed_runtime import SCOPE, frame_scope frame = inspect.currentframe().f_back try: - (frame_scope(frame) or frame.f_locals[SCOPE]).recovery_counts.pop(key, None) + self.recovery_scope(frame).recovery_counts.pop(key, None) finally: del frame diff --git a/tests/test_cli.py b/tests/test_cli.py index c1cf914..897b78a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,6 +7,7 @@ import subprocess import sys import tempfile +import types import unittest from pathlib import Path from unittest.mock import patch @@ -17,11 +18,163 @@ from aiython.config import CAPABILITIES, credential, resolve from aiython.cli import main, run_repl, run_script +from aiython import cli from aiython.models import ConfigError from aiython.setup import ModelChoice, _catalog, _choose_model, setup class CLITests(unittest.TestCase): + def test_module_source_restores_parent_attribute_after_static_import(self): + parent = types.ModuleType('temporary_parent') + child = types.ModuleType('temporary_parent.child') + secondary_parent = types.ModuleType('temporary_secondary_parent') + secondary_child = types.ModuleType('temporary_secondary_parent.child') + secondary_parent.child = secondary_child + previous = object() + parent.child = previous + finder = types.SimpleNamespace(static_modules=[ + (secondary_child, secondary_parent, 'child', False, None), + (child, parent, 'child', True, previous)]) + + def details(name): + parent.child = child + sys.modules[child.__name__] = child + return types.SimpleNamespace(name=name, origin='', loader=None), 'pass', None + + try: + with patch.object(cli, 'ModuleStartFinder', return_value=finder), \ + patch.object(cli, 'module_details', side_effect=details): + cli.module_source(child.__name__, runtime=object()) + self.assertIs(parent.child, previous) + self.assertNotIn(child.__name__, sys.modules) + self.assertFalse(hasattr(secondary_parent, 'child')) + finally: + sys.modules.pop(child.__name__, None) + + def test_repl_startup_reports_open_compile_and_execution_errors(self): + class Terminal(io.StringIO): + def isatty(self): + return True + + with tempfile.TemporaryDirectory() as directory: + startup = Path(directory) / 'startup.py' + startup.write_text('answer = 1\n') + errors = io.StringIO() + with patch.dict(os.environ, {'PYTHONSTARTUP': str(Path(directory) / 'missing.py')}), \ + patch.object(sys, 'stdin', Terminal()), \ + contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(errors): + run_repl(restore_state=True) + self.assertIn('Could not open PYTHONSTARTUP', errors.getvalue()) + + errors = io.StringIO() + with patch.dict(os.environ, {'PYTHONSTARTUP': str(startup)}), \ + patch.object(sys, 'stdin', Terminal()), \ + patch('aiython.cli.Runtime.compile_source', side_effect=SyntaxError('bad startup')), \ + contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(errors): + run_repl(restore_state=True) + self.assertIn('SyntaxError', errors.getvalue()) + + errors = io.StringIO() + with patch.dict(os.environ, {'PYTHONSTARTUP': str(startup)}), \ + patch.object(sys, 'stdin', Terminal()), \ + patch('aiython.repl.AiythonConsole.runcode', side_effect=RuntimeError('startup failed')), \ + contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(errors): + run_repl(restore_state=True) + self.assertIn('startup failed', errors.getvalue()) + with patch.dict(os.environ, {'PYTHONSTARTUP': str(startup)}), \ + patch.object(sys, 'stdin', Terminal()), \ + patch('aiython.repl.AiythonConsole.runcode', side_effect=SystemExit(4)): + with self.assertRaises(SystemExit): + run_repl(restore_state=True) + + def test_existing_interactive_host_state_is_restored(self): + class Terminal(io.StringIO): + def isatty(self): + return True + + sentinel = object() + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'script.py' + path.write_text('answer = 1\n') + with patch.dict(vars(builtins), {'_': sentinel}), \ + patch.dict(vars(sys), {'last_type': sentinel, 'last_value': sentinel, + 'last_exc': sentinel, 'last_traceback': sentinel}), \ + patch.object(sys, 'stdin', Terminal()), \ + patch('aiython.repl.AiythonConsole.interact'): + run_script(path, interactive=True) + self.assertIs(builtins._, sentinel) + self.assertTrue(all(getattr(sys, name) is sentinel for name in + ('last_type', 'last_value', 'last_exc', 'last_traceback'))) + run_repl(restore_state=True) + self.assertIs(builtins._, sentinel) + self.assertTrue(all(getattr(sys, name) is sentinel for name in + ('last_type', 'last_value', 'last_exc', 'last_traceback'))) + + def test_explain_requires_input_and_module_syntax_error_keeps_origin(self): + class Terminal(io.StringIO): + def isatty(self): + return True + + with patch.object(sys, 'stdin', Terminal()), contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + main(['--explain']) + with patch.object(cli, 'module_source', side_effect=SyntaxError('missing filename')): + with self.assertRaisesRegex(SyntaxError, 'missing filename'): + main(['-m', 'missing_module']) + + def test_nonembedded_repl_finishes_at_process_exit(self): + source = ('from unittest.mock import patch\n' + 'from aiython.cli import run_repl\n' + "with patch('aiython.repl.AiythonConsole.interact'):\n" + ' run_repl()\n') + result = subprocess.run([sys.executable, '-c', source], + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_repl_safe_path_and_missing_host_main_restore(self): + source = ('import sys\n' + 'from unittest.mock import patch\n' + 'from aiython.cli import run_repl\n' + "sys.modules.pop('__main__', None)\n" + "with patch('aiython.repl.AiythonConsole.interact'):\n" + ' run_repl(restore_state=True)\n') + result = subprocess.run([sys.executable, '-P', '-c', source], + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_interactive_error_on_legacy_version_skips_last_exc(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'error.py' + path.write_text('raise ValueError("boom")\n') + for version in ((3, 11), (3, 14)): + with self.subTest(version=version), patch.object(sys, 'version_info', version), \ + patch('aiython.repl.AiythonConsole.interact'), \ + patch.object(sys, 'excepthook'): + run_script(path, interactive=True) + + def test_host_execution_error_without_script_frame_is_propagated(self): + source = ('from pathlib import Path\n' + 'from unittest.mock import patch\n' + 'from aiython import cli\n' + 'from aiython.models import ResolvedConfig\n' + 'config = ResolvedConfig(None, Path.cwd())\n' + 'code = compile("pass", "", "exec")\n' + 'def fail(*args):\n' + ' raise ValueError("outside script")\n' + 'with patch.object(cli, "exec", fail, create=True), patch.object(cli.atexit, "register"):\n' + ' try:\n' + ' cli.run_script(Path("entry.py"), config=config, compiled_code=code, ' + 'restore_state=False)\n' + ' except ValueError as error:\n' + ' assert str(error) == "outside script"\n' + ' else:\n' + ' raise AssertionError("host error was swallowed")\n') + result = subprocess.run([sys.executable, '-c', source], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + def test_cli_without_arguments_opens_repl_on_terminal(self): class Terminal(io.StringIO): def isatty(self): diff --git a/tests/test_native_bridge.py b/tests/test_native_bridge.py index 741727a..fcb58e0 100644 --- a/tests/test_native_bridge.py +++ b/tests/test_native_bridge.py @@ -1,13 +1,612 @@ """Bridge contracts that also run without a patched CPython interpreter.""" +import ast import sys +import gc +import weakref +from types import SimpleNamespace import unittest +from pathlib import Path +from unittest.mock import Mock, patch -from aiython.native_bridge import NativeTypeBridge +from aiython.models import ProfileConfig, RecoveryDecision, ResolvedConfig +from aiython.frontend import parse +from aiython import native_bridge +from aiython.native_bridge import NativeTypeBridge, vm_available +from aiython.runtime import Runtime from aiython.type_constraints import TypeViolation class NativeBridgeTests(unittest.TestCase): + def test_callback_store_initializes_module_scope_without_call_event(self): + bridge = NativeTypeBridge() + source = ('value: int = 1\n' + 'bridge.before_store(__import__("inspect").currentframe(), ' + '"value", "invalid")\n') + namespace = {'bridge': bridge} + with self.assertRaises(TypeViolation): + exec(bridge.compile_source(source, ''), namespace) + self.assertIs(bridge.types._module_scopes[id(namespace)][0], namespace) + bridge.before_store(__import__('inspect').currentframe(), 'unknown', 1) + bridge.on_call(__import__('inspect').currentframe()) + bridge.on_return(__import__('inspect').currentframe(), 1) + + def test_native_source_detects_dynamic_type_boundaries(self): + runtime = Runtime(ResolvedConfig(None, Path.cwd())) + try: + for source in ('answer = __builtins__\n', + 'def answer(value: int):\n return value\n'): + with self.subTest(source=source): + self.assertFalse(runtime.bridge.native_source(parse(source, ''))) + if hasattr(__import__('ast'), 'TypeAlias'): + self.assertFalse(runtime.bridge.native_source( + parse('type Answer = int\n', ''))) + parameterized = parse('pass\n', '') + parameterized.tree.body[0].type_params = [object()] + self.assertFalse(runtime.bridge.native_source(parameterized)) + with patch.object(ast, 'TypeAlias', ast.Pass, create=True): + self.assertFalse(runtime.bridge.native_source( + parse('pass\n', ''))) + finally: + runtime.capabilities.close() + + def test_vm_probe_and_scoped_hook_lifecycle_with_emulated_event(self): + def emit_probe(code, namespace): + unrelated = SimpleNamespace(f_code=SimpleNamespace(co_filename='')) + sys._aiython_before_store(unrelated, 'other', 1) + frame = SimpleNamespace(f_code=SimpleNamespace(co_filename='')) + sys._aiython_before_store(frame, 'probe_value', 1) + + previous_availability = native_bridge._VM_AVAILABLE + previous_hook = getattr(sys, '_aiython_before_store', None) + try: + sys._aiython_before_store = lambda frame, name, value: None + with patch.object(native_bridge, '_VM_AVAILABLE', None), \ + patch.object(native_bridge, 'exec', side_effect=emit_probe, create=True): + self.assertTrue(native_bridge.vm_available()) + bridge = NativeTypeBridge() + with bridge.installed(): + self.assertIs(sys._aiython_before_store.__self__, bridge) + finally: + native_bridge._VM_AVAILABLE = previous_availability + if previous_hook is None: + vars(sys).pop('_aiython_before_store', None) + else: + sys._aiython_before_store = previous_hook + + def test_vm_availability_double_check_handles_concurrent_probe(self): + class ConcurrentProbe: + def __enter__(self): + native_bridge._VM_AVAILABLE = True + + def __exit__(self, kind, error, traceback): + return False + + with patch.object(native_bridge, '_VM_AVAILABLE', None), \ + patch.object(native_bridge, '_VM_LOCK', ConcurrentProbe()): + self.assertTrue(native_bridge.vm_available()) + + def test_unregistered_vm_events_do_not_call_previous_handlers(self): + frame = SimpleNamespace(f_code=compile('pass', '', 'exec')) + with patch.object(native_bridge, '_VM_PREVIOUS_STORE', None), \ + patch.object(native_bridge, '_VM_PREVIOUS_CALL', None): + native_bridge._vm_on_call(frame) + native_bridge._vm_before_store(frame, 'value', 1) + + def test_expired_code_cannot_remove_a_new_registry_entry(self): + bridge = NativeTypeBridge() + newer = compile('pass', '', 'exec') + + def register_then_replace(): + old = compile('pass', '', 'exec') + with patch.object(native_bridge, '_VM_HOOK_INSTALLED', True): + native_bridge._register_vm_code(old, bridge, 'old') + key = id(old) + original_reference = native_bridge._VM_CODES[key][0] + native_bridge._VM_CODES[key] = (weakref.ref(newer), bridge, 'new') + return key, original_reference + + key, original_reference = register_then_replace() + try: + gc.collect() + self.assertIsNone(original_reference()) + self.assertEqual(native_bridge._VM_CODES[key][2], 'new') + finally: + native_bridge._VM_CODES.pop(key, None) + + def test_variadic_rebinding_is_checked_locally_and_through_closures(self): + source = '''def outer(*values: int): + def middle(): + def inner(): + nonlocal values + bridge.before_store(__import__('inspect').currentframe(), 'values', ('new',)) + values = ('new',) + return values + return inner + return middle() +def rebind_args(*values: int): + bridge.before_store(__import__('inspect').currentframe(), 'values', ('bad',)) +def rebind_kwargs(**named: int): + bridge.before_store(__import__('inspect').currentframe(), 'named', {'x': 'bad'}) +def grandparent(): + value: int = 1 + def middle(): + def inner(): + nonlocal value + bridge.before_store(__import__('inspect').currentframe(), 'value', 2) + value = 2 + return value + return inner + return middle() +def typed_outer(value: int): + def inner(): + nonlocal value + bridge.before_store(__import__('inspect').currentframe(), 'value', 'bad') + return inner() +def untyped_outer(value): + def inner(): + nonlocal value + bridge.before_store(__import__('inspect').currentframe(), 'value', 'allowed') + value = 'allowed' + return value + return inner() +def no_parameters(): + bridge.on_call(__import__('inspect').currentframe()) +''' + bridge = NativeTypeBridge() + namespace = {'bridge': bridge} + exec(bridge.compile_source(source, ''), namespace) + for action in (lambda: namespace['outer'](1)(), + lambda: namespace['rebind_args'](1), + lambda: namespace['rebind_kwargs'](x=1), + lambda: namespace['typed_outer'](1)): + with self.assertRaises(TypeViolation): + action() + self.assertEqual(namespace['grandparent']()(), 2) + self.assertEqual(namespace['untyped_outer'](1), 'allowed') + namespace['no_parameters']() + bridge.before_mutation(__import__('inspect').currentframe(), 'item', {}, 'key', 1) + + def test_first_final_store_is_allowed_before_binding_exists(self): + source = '''from typing import Final +bridge.before_store(__import__('inspect').currentframe(), 'fixed', 1) +fixed: Final[int] = 1 +''' + bridge = NativeTypeBridge() + namespace = {'bridge': bridge} + exec(bridge.compile_source(source, ''), namespace) + self.assertEqual(namespace['fixed'], 1) + + def test_vm_yield_monitor_fallback_and_exhausted_tool_ids(self): + bridge = NativeTypeBridge() + with patch.object(sys, 'monitoring', None, create=True): + with bridge._yield_monitor(): + pass + busy = SimpleNamespace(events=SimpleNamespace(PY_YIELD=1), + get_tool=lambda number: 'busy') + with patch.object(sys, 'monitoring', busy, create=True): + with self.assertRaisesRegex(RuntimeError, 'No free sys.monitoring tool ID'): + with bridge._yield_monitor(): + pass + events = [] + free = SimpleNamespace( + events=SimpleNamespace(PY_YIELD=8), + get_tool=lambda number: None, + use_tool_id=lambda number, name: events.append(('use', number, name)), + register_callback=lambda number, event, callback: events.append(('callback', callback)), + set_events=lambda number, event: events.append(('events', event)), + free_tool_id=lambda number: events.append(('free', number)), + ) + with patch.object(sys, 'monitoring', free, create=True): + with bridge._yield_monitor(): + self.assertEqual(events[0][0], 'use') + self.assertEqual([entry[0] for entry in events], + ['use', 'callback', 'events', 'events', 'callback', 'free']) + + def test_vm_route_registers_exact_code_for_each_runtime(self): + runtimes = [Runtime(ResolvedConfig(None, Path.cwd()))] + profile = ProfileConfig('default', 'fake', 'fake') + runtimes.append(Runtime(ResolvedConfig(None, Path.cwd(), 'default', + {'default': profile}))) + hooks = {name: getattr(sys, name, None) for name in + ('_aiython_before_store', '_aiython_on_call')} + installed = native_bridge._VM_HOOK_INSTALLED + previous = (native_bridge._VM_PREVIOUS_STORE, native_bridge._VM_PREVIOUS_CALL) + try: + with patch.object(native_bridge, 'vm_available', return_value=True): + for runtime in runtimes: + code = runtime.compile_source('value: int = 1\n', '') + self.assertTrue(runtime.bridge.uses_vm(code)) + self.assertEqual(runtime.bridge._frame_types( + SimpleNamespace(f_code=code)).declarations, {'value': 'int'}) + self.assertEqual(runtime.units[''].source, + 'value: int = 1\n') + finally: + for runtime in runtimes: + runtime.capabilities.close() + native_bridge._VM_HOOK_INSTALLED = installed + native_bridge._VM_PREVIOUS_STORE, native_bridge._VM_PREVIOUS_CALL = previous + for name, value in hooks.items(): + if value is None: + vars(sys).pop(name, None) + else: + setattr(sys, name, value) + + def test_bridge_requires_runtime_to_select_managed_code(self): + bridge = NativeTypeBridge() + unit = parse('value: int = 1\n', '') + with self.assertRaisesRegex(RuntimeError, 'requires a Runtime'): + bridge.native_source(unit) + with self.assertRaisesRegex(RuntimeError, 'requires a Runtime'): + bridge.prepare_unit(unit) + + def test_scoped_native_hooks_restore_prior_sys_callbacks(self): + bridge = NativeTypeBridge() + source = 'value: int = 1\nvalue = "invalid"\n' + code = bridge.compile_source(source, '') + missing = object() + previous = {name: getattr(sys, name, missing) for name in bridge._HOOKS} + if vm_available(): + with bridge.installed(): + with self.assertRaises(TypeViolation): + exec(code, {}) + else: + with self.assertRaisesRegex(RuntimeError, 'requires a CPython build'): + with bridge.installed(): + pass + for name, value in previous.items(): + self.assertIs(getattr(sys, name, missing), value) + + def test_vm_dispatch_during_interpreter_shutdown_is_safe(self): + frame = SimpleNamespace(f_code=compile('pass', '', 'exec')) + with patch.object(native_bridge, '_VM_LOCK', None): + native_bridge._vm_on_call(frame) + native_bridge._vm_before_store(frame, 'value', 1) + + def test_yield_and_generator_return_callbacks_enforce_contracts(self): + source = '''from typing import Generator +def yielded(value) -> Generator[int, None, None]: + bridge.on_yield(None, None, value) +def bad_annotation(value) -> int: + bridge.on_yield(None, None, value) +def returned(value) -> Generator[int, None, int]: + bridge.on_return(__import__('inspect').currentframe(), value) + yield 1 +''' + bridge = NativeTypeBridge() + namespace = {'bridge': bridge} + exec(bridge.compile_source(source, ''), namespace) + namespace['yielded'](1) + with self.assertRaises(TypeViolation): + namespace['yielded']('invalid') + with self.assertRaises(TypeViolation): + namespace['bad_annotation'](1) + self.assertEqual(list(namespace['returned'](1)), [1]) + with self.assertRaises(TypeViolation): + list(namespace['returned']('invalid')) + bridge.on_yield(None, None, 'outside tracked code') + + def test_callbacks_reject_invalid_parameters_returns_globals_and_mutations(self): + source = '''from typing import Final, Generator +answer: int = 1 +fixed: Final[int] = 1 +class Box: + value: int + def __init__(self): + self.value = 1 +def parameter(value: int): + bridge.on_call(__import__('inspect').currentframe()) + bridge.before_store(__import__('inspect').currentframe(), 'value', 'invalid') +def returned(value) -> int: + bridge.on_return(__import__('inspect').currentframe(), value) +def change_global(value): + global answer + bridge.before_store(__import__('inspect').currentframe(), 'answer', value) + answer = value +def change_final(): + global fixed + bridge.before_store(__import__('inspect').currentframe(), 'fixed', 2) + fixed = 2 +def change_attribute(box, value): + bridge.before_mutation(__import__('inspect').currentframe(), 'attr', box, 'value', value) + box.value = value +def outer(): + current: int = 1 + def change(value): + nonlocal current + bridge.before_store(__import__('inspect').currentframe(), 'current', value) + current = value + return change +''' + bridge = NativeTypeBridge() + namespace = {'bridge': bridge} + exec(bridge.compile_source(source, ''), namespace) + with self.assertRaises(TypeViolation): + namespace['parameter']('invalid') + with self.assertRaises(TypeViolation): + namespace['parameter'](1) + with self.assertRaises(TypeViolation): + namespace['returned']('invalid') + namespace['returned'](1) + with self.assertRaises(TypeViolation): + namespace['change_global']('invalid') + namespace['change_global'](2) + self.assertEqual(namespace['answer'], 2) + with self.assertRaises(TypeViolation): + namespace['change_final']() + box = namespace['Box']() + with self.assertRaises(TypeViolation): + namespace['change_attribute'](box, 'invalid') + namespace['change_attribute'](box, 2) + self.assertEqual(box.value, 2) + setter = namespace['outer']() + with self.assertRaises(TypeViolation): + setter('invalid') + setter(3) + + def test_vm_indexes_decorated_nested_class_and_variadic_frames(self): + source = '''def identity(function): + return function +@identity +def checked(value: int, /, *rest: int, named: int, **options: int) -> int: + local: int = value + return local +class Box: + field: int + def method(self, value: int) -> int: + return value +async def async_checked(value: int) -> int: + return value +def outer(): + current: int = 1 + def inner(value: int): + nonlocal current + current = value + return inner +def change_global(value: int): + global answer + answer = value +''' + bridge = NativeTypeBridge() + code = bridge.compile_source(source, '') + self.assertEqual(code, compile(source, '', 'exec', dont_inherit=True)) + frames = bridge.frames + self.assertIs(frames[('', 'checked', 3)], + frames[('', 'checked', 4)]) + self.assertEqual(frames[('', 'checked', 4)].parameters, + {'value': ('int', 'value'), 'rest': ('int', 'args'), + 'named': ('int', 'value'), 'options': ('int', 'kwargs')}) + self.assertEqual(frames[('', 'Box.method', 9)].returns, 'int') + self.assertIn(('', 'async_checked', 11), frames) + self.assertEqual(frames[('', 'outer..inner', 15)] + .outer.declarations['current'], 'int') + self.assertEqual(frames[('', 'change_global', 19)] + .global_names, frozenset({'answer'})) + + def test_vm_routing_only_selects_supported_module_shapes(self): + runtime = Runtime(ResolvedConfig(None, Path.cwd())) + try: + supported = ( + 'value: int = 1\n', + 'value: int\nvalue = 1\n', + '"module docstring"\npass\nvalue: bytes = b"ok"\n', + 'previous = 1\nvalue: int = previous\n', + ) + unsupported = ( + 'value = 1\n', + 'value: list[int] = []\n', + 'value: Other = 1\n', + 'value: int = int("1")\n', + 'value: int = 1\nif True: value = 2\n', + 'value: int = 1\nvalue, other = 2, 3\n', + ) + with patch.object(native_bridge, 'vm_available', return_value=True): + for source in supported: + with self.subTest(source=source): + self.assertTrue(runtime.bridge.vm_source(parse(source, ''))) + for source in unsupported: + with self.subTest(source=source): + self.assertFalse(runtime.bridge.vm_source(parse(source, ''))) + with patch.object(native_bridge, 'vm_available', return_value=False): + self.assertFalse(runtime.bridge.vm_source( + parse('value: int = 1\n', ''))) + finally: + runtime.capabilities.close() + + def test_vm_recovery_compiler_retries_original_statement(self): + class Agent: + def recover(self, request, bridge): + self.exception = request.exception + return RecoveryDecision('complete', 42, True) + + agent = Agent() + profile = ProfileConfig('default', 'fake', 'fake') + config = ResolvedConfig(None, Path.cwd(), 'default', {'default': profile}) + runtime = Runtime(config, agent_factory=lambda _: agent) + try: + source = ('bridge.on_call(__import__("inspect").currentframe())\n' + 'value: int = 1\nanswer = missing_value\n') + unit = parse(source, '') + code = runtime.bridge._compile_vm_recovery(unit) + namespace = {'bridge': runtime.bridge} + exec(code, namespace) + self.assertEqual((namespace['value'], namespace['answer']), (1, 42)) + self.assertIsInstance(agent.exception, NameError) + finally: + runtime.capabilities.close() + + def test_managed_vm_dispatch_is_bound_to_code_identity(self): + first_code = compile('value: int = 1', '', 'exec') + second_code = compile('value: int = 1', '', 'exec') + self.assertEqual(first_code, second_code) + self.assertIsNot(first_code, second_code) + target_code = weakref.ref(first_code) + bridge = Mock() + previous_events = [] + + def previous_store(frame, name, value): + if frame.f_code is target_code(): + previous_events.append(('store', name, value)) + + def previous_call(frame): + if frame.f_code is target_code(): + previous_events.append(('call',)) + original_store = getattr(sys, '_aiython_before_store', None) + original_call = getattr(sys, '_aiython_on_call', None) + installed = native_bridge._VM_HOOK_INSTALLED + saved_previous = (native_bridge._VM_PREVIOUS_STORE, + native_bridge._VM_PREVIOUS_CALL) + try: + native_bridge._VM_HOOK_INSTALLED = False + sys._aiython_before_store = previous_store + sys._aiython_on_call = previous_call + native_bridge._register_vm_code(first_code, bridge, 'info') + self.assertIs(sys._aiython_before_store, native_bridge._vm_before_store) + self.assertIs(sys._aiython_on_call, native_bridge._vm_on_call) + self.assertEqual(native_bridge._vm_entry(first_code)[2], 'info') + self.assertIsNone(native_bridge._vm_entry(second_code)) + frame = SimpleNamespace(f_code=first_code) + native_bridge._vm_on_call(frame) + native_bridge._vm_before_store(frame, 'value', 1) + bridge.on_call.assert_called_once_with(frame) + bridge.before_store.assert_called_once_with(frame, 'value', 1) + self.assertEqual(previous_events, [('call',), ('store', 'value', 1)]) + native_bridge._vm_on_call(SimpleNamespace(f_code=second_code)) + native_bridge._vm_before_store(SimpleNamespace(f_code=second_code), 'value', 2) + self.assertEqual(bridge.on_call.call_count, 1) + self.assertEqual(bridge.before_store.call_count, 1) + bridge.reset_mock() + del frame, first_code + gc.collect() + self.assertFalse(any(entry[1] is bridge for entry in native_bridge._VM_CODES.values())) + finally: + native_bridge._VM_HOOK_INSTALLED = installed + native_bridge._VM_PREVIOUS_STORE, native_bridge._VM_PREVIOUS_CALL = saved_previous + if original_store is None: + vars(sys).pop('_aiython_before_store', None) + else: + sys._aiython_before_store = original_store + if original_call is None: + vars(sys).pop('_aiython_on_call', None) + else: + sys._aiython_on_call = original_call + + def test_callbacks_validate_module_function_closure_and_return(self): + source = '''from typing import Final, Generator +answer: int = 1 +fixed: Final[int] = 1 +class Box: + value: int +def typed(value: int) -> int: + return value +def change_global(value): + global answer + answer = value +def outer(): + current: int = 1 + def change(value): + nonlocal current + current = value + return change +def yields() -> Generator[int, None, None]: + yield 1 +''' + bridge = NativeTypeBridge() + code = bridge.compile_source(source, '') + self.assertEqual(code, compile(source, '', 'exec', dont_inherit=True)) + namespace = {} + exec(code, namespace) + self.assertIn('', bridge.module_types) + + # A module code object is needed for the same frame metadata as VM hooks. + probe = compile('bridge.on_call(__import__("inspect").currentframe())\n' + 'bridge.before_store(__import__("inspect").currentframe(), ' + '"answer", "invalid")', '', 'exec') + with self.assertRaises(TypeViolation): + exec(probe, {'bridge': bridge}) + + function_code = namespace['typed'].__code__ + self.assertEqual(bridge.frames[(function_code.co_filename, + function_code.co_qualname, + function_code.co_firstlineno)].returns, 'int') + + # Use the exact compiled function frame while testing callbacks on stock Python. + callback_source = '''def typed(value: int) -> int: + bridge.on_call(__import__('inspect').currentframe()) + bridge.before_store(__import__('inspect').currentframe(), 'value', value) + bridge.on_return(__import__('inspect').currentframe(), value) +''' + callback_bridge = NativeTypeBridge() + callback_namespace = {'bridge': callback_bridge} + exec(callback_bridge.compile_source(callback_source, ''), callback_namespace) + callback_namespace['typed'](1) + with self.assertRaises(TypeViolation): + callback_namespace['typed']('invalid') + + @unittest.skipUnless(vm_available(), 'requires Aiython VM hooks') + def test_vm_store_and_ai_recovery_share_the_default_execution_path(self): + class Agent: + def __init__(self): + self.errors = [] + + def recover(self, request, bridge): + self.errors.append(request.exception) + return RecoveryDecision('complete', 42, True) + + agent = Agent() + profile = ProfileConfig('default', 'fake', 'fake') + config = ResolvedConfig(None, Path.cwd(), 'default', {'default': profile}) + runtime = Runtime(config, agent_factory=lambda _: agent) + try: + code = runtime.compile_source( + 'value: int = 1\nanswer = missing_value\n', '', entry=True) + self.assertTrue(runtime.bridge.uses_vm(code)) + namespace = {} + exec(code, namespace) + self.assertEqual((namespace['value'], namespace['answer']), (1, 42)) + self.assertEqual(len(agent.errors), 1) + self.assertIsInstance(agent.errors[0], NameError) + checking = Runtime(config, agent_factory=lambda _: agent) + try: + bad = checking.compile_source( + 'value: int = 1\nvalue = "invalid"\n', '', entry=True) + self.assertTrue(checking.bridge.uses_vm(bad)) + with self.assertRaises(TypeViolation): + exec(bad, {}) + self.assertEqual(len(agent.errors), 1) + finally: + checking.capabilities.close() + finally: + runtime.capabilities.close() + + @unittest.skipUnless(vm_available(), 'requires Aiython VM hooks') + def test_default_vm_store_callbacks_keep_runtimes_separate(self): + source = 'value: int = 1\nvalue = "invalid"\n' + filename = '' + first = Runtime(ResolvedConfig(None, Path.cwd())) + second = Runtime(ResolvedConfig(None, Path.cwd())) + try: + first_code = first.compile_source(source, filename) + second_code = second.compile_source(source, filename) + self.assertEqual(first_code, compile(source, filename, 'exec', dont_inherit=True)) + self.assertIsNot(first_code, second_code) + self.assertTrue(first.bridge.uses_vm(first_code)) + self.assertTrue(second.bridge.uses_vm(second_code)) + with patch.object(first.bridge, 'before_store', wraps=first.bridge.before_store) as first_hook, \ + patch.object(second.bridge, 'before_store', wraps=second.bridge.before_store) as second_hook: + with self.assertRaises(TypeViolation): + exec(first_code, {}) + first_calls = first_hook.call_count + self.assertGreater(first_calls, 0) + self.assertEqual(second_hook.call_count, 0) + with self.assertRaises(TypeViolation): + exec(second_code, {}) + self.assertEqual(first_hook.call_count, first_calls) + self.assertGreater(second_hook.call_count, 0) + finally: + first.capabilities.close() + second.capabilities.close() + @unittest.skipUnless(hasattr(sys, 'monitoring'), 'requires Python 3.12+') def test_yield_monitor_rejects_value_and_runs_generator_finally(self): source = '''from typing import Generator diff --git a/tests/test_overhead.py b/tests/test_overhead.py index 8f8d515..e7daaea 100644 --- a/tests/test_overhead.py +++ b/tests/test_overhead.py @@ -16,6 +16,7 @@ from aiython.capabilities import Store, CapabilityResult, Embeddings from aiython.cli import run_script from aiython.models import ProfileConfig, ResolvedConfig +from aiython.native_bridge import vm_available from aiython.providers import LiteLLMProvider from aiython.runtime import Runtime from aiython.type_constraints import ContractCache, TypeViolation, compile_contract @@ -99,7 +100,8 @@ def test_preparation_preserves_native_source_tree_and_type_checks(self): original = compile(source, filename, 'exec', dont_inherit=True) unit = runtime.units[filename] self.assertEqual(compile(unit.tree, filename, 'exec', dont_inherit=True), original) - self.assertNotEqual(code, original) + self.assertEqual(code == original, vm_available()) + self.assertEqual(runtime.bridge.uses_vm(code), vm_available()) with self.assertRaises(TypeViolation): exec(code, {}) finally: @@ -107,7 +109,8 @@ def test_preparation_preserves_native_source_tree_and_type_checks(self): cached_runtime = Runtime(config, stats=True) try: cached_code = cached_runtime.compile_source(source, filename, entry=True) - self.assertEqual(cached_runtime.stats.preparation_cache_hits, 1) + self.assertEqual(cached_runtime.stats.preparation_cache_hits, + 0 if vm_available() else 1) self.assertEqual(compile(cached_runtime.units[filename].tree, filename, 'exec', dont_inherit=True), original) with self.assertRaises(TypeViolation): diff --git a/tests/test_repl.py b/tests/test_repl.py index d36e4a7..7a02e15 100644 --- a/tests/test_repl.py +++ b/tests/test_repl.py @@ -5,6 +5,7 @@ import io from pathlib import Path import unittest +from unittest.mock import patch from aiython.models import ProfileConfig, ResolvedConfig from aiython.repl import AiythonConsole @@ -13,6 +14,48 @@ class InteractiveConsoleTests(unittest.TestCase): + def test_syntax_and_setup_errors_are_reported_without_advancing_cell(self): + with patch.object(self.console, 'compile', side_effect=SyntaxError('bad syntax')): + with patch.object(self.console, 'showsyntaxerror') as report: + self.assertFalse(self.console.runsource('value = 1')) + report.assert_called_once_with('') + with patch.object(self.console, 'compile', side_effect=SyntaxError('bad AI')): + with patch('aiython.repl.parse', side_effect=SyntaxError('bad AI')): + with patch.object(self.console, 'showsyntaxerror') as report: + self.assertFalse(self.console.runsource('choose')) + report.assert_called_once_with('') + with patch.object(self.console, 'compile', side_effect=SyntaxError('bad AI')): + with patch('aiython.repl.parse', side_effect=RuntimeError('parser failed')): + with patch.object(self.console, 'showtraceback') as report: + self.assertFalse(self.console.runsource('choose')) + report.assert_called_once_with() + with patch.object(self.console, 'compile', side_effect=SyntaxError('bad AI')): + with patch('aiython.repl.parse', side_effect=SystemExit(3)): + with self.assertRaises(SystemExit): + self.console.runsource('choose') + for error, reporter in ((SyntaxError('bad transform'), 'showsyntaxerror'), + (RuntimeError('compiler failed'), 'showtraceback')): + with self.subTest(error=error): + with patch.object(self.runtime.bridge, 'prepare_unit', side_effect=error): + with patch.object(self.console, reporter) as report: + self.assertFalse(self.console.runsource('1 + 2')) + if reporter == 'showsyntaxerror': + report.assert_called_once_with('') + else: + report.assert_called_once_with() + with patch.object(self.runtime.bridge, 'prepare_unit', side_effect=SystemExit(4)): + with self.assertRaises(SystemExit): + self.console.runsource('1 + 2') + self.assertEqual(self.console.cell_number, 0) + + def test_native_compiled_cell_runs_without_runtime_binding(self): + code = compile('answer = 3', '', 'exec') + with patch.object(self.runtime.bridge, 'prepare_unit', return_value=(code, True)): + with patch('aiython.repl.bind_runtime') as bind: + self.assertFalse(self.console.runsource('answer = 3')) + bind.assert_not_called() + self.assertEqual(self.namespace['answer'], 3) + def setUp(self): self.had_underscore = '_' in vars(builtins) self.original_underscore = vars(builtins).get('_') diff --git a/tests/test_runtime_edges.py b/tests/test_runtime_edges.py index e691f66..9ca0458 100644 --- a/tests/test_runtime_edges.py +++ b/tests/test_runtime_edges.py @@ -119,6 +119,13 @@ def test_large_source_skips_preparation_cache_and_small_cache_evicts(self): self.manager.compile_source(large, filename) self.assertTrue(large.startswith('x = 1')) + def test_bound_code_cache_can_be_bypassed_and_evicted(self): + source = compile('answer = 1', '', 'exec') + self.manager._bind_compiled(source, '__aiython_runtime__', cache=False) + with patch.object(rt, '_PREPARED_LIMIT', 0): + self.manager._bind_compiled(source, '__aiython_runtime__') + self.assertFalse(self.manager._bound_codes) + def test_dynamic_name_fallback_and_local_lookup(self): transformer = rt.DynamicNames(parse('x = 1', 'test.py')) node = ast.Name('x', ast.Load()) @@ -206,6 +213,51 @@ def test_nonrecoverable_error_gets_location_once_and_missing_origin_passes_throu self.assertTrue(error._aiython_location) self.assertIn(__file__, str(error)) + def test_unscoped_checkpoint_and_python_try_handler(self): + node = self.unit.tree.body[0] + checkpoint = rt.install_checkpoint(self.manager, self.unit, node, + 'unscoped', scoped_retries=False) + tree = ast.fix_missing_locations(ast.Module(body=[checkpoint], type_ignores=[])) + code = compile(tree, self.unit.filename, 'exec') + namespace = {self.unit.runtime_name: self.manager} + with patch.object(self.manager, 'recover', return_value=False) as recover: + exec(code, namespace) + self.assertEqual(recover.call_args.args[2], 1) + + source = '''try: + answer = 1 +except ValueError: + answer = probe() +else: + answer = probe() +finally: + cleaned = True +''' + filename = '' + self.manager.units[filename] = parse(source, filename) + + def probe(): + return self.manager.caller_has_python_handler(inspect.currentframe()) + + namespace = {'probe': probe} + exec(compile(source, filename, 'exec'), namespace) + self.assertTrue(namespace['answer']) + self.assertTrue(namespace['cleaned']) + + def test_recovery_without_profile_reraises_and_retained_scope_is_found(self): + from aiython.typed_runtime import Scope + + runtime = rt.Runtime(ResolvedConfig(None, Path.cwd())) + error = ValueError('original') + with self.assertRaises(ValueError) as caught: + runtime.recover('missing', error) + self.assertIs(caught.exception, error) + frame = inspect.currentframe() + scope = Scope() + runtime.types._module_scopes[id(frame.f_globals)] = (frame.f_globals, scope) + self.assertIs(runtime.recovery_scope(frame), scope) + runtime.capabilities.close() + def test_fallback_attempt_counter_and_origin_traceback(self): self.assertTrue(self.recover(ValueError('failed'), RecoveryDecision('retry'), attempt=None)) try: diff --git a/tests/test_type_constraints_edges.py b/tests/test_type_constraints_edges.py index ab422e0..1b2f3f2 100644 --- a/tests/test_type_constraints_edges.py +++ b/tests/test_type_constraints_edges.py @@ -20,6 +20,186 @@ class ContractEdgeTests(unittest.TestCase): + def test_class_contracts_and_fixed_unpack_are_checked_without_execution(self): + int_contract = tc.compile_contract(int, {}) + self.assertFalse(tc.Contract('class', 'missing', python_type=1) + .accepts_class(int, {})) + self.assertFalse(tc.Contract('class', 'object', python_type=object) + .accepts_class(1, {})) + variable = TypeVar('Variable') + bindings = {} + self.assertTrue(tc.Contract('typevar', 'Variable', (int_contract,), + python_type=variable).accepts_class(int, bindings)) + self.assertIs(bindings[variable], int) + unbounded = TypeVar('Unbounded') + self.assertTrue(tc.Contract('typevar', 'Unbounded', python_type=unbounded) + .accepts_class(str, bindings)) + self.assertIs(bindings[unbounded], str) + + class Private: + _secret: int + + tc.compile_contract(Private, {'Private': Private}).validate(Private()) + compiler = tc.Compiler({}) + packed = compiler.node(ast.parse('Unpack[Packed]', mode='eval').body, + {'Unpack': Unpack, 'Packed': (int_contract,)}) + self.assertEqual(packed.kind, 'unpack_fixed') + starred = compiler.node(ast.Starred(ast.Name('Packed', ast.Load()), ast.Load()), + {'Packed': (int_contract,)}) + self.assertEqual(starred.kind, 'unpack_fixed') + with self.assertRaises(tc.UnsupportedType): + compiler.node(ast.Starred(ast.Name('NotPacked', ast.Load()), ast.Load()), + {'NotPacked': int}) + with self.assertRaises(tc.UnsupportedType): + compiler.generic(Unpack, (int_contract,), 'Unpack[int]', {}) + + def test_descriptor_and_annotation_introspection_edges(self): + self.assertEqual(tc.class_parameters(1), ()) + + def empty_annotator(): + hidden = 1 + + def annotate(format): + return hidden + + del hidden + return annotate + + class Model: + @property + def field(self): + return 1 + + type.__setattr__(Model, '__annotate__', empty_annotator()) + self.assertEqual(tc.annotation_locals(Model), {}) + type.__setattr__(Model, '__annotate__', None) + self.assertEqual(tc.annotation_locals(Model), {}) + + def classdict_annotator(): + __classdict__ = {'field': int} + + def annotate(format): + return __classdict__ + + return annotate + + type.__setattr__(Model, '__annotate__', classdict_annotator()) + self.assertEqual(tc.annotation_locals(Model), {}) + self.assertTrue(tc.descriptor_field(Model, 'field', property)) + self.assertFalse(tc.descriptor_field(Model, 'field', int)) + self.assertTrue(tc.descriptor_field(Model, 'field', "'property'")) + self.assertFalse(tc.descriptor_field(Model, 'field', "'")) + self.assertTrue(tc.descriptor_field(Model, 'field', 'builtins.property')) + self.assertFalse(tc.descriptor_field(Model, 'field', 'list[int]')) + + def test_unsupported_container_shapes_fail_without_consuming_values(self): + namespace = {'typing': typing, 'ChainMap': ChainMap} + self.assertEqual(tc.compile_contract('typing.Collection[int]', namespace) + .schema()['x-python-collection-items']['type'], 'integer') + + class Collection(typing.Collection): + def __len__(self): + return 1 + + def __iter__(self): + yield 1 + + def __contains__(self, item): + return item == 1 + + with self.assertRaises(tc.TypeViolation): + tc.compile_contract('typing.Collection[int]', namespace).validate(Collection()) + + class Mapping(typing.Mapping): + def __iter__(self): + return iter(('a',)) + + def __len__(self): + return 1 + + def __getitem__(self, key): + return 1 + + unsupported = Mapping() + with self.assertRaises(tc.TypeViolation): + tc.compile_contract('typing.Mapping[str, int]', namespace).validate(unsupported) + with self.assertRaises(tc.TypeViolation): + tc.compile_contract('ChainMap[str, int]', namespace).validate(ChainMap(unsupported)) + with self.assertRaises(tc.TypeViolation): + tc.compile_contract('typing.KeysView[str]', namespace).validate(unsupported.keys()) + + class Stream(io.IOBase): + pass + + with self.assertRaises(tc.TypeViolation): + tc.compile_contract('typing.IO[str]', namespace).validate(Stream()) + tc.compile_contract('typing.IO[Any]', namespace | {'Any': Any}).validate(Stream()) + + def test_contract_compiler_rejects_invalid_type_parameters(self): + compiler = tc.Compiler({}) + int_contract = tc.compile_contract(int, {}) + fixed = tc.Contract('unpack_fixed', '*Packed', (int_contract,)) + self.assertIs(compiler.generic(Unpack, (fixed,), 'Unpack[Packed]', {}), fixed) + self.assertEqual(compiler.generic(tuple, (fixed,), 'tuple[*Packed]', {}) + .args, (int_contract,)) + tuple_parameter = TypeVarTuple('TupleParameters') + self.assertEqual(compiler.compile(tuple_parameter, + {tuple_parameter: (int_contract,)}).kind, + 'unpack_fixed') + with self.assertRaises(tc.UnsupportedType): + tc.compile_contract('typing.Callable[int]', {'typing': typing}) + with self.assertRaises(tc.UnsupportedType): + tc.compile_contract('typing.TypeGuard[int, str]', {'typing': typing}) + invalid_generics = ( + (typing.Unpack, (int_contract, int_contract), 'Unpack'), + (typing.IO, (), 'IO'), + (re.Pattern, (int_contract, int_contract), 'Pattern'), + (typing.Hashable, (int_contract,), 'Hashable'), + (tc.dataclasses.InitVar, (int_contract, int_contract), 'InitVar'), + (tuple, (tc.Contract('unpack_any', '*A'), + tc.Contract('unpack_any', '*B')), 'tuple'), + ) + for base, args, label in invalid_generics: + with self.subTest(label=label): + with self.assertRaises(tc.UnsupportedType): + compiler.generic(base, args, label, {}) + self.assertEqual(compiler.generic(typing.Callable, (), 'Callable', {}).kind, + 'callable') + self.assertEqual(tc.compile_contract(tc.dataclasses.KW_ONLY, {}).kind, 'kw_only') + with self.assertRaises(tc.UnsupportedType): + tc.compile_contract(tc.dataclasses.KW_ONLY, {}).validate(1) + + first = TypeVarTuple('First') + second = TypeVarTuple('Second') + with self.assertRaises(tc.UnsupportedType): + compiler.parameter_bindings((first, second), (), {}, 'wrong count') + required = SimpleNamespace(__name__='Required', __default__=getattr(typing, 'NoDefault', None)) + optional = SimpleNamespace(__name__='Optional', __default__=int) + self.assertEqual(compiler.parameter_bindings((first, optional), (), {}, 'wrong count') + ['Optional'].kind, 'int') + self.assertEqual(compiler.parameter_bindings((optional,), (), {}, 'wrong count') + ['Optional'].kind, 'int') + with self.assertRaises(tc.UnsupportedType): + compiler.parameter_bindings((first, required), (), {}, 'wrong count') + with self.assertRaises(tc.UnsupportedType): + compiler.parameter_bindings((first, optional, required), (int_contract,), + {}, 'wrong count') + with self.assertRaises(tc.UnsupportedType): + compiler.parameter_bindings((required,), (int_contract, int_contract), + {}, 'wrong count') + + def test_variadic_generic_parent_keeps_expanded_arguments(self): + parameters = TypeVarTuple('Parameters') + + class Base(Generic[*parameters]): + pass + + class Child(Base[int, str]): + pass + + contract = tc.Compiler({}).class_contract(Child, {}) + self.assertEqual(contract.kind, 'class') + def test_self_field_uses_class_being_validated(self): class Node: next: Self | None diff --git a/tests/test_typed_runtime_edges.py b/tests/test_typed_runtime_edges.py index 98f48f7..3fecd88 100644 --- a/tests/test_typed_runtime_edges.py +++ b/tests/test_typed_runtime_edges.py @@ -1,15 +1,157 @@ import ast +import dataclasses +import enum import inspect +import typing +import weakref from types import SimpleNamespace from typing import ClassVar, Final import unittest +from unittest.mock import patch from aiython.frontend import RUNTIME_NAME +from aiython import typed_runtime as tr from aiython.type_constraints import TypeViolation from aiython.typed_runtime import ExpectedTypes, SCOPE, Scope, TypeRuntime, TypedTransformer class TypeRuntimeEdgeTests(unittest.TestCase): + def test_legacy_functions_expose_explicit_generic_parameters(self): + parameter = typing.TypeVar('__Hidden') + visible = typing.TypeVar('Visible') + + def standalone(value): + return value + + standalone.__type_params__ = (parameter,) + self.runtime.register_function(standalone) + self.assertEqual(self.runtime.function_type_params[standalone.__code__], + (parameter,)) + + class Owner(typing.Generic[parameter, visible]): + def method(self): + return runtime.namespace(inspect.currentframe()) + + runtime = self.runtime + Owner.method.__type_params__ = (parameter,) + self.runtime.register_class(Owner) + namespace = Owner().method() + self.assertIs(namespace['__Hidden'], parameter) + self.assertIs(namespace['_Owner__Hidden'], parameter) + self.assertIs(namespace['Visible'], visible) + self.assertEqual(self.runtime.function_type_params[Owner.method.__code__], + (parameter,)) + self.assertTrue(tr.unconstrained_variadic('*Ts', 'args', {})) + + def test_custom_dataclass_constructor_may_omit_initvar(self): + @dataclasses.dataclass + class Custom: + supplied: dataclasses.InitVar[int] + + def __init__(self): + self.ready = True + + self.runtime.register_class(Custom) + self.assertTrue(Custom().ready) + + def test_method_owner_fallbacks_for_incomplete_mro_and_unrelated_receiver(self): + runtime = self.runtime + + class Owner: + def method(self): + return runtime.method_self_owner(inspect.currentframe()) + + class Other: + method = Owner.method + + runtime.method_owners[Owner.method.__code__] = weakref.WeakSet([Owner]) + self.assertIs(Other().method(), Owner) + + class TypeProxy: + def __call__(self, value): + return type(value) + + def __getattribute__(self, name): + if name == '__getattribute__': + return lambda candidate, attribute: [] + return object.__getattribute__(self, name) + + with patch.object(tr, 'type', TypeProxy(), create=True), \ + patch.object(tr, 'issubclass', lambda candidate, parent: False, create=True): + self.assertIs(Owner().method(), Owner) + + def test_registered_instance_with_unreadable_or_nonmapping_dict(self): + class Unreadable: + def __get__(self, instance, owner): + raise AttributeError('unreadable') + + class NonMapping: + def __get__(self, instance, owner): + return [] + + class First: + __dict__ = Unreadable() + + class Second: + __dict__ = NonMapping() + + self.runtime.register_class(First) + self.runtime.register_class(Second) + frame = inspect.currentframe() + with patch.object(tr.types, 'GetSetDescriptorType', Unreadable): + self.runtime._check_instances(First(), {First}, set(), frame) + with patch.object(tr.types, 'GetSetDescriptorType', NonMapping): + self.runtime._check_instances(Second(), {Second}, set(), frame) + + def test_ambiguous_generic_method_owner_does_not_invent_binding(self): + parameter = typing.TypeVar('Parameter') + + class First(typing.Generic[parameter]): + pass + + class Second(typing.Generic[parameter]): + pass + + def inspect_namespace(value): + return self.runtime.namespace(inspect.currentframe()) + + self.runtime.method_owners[inspect_namespace.__code__] = weakref.WeakSet([First, Second]) + with patch.object(self.runtime, 'method_self_owner', return_value=None): + namespace = inspect_namespace(1) + self.assertNotIn('Parameter', namespace) + + def test_enum_metadata_recursive_guard_and_classmethod_registration(self): + class Choice(enum.Enum): + first: int = 1 + description: str + + self.runtime.register_class(Choice) + value = object() + token = self.runtime._active_instance_checks.set(frozenset({id(value)})) + try: + self.runtime._validate_instance(value, 'value', inspect.currentframe()) + finally: + self.runtime._active_instance_checks.reset(token) + method = classmethod(lambda cls: cls) + self.assertIs(self.runtime.register_function(method), method) + self.assertIs(self.runtime.register_function(42), 42) + + def test_method_owner_is_discovered_when_called_before_registration(self): + class Owner: + def method(self): + return self_runtime.method_self_owner(inspect.currentframe(), discover=True) + + self_runtime = self.runtime + self.assertIs(Owner().method(), Owner) + + def test_final_class_placeholder_cannot_be_reassigned(self): + runtime = self.runtime + with self.assertRaisesRegex(TypeViolation, 'Final binding'): + class Owner: + __aiython_type_scope__ = Scope(final_names={'value'}) + value = runtime.assignment_in(inspect.currentframe(), dataclasses.field(), + 'value') + def setUp(self): self.runtime = TypeRuntime() @@ -156,6 +298,26 @@ class Owner: class TypedTransformerEdgeTests(unittest.TestCase): + def test_legacy_parser_preserves_variadic_and_synthetic_type_parameters(self): + transformer = TypedTransformer() + transformer.body(ast.parse('pass\n').body, + parameters={'values': ('*Ts', 'args')}) + function = ast.parse('def identity(value):\n return value\n').body[0] + function.type_params = [object()] + transformer.visit_FunctionDef(function) + self.assertTrue(function.decorator_list) + self.assertEqual(function.decorator_list[-1].attr, 'register_function') + + def test_docstring_only_class_keeps_scope_exit_and_unmangled_name(self): + tree = ast.parse('class Empty:\n "class documentation"\n') + transformer = TypedTransformer() + transformer.visit(tree) + ast.fix_missing_locations(tree) + compile(tree, '', 'exec') + self.assertIn('exit_scope', ast.unparse(tree)) + transformer.class_name = '_' + self.assertEqual(transformer.attribute_name('__secret'), '__secret') + def test_bare_return_annotation_only_and_named_expression(self): tree = ast.parse('''def answer() -> int: value: int