From 8e823c194c2d13ba25e07a636fc1ca6277d535ac Mon Sep 17 00:00:00 2001 From: Jeremy Howard Date: Wed, 9 Sep 2026 13:16:32 +1000 Subject: [PATCH] Add a Python-independent Rust API for file and notebook editing --- Cargo.toml | 8 +- DEV.md | 30 ++- python/exhash/__init__.py | 186 +------------------ python/exhash/_cli.py | 39 +--- python/exhash/outline.py | 2 +- src/commands.rs | 65 +++++++ src/files.rs | 373 ++++++++++++++++++++++++++++++++++++++ src/lib.rs | 9 +- src/python.rs | 149 +++++++++------ tests/test_exhash.py | 26 +++ 10 files changed, 609 insertions(+), 278 deletions(-) create mode 100644 src/commands.rs create mode 100644 src/files.rs diff --git a/Cargo.toml b/Cargo.toml index d5614a7..3a6e3a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,11 @@ crate-type = ["cdylib", "rlib"] [dependencies] crc32fast = "1" +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["arbitrary_precision"] } +tempfile = "3" regex = "1.7.1" -pyo3 = ">=0.28" +pyo3 = { version = ">=0.28", optional = true } tree-sitter = "0.26.12" tree-sitter-python = "0.25.0" tree-sitter-javascript = "0.25.0" @@ -26,7 +29,8 @@ tree-sitter-zig = "1.1.2" tree-sitter-swift = "0.7.3" [features] -extension-module = ["pyo3/extension-module"] +python = ["dep:pyo3"] +extension-module = ["python", "pyo3/extension-module"] [lints.clippy] too_many_arguments = "allow" diff --git a/DEV.md b/DEV.md index 83defeb..4a84316 100644 --- a/DEV.md +++ b/DEV.md @@ -14,10 +14,12 @@ src/ lib.rs public API, error type, module declarations engine.rs single- and multi-buffer edit engines producing EditResult lnhash.rs lnhash hashing/formatting/parsing - parse.rs command parsing (script, strs, and args modes) + parse.rs compact command parsing (script and args modes) + commands.rs shared structured command fields/parser + files.rs file/cell paths, notebook JSON, views, edit/write orchestration python.rs PyO3 bindings (incl. exhash_argv used by the CLI) python/exhash/ - __init__.py Python wrappers plus file/notebook path resolution and I/O + __init__.py Python validation, result wrappers, and display formatting _cli.py exhash/lnhashview console-script entry points skill.py pyskills entry point exposing exhash APIs for LLM tools tests/ @@ -42,7 +44,9 @@ maturin develop pytest -q ``` -All tests are Python (`tests/`); there are no `cargo test` unit tests. +The existing Python API/CLI regression suite exercises the shared Rust file/cell +implementation. `cargo test` additionally tests the Rust API with no Python feature. +`cargo check --no-default-features` verifies embedding without PyO3. ## Hash verification timing @@ -50,7 +54,15 @@ All tests are Python (`tests/`); there are no `cargo test` unit tests. The `$` (last line) and `%` (whole file) address forms are resolved against the current buffer and do not require hashes. `edit_text_with_sw` exposes configurable shift width for `<` and `>`; `edit_text` defaults to `sw=4`. In CLI and Python file-helper flows, a missing file is treated as empty input only when the parsed command set is valid against an empty buffer (for example `0|0000|a`); otherwise the original file-not-found error is preserved. -Python `file_exhash` resolves optional `path:` and notebook-cell prefixes, loads every referenced buffer, and passes the ordered buffers and commands through one `edit_buffers` binding call. Rust owns validation, buffer state, same- and cross-target transfers, printed marks, diffs, and atomic failure. Python writes successful results and preserves notebook source representation. +Rust `edit_files` resolves optional `path:` and notebook-cell prefixes, loads every +referenced buffer, and calls `edit_buffers_with_sw` once. Rust owns validation, +transfers, diffs, and file/notebook writes. Every command succeeds before writes +begin, but this is not an atomic multi-file transaction: an OS write failure can +leave earlier writes completed. Notebook source form, metadata, outputs, and +trailing newlines are preserved; each notebook is read/written once. JSON uses +arbitrary-precision numbers so unrelated notebook metadata cannot be rounded. +Python `file_exhash` and `cell_exhash` are thin adapters over this core. +File views and `file_exhash` normalize CR, CRLF, and LF line endings on read. Unicode separators remain line content, and Python result wrappers use the Rust engine's original lines so no-op detection and diffs agree. No-op file edits leave the original bytes untouched. `lnhashview` range requests clamp `end` past EOF to the last available line, while invalid `start` values still error. ## Release @@ -79,18 +91,22 @@ No local build is required for release; CI runs the release build, creates a Git ## How the CLIs work -The commands are Python console scripts declared in `[project.scripts]` (`python/exhash/_cli.py`). `exhash` and `exhash-cell` handle argument parsing, atomic file I/O, and delegate compact command parsing and editing to the extension. `lnhashview` and `lnhashview-cell` provide the corresponding address views. `exhash-open` is a fastcore `call_parse` wrapper over the document outline API. +The commands are Python console scripts declared in `[project.scripts]` (`python/exhash/_cli.py`). `exhash` and `exhash-cell` handle argument parsing and delegate compact parsing, +editing, notebook serialization, and atomic replacement to the extension. `lnhashview` and `lnhashview-cell` provide the corresponding address views. `exhash-open` is a fastcore `call_parse` wrapper over the document outline API. ## Command parsing modes The Rust core takes commands three ways: -- Structural (PyO3 `exhash` binding): the Python wrapper validates tuple command specs and passes them through as tuples; `python.rs` builds `Command`/`Subcommand` values directly (`command_from_pyfields`), with no string round-trip. Address strings are parsed by `parse::command_from_parts`; field validation (substitute flags, transliterate counts) is shared with the compact parser via `subst_from_parts`/`translit_from_parts`. Global commands carry their subcommand as a nested tuple; text fields are verbatim, so there is no delimiter choice or escaping anywhere on this path. A trailing `.` line in an `a/i/c` payload is literal text and the binding warns about this common mistake. +- Structural (PyO3 `exhash` binding): the Python wrapper validates tuple command specs and passes them through as tuples; `commands.rs` builds `Command`/`Subcommand` values directly (`command_from_fields`), with no string round-trip. Address strings are parsed by `parse::command_from_parts`; field validation (substitute flags, transliterate counts) is shared with the compact parser via `subst_from_parts`/`translit_from_parts`. Global commands carry their subcommand as a nested tuple; text fields are verbatim, so there is no delimiter choice or escaping anywhere on this path. A trailing `.` line in an `a/i/c` payload is literal text and the binding warns about this common mistake. - Multi-buffer structural (`edit_buffers` binding): Python passes ordered `(target, text)` buffers and target-resolved command tuples. Rust keeps one engine per target and executes the full call, including cross-target `m`/`t`, before returning per-target results. - Compact ex-style strings, where strings are the input medium: - `parse_commands_from_script(&str)`: for script strings; commands are separated by newlines. Single-line `a/i/c` text may be inline; if omitted, following lines up to `.` are used as the text block. - `parse_commands_from_args(&[String], &mut BufRead)`: used by the `exhash` CLI via the `exhash_argv` binding; each arg is a command. Single-line `a/i/c` text may be inline. One command may instead read a multiline text block from stdin through EOF. -File-qualified addresses and notebook cell prefixes are resolved by the Python `file_exhash` wrapper after tuple normalization. The resulting target identifiers are opaque to Rust. +File-qualified addresses and notebook cell prefixes are resolved by `files.rs`. +The public Rust `CommandField` representation supports strings and nested arrays, +so Python and Luau use the same parser without a delimiter/string round trip. +The engine itself still treats target identifiers as opaque strings. Commands preserve newlines in text fields. This is used by `a/i/c` payloads and by `s` pattern/replacement; replacement newlines split lines during editing. Commands without text fields do not take text. In compact strings, substitute parsing keeps Rust regex escapes intact (`\d`, `\w`, etc.) while allowing escaped command delimiters (`\/`); compact transliteration uses `y/src/dst/`. Tuple fields need no escaping at all. diff --git a/python/exhash/__init__.py b/python/exhash/__init__.py index 68fd84f..5051e88 100644 --- a/python/exhash/__init__.py +++ b/python/exhash/__init__.py @@ -1,8 +1,9 @@ "Hash-verified line-addressed text editing. See `exhash.skill` for the workflow guide: view with `lnhashview_*` first, then edit with addresses taken from that view." -import json, re +import re from pathlib import Path from .exhash import line_hash as _line_hash, lnhash as _lnhash, lnhashview as _lnhashview, exhash as _exhash, edit_buffers as _edit_buffers +from .exhash import view_file as _view_file, view_cell as _view_cell, view_cells as _view_cells, edit_files as _edit_files, edit_cell as _edit_cell from fastcore.basics import fail_clean, PrettyString MAXLEN = 180 # Most characters shown per displayed line @@ -33,7 +34,7 @@ def lnhashview(text:str, start:int=None, end:int=None) -> "LnhashView": @fail_clean(*stdexcs) def lnhashview_file(path:str, start:int=None, end:int=None) -> "LnhashView": 'Return lines formatted as space-padded ``lineno|hash|content`` for file at ``path`` (expands ``~``). Optional 1-based ``start``/``end`` filter the range; ``end`` past EOF is clamped.' - return LnhashView(_lnhashview(Path(path).expanduser().read_text(), start, end)) + return LnhashView(_view_file(str(path), start, end)) _NOFIELD = {'d', 'p', 'j', 'sort'} @@ -229,116 +230,6 @@ def _diff_out(res): -def _unescape_path(path): - out, escaped = [], False - for ch in path: - if escaped: - if ch not in ':\\': out.append('\\') - out.append(ch) - escaped = False - elif ch == '\\': escaped = True - else: out.append(ch) - if escaped: out.append('\\') - return ''.join(out) - - -def _split_file_prefix(s): - if _ADDR_RE.match(s): return None, s - escaped = False - for i, ch in enumerate(s): - if escaped: - escaped = False - continue - if ch == '\\': - escaped = True - continue - if ch == ':' and _ADDR_RE.match(s[i + 1:]): - path = _unescape_path(s[:i]) - if not path: raise ValueError('empty filename prefix') - return _norm_path(path), s[i + 1:] - return None, s - - -_CELLPATH_RE = re.compile(r'(.*\.ipynb):([A-Za-z0-9_-]+)') - -def _target_key(target): - path, cell = target - return path if cell is None else f'{path}:{cell}' - - -def _parse_fileaddr(s, default): - s = s.lstrip() - path, rest = _split_file_prefix(s) - target = (path, None) if path else default - if path and (m2 := _CELLPATH_RE.fullmatch(path)): target = (m2.group(1), m2.group(2)) - m = _ADDR_RE.match(rest) - if not m: raise ValueError(f'expected exhash address near {s[:40]!r}') - return target, m.group(0), rest[m.end():] - - -def _parse_file_command(cmd, default): - addr, op, *fields = cmd - src, addr1, rest = _parse_fileaddr(addr, default) - has_comma, addr2 = False, None - if rest.startswith(','): - has_comma = True - src2, addr2, rest = _parse_fileaddr(rest[1:], src) - if src2 != src: raise ValueError('a range must stay within one file or cell') - if rest.strip(): raise ValueError(f'unexpected trailing characters in address: {rest!r}') - parsed = dict(src=src, addr1=addr1, addr2=addr2, has_comma=has_comma, op=op, dest=None, dest_addr=None, local=None) - local_addr = addr1 if addr2 is None else f'{addr1},{addr2}' - if op in ('m', 't'): - dest, dest_addr, tail = _parse_fileaddr(fields[0], src) - if tail.strip(): raise ValueError(f'unexpected trailing characters after destination: {tail!r}') - parsed.update(dest=dest, dest_addr=dest_addr, local=(local_addr, op, dest_addr)) - else: parsed['local'] = (local_addr, op, *fields) - return parsed - - -_UNEXPANDED_RE = re.compile(r'\{[A-Za-z_]\w*\}|\$\{?[A-Za-z_]\w*\}?') - - -def _unexpanded(path): - "A note naming the IPython interpolation left literal in `path`: in a `%%exhash` line, an undefined `{name}`/`$name` is passed through as text, so the path silently becomes nonsense." - m = _UNEXPANDED_RE.search(str(path)) - return f" -- note: {m.group(0)!r} looks like an unexpanded IPython variable (undefined names in a magic line are passed through literally)" if m else "" - - -def _load_buffer(st, target, missing_ok=False): - path, cell = target - if cell is not None: - if path not in st['nbs']: - nbp = Path(path).expanduser() - if not nbp.exists(): raise FileNotFoundError(f'notebook not found: {path}{_unexpanded(path)}') - st['nbs'][path] = json.loads(nbp.read_text()) - c = _find_cell(st['nbs'][path], cell, path) - target = (path, c['id']) - if target not in st['bufs']: - text = _cell_text(c) - st['bufs'][target] = dict(target=target, path=path, cellref=c, trail_nl=text.endswith('\n'), original=text.splitlines(), lines=text.splitlines()) - return st['bufs'][target] - if target in st['bufs']: return st['bufs'][target] - p = Path(path) - try: lines = p.read_text().splitlines() - except FileNotFoundError: - if not missing_ok: raise FileNotFoundError(f'file not found: {path}{_unexpanded(path)} (a new file can only be created with a 0|0000| a/i command)') from None - if not p.parent.exists(): raise FileNotFoundError(f'cannot create {path}: parent directory {p.parent} does not exist{_unexpanded(path)}') from None - lines = [] - st['bufs'][target] = dict(target=target, path=path, cellref=None, original=list(lines), lines=list(lines)) - return st['bufs'][target] - - -def _can_create_missing(parsed): return parsed['addr1'] == '0|0000|' and parsed['op'] in ('a', 'i') - - -def _prepare_file_command(st, parsed): - src = _load_buffer(st, parsed['src'], missing_ok=_can_create_missing(parsed) and parsed['src'][1] is None) - dest = None - if parsed['dest'] is not None: - dest = _load_buffer(st, parsed['dest'], missing_ok=parsed['dest_addr'] == '0|0000|' and parsed['dest'][1] is None) - return (_target_key(src['target']), parsed['local'], _target_key(dest['target']) if dest else None) - - @fail_clean(*stdexcs) def file_exhash(path:str, *cmds:tuple, sw:int=4, inplace:bool=True): r'''Read files and notebook cells, apply file-aware exhash commands, and return per-target results or a combined diff. @@ -371,71 +262,22 @@ def file_exhash(path:str, *cmds:tuple, sw:int=4, inplace:bool=True): ``FileSetEditResult`` is returned with ``files``, ``changed``, ``default_path``, ``res[path]`` (cell targets under ``'path:cellid'``), and ``res.format_diff(context=1)``. ''' - default, st = (_norm_path(path), None), dict(bufs={}, nbs={}) - commands = [_prepare_file_command(st, _parse_file_command(cmd, default)) for cmd in _normalize_cmds(cmds)] - if not st['bufs']: _load_buffer(st, default) - by_key = {_target_key(target): buf for target, buf in st['bufs'].items()} - buffers = [(key, _text_from_lines(buf['lines'])) for key, buf in by_key.items()] - native = _edit_buffers(buffers, commands, sw=sw) - files = {key: FileEditResult(key, by_key[key]['original'], result, cell=by_key[key]['target'][1]) for key, result in native} - for key, result in files.items(): by_key[key]['lines'] = result.lines + native = _edit_files(str(path), _normalize_cmds(cmds), sw=sw, inplace=inplace) + files = {key: FileEditResult(key, result.original_lines, result, cell=cell) for key, cell, result in native} result = FileSetEditResult(files, _norm_path(path)) - if inplace: - nbs_out = {} - for t, buf in st['bufs'].items(): - if buf['original'] == buf['lines']: continue - if buf['cellref'] is None: _write_lines(buf['path'], buf['lines']) - else: - new = '\n'.join(buf['lines']) - if buf['trail_nl'] and new: new += '\n' - c = buf['cellref'] - c['source'] = new.splitlines(keepends=True) if isinstance(c['source'], list) else new - nbs_out[buf['path']] = st['nbs'][buf['path']] - for pth, nb in nbs_out.items(): Path(pth).expanduser().write_text(json.dumps(nb, sort_keys=True, indent=1, ensure_ascii=False) + '\n') - return result._trunc_diff() - return result - - - - -def _find_cell(nb, cell_id, path): - 'The cell in `nb` whose id is ``cell_id`` (exact match or unique prefix).' - cells = [c for c in nb['cells'] if c.get('id','').startswith(cell_id)] - exact = [c for c in cells if c.get('id')==cell_id] - if exact: cells = exact - if not cells: raise KeyError(f'no cell with id {cell_id!r} in {path}') - if len(cells)>1: raise KeyError(f'cell id prefix {cell_id!r} is ambiguous in {path}') - return cells[0] - - -def _load_cell(path, cell_id): - 'Return ``(nb, cell)`` for the cell whose id is ``cell_id`` (exact match or unique prefix).' - nbp = Path(path).expanduser() - if not nbp.exists(): raise FileNotFoundError(f'notebook not found: {path}{_unexpanded(path)}') - nb = json.loads(nbp.read_text()) - return nb, _find_cell(nb, cell_id, path) - - -def _cell_text(cell): - src = cell['source'] - return src if isinstance(src, str) else ''.join(src) + return result._trunc_diff() if inplace else result @fail_clean(*stdexcs) def lnhashview_cell(path:str, cell_id:str, start:int=None, end:int=None) -> "LnhashView": 'Return lines formatted as ``lineno|hash|content`` for the source of notebook cell ``cell_id`` in ipynb file at ``path`` (expands ``~``). ``cell_id`` may be an exact id or unique prefix; optional 1-based ``start``/``end`` filter the range.' - return LnhashView(_lnhashview(_cell_text(_load_cell(path, cell_id)[1]), start, end)) + return LnhashView(_view_cell(str(path), cell_id, start, end)) @fail_clean(*stdexcs) def lnhashview_cells(path:str, *cell_ids:str, start:int=None, end:int=None) -> "LnhashView": 'Return grouped lnhash views for explicit notebook cell ids in the ipynb file at ``path`` (expands ``~``). Each group starts with ``# cell ``; following lines keep normal ``lineno|hash|content`` format.' - out = [] - for cell_id in cell_ids: - _, cell = _load_cell(path, cell_id) - out.append(f"# cell {cell.get('id', cell_id)}") - out += _lnhashview(_cell_text(cell), start, end) - return LnhashView(out) + return LnhashView(_view_cells(str(path), cell_ids, start, end)) @fail_clean(*stdexcs) @@ -453,15 +295,7 @@ def cell_exhash(path:str, cell_id:str, *cmds:tuple, sw:int=4, inplace:bool=True) returns the printed lines as a bare, untruncated ``lnhashview``. Pass ``inplace=False`` to preview instead: the EditResult is returned without touching the file. """ - nb, cell = _load_cell(path, cell_id) - text = _cell_text(cell) - res = exhash(text, cmds, sw=sw) - if not inplace: return res - new = '\n'.join(res['lines']) - if text.endswith('\n') and new: new += '\n' - if new != text: - cell['source'] = new.splitlines(keepends=True) if isinstance(cell['source'], list) else new - Path(path).expanduser().write_text(json.dumps(nb, sort_keys=True, indent=1, ensure_ascii=False) + '\n') - return _diff_out(res) + res = _edit_cell(str(path), cell_id, _normalize_cmds(cmds), sw=sw, inplace=inplace) + return _diff_out(res) if inplace else res from .outline import * diff --git a/python/exhash/_cli.py b/python/exhash/_cli.py index 33bf322..236bbaf 100644 --- a/python/exhash/_cli.py +++ b/python/exhash/_cli.py @@ -1,9 +1,9 @@ "Console-script entry points for exhash tools." -import json, os, re, sys, tempfile +import re, sys from pathlib import Path from fastcore.script import call_parse -from .exhash import exhash_argv as _exhash_argv, lnhashview as _lnhashview +from .exhash import exhash_argv as _exhash_argv, lnhashview as _lnhashview, edit_file_argv as _edit_file_argv, edit_cell_argv as _edit_cell_argv _ADDR_RE = re.compile(r'(?:\$|%|\d+\|[0-9a-fA-F]{4}\|)') @@ -55,20 +55,6 @@ def _read_text_or_die(path): try: return data.decode("utf-8") except UnicodeDecodeError: _die("error: non-UTF8 file rejected") -def _atomic_write(path, content): - p = Path(path).expanduser() - d = str(p.parent) or "." - fd, tmp = tempfile.mkstemp(dir=d, prefix=f".{p.name}.exhash.tmp.") - try: - with os.fdopen(fd, "w", encoding="utf-8") as f: f.write(content) - try: os.chmod(tmp, os.stat(p).st_mode) - except FileNotFoundError: pass - os.replace(tmp, p) - except BaseException: - try: os.unlink(tmp) - except OSError: pass - raise - def _needs_text_block(cmd): "True if `cmd` is an a/i/c command with no inline text (so it reads a stdin block)." m = _ADDR_RE.match(cmd) @@ -115,14 +101,8 @@ def exhash_main(argv=None): return text_block = sys.stdin.read() if (not sys.stdin.isatty() or any(_needs_text_block(c) for c in cmds)) else "" - try: text = _read_text_or_die(file) - except FileNotFoundError: text = "" - try: res = _exhash_argv(text, cmds, text_block, sw) - except ValueError as e: _die(f"error: {e}", 2) - new_text = "\n".join(res.lines) + "\n" if res.lines else "" - if not dry_run: - try: _atomic_write(file, new_text) - except OSError as e: _die(f"error: failed to write {file}: {e}") + try: res = _edit_file_argv(file, cmds, text_block, sw, not dry_run) + except (ValueError, OSError) as e: _die(f"error: {e}", 2) diff = res.format_diff(1) if diff: sys.stdout.write(diff) @@ -180,16 +160,7 @@ def exhash_cell_main(argv=None): if len(argv)-i < 2: _die(EXHASH_CELL_USAGE, 2) file, cell_id, cmds = argv[i], argv[i+1], argv[i+2:] text_block = sys.stdin.read() if any(_needs_text_block(c) for c in cmds) else "" - try: - from . import _cell_text, _load_cell - nb, cell = _load_cell(file, cell_id) - text = _cell_text(cell) - res = _exhash_argv(text, cmds, text_block, sw) - new = '\n'.join(res.lines) - if text.endswith('\n') and new: new += '\n' - if new != text and not dry_run: - cell['source'] = new.splitlines(keepends=True) if isinstance(cell['source'], list) else new - _atomic_write(file, json.dumps(nb, sort_keys=True, indent=1, ensure_ascii=False) + '\n') + try: res = _edit_cell_argv(file, cell_id, cmds, text_block, sw, not dry_run) except Exception as e: _die(f"error: {e}", 2) if (diff := res.format_diff(1)): sys.stdout.write(diff) diff --git a/python/exhash/outline.py b/python/exhash/outline.py index cd4b690..5efff3a 100644 --- a/python/exhash/outline.py +++ b/python/exhash/outline.py @@ -284,7 +284,7 @@ def _addressed(self, nums, lnhashs): def _parse_nb(path): "Build an `NbSection` tree from the ipynb file at `path`: md-cell headings over cells" - from . import _cell_text + def _cell_text(c): return c["source"] if isinstance(c["source"], str) else "".join(c["source"]) path = Path(path).expanduser() nb = json.loads(path.read_text()) cells = [(c.get('id',''), c['cell_type'], _cell_text(c).rstrip('\n')) for c in nb['cells']] diff --git a/src/commands.rs b/src/commands.rs new file mode 100644 index 0000000..a077cc7 --- /dev/null +++ b/src/commands.rs @@ -0,0 +1,65 @@ +//! Structured commands shared by language bindings. +use crate::parse::{ + command_from_parts, parse_buffer_destination_address, parse_destination_address, parse_optional_usize, split_text_payload, subst_from_parts, + translit_from_parts, +}; +use crate::{Command, EditError, Subcommand}; + +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(untagged)] +pub enum CommandField { Str(String), Seq(Vec) } + +pub fn command_from_fields(fields: &[CommandField]) -> Result { + let [CommandField::Str(addr), CommandField::Str(op), rest @ ..] = fields else { + return Err(EditError::new("command must start with (address, op) strings")); + }; + command_from_parts(addr, subcommand_from_fields(op, rest)?) +} + +pub fn buffer_command_from_fields(fields: &[CommandField]) -> Result { + let [CommandField::Str(addr), CommandField::Str(op), CommandField::Str(dest)] = fields else { return command_from_fields(fields); }; + if !matches!(op.as_str(), "m" | "t") { return command_from_fields(fields); } + let op_char = if op == "m" { 'm' } else { 't' }; + let dest = parse_buffer_destination_address(dest, op_char)?; + let sub = if op == "m" { Subcommand::Move { dest } } else { Subcommand::Copy { dest } }; + command_from_parts(addr, sub) +} + +fn str_fields<'a>(op: &str, fields: &'a [CommandField]) -> Result, EditError> { + fields + .iter() + .map(|f| match f { CommandField::Str(s) => Ok(s.as_str()), CommandField::Seq(_) => Err(EditError::new(format!("{op} fields must be strings"))) }) + .collect() +} + +fn subcommand_from_fields(op: &str, fields: &[CommandField]) -> Result { + if let "g" | "g!" | "v" = op { + let [CommandField::Str(pattern), CommandField::Seq(inner)] = fields else { + return Err(EditError::new(format!("{op} takes (pattern, (subcommand, ...))"))); + }; + let [CommandField::Str(iop), irest @ ..] = inner.as_slice() else { return Err(EditError::new("global subcommand must start with an op string")); }; + if matches!(iop.as_str(), "g" | "g!" | "v") { return Err(EditError::new("global commands cannot nest")); } + return Ok(Subcommand::Global { invert: op != "g", pattern: pattern.clone(), cmd: Box::new(subcommand_from_fields(iop, irest)?) }); + } + let f = str_fields(op, fields)?; + match (op, f.as_slice()) { + ("d", []) => Ok(Subcommand::Delete), + ("p", []) => Ok(Subcommand::Print), + ("j", []) => Ok(Subcommand::Join), + ("sort", []) => Ok(Subcommand::Sort), + ("a", [text]) => Ok(Subcommand::Append(split_text_payload(text))), + ("i", [text]) => Ok(Subcommand::Insert(split_text_payload(text))), + ("c", [text]) => Ok(Subcommand::Change(split_text_payload(text))), + ("s", [pat, rep]) => Ok(Subcommand::Substitute(subst_from_parts((*pat).into(), (*rep).into(), "")?)), + ("s", [pat, rep, flags]) => Ok(Subcommand::Substitute(subst_from_parts((*pat).into(), (*rep).into(), flags)?)), + ("y", [source, dest]) => { + let (source, dest) = translit_from_parts((*source).into(), (*dest).into())?; + Ok(Subcommand::Transliterate { source, dest }) + } + ("m", [dest]) => Ok(Subcommand::Move { dest: parse_destination_address(dest, 'm')? }), + ("t", [dest]) => Ok(Subcommand::Copy { dest: parse_destination_address(dest, 't')? }), + (">", rest @ ([] | [_])) => Ok(Subcommand::Indent { levels: parse_optional_usize(rest.first().copied().unwrap_or(""))? }), + ("<", rest @ ([] | [_])) => Ok(Subcommand::Dedent { levels: parse_optional_usize(rest.first().copied().unwrap_or(""))? }), + _ => Err(EditError::new(format!("invalid tuple command: {op:?} with {} field(s)", f.len()))), + } +} diff --git a/src/files.rs b/src/files.rs new file mode 100644 index 0000000..ffefd8f --- /dev/null +++ b/src/files.rs @@ -0,0 +1,373 @@ +//! File and notebook orchestration, independent of Python. All commands are +//! validated/applied in memory before any writes; this is not a multi-file transaction. +use crate::commands::buffer_command_from_fields; +use crate::{BufferCommand, Command, CommandField, EditError, EditResult, edit_buffers_with_sw, edit_text_with_sw}; +use regex::Regex; +use serde_json::Value; +use std::{collections::BTreeMap, fs, io, path::Path, sync::LazyLock}; + +#[derive(Debug)] +pub enum FileError { + Io(io::Error), + Edit(EditError), + Cell(String), + Json(serde_json::Error), +} +impl std::fmt::Display for FileError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(e) => e.fmt(f), + Self::Edit(e) => e.fmt(f), + Self::Cell(e) => e.fmt(f), + Self::Json(e) => e.fmt(f), + } + } +} +impl std::error::Error for FileError {} +impl From for FileError { fn from(e: io::Error) -> Self { Self::Io(e) } } +impl From for FileError { fn from(e: EditError) -> Self { Self::Edit(e) } } +impl From for FileError { fn from(e: serde_json::Error) -> Self { Self::Json(e) } } +type Result = std::result::Result; +fn invalid(message: impl Into) -> FileError { EditError::new(message).into() } + +/// Expand a current-user home prefix and normalize redundant separators/dots, +/// without resolving symlinks or making a relative path absolute. +pub fn normalize_path(path: &str) -> Result { + let expanded = if path == "~" || path.starts_with("~/") { + let home = std::env::var("HOME").map_err(|_| invalid("cannot expand ~: HOME is not set"))?; + format!("{home}{}", &path[1..]) + } else { path.to_owned() }; + let normalized: std::path::PathBuf = Path::new(&expanded).components().filter(|c| !matches!(c, std::path::Component::CurDir)).collect(); + Ok(if normalized.as_os_str().is_empty() { ".".into() } else { normalized.to_string_lossy().into_owned() }) +} + +fn unexpanded(path: &str) -> String { + static RE: LazyLock = LazyLock::new(|| Regex::new(r"\{[A-Za-z_]\w*\}|\$\{?[A-Za-z_]\w*\}?").unwrap()); + RE.find(path) + .map(|m| format!(" -- note: {:?} looks like an unexpanded IPython variable (undefined names in a magic line are passed through literally)", m.as_str())) + .unwrap_or_default() +} +fn missing(message: String) -> FileError { io::Error::new(io::ErrorKind::NotFound, message).into() } + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Target { path: String, cell: Option } +impl Target { fn key(&self) -> String { match &self.cell { Some(id) => format!("{}:{id}", self.path), None => self.path.clone() } } } +fn unescape(path: &str) -> String { + let mut out = String::new(); + let mut chars = path.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + if let Some(next) = chars.next() { + if next != ':' && next != '\\' { out.push('\\'); } + out.push(next); + } else { out.push('\\'); } + } + out +} +fn address(input: &str, default: &Target) -> Result<(Target, String, String)> { + static ADDR: LazyLock = LazyLock::new(|| Regex::new(r"^(?:\$|%|\d+\|[0-9a-fA-F]{4}\|)").unwrap()); + static CELL: LazyLock = LazyLock::new(|| Regex::new(r"^(.*\.ipynb):([A-Za-z0-9_-]+)$").unwrap()); + let input = input.trim_start(); + let mut target = default.clone(); + let mut rest = input; + if !ADDR.is_match(input) { + let mut escaped = false; + for (i, c) in input.char_indices() { + if escaped { + escaped = false; + continue; + } + if c == '\\' { + escaped = true; + continue; + } + if c != ':' || !ADDR.is_match(&input[i + 1..]) { continue; } + if i == 0 { return Err(invalid("empty filename prefix")); } + let path = normalize_path(&unescape(&input[..i]))?; + target = match CELL.captures(&path) { Some(m) => Target { path: m[1].into(), cell: Some(m[2].into()) }, None => Target { path, cell: None } }; + rest = &input[i + 1..]; + break; + } + } + let m = ADDR.find(rest).ok_or_else(|| invalid(format!("expected exhash address near {:?}", input.chars().take(40).collect::())))?; + Ok((target, m.as_str().into(), rest[m.end()..].into())) +} + +/// Resolve a cell by exact ID or unique prefix. Return its index, not a copy. +fn cell_index(nb: &Value, id: &str, path: &str) -> Result { + let cells = nb.get("cells").and_then(Value::as_array).ok_or_else(|| invalid("notebook cells must be an array"))?; + let matches: Vec<_> = cells.iter().enumerate().filter(|(_, c)| c.get("id").and_then(Value::as_str).unwrap_or("").starts_with(id)).collect(); + if let Some((i, _)) = matches.iter().find(|(_, c)| c.get("id").and_then(Value::as_str) == Some(id)) { return Ok(*i); } + match matches.as_slice() { + [(i, _)] => Ok(*i), + [] => Err(FileError::Cell(format!("no cell with id {id:?} in {path}"))), + _ => Err(FileError::Cell(format!("cell id prefix {id:?} is ambiguous in {path}"))), + } +} +fn cell_text(cell: &Value) -> Result { + match cell.get("source") { + Some(Value::String(s)) => Ok(s.clone()), + Some(Value::Array(lines)) => { + lines.iter().map(|v| v.as_str().ok_or_else(|| invalid("cell source must contain strings"))).collect::>>().map(|s| s.concat()) + } + _ => Err(invalid("cell source must be a string or array of strings")), + } +} +fn read_notebook(path: &str) -> Result { + let text = fs::read_to_string(path) + .map_err(|e| if e.kind() == io::ErrorKind::NotFound { missing(format!("notebook not found: {path}{}", unexpanded(path))) } else { e.into() })?; + Ok(serde_json::from_str(&text)?) +} +fn set_source(cell: &mut Value, text: String) { + cell["source"] = + if cell["source"].is_array() { Value::Array(text.split_inclusive('\n').map(|s| Value::String(s.into())).collect()) } else { Value::String(text) }; +} +fn notebook_text(nb: &Value) -> Result> { + // Match Jupyter/Python's sorted, one-space-indented, UTF-8 JSON layout. + let mut sorted = nb.clone(); + sorted.sort_all_objects(); + let mut out = Vec::new(); + let mut serializer = serde_json::Serializer::with_formatter(&mut out, serde_json::ser::PrettyFormatter::with_indent(b" ")); + serde::Serialize::serialize(&sorted, &mut serializer)?; + out.push(b'\n'); + Ok(out) +} +fn output_text(lines: &[String], trailing_newline: bool) -> String { + let mut text = lines.join("\n"); + if trailing_newline && !lines.is_empty() { text.push('\n'); } + text +} + +// Match Python's universal-newline file reads without splitting Unicode content. +fn read_text(path: impl AsRef) -> io::Result { fs::read_to_string(path).map(|s| s.replace("\r\n", "\n").replace('\r', "\n")) } + +pub fn view_file(path: &str, start: Option, end: Option) -> Result> { + let text = read_text(normalize_path(path)?)?; + Ok(crate::lnhashview(&text.lines().collect::>(), start, end)?) +} +pub fn view_cell(path: &str, id: &str, start: Option, end: Option) -> Result> { + let path = normalize_path(path)?; + let nb = read_notebook(&path)?; + let text = cell_text(&nb["cells"][cell_index(&nb, id, &path)?])?; + Ok(crate::lnhashview(&text.lines().collect::>(), start, end)?) +} +pub fn view_cells(path: &str, ids: &[String], start: Option, end: Option) -> Result> { + if ids.is_empty() { return Ok(vec![]); } + let path = normalize_path(path)?; + let nb = read_notebook(&path)?; + let mut out = vec![]; + for id in ids { + let cell = &nb["cells"][cell_index(&nb, id, &path)?]; + out.push(format!("# cell {}", cell["id"].as_str().unwrap_or(id))); + out.extend(crate::lnhashview(&cell_text(cell)?.lines().collect::>(), start, end)?); + } + Ok(out) +} + +/// The edited state of one file or cell. `path` is its resolved target key. +#[derive(Debug)] +pub struct FileEdit { + pub path: String, + pub cell: Option, + pub original_text: String, + pub result: EditResult, +} +impl FileEdit { + pub fn changed(&self) -> bool { self.original_text.lines().ne(self.result.lines.iter().map(String::as_str)) } + pub fn format_diff(&self, context: usize) -> String { + let diff = self.result.format_diff(&self.original_text.lines().collect::>(), context); + if self.changed() { diff.replacen("--- original\n+++ modified\n", &format!("--- {}\n+++ {}\n", self.path, self.path), 1) } else { diff } + } +} +struct Buffer { target: Target, text: String, cell_index: Option } +#[derive(Default)] +struct Files { buffers: Vec, notebooks: BTreeMap } +impl Files { + fn load(&mut self, mut target: Target, missing_ok: bool) -> Result { + let (text, index) = if let Some(id) = &target.cell { + if !self.notebooks.contains_key(&target.path) { self.notebooks.insert(target.path.clone(), read_notebook(&target.path)?); } + let nb = &self.notebooks[&target.path]; + let index = cell_index(nb, id, &target.path)?; + let cell = &nb["cells"][index]; + target.cell = Some(cell["id"].as_str().ok_or_else(|| invalid("cell id must be a string"))?.into()); + (cell_text(cell)?, Some(index)) + } else { + if self.buffers.iter().any(|b| b.target == target) { return Ok(target.key()); } + let text = match read_text(&target.path) { + Ok(text) => text, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + let path = &target.path; + if !missing_ok { + return Err(missing(format!("file not found: {path}{} (a new file can only be created with a 0|0000| a/i command)", unexpanded(path)))); + } + let parent = Path::new(path).parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new(".")); + if !parent.exists() { + return Err(missing(format!("cannot create {path}: parent directory {} does not exist{}", parent.display(), unexpanded(path)))); + } + String::new() + } + Err(e) => return Err(e.into()), + }; + (text, None) + }; + let key = target.key(); + if !self.buffers.iter().any(|b| b.target == target) { self.buffers.push(Buffer { target, text, cell_index: index }); } + Ok(key) + } +} + +/// Apply structured file-aware commands, optionally writing changed files/cells. +/// Cell ID prefixes are resolved once; each notebook is read/written only once. +pub fn edit_files(path: &str, commands: &[Vec], sw: usize, inplace: bool) -> Result> { + let default = Target { path: normalize_path(path)?, cell: None }; + let mut files = Files::default(); + let mut parsed = vec![]; + for fields in commands { + let [CommandField::Str(addr), CommandField::Str(op), rest @ ..] = fields.as_slice() else { + return Err(invalid("command must start with (address, op) strings")); + }; + let (src, addr1, tail) = address(addr, &default)?; + let (local_addr, tail) = if let Some(tail) = tail.strip_prefix(',') { + let (src2, addr2, tail) = address(tail, &src)?; + if src != src2 { return Err(invalid("a range must stay within one file or cell")); } + (format!("{addr1},{addr2}"), tail) + } else { (addr1.clone(), tail) }; + if !tail.trim().is_empty() { return Err(invalid(format!("unexpected trailing characters in address: {tail:?}"))); } + let mut local = vec![CommandField::Str(local_addr), CommandField::Str(op.clone())]; + let (dest, dest_addr) = if matches!(op.as_str(), "m" | "t") { + let [CommandField::Str(dest)] = rest else { return Err(invalid("m/t requires one destination address")); }; + let (dest, dest_addr, tail) = address(dest, &src)?; + if !tail.trim().is_empty() { return Err(invalid(format!("unexpected trailing characters after destination: {tail:?}"))); } + local.push(CommandField::Str(dest_addr.clone())); + (Some(dest), Some(dest_addr)) + } else { + local.extend_from_slice(rest); + (None, None) + }; + let command = buffer_command_from_fields(&local)?; + let target = files.load(src, addr1 == "0|0000|" && matches!(op.as_str(), "a" | "i"))?; + let destination = dest.map(|d| files.load(d, dest_addr.as_deref() == Some("0|0000|"))).transpose()?; + parsed.push(BufferCommand { target, command, destination }); + } + if files.buffers.is_empty() { files.load(default, false)?; } + let buffers = files.buffers.iter().map(|b| (b.target.key(), output_text(&b.text.lines().map(str::to_owned).collect::>(), true))).collect(); + let results = edit_buffers_with_sw(buffers, parsed, sw)?; + let mut out = vec![]; + let mut changed_notebooks = std::collections::BTreeSet::new(); + for (buffer, edited) in files.buffers.iter().zip(results) { + let result = FileEdit { path: edited.target, cell: buffer.target.cell.clone(), original_text: edited.original_text, result: edited.result }; + if inplace && result.changed() { + if let Some(index) = buffer.cell_index { + let cell = &mut files.notebooks.get_mut(&buffer.target.path).unwrap()["cells"][index]; + set_source(cell, output_text(&result.result.lines, buffer.text.ends_with('\n'))); + changed_notebooks.insert(buffer.target.path.clone()); + } else { fs::write(&buffer.target.path, output_text(&result.result.lines, true))?; } + } + out.push(result); + } + for path in changed_notebooks { fs::write(&path, notebook_text(&files.notebooks[&path])?)?; } + Ok(out) +} + +/// Edit a single notebook cell using already parsed, local commands. +pub fn edit_cell(path: &str, id: &str, commands: &[Command], sw: usize, inplace: bool) -> Result { + edit_cell_with_writer(path, id, commands, sw, inplace, |path, text| fs::write(path, text)) +} + +pub(crate) fn edit_cell_with_writer( + path: &str, + id: &str, + commands: &[Command], + sw: usize, + inplace: bool, + write: impl FnOnce(&str, &[u8]) -> io::Result<()>, +) -> Result { + let path = normalize_path(path)?; + let mut nb = read_notebook(&path)?; + let index = cell_index(&nb, id, &path)?; + let cell = &mut nb["cells"][index]; + let text = cell_text(cell)?; + let result = edit_text_with_sw(&text, commands, sw)?; + let resolved_id = cell["id"].as_str().unwrap_or(id).to_owned(); + let new = output_text(&result.lines, text.ends_with('\n')); + if inplace && new != text { + set_source(cell, new); + write(&path, ¬ebook_text(&nb)?)?; + } + Ok(FileEdit { path: format!("{path}:{resolved_id}"), cell: Some(resolved_id), original_text: text, result }) +} + +/// CLI replacement semantics: write a same-directory temporary file, preserve +/// existing permissions, then rename. Python's in-place API keeps its write semantics. +pub(crate) fn atomic_write(path: &str, content: &[u8]) -> io::Result<()> { + use io::Write; + let path = Path::new(path); + let parent = path.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new(".")); + let mut temp = tempfile::NamedTempFile::new_in(parent)?; + temp.write_all(content)?; + match fs::metadata(path) { + Ok(meta) => temp.as_file().set_permissions(meta.permissions())?, + Err(e) if e.kind() == io::ErrorKind::NotFound => (), + Err(e) => return Err(e), + } + temp.persist(path).map_err(|e| e.error)?; + Ok(()) +} + +/// Compact single-file CLI commands, with binary rejection and atomic replacement. +pub fn edit_file_argv(path: &str, args: &[String], text_block: &str, sw: usize, inplace: bool) -> Result { + let path = normalize_path(path)?; + let text = match fs::read_to_string(&path) { Ok(t) => t, Err(e) if e.kind() == io::ErrorKind::NotFound => String::new(), Err(e) => return Err(e.into()) }; + if text.contains('\0') { return Err(invalid("binary file rejected (NUL byte found)")); } + let commands = crate::parse_commands_from_args(args, &mut io::Cursor::new(text_block.as_bytes()))?; + let result = edit_text_with_sw(&text, &commands, sw)?; + if inplace { atomic_write(&path, output_text(&result.lines, true).as_bytes())?; } + Ok(FileEdit { path, cell: None, original_text: text, result }) +} + +#[cfg(test)] +mod tests { + use super::*; + fn fields(values: &[&str]) -> Vec { values.iter().map(|s| CommandField::Str((*s).into())).collect() } + + #[test] + fn file_transfer_preview_and_failure() { + let dir = tempfile::tempdir().unwrap(); + let src = dir.path().join("src.txt").to_string_lossy().into_owned(); + let dst = dir.path().join("dst.txt").to_string_lossy().into_owned(); + fs::write(&src, "alpha\nbeta\n").unwrap(); + let addr = crate::format_lnhash(1, "alpha"); + let cmds = vec![fields(&[&addr, "m", &format!("{dst}:0|0000|")])]; + let preview = edit_files(&src, &cmds, 4, false).unwrap(); + assert_eq!(preview[0].result.lines, ["beta"]); + assert_eq!(preview[1].result.lines, ["alpha"]); + assert!(!Path::new(&dst).exists()); + let mut bad = cmds.clone(); + bad.push(fields(&["1|dead|", "d"])); + assert!(edit_files(&src, &bad, 4, true).is_err()); + assert_eq!(fs::read_to_string(&src).unwrap(), "alpha\nbeta\n"); + assert!(!Path::new(&dst).exists()); + edit_files(&src, &cmds, 4, true).unwrap(); + assert_eq!(fs::read_to_string(&src).unwrap(), "beta\n"); + assert_eq!(fs::read_to_string(&dst).unwrap(), "alpha\n"); + } + + #[test] + fn notebook_preserves_metadata_and_source() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("nb.ipynb").to_string_lossy().into_owned(); + let input = r#"{"cells":[{"id":"aaaa","source":["x=1\n","y=2"],"outputs":[{"value":99999999999999999999999999999999999999}]}],"metadata":{"unicode":"δΈ–η•Œ","ratio":1.00}}"#; + fs::write(&path, input).unwrap(); + let commands = crate::parse_commands_from_script("%s/1/9/").unwrap(); + edit_cell(&path, "aa", &commands, 4, true).unwrap(); + let before: Value = serde_json::from_str(input).unwrap(); + let after: Value = serde_json::from_str(&fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(after["metadata"], before["metadata"]); + assert_eq!(after["cells"][0]["outputs"], before["cells"][0]["outputs"]); + assert_eq!(after["cells"][0]["source"], serde_json::json!(["x=9\n", "y=2"])); + assert!(matches!(view_cell(&path, "missing", None, None), Err(FileError::Cell(_)))); + } +} diff --git a/src/lib.rs b/src/lib.rs index 90b772d..36a7b34 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,16 +1,21 @@ //! exhash β€” Verified Line-Addressed File Editor //! -//! This crate provides the string-based editing engine and command parsing for the -//! `exhash` and `lnhashview` CLIs. +//! Shared text/file/notebook editing and command parsing for Rust embedders and +//! the Python APIs/CLIs. Python bindings are optional (`python` feature). +mod commands; mod engine; +mod files; mod lnhash; mod outline; mod parse; +#[cfg(feature = "python")] mod python; +pub use commands::{CommandField, command_from_fields}; pub use engine::{BufferCommand, BufferEditResult, EditResult, edit_buffers_with_sw, edit_text, edit_text_with_sw}; +pub use files::*; pub use lnhash::{LnHash, format_lnhash, line_hash_u16, lnhashview, parse_lnhash}; pub use outline::{HeadingRow, LinkRow, scan_code, scan_md}; pub use parse::{Address, Command, Subcommand, parse_commands_from_args, parse_commands_from_script}; diff --git a/src/python.rs b/src/python.rs index 7656545..b768a10 100644 --- a/src/python.rs +++ b/src/python.rs @@ -3,11 +3,8 @@ use std::panic::{AssertUnwindSafe, catch_unwind}; use pyo3::exceptions::{PyRuntimeError, PyUserWarning, PyValueError}; use pyo3::prelude::*; -use crate::parse::{ - command_from_parts, parse_buffer_destination_address, parse_destination_address, parse_optional_usize, split_text_payload, subst_from_parts, - translit_from_parts, -}; -use crate::{BufferCommand, Command, EditError, Subcommand}; +use crate::commands::{buffer_command_from_fields, command_from_fields}; +use crate::{BufferCommand, Command, CommandField, EditError, Subcommand}; /// Run a panic-prone pure-Rust step, converting any panic into a clean /// `RuntimeError` instead of surfacing pyo3's `BaseException`-derived @@ -34,6 +31,7 @@ struct EditResultPy { origins: Vec>, #[pyo3(get)] printed: Vec, + #[pyo3(get)] original_text: String, } @@ -66,6 +64,9 @@ fn edit_result_py(original_text: String, result: crate::EditResult) -> EditResul #[pymethods] impl EditResultPy { + #[getter] + fn original_lines(&self) -> Vec<&str> { self.original_text.lines().collect() } + #[pyo3(signature = (context=1))] fn format_diff(&self, py: Python<'_>, context: usize) -> PyResult> { pretty_string(py, self.diff_text(context)) } @@ -150,62 +151,19 @@ enum PyField { Seq(Vec), } -fn command_from_pyfields(fields: &[PyField]) -> Result { - let [PyField::Str(addr), PyField::Str(op), rest @ ..] = fields else { return Err(EditError::new("command must start with (address, op) strings")); }; - command_from_parts(addr, subcommand_from_pyfields(op, rest)?) -} - -fn buffer_command_from_pyfields(fields: &[PyField]) -> Result { - let [PyField::Str(addr), PyField::Str(op), PyField::Str(dest)] = fields else { return command_from_pyfields(fields); }; - if !matches!(op.as_str(), "m" | "t") { return command_from_pyfields(fields); } - let op_char = if op == "m" { 'm' } else { 't' }; - let dest = parse_buffer_destination_address(dest, op_char)?; - let sub = if op == "m" { Subcommand::Move { dest } } else { Subcommand::Copy { dest } }; - command_from_parts(addr, sub) -} - -fn str_fields<'a>(op: &str, fields: &'a [PyField]) -> Result, EditError> { - fields - .iter() - .map(|f| match f { PyField::Str(s) => Ok(s.as_str()), PyField::Seq(_) => Err(EditError::new(format!("{op} fields must be strings"))) }) - .collect() -} - -fn subcommand_from_pyfields(op: &str, fields: &[PyField]) -> Result { - if let "g" | "g!" | "v" = op { - let [PyField::Str(pattern), PyField::Seq(inner)] = fields else { return Err(EditError::new(format!("{op} takes (pattern, (subcommand, ...))"))); }; - let [PyField::Str(iop), irest @ ..] = inner.as_slice() else { return Err(EditError::new("global subcommand must start with an op string")); }; - if matches!(iop.as_str(), "g" | "g!" | "v") { return Err(EditError::new("global commands cannot nest")); } - return Ok(Subcommand::Global { invert: op != "g", pattern: pattern.clone(), cmd: Box::new(subcommand_from_pyfields(iop, irest)?) }); - } - let f = str_fields(op, fields)?; - match (op, f.as_slice()) { - ("d", []) => Ok(Subcommand::Delete), - ("p", []) => Ok(Subcommand::Print), - ("j", []) => Ok(Subcommand::Join), - ("sort", []) => Ok(Subcommand::Sort), - ("a", [text]) => Ok(Subcommand::Append(split_text_payload(text))), - ("i", [text]) => Ok(Subcommand::Insert(split_text_payload(text))), - ("c", [text]) => Ok(Subcommand::Change(split_text_payload(text))), - ("s", [pat, rep]) => Ok(Subcommand::Substitute(subst_from_parts((*pat).into(), (*rep).into(), "")?)), - ("s", [pat, rep, flags]) => Ok(Subcommand::Substitute(subst_from_parts((*pat).into(), (*rep).into(), flags)?)), - ("y", [source, dest]) => { - let (source, dest) = translit_from_parts((*source).into(), (*dest).into())?; - Ok(Subcommand::Transliterate { source, dest }) - } - ("m", [dest]) => Ok(Subcommand::Move { dest: parse_destination_address(dest, 'm')? }), - ("t", [dest]) => Ok(Subcommand::Copy { dest: parse_destination_address(dest, 't')? }), - (">", rest @ ([] | [_])) => Ok(Subcommand::Indent { levels: parse_optional_usize(rest.first().copied().unwrap_or(""))? }), - ("<", rest @ ([] | [_])) => Ok(Subcommand::Dedent { levels: parse_optional_usize(rest.first().copied().unwrap_or(""))? }), - _ => Err(EditError::new(format!("invalid tuple command: {op:?} with {} field(s)", f.len()))), +impl PyField { + fn native(&self) -> CommandField { + match self { Self::Str(s) => CommandField::Str(s.clone()), Self::Seq(v) => CommandField::Seq(v.iter().map(Self::native).collect()) } } } #[pyfunction] #[pyo3(name = "exhash", signature = (text, *cmds, sw=4))] fn py_exhash(py: Python<'_>, text: &str, cmds: Vec>, sw: usize) -> PyResult { - let parsed = guard("parsing commands", || cmds.iter().map(|c| command_from_pyfields(c)).collect::, _>>())? - .map_err(|e| PyValueError::new_err(e.to_string()))?; + let parsed = guard("parsing commands", || { + cmds.iter().map(|c| command_from_fields(&c.iter().map(PyField::native).collect::>())).collect::, _>>() + })? + .map_err(|e| PyValueError::new_err(e.to_string()))?; warn_on_ex_style_dot_terminators(py, &parsed)?; let res = guard("applying edits", || crate::edit_text_with_sw(text, &parsed, sw))?.map_err(|e| PyValueError::new_err(e.to_string()))?; Ok(edit_result_py(text.to_string(), res)) @@ -222,7 +180,9 @@ fn edit_buffers( let parsed = guard("parsing buffer commands", || { commands .into_iter() - .map(|(target, fields, destination)| Ok(BufferCommand { target, command: buffer_command_from_pyfields(&fields)?, destination })) + .map(|(target, fields, destination)| { + Ok(BufferCommand { target, command: buffer_command_from_fields(&fields.iter().map(PyField::native).collect::>())?, destination }) + }) .collect::, EditError>>() })? .map_err(|e| PyValueError::new_err(e.to_string()))?; @@ -240,8 +200,85 @@ fn exhash_argv(text: &str, cmds: Vec, text_block: &str, sw: usize) -> Py Ok(edit_result_py(text.to_string(), res)) } +fn file_error(error: crate::FileError) -> PyErr { + use pyo3::exceptions::{PyFileNotFoundError, PyKeyError, PyOSError}; + match error { + crate::FileError::Io(e) if e.kind() == std::io::ErrorKind::NotFound => PyFileNotFoundError::new_err(e.to_string()), + crate::FileError::Io(e) => PyOSError::new_err(e.to_string()), + crate::FileError::Cell(e) => PyKeyError::new_err(e), + other => PyValueError::new_err(other.to_string()), + } +} + +#[pyfunction] +#[pyo3(signature = (path, start=None, end=None))] +fn view_file(path: &str, start: Option, end: Option) -> PyResult> { crate::view_file(path, start, end).map_err(file_error) } +#[pyfunction] +#[pyo3(signature = (path, cell_id, start=None, end=None))] +fn view_cell(path: &str, cell_id: &str, start: Option, end: Option) -> PyResult> { + crate::view_cell(path, cell_id, start, end).map_err(file_error) +} +#[pyfunction] +#[pyo3(signature = (path, cell_ids, start=None, end=None))] +fn view_cells(path: &str, cell_ids: Vec, start: Option, end: Option) -> PyResult> { + crate::view_cells(path, &cell_ids, start, end).map_err(file_error) +} +#[pyfunction] +#[pyo3(signature = (path, commands, sw=4, inplace=true))] +fn edit_files(py: Python<'_>, path: &str, commands: Vec>, sw: usize, inplace: bool) -> PyResult, EditResultPy)>> { + let commands: Vec> = commands.iter().map(|c| c.iter().map(PyField::native).collect()).collect(); + // Warnings are Python presentation; qualified address parsing lives in Rust. + let warning_commands = commands + .iter() + .filter_map(|c| { + let [CommandField::Str(_), CommandField::Str(op), rest @ ..] = c.as_slice() else { return None; }; + if !matches!(op.as_str(), "a" | "i" | "c" | "g" | "g!" | "v") { return None; } + let mut local = vec![CommandField::Str("%".into()), CommandField::Str(op.clone())]; + local.extend_from_slice(rest); + command_from_fields(&local).ok() + }) + .collect::>(); + warn_on_ex_style_dot_terminators(py, &warning_commands)?; + let results = guard("editing files", || crate::edit_files(path, &commands, sw, inplace))?.map_err(file_error)?; + Ok(results.into_iter().map(|r| (r.path, r.cell, edit_result_py(r.original_text, r.result))).collect()) +} +#[pyfunction] +#[pyo3(signature = (path, cell_id, commands, sw=4, inplace=true))] +fn edit_cell(py: Python<'_>, path: &str, cell_id: &str, commands: Vec>, sw: usize, inplace: bool) -> PyResult { + let parsed = commands + .iter() + .map(|c| command_from_fields(&c.iter().map(PyField::native).collect::>())) + .collect::, _>>() + .map_err(|e| PyValueError::new_err(e.to_string()))?; + warn_on_ex_style_dot_terminators(py, &parsed)?; + let r = guard("editing a cell", || crate::edit_cell(path, cell_id, &parsed, sw, inplace))?.map_err(file_error)?; + Ok(edit_result_py(r.original_text, r.result)) +} +#[pyfunction] +#[pyo3(signature = (path, cell_id, cmds, text_block="", sw=4, inplace=true))] +fn edit_cell_argv(path: &str, cell_id: &str, cmds: Vec, text_block: &str, sw: usize, inplace: bool) -> PyResult { + let parsed = crate::parse_commands_from_args(&cmds, &mut std::io::Cursor::new(text_block.as_bytes())).map_err(|e| PyValueError::new_err(e.to_string()))?; + let r = guard("editing a cell", || crate::files::edit_cell_with_writer(path, cell_id, &parsed, sw, inplace, crate::files::atomic_write))? + .map_err(file_error)?; + Ok(edit_result_py(r.original_text, r.result)) +} + +#[pyfunction] +#[pyo3(signature = (path, cmds, text_block="", sw=4, inplace=true))] +fn edit_file_argv(path: &str, cmds: Vec, text_block: &str, sw: usize, inplace: bool) -> PyResult { + let r = guard("editing a file", || crate::edit_file_argv(path, &cmds, text_block, sw, inplace))?.map_err(file_error)?; + Ok(edit_result_py(r.original_text, r.result)) +} + #[pymodule] fn exhash(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(view_file, m)?)?; + m.add_function(wrap_pyfunction!(view_cell, m)?)?; + m.add_function(wrap_pyfunction!(view_cells, m)?)?; + m.add_function(wrap_pyfunction!(edit_files, m)?)?; + m.add_function(wrap_pyfunction!(edit_cell, m)?)?; + m.add_function(wrap_pyfunction!(edit_cell_argv, m)?)?; + m.add_function(wrap_pyfunction!(edit_file_argv, m)?)?; m.add_class::()?; m.add_function(wrap_pyfunction!(line_hash, m)?)?; m.add_function(wrap_pyfunction!(lnhash, m)?)?; diff --git a/tests/test_exhash.py b/tests/test_exhash.py index 130f1e9..e2774d5 100644 --- a/tests/test_exhash.py +++ b/tests/test_exhash.py @@ -320,6 +320,32 @@ def test_file_exhash_read(tmp_path): assert len(lines) == 2 assert "hello" in lines[0] +@pytest.mark.parametrize('newline', ['\r', '\r\n', '\n']) +def test_file_exhash_universal_newlines(tmp_path, newline): + f = tmp_path/'test.txt' + original = f'alpha{newline}beta{newline}'.encode() + f.write_bytes(original) + assert list(lnhashview_file(f)) == [f'{lnhash(1, "alpha")}alpha', f'{lnhash(2, "beta")}beta'] + assert file_exhash(f, inplace=False).changed == [] + file_exhash(f) + assert f.read_bytes() == original + cmd = (lnhash(2, 'beta'), 'c', 'BETA') + assert file_exhash(f, cmd, inplace=False)[f].lines == ['alpha', 'BETA'] + file_exhash(f, cmd) + assert f.read_bytes() == b'alpha\nBETA\n' + +@pytest.mark.parametrize('separator', ['\u2028', '\u2029', '\x85', '\v', '\f']) +def test_file_exhash_unicode_separator_noop(tmp_path, separator): + f = tmp_path/'test.txt' + text = f'alpha{separator}beta\n' + f.write_text(text) + result = file_exhash(f, inplace=False) + assert result.changed == [] + assert result[f].original_lines == result[f].lines == [text[:-1]] + assert result.format_diff() == '' + assert file_exhash(f) == '' + assert f.read_text() == text + def test_file_exhash_inplace(tmp_path): f = tmp_path / "test.txt" f.write_text("foo\nbar\n")