From 16b066d720325728ca232dd615113ad1b4e02b5f Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:27:36 +0100 Subject: [PATCH 1/6] Add source provenance, source maps, and traceback recovery --- Dockerfile | 4 + doc/advanced/how-the-pipeline-works.md | 60 ++++ doc/guides/cli-usage.md | 37 ++ doc/reference/cli-options.md | 37 ++ doc/reference/config-object.md | 9 + src/cli.lua | 333 +++++++++++++++++- src/prometheus.lua | 4 + src/prometheus/cff/block.lua | 2 + src/prometheus/cff/compile_core.lua | 13 + src/prometheus/cff/compiler.lua | 6 +- src/prometheus/cff/emit.lua | 16 +- src/prometheus/cff/strings.lua | 6 +- src/prometheus/json.lua | 161 +++++++++ src/prometheus/parser.lua | 259 ++++++++++---- src/prometheus/pipeline.lua | 28 +- src/prometheus/source.lua | 57 +++ src/prometheus/sourcemap.lua | 148 ++++++++ src/prometheus/sourcemapwriter.lua | 76 ++++ src/prometheus/steps/AddVararg.lua | 6 +- src/prometheus/steps/AntiTamper.lua | 8 +- src/prometheus/steps/Cff.lua | 4 +- src/prometheus/steps/ConstantArray.lua | 56 +-- src/prometheus/steps/EncryptStrings.lua | 14 +- src/prometheus/steps/NumbersToExpressions.lua | 3 +- src/prometheus/steps/ProxifyLocals.lua | 35 +- src/prometheus/steps/SplitStrings.lua | 32 +- src/prometheus/steps/Watermark.lua | 6 +- src/prometheus/steps/WatermarkCheck.lua | 4 +- src/prometheus/steps/WrapInFunction.lua | 5 +- src/prometheus/tokenizer.lua | 52 +-- src/prometheus/traceback.lua | 258 ++++++++++++++ src/prometheus/unparser.lua | 71 +++- src/prometheus/vm/emit.lua | 4 +- tests/harness/parser-sources.lua | 97 +++++ tests/harness/source-maps.lua | 122 +++++++ tests/harness/source-maps.py | 249 +++++++++++++ tests/harness/step-sources.lua | 190 ++++++++++ 37 files changed, 2291 insertions(+), 181 deletions(-) create mode 100644 src/prometheus/json.lua create mode 100644 src/prometheus/source.lua create mode 100644 src/prometheus/sourcemap.lua create mode 100644 src/prometheus/sourcemapwriter.lua create mode 100644 src/prometheus/traceback.lua create mode 100644 tests/harness/parser-sources.lua create mode 100644 tests/harness/source-maps.lua create mode 100644 tests/harness/source-maps.py create mode 100644 tests/harness/step-sources.lua diff --git a/Dockerfile b/Dockerfile index 784ae330..8be3932a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,10 @@ COPY . /app RUN lua5.1 tests/harness/metadata.lua \ && timeout 60s lua5.1 tests/harness/ast-bindings.lua \ + && timeout 60s lua5.1 tests/harness/parser-sources.lua \ + && timeout 60s lua5.1 tests/harness/step-sources.lua \ + && timeout 60s lua5.1 tests/harness/source-maps.lua \ + && timeout 60s python3 tests/harness/source-maps.py \ && timeout 60s lua5.1 tests/harness/numbers-to-expressions.lua \ && timeout 60s lua5.1 tests/harness/cff-arity.lua \ && timeout 60s lua5.1 tests/harness/cff-native.lua \ diff --git a/doc/advanced/how-the-pipeline-works.md b/doc/advanced/how-the-pipeline-works.md index d067ea21..b654220f 100644 --- a/doc/advanced/how-the-pipeline-works.md +++ b/doc/advanced/how-the-pipeline-works.md @@ -25,3 +25,63 @@ After all steps finish, Prometheus renames identifiers using: ## Logging Pipeline emits informational logs for each phase and step timing, including output size relative to source. + +## Source maps + +Source map generation is disabled by default. Set `SourceMap = true` in a config +to make `Pipeline:apply` return the generated code and map table. Original +source text is embedded by default; set `SourceMapIncludeSources = false` to +omit it. + +```lua +local Prometheus = require("src.prometheus") +local config = { + LuaVersion = "Lua51", + SourceMap = true, + SourceMapIncludeSources = true, + Steps = {}, +} + +local pipeline = Prometheus.Pipeline:fromConfig(config) +local code, map = pipeline:apply(source, "src/main.lua") +map.file = "main.obfuscated.lua" -- Generated file basename +local mapJson = Prometheus.SourceMap.encode(map) +``` + +`Prometheus.SourceMap.decode(mapJson)` parses and validates JSON and returns a +map table. `Prometheus.SourceMap.lookup(map, generatedLine)` returns the +original source path, one-based line, and embedded source text when present. +Lookup uses the first mapped segment on that exact generated line; lines with +no mapped segment return `nil`. Source maps retain approximate line locations +where transformations combine or rewrite syntax. + +`Prometheus.Traceback.format(traceback, map, mapPath)` rewrites matching Lua +and Luau frames and appends source excerpts around mapped locations. If the map +does not embed source text, it reads source files relative to `mapPath`, +including `sourceRoot` when present. Set `map.file` to the generated output +basename so tracebacks can be matched reliably; the CLI sets this field from +`--out` automatically. When source maps are enabled in a CLI config, the CLI +writes `.map` by default; `--source-map ` overrides it. + +## Parser source locations + +`Parser:parse(code, pathOrSourceRecord)` attaches a `sourceSpan` to AST nodes. +The second argument can be a path string or a source record table containing +`path` and `content`; omitting it gives the input a unique internal snippet +label. A span contains a `source` record, `startLine`, `startColumn`, `endLine`, +`endColumn`, `startOffset`, and `endOffset`. Lines and byte columns are +one-based; byte offsets are zero-based and the end offset is exclusive. + +```lua +local Parser = require("src.prometheus.parser") +local parser = Parser:new({ LuaVersion = "Lua51" }) +local ast = parser:parse(source, { + path = "src/main.lua", + content = source, +}) +local firstStatementSpan = ast.body.statements[1].sourceSpan +``` + +Generated snippets use internal paths such as +`prometheus-internal://snippet/1`; these labels distinguish generated source +from user files and can include the snippet text for traceback excerpts. diff --git a/doc/guides/cli-usage.md b/doc/guides/cli-usage.md index aeddc021..b9ad39cc 100644 --- a/doc/guides/cli-usage.md +++ b/doc/guides/cli-usage.md @@ -46,8 +46,45 @@ Enable pretty output: prometheus-lua --preset Minify --pretty ./src/main.lua ``` +Generate a source map with embedded source text: + +```bash +prometheus-lua --preset Medium --out ./output.lua --source-map ./output.lua.map ./src/main.lua +``` + +Use the map to rewrite a traceback saved in a file: + +```bash +prometheus-lua trace --map ./output.lua.map ./traceback.txt +``` + +`trace` also accepts `-` as the traceback path to read from standard input: + +```bash +lua ./output.lua 2>&1 | prometheus-lua trace --map ./output.lua.map - +``` + +The map embeds the original source by default. To omit it, add +`--source-map-no-sources`; `trace` will then look for source files relative to +the map file and its optional `sourceRoot`. The trace formatter appends source +excerpts around resolved locations. It recognizes Lua `file:line:` locations +and Luau `Script 'file', Line N` locations, while leaving C frames and paths +that do not match the map's generated file unchanged. + +If a custom config sets `SourceMap = true`, the CLI writes `.map` +by default. Passing `--source-map ` overrides that sidecar path. + ## Notes - Unknown `--...` options are ignored with a warning. - If no config/preset is passed, Prometheus falls back to `Minify`. - `update` command uses the official installer script from GitHub. +- Source map line mappings are approximate because generated and original + code do not have a one-to-one line relationship. Compact and pretty output + can have different unmapped lines; the formatter only maps exact generated + lines that have a source map segment. +- VM implementation lines use explicit `prometheus-internal://VM/...` source + labels in the displayed trace. Generated lines without mappings retain their + generated location. +- `AntiTamper` is skipped when source maps are enabled, with a warning, because + its generated checks are incompatible with source map output. diff --git a/doc/reference/cli-options.md b/doc/reference/cli-options.md index c95d485c..330ecd86 100644 --- a/doc/reference/cli-options.md +++ b/doc/reference/cli-options.md @@ -7,6 +7,7 @@ | `update` | Runs official installer script to install latest release | | `--version`, `-v` | Prints version (`PROMETHEUS_LUA_VERSION` or `dev`) | | `--help`, `-h`, `help` | Prints CLI help | +| `trace --map ` | Rewrites generated Lua locations using a source map; use `-` instead of a file to read the traceback from standard input | ## Obfuscation options @@ -15,6 +16,8 @@ | `--preset` | `--p` | `` | Use built-in preset by name | | `--config` | `--c` | `` | Load config Lua file | | `--out` | `--o` | `` | Output file path | +| `--source-map` | - | `` | Write a source map to the specified path; enables source maps for this invocation | +| `--source-map-no-sources` | - | none | Omit embedded original source text from the generated map | | `--Lua51` | - | none | Force Lua 5.1 conventions | | `--LuaU` | - | none | Force LuaU conventions | | `--pretty` | - | none | Pretty-print generated output | @@ -26,3 +29,37 @@ - First non-option argument is treated as input file. - A second non-option argument raises an error. - Unknown options are ignored with a warning. + +## Source maps and tracebacks + +`--source-map ` writes a version 3 source map for the generated output. +If source maps are enabled through a config and this option is omitted, the CLI +writes `.map`; the explicit option selects a different path. +Original source text is embedded by default so `trace` can show excerpts without +access to the original files. `--source-map-no-sources` omits that text; the +trace command then looks for the original source relative to the map file, +including any `sourceRoot` recorded in the map. The CLI stores relative source +paths relative to the map file so maps remain usable when saved in a subfolder. + +For example, save a traceback in a file and translate it with: + +```bash +prometheus-lua trace --map ./output.lua.map ./traceback.txt +``` + +To read the traceback from a pipe or the terminal, pass `-`: + +```bash +some-lua-runtime ./output.lua 2>&1 | prometheus-lua trace --map ./output.lua.map - +``` + +Trace mapping uses the first mapped segment on each exact generated line. It +does not infer mappings for lines without a segment, so compact and pretty +output can have approximate coverage. C frames and unrelated file paths are +left as written. Excerpts show two source lines on either side of each mapped +line where available; missing source files are reported in the output. +VM implementation lines can appear under explicit `prometheus-internal://VM/...` +labels, and generated lines without a mapping remain in the traceback as +generated locations. `AntiTamper` is skipped with a warning when source maps +are enabled because its generated checks are incompatible with source map +output. diff --git a/doc/reference/config-object.md b/doc/reference/config-object.md index 98ff0df4..7abdb940 100644 --- a/doc/reference/config-object.md +++ b/doc/reference/config-object.md @@ -6,11 +6,20 @@ Prometheus config table fields: | --- | --- | --- | --- | | `LuaVersion` | string | `"Lua51"` (in `fromConfig`) | Target parser/unparser conventions (`Lua51` or `LuaU`) | | `PrettyPrint` | boolean | `false` | Pretty output mode | +| `SourceMap` | boolean | `false` | Generate a source map and return it as the second result from `Pipeline:apply` | +| `SourceMapIncludeSources` | boolean | `true` | Embed original source text in generated maps when `SourceMap` is enabled | | `VarNamePrefix` | string | `""` | Prefix for generated variable names | | `NameGenerator` | string | `"MangledShuffled"` | Name generator key | | `Seed` | number | `0` | RNG seed; `<=0` means generated seed | | `Steps` | table | `{}` | Ordered step list | +With `SourceMap = true`, `Pipeline:apply(code, filename)` returns the generated +code and a source map table. `SourceMapIncludeSources` defaults to `true`; set +it to `false` to keep source text out of the map. Maps without embedded sources +can still be used when the original files are available relative to the map +file. In the CLI, a config with `SourceMap = true` writes `.map` +unless `--source-map ` selects another location. + ## Steps format ```lua diff --git a/src/cli.lua b/src/cli.lua index 0e8282ed..bf974e63 100644 --- a/src/cli.lua +++ b/src/cli.lua @@ -38,12 +38,15 @@ local function print_help() print("") print("Commands:") print(" update Install latest release via installer script") + print(" trace Rewrite a traceback using a source map") print(" --version, -v Print CLI version") print("") print("Obfuscation options:") print(" --preset, --p Use preset (e.g. Minify, Medium, High)") print(" --config, --c Use custom config Lua file") print(" --out, --o Set output path") + print(" --source-map Write a source map for generated output") + print(" --source-map-no-sources Omit embedded source text from the map") print(" --Lua51 Force Lua 5.1 target") print(" --LuaU Force LuaU target") print(" --pretty Pretty print output") @@ -105,6 +108,193 @@ local function file_exists(file) return f ~= nil end +local function read_file(file) + local handle = io.open(file, "rb") + if not handle then + return nil + end + local content = handle:read("*a") + handle:close() + return content +end + +local function write_file(file, content) + local handle, err = io.open(file, "wb") + if not handle then + return nil, err + end + local ok, writeErr = handle:write(content) + local closeOk, closeErr = handle:close() + if not ok then + return nil, writeErr + end + if closeOk == nil then + return nil, closeErr + end + return true +end + +local function path_basename(path) + return (path:gsub("\\", "/"):match("([^/]+)$")) or path +end + +local function path_info(path) + path = (path or ""):gsub("\\", "/") + local root, rest, absolute = "", path, false + if path:match("^%a:/") then + root, rest, absolute = path:sub(1, 2) .. "/", path:sub(4), true + elseif path:sub(1, 1) == "/" then + root, rest, absolute = "/", path:sub(2), true + end + local parts = {} + for part in rest:gmatch("[^/]+") do + if part == "." or part == "" then + -- Skip redundant path components. + elseif part == ".." then + if #parts > 0 and parts[#parts] ~= ".." then + parts[#parts] = nil + elseif not absolute then + parts[#parts + 1] = part + end + else + parts[#parts + 1] = part + end + end + return root, parts, absolute +end + +local function normalize_path(path) + local root, parts, absolute = path_info(path) + local value = root .. table.concat(parts, "/") + if value == "" then + return "." + end + if absolute and #parts == 0 then + return root + end + return value +end + +local function path_directory(path) + local normalized = normalize_path(path) + local directory = normalized:match("^(.*)/[^/]*$") + if not directory then + return "." + elseif directory == "" then + return "/" + elseif directory:match("^%a:$") then + return directory .. "/" + end + return directory +end + +local function relative_path(fromPath, targetPath) + local fromRoot, fromParts, fromAbsolute = path_info(fromPath) + local targetRoot, targetParts, targetAbsolute = path_info(targetPath) + if fromAbsolute ~= targetAbsolute or fromRoot:lower() ~= targetRoot:lower() then + return normalize_path(targetPath) + end + local common = 0 + while common < #fromParts and common < #targetParts do + local fromPart, targetPart = fromParts[common + 1], targetParts[common + 1] + if fromRoot:match("^%a:/$") then + fromPart, targetPart = fromPart:lower(), targetPart:lower() + end + if fromPart ~= targetPart then + break + end + common = common + 1 + end + local parts = {} + for _ = common + 1, #fromParts do + parts[#parts + 1] = ".." + end + for i = common + 1, #targetParts do + parts[#parts + 1] = targetParts[i] + end + return #parts > 0 and table.concat(parts, "/") or "." +end + +local function is_absolute_path(path) + path = (path or ""):gsub("\\", "/") + return path:sub(1, 1) == "/" or path:match("^%a:/") ~= nil +end + +local function current_directory() + local command = package.config and package.config:sub(1, 1) == "\\" and "cd" or "pwd" + local opened, handle = pcall(io.popen, command) + if not opened or not handle then + return nil + end + local directory = handle:read("*l") + handle:close() + if not directory or directory == "" then + return nil + end + return normalize_path(directory) +end + +local function leading_parent_count(path) + local _, parts, absolute = path_info(path) + if absolute then + return 0 + end + local count = 0 + for _, part in ipairs(parts) do + if part ~= ".." then + break + end + count = count + 1 + end + return count +end + +local function paths_equal(first, second) + first = normalize_path(first) + second = normalize_path(second) + if is_absolute_path(first) ~= is_absolute_path(second) then + local cwd = current_directory() + if not cwd then + error("Could not determine the current directory to compare file paths") + end + if not is_absolute_path(first) then + first = normalize_path(cwd .. "/" .. first) + end + if not is_absolute_path(second) then + second = normalize_path(cwd .. "/" .. second) + end + end + if package.config and package.config:sub(1, 1) == "\\" then + return first:lower() == second:lower() + end + return first == second +end + +local function rebase_map_sources(map, mapPath) + local mapDirectory = path_directory(mapPath) + local cwd + for i, source in ipairs(map.sources or {}) do + if type(source) == "string" + and not source:match("^[%a][%w+.-]*://") + and not is_absolute_path(source) + then + local normalizedSource = normalize_path(source) + local needsCwd = is_absolute_path(mapDirectory) + or leading_parent_count(mapDirectory) ~= leading_parent_count(normalizedSource) + if needsCwd then + cwd = cwd or current_directory() + if not cwd then + error("Could not determine the current directory to rebase source paths") + end + local absoluteSource = normalize_path(cwd .. "/" .. normalizedSource) + map.sources[i] = relative_path(mapDirectory, absoluteSource) + else + map.sources[i] = relative_path(mapDirectory, normalizedSource) + end + end + end +end + string.split = function(str, sep) local fields = {} local pattern = string.format("([^%s]+)", sep) @@ -148,9 +338,80 @@ local function load_chunk(content, chunkName, environment) return load(content, chunkName, "t", environment) end +local function run_trace() + if arg[2] == "--help" or arg[2] == "-h" then + print("Usage: prometheus-lua trace --map ") + print("Read a traceback from a file or standard input and rewrite generated locations.") + return + end + + local mapPath, tracebackPath + local i = 2 + while i <= #arg do + local current = arg[i] + if current == "--map" then + i = i + 1 + if not arg[i] then + Prometheus.Logger:error('Option "--map" requires a path') + end + mapPath = arg[i] + elseif current:sub(1, 1) == "-" and current ~= "-" then + Prometheus.Logger:error(string.format('Unknown trace option "%s"', current)) + elseif tracebackPath then + Prometheus.Logger:error(string.format('Unexpected argument "%s"', current)) + else + tracebackPath = current + end + i = i + 1 + end + + if not mapPath then + Prometheus.Logger:error('Trace requires "--map "') + end + if not tracebackPath then + Prometheus.Logger:error("Trace requires a traceback file or '-' for standard input") + end + local sourceMapApi = Prometheus.SourceMap + if type(sourceMapApi) ~= "table" or type(sourceMapApi.decode) ~= "function" then + Prometheus.Logger:error("Source map support is unavailable in this Prometheus build") + end + + local mapJson = read_file(mapPath) + if mapJson == nil then + Prometheus.Logger:error(string.format('Could not read source map "%s"', mapPath)) + end + local decoded, sourceMap = pcall(sourceMapApi.decode, mapJson) + if not decoded then + Prometheus.Logger:error(string.format('Invalid source map "%s": %s', mapPath, tostring(sourceMap))) + end + + local traceback + if tracebackPath == "-" then + traceback = io.stdin:read("*a") + else + traceback = read_file(tracebackPath) + if traceback == nil then + Prometheus.Logger:error(string.format('Could not read traceback "%s"', tracebackPath)) + end + end + + local Traceback = require("prometheus.traceback") + local formattedOk, formatted = pcall(Traceback.format, traceback, sourceMap, mapPath, sourceMapApi) + if not formattedOk then + Prometheus.Logger:error(string.format("Could not format traceback: %s", tostring(formatted))) + end + io.stdout:write(formatted) +end + local function run_cli() + if arg[1] == "trace" then + run_trace() + return + end + -- CLI local config, sourceFile, outFile, luaVersion, prettyPrint + local sourceMapFile, omitSourceContent Prometheus.colors.enabled = true @@ -191,6 +452,14 @@ local function run_cli() Prometheus.Logger:warn("The output file was specified multiple times!") end outFile = arg[i] + elseif curr == "--source-map" then + i = i + 1 + if not arg[i] or arg[i]:sub(1, 2) == "--" then + Prometheus.Logger:error('Option "--source-map" requires a path') + end + sourceMapFile = arg[i] + elseif curr == "--source-map-no-sources" then + omitSourceContent = true elseif curr == "--nocolors" then Prometheus.colors.enabled = false elseif curr == "--Lua51" then @@ -238,6 +507,12 @@ local function run_cli() -- Add Option to override Lua Version config.LuaVersion = luaVersion or config.LuaVersion config.PrettyPrint = prettyPrint ~= nil and prettyPrint or config.PrettyPrint + if sourceMapFile then + config.SourceMap = true + end + if omitSourceContent then + config.SourceMapIncludeSources = false + end if not file_exists(sourceFile) then Prometheus.Logger:error(string.format('The File "%s" was not found!', sourceFile)) @@ -250,16 +525,64 @@ local function run_cli() outFile = sourceFile .. ".obfuscated.lua" end end + if not sourceMapFile and config.SourceMap == true then + sourceMapFile = outFile .. ".map" + end + if sourceMapFile then + local sameOk, sameOutput = pcall(paths_equal, sourceMapFile, outFile) + if not sameOk then + Prometheus.Logger:error(string.format("Could not validate source map path: %s", tostring(sameOutput))) + end + if sameOutput then + Prometheus.Logger:error("The source map path must differ from the generated output path") + end + sameOk, sameOutput = pcall(paths_equal, sourceMapFile, sourceFile) + if not sameOk then + Prometheus.Logger:error(string.format("Could not validate source map path: %s", tostring(sameOutput))) + end + if sameOutput then + Prometheus.Logger:error("The source map path must differ from the input source path") + end + end - local source = table.concat(lines_from(sourceFile), "\n") + local source = read_file(sourceFile) + if source == nil then + Prometheus.Logger:error(string.format('Could not read input file "%s"', sourceFile)) + end local pipeline = Prometheus.Pipeline:fromConfig(config) - local out = pipeline:apply(source, sourceFile) + local out, sourceMap = pipeline:apply(source, sourceFile) + local sourceMapJson + if sourceMapFile then + if type(sourceMap) ~= "table" then + Prometheus.Logger:error("The pipeline did not return a source map; check that source map support is enabled") + end + if type(Prometheus.SourceMap) ~= "table" or type(Prometheus.SourceMap.encode) ~= "function" then + Prometheus.Logger:error("Source map support is unavailable in this Prometheus build") + end + sourceMap.file = path_basename(outFile) + local rebasedOk, rebaseError = pcall(rebase_map_sources, sourceMap, sourceMapFile) + if not rebasedOk then + Prometheus.Logger:error(string.format("Could not rebase source paths: %s", tostring(rebaseError))) + end + local encoded, mapOrError = pcall(Prometheus.SourceMap.encode, sourceMap) + if not encoded then + Prometheus.Logger:error(string.format("Could not encode source map: %s", tostring(mapOrError))) + end + sourceMapJson = mapOrError + end Prometheus.Logger:info(string.format('Writing output to "%s"', outFile)) -- Write Output - local handle = io.open(outFile, "w") - handle:write(out) - handle:close() + local wrote, writeError = write_file(outFile, out) + if not wrote then + Prometheus.Logger:error(string.format('Could not write output file "%s": %s', outFile, tostring(writeError))) + end + if sourceMapFile then + wrote, writeError = write_file(sourceMapFile, sourceMapJson) + if not wrote then + Prometheus.Logger:error(string.format('Could not write source map "%s": %s', sourceMapFile, tostring(writeError))) + end + end end local ok, err = xpcall(run_cli, function(e) diff --git a/src/prometheus.lua b/src/prometheus.lua index bb3eeeb4..30e4d10e 100644 --- a/src/prometheus.lua +++ b/src/prometheus.lua @@ -56,6 +56,8 @@ local Logger = require("logger"); local Presets = require("presets"); local Config = require("config"); local util = require("prometheus.util"); +local SourceMap = require("prometheus.sourcemap"); +local Traceback = require("prometheus.traceback"); -- Restore package.path package.path = oldPkgPath; @@ -63,6 +65,8 @@ package.path = oldPkgPath; -- Export return { Pipeline = Pipeline; + SourceMap = SourceMap; + Traceback = Traceback; colors = colors; Config = util.readonly(Config); -- Readonly Logger = Logger; diff --git a/src/prometheus/cff/block.lua b/src/prometheus/cff/block.lua index efca1c6c..0b97cc80 100644 --- a/src/prometheus/cff/block.lua +++ b/src/prometheus/cff/block.lua @@ -4,6 +4,7 @@ -- -- Block management for the compiler local util = require("prometheus.util"); +local Source = require("prometheus.source"); local lookupify = util.lookupify; @@ -29,6 +30,7 @@ return function(Compiler) function Compiler:addStatement(statement, writes, reads, usesUpvals, isCleanup) if(self.activeBlock.advanceToNextBlock) then + Source.fill(statement, self.currentOrigin); table.insert(self.activeBlock.statements, { statement = statement, writes = lookupify(writes), diff --git a/src/prometheus/cff/compile_core.lua b/src/prometheus/cff/compile_core.lua index c580c70d..12c5d596 100644 --- a/src/prometheus/cff/compile_core.lua +++ b/src/prometheus/cff/compile_core.lua @@ -50,4 +50,17 @@ return function(Compiler) end logger:error(string.format("%s is not an compliable expression!", expression.kind)); end + + -- Each handler emits operations for its input node. Nested compilation + -- temporarily selects the child's origin and restores the enclosing one. + for _, method in ipairs({'compileStatement', 'compileExpression'}) do + local compile = Compiler[method] + Compiler[method] = function(self, node, ...) + local previous = self.currentOrigin + self.currentOrigin = node.sourceSpan or previous + local result = compile(self, node, ...) + self.currentOrigin = previous + return result + end + end end diff --git a/src/prometheus/cff/compiler.lua b/src/prometheus/cff/compiler.lua index e8aca3c7..a855372a 100644 --- a/src/prometheus/cff/compiler.lua +++ b/src/prometheus/cff/compiler.lua @@ -7,6 +7,7 @@ local Ast = require("prometheus.ast"); local analyzeBindings = require("prometheus.bindinganalysis"); local util = require("prometheus.util"); +local Source = require("prometheus.source"); local lookupify = util.lookupify; local AstKind = Ast.AstKind; @@ -83,6 +84,7 @@ function Compiler:popRegisterUsageInfo() end function Compiler:compile(ast) + self.currentOrigin = nil; analyzeBindings(ast); self.blocks = {}; self.registers = {}; @@ -369,13 +371,13 @@ function Compiler:compile(ast) } })); - return Ast.TopNode(Ast.Block({ + return Source.fill(Ast.TopNode(Ast.Block({ Ast.ReturnStatement{Ast.FunctionCallExpression(functionNode, { astItems[ids[1]], astItems[ids[2]], astItems[ids[3]], astItems[ids[4]], astItems[ids[5]], astItems[ids[6]], astItems[ids[7]], astItems[ids[8]], astItems[ids[9]], astItems[ids[10]], astItems[ids[11]], })}; - })); + })), Source.generated('Cff')); end -- Each wrapper reads its own environment at entry, before tail-calling the VM. diff --git a/src/prometheus/cff/emit.lua b/src/prometheus/cff/emit.lua index 55f1f316..429123d2 100644 --- a/src/prometheus/cff/emit.lua +++ b/src/prometheus/cff/emit.lua @@ -7,6 +7,7 @@ local Ast = require("prometheus.ast"); local util = require("prometheus.util"); local constants = require("prometheus.cff.constants"); +local Source = require("prometheus.source"); local AstKind = Ast.AstKind; local MAX_REGS = constants.MAX_REGS; @@ -39,7 +40,7 @@ return function(Compiler) return slots; end - local function canMergeParallelAssignmentStatements(statA, statB) + local function canMergeParallelAssignmentStatements(statA, statB, sourceMap) if type(statA) ~= "table" or type(statB) ~= "table" then return false; end @@ -56,6 +57,9 @@ return function(Compiler) if type(a) ~= "table" or type(b) ~= "table" then return false; end + if sourceMap and not Source.sameLine(a.sourceSpan, b.sourceSpan) then + return false; + end if a.kind ~= AstKind.AssignmentStatement or b.kind ~= AstKind.AssignmentStatement then return false; end @@ -127,7 +131,7 @@ return function(Compiler) for i = 1, #bRhs do rhs[#aRhs + i] = bRhs[i]; end return { - statement = Ast.AssignmentStatement(lhs, rhs), + statement = Source.inherit(Ast.AssignmentStatement(lhs, rhs), statA.statement), writes = unionLookupTables(statA.writes, statB.writes), reads = unionLookupTables(statA.reads, statB.reads), usesUpvals = statA.usesUpvals or statB.usesUpvals, @@ -135,14 +139,14 @@ return function(Compiler) }; end - local function mergeAdjacentParallelAssignments(blockstats) + local function mergeAdjacentParallelAssignments(blockstats, sourceMap) local merged = {}; local i = 1; while i <= #blockstats do local stat = blockstats[i]; i = i + 1; - while i <= #blockstats and canMergeParallelAssignmentStatements(stat, blockstats[i]) do + while i <= #blockstats and canMergeParallelAssignmentStatements(stat, blockstats[i], sourceMap) do stat = mergeParallelAssignmentStatements(stat, blockstats[i]); i = i + 1; end @@ -216,9 +220,9 @@ return function(Compiler) end end - local mergedBlockStats = mergeAdjacentParallelAssignments(blockstats); + local mergedBlockStats = mergeAdjacentParallelAssignments(blockstats, self.sourceMap); for _=1, 7 do - mergedBlockStats = mergeAdjacentParallelAssignments(mergedBlockStats); + mergedBlockStats = mergeAdjacentParallelAssignments(mergedBlockStats, self.sourceMap); end blockstats = {}; diff --git a/src/prometheus/cff/strings.lua b/src/prometheus/cff/strings.lua index 66b4937f..29192f28 100644 --- a/src/prometheus/cff/strings.lua +++ b/src/prometheus/cff/strings.lua @@ -2,6 +2,7 @@ local Ast = require('prometheus.ast') local visitast = require('prometheus.visitast') local shuffle = require('prometheus.util').shuffle +local Source = require('prometheus.source') local Kind = Ast.AstKind return function(ast) @@ -18,6 +19,7 @@ return function(ast) shuffle(strings) local scope = ast.body.scope local pool = Ast.LocalSymbol() + local generatedSpan = Source.generated('Cff.StringPool') local indices, entries = {}, {} for i, value in ipairs(strings) do indices[value] = i @@ -34,12 +36,12 @@ return function(ast) elseif node.kind == Kind.StringExpression then local index = indices[node.value] if index then - return Ast.IndexExpression(Ast.VariableExpression(pool), Ast.NumberExpression(index)) + return Source.inherit(Ast.IndexExpression(Ast.VariableExpression(pool), Ast.NumberExpression(index)), node) end end end) -- Insert only after replacement so each pooled value remains embedded once. table.insert(ast.body.statements, 1, - Ast.LocalVariableDeclaration({pool}, {Ast.TableConstructorExpression(entries)})) + Source.fill(Ast.LocalVariableDeclaration({pool}, {Ast.TableConstructorExpression(entries)}), generatedSpan)) return ast end diff --git a/src/prometheus/json.lua b/src/prometheus/json.lua new file mode 100644 index 00000000..cfb31108 --- /dev/null +++ b/src/prometheus/json.lua @@ -0,0 +1,161 @@ +-- Small dependency-free JSON codec for source-map files (Lua 5.1 and Luau). +local util = require('prometheus.util') +local Json = {null = {}} +local arrayMeta = {} +function Json.array(value) return setmetatable(value or {}, arrayMeta) end +function Json.isArray(value) return type(value) == 'table' and getmetatable(value) == arrayMeta end + +local function quote(value) + return '"' .. value:gsub('[%z\1-\31\\"]', function(char) + if char == '"' then return '\\"' end + if char == '\\' then return '\\\\' end + return string.format('\\u%04x', char:byte()) + end) .. '"' +end + +function Json.encode(value) + local active = {} + local function encode(item) + if item == Json.null then return 'null' end + local kind = type(item) + if kind == 'string' then return quote(item) end + if kind == 'boolean' then return tostring(item) end + if kind == 'number' then + assert(item == item and math.abs(item) ~= math.huge, 'Invalid JSON number') + return string.format('%.17g', item) + end + assert(kind == 'table', 'Unsupported JSON value: ' .. kind) + assert(not active[item], 'Cyclic JSON value') + active[item] = true + local parts = {} + if getmetatable(item) == arrayMeta then + for i, child in ipairs(item) do parts[i] = encode(child) end + active[item] = nil + return '[' .. table.concat(parts, ',') .. ']' + end + local keys = {} + for key in pairs(item) do + assert(type(key) == 'string', 'JSON object key must be a string') + keys[#keys + 1] = key + end + table.sort(keys) + for i, key in ipairs(keys) do parts[i] = quote(key) .. ':' .. encode(item[key]) end + active[item] = nil + return '{' .. table.concat(parts, ',') .. '}' + end + return encode(value) +end + +function Json.decode(input) + assert(type(input) == 'string', 'JSON input must be a string') + local index, depth = 1, 0 + local function fail(message) error('Invalid JSON at byte ' .. index .. ': ' .. message, 0) end + local function whitespace() + local _, last = input:find('^[ \t\r\n]*', index) + index = (last or index - 1) + 1 + end + local escapes = {['"'] = '"', ['\\'] = '\\', ['/'] = '/', b = '\b', f = '\f', n = '\n', r = '\r', t = '\t'} + local function hex() + local digits = input:sub(index, index + 3) + if #digits ~= 4 or not digits:match('^%x%x%x%x$') then fail('invalid Unicode escape') end + index = index + 4 + return tonumber(digits, 16) + end + local function stringValue() + index = index + 1 + local parts = {} + while index <= #input do + local char = input:sub(index, index) + index = index + 1 + if char == '"' then return table.concat(parts) end + if char == '\\' then + local escape = input:sub(index, index) + index = index + 1 + if escape == 'u' then + local cp = hex() + if cp >= 0xD800 and cp <= 0xDBFF then + if input:sub(index, index + 1) ~= '\\u' then fail('missing low surrogate') end + index = index + 2 + local low = hex() + if low < 0xDC00 or low > 0xDFFF then fail('invalid low surrogate') end + cp = 0x10000 + (cp - 0xD800) * 1024 + low - 0xDC00 + elseif cp >= 0xDC00 and cp <= 0xDFFF then fail('unexpected low surrogate') end + parts[#parts + 1] = util.utf8char(cp) + elseif escapes[escape] then parts[#parts + 1] = escapes[escape] + else fail('invalid escape') end + elseif char:byte() < 32 then fail('unescaped control character') + else parts[#parts + 1] = char end + end + fail('unterminated string') + end + local parse + parse = function() + whitespace() + depth = depth + 1 + if depth > 256 then fail('nesting too deep') end + local char = input:sub(index, index) + local result + if char == '"' then result = stringValue() + elseif char == '[' or char == '{' then + local isArray, closing = char == '[', char == '[' and ']' or '}' + result = isArray and Json.array() or {} + index = index + 1 + whitespace() + if input:sub(index, index) ~= closing then + while true do + local key + if not isArray then + if input:sub(index, index) ~= '"' then fail('expected object key') end + key = stringValue() + whitespace() + if input:sub(index, index) ~= ':' then fail('expected colon') end + index = index + 1 + else key = #result + 1 end + result[key] = parse() + whitespace() + local delimiter = input:sub(index, index) + if delimiter == closing then break end + if delimiter ~= ',' then fail('expected comma or ' .. closing) end + index = index + 1 + whitespace() + end + end + index = index + 1 + elseif input:sub(index, index + 3) == 'true' then index = index + 4; result = true + elseif input:sub(index, index + 4) == 'false' then index = index + 5; result = false + elseif input:sub(index, index + 3) == 'null' then index = index + 4; result = Json.null + else + local start = index + if char == '-' then index = index + 1 end + if input:sub(index, index) == '0' then index = index + 1 + else + local digits = input:match('^[1-9]%d*', index) + if not digits then fail('expected value') end + index = index + #digits + end + if input:sub(index, index) == '.' then + index = index + 1 + local digits = input:match('^%d+', index) + if not digits then fail('expected fractional digits') end + index = index + #digits + end + if input:sub(index, index):match('[eE]') then + index = index + 1 + if input:sub(index, index):match('[+-]') then index = index + 1 end + local digits = input:match('^%d+', index) + if not digits then fail('expected exponent digits') end + index = index + #digits + end + result = tonumber(input:sub(start, index - 1)) + if not result or math.abs(result) == math.huge then fail('invalid number') end + end + depth = depth - 1 + return result + end + local result = parse() + whitespace() + if index <= #input then fail('trailing input') end + return result +end + +return Json diff --git a/src/prometheus/parser.lua b/src/prometheus/parser.lua index bf65bbd7..6c1bb428 100644 --- a/src/prometheus/parser.lua +++ b/src/prometheus/parser.lua @@ -18,6 +18,7 @@ local util = require("prometheus.util"); local Ast = require("prometheus.ast"); local analyzeBindings = require("prometheus.bindinganalysis"); local logger = require("logger"); +local Source = require("prometheus.source"); local AstKind = Ast.AstKind; @@ -25,6 +26,7 @@ local LuaVersion = Enums.LuaVersion; local lookupify = util.lookupify; local TokenKind = Tokenizer.TokenKind; +local unpack = unpack or table.unpack -- Transient parser name environments never become part of the AST. local Bindings = {}; @@ -87,6 +89,59 @@ local function generateWarning(token, message) return "Warning at Position " .. tostring(token.line) .. ":" .. tostring(token.linePos) .. ", " .. message; end +local function sourceSpan(self, startOffset, endOffset) + local startLine, startColumn = self.tokenizer:getPosition(startOffset) + local endLine, endColumn = self.tokenizer:getPosition(endOffset) + return { + source = self.sourceRecord, + startLine = startLine, + startColumn = startColumn, + endLine = endLine, + endColumn = endColumn, + startOffset = startOffset, + endOffset = endOffset, + } +end + +local function tokenSpan(self, token) + if token then return sourceSpan(self, token.startPos, token.endPos) end +end + +local function parserRangeSpan(self, firstIndex, lastIndex) + local first = self.tokens[firstIndex] + local last = self.tokens[lastIndex] + local startOffset = first and first.startPos or self.tokenizer.length + local endOffset = last and last.endPos or startOffset + if lastIndex < firstIndex then endOffset = startOffset end + return sourceSpan(self, startOffset, endOffset) +end + +local function combinedSpan(self, left, right) + local a = left and left.sourceSpan + local b = right and right.sourceSpan + if a and b then + return sourceSpan(self, math.min(a.startOffset, b.startOffset), math.max(a.endOffset, b.endOffset)) + elseif a then + return a + elseif b then + return b + end +end + +local function markCombined(self, node, left, right) + local span = combinedSpan(self, left, right) + if span then node.sourceSpan = span end + return node +end + +local function markThrough(self, node, left, lastToken) + local leftSpan = left and left.sourceSpan + local startOffset = leftSpan and leftSpan.startOffset or (lastToken and lastToken.startPos) or 0 + local endOffset = lastToken and lastToken.endPos or startOffset + node.sourceSpan = sourceSpan(self, startOffset, endOffset) + return node +end + function Parser:new(settings) local luaVersion = (settings and (settings.luaVersion or settings.LuaVersion)) or LuaVersion.LuaU; local parser = { @@ -171,30 +226,48 @@ local function expect(self, kind, source) end -- Parse the given code to an Abstract Syntax Tree -function Parser:parse(code) - self.tokenizer:append(code); - self.tokens = self.tokenizer:scanAll(); - self.length = #self.tokens; - - -- Begin transient name lookup at the chunk boundary - local globalScope = Bindings:new(); - - local ast = Ast.TopNode(self:block(globalScope, false)); - -- File Must be Over when Top Node is Fully Parsed - expect(self, TokenKind.Eof); - - logger:debug("Cleaning up Parser for next Use ...") - -- Clean Up - self.tokenizer:reset(); - self.tokens = {}; - self.index = 0; - self.length = 0; - - logger:debug("Cleanup Done") - -- Keep direct Parser consumers on the same analyzed AST path as the pipeline. - analyzeBindings(ast); - - return ast; +function Parser:parse(code, source) + local previousDisableLog = self.disableLog + self.tokenizer:reset() + self.tokens = {} + self.index = 0 + self.length = 0 + if type(source) == "string" then + self.sourceRecord = Source.new(source, code) + elseif type(source) == "table" then + self.sourceRecord = source + else + self.sourceRecord = Source.internal("snippet", code) + end + + local ok, ast = pcall(function() + self.tokenizer:append(code) + self.tokens = self.tokenizer:scanAll() + self.length = #self.tokens + + -- Begin transient name lookup at the chunk boundary + local globalScope = Bindings:new() + local parsedAst = Ast.TopNode(self:block(globalScope, false)) + -- File Must be Over when Top Node is Fully Parsed + expect(self, TokenKind.Eof) + + -- The top node covers the complete input, including leading and trailing + -- whitespace. All child nodes retain their narrower parser spans. + Source.fill(parsedAst, sourceSpan(self, 0, #code)) + -- Keep direct Parser consumers on the same analyzed AST path as the pipeline. + analyzeBindings(parsedAst) + return parsedAst + end) + + -- Always clear scanner/parser state so a failed parse does not poison reuse. + self.tokenizer:reset() + self.tokens = {} + self.index = 0 + self.length = 0 + self.disableLog = previousDisableLog + self.sourceRecord = nil + if not ok then error(ast, 0) end + return ast end -- Parse a Code Block @@ -321,7 +394,9 @@ function Parser:statement(scope, currentLoop) if(obj.passSelf) then local symbol = funcScope:declare("self", obj.token); - table.insert(args, 1, Ast.VariableExpression(symbol)); + local selfArgument = Ast.VariableExpression(symbol) + selfArgument.sourceSpan = tokenSpan(self, obj.token) + table.insert(args, 1, selfArgument); end local body = self:block(nil, false, funcScope); @@ -371,6 +446,7 @@ function Parser:statement(scope, currentLoop) -- For Statement if(consume(self, TokenKind.Keyword, "for")) then + local forToken = self.tokens[self.index] -- Normal for Statement if(is(self, TokenKind.Symbol, "=", 1)) then local forScope = Bindings:new(scope); @@ -384,6 +460,7 @@ function Parser:statement(scope, currentLoop) expect(self, TokenKind.Symbol, ","); local finalValue = self:expression(scope); local incrementBy = Ast.NumberExpression(1); + incrementBy.sourceSpan = tokenSpan(self, forToken) if(consume(self, TokenKind.Symbol, ",")) then incrementBy = self:expression(scope); end @@ -523,9 +600,10 @@ end function Parser:primaryExpression(scope) local i = self.index; local s = self; + local previousDisableLog = self.disableLog self.disableLog = true; local status, val = pcall(self.expressionFunctionCall, self, scope); - self.disableLog = false; + self.disableLog = previousDisableLog; if(status) then return val; else @@ -574,7 +652,8 @@ function Parser:funcName(scope) local ident = expect(self, TokenKind.Ident); local baseName = ident.value; - local target = scope:resolve(baseName); + local target = scope:resolve(baseName); + target.sourceSpan = tokenSpan(self, ident); local indices = {}; local passSelf = false; @@ -605,7 +684,7 @@ function Parser:expressionOr(scope) if(consume(self, TokenKind.Keyword, "or")) then local rhs = self:expressionOr(scope); - return Ast.OrExpression(lhs, rhs, true); + return markCombined(self, Ast.OrExpression(lhs, rhs, true), lhs, rhs); end return lhs; @@ -616,7 +695,7 @@ function Parser:expressionAnd(scope) if(consume(self, TokenKind.Keyword, "and")) then local rhs = self:expressionAnd(scope); - return Ast.AndExpression(lhs, rhs, true); + return markCombined(self, Ast.AndExpression(lhs, rhs, true), lhs, rhs); end return lhs; @@ -628,37 +707,37 @@ function Parser:expressionComparision(scope) local found = false; if(consume(self, TokenKind.Symbol, "<")) then local rhs = self:expressionStrCat(scope); - curr = Ast.LessThanExpression(curr, rhs, true); + curr = markCombined(self, Ast.LessThanExpression(curr, rhs, true), curr, rhs); found = true; end if(consume(self, TokenKind.Symbol, ">")) then local rhs = self:expressionStrCat(scope); - curr = Ast.GreaterThanExpression(curr, rhs, true); + curr = markCombined(self, Ast.GreaterThanExpression(curr, rhs, true), curr, rhs); found = true; end if(consume(self, TokenKind.Symbol, "<=")) then local rhs = self:expressionStrCat(scope); - curr = Ast.LessThanOrEqualsExpression(curr, rhs, true); + curr = markCombined(self, Ast.LessThanOrEqualsExpression(curr, rhs, true), curr, rhs); found = true; end if(consume(self, TokenKind.Symbol, ">=")) then local rhs = self:expressionStrCat(scope); - curr = Ast.GreaterThanOrEqualsExpression(curr, rhs, true); + curr = markCombined(self, Ast.GreaterThanOrEqualsExpression(curr, rhs, true), curr, rhs); found = true; end if(consume(self, TokenKind.Symbol, "~=")) then local rhs = self:expressionStrCat(scope); - curr = Ast.NotEqualsExpression(curr, rhs, true); + curr = markCombined(self, Ast.NotEqualsExpression(curr, rhs, true), curr, rhs); found = true; end if(consume(self, TokenKind.Symbol, "==")) then local rhs = self:expressionStrCat(scope); - curr = Ast.EqualsExpression(curr, rhs, true); + curr = markCombined(self, Ast.EqualsExpression(curr, rhs, true), curr, rhs); found = true; end until not found; @@ -671,7 +750,7 @@ function Parser:expressionStrCat(scope) if(consume(self, TokenKind.Symbol, "..")) then local rhs = self:expressionStrCat(scope); - return Ast.StrCatExpression(lhs, rhs, true); + return markCombined(self, Ast.StrCatExpression(lhs, rhs, true), lhs, rhs); end return lhs; @@ -684,13 +763,13 @@ function Parser:expressionAddSub(scope) local found = false; if(consume(self, TokenKind.Symbol, "+")) then local rhs = self:expressionMulDivMod(scope); - curr = Ast.AddExpression(curr, rhs, true); + curr = markCombined(self, Ast.AddExpression(curr, rhs, true), curr, rhs); found = true; end if(consume(self, TokenKind.Symbol, "-")) then local rhs = self:expressionMulDivMod(scope); - curr = Ast.SubExpression(curr, rhs, true); + curr = markCombined(self, Ast.SubExpression(curr, rhs, true), curr, rhs); found = true; end until not found; @@ -706,19 +785,19 @@ function Parser:expressionMulDivMod(scope) local found = false; if(consume(self, TokenKind.Symbol, "*")) then local rhs = self:expressionUnary(scope); - curr = Ast.MulExpression(curr, rhs, true); + curr = markCombined(self, Ast.MulExpression(curr, rhs, true), curr, rhs); found = true; end if(consume(self, TokenKind.Symbol, "/")) then local rhs = self:expressionUnary(scope); - curr = Ast.DivExpression(curr, rhs, true); + curr = markCombined(self, Ast.DivExpression(curr, rhs, true), curr, rhs); found = true; end if(consume(self, TokenKind.Symbol, "%")) then local rhs = self:expressionUnary(scope); - curr = Ast.ModExpression(curr, rhs, true); + curr = markCombined(self, Ast.ModExpression(curr, rhs, true), curr, rhs); found = true; end until not found; @@ -751,7 +830,7 @@ function Parser:expressionPow(scope) if(consume(self, TokenKind.Symbol, "^")) then -- Allow unary operators on the rhs (e.g. 2 ^ #x, 2 ^ -x) while preserving right-associativity. ~ SpinnySpiwal local rhs = self:expressionUnary(scope); - return Ast.PowExpression(lhs, rhs, true); + return markCombined(self, Ast.PowExpression(lhs, rhs, true), lhs, rhs); end return lhs; @@ -789,7 +868,9 @@ end function Parser:functionArgList(scope) local args = {}; if(consume(self, TokenKind.Symbol, "...")) then - table.insert(args, Ast.VarargExpression()); + local arg = Ast.VarargExpression(); + arg.sourceSpan = tokenSpan(self, self.tokens[self.index]); + table.insert(args, arg); return args; end @@ -798,11 +879,15 @@ function Parser:functionArgList(scope) local name = ident.value; local symbol = scope:declare(name, ident); - table.insert(args, Ast.VariableExpression(symbol)); + local arg = Ast.VariableExpression(symbol); + arg.sourceSpan = tokenSpan(self, ident); + table.insert(args, arg); while(consume(self, TokenKind.Symbol, ",")) do if(consume(self, TokenKind.Symbol, "...")) then - table.insert(args, Ast.VarargExpression()); + local arg = Ast.VarargExpression(); + arg.sourceSpan = tokenSpan(self, self.tokens[self.index]); + table.insert(args, arg); return args; end @@ -810,7 +895,9 @@ function Parser:functionArgList(scope) name = ident.value; symbol = scope:declare(name, ident); - table.insert(args, Ast.VariableExpression(symbol)); + local arg = Ast.VariableExpression(symbol); + arg.sourceSpan = tokenSpan(self, ident); + table.insert(args, arg); end end @@ -827,9 +914,11 @@ function Parser:expressionFunctionCall(scope, base) -- Normal Function Call local args = {}; if(is(self, TokenKind.String)) then + local stringToken = get(self) args = { - Ast.StringExpression(get(self).value), + Ast.StringExpression(stringToken.value), }; + args[1].sourceSpan = tokenSpan(self, stringToken) elseif(is(self, TokenKind.Symbol, "{")) then args = { self:tableConstructor(scope), @@ -843,7 +932,7 @@ function Parser:expressionFunctionCall(scope, base) return base; end - local node = Ast.FunctionCallExpression(base, args); + local node = markThrough(self, Ast.FunctionCallExpression(base, args), base, self.tokens[self.index]); -- the result of a function call can be indexed if(is(self, TokenKind.Symbol, ".") or is(self, TokenKind.Symbol, "[") or is(self, TokenKind.Symbol, ":")) then @@ -865,18 +954,20 @@ function Parser:expressionIndex(scope, base) while(consume(self, TokenKind.Symbol, "[")) do local expr = self:expression(scope); expect(self, TokenKind.Symbol, "]"); - base = Ast.IndexExpression(base, expr); + base = markThrough(self, Ast.IndexExpression(base, expr), base, self.tokens[self.index]); end -- Parse Indexing Expressions while consume(self, TokenKind.Symbol, ".") do local ident = expect(self, TokenKind.Ident); - base = Ast.IndexExpression(base, Ast.StringExpression(ident.value)); + local key = Ast.StringExpression(ident.value) + key.sourceSpan = tokenSpan(self, ident) + base = markThrough(self, Ast.IndexExpression(base, key), base, ident) while(consume(self, TokenKind.Symbol, "[")) do local expr = self:expression(scope); expect(self, TokenKind.Symbol, "]"); - base = Ast.IndexExpression(base, expr); + base = markThrough(self, Ast.IndexExpression(base, expr), base, self.tokens[self.index]); end end @@ -885,9 +976,11 @@ function Parser:expressionIndex(scope, base) local passSelfFunctionName = expect(self, TokenKind.Ident).value; local args = {}; if(is(self, TokenKind.String)) then + local stringToken = get(self) args = { - Ast.StringExpression(get(self).value), + Ast.StringExpression(stringToken.value), }; + args[1].sourceSpan = tokenSpan(self, stringToken) elseif(is(self, TokenKind.Symbol, "{")) then args = { self:tableConstructor(scope), @@ -900,7 +993,7 @@ function Parser:expressionIndex(scope, base) expect(self, TokenKind.Symbol, ")"); end - local node = Ast.PassSelfFunctionCallExpression(base, passSelfFunctionName, args); + local node = markThrough(self, Ast.PassSelfFunctionCallExpression(base, passSelfFunctionName, args), base, self.tokens[self.index]); -- the result of a function call can be indexed if(is(self, TokenKind.Symbol, ".") or is(self, TokenKind.Symbol, "[") or is(self, TokenKind.Symbol, ":")) then @@ -925,11 +1018,13 @@ end function Parser:expressionLiteral(scope) -- () expression - if(consume(self, TokenKind.Symbol, "(")) then + if(is(self, TokenKind.Symbol, "(")) then + local open = get(self) local expr = self:expression(scope); - expect(self, TokenKind.Symbol, ")"); + local close = expect(self, TokenKind.Symbol, ")"); if expr then expr.isParenthesizedExpression = true; + expr.sourceSpan = sourceSpan(self, open.startPos, close.endPos) end return expr; end @@ -1009,20 +1104,29 @@ function Parser:tableConstructor(scope) expect(self, TokenKind.Symbol, "{"); while (not consume(self, TokenKind.Symbol, "}")) do + local entryStart = self.index + 1 if(consume(self, TokenKind.Symbol, "[")) then local key = self:expression(scope); expect(self, TokenKind.Symbol, "]"); expect(self, TokenKind.Symbol, "="); local value = self:expression(scope); - table.insert(entries, Ast.KeyedTableEntry(key, value)); + local entry = Ast.KeyedTableEntry(key, value) + entry.sourceSpan = parserRangeSpan(self, entryStart, self.index) + table.insert(entries, entry); elseif(is(self, TokenKind.Ident, 0) and is(self, TokenKind.Symbol, "=", 1)) then - local key = Ast.StringExpression(get(self).value); + local ident = get(self) + local key = Ast.StringExpression(ident.value) + key.sourceSpan = tokenSpan(self, ident) expect(self, TokenKind.Symbol, "="); local value = self:expression(scope); - table.insert(entries, Ast.KeyedTableEntry(key, value)); + local entry = Ast.KeyedTableEntry(key, value) + entry.sourceSpan = parserRangeSpan(self, entryStart, self.index) + table.insert(entries, entry); else local value = self:expression(scope); - table.insert(entries, Ast.TableEntry(value)); + local entry = Ast.TableEntry(value) + entry.sourceSpan = parserRangeSpan(self, entryStart, self.index) + table.insert(entries, entry); end @@ -1035,4 +1139,41 @@ function Parser:tableConstructor(scope) return Ast.TableConstructorExpression(entries); end +-- Fill each parser method's result with the token range consumed by that +-- production. Recursive productions have already attached narrower spans to +-- their children, and explicit operator/call/index spans below preserve the +-- exact range for nodes assembled from multiple productions. +local spanProductions = { + "block", "statement", "primaryExpression", "exprList", + "expression", "expressionOr", "expressionAnd", "expressionComparision", + "expressionStrCat", "expressionAddSub", "expressionMulDivMod", + "expressionUnary", "expressionPow", "tableOrFunctionLiteral", + "expressionFunctionLiteral", "expressionFunctionCall", "expressionIndex", + "expressionLiteral", "tableConstructor", +} + +local function wrapSpanProduction(name) + local production = Parser[name] + Parser[name] = function(self, ...) + local firstIndex = self.index + 1 + if name == "statement" then + while self.tokens[firstIndex] + and self.tokens[firstIndex].kind == TokenKind.Symbol + and self.tokens[firstIndex].source == ";" do + firstIndex = firstIndex + 1 + end + end + local result = {production(self, ...)} + local node = result[1] + if type(node) == "table" and AstKind[node.kind] and not node.sourceSpan then + node.sourceSpan = parserRangeSpan(self, firstIndex, self.index) + end + return unpack(result) + end +end + +for _, name in ipairs(spanProductions) do + wrapSpanProduction(name) +end + return Parser diff --git a/src/prometheus/pipeline.lua b/src/prometheus/pipeline.lua index 2bfcd6b3..011aa68d 100644 --- a/src/prometheus/pipeline.lua +++ b/src/prometheus/pipeline.lua @@ -12,6 +12,7 @@ local analyzeBindings = require("prometheus.bindinganalysis"); local renameVariables = require("prometheus.names"); local Unparser = require("prometheus.unparser"); local logger = require("logger"); +local Source = require("prometheus.source"); local NameGenerators = require("prometheus.namegenerators"); @@ -34,6 +35,8 @@ local Pipeline = { DefaultSettings = { LuaVersion = LuaVersion.LuaU; -- The Lua Version to use for the Tokenizer, Parser and Unparser PrettyPrint = false; -- Note that Pretty Print is currently not producing Pretty results + SourceMap = false; + SourceMapIncludeSources = true; Seed = 0; -- The Seed. 0 or below uses the current time as a seed VarNamePrefix = ""; -- The Prefix that every variable will start with } @@ -55,6 +58,8 @@ function Pipeline:new(settings) local pipeline = { LuaVersion = luaVersion; PrettyPrint = prettyPrint; + SourceMap = settings.SourceMap == true; + SourceMapIncludeSources = settings.SourceMapIncludeSources ~= false; VarNamePrefix = prefix; Seed = seed; parser = Parser:new({ @@ -63,6 +68,8 @@ function Pipeline:new(settings) unparser = Unparser:new({ LuaVersion = luaVersion; PrettyPrint = prettyPrint; + SourceMap = settings.SourceMap == true; + SourceMapIncludeSources = settings.SourceMapIncludeSources ~= false; Highlight = settings.Highlight; }); namegenerator = Pipeline.NameGenerators.MangledShuffled; @@ -81,6 +88,8 @@ function Pipeline:fromConfig(config) local pipeline = Pipeline:new({ LuaVersion = config.LuaVersion or LuaVersion.Lua51; PrettyPrint = config.PrettyPrint or false; + SourceMap = config.SourceMap == true; + SourceMapIncludeSources = config.SourceMapIncludeSources ~= false; VarNamePrefix = config.VarNamePrefix or ""; Seed = config.Seed or 0; }); @@ -136,6 +145,9 @@ function Pipeline:setLuaVersion(luaVersion) }); self.unparser = Unparser:new({ LuaVersion = luaVersion; + PrettyPrint = self.PrettyPrint; + SourceMap = self.SourceMap; + SourceMapIncludeSources = self.SourceMapIncludeSources; }); self.conventions = conventions; end @@ -222,7 +234,7 @@ function Pipeline:apply(code, filename) local parserStartTime = gettime(); local sourceLen = string.len(code); - local ast = self.parser:parse(code); + local ast = self.parser:parse(code, filename); local parserTimeDiff = gettime() - parserStartTime; logger:info(string.format("Parsing Done in %.2f seconds", parserTimeDiff)); @@ -235,6 +247,7 @@ function Pipeline:apply(code, filename) if type(newAst) == "table" then ast = newAst; end + Source.fill(ast, Source.generated(step.Name or 'step')); analyzeBindings(ast); logger:info(string.format("Step \"%s\" Done in %.2f seconds", step.Name or "Unnamed", gettime() - stepStartTime)); end @@ -242,26 +255,31 @@ function Pipeline:apply(code, filename) -- Rename Variables Step self:renameVariables(ast); - code = self:unparse(ast); + local map; + code, map = self:unparse(ast); local timeDiff = gettime() - startTime; logger:info(string.format("Obfuscation Done in %.2f seconds", timeDiff)); logger:info(string.format("Generated Code size is %.2f%% of the Source Code size", (string.len(code) / sourceLen)*100)) - return code; + return code, map; end function Pipeline:unparse(ast) local startTime = gettime(); logger:info("Generating Code ..."); - local unparsed = self.unparser:unparse(ast); + -- Options can be changed directly by existing programmatic consumers. + self.unparser.sourceMap = self.SourceMap; + self.unparser.includeSources = self.SourceMapIncludeSources; + self.unparser.prettyPrint = self.PrettyPrint; + local unparsed, map = self.unparser:unparse(ast); local timeDiff = gettime() - startTime; logger:info(string.format("Code Generation Done in %.2f seconds", timeDiff)); - return unparsed; + return unparsed, map; end function Pipeline:renameVariables(ast) diff --git a/src/prometheus/source.lua b/src/prometheus/source.lua new file mode 100644 index 00000000..b2e12311 --- /dev/null +++ b/src/prometheus/source.lua @@ -0,0 +1,57 @@ +-- Source provenance is separate from binding/scope metadata. +local Ast = require('prometheus.ast') +local Source = {} +local counters = {} + +function Source.new(path, content, internal) + return {path = path or 'Anonymous Script', content = content, internal = internal == true} +end + +function Source.internal(label, content) + label = label or 'snippet' + counters[label] = (counters[label] or 0) + 1 + return Source.new('prometheus-internal://' .. label .. '/' .. counters[label], content, true) +end + +function Source.generated(label) + local source = Source.internal(label, '-- Generated ' .. label .. ' code\n') + return {source = source, startLine = 1, startColumn = 1, endLine = 1, + endColumn = 1, startOffset = 0, endOffset = 0} +end + +local fields = {'body', 'statements', 'lhs', 'rhs', 'args', 'expressions', 'entries', + 'key', 'value', 'base', 'index', 'condition', 'true_value', 'false_value', + 'elseifs', 'elsebody', 'initialValue', 'finalValue', 'incrementBy', 'target'} + +-- Walk only syntax edges: symbols, scopes, loop backreferences and cached +-- analysis are intentionally excluded. Also descend through elseif records. +function Source.walk(root, callback) + local seen = {} + local function walk(node) + if type(node) ~= 'table' or seen[node] then return end + seen[node] = true + if Ast.AstKind[node.kind] then callback(node) end + for _, field in ipairs(fields) do walk(node[field]) end + for _, child in ipairs(node) do walk(child) end + end + walk(root) +end + +function Source.fill(node, span) + if span then + Source.walk(node, function(child) + if not child.sourceSpan then child.sourceSpan = span end + end) + end + return node +end + +function Source.inherit(node, original) + return Source.fill(node, original and original.sourceSpan) +end + +function Source.sameLine(a, b) + return a and b and a.source == b.source and a.startLine == b.startLine +end + +return Source diff --git a/src/prometheus/sourcemap.lua b/src/prometheus/sourcemap.lua new file mode 100644 index 00000000..62299463 --- /dev/null +++ b/src/prometheus/sourcemap.lua @@ -0,0 +1,148 @@ +-- Source Map v3 / ECMA-426. Mapping coordinates are zero based on the wire. +local Json = require('prometheus.json') +local SourceMap = {} +local alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' +local digits = {} +for i = 1, #alphabet do digits[alphabet:sub(i, i)] = i - 1 end + +local function vlq(value) + local number = math.abs(value) * 2 + (value < 0 and 1 or 0) + local out = {} + repeat + local digit = number % 32 + number = math.floor(number / 32) + if number > 0 then digit = digit + 32 end + out[#out + 1] = alphabet:sub(digit + 1, digit + 1) + until number == 0 + return table.concat(out) +end + +local function decodeSegment(segment) + local values, value, shift = {}, 0, 0 + for i = 1, #segment do + local digit = digits[segment:sub(i, i)] + assert(digit, 'Invalid source map: invalid Base64 VLQ digit') + value = value + (digit % 32) * 2 ^ shift + assert(value <= 4294967295 and shift <= 30, 'Invalid source map: VLQ overflow') + if digit < 32 then + local negative = value % 2 == 1 + values[#values + 1] = math.floor(value / 2) * (negative and -1 or 1) + value, shift = 0, 0 + else shift = shift + 5 end + end + assert(shift == 0, 'Invalid source map: incomplete VLQ') + assert(#values == 1 or #values == 4 or #values == 5, 'Invalid source map: segment must have 1, 4 or 5 fields') + return values +end + +function SourceMap.build(origins, count, includeSources) + local map = {version = 3, sources = Json.array(), names = Json.array(), mappings = ''} + if includeSources ~= false then map.sourcesContent = Json.array() end + local ids, lines = {}, {} + local lastSource, lastLine = 0, 0 + for i = 1, count do + local origin = origins[i] + if origin then + local source = origin.source + local id = ids[source] + if not id then + id = #map.sources + 1 + ids[source] = id + map.sources[id] = source.path + if map.sourcesContent then map.sourcesContent[id] = source.content or Json.null end + end + local originalLine = origin.startLine - 1 + lines[i] = 'A' .. vlq(id - 1 - lastSource) .. vlq(originalLine - lastLine) .. 'A' + lastSource, lastLine = id - 1, originalLine + else lines[i] = '' end + end + map.mappings = table.concat(lines, ';') + return map +end + +local cache = setmetatable({}, {__mode = 'k'}) +local function array(value, label, itemKind) + assert(type(value) == 'table', 'Invalid source map: ' .. label .. ' must be an array') + local count = 0 + for index, item in pairs(value) do + assert(type(index) == 'number' and index >= 1 and index == math.floor(index), + 'Invalid source map: ' .. label .. ' must be an array') + count = count + 1 + assert(type(item) == itemKind or (label == 'sourcesContent' and item == Json.null), + 'Invalid source map: invalid ' .. label .. ' entry') + end + assert(count == #value, 'Invalid source map: ' .. label .. ' must not be sparse') +end + +local function validate(map) + assert(type(map) == 'table' and map.version == 3, 'Invalid source map: expected version 3') + assert(type(map.sources) == 'table' and type(map.names) == 'table' and type(map.mappings) == 'string', + 'Invalid source map: expected sources, names and mappings') + array(map.sources, 'sources', 'string') + array(map.names, 'names', 'string') + if map.file ~= nil then assert(type(map.file) == 'string', 'Invalid source map: file must be a string') end + if map.sourceRoot ~= nil then assert(type(map.sourceRoot) == 'string', 'Invalid source map: sourceRoot must be a string') end + if map.sourcesContent ~= nil then + assert(type(map.sourcesContent) == 'table' and #map.sourcesContent == #map.sources, 'Invalid source map: sourcesContent must match sources') + array(map.sourcesContent, 'sourcesContent', 'string') + end + local result, sourceIndex, originalLine, originalColumn, nameIndex = {}, 0, 0, 0, 0 + local lineNumber = 0 + for line in (map.mappings .. ';'):gmatch('(.-);') do + lineNumber = lineNumber + 1 + local generatedColumn = 0 + if line ~= '' then + for segment in (line .. ','):gmatch('(.-),') do + local fields = decodeSegment(segment) + assert(fields[1] >= 0, 'Invalid source map: generated columns must be ordered') + generatedColumn = generatedColumn + fields[1] + if #fields > 1 then + sourceIndex = sourceIndex + fields[2] + originalLine = originalLine + fields[3] + originalColumn = originalColumn + fields[4] + assert(sourceIndex >= 0 and sourceIndex < #map.sources and originalLine >= 0 and originalColumn >= 0, + 'Invalid source map: mapping references invalid source or position') + if #fields == 5 then + nameIndex = nameIndex + fields[5] + assert(nameIndex >= 0 and nameIndex < #map.names, 'Invalid source map: invalid name index') + end + if not result[lineNumber] then + local content = map.sourcesContent and map.sourcesContent[sourceIndex + 1] + result[lineNumber] = {source = map.sources[sourceIndex + 1], line = originalLine + 1, + content = content ~= Json.null and content or nil} + end + end + end + end + end + cache[map] = result + return map +end + +function SourceMap.encode(map) + validate(map) + local function copyArray(items) + local copy = Json.array() + for i, item in ipairs(items) do copy[i] = item end + return copy + end + local object = {version = map.version, sources = copyArray(map.sources), names = copyArray(map.names), + mappings = map.mappings, file = map.file, sourceRoot = map.sourceRoot} + if map.sourcesContent then object.sourcesContent = copyArray(map.sourcesContent) end + return Json.encode(object) +end + +function SourceMap.decode(text) + local map = Json.decode(text) + assert(type(map) == 'table' and Json.isArray(map.sources) and Json.isArray(map.names) and + (map.sourcesContent == nil or Json.isArray(map.sourcesContent)), + 'Invalid source map: sources, names and sourcesContent must be JSON arrays') + return validate(map) +end + +function SourceMap.lookup(map, generatedLine) + if not cache[map] then validate(map) end + return cache[map][generatedLine] +end + +return SourceMap diff --git a/src/prometheus/sourcemapwriter.lua b/src/prometheus/sourcemapwriter.lua new file mode 100644 index 00000000..d0e8fca7 --- /dev/null +++ b/src/prometheus/sourcemapwriter.lua @@ -0,0 +1,76 @@ +-- Buffer provenance alongside the existing renderer, then emit whole tokens. +-- Private NUL-delimited tags never appear in output: Lua string values are +-- escaped before rendering and cannot collide with these annotations. +local Tokenizer = require('prometheus.tokenizer') +local Source = require('prometheus.source') +local SourceMap = require('prometheus.sourcemap') +local Writer = {} + +function Writer.render(tagged, spans, luaVersion, includeSources) + local parts, runs, stack = {}, {}, {} + local cursor, offset = 1, 0 + local function append(text) + if #text == 0 then return end + parts[#parts + 1] = text + runs[#runs + 1] = {start = offset, finish = offset + #text, span = stack[#stack]} + offset = offset + #text + end + while true do + local first, last, id = tagged:find('%z([^%z]*)%z', cursor) + if not first then append(tagged:sub(cursor)); break end + append(tagged:sub(cursor, first - 1)) + if id == '/' then + assert(#stack > 0, 'Unbalanced source annotations') + stack[#stack] = nil + else + local span = spans[tonumber(id)] + assert(span, 'Invalid source annotation') + stack[#stack + 1] = span + end + cursor = last + 1 + end + assert(#stack == 0, 'Unbalanced source annotations') + local text = table.concat(parts) + local tokenizer = Tokenizer:new({LuaVersion = luaVersion, AllowInternalIdentifiers = true}) + tokenizer:append(text) + local tokens = tokenizer:scanAll() + local output, origins = {}, {} + local line, runIndex, previousEnd, previousSpan, previousToken = 1, 1, 0, nil, nil + local function emit(fragment, span) + output[#output + 1] = fragment + if span and #fragment > 0 and not fragment:match('^[ \t\r\n]*$') then + origins[line] = origins[line] or span + end + for _ in fragment:gmatch('\n') do + line = line + 1 + -- Actual multiline tokens retain their origin on continuation lines. + if span then origins[line] = span end + end + end + for _, token in ipairs(tokens) do + if token.kind ~= Tokenizer.TokenKind.Eof then + while runs[runIndex] and runs[runIndex].finish <= token.startPos do runIndex = runIndex + 1 end + local span = runs[runIndex] and runs[runIndex].span + -- Lua 5.1 rejects a newline between a callee and its argument '('. + -- Attribute that delimiter to the callee; the call's closing + -- delimiter still carries the enclosing operation's source line. + if token.source == '(' and previousToken and + (previousToken.kind == Tokenizer.TokenKind.Ident or previousToken.kind == Tokenizer.TokenKind.String or + previousToken.source == ')' or previousToken.source == ']' or previousToken.source == '}') then + span = previousSpan + end + local gap = text:sub(previousEnd + 1, token.startPos) + if previousSpan and span and not Source.sameLine(previousSpan, span) and not gap:find('\n', 1, true) then + -- Keep pretty-print indentation; replace only inter-token spacing. + gap = '\n' .. gap + end + emit(gap) + emit(text:sub(token.startPos + 1, token.endPos), span) + previousEnd, previousSpan, previousToken = token.endPos, span, token + end + end + emit(text:sub(previousEnd + 1)) + return table.concat(output), SourceMap.build(origins, line, includeSources) +end + +return Writer diff --git a/src/prometheus/steps/AddVararg.lua b/src/prometheus/steps/AddVararg.lua index 361e4b61..e5def798 100644 --- a/src/prometheus/steps/AddVararg.lua +++ b/src/prometheus/steps/AddVararg.lua @@ -8,6 +8,7 @@ local Step = require("prometheus.step"); local Ast = require("prometheus.ast"); local visitast = require("prometheus.visitast"); local AstKind = Ast.AstKind; +local Source = require("prometheus.source"); local AddVararg = Step:extend(); AddVararg.Description = "This Step Adds Vararg to all Functions"; @@ -18,13 +19,14 @@ AddVararg.SettingsDescriptor = {} function AddVararg:init(_) end function AddVararg:apply(ast) + local generatedSpan = Source.generated("AddVararg"); visitast(ast, nil, function(node) if node.kind == AstKind.FunctionDeclaration or node.kind == AstKind.LocalFunctionDeclaration or node.kind == AstKind.FunctionLiteralExpression then if #node.args < 1 or node.args[#node.args].kind ~= AstKind.VarargExpression then - node.args[#node.args + 1] = Ast.VarargExpression(); + node.args[#node.args + 1] = Source.fill(Ast.VarargExpression(), generatedSpan); end end end) end -return AddVararg; \ No newline at end of file +return AddVararg; diff --git a/src/prometheus/steps/AntiTamper.lua b/src/prometheus/steps/AntiTamper.lua index 5da2e780..9c59ed7d 100644 --- a/src/prometheus/steps/AntiTamper.lua +++ b/src/prometheus/steps/AntiTamper.lua @@ -9,6 +9,7 @@ local RandomStrings = require("prometheus.randomStrings") local Parser = require("prometheus.parser") local Enums = require("prometheus.enums") local logger = require("logger") +local Source = require("prometheus.source") local AntiTamper = Step:extend() AntiTamper.Description = "This Step Breaks your Script when it is modified. This is only effective when using the new VM." @@ -82,8 +83,9 @@ end function AntiTamper:init(settings) end function AntiTamper:apply(ast, pipeline) - if pipeline.PrettyPrint then - logger:warn(string.format('"%s" cannot be used with PrettyPrint, ignoring "%s"', self.Name, self.Name)) + if pipeline.PrettyPrint or pipeline.SourceMap then + local reason = pipeline.PrettyPrint and "PrettyPrint" or "SourceMap" + logger:warn(string.format('"%s" cannot be used with %s, ignoring "%s"', self.Name, reason, self.Name)) return ast end local code = generateSanityCheck() @@ -233,7 +235,7 @@ end repeat until valid; ]] - local parsed = Parser:new({LuaVersion = Enums.LuaVersion.Lua51}):parse(code); + local parsed = Parser:new({LuaVersion = Enums.LuaVersion.Lua51}):parse(code, Source.internal("AntiTamper", code)); local doStat = parsed.body.statements[1]; table.insert(ast.body.statements, 1, doStat); diff --git a/src/prometheus/steps/Cff.lua b/src/prometheus/steps/Cff.lua index 862c3838..7fbafdbe 100644 --- a/src/prometheus/steps/Cff.lua +++ b/src/prometheus/steps/Cff.lua @@ -19,9 +19,10 @@ Cff.SettingsDescriptor = { function Cff:init(_) end -function Cff:apply(ast) +function Cff:apply(ast, pipeline) if self.ExcludeVm and ast.vmRuntime then local compiler = Compiler:new(); + compiler.sourceMap = pipeline and pipeline.SourceMap; compiler.nativeExpressions = {[ast.vmRuntime] = true}; if ast.vmData then compiler.nativeExpressions[ast.vmData] = true; end local flattened = compiler:compile(ast); @@ -34,6 +35,7 @@ function Cff:apply(ast) if flattened then return shareStrings(flattened); end end local compiler = Compiler:new(); + compiler.sourceMap = pipeline and pipeline.SourceMap; return shareStrings(compiler:compile(ast)); end diff --git a/src/prometheus/steps/ConstantArray.lua b/src/prometheus/steps/ConstantArray.lua index 35e34ea6..ae062a78 100644 --- a/src/prometheus/steps/ConstantArray.lua +++ b/src/prometheus/steps/ConstantArray.lua @@ -13,6 +13,7 @@ local visitast = require("prometheus.visitast"); local util = require("prometheus.util") local Parser = require("prometheus.parser"); local enums = require("prometheus.enums") +local Source = require("prometheus.source") local LuaVersion = enums.LuaVersion; local AstKind = Ast.AstKind; @@ -122,7 +123,7 @@ function ConstantArray:createArray() end entries[i] = Ast.TableEntry(Ast.ConstantNode(v)); end - return Ast.TableConstructorExpression(entries); + return Source.fill(Ast.TableConstructorExpression(entries), self.generatedSpan); end function ConstantArray:indexing(index, data) @@ -198,14 +199,15 @@ function ConstantArray:addRotateCode(ast, shift) LuaVersion = LuaVersion.Lua51; }); - local newAst = parser:parse(string.gsub(string.gsub(rotateCode, "SHIFT", tostring(shift)), "LEN", tostring(#self.constants))); + local code = string.gsub(string.gsub(rotateCode, "SHIFT", tostring(shift)), "LEN", tostring(#self.constants)); + local newAst = parser:parse(code, Source.internal("ConstantArray.Rotate", code)); local forStat = newAst.body.statements[1]; visitast(newAst, nil, function(node, data) if node.name == "ARR" then if node.kind == AstKind.AssignmentGlobalVariable then - return Ast.AssignmentVariable(self.arrSymbol); + return Source.inherit(Ast.AssignmentVariable(self.arrSymbol), node); elseif node.kind == AstKind.GlobalVariableExpression then - return Ast.VariableExpression(self.arrSymbol); + return Source.inherit(Ast.VariableExpression(self.arrSymbol), node); end end end) @@ -268,17 +270,17 @@ function ConstantArray:addDecodeCode(ast) LuaVersion = LuaVersion.Lua51; }); - local newAst = parser:parse(base64DecodeCode); + local newAst = parser:parse(base64DecodeCode, Source.internal("ConstantArray.Base64Decoder", base64DecodeCode)); local forStat = newAst.body.statements[1]; visitast(newAst, nil, function(node, data) if(node.kind == AstKind.GlobalVariableExpression or node.kind == AstKind.AssignmentGlobalVariable) then if(node.name == "ARR") then - return node.kind == AstKind.AssignmentGlobalVariable and Ast.AssignmentVariable(self.arrSymbol) or Ast.VariableExpression(self.arrSymbol); + return Source.inherit(node.kind == AstKind.AssignmentGlobalVariable and Ast.AssignmentVariable(self.arrSymbol) or Ast.VariableExpression(self.arrSymbol), node); end if(node.name == "LOOKUP_TABLE") then - return self:createBase64Lookup(); + return Source.fill(self:createBase64Lookup(), self.generatedSpan); end end end) @@ -352,17 +354,17 @@ function ConstantArray:addDecodeCode(ast) LuaVersion = LuaVersion.Lua51; }); - local newAst = parser:parse(base85DecodeCode); + local newAst = parser:parse(base85DecodeCode, Source.internal("ConstantArray.Base85Decoder", base85DecodeCode)); local forStat = newAst.body.statements[1]; visitast(newAst, nil, function(node, data) if(node.kind == AstKind.GlobalVariableExpression or node.kind == AstKind.AssignmentGlobalVariable) then if(node.name == "ARR") then - return node.kind == AstKind.AssignmentGlobalVariable and Ast.AssignmentVariable(self.arrSymbol) or Ast.VariableExpression(self.arrSymbol); + return Source.inherit(node.kind == AstKind.AssignmentGlobalVariable and Ast.AssignmentVariable(self.arrSymbol) or Ast.VariableExpression(self.arrSymbol), node); end if(node.name == "LOOKUP_TABLE") then - return self:createBase85Lookup(); + return Source.fill(self:createBase85Lookup(), self.generatedSpan); end end end) @@ -472,21 +474,21 @@ function ConstantArray:addDecodeCode(ast) LuaVersion = LuaVersion.Lua51; }); - local newAst = parser:parse(mixedDecodeCode); + local newAst = parser:parse(mixedDecodeCode, Source.internal("ConstantArray.MixedDecoder", mixedDecodeCode)); local forStat = newAst.body.statements[1]; visitast(newAst, nil, function(node, data) if(node.kind == AstKind.GlobalVariableExpression or node.kind == AstKind.AssignmentGlobalVariable) then if(node.name == "ARR") then - return node.kind == AstKind.AssignmentGlobalVariable and Ast.AssignmentVariable(self.arrSymbol) or Ast.VariableExpression(self.arrSymbol); + return Source.inherit(node.kind == AstKind.AssignmentGlobalVariable and Ast.AssignmentVariable(self.arrSymbol) or Ast.VariableExpression(self.arrSymbol), node); end if(node.name == "LOOKUP_TABLE_64") then - return self:createBase64Lookup(); + return Source.fill(self:createBase64Lookup(), self.generatedSpan); end if(node.name == "LOOKUP_TABLE_85") then - return self:createBase85Lookup(); + return Source.fill(self:createBase85Lookup(), self.generatedSpan); end end end) @@ -503,7 +505,7 @@ function ConstantArray:createBase64Lookup() i = i + 1; end util.shuffle(entries); - return Ast.TableConstructorExpression(entries); + return Source.fill(Ast.TableConstructorExpression(entries), self.generatedSpan); end function ConstantArray:createBase85Lookup() @@ -514,7 +516,7 @@ function ConstantArray:createBase85Lookup() i = i + 1; end util.shuffle(entries); - return Ast.TableConstructorExpression(entries); + return Source.fill(Ast.TableConstructorExpression(entries), self.generatedSpan); end function ConstantArray:encode(str) @@ -599,6 +601,7 @@ end function ConstantArray:apply(ast, pipeline) initPrefixes(); + self.generatedSpan = Source.generated("ConstantArray"); self.arrSymbol = Ast.LocalSymbol(); self.base64chars = table.concat(util.shuffle{ @@ -685,10 +688,14 @@ function ConstantArray:apply(ast, pipeline) -- Actually insert Statements to get the Constant Values if node.__apply_constant_array then if node.kind == AstKind.StringExpression then - return self:getConstant(node.value, data); + local replacement = self:getConstant(node.value, data); + return Source.inherit(replacement, node); elseif not self.StringsOnly then if node.isConstant then - return node.value ~= nil and self:getConstant(node.value, data); + if node.value ~= nil then + local replacement = self:getConstant(node.value, data); + return Source.inherit(replacement, node); + end end end node.__apply_constant_array = nil; @@ -728,7 +735,7 @@ function ConstantArray:apply(ast, pipeline) addSubArg }); - elems[i] = Ast.KeyedTableEntry( + elems[i] = Ast.KeyedTableEntry( Ast.StringExpression(name), Ast.FunctionLiteralExpression(args, Ast.Block({ Ast.ReturnStatement({ @@ -737,11 +744,11 @@ function ConstantArray:apply(ast, pipeline) })) ) end - table.insert(node.statements, 1, Ast.LocalVariableDeclaration({ + table.insert(node.statements, 1, Source.fill(Ast.LocalVariableDeclaration({ wrappers.symbol }, { Ast.TableConstructorExpression(elems) - })); + }), self.generatedSpan)); end end); @@ -761,7 +768,7 @@ function ConstantArray:apply(ast, pipeline) end -- Create and Add the Function Declaration - table.insert(ast.body.statements, 1, Ast.LocalFunctionDeclaration(self.wrapperSymbol, { + table.insert(ast.body.statements, 1, Source.fill(Ast.LocalFunctionDeclaration(self.wrapperSymbol, { arg }, Ast.Block({ Ast.ReturnStatement({ @@ -770,7 +777,7 @@ function ConstantArray:apply(ast, pipeline) addSubArg ) }); - }))); + })), self.generatedSpan)); -- Resulting Code: -- function xy(a) @@ -793,13 +800,14 @@ function ConstantArray:apply(ast, pipeline) end -- Add the Array Declaration - table.insert(ast.body.statements, 1, Ast.LocalVariableDeclaration({self.arrSymbol}, {self:createArray()})); + table.insert(ast.body.statements, 1, Source.fill(Ast.LocalVariableDeclaration({self.arrSymbol}, {self:createArray()}), self.generatedSpan)); self.arrSymbol = nil; self.wrapperSymbol = nil; self.constants = nil; self.lookup = nil; + self.generatedSpan = nil; end return ConstantArray; diff --git a/src/prometheus/steps/EncryptStrings.lua b/src/prometheus/steps/EncryptStrings.lua index 6bd0b823..00755c0e 100644 --- a/src/prometheus/steps/EncryptStrings.lua +++ b/src/prometheus/steps/EncryptStrings.lua @@ -10,6 +10,7 @@ local Parser = require("prometheus.parser") local Enums = require("prometheus.enums") local visitast = require("prometheus.visitast"); local util = require("prometheus.util") +local Source = require("prometheus.source") local hoistVmStrings = require("prometheus.vm.strings") local AstKind = Ast.AstKind; @@ -194,8 +195,9 @@ function EncryptStrings:apply(ast, _) local Encryptor = self:CreateEncryptionService(); local code = Encryptor.genCode(); - local newAst = Parser:new({ LuaVersion = Enums.LuaVersion.Lua51 }):parse(code); + local newAst = Parser:new({ LuaVersion = Enums.LuaVersion.Lua51 }):parse(code, Source.internal("EncryptStrings.Decryptor", code)); local doStat = newAst.body.statements[1]; + local generatedSpan = Source.generated("EncryptStrings"); local decryptVar = Ast.LocalSymbol("_decrypt"); local stringsVar = Ast.LocalSymbol("_strings"); @@ -211,9 +213,9 @@ function EncryptStrings:apply(ast, _) if(node.kind == AstKind.AssignmentVariable or node.kind == AstKind.VariableExpression or node.kind == AstKind.AssignmentGlobalVariable or node.kind == AstKind.GlobalVariableExpression) then if(node.name == "STRINGS") then if node.kind == AstKind.AssignmentVariable or node.kind == AstKind.AssignmentGlobalVariable then - return Ast.AssignmentVariable(stringsVar); + return Source.inherit(Ast.AssignmentVariable(stringsVar), node); end - return Ast.VariableExpression(stringsVar); + return Source.inherit(Ast.VariableExpression(stringsVar), node); end end end) @@ -221,16 +223,16 @@ function EncryptStrings:apply(ast, _) visitast(ast, nil, function(node, data) if(node.kind == AstKind.StringExpression) then local encrypted, seed = Encryptor.encrypt(node.value); - return Ast.IndexExpression(Ast.VariableExpression(stringsVar), Ast.FunctionCallExpression(Ast.VariableExpression(decryptVar), { + return Source.inherit(Ast.IndexExpression(Ast.VariableExpression(stringsVar), Ast.FunctionCallExpression(Ast.VariableExpression(decryptVar), { Ast.StringExpression(encrypted), Ast.NumberExpression(seed), - })); + })), node); end end) -- Insert to Main Ast table.insert(ast.body.statements, 1, doStat); - table.insert(ast.body.statements, 1, Ast.LocalVariableDeclaration(util.shuffle{ decryptVar, stringsVar }, {})); + table.insert(ast.body.statements, 1, Source.fill(Ast.LocalVariableDeclaration(util.shuffle{ decryptVar, stringsVar }, {}), generatedSpan)); return ast end diff --git a/src/prometheus/steps/NumbersToExpressions.lua b/src/prometheus/steps/NumbersToExpressions.lua index e65c6692..1011b053 100644 --- a/src/prometheus/steps/NumbersToExpressions.lua +++ b/src/prometheus/steps/NumbersToExpressions.lua @@ -13,6 +13,7 @@ local Ast = require("prometheus.ast") local visitast = require("prometheus.visitast") local util = require("prometheus.util") local logger = require("logger") +local Source = require("prometheus.source") local AstKind = Ast.AstKind local NumbersToExpressions = Step:extend() @@ -188,7 +189,7 @@ function NumbersToExpressions:apply(ast) visitast(ast, nil, function(node, _) if node.kind == AstKind.NumberExpression then if math.random() <= self.Threshold then - return self:CreateNumberExpression(node.value, 0) + return Source.inherit(self:CreateNumberExpression(node.value, 0), node) end end end) diff --git a/src/prometheus/steps/ProxifyLocals.lua b/src/prometheus/steps/ProxifyLocals.lua index 0d7c3588..93afb8ad 100644 --- a/src/prometheus/steps/ProxifyLocals.lua +++ b/src/prometheus/steps/ProxifyLocals.lua @@ -8,6 +8,7 @@ local Step = require("prometheus.step") local Ast = require("prometheus.ast") local visitast = require("prometheus.visitast") local RandomLiterals = require("prometheus.randomLiterals") +local Source = require("prometheus.source") local AstKind = Ast.AstKind local ProxifyLocals = Step:extend() @@ -83,13 +84,18 @@ function ProxifyLocals:CreateAssignmentExpression(info, expr) })) table.insert(metatableVals, Ast.KeyedTableEntry(Ast.StringExpression(info.getValue.key), getFunction)) - return Ast.FunctionCallExpression(Ast.VariableExpression(self.setMetatableSymbol), { + local expression = Ast.FunctionCallExpression(Ast.VariableExpression(self.setMetatableSymbol), { Ast.TableConstructorExpression({Ast.KeyedTableEntry(Ast.StringExpression(info.valueName), expr)}), Ast.TableConstructorExpression(metatableVals), }) + -- The outer value is a replacement for the original initializer/RHS. The + -- proxy implementation beneath it belongs to this generated helper source. + expression.sourceSpan = expr and expr.sourceSpan + return Source.fill(expression, self.generatedSpan) end function ProxifyLocals:apply(ast, pipeline) + self.generatedSpan = Source.generated("ProxifyLocals") local localMetatableInfos = {} -- Rewrite exemptions belong to this invocation, including when the same -- step instance is applied repeatedly to an already transformed tree. @@ -114,9 +120,9 @@ function ProxifyLocals:apply(ast, pipeline) localMetatableInfos[self.setMetatableSymbol] = false localMetatableInfos[self.emptyFunctionSymbol] = false self.emptyFunctionUsed = false - local emptyFunctionDeclaration = Ast.LocalVariableDeclaration({self.emptyFunctionSymbol}, { + local emptyFunctionDeclaration = Source.fill(Ast.LocalVariableDeclaration({self.emptyFunctionSymbol}, { Ast.FunctionLiteralExpression({}, Ast.Block({})), - }) + }), self.generatedSpan) skipped[emptyFunctionDeclaration] = true table.insert(ast.body.statements, 1, emptyFunctionDeclaration) @@ -131,13 +137,19 @@ function ProxifyLocals:apply(ast, pipeline) local info = getLocalMetatableInfo(statement.symbol) if info then local declaration = Ast.LocalVariableDeclaration({statement.symbol}, {}) + declaration.sourceSpan = statement.sourceSpan + Source.fill(declaration, self.generatedSpan) skipped[declaration] = true local literal = Ast.FunctionLiteralExpression(statement.args, statement.body) + literal.sourceSpan = statement.sourceSpan + Source.fill(literal, self.generatedSpan) local target = Ast.AssignmentVariable(statement.symbol) ignored[target] = true local assignment = Ast.AssignmentStatement({target}, { self:CreateAssignmentExpression(info, literal), }) + assignment.sourceSpan = statement.sourceSpan + Source.fill(assignment, self.generatedSpan) skipped[assignment] = true node.statements[i] = declaration table.insert(node.statements, i + 1, assignment) @@ -170,7 +182,9 @@ function ProxifyLocals:apply(ast, pipeline) ignored[reference] = true args[1] = info.setValue.constructor(reference, args[1]) self.emptyFunctionUsed = true - return Ast.FunctionCallStatement(Ast.VariableExpression(self.emptyFunctionSymbol), args) + local replacement = Ast.FunctionCallStatement(Ast.VariableExpression(self.emptyFunctionSymbol), args) + replacement.sourceSpan = node.sourceSpan + return Source.fill(replacement, self.generatedSpan) end end end, function(node) @@ -190,14 +204,18 @@ function ProxifyLocals:apply(ast, pipeline) elseif self.LiteralType == "number" then literal = RandomLiterals.Number() elseif self.LiteralType == "string" then literal = RandomLiterals.String(pipeline) else literal = RandomLiterals.Any(pipeline) end - return info.getValue.constructor(node, literal) + local replacement = info.getValue.constructor(node, literal) + replacement.sourceSpan = node.sourceSpan + return Source.fill(replacement, self.generatedSpan) end end if node.kind == AstKind.AssignmentVariable and not ignored[node] and not isGlobal(node) then local info = getLocalMetatableInfo(node.symbol) if info then ignored[node] = true - return Ast.AssignmentIndexing(node, Ast.StringExpression(info.valueName)) + local replacement = Ast.AssignmentIndexing(node, Ast.StringExpression(info.valueName)) + replacement.sourceSpan = node.sourceSpan + return Source.fill(replacement, self.generatedSpan) end end if node.kind == AstKind.FunctionDeclaration and node.symbol then @@ -206,9 +224,10 @@ function ProxifyLocals:apply(ast, pipeline) end end) - table.insert(ast.body.statements, 1, Ast.LocalVariableDeclaration({self.setMetatableSymbol}, { + table.insert(ast.body.statements, 1, Source.fill(Ast.LocalVariableDeclaration({self.setMetatableSymbol}, { Ast.GlobalVariableExpression("setmetatable"), - })) + }), self.generatedSpan)) + self.generatedSpan = nil end return ProxifyLocals diff --git a/src/prometheus/steps/SplitStrings.lua b/src/prometheus/steps/SplitStrings.lua index 26fad732..6742eba9 100644 --- a/src/prometheus/steps/SplitStrings.lua +++ b/src/prometheus/steps/SplitStrings.lua @@ -9,6 +9,7 @@ local Ast = require("prometheus.ast"); local visitAst = require("prometheus.visitast"); local Parser = require("prometheus.parser"); local util = require("prometheus.util"); +local Source = require("prometheus.source"); local enums = require("prometheus.enums") local LuaVersion = enums.LuaVersion; @@ -171,17 +172,17 @@ local function generateCustomFunctionLiteral(variant) -- Custom Function Type 1 if variant == 1 then - local funcDeclNode = parser:parse(custom1Code).body.statements[1]; + local funcDeclNode = parser:parse(custom1Code, Source.internal("SplitStrings.CustomFunction1", custom1Code)).body.statements[1]; local funcBody = funcDeclNode.body; local funcArgs = funcDeclNode.args; - return Ast.FunctionLiteralExpression(funcArgs, funcBody); + return Source.fill(Ast.FunctionLiteralExpression(funcArgs, funcBody), Source.generated("SplitStrings.CustomFunction")); -- Custom Function Type 2 else - local funcDeclNode = parser:parse(custom2Code).body.statements[1]; + local funcDeclNode = parser:parse(custom2Code, Source.internal("SplitStrings.CustomFunction2", custom2Code)).body.statements[1]; local funcBody = funcDeclNode.body; local funcArgs = funcDeclNode.args; - return Ast.FunctionLiteralExpression(funcArgs, funcBody); + return Source.fill(Ast.FunctionLiteralExpression(funcArgs, funcBody), Source.generated("SplitStrings.CustomFunction")); end end @@ -192,18 +193,18 @@ local function generateGlobalCustomFunctionDeclaration(data) -- Custom Function Type 1 if data.customFunctionVariant == 1 then - local funcDeclNode = parser:parse(custom1Code).body.statements[1]; + local funcDeclNode = parser:parse(custom1Code, Source.internal("SplitStrings.GlobalFunction1", custom1Code)).body.statements[1]; local funcBody = funcDeclNode.body; local funcArgs = funcDeclNode.args; - return Ast.LocalVariableDeclaration({data.customFuncSymbol}, - {Ast.FunctionLiteralExpression(funcArgs, funcBody)}); + return Source.fill(Ast.LocalVariableDeclaration({data.customFuncSymbol}, + {Ast.FunctionLiteralExpression(funcArgs, funcBody)}), Source.generated("SplitStrings.GlobalFunction")); -- Custom Function Type 2 else - local funcDeclNode = parser:parse(custom2Code).body.statements[1]; + local funcDeclNode = parser:parse(custom2Code, Source.internal("SplitStrings.GlobalFunction2", custom2Code)).body.statements[1]; local funcBody = funcDeclNode.body; local funcArgs = funcDeclNode.args; - return Ast.LocalVariableDeclaration({data.customFuncSymbol}, - {Ast.FunctionLiteralExpression(funcArgs, funcBody)}); + return Source.fill(Ast.LocalVariableDeclaration({data.customFuncSymbol}, + {Ast.FunctionLiteralExpression(funcArgs, funcBody)}), Source.generated("SplitStrings.GlobalFunction")); end end @@ -212,7 +213,7 @@ function SplitStrings:variant() end function SplitStrings:apply(ast, pipeline) - local data = {}; + local data = { generatedSpan = Source.generated("SplitStrings") }; if(self.ConcatenationType == "table") then @@ -254,7 +255,7 @@ function SplitStrings:apply(ast, pipeline) for i, func in ipairs(data.functionData.localFunctions) do if func.used then local literal = generateCustomFunctionLiteral(func.variant); - table.insert(node.statements, 1, Ast.LocalVariableDeclaration({func.symbol}, {literal})); + table.insert(node.statements, 1, Source.fill(Ast.LocalVariableDeclaration({func.symbol}, {literal}), data.generatedSpan)); end end end @@ -262,6 +263,7 @@ function SplitStrings:apply(ast, pipeline) -- Apply Only to String nodes if(node.kind == Ast.AstKind.StringExpression) then + local original = node; local str = node.value; local chunks = {}; local i = 1; @@ -300,14 +302,14 @@ function SplitStrings:apply(ast, pipeline) end end - return node, true; + return Source.inherit(node, original), true; end end, data) if(self.ConcatenationType == "table") then - table.insert(ast.body.statements, 1, Ast.LocalVariableDeclaration({data.tableConcatSymbol}, - {Ast.IndexExpression(Ast.GlobalVariableExpression("table"), Ast.StringExpression("concat"))})); + table.insert(ast.body.statements, 1, Source.fill(Ast.LocalVariableDeclaration({data.tableConcatSymbol}, + {Ast.IndexExpression(Ast.GlobalVariableExpression("table"), Ast.StringExpression("concat"))}), data.generatedSpan)); elseif(self.ConcatenationType == "custom" and self.CustomFunctionType == "global") then table.insert(ast.body.statements, 1, generateGlobalCustomFunctionDeclaration(data)); end diff --git a/src/prometheus/steps/Watermark.lua b/src/prometheus/steps/Watermark.lua index 603e3bc0..c54fd54c 100644 --- a/src/prometheus/steps/Watermark.lua +++ b/src/prometheus/steps/Watermark.lua @@ -6,6 +6,7 @@ local Step = require("prometheus.step"); local Ast = require("prometheus.ast"); +local Source = require("prometheus.source"); local Watermark = Step:extend(); Watermark.Description = "This Step will add a watermark to the script"; @@ -33,9 +34,10 @@ end function Watermark:apply(ast) local body = ast.body; if string.len(self.Content) > 0 then + local generatedSpan = Source.generated("Watermark"); local watermark = Ast.AssignmentGlobalVariable(self.CustomVariable); local arg = Ast.LocalSymbol("_watermark_arg"); - local statement = Ast.PassSelfFunctionCallStatement(Ast.StringExpression(self.Content), "gsub", { + local statement = Source.fill(Ast.PassSelfFunctionCallStatement(Ast.StringExpression(self.Content), "gsub", { Ast.StringExpression(".+"), Ast.FunctionLiteralExpression({ Ast.VariableExpression(arg) @@ -46,7 +48,7 @@ function Watermark:apply(ast) Ast.VariableExpression(arg) }) })) - }); + }), generatedSpan); table.insert(ast.body.statements, 1, statement) end diff --git a/src/prometheus/steps/WatermarkCheck.lua b/src/prometheus/steps/WatermarkCheck.lua index 48cb352e..ad422a03 100644 --- a/src/prometheus/steps/WatermarkCheck.lua +++ b/src/prometheus/steps/WatermarkCheck.lua @@ -7,6 +7,7 @@ local Step = require("prometheus.step"); local Ast = require("prometheus.ast"); local Watermark = require("prometheus.steps.Watermark"); +local Source = require("prometheus.source"); local WatermarkCheck = Step:extend(); WatermarkCheck.Description = "This Step will add a watermark to the script"; @@ -31,6 +32,7 @@ end function WatermarkCheck:init(_) end function WatermarkCheck:apply(ast, pipeline) + local generatedSpan = Source.generated("WatermarkCheck"); self.CustomVariable = "_" .. callNameGenerator(pipeline.namegenerator, math.random(10000000000, 100000000000)); pipeline:addStep(Watermark:new(self)); @@ -40,7 +42,7 @@ function WatermarkCheck:apply(ast, pipeline) local notEqualsExpression = Ast.NotEqualsExpression(watermark, watermarkExpression); local ifBody = Ast.Block({Ast.ReturnStatement({})}); - table.insert(body.statements, 1, Ast.IfStatement(notEqualsExpression, ifBody, {}, nil)); + table.insert(body.statements, 1, Source.fill(Ast.IfStatement(notEqualsExpression, ifBody, {}, nil), generatedSpan)); end return WatermarkCheck; diff --git a/src/prometheus/steps/WrapInFunction.lua b/src/prometheus/steps/WrapInFunction.lua index c3864771..2f612625 100644 --- a/src/prometheus/steps/WrapInFunction.lua +++ b/src/prometheus/steps/WrapInFunction.lua @@ -6,6 +6,7 @@ local Step = require("prometheus.step"); local Ast = require("prometheus.ast"); +local Source = require("prometheus.source"); local WrapInFunction = Step:extend(); WrapInFunction.Description = "This Step Wraps the Entire Script into a Function"; @@ -28,11 +29,11 @@ function WrapInFunction:apply(ast) for i = 1, self.Iterations, 1 do local body = ast.body; - ast.body = Ast.Block({ + ast.body = Source.fill(Ast.Block({ Ast.ReturnStatement({ Ast.FunctionCallExpression(Ast.FunctionLiteralExpression({Ast.VarargExpression()}, body), {Ast.VarargExpression()}) }); - }); + }), Source.generated("WrapInFunction")); end end diff --git a/src/prometheus/tokenizer.lua b/src/prometheus/tokenizer.lua index f80c8543..580708a1 100644 --- a/src/prometheus/tokenizer.lua +++ b/src/prometheus/tokenizer.lua @@ -45,7 +45,8 @@ Tokenizer.EOF_TOKEN = { } local function token(self, startPos, kind, value) - local line, linePos = self:getPosition(self.index); + local line, linePos = self:getPosition(startPos); + local endLine, endLinePos = self:getPosition(self.index); local annotations = self.annotations self.annotations = {}; return { @@ -56,6 +57,8 @@ local function token(self, startPos, kind, value) source = self.source:sub(startPos + 1, self.index), line = line, linePos = linePos, + endLine = endLine, + endLinePos = endLinePos, annotations = annotations, } end @@ -70,34 +73,37 @@ local function generateWarning(token, message) end function Tokenizer:getPosition(i) - local column = self.columnMap[i] + local position = self.positionMap[i] - if not column then --// `i` is bigger than self.length, this shouldn't happen, but it did. (Theres probably some error in the tokenizer, cant find it.) - column = self.columnMap[#self.columnMap] + if not position then + position = self.positionMap[self.length] or {line = 1, column = 1} end - return column.id, column.charMap[i] + return position.line, position.column end ---// Prepare columnMap for getPosition +--// Prepare byte-coordinate positions for every offset from 0 through length. function Tokenizer:prepareGetPosition() - local columnMap, column = {}, { charMap = {}, id = 1, length = 0 } - - for index = 1, self.length do - local character = string.sub(self.source, index, index) -- NOTE_1: this could use table.clone to reduce amount of NEWTABLE (if that causes any performance issues) - - local columnLength = column.length + 1 - column.length = columnLength - column.charMap[index] = columnLength - - if character == "\n" then - column = { charMap = {}, id = column.id + 1, length = 0 } -- NOTE_1 + local positionMap = {[0] = {line = 1, column = 1}} + local line, column = 1, 1 + local previousNewline + + for offset = 1, self.length do + local character = self.source:sub(offset, offset) + if character == "\r" or character == "\n" then + local paired = (character == "\r" and previousNewline == "\n") + or (character == "\n" and previousNewline == "\r") + if not paired then line = line + 1 end + column = 1 + previousNewline = character + else + column = column + 1 + previousNewline = nil end - - columnMap[index] = column + positionMap[offset] = {line = line, column = column} end - self.columnMap = columnMap + self.positionMap = positionMap end -- Constructor for Tokenizer @@ -146,6 +152,7 @@ function Tokenizer:new(settings) StringStartLookup = lookupify({"\"", "\'"}), annotations = {}, + positionMap = {[0] = {line = 1, column = 1}}, }; setmetatable(tokenizer, self); @@ -160,7 +167,7 @@ function Tokenizer:reset() self.length = 0; self.source = ""; self.annotations = {}; - self.columnMap = {}; + self.positionMap = {[0] = {line = 1, column = 1}}; end -- Append String to this Tokenizer @@ -489,7 +496,8 @@ end function Tokenizer:symbol() local startPos = self.index; - for len = self.MaxSymbolLength, 1, -1 do + local maxLength = math.min(self.MaxSymbolLength, self.length - self.index) + for len = maxLength, 1, -1 do local str = self.source:sub(self.index + 1, self.index + len); if self.SymbolsLookup[str] then self.index = self.index + len; diff --git a/src/prometheus/traceback.lua b/src/prometheus/traceback.lua new file mode 100644 index 00000000..c76d8296 --- /dev/null +++ b/src/prometheus/traceback.lua @@ -0,0 +1,258 @@ +-- Utilities for translating generated Lua traceback locations through a source map. + +local Traceback = {} + +local function normalize_path(path) + return (path or ""):gsub("\\", "/"):gsub("/+", "/") +end + +local function basename(path) + path = normalize_path(path) + return path:match("([^/]+)$") or path +end + +local function trim(value) + return value:match("^%s*(.-)%s*$") +end + +local function map_file_matches(filename, mapFile) + filename = trim(filename):gsub("^@", "") + mapFile = trim(mapFile or "") + if filename == "" or mapFile == "" then + return false + end + + local normalizedFilename = normalize_path(filename) + local normalizedMapFile = normalize_path(mapFile) + return normalizedFilename == normalizedMapFile or basename(normalizedFilename) == basename(normalizedMapFile) +end + +local function source_match(candidate, mapFile) + candidate = trim(candidate) + local normalizedCandidate = normalize_path(candidate:gsub("^@", "")) + local normalizedMapFile = normalize_path(trim(mapFile or "")) + if normalizedCandidate == normalizedMapFile then + return "" + end + + -- Lua runtimes prefix the main chunk error with their executable name, + -- such as "lua: " or "lua5.1: ". Only strip a runtime prefix when its + -- remainder matches the generated filename, so colons in paths are safe. + local prefix, rest = candidate:match("^([%a][%w_.-]*:%s+)(.*)$") + if prefix and map_file_matches(rest, mapFile) then + return prefix + end + if map_file_matches(candidate, mapFile) then + return "" + end + return nil +end + +local function find_lua_location(line, mapFile) + local searchAt = 1 + local selected + while true do + local first, afterNumber, number = line:find(":(%d+):", searchAt) + if not first then + break + end + + local candidate = line:sub(1, first - 1) + local wrapper = source_match(candidate, mapFile) + if wrapper then + selected = { + start = first, + line = tonumber(number), + wrapper = wrapper, + indent = line:match("^(%s*)") or "", + } + end + searchAt = afterNumber + 1 + end + return selected +end + +local function find_luau_location(line, mapFile) + local indent, body = line:match("^(%s*)(.*)$") + local prefix, filename, separator, number, suffix = body:match("^(Script%s+['\"])(.-)(['\"],%s*Line%s+)(%d+)(.*)$") + if prefix and source_match(filename, mapFile) then + return { + indent = indent, + luauPrefix = prefix, + luauSeparator = separator, + line = tonumber(number), + luauSuffix = suffix, + } + end + return nil +end + +local function split_lines(value) + local lines = {} + local start = 1 + while true do + local newline = value:find("\n", start, true) + if not newline then + if start <= #value then + lines[#lines + 1] = value:sub(start):gsub("\r$", "") + end + break + end + lines[#lines + 1] = value:sub(start, newline - 1):gsub("\r$", "") + start = newline + 1 + if start > #value then + break + end + end + return lines, value:sub(-1) == "\n" +end + +local function map_directory(mapPath) + local normalized = normalize_path(mapPath or "") + return normalized:match("^(.*)/[^/]*$") or "." +end + +local function is_absolute(path) + path = normalize_path(path) + return path:sub(1, 1) == "/" or path:match("^%a:/") ~= nil +end + +local function resolve_source_path(source, sourceRoot, mapPath) + local normalizedSource = normalize_path(source) + if is_absolute(normalizedSource) then + return normalizedSource + end + + local root = normalize_path(sourceRoot or "") + if root ~= "" and not is_absolute(root) then + root = map_directory(mapPath) .. "/" .. root + elseif root == "" then + root = map_directory(mapPath) + end + return normalize_path(root .. "/" .. normalizedSource) +end + +local function read_source(source, sourceRoot, mapPath) + local path = resolve_source_path(source, sourceRoot, mapPath) + local file = io.open(path, "rb") + if not file then + return nil, path + end + local content = file:read("*a") + file:close() + return content, path +end + +local function excerpt_lines(content) + local lines = {} + for line in (content .. "\n"):gmatch("(.-)\n") do + lines[#lines + 1] = line:gsub("\r$", "") + end + return lines +end + +local function source_map_api(explicitApi) + if explicitApi and type(explicitApi.lookup) == "function" then + return explicitApi + end + + -- Keeping the explicit API parameter optional makes this module convenient + -- to use directly while the CLI passes the public Prometheus.SourceMap API. + local ok, api = pcall(require, "prometheus.sourcemap") + if ok and type(api) == "table" and type(api.lookup) == "function" then + return api + end + return nil +end + +local function rewrite_lines(lines, map, mapApi) + local rewritten = {} + local locations = {} + for _, line in ipairs(lines) do + local luau = find_luau_location(line, map.file) + if luau then + local location = mapApi.lookup(map, luau.line) + if location then + local indent = luau.indent + line = indent .. luau.luauPrefix .. location.source .. luau.luauSeparator + .. tostring(location.line) .. luau.luauSuffix + locations[#locations + 1] = location + end + else + local location = find_lua_location(line, map.file) + if location then + local mapped = mapApi.lookup(map, location.line) + if mapped then + local before = line:sub(1, location.start - 1) + local prefix = before:match("^(%s*)") or "" + line = prefix .. location.wrapper .. mapped.source .. ":" .. tostring(mapped.line) + .. line:sub(location.start + #tostring(location.line) + 1) + locations[#locations + 1] = mapped + end + end + end + rewritten[#rewritten + 1] = line + end + return rewritten, locations +end + +local function append_excerpts(lines, locations, map, mapPath) + if #locations == 0 then + return lines + end + + local excerptBlocks = {} + local seen = {} + for _, location in ipairs(locations) do + local key = tostring(location.source) .. ":" .. tostring(location.line) + if not seen[key] then + seen[key] = true + local content = location.content + local sourcePath + if content == nil then + content, sourcePath = read_source(location.source, map.sourceRoot, mapPath) + end + + local block = { string.format(" %s:%d", location.source, location.line) } + if content == nil then + block[#block + 1] = " (source excerpt unavailable: could not read " .. tostring(sourcePath) .. ")" + else + local sourceLines = excerpt_lines(content) + local first = math.max(1, location.line - 2) + local last = math.min(#sourceLines, location.line + 2) + for lineNumber = first, last do + local marker = lineNumber == location.line and ">" or " " + block[#block + 1] = string.format(" %s %d | %s", marker, lineNumber, sourceLines[lineNumber]) + end + if #sourceLines == 0 then + block[#block + 1] = " (source excerpt unavailable: source is empty)" + end + end + excerptBlocks[#excerptBlocks + 1] = table.concat(block, "\n") + end + end + + lines[#lines + 1] = "" + lines[#lines + 1] = "Source excerpts:" + for _, block in ipairs(excerptBlocks) do + lines[#lines + 1] = block + end + return lines +end + +function Traceback.format(traceback, map, mapPath, explicitMapApi) + assert(type(traceback) == "string", "traceback must be a string") + assert(type(map) == "table", "source map must be a table") + local mapApi = source_map_api(explicitMapApi) + assert(mapApi, "source map lookup API is unavailable") + local lines, hadTrailingNewline = split_lines(traceback) + local rewritten, locations = rewrite_lines(lines, map, mapApi) + append_excerpts(rewritten, locations, map, mapPath) + local result = table.concat(rewritten, "\n") + if hadTrailingNewline then + result = result .. "\n" + end + return result +end + +return Traceback diff --git a/src/prometheus/unparser.lua b/src/prometheus/unparser.lua index 6396fc6d..b9af0355 100644 --- a/src/prometheus/unparser.lua +++ b/src/prometheus/unparser.lua @@ -17,6 +17,8 @@ local renameVariables = require("prometheus.names"); local Enums = require("prometheus.enums"); local util = require("prometheus.util"); local logger = require("logger"); +local Source = require("prometheus.source"); +local SourceMapWriter = require("prometheus.sourcemapwriter"); local lookupify = util.lookupify; local LuaVersion = Enums.LuaVersion; @@ -41,6 +43,8 @@ function Unparser:new(settings) identCharsLookup = lookupify(conventions.IdentChars); numberCharsLookup = lookupify(conventions.NumberChars); prettyPrint = settings and settings.PrettyPrint or false; + sourceMap = settings and settings.SourceMap or false; + includeSources = not settings or settings.SourceMapIncludeSources ~= false; notIdentPattern = "[^" .. table.concat(conventions.IdentChars, "") .. "]"; numberPattern = "^[" .. table.concat(conventions.NumberChars, "") .. "]"; highlight = settings and settings.Highlight or false; @@ -83,14 +87,14 @@ function Unparser:newline(ws_needed) end function Unparser:whitespaceIfNeeded(following, ws) - if(self.prettyPrint or self.identCharsLookup[string.sub(following, 1, 1)]) then + if(self.prettyPrint or self.identCharsLookup[self:edge(following, false)]) then return ws or self.SPACE; end return ""; end function Unparser:whitespaceIfNeeded2(leading, ws) - if(self.prettyPrint or self.identCharsLookup[string.sub(leading, #leading, #leading)]) then + if(self.prettyPrint or self.identCharsLookup[self:edge(leading, true)]) then return ws or self.SPACE; end return ""; @@ -104,6 +108,24 @@ function Unparser:whitespace(ws) return self.SPACE or ws; end +-- Lexical checks must see rendered characters, not private provenance tags. +function Unparser:edge(text, last) + if self._sourceSpans then + local pattern = last and '%z[^%z]*%z$' or '^%z[^%z]*%z' + while text:find(pattern) do text = text:gsub(pattern, '') end + end + return last and text:sub(-1) or text:sub(1, 1) +end + +function Unparser:tagText(text, node) + if not self._sourceSpans then return text end + local span = node.sourceSpan + assert(span, 'Missing source span on ' .. tostring(node.kind)) + local id = #self._sourceSpans + 1 + self._sourceSpans[id] = span + return '\0' .. id .. '\0' .. text .. '\0/\0' +end + function Unparser:unparse(ast) if(ast.kind ~= AstKind.TopNode) then logger:error("Unparser:unparse expects a TopNode as first argument") @@ -126,6 +148,17 @@ function Unparser:unparse(ast) end, }); + if self.sourceMap then + Source.fill(ast, Source.generated('unparser')) + self._sourceSpans = {}; + local ok, code, map = pcall(function() + local tagged = self:unparseBlock(ast.body) + return SourceMapWriter.render(tagged, self._sourceSpans, self.luaVersion, self.includeSources) + end) + self._sourceSpans = nil + if not ok then error(code, 0) end + return code, map + end return self:unparseBlock(ast.body); end @@ -144,17 +177,17 @@ function Unparser:unparseBlock(block, tabbing) for i, statement in ipairs(block.statements) do if(statement.kind ~= AstKind.NopStatement) then local statementCode = self:unparseStatement(statement, tabbing); - if(not self.prettyPrint and #parts > 0 and string.sub(statementCode, 1, 1) == "(") then + if(not self.prettyPrint and #parts > 0 and self:edge(statementCode, false) == "(") then -- This is so that the following works: -- print("Test");(function() print("Test2") end)(); - statementCode = ";" .. statementCode; + statementCode = self:tagText(";", statement) .. statementCode; end local ws = self:whitespaceIfNeeded2(#parts > 0 and parts[#parts] or "", self:whitespaceIfNeeded(statementCode, self:newline(true))); if i ~= 1 then parts[#parts + 1] = ws; end if(self.prettyPrint) then - statementCode = statementCode .. ";" + statementCode = statementCode .. self:tagText(";", statement) end parts[#parts + 1] = statementCode; end @@ -276,7 +309,7 @@ function Unparser:unparseStatement(statement, tabbing) if(arg.kind == AstKind.VarargExpression) then push("..."); else - push(arg.symbol.name); + push(self:tagText(arg.symbol.name, arg)); end end push(")"); @@ -295,7 +328,7 @@ function Unparser:unparseStatement(statement, tabbing) if(arg.kind == AstKind.VarargExpression) then push("..."); else - push(arg.symbol.name); + push(self:tagText(arg.symbol.name, arg)); end end push(")"); @@ -529,11 +562,11 @@ function Unparser:unparseExpression(expression, tabbing) rhs = "(" .. rhs .. ")" end - if self.numberCharsLookup[string.sub(lhs, #lhs, #lhs)] then + if self.numberCharsLookup[self:edge(lhs, true)] then lhs = lhs .. " " end - return lhs .. self:optionalWhitespace() .. (tostring(rhs):sub(1, 1) == "." and ".. " or "..") .. self:optionalWhitespace() .. rhs + return lhs .. self:optionalWhitespace() .. (self:edge(rhs, false) == "." and ".. " or "..") .. self:optionalWhitespace() .. rhs end local arithmeticOps = { @@ -558,7 +591,7 @@ function Unparser:unparseExpression(expression, tabbing) rhs = "(" .. rhs .. ")"; end - if op == "-" and string.sub(rhs, 1, 1) == "-" then + if op == "-" and self:edge(rhs, false) == "-" then rhs = "(" .. rhs .. ")"; end @@ -583,7 +616,7 @@ function Unparser:unparseExpression(expression, tabbing) rhs = "(" .. rhs .. ")"; end - if string.sub(rhs, 1, 1) == "-" then + if self:edge(rhs, false) == "-" then rhs = "(" .. rhs .. ")"; end @@ -609,7 +642,7 @@ function Unparser:unparseExpression(expression, tabbing) -- Identifier Indexing e.g: x.y instead of x["y"]; if(expression.index.kind == AstKind.StringExpression and self:isValidIdentifier(expression.index.value)) then - return base .. "." .. expression.index.value; + return base .. "." .. self:tagText(expression.index.value, expression.index); end -- Index never needs parens @@ -663,7 +696,7 @@ function Unparser:unparseExpression(expression, tabbing) if(arg.kind == AstKind.VarargExpression) then push("..."); else - push(arg.symbol.name); + push(self:tagText(arg.symbol.name, arg)); end end push(")"); @@ -698,7 +731,7 @@ function Unparser:unparseExpression(expression, tabbing) end if(entry.kind == AstKind.KeyedTableEntry) then if(entry.key.kind == AstKind.StringExpression and self:isValidIdentifier(entry.key.value)) then - push(entry.key.value); + push(self:tagText(entry.key.value, entry.key)); else push("[", self:unparseExpression(entry.key, tableTabbing), "]"); end @@ -737,4 +770,14 @@ function Unparser:unparseExpression(expression, tabbing) logger:error(string.format("\"%s\" is not a valid unparseable expression", expression.kind)); end +-- Reuse the established grammar renderer. Nested annotations supply origins +-- for punctuation/operators from their enclosing node and for child tokens +-- from the child node; the writer removes them before tokenizing the result. +for _, method in ipairs({'unparseBlock', 'unparseStatement', 'unparseExpression'}) do + local render = Unparser[method] + Unparser[method] = function(self, node, ...) + return self:tagText(render(self, node, ...), node) + end +end + return Unparser diff --git a/src/prometheus/vm/emit.lua b/src/prometheus/vm/emit.lua index 19c11caf..9854769b 100644 --- a/src/prometheus/vm/emit.lua +++ b/src/prometheus/vm/emit.lua @@ -4,6 +4,7 @@ local Enums = require('prometheus.enums') local Runtime = require('prometheus.vm.runtime') local Layout = require('prometheus.vm.layout') local Data = require('prometheus.vm.data') +local Source = require('prometheus.source') local Emit = {} function Emit.source(proto, layout) @@ -16,7 +17,8 @@ function Emit.source(proto, layout) end function Emit.ast(proto) - local ast = Parser:new({LuaVersion = Enums.LuaVersion.Lua51}):parse(Emit.source(proto)) + local source = Emit.source(proto) + local ast = Parser:new({LuaVersion = Enums.LuaVersion.Lua51}):parse(source, Source.internal('VM', source)) local invocation = ast.body.statements[#ast.body.statements].args[1] ast.vmRuntime = invocation.base -- Only the large shard literal stays native. Cff protects the reconstruction call/helper. diff --git a/tests/harness/parser-sources.lua b/tests/harness/parser-sources.lua new file mode 100644 index 00000000..8627eabd --- /dev/null +++ b/tests/harness/parser-sources.lua @@ -0,0 +1,97 @@ +local Prometheus = require('src.prometheus') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error + +local Parser = require('prometheus.parser') +local Source = require('prometheus.source') +local Tokenizer = require('prometheus.tokenizer') +local parser = Parser:new({LuaVersion = 'Lua51'}) + +local code = 'local x = a + 2 * (b - 3)\r\nreturn obj.foo(x, "hi")[1]' +local tree = parser:parse(code, 'precise.lua') +local nodes = {} +Source.walk(tree, function(node) + local span = assert(node.sourceSpan, node.kind .. ' has no source span') + assert(span.source.path == 'precise.lua') + assert(span.source.content == code) + assert(span.startOffset <= span.endOffset and span.endOffset <= #code) + nodes[node.kind] = nodes[node.kind] or {} + table.insert(nodes[node.kind], span) +end) + +local function hasSpan(kind, startOffset, endOffset) + for _, span in ipairs(nodes[kind] or {}) do + if span.startOffset == startOffset and span.endOffset == endOffset then return true end + end + return false +end + +assert(hasSpan('TopNode', 0, #code)) +assert(hasSpan('Block', 0, #code)) +assert(hasSpan('AddExpression', 10, 25)) +assert(hasSpan('MulExpression', 14, 25)) +assert(hasSpan('SubExpression', 18, 25)) +assert(hasSpan('FunctionCallExpression', 34, 50)) +assert(hasSpan('IndexExpression', 34, 53)) + +local semicolonTree = parser:parse('local x=nil;\nreturn x+1', 'semicolon-line.lua') +assert(semicolonTree.body.statements[2].sourceSpan.startLine == 2) +assert(semicolonTree.body.statements[2].sourceSpan.startColumn == 1) + +local declarations = parser:parse('function f(arg, ...) return {name=arg, [1]=...} end', 'entries.lua') +local entries, args = 0, 0 +Source.walk(declarations, function(node) + assert(node.sourceSpan and node.sourceSpan.source.path == 'entries.lua') + if node.kind == 'KeyedTableEntry' then entries = entries + 1 end + if node.kind == 'VarargExpression' or (node.kind == 'VariableExpression' and node.sourceSpan.startOffset == 11) then + args = args + 1 + end +end) +assert(entries == 2 and args >= 2, 'Table entries and parameters must have spans') + +local internalA = parser:parse('') +local internalB = parser:parse('') +assert(internalA.sourceSpan.source.internal and internalB.sourceSpan.source.internal) +assert(internalA.sourceSpan.source.path ~= internalB.sourceSpan.source.path) +assert(internalA.sourceSpan.startOffset == 0 and internalA.sourceSpan.endOffset == 0) +local explicitRecord = Source.new('record.lua', 'return 2') +assert(parser:parse(explicitRecord.content, explicitRecord).sourceSpan.source == explicitRecord) + +local tokenizer = Tokenizer:new({LuaVersion = 'Lua51'}) +tokenizer:append('a\r\nb') +local tokens = tokenizer:scanAll() +assert(tokens[1].line == 1 and tokens[1].linePos == 1) +assert(tokens[1].endLine == 1 and tokens[1].endLinePos == 2) +assert(tokens[1].startPos == 0 and tokens[1].endPos == 1) +assert(tokens[2].line == 2 and tokens[2].linePos == 1) +assert(tokens[2].endLine == 2 and tokens[2].endLinePos == 2) +assert(tokens[3].line == 2 and tokens[3].linePos == 2) +assert(tokens[3].endLine == 2 and tokens[3].endLinePos == 2) + +local emptyTokenizer = Tokenizer:new({LuaVersion = 'Lua51'}) +emptyTokenizer:append('') +local eof = emptyTokenizer:scanAll()[1] +assert(eof.startPos == 0 and eof.endPos == 0) +assert(eof.line == 1 and eof.linePos == 1 and eof.endLine == 1 and eof.endLinePos == 1) +local crlfTokenizer = Tokenizer:new({LuaVersion = 'Lua51'}) +crlfTokenizer:append('\r\n') +local crlfEof = crlfTokenizer:scanAll()[1] +assert(crlfEof.line == 2 and crlfEof.linePos == 1) +assert(crlfEof.endLine == 2 and crlfEof.endLinePos == 1) +local lfcrTokenizer = Tokenizer:new({LuaVersion = 'Lua51'}) +lfcrTokenizer:append('a\n\rb') +local lfcrTokens = lfcrTokenizer:scanAll() +assert(lfcrTokens[2].line == 2 and lfcrTokens[2].linePos == 1) +for _, newline in ipairs({'\n\n', '\r\r'}) do + local repeatedTokenizer = Tokenizer:new({LuaVersion = 'Lua51'}) + repeatedTokenizer:append('a' .. newline .. 'b') + local repeatedTokens = repeatedTokenizer:scanAll() + assert(repeatedTokens[2].line == 3 and repeatedTokens[2].linePos == 1) +end + +parser.disableLog = true +assert(not pcall(function() parser:parse('local =') end)) +assert(parser.disableLog, 'parse should preserve the caller logging preference') +parser.disableLog = false +assert(parser:parse('return 1', 'after-error.lua').sourceSpan.source.path == 'after-error.lua') + +print('Parser source span and tokenizer coordinate checks passed') diff --git a/tests/harness/source-maps.lua b/tests/harness/source-maps.lua new file mode 100644 index 00000000..fc0b2467 --- /dev/null +++ b/tests/harness/source-maps.lua @@ -0,0 +1,122 @@ +local Prometheus = require('src.prometheus') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error +local SourceMap = Prometheus.SourceMap +local Source = require('prometheus.source') +local Parser = require('prometheus.parser') +local Ast = require('prometheus.ast') +local Json = require('prometheus.json') + +-- Independent v3 fixture: source/name/line deltas persist between lines, +-- generated columns reset, and column-only segments remain unmapped. +local fixture = SourceMap.decode([[{"version":3,"file":"out.lua","sources":["one.lua","two.lua"],"names":["value"],"sourcesContent":["one",null],"mappings":"AAAAA;ACGA;ADDA;A;KACA"}]]) +assert(SourceMap.lookup(fixture, 1).line == 1) +assert(SourceMap.lookup(fixture, 2).source == 'two.lua') +assert(SourceMap.lookup(fixture, 2).line == 4) +assert(SourceMap.lookup(fixture, 2).content == nil) +assert(SourceMap.lookup(fixture, 3).line == 3) +assert(SourceMap.lookup(fixture, 4) == nil) +assert(SourceMap.lookup(fixture, 5).line == 4) +assert(SourceMap.lookup(SourceMap.decode(SourceMap.encode(fixture)), 5).line == 4) +assert(not pcall(SourceMap.decode, [[{"version":3,"sources":{},"names":[],"mappings":""}]])) +for _, mapping in ipairs({'!', 'g', 'AA', 'AAAAAA', 'ACAA', 'AADA', 'AAAA,', 'AAAA,D'}) do + local map = {version = 3, sources = {'one.lua'}, names = {}, mappings = mapping} + assert(not pcall(SourceMap.encode, map), 'Should reject mapping ' .. mapping) +end +assert(Json.decode('"\\ud83d\\ude00"') == '\240\159\152\128') +for _, input in ipairs({'01', '[1,]', '{"a":}', '"\\ud800"', 'true false'}) do + assert(not pcall(Json.decode, input), 'Should reject JSON ' .. input) +end + +local function pipeline(pretty, steps, dialect) + return Prometheus.Pipeline:fromConfig({LuaVersion = dialect or 'Lua51', Seed = 7, + SourceMap = true, PrettyPrint = pretty, Steps = steps or {}}) +end + +-- Split tokens even where the grammar previously needed no space. The source +-- span on the enclosing operator supplies its line independently of operands. +for _, pretty in ipairs({false, true}) do + local ast = Parser:new({LuaVersion = 'Lua51'}):parse('return 1', 'split.lua') + local expression = Source.inherit(Ast.AddExpression(Ast.NumberExpression(1), Ast.NumberExpression(2)), + ast.body.statements[1].args[1]) + ast.body.statements[1].args[1] = expression + expression.rhs.sourceSpan = {source = expression.sourceSpan.source, startLine = 2, + startColumn = 1, endLine = 2, endColumn = 2, startOffset = 0, endOffset = 1} + local code, map = pipeline(pretty):unparse(ast) + assert(code:find('\n', 1, true)) + assert(assert(loadstring(code))() == 3) + local found = false + for line = 1, select(2, code:gsub('\n', '')) + 1 do + local mapped = SourceMap.lookup(map, line) + if mapped and mapped.line == 2 then found = true end + end + assert(found, 'Missing mapping for RHS token') +end + +local codeExamples = { + 'return 1 .. .2', + 'return 1- -2, - -3', + 'local f=function() return 3 end; f(); (function() return f() end)(); return f()', + 'local t={a="\\00012",["b"]=2}; return #t.a,t.b', + 'local function f(...) return (...) end; return f(nil,2)', + 'local x=1; return function() return x end', + 'return [[first\nsecond]], "\\n\\r\\000"', +} +for _, code in ipairs(codeExamples) do + local original = assert(loadstring(code)) + for _, pretty in ipairs({false, true}) do + local generated, map = pipeline(pretty):apply(code, 'examples.lua') + local result = {assert(loadstring(generated))()} + local expected = {original()} + for i, value in ipairs(expected) do + if type(value) ~= 'function' then assert(result[i] == value, generated) end + end + assert(not generated:find('\0', 1, true), 'Private annotations escaped into output') + assert(SourceMap.decode(SourceMap.encode(map)).version == 3) + end +end + +-- Genuine runtime errors must recover the failing user line through ordinary +-- transformations and Cff. Generated VM frames deliberately remain internal. +local input = 'local x={}\nlocal function fail()\n return x.missing + 1\nend\nreturn fail()\n' +local configs = { + {}, + {{Name = 'NumbersToExpressions', Settings = {Threshold = 1}}}, + {{Name = 'SplitStrings', Settings = {Threshold = 1}}}, + {{Name = 'EncryptStrings', Settings = {}}}, + {{Name = 'ConstantArray', Settings = {Threshold = 1}}}, + {{Name = 'WrapInFunction', Settings = {}}}, + {{Name = 'ProxifyLocals', Settings = {}}}, + {{Name = 'Cff', Settings = {}}}, +} +for _, steps in ipairs(configs) do + for _, pretty in ipairs({false, true}) do + local generated, map = pipeline(pretty, steps):apply(input, 'failing.lua') + local chunk = assert(loadstring(generated, '@mapped.lua')) + local ok, err = pcall(chunk) + assert(not ok, 'Fixture must fail') + local line = tonumber(tostring(err):match('mapped%.lua:(%d+):')) + assert(line, tostring(err)) + local recovered = SourceMap.lookup(map, line) + assert(recovered and recovered.source == 'failing.lua' and recovered.line == 3, + (steps[1] and steps[1].Name or 'Minify') .. ': ' .. tostring(err) .. '\n' .. + (recovered and recovered.source .. ':' .. recovered.line or 'unmapped')) + end +end +for _, pretty in ipairs({false, true}) do + local generated, map = pipeline(pretty, {{Name = 'Vmify'}}):apply(input, 'failing.lua') + local ok, err = pcall(assert(loadstring(generated, '@mapped.lua'))) + assert(not ok) + local recovered = SourceMap.lookup(map, tonumber(tostring(err):match('mapped%.lua:(%d+):'))) + assert(recovered and recovered.source:match('^prometheus%-internal://VM/'), tostring(err)) +end + +local unmappedPipeline = Prometheus.Pipeline:fromConfig({LuaVersion = 'Lua51', Seed = 7, Steps = {}}) +local plain, absent = unmappedPipeline:apply('return 1', 'plain.lua') +assert(absent == nil and not plain:find('\n', 1, true)) +local omitted = pipeline(false) +omitted.SourceMapIncludeSources = false +local _, map = omitted:apply('return 1', 'plain.lua') +assert(map.sourcesContent == nil) +assert(not SourceMap.encode(map):find('sourcesContent', 1, true)) + +print('Source-map codec, token emission, transformation and runtime recovery checks passed') diff --git a/tests/harness/source-maps.py b/tests/harness/source-maps.py new file mode 100644 index 00000000..aab38c8d --- /dev/null +++ b/tests/harness/source-maps.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""Exercise source map generation and traceback rewriting through the CLI.""" +import json +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +CLI = ROOT / "cli.lua" +LUA51 = "lua5.1" +LUAU = "luau" +TIMEOUT = 20 + + +def run(command, cwd, *, input_text=None, timeout=TIMEOUT): + return subprocess.run(command, cwd=cwd, input=input_text, text=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=timeout) + + +def check(command, cwd, *, input_text=None): + result = run(command, cwd, input_text=input_text) + assert result.returncode == 0, f"command failed ({result.returncode}): {command}\n{result.stdout}" + return result.stdout + + +def fail(command, cwd, expected): + result = run(command, cwd) + assert result.returncode != 0, f"command unexpectedly succeeded: {command}\n{result.stdout}" + assert expected in result.stdout, f"missing {expected!r}:\n{result.stdout}" + return result.stdout + + +def lua_cli(*args, cwd): + return [LUA51, str(CLI), *map(str, args)] + + +def traceback_command(map_path, trace_path="-"): + return [LUA51, str(CLI), "trace", "--map", str(map_path), str(trace_path)] + + +def make_failure_sources(directory): + # Shared leading whitespace, quoted string, and multiline literal exercise + # source spans around syntax that occupies more than one generated line. + prelude = '''-- source-map whitespace and literal coverage +local message = "quoted -- text" +local block = [=[ + multiline literal +]=] + +''' + cases = { + "arithmetic": ("local function fail()\n" + " local value = nil\n" + " return value + 1\n" + "end\n\nfail()\n", 9), + "indexing": ("local function fail()\n" + " local value = {}\n" + " return value.missing.member\n" + "end\n\nfail()\n", 9), + "call": ("local function fail()\n" + " local value = 42\n" + " return value()\n" + "end\n\nfail()\n", 9), + "closure": ("local captured = 1\n" + "local function outer()\n" + " return function()\n" + " return captured + nil\n" + " end\n" + "end\n\nouter()()\n", 10), + "multiline-index": ("local function fail()\n" + " local target = nil\n" + " return\n" + " target\n" + " [1]\n" + "end\n\nfail()\n", 10), + } + paths = {} + for name, (body, expected_line) in cases.items(): + path = directory / f"original {name}: source.lua" + path.write_text(prelude + body, encoding="utf-8", newline="") + paths[name] = (path, expected_line, prelude + body) + return paths + + +def generate(directory, source_path, output_path, map_path, target, pretty=False): + args = ["--preset", "Minify", "--out", output_path.name, "--source-map", map_path.name, + "--Lua51" if target == "Lua51" else "--LuaU"] + if pretty: + args.append("--pretty") + args.append(source_path.relative_to(directory)) + check(lua_cli(*args, cwd=directory), directory) + assert output_path.is_file(), output_path + assert map_path.is_file(), map_path + + +with tempfile.TemporaryDirectory(prefix="prometheus-source-maps-") as temporary: + base = Path(temporary) + fixture_dir = base / "fixtures" + fixture_dir.mkdir() + cases = make_failure_sources(fixture_dir) + + # Both source targets and output layouts are generated by the CLI. Each + # output is executed by both available runtimes to collect real tracebacks. + for target in ("Lua51", "LuaU"): + for pretty in (False, True): + for name, (source_path, expected_line, source_text) in cases.items(): + case_dir = base / f"{target}-{int(pretty)}-{name}" + case_dir.mkdir() + local_source = case_dir / source_path.name + local_source.write_bytes(source_path.read_bytes()) + output_path = case_dir / "generated output: sample.lua" + map_path = case_dir / "generated output: sample.lua.map" + generate(case_dir, local_source, output_path, map_path, target, pretty) + + source_map = json.loads(map_path.read_text(encoding="utf-8")) + assert source_map["file"] == output_path.name, source_map + assert source_map.get("sourcesContent", [None])[0] == source_text, source_map + + for runtime in (LUA51, LUAU): + if name == "multiline-index": + baseline = run([runtime, str(local_source)], case_dir) + assert baseline.returncode != 0, baseline.stdout + source_error = re.search(re.escape(local_source.name) + r":(\d+):", baseline.stdout) + assert source_error, baseline.stdout + baseline_line = int(source_error.group(1)) + if name == "multiline-index": + # Lua runtimes may report either the base token or + # closing index token for this single AST span. + assert expected_line <= baseline_line <= expected_line + 1, baseline.stdout + else: + assert baseline_line == expected_line, baseline.stdout + result = run([runtime, str(output_path)], case_dir) + assert result.returncode != 0, f"expected {runtime} failure for {name}" + assert output_path.name in result.stdout, result.stdout + # Run every real runtime traceback through trace, reading + # from stdin, and retain C/unrelated frames verbatim. + raw_trace = result.stdout.rstrip() + ( + "\n\t[C]: in function 'unrelated_c_frame'" + "\n\tother module.lua:77: in function 'unrelated'\n") + formatted = check(traceback_command(map_path), case_dir, input_text=raw_trace) + expected_location = f"original {name}: source.lua:{expected_line}" + if name == "multiline-index": + mapped_lines = (expected_line, expected_line + 1) + assert any(f"original {name}: source.lua:{line}" in formatted + for line in mapped_lines), (runtime, expected_location, formatted) + assert any(f"> {line} | " in formatted for line in mapped_lines), (runtime, formatted) + assert "11 | [1]" in formatted, formatted + else: + assert expected_location in formatted, (runtime, expected_location, formatted) + assert f"> {expected_line} | " in formatted, (runtime, formatted) + assert "[C]: in function 'unrelated_c_frame'" in formatted, formatted + assert "other module.lua:77: in function 'unrelated'" in formatted, formatted + if runtime == LUA51: + assert formatted.startswith("lua5.1: "), formatted + + # With source text omitted, trace resolves a relative source path from the + # map's directory and emits the same excerpt. + fallback_dir = base / "fallback" + (fallback_dir / "sources").mkdir(parents=True) + (fallback_dir / "build").mkdir() + source_path = fallback_dir / "sources" / "without embedded source.lua" + source_text = "local value = nil\nreturn value + 1\n" + source_path.write_text(source_text, encoding="utf-8", newline="") + config_path = fallback_dir / "source-map.config.lua" + config_path.write_text("return { LuaVersion = 'Lua51', Steps = {}, SourceMap = true, " + "SourceMapIncludeSources = false }\n", encoding="utf-8") + output_path = fallback_dir / "build" / "generated.lua" + map_path = fallback_dir / "build" / "generated.lua.map" + check(lua_cli("--config", config_path.name, "--Lua51", "--out", str(output_path), + "sources/without embedded source.lua", cwd=fallback_dir), fallback_dir) + source_map = json.loads(map_path.read_text(encoding="utf-8")) + assert "sourcesContent" not in source_map, source_map + assert source_map["file"] == output_path.name, source_map + assert source_map["sources"] == ["../sources/without embedded source.lua"], source_map + runtime = run([LUA51, str(output_path)], fallback_dir) + assert runtime.returncode != 0, runtime.stdout + formatted = check(traceback_command(map_path), fallback_dir, input_text=runtime.stdout) + assert "without embedded source.lua:2" in formatted, formatted + assert "> 2 | return value + 1" in formatted, formatted + + # Relative input and map paths are rebased lexically without needing the + # process working directory. + relative_output = fallback_dir / "build" / "relative-generated.lua" + check(lua_cli("--config", config_path.name, "--Lua51", "--out", "build/relative-generated.lua", + "sources/without embedded source.lua", cwd=fallback_dir), fallback_dir) + relative_map = json.loads(Path(str(relative_output) + ".map").read_text(encoding="utf-8")) + assert relative_map["sources"] == ["../sources/without embedded source.lua"], relative_map + + # On case-sensitive filesystems, a path component that differs only by + # case is not part of the common prefix when rebasing absolute paths. + case_root = base / "case-sensitive-paths" + upper_source_dir = case_root / "Sources" + lower_map_dir = case_root / "sources" + upper_source_dir.mkdir(parents=True) + try: + lower_map_dir.mkdir() + case_sensitive = not upper_source_dir.samefile(lower_map_dir) + except FileExistsError: + case_sensitive = False + if case_sensitive: + case_source = upper_source_dir / "file.lua" + case_source.write_text(source_text, encoding="utf-8", newline="") + case_output = lower_map_dir / "output.lua" + check(lua_cli("--config", str(config_path), "--Lua51", "--out", str(case_output), + "Sources/file.lua", cwd=case_root), case_root) + case_map_path = Path(str(case_output) + ".map") + case_map = json.loads(case_map_path.read_text(encoding="utf-8")) + assert case_map["sources"] == ["../Sources/file.lua"], case_map + case_runtime = run([LUA51, str(case_output)], case_root) + assert case_runtime.returncode != 0, case_runtime.stdout + case_formatted = check(traceback_command(case_map_path), case_root, + input_text=case_runtime.stdout) + assert "file.lua:2" in case_formatted, case_formatted + assert "> 2 | return value + 1" in case_formatted, case_formatted + + # Invalid maps report a useful CLI error. + invalid_map = fallback_dir / "invalid.map" + invalid_map.write_text("{not valid JSON", encoding="utf-8") + fail(traceback_command(invalid_map), fallback_dir, "Invalid source map") + + # A map path that aliases the generated file must be rejected before any + # output is written over it. + same_path = fallback_dir / "same output.lua" + same_source = "print('keep the Lua output intact')\n" + same_path.write_text(same_source, encoding="utf-8", newline="") + fail(lua_cli("--preset", "Minify", "--out", same_path.name, "--source-map", same_path.name, + same_path.name, cwd=fallback_dir), fallback_dir, "must differ from the generated output path") + assert same_path.read_text(encoding="utf-8") == same_source + + # Luau-only compounds and if-expressions survive both unparser layouts. + luau_source = base / "luau-syntax.lua" + luau_source.write_text('''local value = 1 +value += 2 +local message = if value == 3 then "ok" else "bad" +assert(message == "ok") +''', encoding="utf-8", newline="") + for pretty in (False, True): + luau_dir = base / f"luau-syntax-{int(pretty)}" + luau_dir.mkdir() + local_source = luau_dir / luau_source.name + local_source.write_bytes(luau_source.read_bytes()) + output_path = luau_dir / "luau-output.lua" + map_path = luau_dir / "luau-output.lua.map" + generate(luau_dir, local_source, output_path, map_path, "LuaU", pretty) + check([LUAU, str(output_path)], luau_dir) + +print("Source map CLI integration checks passed") diff --git a/tests/harness/step-sources.lua b/tests/harness/step-sources.lua new file mode 100644 index 00000000..b192fdad --- /dev/null +++ b/tests/harness/step-sources.lua @@ -0,0 +1,190 @@ +local Prometheus = require('src.prometheus') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error + +local oldPath = package.path +package.path = 'src/?.lua;' .. package.path +local Ast = require('prometheus.ast') +local Parser = require('prometheus.parser') +local Source = require('prometheus.source') +local visitast = require('prometheus.visitast') +local Steps = require('prometheus.steps') +local Watermark = require('prometheus.steps.Watermark') +local Emit = require('prometheus.vm.emit') +local Compiler = require('prometheus.vm.compiler') +local shareCffStrings = require('prometheus.cff.strings') +package.path = oldPath + +local parser = Parser:new({LuaVersion = 'Lua51'}) +local originalPath = 'step-sources-input.lua' + +local function parse(code) + return parser:parse(code, originalPath) +end + +local function findNode(root, kind, predicate) + local found + Source.walk(root, function(node) + if not found and node.kind == kind and (not predicate or predicate(node)) then found = node end + end) + return found +end + +local function checkSpans(root, predicate) + local count = 0 + Source.walk(root, function(node) + count = count + 1 + assert(type(node.sourceSpan) == 'table', 'missing source span on ' .. tostring(node.kind)) + assert(predicate(node.sourceSpan, node), 'unexpected source span on ' .. tostring(node.kind)) + end) + assert(count > 0, 'expected AST nodes') +end + +local function original(span) + return span.source.path == originalPath +end + +local function internal(label) + return function(span) + return span.source.internal and span.source.path:find('prometheus-internal://' .. label .. '/', 1, true) == 1 + end +end + +local function hasInternal(root, label) + local found = false + Source.walk(root, function(node) + local span = node.sourceSpan + if span and span.source.internal and span.source.path:find('prometheus-internal://' .. label .. '/', 1, true) == 1 then + found = true + end + end) + return found +end + +-- Replacing one literal with an expression tree keeps the original source span. +local numbers = parse('return 42') +Steps.NumbersToExpressions:new({Threshold = 1, InternalThreshold = 0}):apply(numbers) +checkSpans(numbers.body.statements[1].args[1], original) + +local split = parse('return "abcdef"') +Steps.SplitStrings:new({ConcatenationType = 'strcat', MinLength = 2, MaxLength = 2, Threshold = 1}):apply(split) +checkSpans(split.body.statements[1].args[1], original) + +local customSplitStep = Steps.SplitStrings:new({ConcatenationType = 'custom', CustomFunctionType = 'inline', + MinLength = 2, MaxLength = 2, Threshold = 1}) +local customSplitA = parse('return "abcdef"') +local customSplitB = parse('return "uvwxyz"') +customSplitStep:apply(customSplitA) +customSplitStep:apply(customSplitB) +assert(hasInternal(customSplitA, 'SplitStrings.CustomFunction') + or hasInternal(customSplitA, 'SplitStrings.CustomFunction1') + or hasInternal(customSplitA, 'SplitStrings.CustomFunction2'), 'custom split helper should have an internal source') +local function snippetPath(root) + local path + Source.walk(root, function(node) + local span = node.sourceSpan + if not path and span and span.source.path:find('prometheus-internal://SplitStrings.CustomFunction', 1, true) == 1 then + path = span.source.path + end + end) + return path +end +assert(snippetPath(customSplitA) ~= snippetPath(customSplitB), + 'reusing a SplitStrings step should allocate unique helper source IDs') + +-- ConstantArray keeps transformed value accesses tied to the replaced literal, +-- while decoder and array support code receives internal source identities. +local constants = parse('return "one", "two"') +local constantStep = Steps.ConstantArray:new({Encoding = 'mixed', StringsOnly = true, Rotate = true}) +constantStep:apply(constants) +local constantReturn = findNode(constants, 'ReturnStatement', function(node) return original(node.sourceSpan) end) +checkSpans(constantReturn.args[1], original) +assert(findNode(constants, 'ForStatement'), 'expected generated rotation/decoder loop') +local decoderSource = false +Source.walk(constants, function(node) + local span = node.sourceSpan + if span and span.source.path:find('prometheus-internal://ConstantArray.MixedDecoder/', 1, true) == 1 then + decoderSource = true + end +end) +assert(decoderSource, 'decoder snippet should keep its internal source identity') +local constantAgain = parse('return "three", "four"') +constantStep:apply(constantAgain) +local function firstGeneratedArrayPath(root) + local path + Source.walk(root, function(node) + if not path and node.kind == 'LocalVariableDeclaration' and internal('ConstantArray')(node.sourceSpan) then + path = node.sourceSpan.source.path + end + end) + return path +end +assert(firstGeneratedArrayPath(constants) ~= firstGeneratedArrayPath(constantAgain), + 'reusing a ConstantArray step should allocate unique generated source IDs') + +local encrypted = parse('return "secret"') +Steps.EncryptStrings:new({}):apply(encrypted) +local encryptedReturn = findNode(encrypted, 'ReturnStatement', function(node) return original(node.sourceSpan) end) +checkSpans(encryptedReturn.args[1], original) +local decryptSource = false +Source.walk(encrypted, function(node) + local span = node.sourceSpan + if span and span.source.path:find('prometheus-internal://EncryptStrings.Decryptor/', 1, true) == 1 then + decryptSource = true + end +end) +assert(decryptSource, 'decryptor snippet should keep its internal source identity') + +local proxified = parse('local x = 3; return x') +Steps.ProxifyLocals:new({LiteralType = 'number'}):apply(proxified, { + namegenerator = function() return 'source_test' end, +}) +local proxifiedValue = proxified.body.statements[#proxified.body.statements].args[1] +assert(original(proxifiedValue.sourceSpan), 'proxified expression should retain the replaced expression origin') +assert(findNode(proxified, 'LocalVariableDeclaration', function(node) + return node.sourceSpan.source.path:find('prometheus-internal://ProxifyLocals/', 1, true) == 1 +end), 'proxify support declarations should be marked generated') +checkSpans(proxified, function(span) + return span.source.path == originalPath or span.source.path:find('prometheus-internal://ProxifyLocals/', 1, true) == 1 +end) + +local vararg = parse('local f = function(x) return x end; return f') +Steps.AddVararg:new():apply(vararg) +local addedVararg = findNode(vararg, 'FunctionLiteralExpression').args[2] +assert(addedVararg.kind == 'VarargExpression' and internal('AddVararg')(addedVararg.sourceSpan), + 'added vararg should have its own generated source') + +local wrapped = parse('return 1') +local movedBody = wrapped.body +Steps.WrapInFunction:new({Iterations = 1}):apply(wrapped) +assert(wrapped.body.sourceSpan.source.internal, 'wrapper block should be generated') +assert(wrapped.body.statements[1].args[1].base.body == movedBody, + 'wrapping should preserve the moved source body') +assert(original(movedBody.statements[1].args[1].sourceSpan), 'moved source body origin should be preserved') + +local watermarked = parse('return 1') +Watermark:new({Content = 'mark', CustomVariable = '_mark'}):apply(watermarked) +assert(internal('Watermark')(watermarked.body.statements[1].sourceSpan), 'watermark code should be generated') + +local checked = parse('return 1') +local scheduled +Steps.WatermarkCheck:new({Content = 'mark'}):apply(checked, { + namegenerator = function() return 'mark' end, + addStep = function(_, step) scheduled = step end, +}) +assert(internal('WatermarkCheck')(checked.body.statements[1].sourceSpan), 'watermark check should be generated') +assert(scheduled and scheduled.Name == 'Watermark', 'watermark step should still be scheduled') + +local untouched = parse('return 1') +local antiTamper = Steps.AntiTamper:new({}) +antiTamper:apply(untouched, {PrettyPrint = false, SourceMap = true}) +assert(#untouched.body.statements == 1, 'AntiTamper should be skipped with source maps enabled') + +local shared = parse('return "repeat", "repeat"') +shareCffStrings(shared) +assert(internal('Cff.StringPool')(shared.body.statements[1].sourceSpan), 'Cff string pool should be generated') +assert(original(shared.body.statements[2].args[1].sourceSpan), 'pooled lookup should inherit the replaced literal origin') + +local vmAst = Emit.ast(Compiler:compile(parse('return 7'))) +checkSpans(vmAst, internal('VM')) + +print('Step source propagation checks passed') From fb88eabbc7d78d5ed0552a94d321f321dd059caf Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:29:52 +0100 Subject: [PATCH 2/6] Fix source positions for consecutive paired newlines --- src/prometheus/tokenizer.lua | 4 ++- tests/harness/parser-sources.lua | 42 +++++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/prometheus/tokenizer.lua b/src/prometheus/tokenizer.lua index 580708a1..7c11b5f2 100644 --- a/src/prometheus/tokenizer.lua +++ b/src/prometheus/tokenizer.lua @@ -95,7 +95,9 @@ function Tokenizer:prepareGetPosition() or (character == "\n" and previousNewline == "\r") if not paired then line = line + 1 end column = 1 - previousNewline = character + -- Consume CRLF/LFCR as a pair; its second byte cannot also + -- start a pair with the following newline. + previousNewline = not paired and character or nil else column = column + 1 previousNewline = nil diff --git a/tests/harness/parser-sources.lua b/tests/harness/parser-sources.lua index 8627eabd..bc18c65b 100644 --- a/tests/harness/parser-sources.lua +++ b/tests/harness/parser-sources.lua @@ -1,5 +1,6 @@ local Prometheus = require('src.prometheus') Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error +local SourceMap = Prometheus.SourceMap local Parser = require('prometheus.parser') local Source = require('prometheus.source') @@ -77,10 +78,43 @@ crlfTokenizer:append('\r\n') local crlfEof = crlfTokenizer:scanAll()[1] assert(crlfEof.line == 2 and crlfEof.linePos == 1) assert(crlfEof.endLine == 2 and crlfEof.endLinePos == 1) -local lfcrTokenizer = Tokenizer:new({LuaVersion = 'Lua51'}) -lfcrTokenizer:append('a\n\rb') -local lfcrTokens = lfcrTokenizer:scanAll() -assert(lfcrTokens[2].line == 2 and lfcrTokens[2].linePos == 1) +local pairedNewlines = { + {'\r\n\r\n', 3}, + {'\n\r\n\r', 3}, + {'\r\n\n\r', 3}, + {'\n\r\r\n', 3}, +} +for _, case in ipairs(pairedNewlines) do + local newline, expectedLine = case[1], case[2] + local pairedTokenizer = Tokenizer:new({LuaVersion = 'Lua51'}) + pairedTokenizer:append(newline) + local pairedEof = pairedTokenizer:scanAll()[1] + assert(pairedEof.line == expectedLine and pairedEof.linePos == 1) + assert(pairedEof.endLine == expectedLine and pairedEof.endLinePos == 1) +end +local loadChunk = loadstring or load +for _, case in ipairs(pairedNewlines) do + local newline, expectedLine = case[1], case[2] + local input = 'local x={}' .. newline .. 'return x.missing + 1' .. newline + local mappedAst = Parser:new({LuaVersion = 'Lua51'}):parse(input, 'paired-newlines.lua') + assert(mappedAst.body.statements[2].sourceSpan.startLine == expectedLine) + assert(mappedAst.sourceSpan.endLine == expectedLine + 2) + for _, pretty in ipairs({false, true}) do + local pipeline = Prometheus.Pipeline:fromConfig({LuaVersion = 'Lua51', Seed = 7, + SourceMap = true, PrettyPrint = pretty, Steps = {}}) + local generated, map = pipeline:apply(input, 'paired-newlines.lua') + local chunk = assert(loadChunk(generated, '@paired-output.lua')) + local ok, err = pcall(chunk) + assert(not ok, 'Paired newline source-map fixture must fail') + local generatedLine = tonumber(tostring(err):match('paired%-output%.lua:(%d+):')) + assert(generatedLine, tostring(err)) + local recovered = SourceMap.lookup(map, generatedLine) + assert(recovered and recovered.source == 'paired-newlines.lua' + and recovered.line == expectedLine, + (pretty and 'Pretty' or 'Compact') .. ' map recovered ' + .. (recovered and recovered.source .. ':' .. recovered.line or 'no source')) + end +end for _, newline in ipairs({'\n\n', '\r\r'}) do local repeatedTokenizer = Tokenizer:new({LuaVersion = 'Lua51'}) repeatedTokenizer:append('a' .. newline .. 'b') From 878907e357ee334c510c8799c834772411f48363 Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:31:45 +0100 Subject: [PATCH 3/6] Keep source map JSON valid for binary Lua sources --- doc/reference/cli-options.md | 3 ++ src/prometheus/json.lua | 36 +++++++++++++++++ src/prometheus/sourcemap.lua | 8 +++- tests/harness/source-maps.lua | 13 ++++++ tests/harness/source-maps.py | 74 +++++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 1 deletion(-) diff --git a/doc/reference/cli-options.md b/doc/reference/cli-options.md index 330ecd86..b28055c0 100644 --- a/doc/reference/cli-options.md +++ b/doc/reference/cli-options.md @@ -40,6 +40,9 @@ access to the original files. `--source-map-no-sources` omits that text; the trace command then looks for the original source relative to the map file, including any `sourceRoot` recorded in the map. The CLI stores relative source paths relative to the map file so maps remain usable when saved in a subfolder. +When embedded-source output is enabled, source text that is not valid UTF-8 is +omitted from `sourcesContent` as `null` and the CLI logs a warning. With +`--source-map-no-sources`, no embedded-source warning is emitted. For example, save a traceback in a file and translate it with: diff --git a/src/prometheus/json.lua b/src/prometheus/json.lua index cfb31108..7ad3434b 100644 --- a/src/prometheus/json.lua +++ b/src/prometheus/json.lua @@ -5,7 +5,42 @@ local arrayMeta = {} function Json.array(value) return setmetatable(value or {}, arrayMeta) end function Json.isArray(value) return type(value) == 'table' and getmetatable(value) == arrayMeta end +-- Lua strings can contain arbitrary bytes; JSON strings must contain UTF-8 +-- Unicode scalars. Reject overlong sequences, surrogates and code points above +-- U+10FFFF, while leaving valid source text byte-for-byte unchanged. +function Json.isValidUtf8(value) + if type(value) ~= 'string' then return false end + local index = value:find('[\128-\255]') or #value + 1 + while index <= #value do + local byte = value:byte(index) + if byte < 128 then + index = index + 1 + else + local length, minimum, maximum = 0, 128, 191 + if byte >= 194 and byte <= 223 then length = 2 + elseif byte >= 224 and byte <= 239 then + length = 3 + if byte == 224 then minimum = 160 end + if byte == 237 then maximum = 159 end + elseif byte >= 240 and byte <= 244 then + length = 4 + if byte == 240 then minimum = 144 end + if byte == 244 then maximum = 143 end + else return false end + local second = value:byte(index + 1) + if not second or second < minimum or second > maximum then return false end + for offset = 2, length - 1 do + local continuation = value:byte(index + offset) + if not continuation or continuation < 128 or continuation > 191 then return false end + end + index = index + length + end + end + return true +end + local function quote(value) + assert(Json.isValidUtf8(value), 'Cannot encode JSON string containing invalid UTF-8') return '"' .. value:gsub('[%z\1-\31\\"]', function(char) if char == '"' then return '\\"' end if char == '\\' then return '\\\\' end @@ -48,6 +83,7 @@ end function Json.decode(input) assert(type(input) == 'string', 'JSON input must be a string') + assert(Json.isValidUtf8(input), 'Invalid JSON: invalid UTF-8') local index, depth = 1, 0 local function fail(message) error('Invalid JSON at byte ' .. index .. ': ' .. message, 0) end local function whitespace() diff --git a/src/prometheus/sourcemap.lua b/src/prometheus/sourcemap.lua index 62299463..2bb60f63 100644 --- a/src/prometheus/sourcemap.lua +++ b/src/prometheus/sourcemap.lua @@ -1,5 +1,6 @@ -- Source Map v3 / ECMA-426. Mapping coordinates are zero based on the wire. local Json = require('prometheus.json') +local logger = require('logger') local SourceMap = {} local alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' local digits = {} @@ -49,7 +50,12 @@ function SourceMap.build(origins, count, includeSources) id = #map.sources + 1 ids[source] = id map.sources[id] = source.path - if map.sourcesContent then map.sourcesContent[id] = source.content or Json.null end + if map.sourcesContent then + if source.content ~= nil and not Json.isValidUtf8(source.content) then + logger:warn('Omitting embedded source content for ' .. source.path .. ': invalid UTF-8') + map.sourcesContent[id] = Json.null + else map.sourcesContent[id] = source.content or Json.null end + end end local originalLine = origin.startLine - 1 lines[i] = 'A' .. vlq(id - 1 - lastSource) .. vlq(originalLine - lastLine) .. 'A' diff --git a/tests/harness/source-maps.lua b/tests/harness/source-maps.lua index fc0b2467..00542fea 100644 --- a/tests/harness/source-maps.lua +++ b/tests/harness/source-maps.lua @@ -26,6 +26,19 @@ assert(Json.decode('"\\ud83d\\ude00"') == '\240\159\152\128') for _, input in ipairs({'01', '[1,]', '{"a":}', '"\\ud800"', 'true false'}) do assert(not pcall(Json.decode, input), 'Should reject JSON ' .. input) end +for _, text in ipairs({'', 'ASCII\0text', '\194\128', '\223\191', '\224\160\128', + '\237\159\191', '\239\191\191', '\240\144\128\128', '\244\143\191\191', + 'Unicode: \228\184\173 \240\159\152\128'}) do + assert(Json.isValidUtf8(text)) + assert(Json.decode(Json.encode(text)) == text) +end +for _, text in ipairs({'\255', '\128', '\192\128', '\193\191', '\194', '\194a', + '\224\159\191', '\237\160\128', '\239\191', '\240\143\191\191', + '\244\144\128\128', '\245\128\128\128', 'bad \226\130'}) do + assert(not Json.isValidUtf8(text)) + assert(not pcall(Json.encode, text)) + assert(not pcall(Json.decode, '"' .. text .. '"')) +end local function pipeline(pretty, steps, dialect) return Prometheus.Pipeline:fromConfig({LuaVersion = dialect or 'Lua51', Seed = 7, diff --git a/tests/harness/source-maps.py b/tests/harness/source-maps.py index aab38c8d..7553bae9 100644 --- a/tests/harness/source-maps.py +++ b/tests/harness/source-maps.py @@ -246,4 +246,78 @@ def generate(directory, source_path, output_path, map_path, target, pretty=False generate(luau_dir, local_source, output_path, map_path, "LuaU", pretty) check([LUAU, str(output_path)], luau_dir) + # Maps remain valid UTF-8 JSON when source bytes are not valid UTF-8. The + # original source is represented by JSON null, while valid Unicode remains + # embedded, and generated Lua preserves each source's runtime behavior. + utf8_fixtures = { + "invalid-literal": ( + b'local value = "\xff"\n' + b'assert(string.byte(value) == 255)\n' + b'print("invalid literal ok")\n', + "invalid literal ok", + False, + ), + "invalid-comment": ( + b'-- comment with invalid byte: \xff\n' + b'assert(2 + 2 == 4)\n' + b'print("invalid comment ok")\n', + "invalid comment ok", + False, + ), + "unicode": ( + 'local value = "漢字🙂"\n' + 'assert(value == "漢字🙂")\n' + 'print("unicode ok")\n'.encode("utf-8"), + "unicode ok", + True, + ), + } + for pretty in (False, True): + utf8_dir = base / f"utf8-{int(pretty)}" + utf8_dir.mkdir() + for name, (source_bytes, expected_output, valid_source) in utf8_fixtures.items(): + source_path = utf8_dir / f"{name}.lua" + source_path.write_bytes(source_bytes) + output_path = utf8_dir / f"{name}.generated.lua" + map_path = utf8_dir / f"{name}.generated.lua.map" + args = ["--preset", "Minify", "--out", output_path.name, + "--source-map", map_path.name, "--Lua51"] + if pretty: + args.append("--pretty") + args.append(source_path.name) + build_log = check(lua_cli(*args, cwd=utf8_dir), utf8_dir) + warning = "Omitting embedded source content for" + if valid_source: + assert warning not in build_log, build_log + else: + assert warning in build_log and "invalid UTF-8" in build_log, build_log + + source_map = json.loads(map_path.read_bytes()) + content = source_map["sourcesContent"][0] + if valid_source: + assert content == source_bytes.decode("utf-8"), source_map + else: + assert content is None, source_map + + for runtime in (LUA51, LUAU): + output = check([runtime, str(output_path)], utf8_dir) + assert expected_output in output, (runtime, output) + + # Omitting all sources suppresses the invalid-UTF-8 warning as well as + # sourcesContent in the map. + no_sources_dir = base / "utf8-no-sources" + no_sources_dir.mkdir() + invalid_path = no_sources_dir / "invalid.lua" + invalid_path.write_bytes(utf8_fixtures["invalid-literal"][0]) + output_path = no_sources_dir / "generated.lua" + map_path = no_sources_dir / "generated.lua.map" + build_log = check(lua_cli("--preset", "Minify", "--Lua51", "--out", output_path.name, + "--source-map", map_path.name, "--source-map-no-sources", + invalid_path.name, cwd=no_sources_dir), no_sources_dir) + assert "Omitting embedded source content" not in build_log, build_log + assert "sourcesContent" not in json.loads(map_path.read_bytes()) + for runtime in (LUA51, LUAU): + output = check([runtime, str(output_path)], no_sources_dir) + assert "invalid literal ok" in output, (runtime, output) + print("Source map CLI integration checks passed") From af8f318c4c35b95fa97face27d055396b58407b0 Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:33:56 +0100 Subject: [PATCH 4/6] Handle Lua newline conventions in traceback source excerpts --- src/prometheus/tokenizer.lua | 3 +- src/prometheus/traceback.lua | 13 ++++++-- src/prometheus/util.lua | 7 ++++ tests/harness/source-maps.py | 63 ++++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/src/prometheus/tokenizer.lua b/src/prometheus/tokenizer.lua index 7c11b5f2..a2b8ca23 100644 --- a/src/prometheus/tokenizer.lua +++ b/src/prometheus/tokenizer.lua @@ -91,8 +91,7 @@ function Tokenizer:prepareGetPosition() for offset = 1, self.length do local character = self.source:sub(offset, offset) if character == "\r" or character == "\n" then - local paired = (character == "\r" and previousNewline == "\n") - or (character == "\n" and previousNewline == "\r") + local paired = util.isNewlinePair(previousNewline, character) if not paired then line = line + 1 end column = 1 -- Consume CRLF/LFCR as a pair; its second byte cannot also diff --git a/src/prometheus/traceback.lua b/src/prometheus/traceback.lua index c76d8296..c8057e77 100644 --- a/src/prometheus/traceback.lua +++ b/src/prometheus/traceback.lua @@ -1,6 +1,7 @@ -- Utilities for translating generated Lua traceback locations through a source map. local Traceback = {} +local util = require('prometheus.util') local function normalize_path(path) return (path or ""):gsub("\\", "/"):gsub("/+", "/") @@ -145,8 +146,16 @@ end local function excerpt_lines(content) local lines = {} - for line in (content .. "\n"):gmatch("(.-)\n") do - lines[#lines + 1] = line:gsub("\r$", "") + local start = 1 + while true do + local newline = content:find('[\r\n]', start) + if not newline then + lines[#lines + 1] = content:sub(start) + break + end + lines[#lines + 1] = content:sub(start, newline - 1) + local paired = util.isNewlinePair(content:sub(newline, newline), content:sub(newline + 1, newline + 1)) + start = newline + (paired and 2 or 1) end return lines end diff --git a/src/prometheus/util.lua b/src/prometheus/util.lua index 5d8b6bde..420c262e 100644 --- a/src/prometheus/util.lua +++ b/src/prometheus/util.lua @@ -63,6 +63,12 @@ local function keys(tb) return keyset end +-- Lua treats CRLF and LFCR as single newline sequences. Callers must +-- consume both bytes together so adjacent pairs cannot overlap. +local function isNewlinePair(first, second) + return (first == "\r" and second == "\n") or (first == "\n" and second == "\r") +end + local utf8char; do local string_char = string.char @@ -108,6 +114,7 @@ return { escape = escape, chararray = chararray, keys = keys, + isNewlinePair = isNewlinePair, shuffle = shuffle, utf8char = utf8char, readonly = readonly diff --git a/tests/harness/source-maps.py b/tests/harness/source-maps.py index 7553bae9..50a5efa4 100644 --- a/tests/harness/source-maps.py +++ b/tests/harness/source-maps.py @@ -320,4 +320,67 @@ def generate(directory, source_path, output_path, map_path, target, pretty=False output = check([runtime, str(output_path)], no_sources_dir) assert "invalid literal ok" in output, (runtime, output) + # Newline pairs remain non-overlapping in both generated mappings and + # traceback excerpts. Keep each source byte-for-byte to exercise CR, LF, + # CRLF, LFCR, and mixed paired endings. + newline_sequences = { + "lf": (b"\n", b"\n"), + "cr": (b"\r", b"\r"), + "crlf": (b"\r\n", b"\r\n"), + "lfcr": (b"\n\r", b"\n\r"), + "mixed-pairs": (b"\r\n", b"\n\r"), + } + compatible_newline_runtimes = {LUA51: 0, LUAU: 0} + for convention, (first_ending, second_ending) in newline_sequences.items(): + baseline_dir = base / f"newline-baseline-{convention}" + baseline_dir.mkdir() + baseline_source = baseline_dir / "source.lua" + baseline_bytes = (b"local value=nil" + first_ending + second_ending + + b"return value+1" + first_ending) + baseline_source.write_bytes(baseline_bytes) + compatible = {} + for runtime in (LUA51, LUAU): + baseline = run([runtime, str(baseline_source)], baseline_dir) + source_error = re.search(r"source\.lua:(\d+):", baseline.stdout) + compatible[runtime] = baseline.returncode != 0 and source_error is not None \ + and int(source_error.group(1)) == 3 + if compatible[runtime]: + compatible_newline_runtimes[runtime] += 1 + + for pretty in (False, True): + for embed_sources in (True, False): + case_dir = base / f"newline-{convention}-{int(pretty)}-{int(embed_sources)}" + (case_dir / "build").mkdir(parents=True) + source_path = case_dir / "source.lua" + source_path.write_bytes(baseline_bytes) + output_path = case_dir / "build" / "generated.lua" + map_path = case_dir / "build" / "generated.lua.map" + args = ["--preset", "Minify", "--Lua51", "--out", "build/generated.lua", + "--source-map", "build/generated.lua.map"] + if pretty: + args.append("--pretty") + if not embed_sources: + args.append("--source-map-no-sources") + args.append(source_path.name) + check(lua_cli(*args, cwd=case_dir), case_dir) + + source_map = json.loads(map_path.read_bytes()) + if embed_sources: + assert source_map["sourcesContent"] == [baseline_bytes.decode("ascii")], source_map + else: + assert "sourcesContent" not in source_map, source_map + + for runtime in (LUA51, LUAU): + if not compatible[runtime]: + continue + result = run([runtime, str(output_path)], case_dir) + assert result.returncode != 0, (runtime, convention, pretty, embed_sources, result.stdout) + formatted = check(traceback_command(map_path), case_dir, + input_text=result.stdout) + assert "source.lua:3" in formatted, (runtime, convention, formatted) + assert "> 3 | return value+1" in formatted, (runtime, convention, formatted) + + for runtime, count in compatible_newline_runtimes.items(): + assert count > 0, f"{runtime} did not report the expected line for any newline convention" + print("Source map CLI integration checks passed") From 9f255b8e936bb58e4f4f14d510bffe769f2980f6 Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:35:58 +0100 Subject: [PATCH 5/6] Reduce tokenizer coordinate storage with numeric arrays --- src/prometheus/tokenizer.lua | 22 +++++------ tests/harness/parser-sources.lua | 66 ++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 12 deletions(-) diff --git a/src/prometheus/tokenizer.lua b/src/prometheus/tokenizer.lua index a2b8ca23..b543316c 100644 --- a/src/prometheus/tokenizer.lua +++ b/src/prometheus/tokenizer.lua @@ -73,18 +73,14 @@ local function generateWarning(token, message) end function Tokenizer:getPosition(i) - local position = self.positionMap[i] - - if not position then - position = self.positionMap[self.length] or {line = 1, column = 1} - end - - return position.line, position.column + if not self.positionLines[i] then i = self.length end + return self.positionLines[i] or 1, self.positionColumns[i] or 1 end --// Prepare byte-coordinate positions for every offset from 0 through length. function Tokenizer:prepareGetPosition() - local positionMap = {[0] = {line = 1, column = 1}} + -- Dense numeric arrays avoid allocating a separate table for every byte. + local positionLines, positionColumns = {[0] = 1}, {[0] = 1} local line, column = 1, 1 local previousNewline @@ -101,10 +97,10 @@ function Tokenizer:prepareGetPosition() column = column + 1 previousNewline = nil end - positionMap[offset] = {line = line, column = column} + positionLines[offset], positionColumns[offset] = line, column end - self.positionMap = positionMap + self.positionLines, self.positionColumns = positionLines, positionColumns end -- Constructor for Tokenizer @@ -153,7 +149,8 @@ function Tokenizer:new(settings) StringStartLookup = lookupify({"\"", "\'"}), annotations = {}, - positionMap = {[0] = {line = 1, column = 1}}, + positionLines = {[0] = 1}, + positionColumns = {[0] = 1}, }; setmetatable(tokenizer, self); @@ -168,7 +165,8 @@ function Tokenizer:reset() self.length = 0; self.source = ""; self.annotations = {}; - self.positionMap = {[0] = {line = 1, column = 1}}; + self.positionLines = {[0] = 1}; + self.positionColumns = {[0] = 1}; end -- Append String to this Tokenizer diff --git a/tests/harness/parser-sources.lua b/tests/harness/parser-sources.lua index bc18c65b..f671f6c6 100644 --- a/tests/harness/parser-sources.lua +++ b/tests/harness/parser-sources.lua @@ -122,6 +122,72 @@ for _, newline in ipairs({'\n\n', '\r\r'}) do assert(repeatedTokens[2].line == 3 and repeatedTokens[2].linePos == 1) end +-- Independently calculate coordinates at every byte boundary. Newline pairs +-- consume two opposite newline bytes as one line break; equal adjacent newline +-- bytes remain two separate line breaks. +local function referencePositions(text) + local positions = {[0] = {line = 1, column = 1}} + local line, column, offset = 1, 1, 0 + while offset < #text do + local character = text:sub(offset + 1, offset + 1) + if character == '\r' or character == '\n' then + local nextCharacter = text:sub(offset + 2, offset + 2) + local paired = (character == '\r' and nextCharacter == '\n') + or (character == '\n' and nextCharacter == '\r') + line, column = line + 1, 1 + positions[offset + 1] = {line = line, column = column} + offset = offset + 1 + if paired then + positions[offset + 1] = {line = line, column = column} + offset = offset + 1 + end + else + column = column + 1 + offset = offset + 1 + positions[offset] = {line = line, column = column} + end + end + return positions +end + +local function assertAllBytePositions(tokenizer, text) + local expected = referencePositions(text) + for offset = 0, #text do + local line, column = tokenizer:getPosition(offset) + assert(line == expected[offset].line and column == expected[offset].column, + string.format('Offset %d in %q expected %d:%d, got %d:%d', offset, text, + expected[offset].line, expected[offset].column, line, column)) + end + local eof = expected[#text] + local line, column = tokenizer:getPosition(#text + 1) + assert(line == eof.line and column == eof.column, 'Past-EOF position should fall back to EOF') + line, column = tokenizer:getPosition(-1) + assert(line == eof.line and column == eof.column, 'Negative position should fall back to EOF') +end + +local coordinateSamples = { + 'ASCII byte columns', + 'one\r\ntwo\n\rthree\r\n\r\nend', + 'caf\195\169\r\nline\n\rend', + '\r\n\r\n\n\r\n\r', +} +local coordinateTokenizer = Tokenizer:new({LuaVersion = 'Lua51'}) +for _, sample in ipairs(coordinateSamples) do + coordinateTokenizer:reset() + coordinateTokenizer:append(sample) + assertAllBytePositions(coordinateTokenizer, sample) +end + +-- Appending may split a pair across calls; reset must also clear old coordinates. +coordinateTokenizer:reset() +coordinateTokenizer:append('prefix\r') +assertAllBytePositions(coordinateTokenizer, 'prefix\r') +coordinateTokenizer:append('\nnext\n\r') +assertAllBytePositions(coordinateTokenizer, 'prefix\r\nnext\n\r') +coordinateTokenizer:reset() +coordinateTokenizer:append('q') +assertAllBytePositions(coordinateTokenizer, 'q') + parser.disableLog = true assert(not pcall(function() parser:parse('local =') end)) assert(parser.disableLog, 'parse should preserve the caller logging preference') From d7a878f5a2b1eda8a173df2f209af1d452608635 Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:45:35 +0100 Subject: [PATCH 6/6] Resolve Lua traceback locations with truncated filenames --- src/prometheus/traceback.lua | 11 ++++++++- tests/harness/source-maps.py | 44 ++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/prometheus/traceback.lua b/src/prometheus/traceback.lua index c8057e77..3de6464d 100644 --- a/src/prometheus/traceback.lua +++ b/src/prometheus/traceback.lua @@ -25,7 +25,16 @@ local function map_file_matches(filename, mapFile) local normalizedFilename = normalize_path(filename) local normalizedMapFile = normalize_path(mapFile) - return normalizedFilename == normalizedMapFile or basename(normalizedFilename) == basename(normalizedMapFile) + if normalizedFilename == normalizedMapFile or basename(normalizedFilename) == basename(normalizedMapFile) then + return true + end + -- Lua shortens long chunk filenames to "..." followed by their tail. + -- Require a substantial retained suffix to avoid guessing from short names. + if normalizedFilename:sub(1, 3) == "..." then + local suffix = normalizedFilename:sub(4) + return #suffix >= 32 and normalizedMapFile:sub(-#suffix) == suffix + end + return false end local function source_match(candidate, mapFile) diff --git a/tests/harness/source-maps.py b/tests/harness/source-maps.py index 50a5efa4..40a6be34 100644 --- a/tests/harness/source-maps.py +++ b/tests/harness/source-maps.py @@ -383,4 +383,48 @@ def generate(directory, source_path, output_path, map_path, target, pretty=False for runtime, count in compatible_newline_runtimes.items(): assert count > 0, f"{runtime} did not report the expected line for any newline convention" + # Lua 5.1 truncates long chunk filenames in both the error and traceback + # frame. Source-map matching should use the retained filename suffix. + long_name_dir = base / "long-lua51-name" + long_name_dir.mkdir() + long_basename = "generated-" + "x" * 80 + "-trace-suffix.lua" + for pretty in (False, True): + case_dir = long_name_dir / ("pretty" if pretty else "compact") + case_dir.mkdir() + long_source = case_dir / "original-long.lua" + long_source.write_bytes(b"local value=nil\nreturn value+1\n") + long_output = case_dir / long_basename + long_map = case_dir / (long_basename + ".map") + generate(case_dir, long_source, long_output, long_map, "Lua51", pretty) + xpcall_runner = case_dir / "xpcall-runner.lua" + xpcall_runner.write_text('''local chunk = assert(loadfile(arg[1])) +local ok, message = xpcall(chunk, function(err) + return debug.traceback(tostring(err), 2) +end) +if not ok then + io.stdout:write(message, "\\n") + os.exit(1) +end +''', encoding="utf-8", newline="") + long_trace = run([LUA51, str(xpcall_runner), str(long_output)], case_dir) + assert long_trace.returncode != 0, long_trace.stdout + matching_trace_lines = [line for line in long_trace.stdout.splitlines() + if "trace-suffix.lua" in line] + assert len(matching_trace_lines) >= 2, long_trace.stdout + error_lines = [line for line in matching_trace_lines + if "attempt to perform arithmetic" in line] + frame_lines = [line for line in matching_trace_lines if "in main chunk" in line] + assert error_lines and all("..." in line for line in error_lines), long_trace.stdout + assert frame_lines and all("..." in line for line in frame_lines), long_trace.stdout + + augmented_trace = long_trace.stdout.rstrip() + ( + "\n\t..." + "y" * 40 + "-trace-suffix.lua:2: in function 'unrelated'" + "\n\t...x.lua:2: in function 'short-name'\n") + rewritten = check(traceback_command(long_map), case_dir, input_text=augmented_trace) + rewritten_trace = rewritten.split("\nSource excerpts:\n", 1)[0] + assert rewritten_trace.count("original-long.lua:2") >= 2, rewritten + assert "> 2 | return value+1" in rewritten, rewritten + assert "..." + "y" * 40 + "-trace-suffix.lua:2: in function 'unrelated'" in rewritten, rewritten + assert "...x.lua:2: in function 'short-name'" in rewritten, rewritten + print("Source map CLI integration checks passed")