From 2d2e4d7ebfdec5fc944d35c930368fc75cddbd6c Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Mon, 14 Sep 2026 20:05:54 +0800 Subject: [PATCH 01/16] ci: workspace mcpp bootstrap pin -> 2026.9.14.3 (released + indexed); #634 ledger closed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version group 2 moves to 2026.9.14.3, released from 9b6a1188 and named `latest` by the published xim index artifact `xim-index-ed04041`. The #634 implementation plan and the triage record are marked landed. The ledger closes every row with a reading (E1-E24 in mcpp#637, E25 in mcpp#638, P1-P9 in mcpp-plugins#24, I1 in mcpplibs/mcpp-index#426, R1-R4, V1-V4), and §9 records what landed, the sandbox readings on 2026.9.14.2, 2026.9.14.3 and the 2026.9.14.1 control, the ecosystem review, and two observations this work did not change. --- ...26-09-14-634-cmake-parity-items-by-home.md | 7 +- .../2026-09-14-634-implementation-plan.md | 153 +++++++++++++----- .agents/docs/README.md | 8 +- .xlings.json | 2 +- 4 files changed, 120 insertions(+), 50 deletions(-) diff --git a/.agents/docs/2026-09-14-634-cmake-parity-items-by-home.md b/.agents/docs/2026-09-14-634-cmake-parity-items-by-home.md index 1c4d7c50..7c0cce22 100644 --- a/.agents/docs/2026-09-14-634-cmake-parity-items-by-home.md +++ b/.agents/docs/2026-09-14-634-cmake-parity-items-by-home.md @@ -1,11 +1,14 @@ --- subject: triage -status: active +status: landed --- # A framework's CMake parity list: the twenty-one items of #634, read against the code and routed to where each one belongs -**Status:** active, revision 3. Revision 3 is the implementation's: the plan +**Status:** landed on 2026-09-14, revision 3: mcpp 2026.9.14.2 and 2026.9.14.3 +(mcpp-community/mcpp#637, #638), `mcpp:plugins` 0.10.0 (mcpp-community/mcpp-plugins#24), +openxlings/xim-pkgindex#838 and mcpplibs/mcpp-index#426; the closure and the +sandbox readings are §9 of the implementation plan. Revision 3 is the implementation's: the plan and ledger are `2026-09-14-634-implementation-plan.md`, whose §1 states the refinements adopted before and while implementing; the sections below carry them where a decision's wording changed (§5.2, §5.3, §5.6). Revision 1 classified the items; revision 2 held every diff --git a/.agents/docs/2026-09-14-634-implementation-plan.md b/.agents/docs/2026-09-14-634-implementation-plan.md index d810ce1d..15b0fd71 100644 --- a/.agents/docs/2026-09-14-634-implementation-plan.md +++ b/.agents/docs/2026-09-14-634-implementation-plan.md @@ -1,11 +1,14 @@ --- subject: triage -status: active +status: landed --- # #634 implemented across five repositories: the plan, its review, and the ledger that tracks it -**Status:** active. This record turns the decisions of +**Status:** landed on 2026-09-14 as mcpp 2026.9.14.2 (mcpp-community/mcpp#637) +and 2026.9.14.3 (mcpp-community/mcpp#638), `mcpp:plugins` 0.10.0 +(mcpp-community/mcpp-plugins#24), openxlings/xim-pkgindex#838, #839 and #840, +and mcpplibs/mcpp-index#426; §9 records the closure. This record turns the decisions of `2026-09-14-634-cmake-parity-items-by-home.md` (revision 2, "the triage record" below) into work: one pull request per repository, the order in which they merge and release, the criteria each task is held to, and a ledger whose @@ -30,30 +33,31 @@ that closed it), `dropped` (with the reason). Owner `lead` is the integrating se | id | task (triage §) | owner | depends on | status | |---|---|---|---|---| -| E1 | a matching conditional dependency declaration replaces the unconditional one, compared by identity; the same for `dev-dependencies`, `build-dependencies`, `feature-deps` (§5.1.1) | lead | - | branch: e2e 677 A, B (fails on 2026.9.14.1) | -| E2 | conditional dependency tables labelled by their full section; an option-named selector with a string value warns with the restated form (§5.1.2) | lead | - | branch: e2e 677 C | -| E3 | `[target..targets.] kind` for declared library targets; the `[target.]` sweep reports sub-tables the parser does not read (§5.1.3) | lead | - | branch: e2e 677 D, 678 D; unit `TargetScalarKeys.EveryParsedSubTableIsKnownToTheSweep` | -| E4 | the link-form degradation names the package's statement and, for a row, its selector (§5.1.3, §9 item 5) | lead | E3 | branch: e2e 678 C | -| E5 | a `path`/`git` dependency adopts its manifest's identity and warns once; the record is registered under both keys (§5.2) | lead | - | branch: e2e 679 A, B | -| E6 | two identities over one canonical source are refused before scanning; the scanner's duplicate-provider message names the packages (§5.2) | lead | E5 | branch: e2e 679 C | -| E7 | the six edges in mcpp's `examples/` write the declared identity (§5.2) | lead | E5 | branch: the three examples build with no identity warning, and still build on 2026.9.14.1 | -| E8 | one reader-driven closure for PE, the Android rows and Mach-O, with per-format platform rules (§5.3.1) | W1 | - | branch: unit `test_pack_closure` (11 cases); e2e 667, 668 (macOS CI) | -| E9 | Android stages `lib/` (per ABI), Mach-O stages beside the program; `walked` only when complete, else `not-walked` naming the names and `dir`/`tar` refuse; `needs` lines on every row (§5.3.2-5) | W1 | E8 | branch: e2e 266, 666, 668 (macOS CI), 667, 669 | -| E10 | rpath entries that begin with a loader token (`$ORIGIN`, `@executable_path`, `@loader_path`, `@rpath`) are not anchored, through one function shared by both normalisers (§9 item 11) | W1 | - | branch: unit `test_build_flags`; e2e 670 | -| E11 | an ELF shared library without a declared `soname` is linked with its file name as SONAME (§5.4) | W1 | - | branch: e2e 667, 669 | -| E12 | the static C++ runtime archives are located by asking the driver for the effective target (§5.6.1) | W2 | - | branch: e2e 675 | -| E13 | every runner of `mcpp run` and `mcpp test` receives `MCPP_RUNTIME_FILES` (§5.6.2, format refined in §1) | W2 | - | branch: e2e 672; the `android` job's emulator step | -| E14 | `run`, `test` and `pack` declare `--toolchain` (§5.10) | W2 | - | branch: e2e 671 | -| E15 | `[test] discover` (§5.5) | W2 | - | branch: e2e 673; unit `test_test_targets` | -| E16 | `mcpp run --format ` uses the named runner `` when one exists; a directory distributable that meets no runner is refused before the spawn (§6.3) | W2 | - | branch: e2e 674 | -| E17 | two emissions of one named runner: measured, then decided (§9 item 8; see §1) | W2 | - | branch: decided in §1.9, e2e 684 | -| E18 | `resolution.json` records `graph`; `mcpp why deps` prints it (§5.11) | lead | E1-E6 | branch: e2e 682; 677, 678 and 679 read `graph` | -| E19 | the engine states `android.api-level`, `ios.deployment-target`, `macos.deployment-target`; the refusal names the fact's key and says "this build targets" (§5.9, §9 item 6) | lead | - | branch: e2e 680 | -| E20 | `config.toml [index.repos.]` reaches an existing registry; a payload installed from an overridden index names the source (§7.4, §9 item 7) | lead | - | branch: e2e 681; an existing home re-pointed its index and restored the entry (local probe, §1.9) | -| E21 | `mcpp::pkg_config_libdir()` (§5.7) | lead | - | branch: e2e 683 | -| E22 | CI: the `android-ndk` e2e tests run on a Linux job, and one emulator step runs `mcpp test` on the x86_64 Android row (§1, test coverage) | lead | E9, E12, E13 | branch: `ci-linux-e2e.yml` job `android` | -| E23 | user documentation and its Chinese mirror; normative specification changes (§5 below) | lead, W1, W2 | E1-E21 | doing | -| E24 | triage record revision 3 folded in; this ledger closed for the engine rows; CHANGELOG; version group 1 | lead | E1-E23 | doing | +| E1 | a matching conditional dependency declaration replaces the unconditional one, compared by identity; the same for `dev-dependencies`, `build-dependencies`, `feature-deps` (§5.1.1) | lead | - | merged (#637): e2e 677 A, B (fails on 2026.9.14.1) | +| E2 | conditional dependency tables labelled by their full section; an option-named selector with a string value warns with the restated form (§5.1.2) | lead | - | merged (#637): e2e 677 C | +| E3 | `[target..targets.] kind` for declared library targets; the `[target.]` sweep reports sub-tables the parser does not read (§5.1.3) | lead | - | merged (#637): e2e 677 D, 678 D; unit `TargetScalarKeys.EveryParsedSubTableIsKnownToTheSweep` | +| E4 | the link-form degradation names the package's statement and, for a row, its selector (§5.1.3, §9 item 5) | lead | E3 | merged (#637): e2e 678 C | +| E5 | a `path`/`git` dependency adopts its manifest's identity and warns once; the record is registered under both keys (§5.2) | lead | - | merged (#637): e2e 679 A, B | +| E6 | two identities over one canonical source are refused before scanning; the scanner's duplicate-provider message names the packages (§5.2) | lead | E5 | merged (#637): e2e 679 C | +| E7 | the six edges in mcpp's `examples/` write the declared identity (§5.2) | lead | E5 | merged (#637): the three examples build with no identity warning, and still build on 2026.9.14.1 | +| E8 | one reader-driven closure for PE, the Android rows and Mach-O, with per-format platform rules (§5.3.1) | W1 | - | merged (#637): unit `test_pack_closure` (11 cases); e2e 667, 668 (macOS CI) | +| E9 | Android stages `lib/` (per ABI), Mach-O stages beside the program; `walked` only when complete, else `not-walked` naming the names and `dir`/`tar` refuse; `needs` lines on every row (§5.3.2-5) | W1 | E8 | merged (#637): e2e 266, 666, 668 (macOS CI), 667, 669 | +| E10 | rpath entries that begin with a loader token (`$ORIGIN`, `@executable_path`, `@loader_path`, `@rpath`) are not anchored, through one function shared by both normalisers (§9 item 11) | W1 | - | merged (#637): unit `test_build_flags`; e2e 670 | +| E11 | an ELF shared library without a declared `soname` is linked with its file name as SONAME (§5.4) | W1 | - | merged (#637): e2e 667, 669 | +| E12 | the static C++ runtime archives are located by asking the driver for the effective target (§5.6.1) | W2 | - | merged (#637): e2e 675 | +| E13 | every runner of `mcpp run` and `mcpp test` receives `MCPP_RUNTIME_FILES` (§5.6.2, format refined in §1) | W2 | - | merged (#637): e2e 672; the `android` job's emulator step | +| E14 | `run`, `test` and `pack` declare `--toolchain` (§5.10) | W2 | - | merged (#637): e2e 671 | +| E15 | `[test] discover` (§5.5) | W2 | - | merged (#637): e2e 673; unit `test_test_targets` | +| E16 | `mcpp run --format ` uses the named runner `` when one exists; a directory distributable that meets no runner is refused before the spawn (§6.3) | W2 | - | merged (#637): e2e 674 | +| E17 | two emissions of one named runner: measured, then decided (§9 item 8; see §1) | W2 | - | merged (#637): decided in §1.9, e2e 684 | +| E18 | `resolution.json` records `graph`; `mcpp why deps` prints it (§5.11) | lead | E1-E6 | merged (#637): e2e 682; 677, 678 and 679 read `graph` | +| E19 | the engine states `android.api-level`, `ios.deployment-target`, `macos.deployment-target`; the refusal names the fact's key and says "this build targets" (§5.9, §9 item 6) | lead | - | merged (#637): e2e 680 | +| E20 | `config.toml [index.repos.]` reaches an existing registry; a payload installed from an overridden index names the source (§7.4, §9 item 7) | lead | - | merged (#637): e2e 681; an existing home re-pointed its index and restored the entry (local probe, §1.9) | +| E21 | `mcpp::pkg_config_libdir()` (§5.7) | lead | - | merged (#637): e2e 683 | +| E22 | CI: the `android-ndk` e2e tests run on a Linux job, and one emulator step runs `mcpp test` on the x86_64 Android row (§1, test coverage) | lead | E9, E12, E13 | merged (#637): `ci-linux-e2e.yml` job `android` | +| E23 | user documentation and its Chinese mirror; normative specification changes (§5 below) | lead, W1, W2 | E1-E21 | merged (#637): docs 04, 05, 06, 07, 08, 09, 10, 12, 22, 30, 32, 41, 50 with their Chinese mirrors; SPEC-001 §5.4, SPEC-004 §3.1.1 | +| E24 | triage record revision 3 folded in; this ledger closed for the engine rows; CHANGELOG; version group 1 | lead | E1-E23 | merged (#637): revision 3; CHANGELOG; 2026.9.14.2 | +| E25 | mcpp's xlings invocations do not inherit `XLINGS_ACTIVE_SUBOS` (found by V1, §9) | lead | V1 | merged (#638): unit `XlingsInvocationEnv.TheShellsActiveSubosIsNeverInherited`; e2e 686 fails on 2026.9.14.2 and passes on 2026.9.14.3 | ### 0.2 Payloads: openxlings/xim-pkgindex, one pull request (`feat/634-runners-and-payloads`) @@ -71,35 +75,35 @@ that closed it), `dropped` (with the reason). Owner `lead` is the integrating se | id | task (triage §) | owner | depends on | status | |---|---|---|---|---| -| P1 | `dist-apk` reads the staged closure (`lib/`, `lib//`), deletes its walk and stamp, packs several ABIs into one APK, reports refusals through `mcpp::warning`, and refuses a stage without `needs` lines naming the engine floor (§6.5) | W4 | E9 | doing | -| P2 | `dist-apple` places staged dylibs in the framework directory, adds the link-time rpath through `mcpp::link_flag`, signs ad hoc when no identity is given, and keeps closure members out of the resource directory (§6.1) | W4 | E9, E10 | doing | -| P3 | `dist-apple` supplies `mcpp::runner("app", "macapp-run")` on `*-macos` and declares the payload (§6.3) | W4 | E16, X1 | doing | -| P4 | `dist-apple` `dmg` format (§6.2) | W4 | - | doing | -| P5 | `dist-apk` `aab` format (§6.5) | W4 | P1, X5 | doing | -| P6 | `dist-wix` `setup` format (§6.4) | W4 | X4 | doing | -| P7 | `rules-metal` (§6.6) | W4 | - | doing | -| P8 | CI: the iOS fixture exits non-zero, the diagnostic is portable, P1-P7 have jobs, `MCPP_VERSION` names the released engine (§7.2, §9 item 10) | W4 | R1 | doing | -| P9 | documentation, version 0.10.0, release, GitCode assets (§4 below) | lead | P1-P8 | todo | +| P1 | `dist-apk` reads the staged closure (`lib/`, `lib//`), deletes its walk and stamp, packs several ABIs into one APK, reports refusals through `mcpp::warning`, and refuses a stage without `needs` lines naming the engine floor (§6.5) | W4 | E9 | done (mcpp-plugins#24): Linux job, two packs keep the dependency's library; `native-code: 'arm64-v8a' 'x86_64'`; the floor refusal | +| P2 | `dist-apple` places staged dylibs in the framework directory, adds the link-time rpath through `mcpp::link_flag`, signs ad hoc when no identity is given, and keeps closure members out of the resource directory (§6.1) | W4 | E9, E10 | done (mcpp-plugins#24): macOS job, `@executable_path/../Frameworks`; `codesign --verify --deep --strict` valid; exit 7 through the framework, 134 without it | +| P3 | `dist-apple` supplies `mcpp::runner("app", "macapp-run")` on `*-macos` and declares the payload (§6.3) | W4 | E16, X1 | done (mcpp-plugins#24): macOS job, `mcpp run --format app` exits 7 through `macapp-run` | +| P4 | `dist-apple` `dmg` format (§6.2) | W4 | - | done (mcpp-plugins#24): macOS job, `hdiutil verify` valid; the image root holds the bundle and `Applications` | +| P5 | `dist-apk` `aab` format (§6.5) | W4 | P1, X5 | done (mcpp-plugins#24): Linux job, `bundletool validate`; a universal APK carries the dependency's library | +| P6 | `dist-wix` `setup` format (§6.4) | W4 | X4 | done (mcpp-plugins#24): Windows job, the MSI extracted from the bundle is byte-identical to the built one | +| P7 | `rules-metal` (§6.6) | W4 | - | done (mcpp-plugins#24): macOS job, three `MTLB` libraries; a header edit recompiles only the shaders that include it | +| P8 | CI: the iOS fixture exits non-zero, the diagnostic is portable, P1-P7 have jobs, `MCPP_VERSION` names the released engine (§7.2, §9 item 10) | W4 | R1 | done (mcpp-plugins#24): run 34831041382 on the released 2026.9.14.2; the iOS fixture returns 7 through `simctl-run` | +| P9 | documentation, version 0.10.0, release, GitCode assets (§4 below) | lead | P1-P8 | done: README floors; 0.10.0; tag `v0.10.0` (4525537) | ### 0.4 Index: mcpplibs/mcpp-index, one pull request | id | task | owner | depends on | status | |---|---|---|---|---| -| I1 | `mcpp.plugins` 0.10.0 descriptor entry and its note | lead | R2 | todo | +| I1 | `mcpp.plugins` 0.10.0 descriptor entry and its note | lead | R2 | done: mcpplibs/mcpp-index#426 | | I2 | D1, HuxerUI's six-row descriptors | project | HuxerUI's release | dropped: follows the project's release, and HuxerUI is not modified here | ### 0.5 Release, verification, reply | id | task | depends on | status | |---|---|---|---| -| R1 | engine: merge, tag, `release.yml` on four hosts, `publish-ecosystem`, the bot's index pull request merged, GitCode assets checked (local `gtc` for any missing), bootstrap pin (version group 2) | E24, CI green | todo | -| R2 | plugins: merge, tag `v0.10.0`, release assets on GitHub and GitCode (`gtc`), byte comparison of each downloaded asset | P9, CI green | todo | +| R1 | engine: merge, tag, `release.yml` on four hosts, `publish-ecosystem`, the bot's index pull request merged, GitCode assets checked (local `gtc` for any missing), bootstrap pin (version group 2) | E24, CI green | done: 2026.9.14.2 (adbd8c20) and 2026.9.14.3 (9b6a1188); both releases' eight mirrored assets fetched back byte-identical (2026.9.14.3's `linux-aarch64` and `macosx-arm64` uploaded to GitCode with local `gtc`); xim-pkgindex#839 and #840; the artifacts `xim-index-c01ac75` and `xim-index-ed04041` name each as `latest`; the bootstrap pin moves to 2026.9.14.3 with this closure | +| R2 | plugins: merge, tag `v0.10.0`, release assets on GitHub and GitCode (`gtc`), byte comparison of each downloaded asset | P9, CI green | done: the tag archive equals `git archive v0.10.0`; the GitCode asset, uploaded with `gtc`, is byte-identical to it (302708 bytes, one sha256 `b7cfa4b5`) | | R3 | xim-pkgindex pull request merged, and its artifact read back from a client | X1-X7, CI green | done: a local client installed `xim:android-platform-tools@37.0.1-4` from the index after the merge | -| R4 | mcpp-index pull request merged; `latest` of `mcpp.plugins` read back from a client | I1 | todo | -| V1 | sandbox verification of the released engine (§6.2) | R1, R3 | todo | -| V2 | sandbox verification of plugins 0.10.0 through the index (§6.2) | R2, R4 | todo | -| V3 | the ecosystem review (§7) | V1, V2 | todo | -| V4 | the reply on #634 | V3 | todo | +| R4 | mcpp-index pull request merged; `latest` of `mcpp.plugins` read back from a client | I1 | done: #426 merged (f5e3de6); the artifact `mcpp-index-f5e3de6`, identical on GitHub and GitCode, names `latest` 0.10.0; the sandbox installed 0.10.0 from it (V2) | +| V1 | sandbox verification of the released engine (§6.2) | R1, R3 | done: §9.2 | +| V2 | sandbox verification of plugins 0.10.0 through the index (§6.2) | R2, R4 | done: §9.2 | +| V3 | the ecosystem review (§7) | V1, V2 | done: §9.3 | +| V4 | the reply on #634 | V3 | done: https://github.com/mcpp-community/mcpp/issues/634#issuecomment-5663600579 | ## 1. Review of revision 2 from the implementation's side, and revision 3 @@ -576,3 +580,66 @@ e2e tests that cover the files it changes; a built binary is copied to a fixed path before a suite runs against it. The lead merges W1 and W2 into the feature branch, resolves conflicts, and runs the full suite once on the merged tree. No work tree kills processes by pattern. + +## 9. Closure + +### 9.1 What landed + +| repository | pull request | merge commit | published as | +|---|---|---|---| +| mcpp-community/mcpp | #637 (E1-E24) | `adbd8c20` | mcpp 2026.9.14.2 | +| mcpp-community/mcpp | #638 (E25) | `9b6a1188` | mcpp 2026.9.14.3 | +| mcpp-community/mcpp-plugins | #24 (P1-P9) | `4525537` | `mcpp:plugins` 0.10.0 | +| openxlings/xim-pkgindex | #838 (X1-X7); #839 and #840 (the release bot) | `8f67d875`; `c01ac75b`; `ed040415` | the payloads; `mcpp` 2026.9.14.2 and 2026.9.14.3 | +| mcpplibs/mcpp-index | #426 (I1) | `f5e3de6` | `mcpp.plugins` 0.10.0 | + +HuxerUI and Lib-Live2D are unchanged; the reply on #634 lists what the project +can change. + +### 9.2 Sandbox readings + +One SubOS, `m634-eco`, entered with `xlings subos use m634-eco --sandbox`, with +`xlings config --mirror CN` and `mcpp self config --mirror CN` set inside it. +The engine is installed with `xlings install mcpp@` and addressed by +its store path; `mcpp:plugins` is resolved from the index. The mcpp home inside +the sandbox is removed before each run that the table marks fresh. + +| run | engine | readings | +|---|---|---| +| V1 | 2026.9.14.2, fresh home | 16 pass; S10 and S15 fail; S11 skipped, S13 and S14 not requested. At the failure site the registry had `subos/default` and `subos/m634-eco`, the gtk4 `.pc` files were in the second, and the accessor names the first: the sandbox exports `XLINGS_ACTIVE_SUBOS=m634-eco` and mcpp passed it to its xlings (E25) | +| pre-release | #638 built as a static musl binary, fresh home | S10 and S15 pass. S13 and S14 fail on two probe defects: the check read an intermediate APK (`dist-apk/base.apk`) instead of the path the pack reports as `Packed`, and looked for the manifest at the version directory instead of inside the archive's top directory; the APK the pack reported carried `lib/x86_64/libfw.so` and `lib/arm64-v8a/libfw.so` | +| V1 and V2 | 2026.9.14.3 released, `mcpp:plugins` 0.10.0 from the index, fresh home | 22 pass, 0 fail. S11, run afterwards in the same home: the table reached the existing registry with one line naming it, and its removal restored the entry, 4 of 4 | +| control | 2026.9.14.1, fresh home, same script | 4 pass (S1 twice; S8's `closure = walked`, which that release wrote over an incomplete tree; S14, which reads the index), 18 fail | + +### 9.3 The ecosystem review (V3) + +- Every ledger row is done, with a reading, or dropped with a reason (I2). +- The main heads build green on their own commits: mcpp `9b6a1188` (every + workflow, including the fresh-install, AUR and Homebrew runs the release + triggers), mcpp-plugins `4525537` (three jobs on the released engine), + xim-pkgindex `ed040415`, mcpp-index `f5e3de6`. +- The published artifacts carry the releases: `xim-index-ed04041` names mcpp + 2026.9.14.3 as `latest` on its three platform tables, and + `mcpp-index-f5e3de6`, identical on GitHub and GitCode, names `mcpp.plugins` + 0.10.0 as `latest`. +- Both engine releases' eight mirrored assets and the plugins archive were + fetched back and compared byte for byte; 2026.9.14.3's `linux-aarch64` and + `macosx-arm64` assets were uploaded to GitCode with a local `gtc`. +- The documentation states what the sandbox measured: `mcpp::pkg_config_libdir()` + names a view that holds the payloads' `.pc` files, and from 2026.9.14.3 that + holds inside an activated SubOS as well (docs/91). +- The triage record and this record are marked landed. + +### 9.4 Observed and not changed by this work + +- xlings stages a package that has no install hook by moving every entry of the + shared `data/runtimedir` into the package's directory: + `extract_archive_detailed` returns the destination directory itself, and + `stage_extracted_payload_` moves each entry it finds there. On this host + `mcpplibs-x-openkal/0.8.0` holds 1.6 GiB of other packages' archives and + `mcpplibs-x-openkal-emscripten/0.1.0` 1.5 GiB (59 archives); in the sandbox + `mcpp-x-plugins/0.10.0` received 1.2 GiB. No result above depends on it, and + the change belongs to xlings. +- On Windows, mcpp prints `The system cannot find the path specified.` as a + command starts; the released 2026.9.13.2 prints it too (mcpp-plugins run + 34772006562, twelve times). Not investigated. diff --git a/.agents/docs/README.md b/.agents/docs/README.md index dc059599..cb5af365 100644 --- a/.agents/docs/README.md +++ b/.agents/docs/README.md @@ -58,8 +58,8 @@ Records that declare one. Everything else is listed by date below. ### triage -- [#634 implemented across five repositories: the plan, its review, and the ledger that tracks it](2026-09-14-634-implementation-plan.md) — active -- [A framework's CMake parity list: the twenty-one items of #634, read against the code and routed to where each one belongs](2026-09-14-634-cmake-parity-items-by-home.md) — active +- [#634 implemented across five repositories: the plan, its review, and the ledger that tracks it](2026-09-14-634-implementation-plan.md) — landed +- [A framework's CMake parity list: the twenty-one items of #634, read against the code and routed to where each one belongs](2026-09-14-634-cmake-parity-items-by-home.md) — landed - [What a framework and its ecosystem library still hit in the engine: the ten items of #630, read against the code](2026-09-13-630-what-a-framework-still-hits-in-the-engine.md) — landed - [The engine gaps left open after the SDK batch](2026-09-12-engine-gaps-after-the-sdk-batch.md) — landed - [Six open issues: what each one actually is, and what would answer it](2026-09-11-six-open-issues-analysis.md) — active @@ -68,8 +68,8 @@ Records that declare one. Everything else is listed by date below. ### 2026-09 -- [#634 implemented across five repositories: the plan, its review, and the ledger that tracks it](2026-09-14-634-implementation-plan.md) — active -- [A framework's CMake parity list: the twenty-one items of #634, read against the code and routed to where each one belongs](2026-09-14-634-cmake-parity-items-by-home.md) — active +- [#634 implemented across five repositories: the plan, its review, and the ledger that tracks it](2026-09-14-634-implementation-plan.md) — landed +- [A framework's CMake parity list: the twenty-one items of #634, read against the code and routed to where each one belongs](2026-09-14-634-cmake-parity-items-by-home.md) — landed - [Four upstream asks from a UI framework: what each one is under mcpp's design, and the combined plan](2026-09-13-four-upstream-asks-from-a-ui-framework.md) — landed - [What a framework and its ecosystem library still hit in the engine: the ten items of #630, read against the code](2026-09-13-630-what-a-framework-still-hits-in-the-engine.md) — landed - [The engine gaps left open after the SDK batch](2026-09-12-engine-gaps-after-the-sdk-batch.md) — landed diff --git a/.xlings.json b/.xlings.json index a7762a60..8079f853 100644 --- a/.xlings.json +++ b/.xlings.json @@ -1,5 +1,5 @@ { "workspace": { - "mcpp": "2026.9.14.1" + "mcpp": "2026.9.14.3" } } From 4182fdf2d2f410caf04accf224c6593ad15ac909 Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Mon, 14 Sep 2026 21:57:03 +0800 Subject: [PATCH 02/16] docs(design): the build database of #636 and the two defects on the way to the latest xlings The record of the analysis, the plan and its self-review: xlings stages a hookless package from the shared download directory; a POSIX redirect in the vendored-xlings probe reaches cmd.exe on Windows; and mcpp emits an S1 build database from the plan without writing into the project. --- ...36-build-database-and-the-latest-xlings.md | 469 ++++++++++++++++++ .agents/docs/README.md | 7 +- 2 files changed, 475 insertions(+), 1 deletion(-) create mode 100644 .agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md diff --git a/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md b/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md new file mode 100644 index 00000000..3e13abc4 --- /dev/null +++ b/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md @@ -0,0 +1,469 @@ +--- +subject: design +status: active +--- + +# The build database of #636, and two defects on the way to the latest xlings + +**Status:** proposed on 2026-09-14; adopted the same day as the plan in §6, with +the decisions of §7 settled by the self-review in §8. Implementation is in +progress in one pull request per repository: mcpp-community/mcpp#639 for every +mcpp change, and one xlings pull request for item A. Code was read at mcpp +`9b6a1188` (2026.9.14.3), xlings `59068d6` (2026.9.12.1, the latest release) and +lsp-mcpp `4ea9f81` (S1 profile 0.2.0, S2 0.2.0). Measured: the store of this +host's mcpp registry, and the Windows CI logs of 2026-09-14 on `main`. A +statement marked *(code)* was derived from source and has not been run; §6.3 +lists the runs that settle each one. + +## 0. Ledger + +| | subject | classification | home | vehicle | +|---|---|---|---|---| +| A | xlings moves the whole shared download directory into a package installed without an `install()` hook | defect, general | xlings installer; xpkg spec §6 | xlings 2026.9.14.1 | +| B | mcpp on Windows prints `The system cannot find the path specified.` in every command after the first | defect, general | mcpp's calls of the platform launchers | mcpp#639, with the move of `kXlingsVersion` | +| C | #636, `mcpp emit build-database` | feature | engine for facts and a neutral format; the S1 library for structured options; the consumer for editor policy | mcpp#639 | + +"The latest xlings inside mcpp" is the xlings release that carries A. B ships +with that pin change, because B is what lets a pin change reach an existing +Windows home (§3.3). + +## 1. The rule behind all three + +A behaviour that another component depends on is written in a specification +that component can cite, and it is tested at that boundary. No component +depends on another component's unwritten behaviour or internal layout. + +- **A.** xpkg spec §6 defines what `install()` does and says nothing about a + package without one. The implementation's accident became the contract: 55 + mcpp-index descriptors use pointers of the form `mcpp = "*/…/mcpp.toml"`, + which match only because the archive's top-level directory is never stripped. +- **B.** A command string written for `/bin/sh` is handed to `cmd.exe`. +- **C.** lsp-mcpp today parses mcpp's `build.ninja`, reads `std-module.json` + from the std cache, and parses `mcpp.toml` as text (lsp-mcpp + `src/project/mcpp.cpp:22-40`, `:54`, `:65`). #636 replaces these reads with a + published document. This design keeps that document a projection of the + plan, and keeps editor policy out of the engine. + +## 2. A — a hookless install sweeps the download directory + +### 2.1 Mechanism (xlings `59068d6`) + +1. Every download of a data root lands in one directory, `/runtimedir` + (`src/core/xim/installer.cpp:963-968`, `:2575`), beside its `.lock` + (held while downloading), `.meta` and `.part.*` staging files + (`downloader.cpp:111`, `:484-492`). +2. Every archive is extracted into that same directory (`installer.cpp:2831-2835`); + `extract_archive_detailed` returns the directory itself (`extract.cpp:300`). +3. When `install_dir` is still empty after hooks (always for a package without + `install()`, and for a hook that does nothing), `stage_extracted_payload_` + receives `runtimedir` and moves every entry in it (`installer.cpp:1121-1190`, + called at `:3116-3117`). Its branch that strips a single top-level directory + (`:1135-1137`) never runs for a downloaded archive, because the archive + itself is always a second entry. + +### 2.2 Measured on this host + +- mcpp registry, `~/.mcpp/registry/data/xpkgs`: 221 of 401 version directories + contain `*.lock` sidecars, and their top-level archives alone total 5.2 GiB. + `mcpplibs-x-openkal/0.8.0` holds 1.6 GiB (nine archives, among them gcc 13.3.0 + at 329 MiB with its extracted tree); `mcpplibs-x-openkal-emscripten/0.1.0` + holds 1.5 GiB (59 archives, android-ndk-r30 at 704 MiB, and `perl-5.44.0/`). + The installing clients were xlings 2026.8.27.1 and 2026.9.5.1 + (`.xpkg-install.json`); the second is mcpp's current pin. +- The registry's `runtimedir` is empty (12 KiB). `~/.xlings/data/runtimedir` + holds 7.2 GiB, and none of that home's 296 version directories contains a + sidecar. An empty download directory is the signature of the sweep: mcpp-index + descriptors carry only `url` and `sha256`, so mcpp's registry is where + hookless installs happen. + +Consequences beyond disk: + +- a swept archive has left the download cache and is downloaded again; +- *(code)* a swept `.lock` may be held by another process, and a swept + `.part.*` file is another process's download in progress; +- *(code)* mcpp's default manifest lookup and every `*/…` pointer require + exactly one match (`src/build/prepare.cppm:5610`, `:5626-5648`), so a swept + source tree that carries its own `mcpp.toml` turns a working dependency into + "matched 2 files". + +### 2.3 The rule, added to xpkg-manifest-v1 §6 + +> A package that defines no `install()` receives in `pkginfo.install_dir()` the +> entries of its own archive, laid out as the archive lays them out, and +> nothing else. The archive stays in the download cache. + +The top-level directory is kept. That is the layout every hookless install has +produced (2.1, step 3); the 55 descriptors and mcpp's `*/mcpp.toml` lookup are +written against it, and a stripping layout would need an opt-in field that +nothing asks for. + +### 2.4 The change + +- A package without `install()` extracts into a private directory, + `/.stage/-/`, removed on every exit path. A package + with a hook extracts beside its archive as before; that is its contract (spec + §6, whose example at line 293 derives the extracted directory from + `install_file()`). +- The staging fallback always reads a private extraction. For a hook that left + `install_dir` empty (the patchelf case named at `installer.cpp:3154`), the + archive is extracted a second time, privately. +- `stage_extracted_payload_` moves every entry of the private directory. Its + stripping branch is deleted: with a private directory it would become + reachable and change the layout. +- Script and SubOS packages are unaffected; their default installers fill + `install_dir` before staging. + +### 2.5 Stores that are already swept + +`xlings self doctor` reports a version directory whose top level contains a +`.lock` file. The downloader creates that file only in `runtimedir`, so its +presence in a payload is the sweep's fingerprint. `--fix` reinstalls that +version, which applies 2.3. mcpp adds no code: its registry is an xlings home +that mcpp drives through `XLINGS_HOME` (`src/xlings/xlings.cppm:1235`), and the +remedy is `XLINGS_HOME= xlings self doctor --fix`. + +### 2.6 Criteria + +- **Positive.** In a fresh home, install a package whose hook leaves its archive + in `runtimedir` (for example `xim:gcc`), then a hookless archive package + (`mcpp:plugins`). The plugins directory's entry set equals its archive's + top-level entry set, and `runtimedir` still holds both archives. +- **Negative.** The same sequence on 2026.9.12.1 finds the gcc archive inside + the plugins directory; the e2e is red there before it is green here. +- **Layout.** A single-directory archive installs as `//…`, and a + `*/mcpp.toml` pointer still matches exactly once. +- **Doctor.** A store seeded with a swept directory is reported, and after + `--fix` it equals a fresh install. +- Linux, macOS and Windows (rename within one volume, and the copy fallback). + +## 3. B — a POSIX redirect reaches cmd.exe + +### 3.1 Mechanism + +`vendored_xlings_version` runs `"" --version 2>/dev/null` +(`src/fallback/xlings_binary.cppm:159-160`) through +`mcpp::platform::process::capture`, which on Windows is `_popen`, that is +`cmd.exe /c` (`modules/platform/src/process.cppm:460-476`). cmd resolves +`/dev/null` to `\dev\null` on the current drive and cannot open it. It writes +the message to its own stderr, which `_popen` does not capture, and does not +run the program, so the probe returns an empty version *(code for the last two +effects)*. + +The probe runs in `acquire_xlings_binary` (`xlings_binary.cppm:53-58`) during +configuration loading (`src/config.cppm:753`) whenever the vendored binary +already exists. That is every command that loads the configuration, except the +one that first copies the binary. `mcpp --version` loads no configuration, +which is what xlings#543 observed. The probe and its redirect arrived together +in `fdad165b` (2026.8.8.2, #378) and are in every release since. + +### 3.2 Measured + +Windows CI of 2026-09-14. In the windows-2022 fresh install of the released +2026.9.14.3, the command that copies the vendored xlings prints nothing, and +every later `mcpp run` and `mcpp new --template` prints the line. On `main`, +`ci-windows-e2e` printed it 26 and 24 times and `toolchains + regressions` 16 +times. Every job was green, because no test asserts that the line is absent. + +### 3.3 The silent twin + +An empty version returns early (`xlings_binary.cppm:57`). On Windows the vendored +xlings is therefore never compared with `kXlingsPinnedVersion`, a pin change +never reaches an existing home, and `mcpp self doctor` can only report that it +cannot read the version (`src/doctor.cppm:495-499`). xlings#543 is the field +instance: the `subos_info` block was missing, `xlings self update` and a +reinstall of xlings did not help, and deleting `~/.mcpp` did. The check that +exists for exactly that case never ran *(code; §6, run 2)*. + +### 3.4 Rule and change + +> A program probe is an argument vector. Redirection and the null device belong +> to the platform layer. + +Five command strings carry POSIX grammar on a path that Windows reaches. All +five move to the argument-vector launcher `capture_exec` (`process.cppm:671`; +`posix_spawn` on POSIX, and on Windows a quoted command line with `2>&1`, +`:741`). + +| site | reached on Windows | +|---|---| +| `src/fallback/xlings_binary.cppm:160`, xlings version | every command after the first | +| `src/toolchain/post_install.cppm:853`, `-dumpspecs` for the clean link specs (`prepare.cppm:11833-11835`) | every prepared build with GCC | +| `src/toolchain/gcc.cppm:126`, libstdc++ fallback probe (single-quoted as well) | GCC whose `bits/std.cc` scan misses | +| `src/build/execute.cppm:814`, freestanding size report | freestanding targets | +| `src/pm/publisher.cppm:363`, `sha256sum` | `mcpp publish`, where the tool is also absent | + +`src/build/hermetic.cppm:149` returns early off Linux (`:110`), and the patchelf +walk in `post_install.cppm:105-142` touches ELF files only. `xlings --version` +writes one line to stdout and nothing to stderr, so the merged capture leaves +the version parser unchanged. + +### 3.5 Criteria + +- **Negative**, windows-2022. In a fresh home, the second command's stderr does + not contain `The system cannot find the path specified.` It does on + 2026.9.14.3 (3.2). +- **Positive.** `mcpp self doctor` prints `vendored xlings (pinned

)` on + Windows. +- **Twin.** With a vendored `xlings.exe` older than the pin and a newer system + xlings, the next command prints `Updating vendored xlings -> `. + Linux is the control leg. + +## 4. Moving mcpp to the latest xlings + +1. xlings: 2.3 to 2.6 in one PR, released as 2026.9.14.1 (X) and indexed. +2. mcpp: `kXlingsVersion = X` together with 3.4, in mcpp#639. The constant is the + only edit; `check_version_pins.sh` names the other pin points. +3. Verification on the published mcpp: the CI matrix; a sandbox with a fresh + home (2.6's positive leg inside mcpp's registry, `mcpp:plugins` after a + toolchain install); and a home that carries a swept directory (2.5). + +X also carries everything since 2026.9.5.1, including 2026.9.12.1's changes to +`remove`, `self doctor`, the index overlay and one-time notices. The matrix and +the sandbox are the gate for those; no mcpp change is expected from them. + +## 5. C — the build database (#636) + +### 5.1 Where each part belongs + +| part | content | home | why not elsewhere | +|---|---|---|---| +| facts | translation units and their argument vectors; `provides`, `requires` and the declaration form from mcpp's scanner, `scan_overrides` included; toolchain identity; the std units mcpp compiles; the inputs whose change changes the plan; planning without writing into the project | engine | Only `prepare_build` computes them. A build program runs before the plan exists and sees its own package; a `dist-*` member runs under `mcpp pack` on link outputs. No plugin reaches the finished plan. | +| format | S1 at level 2, rendered from the facts, inside the docs/50 envelope (which is S2 §3.4) | engine, beside `compile_commands.json` | P2977/S1 is a build-system-neutral database, as the CDB is. A plugin would need the engine to export the same facts first (a second contract), a new dispatch point, and a project manifest that declares the plugin, which an editor cannot add to a user's project. | +| structured options | S1 level 3, `ide.options` | the S1 reference library, beside its schema | mcpp holds flags as strings (`src/build/flags.cppm:37-41`) and renders `arguments` by splitting them (`src/build/compile_commands.cppm:221-264`). Options derived inside the engine are a second parser of the same argv that must follow every flag the engine adds. S1 §9 rule 1 already assigns the step to whoever lacks `options`, and lsp-mcpp already parses the three dialects. | +| editor policy | watching and debouncing, time bounds, stale models, fallback to `--configure-only`, status text | consumer (S2 §5) | Behaviour of an editor session, not a property of the build. | + +The answer to the review question: this is not a plugin. The engine keeps the +facts and a neutral format, and nothing in the engine names lsp-mcpp. + +### 5.2 The contract is specifications + +- The document conforms to S1 profile 0.2.0. The issue links the + `feat/lsp-mcpp-v1` branch, no `spec-s1-v*` or `spec-s2-v*` tag exists, and the + schema's `$id` names `github.com/mcpp-community/lsp-mcpp`, which does not + exist. mcpp therefore cites the specification by commit, which is immutable. +- The schema at that commit is vendored into mcpp's tests, with the commit + recorded beside it, and validates every document they produce. No mcpp test + runs lsp-mcpp or reads its fixtures. +- mcpp's own obligations (5.3 to 5.6) are written as SPEC-005 in `docs/specs/`, + with an implementation status per rule, as SPEC-003 does. docs/50 gains the + kind and the command's effects. +- Once the command exists, `build.ninja`, the std cache's `std-module.json` and + the text of `mcpp.toml` are declared not to be interfaces. + +### 5.3 What the engine changes + +`BuildOverrides::work_dir` already means "where mcpp writes": `target/`, +`mcpp.lock`, `compile_commands.json`, `.mcpp/` and the build programs' artifact +directory move together (`src/build/prepare.cppm:1012-1025`), and the host-tool +sub-build uses it in production (`:9317-9330`). The command runs `prepare_build` +with `work_dir = /cache/workdirs/` and +renders the plan in memory. `run_configure_plan`, which writes `build.ninja` and +the CDB and stages BMIs, is not called. + +Five engine changes make this exact. Each removes a leak or a second +derivation, and each lands with its own criterion before the command does. + +1. **Two writes ignore `work_dir`** and use the literal root: the mangling stage + directory (`prepare.cppm:6965`) and the root package's `[build] + generated_files` (`:4185`). The first moves to `work_dir`. The second is a + source file by design, so the command compares instead of writing: identical + content needs no write, and a missing or different file becomes a `warning` + diagnostic that names it. +2. **`prepare_build` compiles the std module** (`:11383`). Its identity (cache + paths and commands) becomes computable without compiling, and the command + uses only the identity. This is the only compilation that planning performs + for a project without build programs. +3. **The std build commands are shell strings** (`src/toolchain/gcc.cppm:182-204`, + `cd && … 2>&1`), and the strings are part of the std cache identity + (`src/toolchain/stdmod.cppm:317-330`). The builders are not changed: a + changed string would invalidate every user's std BMIs on upgrade. The std + units' `work-directory` and `arguments` are recovered from the first command + that names the module source, by the inverse of mcpp's own rendering (the + `cd` prefix, environment assignments and the trailing redirect removed, the + quoting undone). Unit tests drive every builder and assert the round trip, + so a builder that changes shape fails a test rather than the database. +4. **`emit_compile_commands` builds each entry inline.** The per-unit invocation + (`directory`, `source`, `arguments`, `output`) becomes one record that both + the CDB and the database render, so equal `arguments` hold by construction. +5. **The scanner discards the declaration form.** A `module M;` implementation + unit is stored as `requires M` (`src/modgraph/scanner.cppm:941-952`), which + cannot be told apart from a non-module unit that imports `M`. One enum on + `SourceUnit` records the form that was read; a `scan_overrides` unit records + that it is unknown. + +`mcpp.lock`: the command copies the project's lock into `work_dir`, where +`prepare_build` reads it (`:2035-2037`), and compares the copy afterwards. A +difference is a `warning` diagnostic; the project's lock is never written. + +Build programs still run with the package root as their working directory +(`src/build/build_program.cppm:1468`), and a dependency's host tool is still +built into the global tool store when a build program needs it. The no-write +guarantee covers mcpp's writes. A build program that writes outside +`MCPP_OUT_DIR` is outside the guarantee, as it is under `mcpp build`. + +### 5.4 Mapping + +| S1 | from | note | +|---|---|---| +| toolchain id | `--` | | +| `family` | `Toolchain::compiler_name()` (`modules/toolchain-model/src/model.cppm:261-268`): `gcc`, `clang`, `msvc` | not `compiler_family()`, which answers `llvm` | +| `driver`, `version`, `target`, `sysroot` | `binaryPath`, `version`, `targetTriple`, `sysroot` | | +| `stdlib` | `name` and `version` from `stdlibId` and `stdlibVersion`; no `module-metadata` | std resolves through units (next row) | +| set `mcpp:std` | units for `stdModuleSource` and `stdCompatSource`, carrying the commands of 5.3 (3) | One rule for GCC's `bits/std.cc`, libc++'s `std.cppm`, MSVC's `std.ixx` and a package's own `std.cppm`; S1 §6.1 lets units outrank a manifest. `mcpp:` is the engine's reserved namespace (SPEC-002), so the name cannot collide with a package. | +| sets | one per package; test targets' sources in `:test` | `family-name` is the package; `ide.configuration` is `BuildContext::profile`; `ide.kind` comes from the package's declared targets | +| `visible-sets` | every other set | The engine resolves imports over one flat graph per invocation (`scanner.cppm:1289-1319`). A narrower closure would describe a rule the build does not enforce; if the engine later refuses undeclared imports, the database inherits it. | +| units | every `CompileUnit` except NASM units, as in the CDB (`compile_commands.cppm:225-228`) | rendered from the record of 5.3 (4) | +| `provides` | `providesModule` mapped to `""` | S1-8-6 permits an empty path for a producer that performs no build | +| `requires` | `imports` | partitions are already written in full | +| `ide.role` | the declaration form of 5.3 (5) | `unknown` for `scan_overrides` units | + +### 5.5 `watch` and `inputs-fingerprint` + +- `watch` lists what the resolution read and a user edits: the manifests of the + root, the members and the path dependencies (absolute when outside the + workspace); `mcpp.lock`; every `build.mcpp` and the inputs it declared through + `rerun_if_changed` and `rerun_if_changed_glob`; each package's source globs, + as LSP patterns; and `$MCPP_HOME/config.toml`, which holds the default + toolchain. Environment variables that steer resolution, such as + `MCPP_TOOLCHAIN`, cannot be watched, and SPEC-005 names them. +- `inputs-fingerprint` is a digest of the contents that `watch` matches at the + time of the run, the mcpp version and the selector. It is not the build + fingerprint (`modules/toolchain-model/src/fingerprint.cppm`), which answers a + different question and does not include `mcpp.lock` (`prepare.cppm:11360`). +- The command writes no file that `watch` names; this follows from 5.3. +- Reusing an earlier result while the fingerprint is unchanged is not part of + the first version. If it is added, its criterion is the fast path's: A, then + B, then A. + +### 5.6 Output, selectors, failures and effects + +- Without `--format`, the command prints the bare S1 document, as `mcpp emit + xpkg` prints a bare descriptor; `-o ` writes that document atomically + instead. `--format json` prints the envelope of docs/50 with the document in + `data.database`. +- The selectors are those of `mcpp build` (`--target`, `--toolchain`, + `--profile`, `-p`/`--package`, `--workspace`); the issue's `--member` is + `--package`. +- `--workspace` yields one document whose sets are named `/`. + Each member is planned separately (`src/cli/cmd_build.cppm`), so a dependency + shared by two members appears once per member, with that member's arguments. +- A failure prints one envelope without `data`, whose `diagnostics` carry a + stable `code`, and exits 1 (docs/50 §3). Logs go to stderr. A workspace in + which one member fails to plan fails as a whole, and the diagnostic names the + member. +- `--protocol-version` advertises `"mcpp.build-database": 1` and, for + `emit build-database`, the effects `read-project`, `network`, + `write-global-cache` and `exec-build-script`, never `write-project`. The + envelope's `effects` lists what this run did. +- `--format ndjson` remains an error (exit 2) until the issue's second phase. + +### 5.7 Criteria + +1. Every document produced for `examples/`, for a project with `tests/` and a + dev-dependency, and for a workspace validates against the vendored S1 + schema; every set has `ide.toolchain` and every unit has `ide.role`. +2. One plan rendered as CDB and as database gives equal `arguments` for every + unit. The unit test is shown to fail when one renderer receives a different + flag string. +3. The project tree's content hash is equal before and after the command. The + same measurement around `--configure-only` differs, which shows that the + measurement can see a write. +4. The `mcpp:std` units name `llvm-generated/std.cppm` with + openkal-llvm-runtime, `bits/std.cc` with `gcc@16.1.0` and `std.ixx` with + `msvc@system`; on the three CI hosts, each unit's `arguments` compile its + source when run in its `work-directory`. +5. A project with a syntax error, and a project never built, both produce a + document. +6. With mcpp's own tree as input (it has no build program), no compiler process + starts, on a cold std cache and on a warm one; a driver wrapper that fails + when invoked is the probe. The wall time on a warm cache is of the order of + `--configure-only`. +7. `tests/unit/test_wire.cpp` pins the kind's key set; `--format ndjson` exits 2 + with empty stdout. + +### 5.8 Not in mcpp + +S1 level 3 options; S2 stream mode and the `ndjson` progress stream; editor-side +watching, time bounds and fallback; S3 and S4. The issue's follow-up asks for a +project-level default target, which exists as `[build] target` +(`docs/04-mcpp-toml.md:397`, `prepare.cppm:2535-2536`); whether it serves a +repository that needs `x86_64-windows-gnu` on Windows hosts only is a usage +question for lsp-mcpp. + +## 6. Plan + +### 6.1 Repositories and pull requests + +| repository | pull request | content | +|---|---|---| +| openxlings/xlings | one PR, version 2026.9.14.1 | 2.3 to 2.6: the staging rule in xpkg spec §6, private extraction, the doctor check, tests | +| openxlings/xim-pkgindex | the `xlings` bump; later the bot's `mcpp` bump | index entries for the two releases | +| xlings-res (GitHub and GitCode) | none | release mirrors; the GitCode leg is completed locally with `gtc` where CI leaves it short | +| mcpp-community/mcpp | #639, renamed | B, the `kXlingsVersion` move, C (the five engine changes, the command, SPEC-005, docs/50 and its translation, tests), the release version | +| mcpp-community/mcpp-index | none expected | verification only: packages with `*/…` pointers install and build under the new xlings | +| lsp-mcpp | none | its consumer already reads a level 2 document and completes it (`src/project/mcpp.cpp`, `enrich_database`); S1 is cited by commit and its schema is vendored | + +### 6.2 Tasks and dependencies + +| id | task | depends on | +|---|---|---| +| X1 | xlings: private extraction and the staging rule; strip branch removed | none | +| X2 | xlings: spec sentence in xpkg-manifest-v1 §6 | none | +| X3 | xlings: doctor reports a swept payload; `--fix` reinstalls it | X1 | +| X4 | xlings: e2e for the rule (positive, negative, layout) and the doctor | X1, X3 | +| X5 | xlings: PR, CI, merge, release 2026.9.14.1, mirrors, index | X1 to X4 | +| M1 | mcpp: the five probes of 3.4 through `capture_exec` | none | +| M2 | mcpp: e2e for B (no message on the second command, the doctor reads the version, an older vendored xlings is replaced) | M1 | +| M3 | mcpp: the five engine changes of 5.3 | none | +| M4 | mcpp: `emit build-database` (renderer, `watch`, fingerprint, lock comparison, effects, kind) | M3 | +| M5 | mcpp: unit, contract and e2e tests for C, with the vendored S1 schema and a validator that refuses keywords it does not implement | M4 | +| M6 | mcpp: SPEC-005, docs/50 and its translation, docs/09, the three specification indexes | M4 | +| M7 | mcpp: `kXlingsVersion = 2026.9.14.1` | X5 | +| M8 | mcpp: #639 renamed; CI green; self-review; merge | M1 to M7 | +| R1 | mcpp release; mirrors (GitCode completed with local `gtc`); index merge | M8 | +| V1 | sandbox (`xlings subos use --sandbox --cmd`, CN mirror): a fresh home installs the released mcpp, builds, emits a database, and its registry holds no swept payload | R1 | +| V2 | sandbox: mcpp-index packages with `*/…` pointers, a package-provided `std`, and `xlings self doctor` on a swept store | R1, X5 | +| V3 | reply on #636; records | V1, V2 | + +M1 to M6 proceed in parallel with X1 to X5. M7 is the only mcpp task that waits +on another repository, and it is small. + +### 6.3 Runs that settle the *(code)* statements + +1. windows-2022: the message of 3.1 is absent after M1 (M2's negative leg). +2. Every host: an older vendored xlings is replaced after M1 (M2's twin leg; + the Linux leg is the control). +3. xlings: the concurrent download of 2.2 is not measured; 2.4 removes the + sweep, which removes the question. + +## 7. Decisions + +1. **Level 2 from mcpp; level 3 from whoever lacks `options`.** S1 §9 rule 1 + makes `options` authoritative when present, so an incomplete `options` object + is worse than none, and the engine holds no complete structured form. +2. **`visible-sets` states the flat graph the engine enforces.** +3. **Sets per package**, plus `:test` for the sources of test targets + and `mcpp:std`; in a workspace every name is prefixed with `/`. +4. **`provides` paths are empty**, because the command performs no build. +5. **`work_dir` lives in the global cache**; a root `generated_files` entry that + is missing or stale is a warning, never a write. +6. **SPEC-005** is a new specification in `docs/specs/`. +7. **The command is `mcpp emit build-database`**, beside `emit xpkg`. +8. **The std cache identity does not change** (5.3, item 3). +9. **S1 is cited by commit.** The schema is vendored with the commit it was taken + from; tagging the specification is its owner's step and blocks nothing here. + +## 8. Self-review + +| angle | finding | effect on the plan | +|---|---|---| +| architecture | The engine gains one command and five general changes; no consumer name appears in mcpp; xlings owns its store and its repair | none | +| stability | Rewriting the std builders as data would change the strings that name the std cache, and every home would rebuild std on upgrade | 5.3 item 3 recovers argv from the strings instead | +| stability | A private extraction directory shared by two installs of one package would collide | the directory name carries the process id; removal on every exit path | +| simplicity | Level 3 would need a second parser of every flag the engine emits | level 2 (7.1) | +| user experience | An editor needs the document without a file; a script may want a file | bare document by default, `-o` for a file, `--format json` for the envelope (5.6) | +| compatibility | The CDB must not change; `--protocol-version` gains keys only; new stores keep the un-stripped layout the 55 pointers depend on | a golden test for the CDB bytes; 2.3 keeps the top-level directory | +| cross-platform | B exists only on Windows, so its criteria run on windows-2022; C's paths use the CDB's spelling on every host | M2 includes a Windows leg; units reuse the CDB record | +| consistency | One envelope builder, one selector parser, one unit record, one std command source | none | +| seamless upgrade | A pin move reaches an existing Windows home only through B's fix; swept stores keep working and are repaired on request | B and M7 ship in one pull request; `self doctor --fix` | +| test coverage | Each criterion has a leg that fails on the released binary; the schema validator refuses keywords it does not implement, so a schema change cannot pass unvalidated | M2 and M5 | diff --git a/.agents/docs/README.md b/.agents/docs/README.md index cb5af365..ad8c3412 100644 --- a/.agents/docs/README.md +++ b/.agents/docs/README.md @@ -18,7 +18,7 @@ superseded_by: 2026-09-07-....md # when status is superseded --- ``` -286 records. +287 records. ## By subject @@ -28,6 +28,10 @@ Records that declare one. Everything else is listed by date below. - [Four upstream asks from a UI framework: what each one is under mcpp's design, and the combined plan](2026-09-13-four-upstream-asks-from-a-ui-framework.md) — landed +### design + +- [The build database of #636, and two defects on the way to the latest xlings](2026-09-14-636-build-database-and-the-latest-xlings.md) — active + ### docs - [The documentation as a book: a chapter-by-chapter design](2026-09-08-the-documentation-as-a-book.md) — active @@ -68,6 +72,7 @@ Records that declare one. Everything else is listed by date below. ### 2026-09 +- [The build database of #636, and two defects on the way to the latest xlings](2026-09-14-636-build-database-and-the-latest-xlings.md) — active - [#634 implemented across five repositories: the plan, its review, and the ledger that tracks it](2026-09-14-634-implementation-plan.md) — landed - [A framework's CMake parity list: the twenty-one items of #634, read against the code and routed to where each one belongs](2026-09-14-634-cmake-parity-items-by-home.md) — landed - [Four upstream asks from a UI framework: what each one is under mcpp's design, and the combined plan](2026-09-13-four-upstream-asks-from-a-ui-framework.md) — landed From 62bc7526df680234106a4652bb1220d74130b41e Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Mon, 14 Sep 2026 21:57:03 +0800 Subject: [PATCH 03/16] fix(platform,xlings): a program probe is an argument vector, not a /bin/sh string The vendored-xlings version probe ran ` --version 2>/dev/null` as a command string. On Windows every command string reaches cmd.exe, which cannot open /dev/null: it printed "The system cannot find the path specified." in every command after the first, did not run xlings, and returned an empty version, so a Windows home never replaced a vendored xlings older than the pin. mcpp.platform.process gains capture_stdout: the program runs directly, its standard output is captured, its standard error is discarded and its standard input is empty; on Windows the redirect names cmd.exe's own null device. The xlings probe and the four other program probes that carried POSIX grammar on a path Windows reaches use it: the clean link specs, the libstdc++ fallback probe, the freestanding size report and the publish digest. Tests: capture_stdout unit tests (stdout only, exit code, empty stdin, environment, empty output for a missing program, the Windows command line); a version-probe unit test that runs a .bat through the real launcher on Windows; e2e 687 (no path error on the second command; an older vendored binary is replaced). --- modules/platform/src/process.cppm | 112 ++++++++++++++++++ src/build/execute.cppm | 11 +- src/fallback/xlings_binary.cppm | 8 +- src/pm/publisher.cppm | 7 +- src/toolchain/gcc.cppm | 11 +- src/toolchain/post_install.cppm | 8 +- ...ored_xlings_probe_is_an_argument_vector.sh | 75 ++++++++++++ tests/unit/test_process_run_exec.cpp | 50 ++++++++ tests/unit/test_xlings_version_pin.cpp | 35 ++++++ 9 files changed, 299 insertions(+), 18 deletions(-) create mode 100755 tests/e2e/687_the_vendored_xlings_probe_is_an_argument_vector.sh diff --git a/modules/platform/src/process.cppm b/modules/platform/src/process.cppm index ce2145ff..60572608 100644 --- a/modules/platform/src/process.cppm +++ b/modules/platform/src/process.cppm @@ -32,6 +32,7 @@ module; // Linux and macOS launchers do a direct exec (see run_exec / capture_exec // below); only Windows keeps the std::system shell path (#248). #include // pipe, dup2, close, read +#include // O_RDONLY, O_WRONLY for capture_stdout's /dev/null #include // waitpid #include // posix_spawnp, posix_spawn_file_actions_* (incl. addchdir_np) // The deadline runners' headers (signal.h, errno, poll.h, fcntl.h, time.h) @@ -110,6 +111,37 @@ RunResult capture_exec( std::string_view cwd = {}, int* spawn_error = nullptr); +// A PROGRAM PROBE: run `argv` directly, capture its standard output only, give +// it an empty standard input, and discard its standard error. +// +// This is the argument-vector form of ` 2>/dev/null`, and it +// exists because that string form is written in one shell's grammar. On +// Windows every command string reaches cmd.exe, which resolves `/dev/null` to +// `\dev\null` on the current drive, cannot open it, prints "The system cannot +// find the path specified." on its own stderr and does not run the program. +// Measured on windows-2022 with mcpp 2026.9.14.3: the vendored-xlings version +// probe printed that line in every command after the first, and returned an +// empty version, which silently disabled the pin check it existed for. The +// null device and the redirection belong to this layer; a caller states only +// the program and its arguments. +// +// The output is the program's standard output and nothing else. A program +// that cannot be started yields exit code 127 and EMPTY output: a probe parses +// what it captured, so a launcher error message in `output` would be read as +// the program's answer. `spawn_error` has run_exec's contract. +RunResult capture_stdout( + const std::vector& argv, + const std::vector>& extraEnv = {}, + int* spawn_error = nullptr); + +// capture_stdout for a host tool: the target runtime library search variable is +// cleared for the child, as capture_host_tool does for a command string. +RunResult capture_host_tool_stdout(const std::vector& argv); + +// The command line capture_stdout hands to cmd.exe on Windows. Host-independent +// so the Windows rendering is testable from any platform. +std::string windows_stdout_probe_command(const std::vector& argv); + // Deadline variants: kill the child once `deadline` elapses and set // *timed_out. A zero deadline means no limit. // @@ -753,6 +785,86 @@ RunResult capture_exec( #endif } +// Host-independent (see the declaration): always the Windows shape. The +// redirect names cmd.exe's own null device, and the argv is quoted by the one +// shaper every Windows launch in this file uses. +std::string windows_stdout_probe_command(const std::vector& argv) { + return windows_command_from_argv(argv) + " 2>nul"; +} + +RunResult capture_stdout( + const std::vector& argv, + const std::vector>& extraEnv, + int* spawn_error) +{ + RunResult result; + if (spawn_error) *spawn_error = 0; + if (argv.empty()) { result.exit_code = 127; return result; } +#if defined(__linux__) || defined(__APPLE__) + int fds[2]; + if (::pipe(fds) != 0) { result.exit_code = 127; return result; } + + auto envStore = merged_environ(extraEnv); + std::vector envp; + for (auto& s : envStore) envp.push_back(s.data()); + envp.push_back(nullptr); + std::vector cargv; + for (auto& a : argv) cargv.push_back(const_cast(a.c_str())); + cargv.push_back(nullptr); + + posix_spawn_file_actions_t fa; + ::posix_spawn_file_actions_init(&fa); + ::posix_spawn_file_actions_addopen(&fa, 0, "/dev/null", O_RDONLY, 0); + ::posix_spawn_file_actions_adddup2(&fa, fds[1], 1); // stdout -> pipe + ::posix_spawn_file_actions_addopen(&fa, 2, "/dev/null", O_WRONLY, 0); + ::posix_spawn_file_actions_addclose(&fa, fds[0]); + ::posix_spawn_file_actions_addclose(&fa, fds[1]); + + // Owned as capture_exec's child is: a probe that outlives an interrupted + // mcpp is the same orphan, only smaller. + posix_spawnattr_t attr; + ::posix_spawnattr_init(&attr); + ::posix_spawnattr_setpgroup(&attr, 0); + ::posix_spawnattr_setflags(&attr, POSIX_SPAWN_SETPGROUP); + + pid_t pid = 0; + int sp = ::posix_spawnp(&pid, cargv[0], &fa, &attr, cargv.data(), envp.data()); + ::posix_spawnattr_destroy(&attr); + ::posix_spawn_file_actions_destroy(&fa); + ::close(fds[1]); + if (sp != 0) { + ::close(fds[0]); + result.exit_code = 127; + if (spawn_error) *spawn_error = sp; + return result; + } + mcpp::platform::unixproc::guard_group_on_signal(pid); + + std::array buf{}; + ssize_t n; + while ((n = ::read(fds[0], buf.data(), buf.size())) > 0) + result.output.append(buf.data(), static_cast(n)); + ::close(fds[0]); + int status = 0; + while (::waitpid(pid, &status, 0) < 0) { /* EINTR retry */ } + mcpp::platform::unixproc::unguard_group(pid); + result.exit_code = normalize_exit_code(status); + return result; +#else + // cmd.exe reports a program it cannot find on stderr, which the redirect + // discards, and returns 9009; the output stays empty either way. + return capture_with_env(windows_stdout_probe_command(argv), extraEnv); +#endif +} + +RunResult capture_host_tool_stdout(const std::vector& argv) { + auto key = mcpp::platform::env::host_tool_runtime_library_path_key(); + std::optional runtime_env; + if (!key.empty()) + runtime_env.emplace(key, std::nullopt); + return capture_stdout(argv); +} + // ─── The ONE place the platform question is asked for a bounded run ──────── // // Both launchers answer the same contract behind a `std`-free interface (see diff --git a/src/build/execute.cppm b/src/build/execute.cppm index 9c3d7daf..95f5a8eb 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -810,11 +810,12 @@ void report_freestanding_size(const BuildContext& ctx) { auto art = ctx.outputDir / lu.output; std::error_code ec; if (!std::filesystem::exists(art, ec)) continue; - auto out = mcpp::xlings::run_capture(std::format( - "{} {} 2>/dev/null", mcpp::xlings::shq(tool.string()), - mcpp::xlings::shq(art.string()))); - if (!out) continue; - auto s = mcpp::freestanding::parse_size_output(*out); + // An argument vector rather than a `2>/dev/null` command string, + // which cmd.exe cannot open on a Windows host. + auto out = mcpp::platform::process::capture_stdout( + {tool.string(), art.string()}); + if (out.exit_code != 0 && out.output.empty()) continue; + auto s = mcpp::freestanding::parse_size_output(out.output); if (!s) continue; mcpp::ui::info("Size", std::format( "{} text {} data {} bss {} total {}", diff --git a/src/fallback/xlings_binary.cppm b/src/fallback/xlings_binary.cppm index 43f4ce56..0964d103 100644 --- a/src/fallback/xlings_binary.cppm +++ b/src/fallback/xlings_binary.cppm @@ -156,8 +156,12 @@ acquire_xlings_binary(const std::filesystem::path& destBin, bool quiet = false, std::string vendored_xlings_version(const std::filesystem::path& bin) { std::error_code ec; if (!std::filesystem::exists(bin, ec)) return {}; - auto r = mcpp::platform::process::capture(std::format( - "{} --version 2>/dev/null", mcpp::platform::shell::quote(bin.string()))); + // An argument vector, not a command string. The string form carried + // `2>/dev/null`, which cmd.exe cannot open: on Windows it printed "The + // system cannot find the path specified." in every command after the first, + // did not run xlings, and so returned an empty version, which made the + // pin comparison in acquire_xlings_binary return early forever. + auto r = mcpp::platform::process::capture_stdout({bin.string(), "--version"}); if (r.exit_code != 0) return {}; // Output carries ANSI colour; take the first dotted-numeric run. std::string out; diff --git a/src/pm/publisher.cppm b/src/pm/publisher.cppm index 206453b4..6f698e6c 100644 --- a/src/pm/publisher.cppm +++ b/src/pm/publisher.cppm @@ -360,9 +360,10 @@ std::string release_tarball_url(std::string_view repo, std::string sha256_of_file(const std::filesystem::path& file) { if (!std::filesystem::exists(file)) return {}; - auto cmd = std::format("sha256sum {} 2>/dev/null", - mcpp::platform::shell::quote(file.string())); - auto r = mcpp::platform::process::capture_host_tool(cmd); + // An argument vector rather than a `2>/dev/null` command string, which + // cmd.exe cannot open on a Windows host. + auto r = mcpp::platform::process::capture_host_tool_stdout( + {"sha256sum", file.string()}); if (r.exit_code != 0) return {}; // sha256sum format: "<64-hex> \n" auto sp = r.output.find(' '); diff --git a/src/toolchain/gcc.cppm b/src/toolchain/gcc.cppm index 1a425995..5ea3fbb7 100644 --- a/src/toolchain/gcc.cppm +++ b/src/toolchain/gcc.cppm @@ -123,11 +123,12 @@ std::optional find_std_module_source( } } - auto cmd = std::format("'{}' -print-file-name=libstdc++.so 2>/dev/null", - cxx_binary.string()); - auto r = mcpp::toolchain::run_capture(cmd); - if (r) { - auto trimmed = mcpp::toolchain::trim_line(*r); + // An argument vector: the command-string form was written for /bin/sh + // (single quotes and `2>/dev/null`), and cmd.exe reads neither. + auto r = mcpp::platform::process::capture_host_tool_stdout( + {cxx_binary.string(), "-print-file-name=libstdc++.so"}); + if (r.exit_code == 0) { + auto trimmed = mcpp::toolchain::trim_line(r.output); if (!trimmed.empty()) { std::filesystem::path libpath = trimmed; auto root2 = libpath.parent_path().parent_path(); diff --git a/src/toolchain/post_install.cppm b/src/toolchain/post_install.cppm index 73790f4c..8e0f49ab 100644 --- a/src/toolchain/post_install.cppm +++ b/src/toolchain/post_install.cppm @@ -849,9 +849,11 @@ std::filesystem::path write_clean_link_specs( // regenerating it every build would spawn a process for a constant. if (std::filesystem::exists(out, ec)) return out; - auto r = mcpp::platform::process::capture(std::format( - "{} -dumpspecs 2>/dev/null", - mcpp::platform::shell::quote(compilerBin.string()))); + // An argument vector: the command-string form carried `2>/dev/null`, which + // cmd.exe cannot open, so on a Windows host this probe printed a spurious + // path error in every prepared GCC build and never produced the file. + auto r = mcpp::platform::process::capture_stdout( + {compilerBin.string(), "-dumpspecs"}); if (r.exit_code != 0 || r.output.empty()) return {}; // `*link:` is a section header on its own line; its body is the next line. diff --git a/tests/e2e/687_the_vendored_xlings_probe_is_an_argument_vector.sh b/tests/e2e/687_the_vendored_xlings_probe_is_an_argument_vector.sh new file mode 100755 index 00000000..8d4280e9 --- /dev/null +++ b/tests/e2e/687_the_vendored_xlings_probe_is_an_argument_vector.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# requires: +# 687 -- the vendored-xlings version probe is an argument vector. +# +# It was the command string ` --version 2>/dev/null`. On Windows every +# command string reaches cmd.exe, which cannot open `/dev/null`: it printed "The +# system cannot find the path specified." in every command after the first, did +# not run xlings, and returned an empty version. acquire_xlings_binary reads an +# empty version as "unknown, keep it", so a Windows home never replaced a +# vendored xlings older than the pin. Measured on windows-2022 with 2026.9.14.3. +# +# Criteria, in a home whose vendored xlings exists before the command runs: +# A. The command's stderr carries no path error. The denominator is that the +# vendored binary exists, so the probe did run. +# B. A vendored binary whose `--version` answers a version older than the pin +# is replaced from MCPP_VENDORED_XLINGS, and the command says so. The stand-in +# is the ninja payload, whose `--version` prints a dotted version older than +# any dated xlings. Not `subos/default/bin/ninja`: that is an xlings shim, one +# multicall binary that answers as xlings once it is named `xlings`, and for +# the same reason the real xlings is kept under its own name. +# A and B discriminate on Windows; elsewhere they are the control legs. +set -e + +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT + +fail() { echo "FAIL: $1"; shift; for f in "$@"; do echo "--- $f ---"; cat "$f" 2>/dev/null; done; exit 1; } + +EXE="" +case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) EXE=".exe" ;; esac + +export MCPP_HOME="$TMP/mcpp-home" +# Offline: configuration loading still acquires the vendored xlings (it copies +# a local binary), and no bootstrap step reaches the network. +export MCPP_OFFLINE=1 +source "$(dirname "$0")/_inherit_toolchain.sh" +cd "$TMP" + +VENDORED="$MCPP_HOME/registry/bin/xlings$EXE" + +"$MCPP" self env > first.out 2> first.err || true +[ -f "$VENDORED" ] || fail "the first command did not vendor xlings at $VENDORED" first.out first.err + +# ── A ────────────────────────────────────────────────────────────────────── +"$MCPP" self env > second.out 2> second.err || true +if grep -qi 'cannot find the path specified' second.err; then + fail "A: the version probe reached a shell that could not open its redirect" second.err +fi +echo "ok: A, the probe ran and wrote nothing to stderr" + +# ── B ────────────────────────────────────────────────────────────────────── +NINJA="" +for cand in "$MCPP_HOME"/registry/data/xpkgs/xim-x-ninja/*/ninja$EXE \ + "$MCPP_HOME"/registry/data/xpkgs/xim-x-ninja/*/bin/ninja$EXE; do + if [ -f "$cand" ]; then NINJA="$cand"; break; fi +done +[ -n "$NINJA" ] || fail "B: no ninja binary to stand in for an older xlings under $MCPP_HOME/registry" +older=$("$NINJA" --version 2>/dev/null | head -1) +case "$older" in + [0-9]*.*) ;; + *) fail "B: the stand-in '$NINJA' answered '$older', not a dotted version" ;; +esac + +mkdir -p "$TMP/real" +cp "$VENDORED" "$TMP/real/xlings$EXE" +rm -f "$VENDORED" +cp "$NINJA" "$VENDORED" +chmod +x "$VENDORED" 2>/dev/null || true + +MCPP_VENDORED_XLINGS="$TMP/real/xlings$EXE" "$MCPP" self env > third.out 2> third.err || true +grep -q "vendored xlings $older -> " third.err \ + || fail "B: a vendored xlings answering $older was not replaced" third.out third.err +"$VENDORED" --version 2>/dev/null | grep -q '^xlings ' \ + || fail "B: after the replacement the vendored binary is not xlings" third.err +echo "ok: B, a vendored binary older than the pin was replaced" diff --git a/tests/unit/test_process_run_exec.cpp b/tests/unit/test_process_run_exec.cpp index 633a4dbe..36db0d4f 100644 --- a/tests/unit/test_process_run_exec.cpp +++ b/tests/unit/test_process_run_exec.cpp @@ -215,6 +215,47 @@ TEST(CaptureExecDeadline, SpawnFailureIsInOutputWhenCallerDoesNotAsk) { EXPECT_NE(r.output.find("error 2"), std::string::npos) << r.output; } +// ── capture_stdout: the argument-vector form of ` 2>/dev/null` ── + +TEST(CaptureStdout, CapturesStandardOutputOnly) { + auto r = process::capture_stdout( + {"/bin/sh", "-c", "echo out; echo 9.9.9-on-stderr 1>&2"}); + EXPECT_EQ(r.exit_code, 0); + EXPECT_EQ(r.output, "out\n"); +} + +TEST(CaptureStdout, PropagatesExitCode) { + auto r = process::capture_stdout({"/bin/sh", "-c", "echo partial; exit 3"}); + EXPECT_EQ(r.exit_code, 3); + EXPECT_EQ(r.output, "partial\n"); +} + +// A probe parses what it captured, so a launcher message must never be read +// as the program's answer: a missing program leaves the output empty. +TEST(CaptureStdout, MissingProgramLeavesOutputEmpty) { + int spawnErr = 0; + auto r = process::capture_stdout({"/no/such/program/mcpp-probe-2026.1.2.3"}, + {}, &spawnErr); + EXPECT_EQ(r.exit_code, 127); + EXPECT_EQ(spawnErr, ENOENT); + EXPECT_TRUE(r.output.empty()) << r.output; +} + +TEST(CaptureStdout, StandardInputIsEmpty) { + // `cat` returns at once with nothing when stdin is /dev/null; with an + // inherited terminal or pipe it would block or echo the parent's input. + auto r = process::capture_stdout({"/bin/sh", "-c", "cat; echo done"}); + EXPECT_EQ(r.exit_code, 0); + EXPECT_EQ(r.output, "done\n"); +} + +TEST(CaptureStdout, ExtraEnvironmentReachesTheChild) { + auto r = process::capture_stdout({"/bin/sh", "-c", "printf %s \"$MCPP_PROBE_ENV\""}, + {{"MCPP_PROBE_ENV", "reached"}}); + EXPECT_EQ(r.exit_code, 0); + EXPECT_EQ(r.output, "reached"); +} + #else // _WIN32 TEST(RunExec, WindowsCoveredByIntegration) { @@ -224,3 +265,12 @@ TEST(RunExec, WindowsCoveredByIntegration) { } #endif + +// Host-independent: the command line capture_stdout gives cmd.exe names cmd's +// own null device, never a POSIX path cmd would try to open as `\dev\null`. +TEST(CaptureStdout, WindowsCommandNamesCmdNullDevice) { + auto line = process::windows_stdout_probe_command( + {"C:\\Program Files\\xlings\\xlings.exe", "--version"}); + EXPECT_EQ(line, "\"C:\\Program Files\\xlings\\xlings.exe\" \"--version\" 2>nul"); + EXPECT_EQ(line.find("/dev/null"), std::string::npos) << line; +} diff --git a/tests/unit/test_xlings_version_pin.cpp b/tests/unit/test_xlings_version_pin.cpp index 131c5e8a..9156ca3e 100644 --- a/tests/unit/test_xlings_version_pin.cpp +++ b/tests/unit/test_xlings_version_pin.cpp @@ -13,6 +13,7 @@ // one, and an unparseable version is not evidence of being behind. #include +#include import std; import mcpp.fallback.xlings_binary; @@ -63,4 +64,38 @@ TEST(XlingsVersionPin, DatedSchemeIsNewerThanTheOldOne) { EXPECT_TRUE(fb::version_is_older("0.4.51", "0.4.54")); } +// THE PROBE READS STANDARD OUTPUT, AND ON WINDOWS IT RUNS AT ALL. +// +// The version used to be read through the command string +// ` --version 2>/dev/null`. cmd.exe cannot open `/dev/null`, so on Windows +// the probe printed "The system cannot find the path specified.", never ran +// xlings, and returned an empty version -- which acquire_xlings_binary reads as +// "unknown, keep it", so a Windows home never moved to a newer pin. On Windows +// this test runs a .bat through the real launcher, which is the path that +// failed; elsewhere a shell script. Both print a dotted number on stderr first, +// which must not be taken for the version. +TEST(XlingsVersionPin, ProbeReadsStandardOutputThroughTheLauncher) { + auto dir = std::filesystem::temp_directory_path() + / std::format("mcpp probe {}", std::chrono::steady_clock::now().time_since_epoch().count()); + std::filesystem::create_directories(dir); +#if defined(_WIN32) + auto fake = dir / "xlings.bat"; + { + std::ofstream os(fake, std::ios::binary); + os << "@echo off\r\necho warning 9.9.9 1>&2\r\necho xlings 2026.1.2.3\r\n"; + } +#else + auto fake = dir / "xlings"; + { + std::ofstream os(fake, std::ios::binary); + os << "#!/bin/sh\necho 'warning 9.9.9' 1>&2\nprintf 'xlings 2026.1.2.3\\n'\n"; + } + std::filesystem::permissions(fake, std::filesystem::perms::owner_all, + std::filesystem::perm_options::replace); +#endif + EXPECT_EQ(fb::vendored_xlings_version(fake), "2026.1.2.3"); + std::error_code ec; + std::filesystem::remove_all(dir, ec); +} + } // namespace From 9e31f50a094b53fc51319fca6ae87b0697f10f05 Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Mon, 14 Sep 2026 23:58:08 +0800 Subject: [PATCH 04/16] feat(emit): mcpp emit build-database, the plan as a build database written into nothing (#636) `mcpp emit build-database` plans as `mcpp build --configure-only` plans, with the same selectors, and prints the plan as an S1 build database (C++ Build Database: IDE Profile 0.2.0, a profile of WG21 P2977R2) without writing into the project. `--spec compile-commands` prints compile_commands.json entries instead; `--format json` wraps the document in the mcpp.build-database envelope; `-o ` writes it atomically. docs/specs/build-database.md (SPEC-005) states the rules. The document is S1 level 2: one set per package plus :test and mcpp:std; visible-sets lists every other set, because the engine resolves imports over one module graph; each unit's arguments come from the record the compile database renders; ide.role is the declaration form the scanner read. Five engine changes make the no-write planning exact: - BuildOverrides::work_dir had two writes that ignored it (the multi-version mangling stage, and the root package's generated_files, which are now compared rather than written under plan_only); - the std module is described without being compiled (describe_std_module, the derivation ensure_built builds from); - the std units' commands are recovered from the commands mcpp runs, so the std cache identity does not change; - compile_commands and the build database render one UnitInvocation record; - the scanner records the module declaration form (ModuleDeclaration). mcpp.lock is read from the project and never written back; a difference is the warning MCPP_LOCK_WOULD_CHANGE. Planning narration goes to stderr through a descriptor-level redirect. A failure envelope omits data. Tests: BuildDatabase.* unit tests drive the real gcc and clang std command builders; Scanner.DeclarationFormIsRecordedAsRead; CompileCommandsEmit.UnitInvocationsAreTheCompileDatabaseArguments; WireGolden.NullDataIsOmitted; e2e 688 validates the output against the vendored S1 schema and uses --configure-only as the control leg for the no-write criterion. --- CHANGELOG.md | 33 ++ docs/01-getting-started.md | 13 + docs/50-machine-output.md | 40 ++ docs/README.md | 1 + docs/specs/README.md | 1 + docs/specs/build-database.md | 158 +++++ docs/zh/01-getting-started.md | 10 + docs/zh/50-machine-output.md | 34 ++ docs/zh/README.md | 1 + modules/platform/src/terminal.cppm | 47 ++ src/build/build_database.cppm | 556 ++++++++++++++++++ src/build/build_program.cppm | 43 ++ src/build/compile_commands.cppm | 64 +- src/build/plan.cppm | 4 + src/build/prepare.cppm | 179 ++++-- src/cli.cppm | 47 +- src/cli/cmd_build.cppm | 255 +++++++- src/modgraph/graph.cppm | 29 + src/modgraph/p1689.cppm | 15 + src/modgraph/scanner.cppm | 7 + src/toolchain/stdmod.cppm | 109 +++- src/wire.cppm | 17 +- tests/e2e/688_emit_build_database.sh | 250 ++++++++ tests/e2e/_json_schema_subset.py | 154 +++++ tests/e2e/fixtures/s1/README.md | 19 + .../fixtures/s1/s1-build-database.schema.json | 427 ++++++++++++++ tests/unit/test_build_database.cpp | 156 +++++ tests/unit/test_compile_commands.cpp | 46 ++ tests/unit/test_modgraph.cpp | 29 + tests/unit/test_wire.cpp | 20 +- 30 files changed, 2675 insertions(+), 89 deletions(-) create mode 100644 docs/specs/build-database.md create mode 100644 src/build/build_database.cppm create mode 100755 tests/e2e/688_emit_build_database.sh create mode 100644 tests/e2e/_json_schema_subset.py create mode 100644 tests/e2e/fixtures/s1/README.md create mode 100644 tests/e2e/fixtures/s1/s1-build-database.schema.json create mode 100644 tests/unit/test_build_database.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 27882775..f2a29f23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,39 @@ ## [Unreleased] +### `mcpp emit build-database`:不写工程目录的构建数据库(#636) + +新命令按 `mcpp build --configure-only` 的方式、用相同的选择器规划,把计划打印为 +S1「C++ Build Database: IDE Profile」0.2.0(WG21 P2977R2 的一个 profile)文档,不写入 +工程目录;`--spec compile-commands` 改为打印 `compile_commands.json` 的条目, +`--format json` 用 `mcpp.build-database` 信封包裹,`-o ` 原子写入文件。规则见 +SPEC-005(`docs/specs/build-database.md`)。 + +- 文档满足 S1 等级 2:每个包一个集合,另有 `<包>:test` 与 `mcpp:std`;`visible-sets` + 列出其余所有集合,因为引擎在一张模块图上解析 import;每个单元的 `arguments` 与 + `compile_commands.json` 取自同一条记录;`ide.role` 取自扫描器读到的声明形式。 +- 规划写到 `$MCPP_HOME/cache/build-database/`:`BuildOverrides::work_dir` 此前有 + 两处写入不跟随它(多版本改名的暂存目录、根包 `generated_files`),现已修正;标准库 + 模块只描述不编译(`describe_std_module`);`mcpp.lock` 从工程读取、从不写回,不一致时 + 给出 `MCPP_LOCK_WOULD_CHANGE` 警告。 +- 标准库模块单元的命令从 mcpp 实际运行的构建命令中还原,std 缓存目录的身份不变。 +- 信封增加约定:失败时省略 `data`。`--protocol-version` 声明该命令的效应,不含 + `write-project`。 +- 测试:单测 `BuildDatabase.*`、`Scanner.DeclarationFormIsRecordedAsRead`、 + `CompileCommandsEmit.UnitInvocationsAreTheCompileDatabaseArguments`、 + `WireGolden.NullDataIsOmitted`;e2e 688 以随仓的 S1 schema 校验输出,并以 + `--configure-only` 作对照腿证明工程目录未被写入。 + +### Windows 上不再每条命令都打印 "The system cannot find the path specified." + +vendored xlings 的版本探针以命令串 ` --version 2>/dev/null` 运行。Windows 上 +命令串交给 cmd.exe,它打不开 `/dev/null`:除首次外的每条命令都打印这句提示,xlings +没有运行,探针返回空版本,于是 vendored xlings 从不按 pin 更新。`mcpp.platform.process` +增加 `capture_stdout`:直接运行程序、只捕获标准输出、丢弃标准错误、标准输入为空, +Windows 上重定向写 cmd.exe 自己的空设备;xlings 版本探针与另外四个带 POSIX 语法、 +Windows 可达的探针改用它。(单测 `CaptureStdout.*`、 +`XlingsVersionPin.ProbeReadsStandardOutputThroughTheLauncher`,e2e 687) + ### 在已激活的 xlings subos 中运行时,registry 不再被重定向(2026.9.14.3) `xlings subos use ` 打开的 shell 导出 `XLINGS_ACTIVE_SUBOS`,xlings 解析 subos diff --git a/docs/01-getting-started.md b/docs/01-getting-started.md index cf2e3166..1c9b0475 100644 --- a/docs/01-getting-started.md +++ b/docs/01-getting-started.md @@ -152,6 +152,19 @@ Run it only in a trusted workspace. The process exit code and the resulting `compile_commands.json` are the stable integration contract; stdout remains human-readable. +An editor that must not write into the project asks for the same plan on stdout +*(mcpp 2026.9.15.1+)*: + +```bash +mcpp emit build-database --format json +``` + +The document is an S1 build database: every translation unit with its compile +command, the modules it provides and requires, the toolchain and the standard +library units. `--spec compile-commands` prints `compile_commands.json` entries +instead. The fields are listed in [50 — Machine-Readable Output](50-machine-output.md), +and the rules in [SPEC-005](specs/build-database.md). + ## Incremental Compilation and Testing ```bash diff --git a/docs/50-machine-output.md b/docs/50-machine-output.md index a9c79d93..a0dfada4 100644 --- a/docs/50-machine-output.md +++ b/docs/50-machine-output.md @@ -59,6 +59,10 @@ for `mcpp.xpkg`, with no way to tell which actually changed. `effects` is always present. An empty array means "nothing"; an absent array would mean "unknown", which is a different claim. +`data` is present whenever the command produced its document. A command that +failed before it had one omits `data` rather than sending an empty object, and +its `diagnostics` state why; `mcpp.build-database` is such a kind (§8). + ### Diagnostics ```jsonc @@ -403,6 +407,42 @@ payload and run a dependency's build program. A client gates on that table *before* running anything, so an omission would be a safety claim that is not true. +### `mcpp.build-database` — the build plan as a build database *(mcpp 2026.9.15.1+)* + +``` +mcpp emit build-database [--spec s1|compile-commands] --format json +``` + +It plans as `mcpp build --configure-only` plans, with the same selectors, and +writes nothing into the project. `data` is: + +| field | | +|---|---| +| `spec` | `{"name": "s1", "version": "0.2.0"}`, or `{"name": "compile-commands"}` with `--spec compile-commands` | +| `database` | the document of that specification: an S1 build database, or the entries `mcpp build --configure-only` writes to `compile_commands.json` | +| `watch` | the inputs whose change can change the document: paths and glob patterns relative to the workspace root, or absolute paths | +| `inputs-fingerprint` | `fnv1a:<16 hex digits>`, a digest of those inputs, the mcpp version and the selectors | + +Without `--format`, the command prints the document alone; `-o ` writes +whatever it would print to `` instead. The content of the document, the +no-write guarantee and the `watch` rules are +[SPEC-005](specs/build-database.md). + +A failure omits `data` and exits 1, with the diagnostic code +`MCPP_BUILD_DATABASE_NO_PROJECT` outside a project or +`MCPP_BUILD_DATABASE_PLAN_FAILED` when planning fails. Warnings leave the +document in place: + +| code | | +|---|---| +| `MCPP_LOCK_WOULD_CHANGE` | the resolution differs from the project's `mcpp.lock`, which the command does not write | +| `MCPP_GENERATED_FILE_NOT_MATERIALIZED` | a root `[build] generated_files` entry is missing or stale on disk, and the command does not write it | +| `MCPP_BUILD_DATABASE_STD_UNIT_UNDESCRIBED` | no standard-library build command names its module source, so that unit is not listed | + +`--protocol-version` declares `init-mcpp-home`, `read-project`, `network`, +`write-global-cache` and `exec-build-script` for the command, and never +`write-project`. + ### `mcpp test --message-format json` — the test stream ``` diff --git a/docs/README.md b/docs/README.md index b54e8c9e..0cdb72ed 100644 --- a/docs/README.md +++ b/docs/README.md @@ -160,3 +160,4 @@ downstream tooling. - [SPEC-002 — The target side: reserved namespace, five layers, three rules](specs/target-side.md) - [SPEC-003 — The exit-code contract](specs/exit-codes.md) - [SPEC-004 — `mcpp.toml` planes, conditioning shape, resolution axes and naming](specs/manifest-semantics.md) + - [SPEC-005 — The build database `mcpp emit build-database` prints](specs/build-database.md) diff --git a/docs/specs/README.md b/docs/specs/README.md index 66f63b84..4e498e77 100644 --- a/docs/specs/README.md +++ b/docs/specs/README.md @@ -34,6 +34,7 @@ | [SPEC-002](target-side.md) | 目标侧模型与能力声明(`mcpp:` 保留命名空间、五层、三条规则) | 评审中 v1.0 | 2026-08-24 | mcpp >= 2026.8.24.2 | | [SPEC-003](exit-codes.md) | 退出码契约(分类、语义、稳定性承诺) | 评审中 v1.0 | 2026-09-01 | mcpp >= 2026.9.1.1 | | [SPEC-004](manifest-semantics.md) | `mcpp.toml` 的平面划分、条件化形状、解析轴与命名规约 | 草案 v1.0 | 2026-09-07 | 条件化形状 mcpp >= 2026.8.29.1;目标轴未实现 | +| [SPEC-005](build-database.md) | 构建数据库:`mcpp emit build-database` 的内容、取值规则与不写工程目录的保证 | 评审中 v1.0 | 2026-09-14 | mcpp >= 2026.9.15.1 | ## 文档约定 diff --git a/docs/specs/build-database.md b/docs/specs/build-database.md new file mode 100644 index 00000000..a1d914c0 --- /dev/null +++ b/docs/specs/build-database.md @@ -0,0 +1,158 @@ +# SPEC-005:构建数据库(`mcpp emit build-database`) + +| 项 | 值 | +|---|---| +| 规范编号 | SPEC-005 | +| 标题 | mcpp 输出的构建数据库:内容、取值规则与不写工程目录的保证 | +| 状态 | 评审中 v1.0 | +| 版本 | 1.0 | +| 最后修改 | 2026-09-14 | +| 对应实现 | mcpp >= 2026.9.15.1 | +| 相关设计文档 | `.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md` | +| 相关 issue | #636 | +| 依据的外部规范 | S1「C++ Build Database: IDE Profile」profile 0.2.0 与 S2 0.2.0 §3.4,取自 https://github.com/Sunrisepeak/lsp-mcpp-private 提交 `b82859d`(schema 自提交 `28ecd6e` 起未变);JSON Compilation Database | + +## 0. 适用范围 + +本规范规定 `mcpp emit build-database` 输出的文档、文档中每个字段取自构建计划的 +哪一部分,以及这条命令对工程目录的保证。文档格式由 S1 与 JSON Compilation +Database 定义,本规范不重复它们的字段定义,只规定 mcpp 作为生产方的义务。消费方 +的行为(监视、防抖、超时、把文档补全到 S1 等级 3)不属于本规范。 + +规范用语与实现状态标记见 [规范索引](README.md)。 + +## 1. 命令与文档 + +| 调用 | 标准输出 | +|---|---| +| `mcpp emit build-database` | S1 文档 | +| `mcpp emit build-database --spec compile-commands` | JSON Compilation Database | +| 以上任一加 `--format json` | [docs/50](../50-machine-output.md) 的信封,文档在 `data.database` | +| 以上任一加 `-o ` | 无;原本写到标准输出的内容原子地写入 `` | + +- **R1.1** `--spec` 的取值为 `s1`(默认)与 `compile-commands`。其他取值是用法错误: + 标准输出为空,退出码为 2。**已实现** +- **R1.2** 选择器与 `mcpp build` 相同:`--target`、`--toolchain`、`--profile`、 + `--release`、`--dev`、`--features`、`--cap`、`--accel`、`--no-accel`、`--static`、 + `--strict`、`-p`/`--package`、`--workspace`。同一组选择器下,文档描述的构建计划 + 与 `mcpp build --configure-only` 计算的计划相同;包有测试时,计划包含测试目标与 + dev-dependency。**已实现** +- **R1.3** 规划过程的叙述写到标准错误,标准输出只有文档或信封。**已实现** + +## 2. 不写工程目录 + +- **R2.1** 命令**禁止**写入工程目录,即根包、工作区成员与 path 依赖的源码树。规划 + 写入 `$MCPP_HOME/cache/build-database/`,`` 由工程根与成员决定。该目录 + 是缓存,可以随时删除。**已实现** +- **R2.2** 标准库模块被描述而不被编译。对没有构建程序的工程,命令不启动任何编译器。 + **已实现** +- **R2.3** `mcpp.lock` 从工程根读取,从不写回。规划得出的解析与工程中的锁不一致, + 或工程中没有锁而规划会写出一份时,输出警告 `MCPP_LOCK_WOULD_CHANGE`。**已实现** +- **R2.4** 根包 `[build] generated_files` 中缺失或内容与声明不一致的文件不被写入, + 每个输出一条警告 `MCPP_GENERATED_FILE_NOT_MATERIALIZED`。**已实现** +- **R2.5** 构建程序照常运行,工作目录为包根,与 `mcpp build` 相同;构建程序在 + `MCPP_OUT_DIR` 之外写入的内容不在本保证之内。依赖提供的宿主工具照常构建到全局 + 工具库。**已实现** +- **R2.6** `mcpp --protocol-version` 为这条命令声明 `init-mcpp-home`、`read-project`、 + `network`、`write-global-cache` 与 `exec-build-script`,不声明 `write-project`。 + **已实现** + +## 3. S1 文档 + +mcpp 输出的 S1 文档满足 S1 等级 2,不输出 `ide.options`。等级 3 所需的结构化选项由 +缺少 `options` 的一方按 S1 §9 规则 1 从 `arguments` 推导。 + +### 3.1 工具链 + +- **R3.1** `ide.toolchains` 的键为 `--`,其中 `family` 是 + mcpp 的族名(`gcc`、`llvm`、`msvc`),`triple` 是编译器自身的拼写。键对消费方不 + 透明,在一份文档内稳定。**已实现** +- **R3.2** `family` 为 `gcc`、`clang` 或 `msvc`。`driver` 为构建调用的驱动的绝对路径; + `target` 为编译器自身拼写的目标三元组;构建使用 sysroot 时给出 `sysroot`;`stdlib` + 给出 `name`(`libstdc++`、`libc++`、`msvc-stl` 或 `other`)与 `version`,不给出 + `module-metadata`,标准库模块经 §3.4 的单元解析。**已实现** + +### 3.2 集合 + +- **R3.3** 每个包一个集合,名为包的限定名(`.` 或 ``)。根包 + 测试目标的源文件归入集合 `<包>:test`,标准库模块的单元归入集合 `mcpp:std`。工作区 + 文档中,每个集合名带前缀 `<成员>/`。**已实现** +- **R3.4** `visible-sets` 列出同一成员的其余所有集合。引擎在一次调用的一张模块图上 + 解析 import,更窄的闭包会描述一条构建并不执行的规则。**已实现** +- **R3.5** `family-name` 为包名,`mcpp:std` 集合的为 `mcpp:std`;`ide.configuration` + 为 profile 名;`ide.kind` 在测试集合为 `test`,在根包集合按其目标为 `library`、 + `executable` 或 `other`,在依赖包集合与 `mcpp:std` 为 `library`。**已实现** +- **R3.6** 不输出 `baseline-arguments` 与 `local-arguments`。**已实现** + +### 3.3 翻译单元 + +- **R3.7** 除 NASM 单元外,构建计划中的每个编译单元是一个翻译单元。`source`、 + `work-directory`、`arguments`、`object` 与 `compile_commands.json` 中对应条目的 + `file`、`directory`、`arguments`、`output` 取自同一条记录,因而逐字相同。 + **已实现** +- **R3.8** `provides` 把单元提供的模块名映射到空字符串,命令不执行构建(S1-8-6); + `requires` 为单元导入的模块名,分区写全名 `M:P`。不输出 `private`,即 `false`。 + **已实现** +- **R3.9** `ide.role` 取自扫描器读到的模块声明形式: + + | 声明 | `ide.role` | + |---|---| + | 无模块声明 | `non-module` | + | `export module M;` | `module-interface` | + | `export module M:P;` | `module-partition-interface` | + | `module M:P;` | `module-partition-implementation` | + | `module M;` | `module-implementation` | + | `scan_overrides` 声明的单元;P1689 扫描中无法区分实现单元与导入者的单元 | `unknown` | + + **已实现** + +### 3.4 标准库模块 + +- **R3.10** 构建导入 `std` 时,`mcpp:std` 集合包含 `std` 的单元;工具链有 `std.compat` + 的构建命令时,还包含 `std.compat` 的单元。`provides` 分别为 `std` 与 `std.compat`, + `std.compat` 的 `requires` 为 `std`,角色均为 `module-interface`。该规则对工具链 + 自带的标准库(GCC 的 `bits/std.cc`、libc++ 的 `std.cppm`、MSVC STL 的 `std.ixx`)与 + 依赖包提供的 `std.cppm` 相同。**已实现** +- **R3.11** 这些单元的 `arguments` 与 `work-directory` 来自 mcpp 构建该模块时运行的 + 命令:mcpp 为宿主 shell 渲染的命令去掉 `cd`、环境变量赋值与重定向,再撤销引号。 + 命令中找不到该源文件时,不列出该单元,并输出警告 + `MCPP_BUILD_DATABASE_STD_UNIT_UNDESCRIBED`。**已实现** + +## 4. `--spec compile-commands` + +- **R4.1** 文档为 `mcpp build --configure-only` 在同一组选择器下写入 + `compile_commands.json` 的条目,差别只在输出路径位于 §2 的工作目录之下。标准库模块 + 的单元不在其中。**已实现** + +## 5. 信封 + +- **R5.1** `kind` 为 `mcpp.build-database`,`kindVersion` 为 1。`data` 含 `spec` + (`{"name": "s1", "version": "0.2.0"}` 或 `{"name": "compile-commands"}`)、 + `database`、`watch` 与 `inputs-fingerprint`。**已实现** +- **R5.2** 失败时信封不含 `data`,`diagnostics` 至少含一条 `error`,退出码为 1:不在 + 工程中为 `MCPP_BUILD_DATABASE_NO_PROJECT`,规划失败为 + `MCPP_BUILD_DATABASE_PLAN_FAILED`,工作区中其消息指出成员。工作区中任一成员规划 + 失败,整次命令失败。**已实现** +- **R5.3** 信封的 `effects` 为 `read-project` 与 `write-global-cache`,运行了构建程序时 + 另有 `exec-build-script`。**已实现** + +## 6. `watch` 与 `inputs-fingerprint` + +- **R6.1** `watch` 列出:工作区根与每个源码包的 `mcpp.toml`;`mcpp.lock`;存在时的 + `build.mcpp`;每个源码包的源文件 glob;测试发现的 glob(`[test] discover`,默认 + `tests/**/*.cpp`);构建程序声明的输入文件与 glob;`$MCPP_HOME/config.toml`。位于 + 工作区根之下的条目写成相对工作区根、以 `/` 分隔的路径或 glob;之外的条目写成绝对 + 路径,其 glob 展开为运行时匹配到的文件。**已实现** +- **R6.2** 不监视:环境变量,包括 `MCPP_TOOLCHAIN`、`MCPP_HOME` 与构建程序声明的 + 环境变量;存储中的包与 git 依赖的检出,它们的版本或提交写在已监视的清单与锁中。 + **已实现** +- **R6.3** `inputs-fingerprint` 形如 `fnv1a:<16 位十六进制>`,是 mcpp 版本、选择器与 + `watch` 在运行时匹配到的每个文件的路径与内容的摘要;这些输入不变时它不变。它与 + 构建指纹无关。**已实现** +- **R6.4** 命令不写入 `watch` 列出的任何文件,这由 §2 保证。**已实现** + +## 7. 变更记录 + +| 版本 | 日期 | 变更 | +|---|---|---| +| 1.0 | 2026-09-14 | 首版(#636)。 | diff --git a/docs/zh/01-getting-started.md b/docs/zh/01-getting-started.md index 88daf05c..4ee3852a 100644 --- a/docs/zh/01-getting-started.md +++ b/docs/zh/01-getting-started.md @@ -136,6 +136,16 @@ target 和 toolchain 解析结果。生成的 `compile_commands.json` 同时覆 workspace 中运行。插件稳定依赖进程退出码和生成的 `compile_commands.json`,标准输出仍是 面向人的文本,不作为机器协议。 +不允许写入工程目录的编辑器在标准输出上取得同一份计划 *(mcpp 2026.9.15.1+)*: + +```bash +mcpp emit build-database --format json +``` + +文档是 S1 构建数据库:每个翻译单元及其编译命令、它提供与导入的模块、工具链以及标准库 +模块单元。`--spec compile-commands` 改为输出 `compile_commands.json` 的条目。字段见 +[50 —— 机器可读输出](50-machine-output.md),规则见 [SPEC-005](../specs/build-database.md)。 + ## 增量编译与测试 [08 —— 测试](08-testing.md) 是拥有这个主题的章节;下面只是本教程需要的那一步。 diff --git a/docs/zh/50-machine-output.md b/docs/zh/50-machine-output.md index 6699fe79..cebd1171 100644 --- a/docs/zh/50-machine-output.md +++ b/docs/zh/50-machine-output.md @@ -44,6 +44,9 @@ mcpp 不支持所请求的内容。 `effects` 永远存在。空数组表示「没有」;字段缺失会表示「未知」,那是另一个断言。 +命令产出了它的文档时 `data` 存在。命令在得到文档之前就失败时省略 `data`,而不是 +给一个空对象,原因写在 `diagnostics` 里;`mcpp.build-database` 就是这样的 kind(§8)。 + ### 诊断 ```jsonc @@ -355,6 +358,37 @@ mcpp why toolchain [--target ] [--toolchain ] --format json 解析,而那可能拉取包、安装载荷、并运行某个依赖的构建程序。客户端是在**运行之前** 读这张表来决定放不放行的,漏报一项就是一句不成立的安全承诺。 +### `mcpp.build-database` —— 以构建数据库表达的构建计划 *(mcpp 2026.9.15.1+)* + +``` +mcpp emit build-database [--spec s1|compile-commands] --format json +``` + +它按 `mcpp build --configure-only` 的方式、用相同的选择器规划,不写入工程目录。 +`data` 为: + +| 字段 | | +|---|---| +| `spec` | `{"name": "s1", "version": "0.2.0"}`;使用 `--spec compile-commands` 时为 `{"name": "compile-commands"}` | +| `database` | 该规范的文档:S1 构建数据库,或 `mcpp build --configure-only` 写入 `compile_commands.json` 的条目 | +| `watch` | 变化后可能改变文档的输入:相对工作区根的路径与 glob,或绝对路径 | +| `inputs-fingerprint` | `fnv1a:<16 位十六进制>`,上述输入、mcpp 版本与选择器的摘要 | + +不带 `--format` 时命令只输出文档;`-o ` 把原本输出的内容写入 ``。文档的 +内容、不写工程目录的保证与 `watch` 的规则见 [SPEC-005](../specs/build-database.md)。 + +失败时省略 `data` 并以 1 退出:不在工程中时诊断码为 `MCPP_BUILD_DATABASE_NO_PROJECT`, +规划失败时为 `MCPP_BUILD_DATABASE_PLAN_FAILED`。警告不影响文档: + +| 诊断码 | | +|---|---| +| `MCPP_LOCK_WOULD_CHANGE` | 解析结果与工程的 `mcpp.lock` 不一致,命令不写这个文件 | +| `MCPP_GENERATED_FILE_NOT_MATERIALIZED` | 根包 `[build] generated_files` 中的某个文件缺失或内容过期,命令不写这个文件 | +| `MCPP_BUILD_DATABASE_STD_UNIT_UNDESCRIBED` | 没有标准库构建命令点名其模块源文件,该单元不被列出 | + +`--protocol-version` 为这条命令声明 `init-mcpp-home`、`read-project`、`network`、 +`write-global-cache` 与 `exec-build-script`,从不声明 `write-project`。 + ### `mcpp test --message-format json` —— 测试流 ``` diff --git a/docs/zh/README.md b/docs/zh/README.md index 95542eda..9835cfa1 100644 --- a/docs/zh/README.md +++ b/docs/zh/README.md @@ -155,3 +155,4 @@ - [SPEC-002 —— 目标侧模型:保留命名空间、五层、三条规则](../specs/target-side.md) - [SPEC-003 —— 退出码契约](../specs/exit-codes.md) - [SPEC-004 —— `mcpp.toml` 的平面划分、条件化形状、解析轴与命名规约](../specs/manifest-semantics.md) + - [SPEC-005 —— `mcpp emit build-database` 输出的构建数据库](../specs/build-database.md) diff --git a/modules/platform/src/terminal.cppm b/modules/platform/src/terminal.cppm index 0459afc3..8d5f6203 100644 --- a/modules/platform/src/terminal.cppm +++ b/modules/platform/src/terminal.cppm @@ -11,6 +11,11 @@ module; #include #include #endif +#if defined(_WIN32) +#include // _dup, _dup2, _close +#else +#include // dup, dup2, close +#endif export module mcpp.platform.terminal; @@ -25,6 +30,25 @@ bool is_tty(); // falls back to $COLUMNS, then defaults to 80. std::size_t cols(); +// EVERYTHING WRITTEN TO STANDARD OUTPUT GOES TO STANDARD ERROR UNTIL THIS IS +// DESTROYED. +// +// The redirection is made at the file descriptor, so a child process that +// inherits standard output follows it, and so does narration that prints to +// stdout without consulting any quiet flag. A command whose standard output is +// a document (`mcpp emit build-database`) plans under one of these and prints +// the document after it is gone: people still see the progress, on stderr, and +// the document arrives alone. +class StdoutToStderr { +public: + StdoutToStderr(); + ~StdoutToStderr(); + StdoutToStderr(const StdoutToStderr&) = delete; + StdoutToStderr& operator=(const StdoutToStderr&) = delete; +private: + int saved_ = -1; +}; + } // namespace mcpp::platform::terminal namespace mcpp::platform::terminal { @@ -49,4 +73,27 @@ std::size_t cols() { return 80; } +StdoutToStderr::StdoutToStderr() { + std::fflush(stdout); +#if defined(_WIN32) + saved_ = ::_dup(1); + if (saved_ >= 0) ::_dup2(2, 1); +#else + saved_ = ::dup(1); + if (saved_ >= 0) ::dup2(2, 1); +#endif +} + +StdoutToStderr::~StdoutToStderr() { + std::fflush(stdout); + if (saved_ < 0) return; +#if defined(_WIN32) + ::_dup2(saved_, 1); + ::_close(saved_); +#else + ::dup2(saved_, 1); + ::close(saved_); +#endif +} + } // namespace mcpp::platform::terminal diff --git a/src/build/build_database.cppm b/src/build/build_database.cppm new file mode 100644 index 00000000..d7fd5f75 --- /dev/null +++ b/src/build/build_database.cppm @@ -0,0 +1,556 @@ +// mcpp.build.build_database — the plan, rendered as an S1 build database. +// +// S1 ("C++ Build Database: IDE Profile", profile 0.2.0) is a profile of the +// WG21 P2977R2 build database format; docs/specs/build-database.md (SPEC-005) +// states what mcpp guarantees as its producer. The document is a projection of +// what prepare_build planned: every field is read from a BuildContext, and +// nothing in this module compiles, runs or writes anything. +// +// Three rules shape the projection, and each one is a statement about the +// engine rather than about the format: +// +// - `arguments` come from the record the compile database renders +// (mcpp.build.compile_commands::unit_invocations), so the two databases +// cannot list different command lines for one unit. +// - `visible-sets` is every other set of the planned member. The engine +// resolves imports over one graph per invocation; a narrower closure would +// describe a rule the build does not enforce. +// - The standard library modules are translation units of their own set, +// carrying the command mcpp runs to build them. That is one rule for every +// toolchain and for a package that ships its own `std.cppm`, and S1 lets +// units outrank a module manifest. + +export module mcpp.build.build_database; + +import std; +import mcpp.build.build_program; +import mcpp.build.compile_commands; +import mcpp.build.flags; +import mcpp.build.plan; +import mcpp.build.prepare; +import mcpp.home; +import mcpp.libs.json; +import mcpp.manifest; +import mcpp.modgraph.graph; +import mcpp.modgraph.scanner; +import mcpp.platform; +import mcpp.toolchain.fingerprint; +import mcpp.toolchain.model; +import mcpp.toolchain.stdmod; + +export namespace mcpp::build::database { + +inline constexpr std::string_view kProfileVersion = "0.2.0"; +inline constexpr std::string_view kStdSetName = "mcpp:std"; + +// One planned member of the document. +struct Member { + const BuildContext* ctx = nullptr; + // Prepended to every set name: empty for a single package, `/` for + // a member of a workspace document. + std::string setPrefix; + // Where the planning pass wrote (BuildOverrides::work_dir): the build + // programs' caches, and so their declared inputs, are under it. + std::filesystem::path workDir; + // Test discovery (`[test] discover`, relative to `testRoot`): a new file + // matching it is a new test target, and so a new unit in the document. + std::filesystem::path testRoot; + std::vector testDiscover; +}; + +struct Rendered { + nlohmann::json database; + // The same units as a JSON Compilation Database: the entries `mcpp build + // --configure-only` writes to compile_commands.json, from the same record. + nlohmann::json compileCommands = nlohmann::json::array(); + std::vector watch; + std::string inputsFingerprint; + // Conditions found while rendering; reported as warnings by the command. + std::vector notes; +}; + +// The S1 document for `members`, and the inputs whose change changes it. +// `workspaceRoot` anchors the relative `watch` patterns; `selector` is the +// command's own selection (target, toolchain, profile, members), which enters +// the fingerprint because the same files answer differently under another one. +Rendered render(std::span members, + const std::filesystem::path& workspaceRoot, + std::string_view selector); + +// ── Exported for tests: each is a rule SPEC-005 states. ────────────────── + +// The S1 role of a declaration form (S1 §8.2). +std::string_view role_name(mcpp::modgraph::ModuleDeclaration declaration); + +// The S1 `stdlib.name` for mcpp's standard library identifier. +std::string_view stdlib_name(std::string_view stdlibId); + +// `--`, with mcpp's family name (`llvm`, as in +// `llvm@22.1.8`) and the triple as the compiler spells it: stable within a +// document, and opaque to a consumer (S1 section 5.1). +std::string toolchain_id(const mcpp::toolchain::Toolchain& tc, + std::string_view compilerTriple); + +// Splits a command string mcpp rendered for the host shell into words, undoing +// its quoting: POSIX `sh` rules, or the Microsoft C runtime's rules on Windows. +std::vector split_command_words(std::string_view command, bool windows); + +// The working directory and argument vector of the command in `commands` whose +// words name `source`, recovered from the rendering: a leading `cd`, an `env` +// word and environment assignments, and redirections are removed. A driver +// path that the rendering left unquoted despite a space is rejoined. Empty when +// no command names the source. +struct Invocation { + std::filesystem::path workDirectory; + std::vector arguments; +}; +std::optional recover_invocation(const std::vector& commands, + const std::filesystem::path& source, + const std::filesystem::path& driver, + const std::filesystem::path& defaultDirectory, + bool windows); + +} // namespace mcpp::build::database + +namespace mcpp::build::database { + +namespace { + +std::string native_string(const std::filesystem::path& p) { + auto n = p; + n.make_preferred(); + return n.string(); +} + +std::string qualified_name(const mcpp::manifest::Manifest& m) { + return m.package.namespace_.empty() + ? m.package.name + : m.package.namespace_ + "." + m.package.name; +} + +bool is_assignment(std::string_view w) { + auto eq = w.find('='); + if (eq == std::string_view::npos || eq == 0) return false; + if (!(std::isalpha(static_cast(w[0])) || w[0] == '_')) return false; + for (std::size_t i = 1; i < eq; ++i) { + const unsigned char c = static_cast(w[i]); + if (!(std::isalnum(c) || c == '_')) return false; + } + return true; +} + +// `2>&1`, `>file`, `nul`, ``, `2>`, `<`, +// `>>`: the target is the next word. +enum class Redirect { None, Attached, Detached }; +Redirect redirect_kind(std::string_view w) { + std::size_t i = 0; + while (i < w.size() && std::isdigit(static_cast(w[i]))) ++i; + if (i >= w.size() || (w[i] != '>' && w[i] != '<')) return Redirect::None; + std::size_t j = i + 1; + if (j < w.size() && w[j] == w[i]) ++j; // `>>` + return j == w.size() ? Redirect::Detached : Redirect::Attached; +} + +bool names_source(std::string_view word, const std::filesystem::path& source, + const std::filesystem::path& cwd) { + std::filesystem::path w{std::string(word)}; + const auto want = source.lexically_normal(); + if (w.lexically_normal() == want) return true; + if (!w.is_absolute() && !cwd.empty() && (cwd / w).lexically_normal() == want) + return true; + return false; +} + +nlohmann::json toolchain_json(const mcpp::toolchain::Toolchain& tc, + std::string_view compilerTriple) { + nlohmann::json j{ + {"family", std::string(tc.compiler_name())}, + {"version", tc.version}, + {"driver", tc.binaryPath.string()}, + {"target", std::string(compilerTriple)}, + }; + if (!tc.sysroot.empty()) j["sysroot"] = native_string(tc.sysroot); + if (!tc.stdlibId.empty()) { + nlohmann::json stdlib{{"name", std::string(stdlib_name(tc.stdlibId))}}; + if (!tc.stdlibVersion.empty()) stdlib["version"] = tc.stdlibVersion; + j["stdlib"] = std::move(stdlib); + } + return j; +} + +std::string target_kind(const mcpp::manifest::Manifest& m) { + bool library = false, program = false; + for (auto const& t : m.targets) { + using K = mcpp::manifest::Target::Kind; + if (t.kind == K::Library || t.kind == K::SharedLibrary) library = true; + if (t.kind == K::Binary || t.kind == K::Application) program = true; + } + return library ? "library" : program ? "executable" : "other"; +} + +// The `watch` spelling of a path: relative to the workspace root, with `/`, when +// it is under it; absolute and native otherwise (S2 accepts no absolute glob). +std::optional relative_to(const std::filesystem::path& p, + const std::filesystem::path& root) { + auto rel = p.lexically_normal().lexically_relative(root.lexically_normal()); + if (rel.empty()) return std::nullopt; + auto first = rel.begin(); + if (first != rel.end() && *first == "..") return std::nullopt; + auto s = rel.generic_string(); + return s == "." ? std::string{} : s; +} + +struct SetData { + std::string familyName; + std::string kind; + nlohmann::json units = nlohmann::json::array(); +}; + +} // namespace + +std::string_view role_name(mcpp::modgraph::ModuleDeclaration declaration) { + using D = mcpp::modgraph::ModuleDeclaration; + switch (declaration) { + case D::None: return "non-module"; + case D::Interface: return "module-interface"; + case D::InterfacePartition: return "module-partition-interface"; + case D::ImplementationPartition: return "module-partition-implementation"; + case D::Implementation: return "module-implementation"; + case D::Unknown: return "unknown"; + } + return "unknown"; +} + +std::string_view stdlib_name(std::string_view stdlibId) { + if (stdlibId == "libstdc++") return "libstdc++"; + if (stdlibId == "libc++") return "libc++"; + if (stdlibId.starts_with("msvc")) return "msvc-stl"; + return "other"; +} + +std::string toolchain_id(const mcpp::toolchain::Toolchain& tc, + std::string_view compilerTriple) { + return std::format("{}-{}-{}", tc.compiler_family(), tc.version, compilerTriple); +} + +std::vector split_command_words(std::string_view s, bool windows) { + std::vector out; + std::string cur; + bool started = false; + auto flush = [&] { + if (started) out.push_back(std::move(cur)); + cur.clear(); + started = false; + }; + if (windows) { + bool quoted = false; + for (std::size_t i = 0; i < s.size(); ++i) { + const char c = s[i]; + if (c == '\\') { + std::size_t j = i; + while (j < s.size() && s[j] == '\\') ++j; + const std::size_t count = j - i; + started = true; + if (j < s.size() && s[j] == '"') { + cur.append(count / 2, '\\'); + if (count % 2 == 1) { cur.push_back('"'); i = j; } + else { i = j - 1; } + continue; + } + cur.append(count, '\\'); + i = j - 1; + continue; + } + if (c == '"') { quoted = !quoted; started = true; continue; } + if (!quoted && (c == ' ' || c == '\t')) { flush(); continue; } + cur.push_back(c); + started = true; + } + flush(); + return out; + } + char quote = 0; + for (std::size_t i = 0; i < s.size(); ++i) { + const char c = s[i]; + if (quote == '\'') { + if (c == '\'') quote = 0; else cur.push_back(c); + continue; + } + if (quote == '"') { + if (c == '"') { quote = 0; continue; } + if (c == '\\' && i + 1 < s.size() + && (s[i + 1] == '"' || s[i + 1] == '\\' || s[i + 1] == '$' + || s[i + 1] == '`')) { + cur.push_back(s[++i]); + continue; + } + cur.push_back(c); + continue; + } + if (c == '\'' || c == '"') { quote = c; started = true; continue; } + if (c == '\\' && i + 1 < s.size()) { cur.push_back(s[++i]); started = true; continue; } + if (c == ' ' || c == '\t' || c == '\n') { flush(); continue; } + cur.push_back(c); + started = true; + } + flush(); + return out; +} + +std::optional recover_invocation(const std::vector& commands, + const std::filesystem::path& source, + const std::filesystem::path& driver, + const std::filesystem::path& defaultDirectory, + bool windows) { + const std::string driverText = driver.string(); + for (auto const& command : commands) { + auto words = split_command_words(command, windows); + std::vector> segments(1); + for (auto& w : words) { + if (w == "&&") segments.emplace_back(); + else segments.back().push_back(std::move(w)); + } + std::filesystem::path cwd; + for (auto& seg : segments) { + if (seg.empty()) continue; + if (seg.front() == "cd") { + std::size_t k = 1; + if (k < seg.size() && (seg[k] == "/d" || seg[k] == "/D")) ++k; + if (k < seg.size()) cwd = std::filesystem::path{seg[k]}; + continue; + } + std::size_t b = 0; + if (b < seg.size() && seg[b] == "env") ++b; + while (b < seg.size() && is_assignment(seg[b])) ++b; + std::vector argv; + for (std::size_t k = b; k < seg.size(); ++k) { + switch (redirect_kind(seg[k])) { + case Redirect::Attached: continue; + case Redirect::Detached: ++k; continue; + case Redirect::None: argv.push_back(seg[k]); + } + } + if (argv.empty()) continue; + // A driver path rendered without quotes splits at its spaces. + if (argv.front() != driverText + && driverText.find(' ') != std::string::npos) { + std::string joined = argv.front(); + std::size_t k = 1; + while (k < argv.size() && joined.size() < driverText.size()) { + joined += ' '; + joined += argv[k]; + ++k; + } + if (joined == driverText) { + argv.erase(argv.begin() + 1, argv.begin() + static_cast(k)); + argv.front() = driverText; + } + } + const bool named = std::ranges::any_of(argv, [&](const std::string& w) { + return names_source(w, source, cwd.empty() ? defaultDirectory : cwd); + }); + if (!named) continue; + return Invocation{cwd.empty() ? defaultDirectory : cwd, std::move(argv)}; + } + } + return std::nullopt; +} + +Rendered render(std::span members, + const std::filesystem::path& workspaceRoot, + std::string_view selector) { + Rendered r; + const bool windows = mcpp::platform::is_windows; + nlohmann::json toolchains = nlohmann::json::object(); + nlohmann::json sets = nlohmann::json::array(); + + std::set watch; + std::set inputFiles; + + auto watch_file = [&](const std::filesystem::path& p) { + const auto normal = p.lexically_normal(); + std::error_code ec; + if (std::filesystem::is_regular_file(normal, ec)) inputFiles.insert(normal); + if (auto rel = relative_to(normal, workspaceRoot); rel && !rel->empty()) + watch.insert(*rel); + else + watch.insert(native_string(normal)); + }; + auto watch_glob = [&](const std::filesystem::path& root, std::string_view glob) { + if (glob.empty() || glob.starts_with('!')) return; + const auto matches = mcpp::modgraph::expand_glob(root, glob); + for (auto const& f : matches) inputFiles.insert(f.lexically_normal()); + if (auto rel = relative_to(root, workspaceRoot)) { + watch.insert(rel->empty() ? std::string(glob) + : *rel + "/" + std::string(glob)); + } else { + for (auto const& f : matches) watch.insert(native_string(f.lexically_normal())); + } + }; + + watch_file(workspaceRoot / "mcpp.toml"); + watch_file(mcpp::home::root() / "config.toml"); + + for (auto const& member : members) { + const auto& ctx = *member.ctx; + // The triple as the compiler spells it (`x86_64-pc-windows-msvc`), + // which is what S1 asks for; mcpp's own spelling when none was resolved. + const std::string compilerTriple = ctx.plan.targetSide.llvmTriple.empty() + ? ctx.tc.targetTriple : ctx.plan.targetSide.llvmTriple; + const auto tcId = toolchain_id(ctx.tc, compilerTriple); + if (!toolchains.contains(tcId)) + toolchains[tcId] = toolchain_json(ctx.tc, compilerTriple); + + const auto rootName = qualified_name(ctx.manifest); + std::set testSources; + for (auto const& t : ctx.manifest.targets) { + if (t.kind != mcpp::manifest::Target::TestBinary || t.main.empty()) continue; + std::filesystem::path main{t.main}; + testSources.insert((main.is_absolute() ? main : ctx.projectRoot / main) + .lexically_normal()); + } + + std::vector order; + std::map groups; + auto set_for = [&](const std::string& name, const std::string& family, + const std::string& kind) -> SetData& { + auto [it, inserted] = groups.try_emplace(name); + if (inserted) { + order.push_back(name); + it->second.familyName = family; + it->second.kind = kind; + } + return it->second; + }; + + const auto flags = mcpp::build::compute_flags(ctx.plan); + for (auto& inv : mcpp::build::unit_invocations(ctx.plan, flags)) { + const auto& cu = *inv.unit; + const bool isTest = testSources.contains(cu.source.lexically_normal()); + const std::string package = cu.packageName.empty() ? rootName : cu.packageName; + const std::string kind = isTest ? "test" + : package == rootName ? target_kind(ctx.manifest) + : "library"; + auto& set = set_for(member.setPrefix + package + (isTest ? ":test" : ""), + package, kind); + nlohmann::json provides = nlohmann::json::object(); + if (cu.providesModule) provides[*cu.providesModule] = ""; + nlohmann::json requires_ = nlohmann::json::array(); + for (auto const& name : cu.imports) requires_.push_back(name); + r.compileCommands.push_back(nlohmann::json{ + {"directory", inv.directory}, + {"file", inv.file}, + {"arguments", inv.arguments}, + {"output", inv.output}, + }); + set.units.push_back(nlohmann::json{ + {"source", std::move(inv.file)}, + {"work-directory", std::move(inv.directory)}, + {"arguments", std::move(inv.arguments)}, + {"object", std::move(inv.output)}, + {"provides", std::move(provides)}, + {"requires", std::move(requires_)}, + {"ide", {{"role", std::string(role_name(cu.declaration))}}}, + }); + } + + if (ctx.stdModule) { + const auto& sm = *ctx.stdModule; + auto add_std = [&](const std::filesystem::path& source, + const std::vector& commands, + const std::filesystem::path& object, + std::string_view module, + std::vector requires_) { + if (source.empty() || commands.empty()) return; + auto inv = recover_invocation(commands, source, ctx.tc.binaryPath, + sm.cacheDir, windows); + if (!inv) { + r.notes.push_back({"MCPP_BUILD_DATABASE_STD_UNIT_UNDESCRIBED", + std::format("no command that builds the {} module names its " + "source '{}'; the unit is not listed", + module, source.string())}); + return; + } + auto& set = set_for(member.setPrefix + std::string(kStdSetName), + std::string(kStdSetName), "library"); + set.units.push_back(nlohmann::json{ + {"source", native_string(source)}, + {"work-directory", native_string(inv->workDirectory)}, + {"arguments", std::move(inv->arguments)}, + {"object", native_string(object)}, + {"provides", {{std::string(module), ""}}}, + {"requires", std::move(requires_)}, + {"ide", {{"role", "module-interface"}}}, + }); + }; + add_std(ctx.tc.stdModuleSource, sm.stdCommands, sm.objectPath, "std", {}); + add_std(ctx.tc.stdCompatSource, sm.compatCommands, sm.compatObjectPath, + "std.compat", {"std"}); + } + + for (auto const& name : order) { + auto& set = groups.at(name); + nlohmann::json visible = nlohmann::json::array(); + for (auto const& other : order) + if (other != name) visible.push_back(other); + sets.push_back(nlohmann::json{ + {"name", name}, + {"family-name", set.familyName}, + {"visible-sets", std::move(visible)}, + {"translation-units", std::move(set.units)}, + {"ide", { + {"toolchain", tcId}, + {"configuration", ctx.profile}, + {"kind", set.kind}, + }}, + }); + } + + watch_file(ctx.projectRoot / "mcpp.lock"); + for (auto const& sp : ctx.sourcePackages) { + watch_file(sp.root / "mcpp.toml"); + std::error_code ec; + if (std::filesystem::exists(sp.root / "build.mcpp", ec)) + watch_file(sp.root / "build.mcpp"); + for (auto const& g : sp.sources) watch_glob(sp.root, g); + } + for (auto const& g : member.testDiscover) watch_glob(member.testRoot, g); + for (auto const& declared : mcpp::build::declared_program_inputs(member.workDir)) { + for (auto const& f : declared.files) watch_file(f); + for (auto const& g : declared.globs) watch_glob(declared.root, g); + } + for (auto const& note : ctx.planNotes) r.notes.push_back(note); + } + + r.database = nlohmann::json{ + {"version", 1}, + {"revision", 0}, + {"ide", { + {"profile-version", std::string(kProfileVersion)}, + {"generator", { + {"name", "mcpp"}, + {"version", std::string(mcpp::toolchain::MCPP_VERSION)}, + }}, + {"toolchains", std::move(toolchains)}, + }}, + {"sets", std::move(sets)}, + }; + + r.watch.assign(watch.begin(), watch.end()); + + std::string digest = "mcpp-build-database-inputs-v1\x1f"; + digest += mcpp::toolchain::MCPP_VERSION; + digest += '\x1f'; + digest += selector; + digest += '\n'; + for (auto const& f : inputFiles) { + digest += f.generic_string(); + digest += '\x1f'; + digest += mcpp::toolchain::hash_file(f); + digest += '\n'; + } + r.inputsFingerprint = "fnv1a:" + mcpp::toolchain::hash_string(digest); + return r; +} + +} // namespace mcpp::build::database diff --git a/src/build/build_program.cppm b/src/build/build_program.cppm index b5a20956..f14fda7c 100644 --- a/src/build/build_program.cppm +++ b/src/build/build_program.cppm @@ -384,6 +384,19 @@ std::expected run_build_program( // so a dependency's input is evaluated against the dependency's tree. bool program_inputs_stale(const std::filesystem::path& projectRoot); +// The inputs the build programs under `/target/.build-mcpp` declared +// on their last run, read from the same caches program_inputs_stale compares: +// declared files (absolute), glob patterns (relative to `root`) and environment +// variable names. The build database lists them as inputs that change the plan. +struct DeclaredProgramInputs { + std::filesystem::path root; + std::vector files; + std::vector globs; + std::vector envs; +}; +std::vector +declared_program_inputs(const std::filesystem::path& workRoot); + } // namespace mcpp::build namespace mcpp::build { @@ -1574,6 +1587,36 @@ std::expected run_build_program( return {}; } +std::vector +declared_program_inputs(const fs::path& workRoot) { + std::vector out; + std::error_code ec; + const fs::path base = workRoot / "target" / ".build-mcpp"; + if (!fs::exists(base, ec)) return out; + // The walk program_inputs_stale makes, bounded the same way. + fs::recursive_directory_iterator it( + base, fs::directory_options::skip_permission_denied, ec); + if (ec) return out; + for (; it != fs::recursive_directory_iterator(); it.increment(ec)) { + if (ec) break; + if (it.depth() >= 3) { it.disable_recursion_pending(); continue; } + if (it->path().filename() != "build.mcpp.cache") continue; + auto rec = read_cache(it->path().parent_path()); + if (!rec.loaded || rec.rootPath.empty()) continue; + DeclaredProgramInputs d; + d.root = fs::path{rec.rootPath}; + for (auto const& [h, rel] : rec.inputs) + d.files.emplace_back(abs_against_root(d.root, rel)); + for (auto const& [h, pattern] : rec.globs) d.globs.push_back(pattern); + for (auto const& [h, name] : rec.envs) d.envs.push_back(name); + out.push_back(std::move(d)); + } + std::ranges::sort(out, {}, [](const DeclaredProgramInputs& d) { + return d.root.generic_string(); + }); + return out; +} + bool program_inputs_stale(const fs::path& projectRoot) { std::error_code ec; const fs::path base = projectRoot / "target" / ".build-mcpp"; diff --git a/src/build/compile_commands.cppm b/src/build/compile_commands.cppm index 0bee6eac..e6ae9be7 100644 --- a/src/build/compile_commands.cppm +++ b/src/build/compile_commands.cppm @@ -30,6 +30,23 @@ export namespace mcpp::build { // rather than by inspection of a whole generated document. std::vector split_flags(std::string_view s); +// ONE TRANSLATION UNIT AS THE COMPILER IS INVOKED FOR IT. +// +// The compile database and the build database (mcpp.build.build_database) both +// render this record, so the `arguments` one lists for a unit cannot differ from +// the other's. NASM units have no record: no consumer of either format can +// interpret their command line. +struct UnitInvocation { + const CompileUnit* unit = nullptr; // into the plan it came from + std::string directory; + std::string file; + std::vector arguments; // driver first + std::string output; +}; + +std::vector unit_invocations(const BuildPlan& plan, + const CompileFlags& flags); + // Generate compile_commands.json content as a string. std::string emit_compile_commands(const BuildPlan& plan, const CompileFlags& flags); @@ -218,8 +235,10 @@ CompileCommandsWriteError write_error(std::string message) { } // namespace -std::string emit_compile_commands(const BuildPlan& plan, const CompileFlags& flags) { - nlohmann::json entries = nlohmann::json::array(); +std::vector unit_invocations(const BuildPlan& plan, + const CompileFlags& flags) { + std::vector out; + out.reserve(plan.compileUnits.size()); for (auto& cu : plan.compileUnits) { // NASM units carry a command line no CDB consumer (clangd, …) can @@ -235,27 +254,42 @@ std::string emit_compile_commands(const BuildPlan& plan, const CompileFlags& fla : isCSource ? flags.cc : flags.cxx; - auto output_path = native_string(plan.outputDir / cu.object); + UnitInvocation inv; + inv.unit = &cu; + inv.output = native_string(plan.outputDir / cu.object); + inv.directory = native_string(plan.projectRoot); + inv.file = native_string(cu.source); // Build arguments array. - nlohmann::json args = nlohmann::json::array(); - args.push_back(compiler.string()); + inv.arguments.push_back(compiler.string()); for (auto& f : local_include_args(cu)) - args.push_back(std::move(f)); + inv.arguments.push_back(std::move(f)); for (auto& f : split_flags(flagStr)) - args.push_back(std::move(f)); + inv.arguments.push_back(std::move(f)); for (auto& f : package_flag_args(cu, isCSource)) - args.push_back(std::move(f)); - args.push_back("-c"); - args.push_back(native_string(cu.source)); - args.push_back("-o"); - args.push_back(output_path); + inv.arguments.push_back(std::move(f)); + inv.arguments.push_back("-c"); + inv.arguments.push_back(inv.file); + inv.arguments.push_back("-o"); + inv.arguments.push_back(inv.output); + + out.push_back(std::move(inv)); + } + return out; +} + +std::string emit_compile_commands(const BuildPlan& plan, const CompileFlags& flags) { + nlohmann::json entries = nlohmann::json::array(); + + for (auto& inv : unit_invocations(plan, flags)) { + nlohmann::json args = nlohmann::json::array(); + for (auto& a : inv.arguments) args.push_back(std::move(a)); nlohmann::json entry; - entry["directory"] = native_string(plan.projectRoot); - entry["file"] = native_string(cu.source); + entry["directory"] = std::move(inv.directory); + entry["file"] = std::move(inv.file); entry["arguments"] = std::move(args); - entry["output"] = output_path; + entry["output"] = std::move(inv.output); entries.push_back(std::move(entry)); } diff --git a/src/build/plan.cppm b/src/build/plan.cppm index ee6555a2..cd42e0fa 100644 --- a/src/build/plan.cppm +++ b/src/build/plan.cppm @@ -54,6 +54,9 @@ struct CompileUnit { // Unit came from a scan_overrides declaration — plan-vs-ddi // verification is mandatory for it (ninja_backend emits --expect-*). bool scanOverridden = false; + // The module declaration form the scanner read (mcpp.modgraph.graph). + // Stated as the unit's role by the build database. + mcpp::modgraph::ModuleDeclaration declaration = mcpp::modgraph::ModuleDeclaration::None; // This unit's outputs are already in the global cache: the backend emits // `stage_file` edges from the cache instead of a compile edge (and skips // the P1689 scan for it entirely). The unit itself stays in the plan so @@ -1521,6 +1524,7 @@ make_plan(const mcpp::manifest::Manifest& manifest, } for (auto& req : u.requires_) cu.imports.push_back(req.logicalName); cu.scanOverridden = u.scanOverridden; + cu.declaration = u.declaration; plan.compileUnits.push_back(std::move(cu)); } diff --git a/src/build/prepare.cppm b/src/build/prepare.cppm index 820a6912..820867d7 100644 --- a/src/build/prepare.cppm +++ b/src/build/prepare.cppm @@ -146,9 +146,15 @@ inline void warn_unknown_xpkg_keys(const mcpp::manifest::Manifest& dm, } } +// `stale`, when given, turns the function into a comparison: nothing is +// created or written, and every declared file that is missing or differs from +// its declared content is appended. A build that describes itself rather than +// running (BuildOverrides::plan_only) reads the root package's generated files +// this way, because they live in the source tree it promises not to write. std::expected materialize_generated_files(const std::filesystem::path& root, - const mcpp::manifest::Manifest& manifest) + const mcpp::manifest::Manifest& manifest, + std::vector* stale = nullptr) { for (auto const& [relPath, content] : manifest.buildConfig.generatedFiles) { if (relPath.empty()) { @@ -176,13 +182,6 @@ materialize_generated_files(const std::filesystem::path& root, } auto out = root / relPath.lexically_normal(); - std::error_code ec; - std::filesystem::create_directories(out.parent_path(), ec); - if (ec) { - return std::unexpected(std::format( - "cannot create directory for generated file '{}': {}", - out.string(), ec.message())); - } // Skip the write when the on-disk content is already identical: ninja // is mtime-driven, and an unconditional rewrite bumps the mtime every @@ -202,7 +201,18 @@ materialize_generated_files(const std::filesystem::path& root, } } } + if (stale) { + stale->push_back(out); + continue; + } + std::error_code ec; + std::filesystem::create_directories(out.parent_path(), ec); + if (ec) { + return std::unexpected(std::format( + "cannot create directory for generated file '{}': {}", + out.string(), ec.message())); + } std::ofstream os(out, std::ios::binary); if (!os) { return std::unexpected(std::format( @@ -838,6 +848,14 @@ export std::string_view cache_mode_name(CacheMode m) { } } +// A condition a planning pass reports instead of acting on (plan_only): the +// code is stable and the message is for people. Emitted as warning diagnostics +// by the command that asked for the plan. +export struct PlanNote { + std::string code; + std::string message; +}; + export struct BuildContext { // THE PER-MACHINE JOB DEFAULT, carried so it is read once. // @@ -901,6 +919,22 @@ export struct BuildContext { std::filesystem::path outputDir; std::filesystem::path stdBmi; std::filesystem::path stdObject; + // plan_only: what the std module build WOULD be (sources are on the + // toolchain), set when the graph imports std. A build compiles it instead + // and leaves this empty. + std::optional stdModule; + // The packages this build read from an editable source tree: the root, and + // every package whose root is neither in a store nor in a hash-addressed git + // checkout (the same test depSourceRoots applies). `sources` are the + // package's source globs, relative to `root`. Read by the build database for + // the inputs it lists. + struct SourcePackage { + std::string name; + std::filesystem::path root; + std::vector sources; + }; + std::vector sourcePackages; + std::vector planNotes; mcpp::build::BuildPlan plan; // The scanned module graph. Only `mcpp pack` reads it — see the note at // the assignment for why the plan cannot answer its question. @@ -1023,6 +1057,17 @@ export struct BuildOverrides { // only some would be worse than moving none — a half-redirected build // writes into the shared root anyway, just less visibly. std::filesystem::path work_dir; + // PLANNING TO DESCRIBE, NOT TO BUILD (`mcpp emit build-database`). + // + // With `work_dir` pointed outside the project, three things still reached + // it or ran a compiler, and this switch settles each: the std module is + // described (mcpp::toolchain::describe_std_module) instead of compiled; the + // root package's `[build] generated_files`, which are sources and live in + // the source tree, are compared instead of written, and a missing or stale + // one is recorded in BuildContext::planNotes; and mcpp.lock is READ from the + // project root, as the resolution input it is, while the lock this planning + // produces is written under `work_dir`. Build programs still run. + bool plan_only = false; // #355 tool provisioning re-enters prepare_build for the tool package. A // tool package's own build.mcpp may legitimately want another tool (gRPC's // wants protoc), so the depth cannot be 1 — but an unbounded chain is a @@ -2039,6 +2084,7 @@ prepare_build(bool print_fingerprint, std::error_code wdEc; std::filesystem::create_directories(workRoot, wdEc); } + std::vector planNotes; if (m->package.sourceProvenance.empty()) { m->package.sourceProvenance = @@ -2152,7 +2198,9 @@ prepare_build(bool print_fingerprint, std::map gitLockAnchors; std::map packageIdentityLockAnchors; { - auto lockPath = workRoot / "mcpp.lock"; + // Read where the project keeps it. A planning pass that writes + // elsewhere (plan_only) still resolves against the project's lock. + auto lockPath = (overrides.plan_only ? *root : workRoot) / "mcpp.lock"; if (std::filesystem::exists(lockPath)) { if (auto lock = mcpp::pm::load(lockPath); lock) { for (auto const& p : lock->packages) { @@ -4182,9 +4230,17 @@ prepare_build(bool print_fingerprint, // build.mcpp itself. (The per-dependency call sits in the dep resolution // loop below; the root manifest needs its own.) if (!m->buildConfig.generatedFiles.empty()) { - if (auto r = materialize_generated_files(*root, *m); !r) { + std::vector staleGenerated; + if (auto r = materialize_generated_files( + *root, *m, overrides.plan_only ? &staleGenerated : nullptr); !r) { return std::unexpected(r.error()); } + for (auto const& path : staleGenerated) + planNotes.push_back({"MCPP_GENERATED_FILE_NOT_MATERIALIZED", + std::format("'{}' is declared in [build] generated_files and its " + "content on disk differs from the declaration; this " + "command does not write the project, and `mcpp build` " + "writes it", path.string())}); } // Canonical rendering of the resolved target (for the env contract). @@ -6962,7 +7018,10 @@ prepare_build(bool print_fingerprint, // /target/.mangled//__self__/ ← rewritten consumer source auto& consumerManifest = *dep_manifests[item.consumerDepIndex]; auto consumerRoot = packages[item.consumerDepIndex + 1].root; - auto stageBase = *root / "target" / ".mangled" + // Under the write root, not the source root: the stage + // is build output, and BuildOverrides::work_dir promises + // that everything the build writes moves with it. + auto stageBase = workRoot / "target" / ".mangled" / consumerManifest.package.name; auto secStage = stageBase / std::format("{}__{}", key.shortName, spec.version); @@ -11366,6 +11425,7 @@ prepare_build(bool print_fingerprint, std::filesystem::path stdObjectPath; std::filesystem::path stdCompatBmiPath; std::filesystem::path stdCompatObjectPath; + std::optional describedStdModule; if (needsStdModule) { // The std BMI must be compiled with the SAME dialect set its // importers use (issue #210: -freflection gates libstdc++'s — @@ -11380,33 +11440,52 @@ prepare_build(bool print_fingerprint, const auto stdCrt = mcpp::toolchain::msvc_crt_flag( stdDialect, mcpp::toolchain::msvc_wants_static_crt( m->buildConfig.linkage, m->buildConfig.cxxRuntime)); - auto sm = mcpp::toolchain::ensure_built( - *tc, m->package.standard, stdFlagAndDialect, - mcpp::platform::macos::deployment_target( - m->buildConfig.macosDeploymentTarget), - mcpp::toolchain::default_cache_root(), stdCrt); - if (!sm) { - // THE ONE CODE IN THE TAXONOMY THAT NOTHING WROTE. - // - // `Code::StdModulePrecompile` has existed, with a name and a - // comment, since the taxonomy was written; `grep` for it found the - // declaration and the `name()` arm and no third site. So every - // std-module refusal reported `other`, which is the bucket - // refusal.cppm defines as "a refusal that has not been given a code - // yet" -- a visible admission, and one nobody had cashed. - // - // Measured: `tests/matrix/expected.tsv` carried exactly ONE `other` - // row out of 176, `x86_64-windows-msvc x llvm@22.1.8` in graph mode, - // and `scan.sh` printed it under "无名拒绝" on every Windows run. - // The sentence was right and the classification was missing -- - // the same shape `Code::HostToolToolchain` was added for. - refusal::record(refusal::Code::StdModulePrecompile); - return std::unexpected(sm.error().message); - } - stdBmiPath = sm->bmiPath; - stdObjectPath = sm->objectPath; - stdCompatBmiPath = sm->compatBmiPath; - stdCompatObjectPath = sm->compatObjectPath; + if (overrides.plan_only) { + // Described, not compiled: the paths and commands are the ones + // ensure_built would use, from the one derivation in stdmod.cppm. + auto described = mcpp::toolchain::describe_std_module( + *tc, m->package.standard, stdFlagAndDialect, + mcpp::platform::macos::deployment_target( + m->buildConfig.macosDeploymentTarget), + mcpp::toolchain::default_cache_root(), stdCrt); + if (!described) { + refusal::record(refusal::Code::StdModulePrecompile); + return std::unexpected(described.error().message); + } + stdBmiPath = described->bmiPath; + stdObjectPath = described->objectPath; + stdCompatBmiPath = described->compatBmiPath; + stdCompatObjectPath = described->compatObjectPath; + describedStdModule = std::move(*described); + } else { + auto sm = mcpp::toolchain::ensure_built( + *tc, m->package.standard, stdFlagAndDialect, + mcpp::platform::macos::deployment_target( + m->buildConfig.macosDeploymentTarget), + mcpp::toolchain::default_cache_root(), stdCrt); + if (!sm) { + // THE ONE CODE IN THE TAXONOMY THAT NOTHING WROTE. + // + // `Code::StdModulePrecompile` has existed, with a name and a + // comment, since the taxonomy was written; `grep` for it found the + // declaration and the `name()` arm and no third site. So every + // std-module refusal reported `other`, which is the bucket + // refusal.cppm defines as "a refusal that has not been given a code + // yet" -- a visible admission, and one nobody had cashed. + // + // Measured: `tests/matrix/expected.tsv` carried exactly ONE `other` + // row out of 176, `x86_64-windows-msvc x llvm@22.1.8` in graph mode, + // and `scan.sh` printed it under "无名拒绝" on every Windows run. + // The sentence was right and the classification was missing -- + // the same shape `Code::HostToolToolchain` was added for. + refusal::record(refusal::Code::StdModulePrecompile); + return std::unexpected(sm.error().message); + } + stdBmiPath = sm->bmiPath; + stdObjectPath = sm->objectPath; + stdCompatBmiPath = sm->compatBmiPath; + stdCompatObjectPath = sm->compatObjectPath; + } } if (print_fingerprint) { @@ -11435,6 +11514,7 @@ prepare_build(bool print_fingerprint, ctx.outputDir = target_dir(*tc, fp, workRoot); ctx.stdBmi = stdBmiPath; ctx.stdObject = stdObjectPath; + ctx.stdModule = std::move(describedStdModule); // Every directory a package payload may legitimately have been INSTALLED // into. There is more than one: the global registry, plus the two // project-local data roots a custom git index installs into @@ -11467,12 +11547,28 @@ prepare_build(bool print_fingerprint, std::vector owned = storeRoots; owned.push_back(mcpp::home::root()); std::vector roots; - for (std::size_t i = 1; i < packages.size(); ++i) { + // The same enumeration answers a second reader: which packages were + // read from an editable tree, with their source globs (the build + // database lists them as the inputs that change the plan). + auto qualified = [](const mcpp::manifest::Manifest& pm) { + return pm.package.namespace_.empty() + ? pm.package.name + : pm.package.namespace_ + "." + pm.package.name; + }; + for (std::size_t i = 0; i < packages.size(); ++i) { const auto& pkgRoot = packages[i].root; if (pkgRoot.empty()) continue; - if (mcpp::build::path_is_under_any(pkgRoot, owned)) continue; + if (i > 0 && mcpp::build::path_is_under_any(pkgRoot, owned)) continue; auto normalized = pkgRoot.lexically_normal(); - if (normalized == root->lexically_normal()) continue; + const bool known = std::ranges::any_of(ctx.sourcePackages, + [&](const BuildContext::SourcePackage& sp) { + return sp.root.lexically_normal() == normalized; + }); + if (!known) + ctx.sourcePackages.push_back({qualified(packages[i].manifest), + normalized, + packages[i].manifest.modules.sources}); + if (i == 0 || normalized == root->lexically_normal()) continue; if (std::find(roots.begin(), roots.end(), normalized) == roots.end()) roots.push_back(std::move(normalized)); } @@ -13318,6 +13414,7 @@ prepare_build(bool print_fingerprint, lu.targetName)); } + ctx.planNotes = std::move(planNotes); return ctx; } diff --git a/src/cli.cppm b/src/cli.cppm index b057a359..915b59dc 100644 --- a/src/cli.cppm +++ b/src/cli.cppm @@ -629,7 +629,7 @@ int run(int argc, char** argv) { // ─── emit (one nested subcommand: xpkg) ──────────────────────── .subcommand(cl::App("emit") - .description("Generate a document describing this project (xpkg, sbom)") + .description("Generate a document describing this project (xpkg, sbom, build-database)") .subcommand(cl::App("xpkg") .description("Generate xpkg Lua entry") .option(cl::Option("version").short_name('V').takes_value().value_name("VER") @@ -648,9 +648,45 @@ int run(int argc, char** argv) { .description("Write a CycloneDX bill of materials for the recorded resolution") .option(cl::Option("output").short_name('o').takes_value().value_name("FILE") .help("Write to file instead of stdout"))) + // The plan as an S1 build database, planned as `build + // --configure-only` plans it and written into nothing; the build + // selectors are `build`'s own (docs/specs/build-database.md). + .subcommand(cl::App("build-database") + .description("Print the build plan as an S1 build database, without writing into the project") + .option(cl::Option("spec").takes_value().value_name("NAME") + .help("Document specification: s1 (default; the S1 IDE profile of P2977R2) | compile-commands")) + .option(cl::Option("format").takes_value().value_name("json") + .help("Machine-readable output (enveloped; see docs/50-machine-output.md)")) + .option(cl::Option("output").short_name('o').takes_value().value_name("FILE") + .help("Write to file instead of stdout")) + .option(cl::Option("target").takes_value().value_name("TRIPLE") + .help("Describe the build for (same axis as `mcpp build --target`)")) + .option(cl::Option("toolchain").takes_value().value_name("SPEC") + .help("Describe the build with this toolchain, e.g. llvm@22.1.8")) + .option(cl::Option("accel").takes_value().value_name("SPEC") + .help("Device backends and architectures (same axis as `mcpp build --accel`)")) + .option(cl::Option("no-accel") + .help("Describe the variant built for no accelerator")) + .option(cl::Option("static").help("Describe the build with --static")) + .option(cl::Option("package").short_name('p').takes_value().value_name("NAME") + .help("Describe only the named workspace member")) + .option(cl::Option("profile").takes_value().value_name("NAME") + .help("Build profile: dev (default) | release | dist | <[profile.*] name>")) + .option(cl::Option("release").help("Shorthand for --profile release")) + .option(cl::Option("dev").help("Shorthand for --profile dev")) + .option(cl::Option("features").takes_value().value_name("LIST") + .help("Activate root-package features (comma-separated)")) + .option(cl::Option("cap").takes_value().value_name("LIST") + .help("Pin capability providers (e.g. blas=openblas,lapack=mkl)")) + .option(cl::Option("strict") + .help("Treat manifest schema warnings (unknown feature/platform) as errors")) + .option(cl::Option("workspace") + .help("Describe all workspace members in one document"))) .action(wrap_rc([&dispatch_sub](const cl::ParsedArgs& p) { return dispatch_sub("emit", p, {{"xpkg", cmd_emit_xpkg}, - {"sbom", mcpp::cli::cmd_sbom}}); + {"sbom", mcpp::cli::cmd_sbom}, + {"build-database", + mcpp::cli::cmd_emit_build_database}}); }))) // ─── xpkg (descriptor tooling: parse) ────────────────────────── @@ -1027,6 +1063,13 @@ int run(int argc, char** argv) { {"why toolchain", {Effect::InitMcppHome, Effect::ReadProject, Effect::Network, Effect::WriteGlobalCache, Effect::ExecBuildScript}}, + // The same resolution as `why toolchain` and as `build + // --configure-only`, and therefore the same declaration, with one + // difference that is the command's reason to exist: no + // `write-project`. Planning writes under the mcpp home. + {"emit build-database", {Effect::InitMcppHome, Effect::ReadProject, + Effect::Network, Effect::WriteGlobalCache, + Effect::ExecBuildScript}}, }; }; diff --git a/src/cli/cmd_build.cppm b/src/cli/cmd_build.cppm index 69d018ab..78f68854 100644 --- a/src/cli/cmd_build.cppm +++ b/src/cli/cmd_build.cppm @@ -19,12 +19,20 @@ import mcpp.build.coff_exports; import mcpp.build.stage; import mcpp.build.schedule.detach_codegen; import mcpp.build.test_targets; +import mcpp.build.build_database; +import mcpp.build.build_program; import mcpp.dyndep; +import mcpp.home; import mcpp.hooks; +import mcpp.libs.json; import mcpp.log; +import mcpp.platform; +import mcpp.platform.terminal; import mcpp.project; import mcpp.manifest; +import mcpp.toolchain.fingerprint; import mcpp.ui; +import mcpp.wire; namespace mcpp::cli { @@ -83,12 +91,11 @@ int run_build_with_hooks(mcpp::build::BuildContext& ctx, bool verbose, return rc != 0 ? rc : ((spanOk && hookOk) ? 0 : 1); } -export int cmd_build(const mcpplibs::cmdline::ParsedArgs& parsed) { - bool verbose = parsed.is_flag_set("verbose") || mcpp::log::is_verbose(); - bool print_fp = parsed.is_flag_set("print-fingerprint"); - bool no_cache = parsed.is_flag_set("no-cache"); - bool configure_only = parsed.is_flag_set("configure-only"); - +// The build selectors, read once for every command that plans a build: `mcpp +// build` and `mcpp emit build-database` select the same plan from the same +// flags, so the database describes the build the same flags would run. +mcpp::build::BuildOverrides overrides_from_selectors( + const mcpplibs::cmdline::ParsedArgs& parsed) { mcpp::build::BuildOverrides ov; if (auto t = parsed.value("target")) ov.target_triple = *t; // --accel / --no-accel stand to `[build] accel` exactly as --target stands @@ -99,11 +106,6 @@ export int cmd_build(const mcpplibs::cmdline::ParsedArgs& parsed) { if (parsed.is_flag_set("no-accel")) ov.accel = "(none)"; else if (auto a = parsed.value("accel")) ov.accel = *a; if (auto p = parsed.value("package")) ov.package_filter = *p; - // --cache global|local|off. --no-cache is the deprecated alias for off; the - // old flag only ever cleared target/, which says nothing about a cache, so - // it is expressed in terms of the new one rather than kept as a second axis. - if (auto c = parsed.value("cache")) ov.cache_mode = *c; - else if (no_cache) ov.cache_mode = "off"; // Profile selection precedence: --profile NAME > --release / --dev > the // project default ([build].default-profile) > "release", resolved in // prepare_build. --release/--dev are shorthands only. @@ -114,6 +116,21 @@ export int cmd_build(const mcpplibs::cmdline::ParsedArgs& parsed) { if (auto cp = parsed.value("cap")) ov.capabilities = *cp; ov.strict = parsed.is_flag_set("strict"); ov.force_static = parsed.is_flag_set("static"); + return ov; +} + +export int cmd_build(const mcpplibs::cmdline::ParsedArgs& parsed) { + bool verbose = parsed.is_flag_set("verbose") || mcpp::log::is_verbose(); + bool print_fp = parsed.is_flag_set("print-fingerprint"); + bool no_cache = parsed.is_flag_set("no-cache"); + bool configure_only = parsed.is_flag_set("configure-only"); + + mcpp::build::BuildOverrides ov = overrides_from_selectors(parsed); + // --cache global|local|off. --no-cache is the deprecated alias for off; the + // old flag only ever cleared target/, which says nothing about a cache, so + // it is expressed in terms of the new one rather than kept as a second axis. + if (auto c = parsed.value("cache")) ov.cache_mode = *c; + else if (no_cache) ov.cache_mode = "off"; // Fan-out prefixes every diagnostic with the member it came from; the // single-package path has nothing to disambiguate and passes "". @@ -207,6 +224,222 @@ export int cmd_build(const mcpplibs::cmdline::ParsedArgs& parsed) { return run_build_with_hooks(*ctx, verbose, no_cache, ov.target_triple); } +// ─── `mcpp emit build-database` ───────────────────────────────────────────── +// +// The plan `mcpp build --configure-only` computes, printed as an S1 build +// database instead of being configured: nothing is written into the project +// (docs/specs/build-database.md, SPEC-005). Planning writes under a work +// directory in the mcpp home, and the std module is described, not compiled. +namespace { + +// One work directory per project root and member, under the mcpp home: the +// planning pass writes its lock, its build programs' artifacts and its output +// directory there, never inside the project. +std::filesystem::path build_database_work_dir(const std::filesystem::path& root, + std::string_view member) { + return mcpp::home::root() / "cache" / "build-database" + / mcpp::toolchain::hash_string(root.lexically_normal().generic_string() + + "\x1f" + std::string(member)); +} + +std::optional read_whole_file(const std::filesystem::path& p) { + std::ifstream is(p, std::ios::binary); + if (!is) return std::nullopt; + return std::string((std::istreambuf_iterator(is)), + std::istreambuf_iterator()); +} + +} // namespace + +export int cmd_emit_build_database(const mcpplibs::cmdline::ParsedArgs& parsed) { + using mcpp::wire::Diagnostic; + using mcpp::wire::Effect; + using mcpp::wire::Severity; + + const auto formatValue = parsed.value("format"); + if (formatValue && !mcpp::wire::parse_format(*formatValue)) { + std::println(stderr, "error: {}", mcpp::wire::unsupported_format(*formatValue)); + return 2; + } + const bool envelope = formatValue.has_value(); + // Which specification the document follows. The plan is one; the document + // is a rendering of it, named by its specification rather than by any + // consumer. `s1` is the default. + const std::string spec = parsed.value("spec").value_or("s1"); + if (spec != "s1" && spec != "compile-commands") { + std::println(stderr, "error: unsupported --spec '{}'; expected: s1, compile-commands", + spec); + return 2; + } + const auto outputPath = parsed.value("output"); + const mcpp::build::BuildOverrides ov = overrides_from_selectors(parsed); + + std::vector diagnostics; + auto publish = [&](const std::string& text) -> int { + if (!outputPath) { std::print("{}", text); return 0; } + const std::filesystem::path out{*outputPath}; + auto tmp = out; + tmp += std::format(".tmp-{}", std::chrono::steady_clock::now() + .time_since_epoch().count()); + { + std::ofstream os(tmp, std::ios::binary); + os << text; + if (!os) { + std::println(stderr, "error: cannot write '{}'", tmp.string()); + return 1; + } + } + std::error_code ec; + if (!mcpp::platform::fs::replace_file(tmp, out, ec)) { + std::filesystem::remove(tmp, ec); + std::println(stderr, "error: cannot replace '{}'", out.string()); + return 1; + } + return 0; + }; + // A failure is one envelope with diagnostics and no `data` (S2 0.2.0 §3.4: + // a command without data has failed), and exit 1. + auto failed = [&](std::string code, std::string message) -> int { + diagnostics.push_back({std::move(code), Severity::Error, std::move(message)}); + if (!envelope) { + for (auto const& d : diagnostics) + std::println(stderr, "{}: {}", + mcpp::wire::severity_name(d.severity), d.message); + return 1; + } + const auto text = mcpp::wire::to_json(mcpp::wire::Envelope{ + .kind = "mcpp.build-database", + .effects = {Effect::ReadProject}, + .data = nullptr, + .diagnostics = diagnostics, + }).dump(2) + "\n"; + (void)publish(text); + return 1; + }; + + auto root = mcpp::project::find_manifest_root(std::filesystem::current_path()); + if (!root) + return failed("MCPP_BUILD_DATABASE_NO_PROJECT", + "no mcpp.toml found in current directory or any parent"); + + std::vector> requests; + if (auto members = workspace_fanout_members(parsed.is_flag_set("workspace"), + ov.package_filter)) { + for (auto const& mp : *members) { + auto mo = ov; + mo.package_filter = mp; + requests.emplace_back(mp, std::move(mo)); + } + } else { + requests.emplace_back(std::string{}, ov); + } + + std::vector contexts; + std::vector workDirs; + std::vector prefixes; + std::vector>> testDiscovery; + std::optional planError; + { + // Planning narrates on stdout and may start programs that inherit it; + // the document is printed after this scope, alone. + mcpp::platform::terminal::StdoutToStderr narration; + for (auto& [member, mo] : requests) { + auto discovered = mcpp::build::discover_test_targets(*root, mo.package_filter); + if (!discovered) { + planError = member.empty() ? discovered.error() + : std::format("{}: {}", member, discovered.error()); + break; + } + // As `--configure-only`: tests and dev-dependencies are part of the + // surface an editor needs. + const bool includeDevDeps = !discovered->targets.empty(); + auto discovery = std::pair{discovered->packageRoot, discovered->discover}; + mo.plan_only = true; + mo.work_dir = build_database_work_dir(*root, mo.package_filter); + std::error_code ec; + std::filesystem::remove(mo.work_dir / "mcpp.lock", ec); + auto ctx = mcpp::build::prepare_build(/*print_fingerprint=*/false, + includeDevDeps, + std::move(discovered->targets), mo); + if (!ctx) { + planError = member.empty() ? ctx.error() + : std::format("{}: {}", member, ctx.error()); + break; + } + contexts.push_back(std::move(*ctx)); + workDirs.push_back(mo.work_dir); + prefixes.push_back(member.empty() ? std::string{} : member + "/"); + testDiscovery.push_back(std::move(discovery)); + } + } + if (planError) return failed("MCPP_BUILD_DATABASE_PLAN_FAILED", *planError); + + // The lock this planning produced, against the project's. The project's is + // never written; a difference is reported. + for (std::size_t i = 0; i < contexts.size(); ++i) { + const auto now = read_whole_file(workDirs[i] / "mcpp.lock"); + if (!now) continue; + const auto projectLock = contexts[i].projectRoot / "mcpp.lock"; + const auto was = read_whole_file(projectLock); + if (!was) + diagnostics.push_back({"MCPP_LOCK_WOULD_CHANGE", Severity::Warning, + std::format("'{}' does not exist; `mcpp build` would create it", + projectLock.string())}); + else if (*was != *now) + diagnostics.push_back({"MCPP_LOCK_WOULD_CHANGE", Severity::Warning, + std::format("this resolution differs from '{}'; `mcpp build` " + "would update it", projectLock.string())}); + } + + std::vector members; + bool ranBuildPrograms = false; + for (std::size_t i = 0; i < contexts.size(); ++i) { + members.push_back({&contexts[i], prefixes[i], workDirs[i], + testDiscovery[i].first, testDiscovery[i].second}); + if (!mcpp::build::declared_program_inputs(workDirs[i]).empty()) + ranBuildPrograms = true; + for (auto const& sp : contexts[i].sourcePackages) { + std::error_code ec; + if (std::filesystem::exists(sp.root / "build.mcpp", ec)) ranBuildPrograms = true; + } + } + const auto selector = std::format( + "spec={}\x1ftarget={}\x1ftoolchain={}\x1fprofile={}\x1ffeatures={}\x1f" + "cap={}\x1faccel={}\x1fstatic={}\x1fpackage={}\x1fworkspace={}", + spec, ov.target_triple, mcpp::platform::env::get("MCPP_TOOLCHAIN").value_or(""), + ov.profile, ov.features, ov.capabilities, ov.accel, ov.force_static, + ov.package_filter, parsed.is_flag_set("workspace")); + auto rendered = mcpp::build::database::render(members, *root, selector); + for (auto& note : rendered.notes) + diagnostics.push_back({std::move(note.code), Severity::Warning, + std::move(note.message)}); + + auto document = spec == "s1" ? std::move(rendered.database) + : std::move(rendered.compileCommands); + if (!envelope) { + for (auto const& d : diagnostics) + std::println(stderr, "{}: {}", mcpp::wire::severity_name(d.severity), + d.message); + return publish(document.dump(2) + "\n"); + } + std::vector effects{Effect::ReadProject, Effect::WriteGlobalCache}; + if (ranBuildPrograms) effects.push_back(Effect::ExecBuildScript); + nlohmann::json specJson{{"name", spec}}; + if (spec == "s1") + specJson["version"] = std::string(mcpp::build::database::kProfileVersion); + return publish(mcpp::wire::to_json(mcpp::wire::Envelope{ + .kind = "mcpp.build-database", + .effects = std::move(effects), + .data = nlohmann::json{ + {"spec", std::move(specJson)}, + {"database", std::move(document)}, + {"watch", std::move(rendered.watch)}, + {"inputs-fingerprint", std::move(rendered.inputsFingerprint)}, + }, + .diagnostics = diagnostics, + }).dump(2) + "\n"); +} + export int cmd_run(const mcpplibs::cmdline::ParsedArgs& parsed, std::span passthrough) { // The action lambda has already split argv at the first "--" and diff --git a/src/modgraph/graph.cppm b/src/modgraph/graph.cppm index a315ce4d..01899c1c 100644 --- a/src/modgraph/graph.cppm +++ b/src/modgraph/graph.cppm @@ -13,6 +13,32 @@ struct ModuleId { auto operator<=>(const ModuleId&) const = default; }; +// THE MODULE DECLARATION A UNIT CARRIES, AS THE SCANNER READ IT. +// +// `provides`, `providesInterface` and `requires_` describe the module GRAPH, and +// one declaration form is not recoverable from them: `module M;` (an +// implementation unit of M) is recorded as `requires M`, exactly as a unit that +// only says `import M;`. The form is what a build database states as a unit's +// role, and a reader has to be told it rather than guess it (S1 profile 0.2.0 +// §8.2: a producer that cannot decide writes `unknown`). +// +// None no module declaration +// Interface `export module M;` +// InterfacePartition `export module M:P;` +// ImplementationPartition `module M:P;` +// Implementation `module M;` +// Unknown the declaration was not read: a `scan_overrides` +// entry, or a P1689 record that cannot tell an +// implementation unit from an importer +enum class ModuleDeclaration { + None, + Interface, + InterfacePartition, + ImplementationPartition, + Implementation, + Unknown, +}; + struct SourceUnit { std::filesystem::path path; // mcpp#233: path relative to this unit's PACKAGE ROOT (not the primary @@ -61,6 +87,9 @@ struct SourceUnit { // to prevent. Unknown now warns, naming the file and the reason. std::optional providesInterface; std::vector requires_; + // The declaration form (see ModuleDeclaration). Read by the build database + // renderer (mcpp.build.build_database), which states it as the unit's role. + ModuleDeclaration declaration = ModuleDeclaration::None; // The unit's ROLE, decided once by the scanner from the owning package's // extension table and carried from here on. Every downstream consumer // (planner, backend, compile_commands, the asm dialect check) reads this diff --git a/src/modgraph/p1689.cppm b/src/modgraph/p1689.cppm index 7d7aa5af..4e6e7e55 100644 --- a/src/modgraph/p1689.cppm +++ b/src/modgraph/p1689.cppm @@ -417,6 +417,21 @@ scan_file(const std::filesystem::path& source, for (auto& r : rule->requires_) { u.requires_.push_back(ModuleId{ r }); } + // The declaration form, only as far as the record states it. A provided + // module with `is-interface` names its form; a unit that requires modules + // and provides none is either `module M;` or an importer, and P1689 does not + // say which; a unit with neither has no module declaration. + if (u.provides && u.providesInterface) { + const bool partition = + u.provides->logicalName.find(':') != std::string::npos; + u.declaration = *u.providesInterface + ? (partition ? ModuleDeclaration::InterfacePartition + : ModuleDeclaration::Interface) + : (partition ? ModuleDeclaration::ImplementationPartition + : ModuleDeclaration::Unknown); + } else if (u.provides || !u.requires_.empty()) { + u.declaration = ModuleDeclaration::Unknown; + } return u; } diff --git a/src/modgraph/scanner.cppm b/src/modgraph/scanner.cppm index 5014465b..b7f772d4 100644 --- a/src/modgraph/scanner.cppm +++ b/src/modgraph/scanner.cppm @@ -916,6 +916,9 @@ std::expected scan_file(const std::filesystem::path& file } u.provides = ModuleId{name}; u.providesInterface = true; // read from the keyword, not assumed + u.declaration = name.find(':') != std::string::npos + ? ModuleDeclaration::InterfacePartition + : ModuleDeclaration::Interface; } else { // A non-exporting `module …;` is TWO different declarations // wearing one spelling, and they were treated as one: @@ -947,8 +950,10 @@ std::expected scan_file(const std::filesystem::path& file } u.provides = ModuleId{name}; u.providesInterface = false; + u.declaration = ModuleDeclaration::ImplementationPartition; } else if (!u.provides) { u.requires_.push_back(ModuleId{name}); + u.declaration = ModuleDeclaration::Implementation; } } // The module this TU belongs to, for resolving `import :part;` @@ -1166,6 +1171,8 @@ void scan_one_into(ScanResult& result, u.relPath = std::filesystem::relative(f, root); u.packageName = qualifiedName; u.scanOverridden = true; + // The override names modules; it does not carry the declaration. + u.declaration = ModuleDeclaration::Unknown; // A declared unit still gets its role from the same classifier — // scan_overrides overrides what was SCANNED, not what the file is. u.kind = mcpp::classify(f, extTable); diff --git a/src/toolchain/stdmod.cppm b/src/toolchain/stdmod.cppm index a8b26377..3927e3f7 100644 --- a/src/toolchain/stdmod.cppm +++ b/src/toolchain/stdmod.cppm @@ -54,6 +54,23 @@ struct StdModule { struct StdModError { std::string message; }; +// WHAT ensure_built WOULD BUILD, WITHOUT BUILDING IT. +// +// The cache directory, the artifact paths and the build commands are all +// derived before any compiler runs; deriving them creates no directory and +// starts no process. A caller that describes a build rather than performing it +// (the build database) reads this, and ensure_built reads the same derivation, +// so the two cannot name different directories or commands. +struct StdModuleDescription { + std::filesystem::path cacheDir; + std::filesystem::path bmiPath; + std::filesystem::path objectPath; + std::filesystem::path compatBmiPath; // empty without std.compat + std::filesystem::path compatObjectPath; // empty without std.compat + std::vector stdCommands; // as run, in order + std::vector compatCommands; // as run, in order +}; + std::filesystem::path default_cache_root(); // Build std module if not already cached. Returns paths to BMI + object. @@ -75,6 +92,15 @@ std::expected ensure_built( // CRT models cannot share a cache directory. std::string_view msvc_crt_flag = {}); +// The derivation ensure_built builds from, with the same parameters. +std::expected describe_std_module( + const Toolchain& tc, + std::string_view cpp_standard, + std::string_view cpp_standard_flag, + std::string_view macos_deployment_target = {}, + const std::filesystem::path& cache_root = default_cache_root(), + std::string_view msvc_crt_flag = {}); + } // namespace mcpp::toolchain namespace mcpp::toolchain { @@ -218,7 +244,14 @@ std::filesystem::path default_cache_root() { return mcpp::home::cache_root(); } -std::expected ensure_built( +namespace { + +struct StdDerivation { + StdModuleDescription description; + nlohmann::json metadata; +}; + +std::expected derive_std_module( const Toolchain& tc, std::string_view cpp_standard, std::string_view cpp_standard_flag, @@ -232,7 +265,6 @@ std::expected ensure_built( } const bool isMsvc = tc.compiler == CompilerId::MSVC; - StdModule sm; // Build sysroot + include flags for std module precompilation, derived // from the shared toolchain link model (same resolver as flags.cppm — @@ -321,14 +353,63 @@ std::expected ensure_built( const auto stdRoot = cache_root / "std"; auto normalized = derive(stdRoot / kStdKeyPlaceholder).metadata; auto identity = std_identity_key(normalized); - sm.cacheDir = stdRoot / identity; - - auto derived = derive(sm.cacheDir); - sm.bmiPath = derived.bmiPath; - sm.objectPath = derived.objectPath; - const auto& stdCommands = derived.stdCommands; - const auto& compatCommands = derived.compatCommands; - const auto& metadata = derived.metadata; + + StdDerivation out; + auto& desc = out.description; + desc.cacheDir = stdRoot / identity; + auto derived = derive(desc.cacheDir); + desc.bmiPath = derived.bmiPath; + desc.objectPath = derived.objectPath; + desc.stdCommands = std::move(derived.stdCommands); + desc.compatCommands = std::move(derived.compatCommands); + if (!desc.compatCommands.empty()) { + desc.compatBmiPath = isMsvc + ? mcpp::toolchain::msvc::std_compat_bmi_path(desc.cacheDir) + : mcpp::toolchain::clang::std_compat_bmi_path(desc.cacheDir); + desc.compatObjectPath = + desc.cacheDir / (isMsvc ? "std.compat.obj" : "std.compat.o"); + } + out.metadata = std::move(derived.metadata); + return out; +} + +} // namespace + +std::expected describe_std_module( + const Toolchain& tc, + std::string_view cpp_standard, + std::string_view cpp_standard_flag, + std::string_view macos_deployment_target, + const std::filesystem::path& cache_root, + std::string_view msvc_crt_flag) +{ + auto d = derive_std_module(tc, cpp_standard, cpp_standard_flag, + macos_deployment_target, cache_root, msvc_crt_flag); + if (!d) return std::unexpected(d.error()); + return std::move(d->description); +} + +std::expected ensure_built( + const Toolchain& tc, + std::string_view cpp_standard, + std::string_view cpp_standard_flag, + std::string_view macos_deployment_target, + const std::filesystem::path& cache_root, + std::string_view msvc_crt_flag) +{ + auto derivation = derive_std_module(tc, cpp_standard, cpp_standard_flag, + macos_deployment_target, cache_root, + msvc_crt_flag); + if (!derivation) return std::unexpected(derivation.error()); + const auto& desc = derivation->description; + + StdModule sm; + sm.cacheDir = desc.cacheDir; + sm.bmiPath = desc.bmiPath; + sm.objectPath = desc.objectPath; + const auto& stdCommands = desc.stdCommands; + const auto& compatCommands = desc.compatCommands; + const auto& metadata = derivation->metadata; auto metaPath = metadata_path(sm.cacheDir); bool std_cached = std::filesystem::exists(sm.bmiPath) && std::filesystem::exists(sm.objectPath) @@ -354,17 +435,15 @@ std::expected ensure_built( // Build std.compat after std (std.compat imports std; Clang + MSVC). if (!compatCommands.empty()) { - auto compatBmi = isMsvc - ? mcpp::toolchain::msvc::std_compat_bmi_path(sm.cacheDir) - : mcpp::toolchain::clang::std_compat_bmi_path(sm.cacheDir); + const auto& compatBmi = desc.compatBmiPath; if (rebuiltStd || !std::filesystem::exists(compatBmi) || !metadata_matches(metaPath, metadata)) { if (auto out = run_commands(compatCommands, tc); !out) { return std::unexpected(out.error()); } } - sm.compatBmiPath = compatBmi; - sm.compatObjectPath = sm.cacheDir / (isMsvc ? "std.compat.obj" : "std.compat.o"); + sm.compatBmiPath = desc.compatBmiPath; + sm.compatObjectPath = desc.compatObjectPath; } if (auto r = write_metadata(metaPath, metadata); !r) { diff --git a/src/wire.cppm b/src/wire.cppm index 0669e351..df94a93d 100644 --- a/src/wire.cppm +++ b/src/wire.cppm @@ -67,7 +67,7 @@ inline constexpr int kEnvelopeVersion = 1; // from what comes back. struct KindVersion { std::string_view kind; int version; }; -inline constexpr std::array kKinds{{ +inline constexpr std::array kKinds{{ {"mcpp.env", 1}, {"mcpp.xpkg", 1}, {"mcpp.cache", 1}, @@ -79,6 +79,11 @@ inline constexpr std::array kKinds{{ // `mcpp toolchain list --format json`: which toolchains are installed and // which target rows this host serves, with their status. {"mcpp.toolchain.list", 1}, + // `mcpp emit build-database --format json`: the plan as an S1 build + // database (`data.database`), the inputs whose change changes it + // (`data.watch`) and their digest (`data.inputs-fingerprint`). Written + // into nothing; see docs/specs/build-database.md. + {"mcpp.build-database", 1}, }}; // What running a command does, beyond writing to stdout. @@ -144,6 +149,9 @@ struct Diagnostic { struct Envelope { std::string_view kind; std::vector effects; + // A null `data` is omitted from the envelope. That is how a command whose + // failure leaves nothing to describe says so: its envelope carries only + // diagnostics (docs/50 §2). nlohmann::json data = nlohmann::json::object(); std::vector diagnostics; }; @@ -177,7 +185,7 @@ inline nlohmann::json to_json(const Envelope& e) { nlohmann::json diags = nlohmann::json::array(); for (auto const& d : e.diagnostics) diags.push_back(to_json(d)); - return nlohmann::json{ + nlohmann::json out{ {"schemaVersion", kEnvelopeVersion}, {"kind", std::string(e.kind)}, {"kindVersion", kind_version(e.kind)}, @@ -186,9 +194,10 @@ inline nlohmann::json to_json(const Envelope& e) { {"version", std::string(mcpp::MCPP_VERSION)}, {"protocol", {{"min", kEnvelopeVersion}, {"max", kEnvelopeVersion}}}, }}, - {"data", e.data}, - {"diagnostics", std::move(diags)}, }; + if (!e.data.is_null()) out["data"] = e.data; + out["diagnostics"] = std::move(diags); + return out; } // Serialise and write to stdout. Two spaces, trailing newline: a client reads diff --git a/tests/e2e/688_emit_build_database.sh b/tests/e2e/688_emit_build_database.sh new file mode 100755 index 00000000..b608cdd4 --- /dev/null +++ b/tests/e2e/688_emit_build_database.sh @@ -0,0 +1,250 @@ +#!/usr/bin/env bash +# requires: python3 +# 688 -- `mcpp emit build-database` prints the plan as an S1 build database and +# writes nothing into the project (docs/specs/build-database.md, SPEC-005). +# +# The project has every module declaration form, a test, a path +# dev-dependency and `import std`. Criteria: +# A. `--format json`: kind `mcpp.build-database`, kindVersion 1, `data.spec` +# names S1 0.2.0, effects without `write-project`, and `--protocol-version` +# declares the command without it. +# B. The project tree is byte-identical before and after. The control leg at +# the end runs `build --configure-only` on the same tree and sees it change, +# which shows the measurement can see a write. +# C. The document validates against the vendored S1 schema. +# D. Each unit's role is its declaration form; sets are per package plus +# `:test` and `mcpp:std`, and each set sees every other set. +# E. The `mcpp:std` unit names an existing source that provides `std`. +# F. `--spec compile-commands` lists the same arguments as the S1 units, and, +# once the output directory is mapped, the arguments `build +# --configure-only` writes to compile_commands.json. +# G. Usage errors write nothing to stdout and exit 2; outside a project the +# envelope has no `data` and exits 1. +# H. The bare document goes to stdout, or to `-o ` with stdout empty. +# I. `watch` names the manifests, the lock, the source and test globs; the +# fingerprint is stable and follows a source edit. +set -e + +TMP=$(mktemp -d) # the measured tree: the project and its dev-dependency +OUT=$(mktemp -d) # everything this script writes, outside the measured tree +cleanup() { + if [ -n "${WORK_DIR:-}" ] && [ -d "$WORK_DIR" ]; then rm -rf "$WORK_DIR"; fi + rm -rf "$TMP" "$OUT" +} +trap cleanup EXIT + +fail() { echo "FAIL: $1"; shift; for f in "$@"; do echo "--- $f ---"; cat "$f" 2>/dev/null; done; exit 1; } +HERE="$(cd "$(dirname "$0")" && pwd)" +SCHEMA="$HERE/fixtures/s1/s1-build-database.schema.json" +VALIDATE="$HERE/_json_schema_subset.py" +PY=python3 + +mkdir -p "$TMP/devkit/include" "$TMP/hello/src" "$TMP/hello/tests" +cat > "$TMP/devkit/mcpp.toml" <<'EOF' +[package] +name = "devkit" +version = "0.1.0" + +[build] +include_dirs = ["include"] +EOF +echo '#define DEVKIT_MARKER 1' > "$TMP/devkit/include/devkit.hpp" + +cd "$TMP/hello" +cat > mcpp.toml <<'EOF' +[package] +name = "hello" +version = "0.1.0" +standard = "c++23" + +[dev-dependencies] +devkit = { path = "../devkit" } +EOF +printf 'export module hello.greet;\nexport import :detail;\nimport std;\nexport std::string greet();\n' > src/greet.cppm +printf 'export module hello.greet:detail;\nexport int answer() { return 42; }\n' > src/detail.cppm +printf 'module hello.greet:impl;\nint hidden() { return 7; }\n' > src/impl.cppm +printf 'module hello.greet;\nimport :impl;\nstd::string greet() { return "hi"; }\n' > src/greet_impl.cpp +printf 'import hello.greet;\nimport std;\nint main( { std::println("{}", greet()); }\n' > src/main.cpp +printf '#include \nint main() { return DEVKIT_MARKER - 1; }\n' > tests/test_smoke.cpp + +tree_digest() { + "$PY" - "$TMP" <<'EOF' +import hashlib, os, sys +root = sys.argv[1] +h = hashlib.sha256() +for dirpath, dirnames, filenames in os.walk(root): + dirnames.sort() + rel = os.path.relpath(dirpath, root) + h.update(("D " + rel + "\n").encode()) + for name in sorted(filenames): + with open(os.path.join(dirpath, name), "rb") as f: + h.update(("F " + os.path.join(rel, name) + " ").encode() + hashlib.sha256(f.read()).hexdigest().encode() + b"\n") +print(h.hexdigest()) +EOF +} + +before=$(tree_digest) + +# ── A ────────────────────────────────────────────────────────────────────── +"$MCPP" emit build-database --format json > "$OUT/env.json" 2> "$OUT/env.err" \ + || fail "A: emit build-database exited non-zero" "$OUT/env.err" "$OUT/env.json" +after=$(tree_digest) + +"$PY" - "$OUT/env.json" <<'EOF' || fail "A: the envelope" "$OUT/env.json" +import json, sys +e = json.load(open(sys.argv[1])) +assert e["kind"] == "mcpp.build-database", e["kind"] +assert e["kindVersion"] == 1, e["kindVersion"] +assert "write-project" not in e["effects"], e["effects"] +assert "read-project" in e["effects"], e["effects"] +d = e["data"] +assert d["spec"] == {"name": "s1", "version": "0.2.0"}, d["spec"] +assert d["database"]["ide"]["profile-version"] == "0.2.0" +assert d["inputs-fingerprint"].startswith("fnv1a:"), d["inputs-fingerprint"] +EOF +"$MCPP" --protocol-version > "$OUT/proto.json" +"$PY" - "$OUT/proto.json" <<'EOF' || fail "A: the protocol document" "$OUT/proto.json" +import json, sys +p = json.load(open(sys.argv[1])) +assert p["kinds"]["mcpp.build-database"] == 1 +fx = p["commands"]["emit build-database"]["effects"] +assert "write-project" not in fx and "exec-build-script" in fx, fx +EOF +echo "ok: A, the envelope and the declared effects" + +# ── B ────────────────────────────────────────────────────────────────────── +[ "$before" = "$after" ] || fail "B: the project tree changed" +echo "ok: B, the project tree is unchanged" + +# ── C ────────────────────────────────────────────────────────────────────── +"$PY" -c 'import json,sys; json.dump(json.load(open(sys.argv[1]))["data"]["database"], open(sys.argv[2], "w"))' \ + "$OUT/env.json" "$OUT/db.json" +"$PY" "$VALIDATE" "$SCHEMA" "$OUT/db.json" > "$OUT/validate.out" \ + || fail "C: the document does not validate against S1 0.2.0" "$OUT/validate.out" +echo "ok: C, the document validates against the S1 schema" + +# ── D, E ─────────────────────────────────────────────────────────────────── +"$PY" - "$OUT/env.json" <<'EOF' || fail "D/E: sets and roles" "$OUT/env.json" +import json, os, sys +db = json.load(open(sys.argv[1]))["data"]["database"] +sets = {s["name"]: s for s in db["sets"]} +assert set(sets) == {"hello", "hello:test", "mcpp:std"}, sorted(sets) +for name, s in sets.items(): + assert sorted(s["visible-sets"]) == sorted(n for n in sets if n != name), (name, s["visible-sets"]) + assert s["ide"]["toolchain"] in db["ide"]["toolchains"], s["ide"] +want = { + "greet.cppm": "module-interface", + "detail.cppm": "module-partition-interface", + "impl.cppm": "module-partition-implementation", + "greet_impl.cpp": "module-implementation", + "main.cpp": "non-module", +} +roles = {os.path.basename(u["source"]): u["ide"]["role"] for u in sets["hello"]["translation-units"]} +assert roles == want, roles +assert sets["hello"]["ide"]["kind"] == "executable", sets["hello"]["ide"] +tests = [os.path.basename(u["source"]) for u in sets["hello:test"]["translation-units"]] +assert tests == ["test_smoke.cpp"], tests +assert sets["hello:test"]["ide"]["kind"] == "test" +greet = next(u for u in sets["hello"]["translation-units"] if u["source"].endswith("greet.cppm")) +assert greet["provides"] == {"hello.greet": ""}, greet["provides"] +assert greet["requires"] == ["hello.greet:detail", "std"], greet["requires"] +std = [u for u in sets["mcpp:std"]["translation-units"] if "std" in u["provides"]] +assert len(std) == 1, sets["mcpp:std"] +assert os.path.isfile(std[0]["source"]), std[0]["source"] +driver = next(iter(db["ide"]["toolchains"].values()))["driver"] +assert std[0]["arguments"][0] == driver, (std[0]["arguments"][0], driver) +assert std[0]["source"] in std[0]["arguments"] or any(a.endswith(os.path.basename(std[0]["source"])) for a in std[0]["arguments"]), std[0]["arguments"] +EOF +echo "ok: D, roles and sets; E, the std unit" + +# ── F (the S1 and compile-commands renderings agree) ────────────────────── +"$MCPP" emit build-database --spec compile-commands > "$OUT/cc.json" 2> "$OUT/cc.err" \ + || fail "F: --spec compile-commands exited non-zero" "$OUT/cc.err" +"$PY" - "$OUT/env.json" "$OUT/cc.json" <<'EOF' || fail "F: the two renderings disagree" "$OUT/cc.json" +import json, sys +db = json.load(open(sys.argv[1]))["data"]["database"] +cc = json.load(open(sys.argv[2])) +s1 = {u["source"]: u["arguments"] for s in db["sets"] if s["name"] != "mcpp:std" for u in s["translation-units"]} +cdb = {e["file"]: e["arguments"] for e in cc} +assert s1 == cdb, (sorted(s1), sorted(cdb)) +EOF +echo "ok: F, the S1 units and the compile-commands entries carry the same arguments" + +# ── G ────────────────────────────────────────────────────────────────────── +set +e +"$MCPP" emit build-database --format ndjson > "$OUT/g1.out" 2> "$OUT/g1.err"; rc1=$? +"$MCPP" emit build-database --spec nope > "$OUT/g2.out" 2> "$OUT/g2.err"; rc2=$? +mkdir -p "$OUT/empty" && (cd "$OUT/empty" && "$MCPP" emit build-database --format json > "$OUT/g3.out" 2> "$OUT/g3.err"); rc3=$? +set -e +[ "$rc1" = 2 ] && [ ! -s "$OUT/g1.out" ] || fail "G: --format ndjson rc=$rc1" "$OUT/g1.out" "$OUT/g1.err" +[ "$rc2" = 2 ] && [ ! -s "$OUT/g2.out" ] || fail "G: --spec nope rc=$rc2" "$OUT/g2.out" "$OUT/g2.err" +[ "$rc3" = 1 ] || fail "G: outside a project rc=$rc3" "$OUT/g3.out" "$OUT/g3.err" +"$PY" - "$OUT/g3.out" <<'EOF' || fail "G: the failure envelope" "$OUT/g3.out" +import json, sys +e = json.load(open(sys.argv[1])) +assert "data" not in e, e +assert e["diagnostics"][0]["code"] == "MCPP_BUILD_DATABASE_NO_PROJECT", e["diagnostics"] +assert e["diagnostics"][0]["severity"] == "error" +EOF +echo "ok: G, usage errors and the failure envelope" + +# ── H ────────────────────────────────────────────────────────────────────── +"$MCPP" emit build-database > "$OUT/bare.json" 2> "$OUT/bare.err" || fail "H: bare document" "$OUT/bare.err" +"$MCPP" emit build-database -o "$OUT/file.json" > "$OUT/o.out" 2> "$OUT/o.err" || fail "H: -o" "$OUT/o.err" +[ ! -s "$OUT/o.out" ] || fail "H: -o also wrote to stdout" "$OUT/o.out" +"$PY" - "$OUT/bare.json" "$OUT/file.json" <<'EOF' || fail "H: bare and -o documents" "$OUT/bare.json" "$OUT/file.json" +import json, sys +a = json.load(open(sys.argv[1])); b = json.load(open(sys.argv[2])) +assert a["version"] == 1 and "sets" in a and a == b +EOF +echo "ok: H, the bare document and -o" + +# ── I ────────────────────────────────────────────────────────────────────── +"$MCPP" emit build-database --format json > "$OUT/env2.json" 2> /dev/null || fail "I: second run" +printf '\n// edited\n' >> src/detail.cppm +"$MCPP" emit build-database --format json > "$OUT/env3.json" 2> /dev/null || fail "I: after an edit" +"$PY" - "$OUT/env.json" "$OUT/env2.json" "$OUT/env3.json" <<'EOF' || fail "I: watch and fingerprint" "$OUT/env.json" +import json, sys +e1, e2, e3 = (json.load(open(p))["data"] for p in sys.argv[1:4]) +w = e1["watch"] +for entry in ("mcpp.toml", "mcpp.lock", "tests/**/*.cpp"): + assert entry in w, (entry, w) +assert any(x.startswith("src/**/") for x in w), w +assert any(x.replace("\\", "/").endswith("devkit/mcpp.toml") for x in w), w +assert e1["inputs-fingerprint"] == e2["inputs-fingerprint"], "the fingerprint is not stable" +assert e1["inputs-fingerprint"] != e3["inputs-fingerprint"], "the fingerprint did not follow an edit" +EOF +echo "ok: I, watch and the inputs fingerprint" + +# The work directory the planning used, for cleanup: the object paths are under +# it, and it is not inside the project. +WORK_DIR=$("$PY" -c ' +import json, sys +db = json.load(open(sys.argv[1]))["data"]["database"] +obj = next(u["object"] for s in db["sets"] if s["name"] == "hello" for u in s["translation-units"]) +norm = obj.replace("\\", "/") +print(obj[: norm.index("/target/")]) +' "$OUT/env.json") + +# ── F (against configure-only) and the control leg of B ─────────────────── +"$MCPP" build --configure-only > "$OUT/conf.out" 2>&1 || fail "F: configure-only" "$OUT/conf.out" +[ "$(tree_digest)" != "$before" ] || fail "B control: configure-only did not change the tree" +[ -f compile_commands.json ] || fail "F: configure-only wrote no compile_commands.json" "$OUT/conf.out" +"$PY" - "$OUT/cc.json" compile_commands.json <<'EOF' || fail "F: compile-commands differs from configure-only" compile_commands.json +import json, re, sys +emitted = json.load(open(sys.argv[1])) +written = json.load(open(sys.argv[2])) +# The one difference by construction is where the build writes: the planning +# pass writes under its work directory, configure-only under the project. +def write_root(entry): + return re.split(r"[\\/]target[\\/]", entry["output"])[0] +work, project = write_root(emitted[0]), write_root(written[0]) +def mapped(args): + return [a.replace(work, project) for a in args] +e = {x["file"]: mapped(x["arguments"]) for x in emitted} +w = {x["file"]: x["arguments"] for x in written} +assert set(e) <= set(w), (sorted(e), sorted(w)) +for f in e: + assert e[f] == w[f], (f, e[f], w[f]) +EOF +echo "ok: F, the arguments are configure-only's; B control, configure-only writes the project" diff --git a/tests/e2e/_json_schema_subset.py b/tests/e2e/_json_schema_subset.py new file mode 100644 index 00000000..94c2d786 --- /dev/null +++ b/tests/e2e/_json_schema_subset.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Validate a JSON document against a JSON Schema (draft 2020-12 subset). + +Usage: _json_schema_subset.py + +Exit status: 0 when the document is valid, 1 when it is not (each violation is +printed with its JSON path), 3 when the schema uses a keyword this validator +does not implement. The last case is deliberate: a keyword that is silently +skipped is a rule that is silently not checked. +""" +import json +import re +import sys + +ANNOTATIONS = {"$schema", "$id", "title", "description", "$comment", "examples", "default"} +IMPLEMENTED = { + "$defs", "$ref", "type", "enum", "const", "required", "properties", + "additionalProperties", "items", "minItems", "uniqueItems", "minLength", + "minimum", "pattern", "oneOf", "not", "propertyNames", +} + + +def keywords(node, found): + if isinstance(node, dict): + for key, value in node.items(): + found.add(key) + if key in ("properties", "$defs"): + for sub in value.values(): + keywords(sub, found) + elif key in ("enum", "const", "required", "examples", "default"): + continue + else: + keywords(value, found) + elif isinstance(node, list): + for item in node: + keywords(item, found) + + +def type_ok(value, name): + if name == "object": + return isinstance(value, dict) + if name == "array": + return isinstance(value, list) + if name == "string": + return isinstance(value, str) + if name == "integer": + return isinstance(value, int) and not isinstance(value, bool) + if name == "number": + return isinstance(value, (int, float)) and not isinstance(value, bool) + if name == "boolean": + return isinstance(value, bool) + if name == "null": + return value is None + raise ValueError(f"unknown type {name}") + + +class Validator: + def __init__(self, root): + self.root = root + + def resolve(self, ref): + if not ref.startswith("#/"): + raise ValueError(f"only local references are supported: {ref}") + node = self.root + for part in ref[2:].split("/"): + node = node[part] + return node + + def check(self, schema, value, path, errors): + if schema is True: + return + if schema is False: + errors.append(f"{path}: not allowed") + return + if "$ref" in schema: + self.check(self.resolve(schema["$ref"]), value, path, errors) + if "type" in schema: + names = schema["type"] if isinstance(schema["type"], list) else [schema["type"]] + if not any(type_ok(value, n) for n in names): + errors.append(f"{path}: expected {names}, got {type(value).__name__}") + return + if "const" in schema and value != schema["const"]: + errors.append(f"{path}: expected {schema['const']!r}") + if "enum" in schema and value not in schema["enum"]: + errors.append(f"{path}: {value!r} is not one of {schema['enum']}") + if isinstance(value, str): + if "minLength" in schema and len(value) < schema["minLength"]: + errors.append(f"{path}: shorter than {schema['minLength']}") + if "pattern" in schema and not re.search(schema["pattern"], value): + errors.append(f"{path}: does not match {schema['pattern']}") + if isinstance(value, (int, float)) and not isinstance(value, bool): + if "minimum" in schema and value < schema["minimum"]: + errors.append(f"{path}: less than {schema['minimum']}") + if isinstance(value, list): + if "minItems" in schema and len(value) < schema["minItems"]: + errors.append(f"{path}: fewer than {schema['minItems']} items") + if schema.get("uniqueItems"): + seen = [json.dumps(v, sort_keys=True) for v in value] + if len(seen) != len(set(seen)): + errors.append(f"{path}: items are not unique") + if "items" in schema: + for i, item in enumerate(value): + self.check(schema["items"], item, f"{path}[{i}]", errors) + if isinstance(value, dict): + for key in schema.get("required", []): + if key not in value: + errors.append(f"{path}: missing required '{key}'") + props = schema.get("properties", {}) + for key, item in value.items(): + if key in props: + self.check(props[key], item, f"{path}.{key}", errors) + elif "additionalProperties" in schema: + self.check(schema["additionalProperties"], item, f"{path}.{key}", errors) + if "propertyNames" in schema: + self.check(schema["propertyNames"], key, f"{path}.<{key}>", errors) + if "oneOf" in schema: + matched = 0 + for alternative in schema["oneOf"]: + sub = [] + self.check(alternative, value, path, sub) + if not sub: + matched += 1 + if matched != 1: + errors.append(f"{path}: matches {matched} of the oneOf alternatives, not exactly one") + if "not" in schema: + sub = [] + self.check(schema["not"], value, path, sub) + if not sub: + errors.append(f"{path}: matches a schema it must not match") + + +def main(): + if len(sys.argv) != 3: + print(__doc__, file=sys.stderr) + return 2 + with open(sys.argv[1], encoding="utf-8") as f: + schema = json.load(f) + with open(sys.argv[2], encoding="utf-8") as f: + document = json.load(f) + found = set() + keywords(schema, found) + unknown = sorted(found - IMPLEMENTED - ANNOTATIONS) + if unknown: + print(f"the schema uses keywords this validator does not implement: {unknown}") + return 3 + errors = [] + Validator(schema).check(schema, document, "$", errors) + for e in errors: + print(e) + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/e2e/fixtures/s1/README.md b/tests/e2e/fixtures/s1/README.md new file mode 100644 index 00000000..cdbeb455 --- /dev/null +++ b/tests/e2e/fixtures/s1/README.md @@ -0,0 +1,19 @@ +# S1 build database schema (vendored) + +`s1-build-database.schema.json` is the JSON Schema of S1, "C++ Build Database: +IDE Profile", profile version 0.2.0, copied without modification from +`specs/schema/s1-build-database.schema.json` of +https://github.com/Sunrisepeak/lsp-mcpp-private at commit +`b82859d993f21746c73334e1c9da980c92dc6a3f` (the file is unchanged since commit +`28ecd6e3bf440884a63bddaf8d57d2aeb8d64b75`). The specification and its schema are +licensed under the Apache License 2.0. + +The file is vendored so that mcpp's tests validate what `mcpp emit +build-database` prints against a fixed version of the specification, without +running or reading anything else from that repository. A new profile version +replaces the file and this record together. + +`tests/e2e/_json_schema_subset.py` validates a document against it. The validator +implements the keywords this schema uses and refuses any other, so a schema that +starts using a keyword the validator does not implement fails instead of +passing unchecked. diff --git a/tests/e2e/fixtures/s1/s1-build-database.schema.json b/tests/e2e/fixtures/s1/s1-build-database.schema.json new file mode 100644 index 00000000..eefdf7c2 --- /dev/null +++ b/tests/e2e/fixtures/s1/s1-build-database.schema.json @@ -0,0 +1,427 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/mcpp-community/lsp-mcpp/specs/schema/s1-build-database/0.2.0", + "title": "C++ Build Database: IDE Profile 0.2.0 (S1)", + "description": "A P2977R2-compatible build database with the IDE profile fields of S1. Unknown fields are permitted everywhere; consumers ignore them.", + "type": "object", + "required": [ + "version", + "revision", + "sets" + ], + "properties": { + "version": { + "const": 1 + }, + "revision": { + "type": "integer", + "minimum": 0 + }, + "ide": { + "$ref": "#/$defs/documentIde" + }, + "sets": { + "type": "array", + "items": { + "$ref": "#/$defs/set" + } + } + }, + "$defs": { + "stringArray": { + "type": "array", + "items": { + "type": "string" + } + }, + "nonEmptyString": { + "type": "string", + "minLength": 1 + }, + "extensions": { + "type": "object", + "description": "Vendor extensions keyed by reverse domain name or registered short name." + }, + "documentIde": { + "type": "object", + "required": [ + "profile-version", + "toolchains" + ], + "properties": { + "profile-version": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(-[0-9A-Za-z.-]+)?(\\+[0-9A-Za-z.-]+)?$" + }, + "generator": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/$defs/nonEmptyString" + }, + "version": { + "type": "string" + } + } + }, + "toolchains": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/toolchain" + } + }, + "extensions": { + "$ref": "#/$defs/extensions" + } + } + }, + "toolchain": { + "type": "object", + "required": [ + "family", + "version", + "driver", + "target" + ], + "properties": { + "family": { + "enum": [ + "gcc", + "clang", + "msvc", + "clang-cl", + "other" + ] + }, + "version": { + "$ref": "#/$defs/nonEmptyString" + }, + "build-id": { + "type": "string" + }, + "driver": { + "$ref": "#/$defs/nonEmptyString" + }, + "target": { + "$ref": "#/$defs/nonEmptyString" + }, + "sysroot": { + "type": "string" + }, + "stdlib": { + "$ref": "#/$defs/stdlib" + }, + "config-files": { + "$ref": "#/$defs/stringArray" + }, + "introspection": { + "type": "array", + "items": { + "type": "object", + "required": [ + "command" + ], + "properties": { + "command": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "output": { + "type": "string" + } + } + } + } + } + }, + "stdlib": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "enum": [ + "libstdc++", + "libc++", + "msvc-stl", + "other" + ] + }, + "version": { + "type": "string" + }, + "module-metadata": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "set": { + "type": "object", + "required": [ + "name", + "visible-sets", + "translation-units" + ], + "properties": { + "name": { + "$ref": "#/$defs/nonEmptyString" + }, + "family-name": { + "type": "string" + }, + "visible-sets": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmptyString" + }, + "uniqueItems": true + }, + "baseline-arguments": { + "$ref": "#/$defs/stringArray" + }, + "translation-units": { + "type": "array", + "items": { + "$ref": "#/$defs/translationUnit" + } + }, + "ide": { + "$ref": "#/$defs/setIde" + } + } + }, + "setIde": { + "type": "object", + "required": [ + "toolchain" + ], + "properties": { + "toolchain": { + "$ref": "#/$defs/nonEmptyString" + }, + "configuration": { + "type": "string" + }, + "kind": { + "enum": [ + "library", + "executable", + "test", + "other" + ] + }, + "options": { + "$ref": "#/$defs/semanticOptions" + }, + "module-metadata": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmptyString" + } + }, + "extensions": { + "$ref": "#/$defs/extensions" + } + } + }, + "translationUnit": { + "type": "object", + "required": [ + "source", + "work-directory", + "arguments" + ], + "properties": { + "source": { + "$ref": "#/$defs/nonEmptyString" + }, + "work-directory": { + "$ref": "#/$defs/nonEmptyString" + }, + "arguments": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "local-arguments": { + "$ref": "#/$defs/stringArray" + }, + "object": { + "type": "string" + }, + "private": { + "type": "boolean" + }, + "provides": { + "type": "object", + "propertyNames": { + "$ref": "#/$defs/moduleName" + }, + "additionalProperties": { + "type": "string" + } + }, + "requires": { + "type": "array", + "items": { + "$ref": "#/$defs/moduleName" + }, + "uniqueItems": true + }, + "ide": { + "$ref": "#/$defs/unitIde" + } + } + }, + "moduleName": { + "type": "string", + "description": "A module name M or a partition name M:P, where M and P are dot-separated identifiers. Identifier characters are not restricted to ASCII.", + "pattern": "^[^\\s:.]+(\\.[^\\s:.]+)*(:[^\\s:.]+(\\.[^\\s:.]+)*)?$" + }, + "unitIde": { + "type": "object", + "required": [ + "role" + ], + "properties": { + "role": { + "$ref": "#/$defs/role" + }, + "options": { + "$ref": "#/$defs/semanticOptions" + }, + "extensions": { + "$ref": "#/$defs/extensions" + } + } + }, + "role": { + "enum": [ + "module-interface", + "module-partition-interface", + "module-partition-implementation", + "module-implementation", + "non-module", + "unknown", + "header-unit" + ] + }, + "macro": { + "oneOf": [ + { + "type": "object", + "required": [ + "define" + ], + "not": { + "required": [ + "undefine" + ] + }, + "properties": { + "define": { + "$ref": "#/$defs/nonEmptyString" + }, + "value": { + "type": [ + "string", + "null" + ] + } + } + }, + { + "type": "object", + "required": [ + "undefine" + ], + "not": { + "required": [ + "define" + ] + }, + "properties": { + "undefine": { + "$ref": "#/$defs/nonEmptyString" + } + } + } + ] + }, + "includeDirectories": { + "type": "object", + "properties": { + "user": { + "$ref": "#/$defs/stringArray" + }, + "quote": { + "$ref": "#/$defs/stringArray" + }, + "system": { + "$ref": "#/$defs/stringArray" + }, + "after": { + "$ref": "#/$defs/stringArray" + } + } + }, + "semanticOptions": { + "type": "object", + "properties": { + "language-standard": { + "type": "string", + "pattern": "^c\\+\\+[0-9a-z]+$" + }, + "language-extensions": { + "enum": [ + "none", + "gnu", + "ms" + ] + }, + "macros": { + "type": "array", + "items": { + "$ref": "#/$defs/macro" + } + }, + "include-directories": { + "$ref": "#/$defs/includeDirectories" + }, + "forced-includes": { + "$ref": "#/$defs/stringArray" + }, + "exceptions": { + "type": "boolean" + }, + "rtti": { + "type": "boolean" + }, + "raw-semantic-arguments": { + "type": "object", + "propertyNames": { + "enum": [ + "gcc", + "clang", + "msvc", + "clang-cl", + "other" + ] + }, + "additionalProperties": { + "$ref": "#/$defs/stringArray" + } + } + } + } + } +} diff --git a/tests/unit/test_build_database.cpp b/tests/unit/test_build_database.cpp new file mode 100644 index 00000000..9c9f38be --- /dev/null +++ b/tests/unit/test_build_database.cpp @@ -0,0 +1,156 @@ +// The rules docs/specs/build-database.md (SPEC-005) states that can be pinned +// without planning a project: the role of each declaration form, the standard +// library names, the toolchain id, and the recovery of the std units' argument +// vectors from the commands mcpp renders. The recovery tests drive the REAL +// command builders, so a builder that changes shape fails here rather than in +// a document an editor reads. + +#include + +import std; +import mcpp.build.build_database; +import mcpp.modgraph.graph; +import mcpp.platform; +import mcpp.toolchain.clang; +import mcpp.toolchain.gcc; +import mcpp.toolchain.model; + +namespace db = mcpp::build::database; +using mcpp::modgraph::ModuleDeclaration; +using namespace mcpp::toolchain; + +namespace { + +const bool kWindows = mcpp::platform::is_windows; + +std::filesystem::path host_path(std::string_view posix, std::string_view windows) { + return std::filesystem::path{std::string(kWindows ? windows : posix)}; +} + +bool contains(const std::vector& words, const std::string& w) { + return std::ranges::find(words, w) != words.end(); +} + +} // namespace + +TEST(BuildDatabase, RoleOfEachDeclarationForm) { + EXPECT_EQ(db::role_name(ModuleDeclaration::None), "non-module"); + EXPECT_EQ(db::role_name(ModuleDeclaration::Interface), "module-interface"); + EXPECT_EQ(db::role_name(ModuleDeclaration::InterfacePartition), + "module-partition-interface"); + EXPECT_EQ(db::role_name(ModuleDeclaration::ImplementationPartition), + "module-partition-implementation"); + EXPECT_EQ(db::role_name(ModuleDeclaration::Implementation), "module-implementation"); + EXPECT_EQ(db::role_name(ModuleDeclaration::Unknown), "unknown"); +} + +TEST(BuildDatabase, StandardLibraryNames) { + EXPECT_EQ(db::stdlib_name("libstdc++"), "libstdc++"); + EXPECT_EQ(db::stdlib_name("libc++"), "libc++"); + EXPECT_EQ(db::stdlib_name("msvc-stl"), "msvc-stl"); + EXPECT_EQ(db::stdlib_name("newlib"), "other"); +} + +TEST(BuildDatabase, ToolchainIdNamesFamilyVersionAndTheCompilersTriple) { + Toolchain tc; + tc.compiler = CompilerId::Clang; + tc.version = "22.1.8"; + tc.targetTriple = "x86_64-windows-msvc"; + // mcpp's family name in the opaque id (`llvm`, as in `llvm@22.1.8`), and the + // triple as the compiler spells it; the S1 `family` field says `clang`. + EXPECT_EQ(db::toolchain_id(tc, "x86_64-pc-windows-msvc"), + "llvm-22.1.8-x86_64-pc-windows-msvc"); + EXPECT_EQ(tc.compiler_name(), "clang"); +} + +TEST(BuildDatabase, PosixWordsUndoShellQuoting) { + auto w = db::split_command_words( + "cd '/c d' && env LD_LIBRARY_PATH='/r t' 'it'\\''s' \"a\\\"b\" x\\ y 2>&1", + /*windows=*/false); + std::vector want{"cd", "/c d", "&&", "env", "LD_LIBRARY_PATH=/r t", + "it's", "a\"b", "x y", "2>&1"}; + EXPECT_EQ(w, want); +} + +TEST(BuildDatabase, WindowsWordsFollowTheRuntimeRules) { + auto w = db::split_command_words( + "cd /d \"C:\\a b\" && \"C:\\t\\clang++.exe\" \"x\\\\\\\"y\" C:\\s\\std.ixx 2>&1", + /*windows=*/true); + std::vector want{"cd", "/d", "C:\\a b", "&&", "C:\\t\\clang++.exe", + "x\\\"y", "C:\\s\\std.ixx", "2>&1"}; + EXPECT_EQ(w, want); +} + +TEST(BuildDatabase, RecoversTheGccStdUnitFromItsBuilder) { + Toolchain tc; + tc.compiler = CompilerId::GCC; + tc.version = "16.1.0"; + tc.binaryPath = host_path("/opt/my tools/bin/g++", "C:\\my tools\\bin\\g++.exe"); + tc.targetTriple = "x86_64-linux-gnu"; + tc.stdModuleSource = host_path("/opt/my tools/include/c++/16.1.0/bits/std.cc", + "C:\\my tools\\include\\c++\\16.1.0\\bits\\std.cc"); + tc.compilerRuntimeDirs = { host_path("/opt/rt lib", "C:\\rt") }; + const auto cache = host_path("/home/u/.mcpp/cache/std/k1", "C:\\u\\.mcpp\\std\\k1"); + + auto commands = gcc::std_module_build_commands(tc, cache, "", "-std=c++23"); + ASSERT_FALSE(commands.empty()); + auto inv = db::recover_invocation(commands, tc.stdModuleSource, tc.binaryPath, + "/unused", kWindows); + ASSERT_TRUE(inv.has_value()) << commands.front(); + EXPECT_EQ(inv->workDirectory, cache) << commands.front(); + ASSERT_FALSE(inv->arguments.empty()); + EXPECT_EQ(inv->arguments.front(), tc.binaryPath.string()) << commands.front(); + EXPECT_TRUE(contains(inv->arguments, tc.stdModuleSource.string())) << commands.front(); + EXPECT_TRUE(contains(inv->arguments, "-std=c++23")) << commands.front(); + EXPECT_FALSE(contains(inv->arguments, "2>&1")) << commands.front(); + EXPECT_FALSE(contains(inv->arguments, "env")) << commands.front(); + for (auto const& a : inv->arguments) + EXPECT_FALSE(a.starts_with("LD_LIBRARY_PATH=")) << a; +} + +TEST(BuildDatabase, RecoversTheClangStdUnitsFromTheirBuilders) { + Toolchain tc; + tc.compiler = CompilerId::Clang; + tc.version = "22.1.8"; + tc.binaryPath = host_path("/opt/llvm/bin/clang++", "C:\\llvm\\bin\\clang++.exe"); + tc.targetTriple = "x86_64-linux-gnu"; + tc.stdModuleSource = host_path("/opt/llvm/share/libc++/v1/std.cppm", + "C:\\llvm\\share\\libc++\\v1\\std.cppm"); + tc.stdCompatSource = host_path("/opt/llvm/share/libc++/v1/std.compat.cppm", + "C:\\llvm\\share\\libc++\\v1\\std.compat.cppm"); + const auto cache = host_path("/home/u/.mcpp/cache/std/k2", "C:\\u\\.mcpp\\std\\k2"); + const auto bmi = cache / "pcm.cache" / "std.pcm"; + + auto stdCommands = clang::std_module_build_commands(tc, cache, bmi, "", "-std=c++23"); + auto inv = db::recover_invocation(stdCommands, tc.stdModuleSource, tc.binaryPath, + cache, kWindows); + ASSERT_TRUE(inv.has_value()) << stdCommands.front(); + EXPECT_EQ(inv->arguments.front(), tc.binaryPath.string()); + EXPECT_TRUE(contains(inv->arguments, "--precompile")) << stdCommands.front(); + EXPECT_TRUE(contains(inv->arguments, tc.stdModuleSource.string())); + EXPECT_EQ(inv->workDirectory, cache); + + auto compatCommands = clang::std_compat_build_commands( + tc, cache, clang::std_compat_bmi_path(cache), bmi, "", "-std=c++23"); + auto compat = db::recover_invocation(compatCommands, tc.stdCompatSource, + tc.binaryPath, cache, kWindows); + ASSERT_TRUE(compat.has_value()) << compatCommands.front(); + EXPECT_TRUE(contains(compat->arguments, tc.stdCompatSource.string())); +} + +TEST(BuildDatabase, NoCommandNamingTheSourceRecoversNothing) { + auto inv = db::recover_invocation({"cd /c && /bin/g++ -c other.cc -o o.o"}, + "/src/std.cc", "/bin/g++", "/c", false); + EXPECT_FALSE(inv.has_value()); +} + +TEST(BuildDatabase, AnUnquotedDriverWithASpaceIsRejoined) { + auto inv = db::recover_invocation( + {"C:\\Program Files\\LLVM\\bin\\clang++.exe -std=c++23 --precompile " + "\"C:\\s\\std.cppm\" -o \"C:\\c\\std.pcm\""}, + "C:\\s\\std.cppm", "C:\\Program Files\\LLVM\\bin\\clang++.exe", "C:\\c", + /*windows=*/true); + ASSERT_TRUE(inv.has_value()); + EXPECT_EQ(inv->arguments.front(), "C:\\Program Files\\LLVM\\bin\\clang++.exe"); + EXPECT_EQ(inv->arguments[1], "-std=c++23"); +} diff --git a/tests/unit/test_compile_commands.cpp b/tests/unit/test_compile_commands.cpp index 40a4532b..e21c81c8 100644 --- a/tests/unit/test_compile_commands.cpp +++ b/tests/unit/test_compile_commands.cpp @@ -416,3 +416,49 @@ TEST(CompileCommandsWriter, ExistingUnreadableDatabaseIsNotOverwritten) { std::filesystem::permissions(path, std::filesystem::perms::owner_all, std::filesystem::perm_options::replace, permissionEc); } + +// ── One record, two databases ─────────────────────────────────────────────── +// +// The compile database and the build database render the same per-unit record, +// so `arguments` cannot differ between them. The second half shows the +// comparison can fail: the record under other flags is not the CDB's. +TEST(CompileCommandsEmit, UnitInvocationsAreTheCompileDatabaseArguments) { + BuildPlan plan; + plan.projectRoot = "/p"; + plan.outputDir = "/p/target"; + plan.compileUnits.push_back({ + .source = std::filesystem::path("/p/src/main.cpp"), + .kind = mcpp::SourceKind::Cxx, + .object = std::filesystem::path("obj") / "main.o", + .packageName = "demo", + .localIncludeDirs = { std::filesystem::path("/p/include") }, + .packageCxxflags = { "-DDEMO=1" }, + }); + plan.compileUnits.push_back({ + .source = std::filesystem::path("/p/src/c.c"), + .kind = mcpp::SourceKind::C, + .object = std::filesystem::path("obj") / "c.o", + .packageName = "demo", + }); + CompileFlags flags; + flags.cxxBinary = "/usr/bin/g++"; + flags.ccBinary = "/usr/bin/gcc"; + flags.cxx = "-std=c++23 '-DQUOTED=a b'"; + flags.cc = "-std=c11"; + + auto j = nlohmann::json::parse(emit_compile_commands(plan, flags)); + auto invs = unit_invocations(plan, flags); + ASSERT_EQ(j.size(), invs.size()); + for (std::size_t i = 0; i < invs.size(); ++i) { + EXPECT_EQ(j[i]["arguments"].get>(), invs[i].arguments); + EXPECT_EQ(j[i]["file"].get(), invs[i].file); + EXPECT_EQ(j[i]["directory"].get(), invs[i].directory); + EXPECT_EQ(j[i]["output"].get(), invs[i].output); + EXPECT_EQ(invs[i].unit, &plan.compileUnits[i]); + } + + CompileFlags other = flags; + other.cxx = "-std=c++26"; + EXPECT_NE(j[0]["arguments"].get>(), + unit_invocations(plan, other)[0].arguments); +} diff --git a/tests/unit/test_modgraph.cpp b/tests/unit/test_modgraph.cpp index 5301ade7..186d1a78 100644 --- a/tests/unit/test_modgraph.cpp +++ b/tests/unit/test_modgraph.cpp @@ -99,6 +99,35 @@ TEST(Scanner, InterfacePartitionIsMarkedAsAnInterface) { std::filesystem::remove_all(dir); } +// ─── the declaration form (build database roles) ───────────────────────── +// +// `module M;` is stored as `requires M`, exactly as `import M;` is, so the form +// cannot be recovered from provides/requires. The scanner records what it read. +TEST(Scanner, DeclarationFormIsRecordedAsRead) { + auto dir = make_tempdir("scan-declform"); + std::filesystem::create_directories(dir / "src"); + struct Case { const char* file; const char* text; ModuleDeclaration want; }; + const Case cases[] = { + {"iface.cppm", "export module mathkit;\nexport int f();\n", + ModuleDeclaration::Interface}, + {"ipart.cppm", "export module mathkit:api;\nexport int g();\n", + ModuleDeclaration::InterfacePartition}, + {"impart.cppm", "module mathkit:secret;\nint h() { return 1; }\n", + ModuleDeclaration::ImplementationPartition}, + {"impl.cpp", "module mathkit;\nint f() { return 2; }\n", + ModuleDeclaration::Implementation}, + {"user.cpp", "import mathkit;\nint main() { return f(); }\n", + ModuleDeclaration::None}, + }; + for (auto const& c : cases) { + write(dir / "src" / c.file, c.text); + auto u = scan_file(dir / "src" / c.file, "pkg", mcpp::builtin_extension_table()); + ASSERT_TRUE(u.has_value()) << c.file; + EXPECT_EQ(static_cast(u->declaration), static_cast(c.want)) << c.file; + } + std::filesystem::remove_all(dir); +} + TEST(Scanner, PlainImplementationUnitStillRequiresItsInterface) { auto dir = make_tempdir("scan-implunit"); std::filesystem::create_directories(dir / "src"); diff --git a/tests/unit/test_wire.cpp b/tests/unit/test_wire.cpp index 3c4811dd..845e424e 100644 --- a/tests/unit/test_wire.cpp +++ b/tests/unit/test_wire.cpp @@ -240,8 +240,26 @@ TEST(WireGolden, SeverityNamesAreStable) { // whether to bother calling; dropping one silently is a breaking change. TEST(WireGolden, DeclaredKinds) { auto j = w::protocol_document({}); - expect_has_keys(j["kinds"], {"mcpp.env", "mcpp.xpkg", "mcpp.cache"}, + expect_has_keys(j["kinds"], {"mcpp.env", "mcpp.xpkg", "mcpp.cache", + "mcpp.build-database"}, "the kind list"); + EXPECT_EQ(j["kinds"]["mcpp.build-database"], 1); +} + +// A failure that leaves nothing to describe omits `data`: a consumer of the +// build database reads "no data" as "the command failed" (S2 0.2.0 section 3.4), +// so an empty object in its place would read as a success with nothing in it. +TEST(WireGolden, NullDataIsOmitted) { + auto failed = w::to_json(w::Envelope{ + .kind = "mcpp.build-database", + .effects = {w::Effect::ReadProject}, + .data = nullptr, + .diagnostics = {{"MCPP_BUILD_DATABASE_PLAN_FAILED", w::Severity::Error, "x"}}, + }); + EXPECT_FALSE(failed.contains("data")) << failed.dump(); + EXPECT_TRUE(failed.contains("diagnostics")); + auto ok = w::to_json(w::Envelope{.kind = "mcpp.env"}); + EXPECT_TRUE(ok.contains("data")) << ok.dump(); } } // namespace From 0421d32d7f7b2a8b323bbfbe7db222d0e38b10ac Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:00:37 +0800 Subject: [PATCH 05/16] fix(emit): a store package's build program inputs are not watched A store package's build program may declare inputs, and they were listed in `watch`; its files are fixed by the version its manifest and lock name, so only an editable package's declared inputs are inputs that can change. Found by running the command over lsp-mcpp, whose openkal-musl dependency declares its own mcpp.toml. --- src/build/build_database.cppm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/build/build_database.cppm b/src/build/build_database.cppm index d7fd5f75..5f99795e 100644 --- a/src/build/build_database.cppm +++ b/src/build/build_database.cppm @@ -515,7 +515,12 @@ Rendered render(std::span members, for (auto const& g : sp.sources) watch_glob(sp.root, g); } for (auto const& g : member.testDiscover) watch_glob(member.testRoot, g); + // Only an editable package's build program inputs can change: a store + // package's are fixed by the version its manifest and lock name. + std::set editableRoots; + for (auto const& sp : ctx.sourcePackages) editableRoots.insert(sp.root.lexically_normal()); for (auto const& declared : mcpp::build::declared_program_inputs(member.workDir)) { + if (!editableRoots.contains(declared.root.lexically_normal())) continue; for (auto const& f : declared.files) watch_file(f); for (auto const& g : declared.globs) watch_glob(declared.root, g); } From 5cfb03af9b79fa103a60d258beb4c7e24dce5003 Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:01:59 +0800 Subject: [PATCH 06/16] test(emit): the MSVC std units are recovered from their builder Drives msvc::std_module_build_commands and std_compat_build_commands, the command shape a Windows host with msvc@system runs (cd /d, a quoted cl.exe, /ifcOutput, /Fo:, 2>&1), through recover_invocation. --- tests/unit/test_build_database.cpp | 32 ++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/unit/test_build_database.cpp b/tests/unit/test_build_database.cpp index 9c9f38be..fa00df15 100644 --- a/tests/unit/test_build_database.cpp +++ b/tests/unit/test_build_database.cpp @@ -14,6 +14,7 @@ import mcpp.platform; import mcpp.toolchain.clang; import mcpp.toolchain.gcc; import mcpp.toolchain.model; +import mcpp.toolchain.msvc; namespace db = mcpp::build::database; using mcpp::modgraph::ModuleDeclaration; @@ -138,6 +139,37 @@ TEST(BuildDatabase, RecoversTheClangStdUnitsFromTheirBuilders) { EXPECT_TRUE(contains(compat->arguments, tc.stdCompatSource.string())); } +TEST(BuildDatabase, RecoversTheMsvcStdUnitsFromTheirBuilders) { + Toolchain tc; + tc.compiler = CompilerId::MSVC; + tc.version = "19.44.35207"; + tc.binaryPath = host_path( + "/vs/VC/Tools/MSVC/14.44.35207/bin/Hostx64/x64/cl.exe", + "C:\\Program Files\\Microsoft Visual Studio\\VC\\Tools\\MSVC\\14.44.35207\\bin\\Hostx64\\x64\\cl.exe"); + tc.targetTriple = "x86_64-windows-msvc"; + tc.stdModuleSource = host_path("/vs/VC/Tools/MSVC/14.44.35207/modules/std.ixx", + "C:\\Program Files\\Microsoft Visual Studio\\VC\\Tools\\MSVC\\14.44.35207\\modules\\std.ixx"); + tc.stdCompatSource = host_path("/vs/VC/Tools/MSVC/14.44.35207/modules/std.compat.ixx", + "C:\\Program Files\\Microsoft Visual Studio\\VC\\Tools\\MSVC\\14.44.35207\\modules\\std.compat.ixx"); + const auto cache = host_path("/home/u/.mcpp/cache/std/k3", "C:\\u\\.mcpp\\std\\k3"); + + auto stdCommands = msvc::std_module_build_commands(tc, cache, "/std:c++latest", "/MD"); + auto inv = db::recover_invocation(stdCommands, tc.stdModuleSource, tc.binaryPath, + "/unused", kWindows); + ASSERT_TRUE(inv.has_value()) << stdCommands.front(); + EXPECT_EQ(inv->workDirectory, cache) << stdCommands.front(); + EXPECT_EQ(inv->arguments.front(), tc.binaryPath.string()) << stdCommands.front(); + EXPECT_TRUE(contains(inv->arguments, "/std:c++latest")) << stdCommands.front(); + EXPECT_TRUE(contains(inv->arguments, tc.stdModuleSource.string())) << stdCommands.front(); + EXPECT_FALSE(contains(inv->arguments, "2>&1")) << stdCommands.front(); + + auto compatCommands = msvc::std_compat_build_commands(tc, cache, "/std:c++latest", "/MD"); + auto compat = db::recover_invocation(compatCommands, tc.stdCompatSource, + tc.binaryPath, "/unused", kWindows); + ASSERT_TRUE(compat.has_value()) << compatCommands.front(); + EXPECT_TRUE(contains(compat->arguments, "/reference")) << compatCommands.front(); +} + TEST(BuildDatabase, NoCommandNamingTheSourceRecoversNothing) { auto inv = db::recover_invocation({"cd /c && /bin/g++ -c other.cc -o o.o"}, "/src/std.cc", "/bin/g++", "/c", false); From 39053b9abf99346f0676f0f29ff3a4e776904e80 Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:07:18 +0800 Subject: [PATCH 07/16] fix(emit): the selector digest separates values with newlines A \x1f separator followed by `f`, `c` or `a` (`\x1ffeatures`, `\x1fcap`, `\x1faccel`) is one longer hex escape; clang refuses it as out of range and every clang host failed to build the branch. GCC accepted it, which is why the Linux build did not see it. The sandbox verification script for the release is added alongside. --- .agents/docs/2026-09-14-636-verify.sh | 132 ++++++++++++++++++++++++++ src/cli/cmd_build.cppm | 6 +- 2 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 .agents/docs/2026-09-14-636-verify.sh diff --git a/.agents/docs/2026-09-14-636-verify.sh b/.agents/docs/2026-09-14-636-verify.sh new file mode 100644 index 00000000..b2665ba0 --- /dev/null +++ b/.agents/docs/2026-09-14-636-verify.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Ecosystem verification for the design record +# 2026-09-14-636-build-database-and-the-latest-xlings.md, against a PUBLISHED +# mcpp and the xlings it pins. +# +# B64=$(base64 -w0 .agents/docs/2026-09-14-636-verify.sh) +# xlings subos new verify-636 # once +# xlings subos use verify-636 --sandbox --cmd \ +# "echo $B64 | base64 -d > /tmp/v.sh && MCPP_VERIFY_VERSION= XLINGS_VERIFY_VERSION= bash /tmp/v.sh" +# +# The sandbox starts from an empty $HOME and a fresh /tmp and shares the xlings +# data directory, so the published mcpp is addressed by its store path. The mcpp +# home is removed first: every section reads this release, not a previous run. +# A section that cannot run says so and is listed again in the summary. +set -u + +VER="${MCPP_VERIFY_VERSION:?set MCPP_VERIFY_VERSION}" +XVER="${XLINGS_VERIFY_VERSION:?set XLINGS_VERIFY_VERSION}" +STORE="${MCPP_VERIFY_BIN:-$HOME/.xlings/data/xpkgs/xim-x-mcpp/$VER/bin/mcpp}" + +fails=0 +skipped="" +fail() { printf 'ASSERT-FAIL: %s\n' "$1"; fails=$((fails + 1)); } +ok() { printf 'ok: %s\n' "$1"; } +section() { printf '\n== %s ==\n' "$1"; } +skip() { printf 'NOT RUN: %s\n' "$1"; skipped="$skipped + - $1"; } + +root=/tmp/verify-636 +rm -rf "$root" "$HOME/.mcpp"; mkdir -p "$root" +REG="$HOME/.mcpp/registry" + +section "A. the published mcpp, the CN mirror, and the xlings it vendors" +if [ ! -x "$STORE" ]; then + fail "no mcpp at $STORE"; printf '\nfails=%d (nothing else can run)\n' "$fails"; exit 1 +fi +got=$("$STORE" --version 2>&1 | head -1) +case "$got" in *"$VER"*) ok "mcpp --version says $got" ;; *) fail "mcpp --version says '$got', expected $VER" ;; esac +"$STORE" self config --mirror CN > "$root/mirror.log" 2>&1 && ok "mcpp self config --mirror CN" || { fail "mcpp self config --mirror CN"; tail -3 "$root/mirror.log"; } +grep -q '"mirror": "CN"' "$REG/.xlings.json" 2>/dev/null && ok "the registry's xlings configuration reads mirror CN" || fail "the registry's .xlings.json does not read mirror CN" +"$STORE" self env > "$root/env.log" 2>&1 +grep -q "xlings pinned = $XVER" "$root/env.log" && ok "the pin is xlings $XVER" || { fail "mcpp self env does not pin xlings $XVER"; grep 'xlings pinned' "$root/env.log"; } +vend=$("$REG/bin/xlings" --version 2>/dev/null | head -1) +case "$vend" in *"$XVER"*) ok "the vendored xlings is $vend" ;; *) fail "the vendored xlings is '$vend', expected $XVER" ;; esac + +section "B. a hookless package after a hook package: its own archive, nothing else" +d=$root/b; rm -rf "$d"; mkdir -p "$d/src" +cat > "$d/mcpp.toml" <<'EOF' +[package] +name = "eco636" +version = "0.1.0" +standard = "c++23" + +[dependencies] +mcpplibs.cmdline = "0.0.1" +EOF +printf 'import std;\nimport mcpplibs.cmdline;\nint main() { std::println("1-2-3"); }\n' > "$d/src/main.cpp" +if (cd "$d" && "$STORE" build > build.log 2>&1); then + ok "a project with a toolchain install and a hookless mcpp-index dependency builds" + swept=$(find "$REG/data/xpkgs" -mindepth 3 -maxdepth 3 -name '*.lock' 2>/dev/null) + [ -z "$swept" ] && ok "no store payload carries a download sidecar" || { fail "store payloads carry download sidecars"; printf '%s\n' "$swept" | head -5; } + cmd=$(find "$REG/data/xpkgs" -maxdepth 2 -path '*cmdline*' -type d | tail -1) + if [ -n "$cmd" ]; then + top=$(ls -A "$cmd" | grep -v '^\.' | tr '\n' ' ') + case "$top" in + *.tar.gz*|*.zip*) fail "the cmdline payload holds an archive: $top" ;; + *) ok "the cmdline payload's top level is its own archive's entries: $top" ;; + esac + [ -n "$(find "$cmd" -mindepth 2 -maxdepth 2 -name mcpp.toml)" ] && ok "the top-level directory is kept, so */mcpp.toml matches" || fail "no */mcpp.toml under $cmd" + else + skip "B: the cmdline payload directory was not found" + fi + (cd "$d" && "$STORE" run > run.log 2>&1); grep -qx '1-2-3' "$d/run.log" && ok "mcpp run prints 1-2-3" || { fail "mcpp run"; tail -3 "$d/run.log"; } +else + fail "the dependency build failed"; tail -8 "$d/build.log" +fi + +section "C. mcpp emit build-database writes nothing and describes the plan" +d=$root/c; rm -rf "$d"; mkdir -p "$d/src" "$d/tests" +cat > "$d/mcpp.toml" <<'EOF' +[package] +name = "hello" +version = "0.1.0" +standard = "c++23" +EOF +printf 'export module hello.greet;\nexport import :detail;\nimport std;\nexport std::string greet();\n' > "$d/src/greet.cppm" +printf 'export module hello.greet:detail;\nexport int answer() { return 42; }\n' > "$d/src/detail.cppm" +printf 'module hello.greet;\nstd::string greet() { return "hi"; }\n' > "$d/src/greet_impl.cpp" +printf 'import hello.greet;\nimport std;\nint main() { std::println("{}", greet()); }\n' > "$d/src/main.cpp" +printf 'int main() { return 0; }\n' > "$d/tests/test_smoke.cpp" +before=$(cd "$d" && find . -type f | sort | xargs sha256sum | sha256sum) +if (cd "$d" && "$STORE" emit build-database --format json > "$root/c.json" 2> "$root/c.err"); then + after=$(cd "$d" && find . -type f | sort | xargs sha256sum | sha256sum) + [ "$before" = "$after" ] && ok "the project tree is unchanged" || fail "the project tree changed" + if python3 - "$root/c.json" <<'EOF' +import json, os, sys +e = json.load(open(sys.argv[1])) +assert e["kind"] == "mcpp.build-database" and "write-project" not in e["effects"] +db = e["data"]["database"] +sets = {s["name"]: s for s in db["sets"]} +assert set(sets) == {"hello", "hello:test", "mcpp:std"}, sorted(sets) +roles = {os.path.basename(u["source"]): u["ide"]["role"] for u in sets["hello"]["translation-units"]} +assert roles == {"greet.cppm": "module-interface", "detail.cppm": "module-partition-interface", + "greet_impl.cpp": "module-implementation", "main.cpp": "non-module"}, roles +std = sets["mcpp:std"]["translation-units"][0] +assert std["provides"] == {"std": ""} and os.path.isfile(std["source"]), std +print("sets", sorted(sets), "units", sum(len(s["translation-units"]) for s in db["sets"])) +EOF + then ok "the S1 document has the sets, roles and std unit SPEC-005 states" + else fail "the S1 document"; fi +else + fail "emit build-database failed"; tail -5 "$root/c.err" +fi + +section "D. xlings reports and repairs a payload that holds another package's download" +cmd=$(find "$REG/data/xpkgs" -maxdepth 2 -path '*cmdline*' -type d | tail -1) +if [ -n "$cmd" ]; then + printf 'x' > "$cmd/intruder-1.0-linux-x86_64.tar.gz"; : > "$cmd/intruder-1.0-linux-x86_64.tar.gz.lock" + XLINGS_HOME="$REG" "$REG/bin/xlings" self doctor > "$root/doctor.log" 2>&1 + grep -qi 'intruder\|swept\|download' "$root/doctor.log" && ok "self doctor reports the swept payload" || { fail "self doctor did not report it"; tail -8 "$root/doctor.log"; } + XLINGS_HOME="$REG" "$REG/bin/xlings" self doctor --fix > "$root/fix.log" 2>&1 + [ ! -e "$cmd/intruder-1.0-linux-x86_64.tar.gz.lock" ] && ok "self doctor --fix leaves no download sidecar in the payload" || { fail "the sidecar is still there after --fix"; tail -8 "$root/fix.log"; } +else + skip "D: section B produced no cmdline payload to seed" +fi + +section "E. Windows: the probe reaches no shell" +skip "E: a Linux sandbox cannot run cmd.exe; e2e 687 and XlingsVersionPin.ProbeReadsStandardOutputThroughTheLauncher are the Windows criteria (CI)" + +printf '\n== summary ==\nfails=%d\n' "$fails" +[ -n "$skipped" ] && printf 'not run:%s\n' "$skipped" +[ "$fails" -eq 0 ] diff --git a/src/cli/cmd_build.cppm b/src/cli/cmd_build.cppm index 78f68854..c45ce4cc 100644 --- a/src/cli/cmd_build.cppm +++ b/src/cli/cmd_build.cppm @@ -403,9 +403,11 @@ export int cmd_emit_build_database(const mcpplibs::cmdline::ParsedArgs& parsed) if (std::filesystem::exists(sp.root / "build.mcpp", ec)) ranBuildPrograms = true; } } + // One line per selector: a value never spans lines, and a `\x1f` separator + // before `f`, `c` or `a` reads as a longer hex escape (clang refuses it). const auto selector = std::format( - "spec={}\x1ftarget={}\x1ftoolchain={}\x1fprofile={}\x1ffeatures={}\x1f" - "cap={}\x1faccel={}\x1fstatic={}\x1fpackage={}\x1fworkspace={}", + "spec={}\ntarget={}\ntoolchain={}\nprofile={}\nfeatures={}\n" + "cap={}\naccel={}\nstatic={}\npackage={}\nworkspace={}", spec, ov.target_triple, mcpp::platform::env::get("MCPP_TOOLCHAIN").value_or(""), ov.profile, ov.features, ov.capabilities, ov.accel, ov.force_static, ov.package_filter, parsed.is_flag_set("workspace")); From 36124cb504ef84849f3cbca5405be8de4fcf9998 Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:36:43 +0800 Subject: [PATCH 08/16] fix(scan): an entry source outside the globs is read by the scanner A target's entry source that no `sources` glob matched (a discovered test, a `main` outside the globs) had its imports read from line-leading `import` alone, in make_plan and again in the standard library check before planning. An import inside a comment or a raw string was therefore planned as one. Validating the lsp-mcpp repository's own build database against S1 found it: its scanner test was planned as importing three modules no source provides. scan_entry_file reads such a file with scan_file. The scanner's refusals (an import inside `#if`, a header unit, an extension without a role) were never applied on this path and are not applied now: a file it refuses yields the line-leading imports of its code, comments and raw strings removed, with the declaration form Unknown. Both readers now call it, so they cannot disagree about `import std`. Refs #636 --- src/build/plan.cppm | 32 ++++++++--------------- src/build/prepare.cppm | 49 +++++++----------------------------- src/modgraph/scanner.cppm | 38 ++++++++++++++++++++++++++++ tests/unit/test_modgraph.cpp | 44 ++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 62 deletions(-) diff --git a/src/build/plan.cppm b/src/build/plan.cppm index cd42e0fa..4c1c181c 100644 --- a/src/build/plan.cppm +++ b/src/build/plan.cppm @@ -1919,28 +1919,16 @@ make_plan(const mcpp::manifest::Manifest& manifest, mcpp::modgraph::normalize_include_flags(projectRoot, main_cu.packageCflags); mcpp::modgraph::normalize_include_flags(projectRoot, main_cu.packageCxxflags); - // We didn't scan main.cpp earlier (it's not in scanner output unless globbed in). - // Best-effort: scan its imports here. - std::ifstream is(*lu.entryMain); - std::string line; - while (std::getline(is, line)) { - auto trim = [](std::string s) { - while (!s.empty() && std::isspace(static_cast(s.front()))) s.erase(0, 1); - while (!s.empty() && std::isspace(static_cast(s.back()))) s.pop_back(); - return s; - }; - line = trim(line); - if (line.starts_with("import ")) { - std::string name; - std::size_t i = 7; - while (i < line.size() && (std::isalnum(static_cast(line[i])) - || line[i] == '_' || line[i] == '.')) { - name.push_back(line[i]); - ++i; - } - if (!name.empty()) main_cu.imports.push_back(name); - } - } + // The entry is in the package scan only when a `sources` glob + // matched it; otherwise it is scanned here. The unit is built as one + // that provides nothing, so a declaration the scanner reads as + // providing a module is recorded as undecided rather than as a + // role the plan does not build. + const auto entry = mcpp::modgraph::scan_entry_file( + *lu.entryMain, main_cu.packageName, rootExtTable); + for (auto const& req : entry.requires_) main_cu.imports.push_back(req.logicalName); + main_cu.declaration = entry.provides ? mcpp::modgraph::ModuleDeclaration::Unknown + : entry.declaration; // mcpp#240: the entry main may ALSO have been scanned (globbed into // [modules].sources — the near-universal `src/**/*.cpp`). When it diff --git a/src/build/prepare.cppm b/src/build/prepare.cppm index 820867d7..f648de66 100644 --- a/src/build/prepare.cppm +++ b/src/build/prepare.cppm @@ -648,43 +648,6 @@ bool is_std_module(std::string_view name) { return name == "std" || name == "std.compat"; } -std::string trim_copy(std::string s) { - while (!s.empty() && std::isspace(static_cast(s.front()))) - s.erase(0, 1); - while (!s.empty() && std::isspace(static_cast(s.back()))) - s.pop_back(); - return s; -} - -bool source_file_imports_std(const std::filesystem::path& path) { - std::ifstream is(path); - if (!is) return false; - - std::string line; - while (std::getline(is, line)) { - line = trim_copy(std::move(line)); - std::size_t i = std::string::npos; - if (line.starts_with("import ")) { - i = 7; - } else if (line.starts_with("export import ")) { - i = 14; - } - if (i == std::string::npos) continue; - while (i < line.size() && std::isspace(static_cast(line[i]))) - ++i; - - std::string name; - while (i < line.size() - && (std::isalnum(static_cast(line[i])) - || line[i] == '_' || line[i] == '.' || line[i] == ':')) { - name.push_back(line[i]); - ++i; - } - if (is_std_module(name)) return true; - } - return false; -} - bool graph_or_targets_import_std(const mcpp::modgraph::Graph& graph, const mcpp::manifest::Manifest& manifest, const std::filesystem::path& projectRoot) { @@ -696,10 +659,16 @@ bool graph_or_targets_import_std(const mcpp::modgraph::Graph& graph, } // Some target entry files can be added to the plan after the package scan. - // Check them here so std BMI setup matches what make_plan will compile. + // Check them here so std BMI setup matches what make_plan will compile: they + // are read by the same scan_entry_file make_plan reads them with. + const auto extTable = mcpp::extension_table_for(manifest.buildConfig.moduleExtensions, + manifest.buildConfig.deviceExtensions); for (auto& t : manifest.targets) { - if (!t.main.empty() && source_file_imports_std(projectRoot / t.main)) - return true; + if (t.main.empty()) continue; + const auto entry = mcpp::modgraph::scan_entry_file(projectRoot / t.main, + manifest.package.name, extTable); + for (auto const& req : entry.requires_) + if (is_std_module(req.logicalName)) return true; } return false; } diff --git a/src/modgraph/scanner.cppm b/src/modgraph/scanner.cppm index b7f772d4..f419e5b5 100644 --- a/src/modgraph/scanner.cppm +++ b/src/modgraph/scanner.cppm @@ -63,6 +63,16 @@ std::expected scan_file(const std::filesystem::path& file const std::string& packageName, const mcpp::ExtensionTable& extTable); +// Scan the entry source of a target that no `sources` glob matched: a +// discovered test, or a `main` outside the globs. The unit is scan_file's when +// scan_file accepts the file. Its refusals (an import inside `#if`, a header +// unit, an extension without a role) were never applied to such a file and are +// not applied here: a file it refuses yields the line-leading imports of its +// code, with comments and raw strings removed, and the declaration form Unknown. +SourceUnit scan_entry_file(const std::filesystem::path& file, + const std::string& packageName, + const mcpp::ExtensionTable& extTable); + // Scan the entire package: collects all sources via manifest globs and returns a Graph. struct ScanResult { Graph graph; @@ -1005,6 +1015,34 @@ std::expected scan_file(const std::filesystem::path& file return u; } +SourceUnit scan_entry_file(const std::filesystem::path& file, + const std::string& packageName, + const mcpp::ExtensionTable& extTable) +{ + if (auto scanned = scan_file(file, packageName, extTable)) return std::move(*scanned); + + SourceUnit u; + u.path = file; + u.packageName = packageName; + u.kind = mcpp::classify(file, extTable); + u.declaration = ModuleDeclaration::Unknown; + std::ifstream is(file); + bool in_raw = false, in_block = false; + std::string raw_close, line; + while (std::getline(is, line)) { + const std::string code = strip_noncode(line, in_block, in_raw, raw_close); + std::string_view r = trim(code); + if (r.starts_with("export ") || r.starts_with("export\t")) r = trim(r.substr(6)); + if (!r.starts_with("import ") && !r.starts_with("import\t")) continue; + r = trim(r.substr(6)); + std::string name; + for (std::size_t i = 0; i < r.size() && is_module_name_char(r[i]); ++i) + name.push_back(r[i]); + if (!name.empty() && name.front() != ':') u.requires_.push_back(ModuleId{name}); + } + return u; +} + namespace { std::vector diff --git a/tests/unit/test_modgraph.cpp b/tests/unit/test_modgraph.cpp index 186d1a78..de930fad 100644 --- a/tests/unit/test_modgraph.cpp +++ b/tests/unit/test_modgraph.cpp @@ -128,6 +128,50 @@ TEST(Scanner, DeclarationFormIsRecordedAsRead) { std::filesystem::remove_all(dir); } +// ─── an entry source no `sources` glob matched ───────────────────────────── +// +// A discovered test, or a `main` outside the globs, is not in the package scan. +// Its imports were read from line-leading `import` alone, so an import inside a +// comment or a raw string was recorded as one: a scanner test in lsp-mcpp was +// planned as importing three modules no source provides. +TEST(Scanner, AnEntrySourceIsReadByTheScanner) { + auto dir = make_tempdir("scan-entry"); + write(dir / "t.cpp", + "import std;\n" + "/* a comment\n" + " import in.comment; */\n" + "auto s = R\"x(\n" + "import in.raw;\n" + ")x\";\n" + "import real;\n" + "int main() {}\n"); + auto u = scan_entry_file(dir / "t.cpp", "pkg", mcpp::builtin_extension_table()); + std::vector names; + for (auto const& r : u.requires_) names.push_back(r.logicalName); + EXPECT_EQ(names, (std::vector{"std", "real"})); + EXPECT_EQ(static_cast(u.declaration), static_cast(ModuleDeclaration::None)); + std::filesystem::remove_all(dir); +} + +// The scanner's refusals were never applied to such a file, and are not now: a +// file it refuses keeps its line-leading imports, and its form is not decided. +TEST(Scanner, AnEntrySourceTheScannerRefusesKeepsItsImports) { + auto dir = make_tempdir("scan-entry-refused"); + write(dir / "t.cpp", + "#ifdef WITH_EXTRA\n" + "import extra;\n" + "#endif\n" + "export import std;\n" + "int main() {}\n"); + ASSERT_FALSE(scan_file(dir / "t.cpp", "pkg", mcpp::builtin_extension_table()).has_value()); + auto u = scan_entry_file(dir / "t.cpp", "pkg", mcpp::builtin_extension_table()); + std::vector names; + for (auto const& r : u.requires_) names.push_back(r.logicalName); + EXPECT_EQ(names, (std::vector{"extra", "std"})); + EXPECT_EQ(static_cast(u.declaration), static_cast(ModuleDeclaration::Unknown)); + std::filesystem::remove_all(dir); +} + TEST(Scanner, PlainImplementationUnitStillRequiresItsInterface) { auto dir = make_tempdir("scan-implunit"); std::filesystem::create_directories(dir / "src"); From 5867295e094f6bb5447dbfa17c67db42c58ff2b9 Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:36:43 +0800 Subject: [PATCH 09/16] feat(emit): baseline and local arguments, private, and config-files The S1 SHOULD fields the lsp-mcpp validator reads, with values the plan determines: - `baseline-arguments` and `local-arguments`. A unit's arguments are its driver, the set's baseline, its local arguments and its own trailing `-c -o `, whose operands name the unit's source and object from its work directory. The baseline is the longest prefix every unit of the set shares; a prefix keeps argument order, which decides include search and macro definitions. - `private` is false: every module is visible to every set. - `config-files` lists what the driver reads without being named: the clang `.cfg` of resolve_clang_driver unless the units pass `--no-default-config`, and GCC's `lib/gcc///specs` (or the major version) beside the driver. e2e 688 gains J (the decomposition, private, config-files) and K (the discovered test's imports inside a comment and a raw string are not `requires`); its F mapping compares paths with one separator, and the fixture's main.cpp is well-formed. SPEC-005 R3.2a, R3.6, R3.8 and R3.9a state the rules. Refs #636 --- ...36-build-database-and-the-latest-xlings.md | 15 ++- docs/specs/build-database.md | 19 +++- src/build/build_database.cppm | 104 +++++++++++++++--- tests/e2e/688_emit_build_database.sh | 54 +++++++-- 4 files changed, 168 insertions(+), 24 deletions(-) diff --git a/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md b/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md index 3e13abc4..7bf59779 100644 --- a/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md +++ b/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md @@ -305,18 +305,31 @@ guarantee covers mcpp's writes. A build program that writes outside | S1 | from | note | |---|---|---| -| toolchain id | `--` | | +| toolchain id | `--`, the triple as the compiler spells it | opaque to a consumer; `llvm`, as in `llvm@22.1.8` | | `family` | `Toolchain::compiler_name()` (`modules/toolchain-model/src/model.cppm:261-268`): `gcc`, `clang`, `msvc` | not `compiler_family()`, which answers `llvm` | | `driver`, `version`, `target`, `sysroot` | `binaryPath`, `version`, `targetTriple`, `sysroot` | | | `stdlib` | `name` and `version` from `stdlibId` and `stdlibVersion`; no `module-metadata` | std resolves through units (next row) | +| `config-files` | clang: the `.cfg` of `resolve_clang_driver`, unless the units pass `--no-default-config`; GCC: `lib/gcc///specs` beside the driver | read from the layout the driver searches; no driver is run | | set `mcpp:std` | units for `stdModuleSource` and `stdCompatSource`, carrying the commands of 5.3 (3) | One rule for GCC's `bits/std.cc`, libc++'s `std.cppm`, MSVC's `std.ixx` and a package's own `std.cppm`; S1 §6.1 lets units outrank a manifest. `mcpp:` is the engine's reserved namespace (SPEC-002), so the name cannot collide with a package. | | sets | one per package; test targets' sources in `:test` | `family-name` is the package; `ide.configuration` is `BuildContext::profile`; `ide.kind` comes from the package's declared targets | | `visible-sets` | every other set | The engine resolves imports over one flat graph per invocation (`scanner.cppm:1289-1319`). A narrower closure would describe a rule the build does not enforce; if the engine later refuses undeclared imports, the database inherits it. | | units | every `CompileUnit` except NASM units, as in the CDB (`compile_commands.cppm:225-228`) | rendered from the record of 5.3 (4) | +| `baseline-arguments`, `local-arguments` | a unit's arguments are its driver, the set's baseline, its local arguments and its own `-c -o `; the baseline is the longest prefix every unit of the set shares | a prefix keeps argument order, which decides include search and macro definitions; a common subset would not | +| `private` | `false` | every module is visible to every set, as the `visible-sets` row states | | `provides` | `providesModule` mapped to `""` | S1-8-6 permits an empty path for a producer that performs no build | | `requires` | `imports` | partitions are already written in full | | `ide.role` | the declaration form of 5.3 (5) | `unknown` for `scan_overrides` units | +A target's entry source that no `sources` glob matched (a discovered test, a +`main` outside the globs) had its imports read from line-leading `import` alone, +so an import in a comment or a raw string was planned as one. Validating the +lsp-mcpp repository's own database against S1 found it: its scanner test +required three modules no source provides. The entry is now read by the +scanner (`scan_entry_file`); a file the scanner refuses, which this path never +refused, keeps its line-leading imports with the role `unknown`. The standard +library check before planning reads entries the same way, so the two cannot +disagree about `import std`. + ### 5.5 `watch` and `inputs-fingerprint` - `watch` lists what the resolution read and a user edits: the manifests of the diff --git a/docs/specs/build-database.md b/docs/specs/build-database.md index a1d914c0..f33796bb 100644 --- a/docs/specs/build-database.md +++ b/docs/specs/build-database.md @@ -6,7 +6,7 @@ | 标题 | mcpp 输出的构建数据库:内容、取值规则与不写工程目录的保证 | | 状态 | 评审中 v1.0 | | 版本 | 1.0 | -| 最后修改 | 2026-09-14 | +| 最后修改 | 2026-09-15 | | 对应实现 | mcpp >= 2026.9.15.1 | | 相关设计文档 | `.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md` | | 相关 issue | #636 | @@ -71,6 +71,10 @@ mcpp 输出的 S1 文档满足 S1 等级 2,不输出 `ide.options`。等级 3 `target` 为编译器自身拼写的目标三元组;构建使用 sysroot 时给出 `sysroot`;`stdlib` 给出 `name`(`libstdc++`、`libc++`、`msvc-stl` 或 `other`)与 `version`,不给出 `module-metadata`,标准库模块经 §3.4 的单元解析。**已实现** +- **R3.2a** `config-files` 列出驱动在命令行之外读取的配置文件,空数组表示没有:clang + 驱动旁的 `<驱动名>.cfg`,单元带 `--no-default-config` 时不列出;GCC 驱动库目录中 + `lib/gcc//<版本>/specs`,版本目录也可以只写主版本号。取值来自驱动 + 搜索的目录布局,命令不运行驱动。**已实现** ### 3.2 集合 @@ -82,7 +86,11 @@ mcpp 输出的 S1 文档满足 S1 等级 2,不输出 `ide.options`。等级 3 - **R3.5** `family-name` 为包名,`mcpp:std` 集合的为 `mcpp:std`;`ide.configuration` 为 profile 名;`ide.kind` 在测试集合为 `test`,在根包集合按其目标为 `library`、 `executable` 或 `other`,在依赖包集合与 `mcpp:std` 为 `library`。**已实现** -- **R3.6** 不输出 `baseline-arguments` 与 `local-arguments`。**已实现** +- **R3.6** 单元的 `arguments` 依次是驱动、集合的 `baseline-arguments`、单元的 + `local-arguments`,以及单元自己结尾的 `-c -o `(若有;两个操作数 + 相对 `work-directory` 指向 `source` 与 `object`)。`baseline-arguments` 是集合中每个 + 单元去掉驱动与该结尾后的最长公共前缀。取前缀而不取公共子集,因为参数顺序决定 + 头文件搜索与宏定义。**已实现** ### 3.3 翻译单元 @@ -91,7 +99,7 @@ mcpp 输出的 S1 文档满足 S1 等级 2,不输出 `ide.options`。等级 3 `file`、`directory`、`arguments`、`output` 取自同一条记录,因而逐字相同。 **已实现** - **R3.8** `provides` 把单元提供的模块名映射到空字符串,命令不执行构建(S1-8-6); - `requires` 为单元导入的模块名,分区写全名 `M:P`。不输出 `private`,即 `false`。 + `requires` 为单元导入的模块名,分区写全名 `M:P`。`private` 为 `false`,理由同 R3.4。 **已实现** - **R3.9** `ide.role` 取自扫描器读到的模块声明形式: @@ -105,6 +113,11 @@ mcpp 输出的 S1 文档满足 S1 等级 2,不输出 `ide.options`。等级 3 | `scan_overrides` 声明的单元;P1689 扫描中无法区分实现单元与导入者的单元 | `unknown` | **已实现** +- **R3.9a** 没有被 `sources` glob 匹配的目标入口源文件(发现的测试、glob 之外的 + `main`)与包源文件由同一扫描器读取,注释与原始字符串中的 `import` 不是导入。扫描器 + 拒绝的入口文件(`#if` 块中的 `import`、头文件单元)从未在这条路径上被拒绝,现在也 + 不被拒绝:`requires` 为其代码中行首的 `import`,`ide.role` 为 `unknown`。入口声明 + 自身提供模块时,`ide.role` 为 `unknown`,因为构建不为该单元产出 BMI。**已实现** ### 3.4 标准库模块 diff --git a/src/build/build_database.cppm b/src/build/build_database.cppm index 5f99795e..5b7a9a50 100644 --- a/src/build/build_database.cppm +++ b/src/build/build_database.cppm @@ -35,6 +35,7 @@ import mcpp.modgraph.graph; import mcpp.modgraph.scanner; import mcpp.platform; import mcpp.toolchain.fingerprint; +import mcpp.toolchain.linkmodel; import mcpp.toolchain.model; import mcpp.toolchain.stdmod; @@ -151,23 +152,95 @@ Redirect redirect_kind(std::string_view w) { return j == w.size() ? Redirect::Detached : Redirect::Attached; } -bool names_source(std::string_view word, const std::filesystem::path& source, +bool names_path(std::string_view word, const std::filesystem::path& path, const std::filesystem::path& cwd) { std::filesystem::path w{std::string(word)}; - const auto want = source.lexically_normal(); + const auto want = path.lexically_normal(); if (w.lexically_normal() == want) return true; if (!w.is_absolute() && !cwd.empty() && (cwd / w).lexically_normal() == want) return true; return false; } +// S1 `config-files`: the configuration files the driver reads without being +// named on a command line. A clang driver reads the `.cfg` beside it unless a +// unit passes `--no-default-config`, which mcpp does whenever that file exists +// (mcpp.toolchain.linkmodel::resolve_clang_driver). A GCC driver reads the +// `specs` file in its library directory for its own machine and version, which +// a distribution may spell with the major version only. Found from the layout +// the driver searches; no driver is run. +nlohmann::json config_files(const mcpp::toolchain::Toolchain& tc, + const std::vector& invocations) { + nlohmann::json out = nlohmann::json::array(); + using C = mcpp::toolchain::CompilerId; + std::error_code ec; + if (tc.compiler == C::Clang) { + const auto dm = mcpp::toolchain::resolve_clang_driver(tc); + const bool read = std::ranges::any_of(invocations, [](auto const& inv) { + return std::ranges::find(inv.arguments, "--no-default-config") == inv.arguments.end(); + }); + if (dm.hasCfg && read) out.push_back(native_string(dm.cfgPath)); + } else if (tc.compiler == C::GCC && !tc.targetTriple.empty()) { + const auto machine = tc.binaryPath.parent_path().parent_path() + / "lib" / "gcc" / tc.targetTriple; + const auto major = tc.version.substr(0, tc.version.find('.')); + for (auto const& version : {tc.version, major}) { + const auto specs = machine / version / "specs"; + if (!version.empty() && std::filesystem::is_regular_file(specs, ec)) { + out.push_back(native_string(specs)); + break; + } + } + } + return out; +} + +// S1 `baseline-arguments` and `local-arguments`. A unit's arguments are its +// driver, then the set's baseline, then its local arguments, then its own +// trailing `-c -o ` when it has one: the baseline is the longest +// prefix that every unit of the set shares after the driver and without that +// tail. A prefix keeps the order of the arguments, which decides include search +// and macro definitions; a common subset would not. +void split_baseline(nlohmann::json& set) { + std::vector> semantic; + for (auto const& u : set["translation-units"]) { + auto args = u["arguments"].get>(); + std::vector rest(args.size() > 1 ? args.begin() + 1 : args.end(), args.end()); + const std::filesystem::path cwd{u["work-directory"].get()}; + const auto n = rest.size(); + if (n >= 4 && rest[n - 4] == "-c" && rest[n - 2] == "-o" + && names_path(rest[n - 3], u["source"].get(), cwd) + && names_path(rest[n - 1], u["object"].get(), cwd)) + rest.resize(n - 4); + semantic.push_back(std::move(rest)); + } + std::vector baseline; + if (!semantic.empty()) { + baseline = semantic.front(); + for (auto const& unit : semantic) { + std::size_t k = 0; + while (k < baseline.size() && k < unit.size() && baseline[k] == unit[k]) ++k; + baseline.resize(k); + } + } + std::size_t i = 0; + for (auto& u : set["translation-units"]) { + const auto& unit = semantic[i++]; + u["local-arguments"] = std::vector( + unit.begin() + static_cast(baseline.size()), unit.end()); + } + set["baseline-arguments"] = std::move(baseline); +} + nlohmann::json toolchain_json(const mcpp::toolchain::Toolchain& tc, - std::string_view compilerTriple) { + std::string_view compilerTriple, + const std::vector& invocations) { nlohmann::json j{ - {"family", std::string(tc.compiler_name())}, - {"version", tc.version}, - {"driver", tc.binaryPath.string()}, - {"target", std::string(compilerTriple)}, + {"family", std::string(tc.compiler_name())}, + {"version", tc.version}, + {"driver", tc.binaryPath.string()}, + {"target", std::string(compilerTriple)}, + {"config-files", config_files(tc, invocations)}, }; if (!tc.sysroot.empty()) j["sysroot"] = native_string(tc.sysroot); if (!tc.stdlibId.empty()) { @@ -347,7 +420,7 @@ std::optional recover_invocation(const std::vector& com } } const bool named = std::ranges::any_of(argv, [&](const std::string& w) { - return names_source(w, source, cwd.empty() ? defaultDirectory : cwd); + return names_path(w, source, cwd.empty() ? defaultDirectory : cwd); }); if (!named) continue; return Invocation{cwd.empty() ? defaultDirectory : cwd, std::move(argv)}; @@ -398,8 +471,6 @@ Rendered render(std::span members, const std::string compilerTriple = ctx.plan.targetSide.llvmTriple.empty() ? ctx.tc.targetTriple : ctx.plan.targetSide.llvmTriple; const auto tcId = toolchain_id(ctx.tc, compilerTriple); - if (!toolchains.contains(tcId)) - toolchains[tcId] = toolchain_json(ctx.tc, compilerTriple); const auto rootName = qualified_name(ctx.manifest); std::set testSources; @@ -424,7 +495,10 @@ Rendered render(std::span members, }; const auto flags = mcpp::build::compute_flags(ctx.plan); - for (auto& inv : mcpp::build::unit_invocations(ctx.plan, flags)) { + auto invocations = mcpp::build::unit_invocations(ctx.plan, flags); + if (!toolchains.contains(tcId)) + toolchains[tcId] = toolchain_json(ctx.tc, compilerTriple, invocations); + for (auto& inv : invocations) { const auto& cu = *inv.unit; const bool isTest = testSources.contains(cu.source.lexically_normal()); const std::string package = cu.packageName.empty() ? rootName : cu.packageName; @@ -448,6 +522,7 @@ Rendered render(std::span members, {"work-directory", std::move(inv.directory)}, {"arguments", std::move(inv.arguments)}, {"object", std::move(inv.output)}, + {"private", false}, {"provides", std::move(provides)}, {"requires", std::move(requires_)}, {"ide", {{"role", std::string(role_name(cu.declaration))}}}, @@ -478,6 +553,7 @@ Rendered render(std::span members, {"work-directory", native_string(inv->workDirectory)}, {"arguments", std::move(inv->arguments)}, {"object", native_string(object)}, + {"private", false}, {"provides", {{std::string(module), ""}}}, {"requires", std::move(requires_)}, {"ide", {{"role", "module-interface"}}}, @@ -493,7 +569,7 @@ Rendered render(std::span members, nlohmann::json visible = nlohmann::json::array(); for (auto const& other : order) if (other != name) visible.push_back(other); - sets.push_back(nlohmann::json{ + nlohmann::json setJson{ {"name", name}, {"family-name", set.familyName}, {"visible-sets", std::move(visible)}, @@ -503,7 +579,9 @@ Rendered render(std::span members, {"configuration", ctx.profile}, {"kind", set.kind}, }}, - }); + }; + split_baseline(setJson); + sets.push_back(std::move(setJson)); } watch_file(ctx.projectRoot / "mcpp.lock"); diff --git a/tests/e2e/688_emit_build_database.sh b/tests/e2e/688_emit_build_database.sh index b608cdd4..b15cf48e 100755 --- a/tests/e2e/688_emit_build_database.sh +++ b/tests/e2e/688_emit_build_database.sh @@ -23,6 +23,12 @@ # H. The bare document goes to stdout, or to `-o ` with stdout empty. # I. `watch` names the manifests, the lock, the source and test globs; the # fingerprint is stable and follows a source edit. +# J. Each unit's arguments are its driver, its set's `baseline-arguments`, its +# `local-arguments` and its `-c -o `; `private` is false; +# each toolchain's `config-files` names existing files, and no `.cfg` that +# the units bypass with `--no-default-config`. +# K. The discovered test is scanned as a package source is: the imports inside +# its comment and its raw string are not in `requires`. set -e TMP=$(mktemp -d) # the measured tree: the project and its dev-dependency @@ -64,8 +70,8 @@ printf 'export module hello.greet;\nexport import :detail;\nimport std;\nexport printf 'export module hello.greet:detail;\nexport int answer() { return 42; }\n' > src/detail.cppm printf 'module hello.greet:impl;\nint hidden() { return 7; }\n' > src/impl.cppm printf 'module hello.greet;\nimport :impl;\nstd::string greet() { return "hi"; }\n' > src/greet_impl.cpp -printf 'import hello.greet;\nimport std;\nint main( { std::println("{}", greet()); }\n' > src/main.cpp -printf '#include \nint main() { return DEVKIT_MARKER - 1; }\n' > tests/test_smoke.cpp +printf 'import hello.greet;\nimport std;\nint main() { std::println("{}", greet()); }\n' > src/main.cpp +printf '#include \n/*\nimport in.comment;\n*/\nconst char* text = R"x(\nimport in.raw;\n)x";\nint main() { return DEVKIT_MARKER - 1; }\n' > tests/test_smoke.cpp tree_digest() { "$PY" - "$TMP" <<'EOF' @@ -157,6 +163,36 @@ assert std[0]["source"] in std[0]["arguments"] or any(a.endswith(os.path.basenam EOF echo "ok: D, roles and sets; E, the std unit" +# ── J, K ─────────────────────────────────────────────────────────────────── +"$PY" - "$OUT/env.json" <<'EOF' || fail "J/K: argument decomposition, config-files, test scan" "$OUT/env.json" +import json, os, sys +db = json.load(open(sys.argv[1]))["data"]["database"] +def same(word, path, cwd): + return os.path.normpath(os.path.join(cwd, word)) == os.path.normpath(path) +bypassed = False +for s in db["sets"]: + base = s["baseline-arguments"] + for u in s["translation-units"]: + a, local = u["arguments"], u["local-arguments"] + head = [a[0]] + base + local + assert a[:len(head)] == head, (s["name"], u["source"], base, local, a) + tail = a[len(head):] + assert tail == [] or (len(tail) == 4 and tail[0] == "-c" and tail[2] == "-o" + and same(tail[1], u["source"], u["work-directory"]) + and same(tail[3], u["object"], u["work-directory"])), (u["source"], tail) + assert u["private"] is False, u + bypassed = bypassed or "--no-default-config" in a +hello = next(s for s in db["sets"] if s["name"] == "hello") +assert "-std=c++23" in hello["baseline-arguments"] or "/std:c++latest" in hello["baseline-arguments"], hello["baseline-arguments"] +for tid, t in db["ide"]["toolchains"].items(): + for f in t["config-files"]: + assert os.path.isabs(f) and os.path.isfile(f), (tid, f) + assert not (bypassed and f.endswith(".cfg")), (tid, f) +test = next(u for s in db["sets"] if s["name"] == "hello:test" for u in s["translation-units"]) +assert test["requires"] == [], test["requires"] +EOF +echo "ok: J, baseline and local arguments, private, config-files; K, the test's imports" + # ── F (the S1 and compile-commands renderings agree) ────────────────────── "$MCPP" emit build-database --spec compile-commands > "$OUT/cc.json" 2> "$OUT/cc.err" \ || fail "F: --spec compile-commands exited non-zero" "$OUT/cc.err" @@ -231,18 +267,22 @@ print(obj[: norm.index("/target/")]) [ "$(tree_digest)" != "$before" ] || fail "B control: configure-only did not change the tree" [ -f compile_commands.json ] || fail "F: configure-only wrote no compile_commands.json" "$OUT/conf.out" "$PY" - "$OUT/cc.json" compile_commands.json <<'EOF' || fail "F: compile-commands differs from configure-only" compile_commands.json -import json, re, sys +import json, sys emitted = json.load(open(sys.argv[1])) written = json.load(open(sys.argv[2])) # The one difference by construction is where the build writes: the planning # pass writes under its work directory, configure-only under the project. +def slash(text): + # One spelling for the comparison: a Windows argument may name a path with + # either separator, and the mapping below is textual. + return text.replace("\\", "/") def write_root(entry): - return re.split(r"[\\/]target[\\/]", entry["output"])[0] + return slash(entry["output"]).split("/target/")[0] work, project = write_root(emitted[0]), write_root(written[0]) def mapped(args): - return [a.replace(work, project) for a in args] -e = {x["file"]: mapped(x["arguments"]) for x in emitted} -w = {x["file"]: x["arguments"] for x in written} + return [slash(a).replace(work, project) for a in args] +e = {slash(x["file"]): mapped(x["arguments"]) for x in emitted} +w = {slash(x["file"]): [slash(a) for a in x["arguments"]] for x in written} assert set(e) <= set(w), (sorted(e), sorted(w)) for f in e: assert e[f] == w[f], (f, e[f], w[f]) From daae1294c99f45f874ec046bd3c96d440ed453f6 Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:36:43 +0800 Subject: [PATCH 10/16] test(e2e): 687 plans in a home with its own configuration The macOS runner's inherited configuration names its ~/.xlings shim as the xlings binary, so nothing was vendored and the precondition failed before either criterion ran. The home now uses its own configuration, whose binary is the vendored one. Refs #636 --- .../687_the_vendored_xlings_probe_is_an_argument_vector.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e/687_the_vendored_xlings_probe_is_an_argument_vector.sh b/tests/e2e/687_the_vendored_xlings_probe_is_an_argument_vector.sh index 8d4280e9..c5aa6b00 100755 --- a/tests/e2e/687_the_vendored_xlings_probe_is_an_argument_vector.sh +++ b/tests/e2e/687_the_vendored_xlings_probe_is_an_argument_vector.sh @@ -33,7 +33,10 @@ export MCPP_HOME="$TMP/mcpp-home" # Offline: configuration loading still acquires the vendored xlings (it copies # a local binary), and no bootstrap step reaches the network. export MCPP_OFFLINE=1 -source "$(dirname "$0")/_inherit_toolchain.sh" +# The home's own configuration, whose xlings binary is the vendored one. An +# inherited configuration may name another (the macOS runner's names its +# `~/.xlings` shim), and then nothing is vendored and nothing is probed. +MCPP_INHERIT_CONFIG=0 source "$(dirname "$0")/_inherit_toolchain.sh" cd "$TMP" VENDORED="$MCPP_HOME/registry/bin/xlings$EXE" From b29f19cab343008b89642ebc3d193cec5e329c97 Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:41:44 +0800 Subject: [PATCH 11/16] chore: bump version to 2026.9.15.1 mcpp.toml and MCPP_VERSION move together; the bootstrap pin stays at the released 2026.9.14.3. The changelog names the release on the three entries that land in it, and records the S1 fields and the entry-source scan. Refs #636 --- ...36-build-database-and-the-latest-xlings.md | 2 +- CHANGELOG.md | 19 +++++++++++++++++-- mcpp.toml | 2 +- modules/versioning/src/version.cppm | 2 +- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md b/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md index 7bf59779..3e2d7a23 100644 --- a/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md +++ b/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md @@ -430,7 +430,7 @@ question for lsp-mcpp. | M3 | mcpp: the five engine changes of 5.3 | none | | M4 | mcpp: `emit build-database` (renderer, `watch`, fingerprint, lock comparison, effects, kind) | M3 | | M5 | mcpp: unit, contract and e2e tests for C, with the vendored S1 schema and a validator that refuses keywords it does not implement | M4 | -| M6 | mcpp: SPEC-005, docs/50 and its translation, docs/09, the three specification indexes | M4 | +| M6 | mcpp: SPEC-005, docs/50 and docs/01 with their translations, the three specification indexes | M4 | | M7 | mcpp: `kXlingsVersion = 2026.9.14.1` | X5 | | M8 | mcpp: #639 renamed; CI green; self-review; merge | M1 to M7 | | R1 | mcpp release; mirrors (GitCode completed with local `gtc`); index merge | M8 | diff --git a/CHANGELOG.md b/CHANGELOG.md index f2a29f23..81687642 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ## [Unreleased] -### `mcpp emit build-database`:不写工程目录的构建数据库(#636) +### `mcpp emit build-database`:不写工程目录的构建数据库(#636,2026.9.15.1) 新命令按 `mcpp build --configure-only` 的方式、用相同的选择器规划,把计划打印为 S1「C++ Build Database: IDE Profile」0.2.0(WG21 P2977R2 的一个 profile)文档,不写入 @@ -21,6 +21,10 @@ SPEC-005(`docs/specs/build-database.md`)。 模块只描述不编译(`describe_std_module`);`mcpp.lock` 从工程读取、从不写回,不一致时 给出 `MCPP_LOCK_WOULD_CHANGE` 警告。 - 标准库模块单元的命令从 mcpp 实际运行的构建命令中还原,std 缓存目录的身份不变。 +- S1 的 SHOULD 字段取自计划:单元的 `arguments` 依次是驱动、集合的 + `baseline-arguments`(集合内最长公共前缀)、单元的 `local-arguments` 与结尾的 + `-c -o `;`private` 为 `false`;`config-files` 列出驱动在命令行之外 + 读取的文件(未被 `--no-default-config` 绕过的 clang `.cfg`,GCC 库目录中的 `specs`)。 - 信封增加约定:失败时省略 `data`。`--protocol-version` 声明该命令的效应,不含 `write-project`。 - 测试:单测 `BuildDatabase.*`、`Scanner.DeclarationFormIsRecordedAsRead`、 @@ -28,7 +32,18 @@ SPEC-005(`docs/specs/build-database.md`)。 `WireGolden.NullDataIsOmitted`;e2e 688 以随仓的 S1 schema 校验输出,并以 `--configure-only` 作对照腿证明工程目录未被写入。 -### Windows 上不再每条命令都打印 "The system cannot find the path specified." +### 未被 `sources` 匹配的目标入口由扫描器读取(2026.9.15.1) + +发现的测试与 glob 之外的 `main` 不在包扫描中,它们的 import 此前只按行首的 `import` +读取(`make_plan` 与规划前的标准库检查各一份),注释与原始字符串中的 `import` 也被当作 +导入。以 S1 校验 lsp-mcpp 仓库自身的构建数据库时发现:其扫描器测试被规划为导入三个 +没有任何源文件提供的模块。现在两处都经 `scan_entry_file` 读取入口:扫描器接受的文件 +取扫描器的结果;扫描器拒绝的文件(`#if` 块中的 `import`、头文件单元)在这条路径上从未 +被拒绝,现在也不被拒绝,取其代码中行首的 import,声明形式为 `Unknown`。(单测 +`Scanner.AnEntrySourceIsReadByTheScanner`、`Scanner.AnEntrySourceTheScannerRefusesKeepsItsImports`, +e2e 688 判据 K) + +### Windows 上不再每条命令都打印 "The system cannot find the path specified."(2026.9.15.1) vendored xlings 的版本探针以命令串 ` --version 2>/dev/null` 运行。Windows 上 命令串交给 cmd.exe,它打不开 `/dev/null`:除首次外的每条命令都打印这句提示,xlings diff --git a/mcpp.toml b/mcpp.toml index 55718edc..9cb3c4d6 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -1,6 +1,6 @@ [package] name = "mcpp" -version = "2026.9.14.3" +version = "2026.9.15.1" description = "Modern C++ build & package management tool" license = "Apache-2.0" authors = ["mcpp-community"] diff --git a/modules/versioning/src/version.cppm b/modules/versioning/src/version.cppm index c3333d7f..5f6a8855 100644 --- a/modules/versioning/src/version.cppm +++ b/modules/versioning/src/version.cppm @@ -31,6 +31,6 @@ import std; export namespace mcpp { -inline constexpr std::string_view MCPP_VERSION = "2026.9.14.3"; +inline constexpr std::string_view MCPP_VERSION = "2026.9.15.1"; } // namespace mcpp From 1aaad4559fc4f4cf1d1bbf3845e6de4b08f83673 Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:43:31 +0800 Subject: [PATCH 12/16] chore(xlings): pin 2026.9.14.1 kXlingsVersion and every pin check_version_pins.sh names move to the xlings release in which a package without an install hook receives its own archive (openxlings/xlings#596). The constant's comment records it as the third reason for the floor. Refs #636 --- .github/actions/bootstrap-mcpp/action.yml | 2 +- .github/actions/setup-macos-llvm/action.yml | 2 +- .github/workflows/bootstrap-macos.yml | 2 +- .github/workflows/ci-fresh-install.yml | 6 +++--- .github/workflows/ci-linux-e2e.yml | 2 +- .github/workflows/cross-build-test.yml | 4 ++-- .github/workflows/release.yml | 14 +++++++------- CHANGELOG.md | 9 +++++++++ src/xlings/xlings.cppm | 8 +++++++- 9 files changed, 32 insertions(+), 17 deletions(-) diff --git a/.github/actions/bootstrap-mcpp/action.yml b/.github/actions/bootstrap-mcpp/action.yml index c549b0ad..ee985a62 100644 --- a/.github/actions/bootstrap-mcpp/action.yml +++ b/.github/actions/bootstrap-mcpp/action.yml @@ -25,7 +25,7 @@ inputs: # `package.name`, so one of the two was simply unreachable — and which one # depended on the machine, which is why CI failed on `compat:lua` on # Windows and `mcpplibs.capi:lua` on Linux. Never pin below that. - default: '2026.9.5.1' + default: '2026.9.14.1' cache-target: description: also restore/save target/ (build artifacts + BMIs) required: false diff --git a/.github/actions/setup-macos-llvm/action.yml b/.github/actions/setup-macos-llvm/action.yml index a8871df7..63e4c81f 100644 --- a/.github/actions/setup-macos-llvm/action.yml +++ b/.github/actions/setup-macos-llvm/action.yml @@ -15,7 +15,7 @@ inputs: # Floor imposed by the index, not a routine bump — see # .github/actions/bootstrap-mcpp/action.yml for why 0.4.69 is required # (two packages named `lua` in one repo need openxlings/xlings#381). - default: '2026.9.5.1' + default: '2026.9.14.1' runs: using: composite diff --git a/.github/workflows/bootstrap-macos.yml b/.github/workflows/bootstrap-macos.yml index 9e177681..8076e513 100644 --- a/.github/workflows/bootstrap-macos.yml +++ b/.github/workflows/bootstrap-macos.yml @@ -17,7 +17,7 @@ jobs: # Dormant (workflow_dispatch only), but kept in step with the rest — # check_version_pins.sh holds it there. Floor: 0.4.69, below which the # index cannot resolve two packages that share a short name. - XLINGS_VERSION: '2026.9.5.1' + XLINGS_VERSION: '2026.9.14.1' steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/ci-fresh-install.yml b/.github/workflows/ci-fresh-install.yml index d3683070..dc88e9fd 100644 --- a/.github/workflows/ci-fresh-install.yml +++ b/.github/workflows/ci-fresh-install.yml @@ -152,7 +152,7 @@ jobs: env: XLINGS_NON_INTERACTIVE: '1' run: | - curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.sh | bash -s v2026.9.5.1 + curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.sh | bash -s v2026.9.14.1 echo "$HOME/.xlings/subos/current/bin" >> "$GITHUB_PATH" - name: Install mcpp and config mirror @@ -312,7 +312,7 @@ jobs: - name: Install xlings + mcpp run: | - curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.sh | bash -s v2026.9.5.1 + curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.sh | bash -s v2026.9.14.1 # Deliberately NOT writing to $GITHUB_PATH here. On container # images that declare no PATH in their config (opensuse/ # tumbleweed), appending a single dir to GITHUB_PATH makes the @@ -383,7 +383,7 @@ jobs: # (older ones carry minos=15 and refuse to start). # v0.4.51+: in-process sha256 — this image has no sha256sum # binary, so pinned fetches failed before it. - curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.sh | bash -s v2026.9.5.1 + curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.sh | bash -s v2026.9.14.1 echo "$HOME/.xlings/subos/current/bin" >> "$GITHUB_PATH" - name: Install mcpp and config mirror diff --git a/.github/workflows/ci-linux-e2e.yml b/.github/workflows/ci-linux-e2e.yml index 7ba672c7..9dfd7b30 100644 --- a/.github/workflows/ci-linux-e2e.yml +++ b/.github/workflows/ci-linux-e2e.yml @@ -384,7 +384,7 @@ jobs: - name: Bootstrap xlings + released mcpp run: | - curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.sh | bash -s v2026.9.5.1 + curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.sh | bash -s v2026.9.14.1 export PATH="$HOME/.xlings/subos/current/bin:$PATH" xlings update xlings install mcpp -y -g diff --git a/.github/workflows/cross-build-test.yml b/.github/workflows/cross-build-test.yml index 77ccf587..7e415a22 100644 --- a/.github/workflows/cross-build-test.yml +++ b/.github/workflows/cross-build-test.yml @@ -135,7 +135,7 @@ jobs: # release assets were uploaded in a broken state (records present, # blobs missing → 404 on GET); re-uploaded clean. The stale-INDEX # half is handled by the marker-clear below. - XLINGS_VERSION: '2026.9.5.1' + XLINGS_VERSION: '2026.9.14.1' run: | tarball="xlings-${XLINGS_VERSION}-linux-x86_64.tar.gz" bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ @@ -289,7 +289,7 @@ jobs: - name: Bootstrap mcpp via xlings env: XLINGS_NON_INTERACTIVE: '1' - XLINGS_VERSION: '2026.9.5.1' + XLINGS_VERSION: '2026.9.14.1' run: | tarball="xlings-${XLINGS_VERSION}-linux-x86_64.tar.gz" bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d8ec4b5e..7f86b65b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -96,7 +96,7 @@ jobs: # Pin xlings to a known-good version. The upstream install # script always grabs `latest` (no version override), so we # download + self-install manually to avoid broken releases. - XLINGS_VERSION: '2026.9.5.1' + XLINGS_VERSION: '2026.9.14.1' run: | if [ ! -x "$HOME/.xlings/subos/default/bin/xlings" ]; then tarball="xlings-${XLINGS_VERSION}-linux-x86_64.tar.gz" @@ -302,7 +302,7 @@ jobs: - name: Bootstrap mcpp via xlings env: XLINGS_NON_INTERACTIVE: '1' - XLINGS_VERSION: '2026.9.5.1' + XLINGS_VERSION: '2026.9.14.1' run: | tarball="xlings-${XLINGS_VERSION}-linux-x86_64.tar.gz" bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ @@ -373,7 +373,7 @@ jobs: # below are pinned to the same version as XLINGS_VERSION; they are # NOT interpolated from it, so check_version_pins.sh scans for them # explicitly (they were absent from the old lock-step comment). - XLA="xlings-2026.9.5.1-linux-aarch64.tar.gz" + XLA="xlings-2026.9.14.1-linux-aarch64.tar.gz" # NOT fetch_release.sh: this asset is OPTIONAL and the `if` is the # point — an arch with no prebuilt xlings must fall through quietly, # while the helper retries a 404 five times before giving up. The one @@ -382,9 +382,9 @@ jobs: # cover it. if curl -fsSL --retry 3 --retry-delay 2 --retry-all-errors \ --connect-timeout 20 --max-time 600 -o "/tmp/$XLA" \ - "https://github.com/openxlings/xlings/releases/download/v2026.9.5.1/$XLA"; then + "https://github.com/openxlings/xlings/releases/download/v2026.9.14.1/$XLA"; then tar -xzf "/tmp/$XLA" -C /tmp - XLBIN=$(find /tmp/xlings-2026.9.5.1-linux-aarch64 -path '*/bin/xlings' -type f | head -1) + XLBIN=$(find /tmp/xlings-2026.9.14.1-linux-aarch64 -path '*/bin/xlings' -type f | head -1) if [ -n "$XLBIN" ]; then mkdir -p "$STAGING/$WRAPPER/registry/bin" cp "$XLBIN" "$STAGING/$WRAPPER/registry/bin/xlings" @@ -462,7 +462,7 @@ jobs: - name: Bootstrap mcpp via xlings env: XLINGS_NON_INTERACTIVE: '1' - XLINGS_VERSION: '2026.9.5.1' + XLINGS_VERSION: '2026.9.14.1' run: | if [ ! -x "$HOME/.xlings/subos/default/bin/xlings" ]; then WORK=$(mktemp -d) @@ -645,7 +645,7 @@ jobs: shell: bash env: XLINGS_NON_INTERACTIVE: '1' - XLINGS_VERSION: '2026.9.5.1' + XLINGS_VERSION: '2026.9.14.1' run: | # Captured before the `cd` below, in POSIX form: this step never # returns to the workspace, and GITHUB_WORKSPACE is a backslash diff --git a/CHANGELOG.md b/CHANGELOG.md index 81687642..bf702c16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,15 @@ SPEC-005(`docs/specs/build-database.md`)。 `WireGolden.NullDataIsOmitted`;e2e 688 以随仓的 S1 schema 校验输出,并以 `--configure-only` 作对照腿证明工程目录未被写入。 +### 内置 xlings 升至 2026.9.14.1(2026.9.15.1) + +此前没有 `install()` 的包在安装时得到的是整个下载目录,而不是它自己的归档:宿主机上 +openkal 0.8.0 的包目录因此多占 1.6 G 其他包的下载物。xlings 2026.9.14.1 让这类包在 +私有目录中解压自己的归档,并由 `xlings self doctor` 报告已被污染的包目录、`--fix` +重装(openxlings/xlings#596)。已有的 registry 用 +`XLINGS_HOME= xlings self doctor --fix` 修复。`kXlingsVersion` 与 +`check_version_pins.sh` 列出的全部 pin 随之前移。 + ### 未被 `sources` 匹配的目标入口由扫描器读取(2026.9.15.1) 发现的测试与 glob 之外的 `main` 不在包扫描中,它们的 import 此前只按行首的 `import` diff --git a/src/xlings/xlings.cppm b/src/xlings/xlings.cppm index 9afb7f87..20eca1e7 100644 --- a/src/xlings/xlings.cppm +++ b/src/xlings/xlings.cppm @@ -83,7 +83,13 @@ namespace pinned { // correctly — which is why this is a floor and not a preference, and why // an index must still not publish a deliberately colliding // `@` (see .agents/docs/2026-08-30-cross-repo-fix-plan §1). - inline constexpr std::string_view kXlingsVersion = "2026.9.5.1"; + // + // Third, at 2026.9.14.1 (mcpp#636). Below it, a package without an + // `install()` received the whole download directory rather than its own + // archive: on a host store, openkal 0.8.0 held 1.6 G of other packages' + // downloads. A registry that already holds such a payload is repaired by + // `XLINGS_HOME= xlings self doctor --fix`. + inline constexpr std::string_view kXlingsVersion = "2026.9.14.1"; inline constexpr std::string_view kNasmVersion = "3.02"; } From 653eb1f984d73641e2741dcc0336c38f3125365d Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:50:56 +0800 Subject: [PATCH 13/16] docs(design): the execution record of #636 What landed in each repository, the five findings made while implementing, and the measurements taken so far, including lsp-mcpp's validator and its conformance runner against this mcpp. Refs #636 --- ...36-build-database-and-the-latest-xlings.md | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md b/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md index 3e2d7a23..94ff9a55 100644 --- a/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md +++ b/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md @@ -480,3 +480,49 @@ on another repository, and it is small. | consistency | One envelope builder, one selector parser, one unit record, one std command source | none | | seamless upgrade | A pin move reaches an existing Windows home only through B's fix; swept stores keep working and are repaired on request | B and M7 ship in one pull request; `self doctor --fix` | | test coverage | Each criterion has a leg that fails on the released binary; the schema validator refuses keywords it does not implement, so a schema change cannot pass unvalidated | M2 and M5 | + +## 9. Execution record + +### 9.1 What landed, by repository + +| repository | vehicle | content | +|---|---|---| +| openxlings/xlings | #596, squash `3cd8061`, version 2026.9.14.1 | 2.3 to 2.6: the §6 sentence, private extraction, the strip branch removed, `self doctor` finding `SweptPayload` with remove-then-reinstall, e2e for the rule and the doctor | +| mcpp-community/mcpp | #639, version 2026.9.15.1 | B, C, the pin of §4, and the findings of 9.2 | + +### 9.2 Found during implementation + +1. **A digest string that GCC accepted and clang refused.** The selector digest + joined values with `"\x1f"` followed by a letter (`"\x1ffeatures"`), which + is one out-of-range hex escape. GCC truncated it; clang on macOS and Windows + refused the file. The separator is now a newline. +2. **The four S1 SHOULD fields.** lsp-mcpp's validator (`specs/tools/validate.py`, + `s1_semantics`) requires `config-files`, `baseline-arguments`, + `local-arguments` and `private`, which the first rendering omitted. Each is + now derived from the plan (5.4). +3. **An entry source read by a second parser.** Validating lsp-mcpp's own + database found a scanner test planned as importing `also.fake`, `in.raw` and + `real`, which are text inside its raw string literal. The entry of a target + that no `sources` glob matched was read by line-leading `import`, in two + places. Both now call `scan_entry_file` (5.4). A test source placed in + `src/` did not reproduce it, which is why the fixtures had never shown it. +4. **A precondition met by one runner's configuration.** e2e 687 inherited + the developer configuration, and the macOS runner's configuration names its + `~/.xlings` shim as the xlings binary: nothing was vendored and the test + stopped before either criterion. It now plans in a home with its own + configuration. +5. **The swept-payload fingerprint.** The first xlings rendering flagged any + top-level file with a download extension or a `.meta` name, so a package + shipping `setup.exe` would be reinstalled on every `--fix`. The fingerprint + is now a zero-length `.lock` whose `` is a sibling, has a download + extension, or has a `.meta` sibling. + +### 9.3 Measurements + +| run | reading | +|---|---| +| lsp-mcpp `s1_semantics` and `mcpp_contract`, schema, argument decomposition | 0 failures on a GCC 16 project, an LLVM 22 project, and the lsp-mcpp repository (11 sets, `std` from openkal-llvm-runtime 0.9.6) | +| lsp-mcpp conformance runner with this mcpp in place of `lsp-mcpp-mock-mcpp` | `mcpp-emit` 12/12, `mcpp-emit-package-std` 13/13; a watch scenario on the real manifest 10/10 (an edit reloads; a broken manifest leaves the model stale with `MCPP_BUILD_DATABASE_PLAN_FAILED`; the repair makes it fresh) | +| `mcpp emit build-database` on the lsp-mcpp repository | 0.84 s; `git status` unchanged | +| e2e 687 on windows-2022 (39053b9a) | A and B pass: the second command prints no path error, and an older vendored xlings is replaced | +| e2e subset touching `mcpp test` and entry mains, Linux | 60 pass, 6 skipped for capability, 0 fail | From 7907e3a6c1bf5e5d469183a56f2954635cbc74f7 Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:52:30 +0800 Subject: [PATCH 14/16] docs(verify): a doctor control, the lock file named, and a dependency project Section D reads a clean store first, then requires the finding's label and the seeded lock file's name, and after --fix requires the reinstalled payload: no seeded file and */mcpp.toml present. Section C2 emits the build database of section B's project: the dependency's set, no watched store path, the S1 SHOULD fields, and an unchanged tree with target/ present. Sections that need python3 say so when it is missing. Refs #636 --- .agents/docs/2026-09-14-636-verify.sh | 58 ++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/.agents/docs/2026-09-14-636-verify.sh b/.agents/docs/2026-09-14-636-verify.sh index b2665ba0..4257571d 100644 --- a/.agents/docs/2026-09-14-636-verify.sh +++ b/.agents/docs/2026-09-14-636-verify.sh @@ -89,7 +89,9 @@ printf 'module hello.greet;\nstd::string greet() { return "hi"; }\n' > "$d/src/g printf 'import hello.greet;\nimport std;\nint main() { std::println("{}", greet()); }\n' > "$d/src/main.cpp" printf 'int main() { return 0; }\n' > "$d/tests/test_smoke.cpp" before=$(cd "$d" && find . -type f | sort | xargs sha256sum | sha256sum) -if (cd "$d" && "$STORE" emit build-database --format json > "$root/c.json" 2> "$root/c.err"); then +if ! command -v python3 >/dev/null 2>&1; then + skip "C: no python3 in the sandbox to read the document" +elif (cd "$d" && "$STORE" emit build-database --format json > "$root/c.json" 2> "$root/c.err"); then after=$(cd "$d" && find . -type f | sort | xargs sha256sum | sha256sum) [ "$before" = "$after" ] && ok "the project tree is unchanged" || fail "the project tree changed" if python3 - "$root/c.json" <<'EOF' @@ -112,14 +114,60 @@ else fail "emit build-database failed"; tail -5 "$root/c.err" fi +section "C2. the build database of a project with an mcpp-index dependency" +d=$root/b +if [ ! -f "$d/mcpp.toml" ] || ! command -v python3 >/dev/null 2>&1; then + skip "C2: section B's project or python3 is missing" +else + before=$(cd "$d" && find . -type f | sort | xargs sha256sum | sha256sum) + if (cd "$d" && "$STORE" emit build-database --format json > "$root/c2.json" 2> "$root/c2.err"); then + after=$(cd "$d" && find . -type f | sort | xargs sha256sum | sha256sum) + [ "$before" = "$after" ] && ok "the built project's tree is unchanged, target/ included" || fail "the built project's tree changed" + if python3 - "$root/c2.json" "$REG" <<'EOF' +import json, os, sys +e = json.load(open(sys.argv[1])); reg = os.path.realpath(sys.argv[2]) +db, watch = e["data"]["database"], e["data"]["watch"] +sets = {s["name"]: s for s in db["sets"]} +assert {"eco636", "mcpplibs.cmdline", "mcpp:std"} <= set(sets), sorted(sets) +assert all(not os.path.realpath(w).startswith(reg) for w in watch if os.path.isabs(w)), watch +for tid, t in db["ide"]["toolchains"].items(): + assert all(os.path.isfile(f) for f in t["config-files"]), (tid, t["config-files"]) +for s in db["sets"]: + for u in s["translation-units"]: + head = [u["arguments"][0]] + s["baseline-arguments"] + u["local-arguments"] + assert u["arguments"][:len(head)] == head and u["private"] is False, (s["name"], u["source"]) +print("sets", sorted(sets), "watch", len(watch)) +EOF + then ok "the document has the dependency's set, watches no store path, and carries the S1 SHOULD fields" + else fail "the dependency project's document"; fi + else + fail "emit build-database failed on the dependency project"; tail -5 "$root/c2.err" + fi +fi + section "D. xlings reports and repairs a payload that holds another package's download" cmd=$(find "$REG/data/xpkgs" -maxdepth 2 -path '*cmdline*' -type d | tail -1) +strip_ansi() { sed 's/\x1b\[[0-9;]*[A-Za-z]//g'; } if [ -n "$cmd" ]; then + # The control: the store section B produced has no finding of this kind. + XLINGS_HOME="$REG" "$REG/bin/xlings" self doctor 2>&1 | strip_ansi > "$root/doctor0.log" + grep -qi 'swept payload' "$root/doctor0.log" && { fail "self doctor reports a swept payload before any was seeded"; grep -i -A2 'swept' "$root/doctor0.log" | head -6; } || ok "the fresh store has no swept-payload finding" + # The downloader's shape: an archive beside its zero-length lock. printf 'x' > "$cmd/intruder-1.0-linux-x86_64.tar.gz"; : > "$cmd/intruder-1.0-linux-x86_64.tar.gz.lock" - XLINGS_HOME="$REG" "$REG/bin/xlings" self doctor > "$root/doctor.log" 2>&1 - grep -qi 'intruder\|swept\|download' "$root/doctor.log" && ok "self doctor reports the swept payload" || { fail "self doctor did not report it"; tail -8 "$root/doctor.log"; } - XLINGS_HOME="$REG" "$REG/bin/xlings" self doctor --fix > "$root/fix.log" 2>&1 - [ ! -e "$cmd/intruder-1.0-linux-x86_64.tar.gz.lock" ] && ok "self doctor --fix leaves no download sidecar in the payload" || { fail "the sidecar is still there after --fix"; tail -8 "$root/fix.log"; } + XLINGS_HOME="$REG" "$REG/bin/xlings" self doctor 2>&1 | strip_ansi > "$root/doctor.log" + if grep -qi 'swept payload' "$root/doctor.log" && grep -qF "intruder-1.0-linux-x86_64.tar.gz.lock" "$root/doctor.log"; then + ok "self doctor reports the swept payload and names the lock file" + else + fail "self doctor did not report the seeded payload"; tail -12 "$root/doctor.log" + fi + XLINGS_HOME="$REG" "$REG/bin/xlings" self doctor --fix 2>&1 | strip_ansi > "$root/fix.log" + cmd=$(find "$REG/data/xpkgs" -maxdepth 2 -path '*cmdline*' -type d | tail -1) + if [ -n "$cmd" ] && [ ! -e "$cmd/intruder-1.0-linux-x86_64.tar.gz.lock" ] && [ ! -e "$cmd/intruder-1.0-linux-x86_64.tar.gz" ] \ + && [ -n "$(find "$cmd" -mindepth 2 -maxdepth 2 -name mcpp.toml)" ]; then + ok "self doctor --fix reinstalls the payload: no seeded file, and */mcpp.toml is back" + else + fail "the payload after --fix"; ls -A "$cmd" 2>/dev/null | head; tail -12 "$root/fix.log" + fi else skip "D: section B produced no cmdline payload to seed" fi From db74e783a6596395b5e50459a977ef03068620bd Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:54:59 +0800 Subject: [PATCH 15/16] docs(specs): SPEC-005's index row carries its last modification date Refs #636 --- docs/specs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/specs/README.md b/docs/specs/README.md index 4e498e77..7cb237f8 100644 --- a/docs/specs/README.md +++ b/docs/specs/README.md @@ -34,7 +34,7 @@ | [SPEC-002](target-side.md) | 目标侧模型与能力声明(`mcpp:` 保留命名空间、五层、三条规则) | 评审中 v1.0 | 2026-08-24 | mcpp >= 2026.8.24.2 | | [SPEC-003](exit-codes.md) | 退出码契约(分类、语义、稳定性承诺) | 评审中 v1.0 | 2026-09-01 | mcpp >= 2026.9.1.1 | | [SPEC-004](manifest-semantics.md) | `mcpp.toml` 的平面划分、条件化形状、解析轴与命名规约 | 草案 v1.0 | 2026-09-07 | 条件化形状 mcpp >= 2026.8.29.1;目标轴未实现 | -| [SPEC-005](build-database.md) | 构建数据库:`mcpp emit build-database` 的内容、取值规则与不写工程目录的保证 | 评审中 v1.0 | 2026-09-14 | mcpp >= 2026.9.15.1 | +| [SPEC-005](build-database.md) | 构建数据库:`mcpp emit build-database` 的内容、取值规则与不写工程目录的保证 | 评审中 v1.0 | 2026-09-15 | mcpp >= 2026.9.15.1 | ## 文档约定 From 2f5b71c44b44daa7dad4a773e5d636088577421f Mon Sep 17 00:00:00 2001 From: speak-agent <248744407+speak-agent@users.noreply.github.com> Date: Tue, 15 Sep 2026 01:08:26 +0800 Subject: [PATCH 16/16] fix(emit): a plan that builds nothing writes no link specs prepare_build ran `g++ -dumpspecs` to write mcpp-clean-link.specs under plan_only. The file is read only by the link line, so the planning pass neither needs it nor should run the driver for it, and the compile arguments are the same without it (e2e 688 F still compares them with configure-only's). Found by an independent review of #639. e2e 688 criterion L measures what SPEC-005 R2.2 states: the fresh work directory holds the resolution record and no object, BMI or link specs, and a home whose build cache is empty stays without objects or BMIs while its document lists the std unit. Its control leg runs configure-only on a copy in the same home and sees the std module compiled. R2.2 now says what is true: nothing is compiled, and the toolchain is still queried as for `mcpp build`. Refs #636 --- ...36-build-database-and-the-latest-xlings.md | 21 ++++++--- docs/specs/build-database.md | 5 ++- src/build/prepare.cppm | 5 ++- tests/e2e/688_emit_build_database.sh | 45 +++++++++++++++++++ 4 files changed, 68 insertions(+), 8 deletions(-) diff --git a/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md b/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md index 94ff9a55..4c453ef7 100644 --- a/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md +++ b/.agents/docs/2026-09-14-636-build-database-and-the-latest-xlings.md @@ -387,10 +387,14 @@ disagree about `import std`. source when run in its `work-directory`. 5. A project with a syntax error, and a project never built, both produce a document. -6. With mcpp's own tree as input (it has no build program), no compiler process - starts, on a cold std cache and on a warm one; a driver wrapper that fails - when invoked is the probe. The wall time on a warm cache is of the order of - `--configure-only`. +6. Nothing is compiled and no link input is written, on a cold std cache and on + a warm one: the planning pass's work directory holds no object, BMI or + `mcpp-clean-link.specs`, and a home whose build cache is empty gains no object + or BMI, while `--configure-only` in the same home compiles the std module (the + control). The driver still answers the queries toolchain resolution makes + (`--version`, `-dumpmachine`, `-print-sysroot`), as it does for `mcpp build`, + so a driver wrapper that fails whenever it is invoked cannot be the probe. + The wall time on a warm cache is of the order of `--configure-only`. 7. `tests/unit/test_wire.cpp` pins the kind's key set; `--format ndjson` exits 2 with empty stdout. @@ -511,7 +515,14 @@ on another repository, and it is small. `~/.xlings` shim as the xlings binary: nothing was vendored and the test stopped before either criterion. It now plans in a home with its own configuration. -5. **The swept-payload fingerprint.** The first xlings rendering flagged any +5. **A link input written by a plan that links nothing.** The independent review + of #639 found `prepare_build` running `g++ -dumpspecs` to write + `mcpp-clean-link.specs` under `plan_only`. Criterion 6 as first written + (a driver wrapper that fails when invoked) could not have caught it: toolchain + resolution queries the driver on every plan. The call is now skipped under + `plan_only`, the link specs being read only by the link line, and e2e 688 + criterion L measures the work directory and a cold home instead. +6. **The swept-payload fingerprint.** The first xlings rendering flagged any top-level file with a download extension or a `.meta` name, so a package shipping `setup.exe` would be reinstalled on every `--fix`. The fingerprint is now a zero-length `.lock` whose `` is a sibling, has a download diff --git a/docs/specs/build-database.md b/docs/specs/build-database.md index f33796bb..53f9bd88 100644 --- a/docs/specs/build-database.md +++ b/docs/specs/build-database.md @@ -44,8 +44,9 @@ Database 定义,本规范不重复它们的字段定义,只规定 mcpp 作为生 - **R2.1** 命令**禁止**写入工程目录,即根包、工作区成员与 path 依赖的源码树。规划 写入 `$MCPP_HOME/cache/build-database/`,`` 由工程根与成员决定。该目录 是缓存,可以随时删除。**已实现** -- **R2.2** 标准库模块被描述而不被编译。对没有构建程序的工程,命令不启动任何编译器。 - **已实现** +- **R2.2** 命令不编译:标准库模块被描述而不被编译,也不生成只供链接使用的输入(GCC 的 + `mcpp-clean-link.specs`)。工具链照常被查询(版本、目标三元组、sysroot 等),与 + `mcpp build` 相同;对没有构建程序的工程,驱动只为这些查询运行。**已实现** - **R2.3** `mcpp.lock` 从工程根读取,从不写回。规划得出的解析与工程中的锁不一致, 或工程中没有锁而规划会写出一份时,输出警告 `MCPP_LOCK_WOULD_CHANGE`。**已实现** - **R2.4** 根包 `[build] generated_files` 中缺失或内容与声明不一致的文件不被写入, diff --git a/src/build/prepare.cppm b/src/build/prepare.cppm index f648de66..041f90e2 100644 --- a/src/build/prepare.cppm +++ b/src/build/prepare.cppm @@ -11895,7 +11895,10 @@ prepare_build(bool print_fingerprint, // moved in — an earlier assignment was silently overwritten by that move, // which produced a generated file that nothing ever passed to the driver. // Generated here rather than in compute_flags, which runs twice per build. - if (tc->compiler == mcpp::toolchain::CompilerId::GCC) + // A link input only: a plan that builds nothing (`plan_only`) neither reads + // it nor runs the driver to produce it, and its compile arguments are the + // same without it. + if (tc->compiler == mcpp::toolchain::CompilerId::GCC && !overrides.plan_only) ctx.plan.gccCleanSpecs = mcpp::toolchain::write_clean_link_specs( tc->binaryPath, ctx.outputDir); diff --git a/tests/e2e/688_emit_build_database.sh b/tests/e2e/688_emit_build_database.sh index b15cf48e..0ccffd86 100755 --- a/tests/e2e/688_emit_build_database.sh +++ b/tests/e2e/688_emit_build_database.sh @@ -29,6 +29,12 @@ # the units bypass with `--no-default-config`. # K. The discovered test is scanned as a package source is: the imports inside # its comment and its raw string are not in `requires`. +# L. The planning pass compiles nothing and writes no link input: its work +# directory, fresh for this project, holds the resolution record and no +# object, BMI or `mcpp-clean-link.specs`; in a home whose std cache is cold +# the document still lists the std unit and the home gains no object or +# BMI. The control leg at the end runs `build --configure-only` on a copy +# in that home and sees the std module compiled there. set -e TMP=$(mktemp -d) # the measured tree: the project and its dev-dependency @@ -262,6 +268,34 @@ norm = obj.replace("\\", "/") print(obj[: norm.index("/target/")]) ' "$OUT/env.json") +# ── L ────────────────────────────────────────────────────────────────────── +[ -n "$(find "$WORK_DIR" -type f -name resolution.json 2>/dev/null)" ] \ + || fail "L: no resolution record under the work directory $WORK_DIR" +written=$(find "$WORK_DIR" -type f \( -name '*.o' -o -name '*.obj' -o -name '*.gcm' \ + -o -name '*.pcm' -o -name '*.ifc' -o -name 'mcpp-clean-link.specs' \) | head -5) +[ -z "$written" ] || fail "L: the planning pass wrote compile or link outputs: $written" + +# A home with the machine's toolchain payloads and an empty build cache. +COLD="$OUT/cold-home" +compiled_in_cold_home() { + find "$COLD" -path "$COLD/registry" -prune -o -type f \( -name '*.o' -o -name '*.obj' \ + -o -name '*.gcm' -o -name '*.pcm' -o -name '*.ifc' \) -print | head -5 +} +( + export MCPP_HOME="$COLD" MCPP_OFFLINE=1 + source "$HERE/_inherit_toolchain.sh" + "$MCPP" emit build-database --format json > "$OUT/cold.json" 2> "$OUT/cold.err" +) || fail "L: emit in a home with a cold std cache" "$OUT/cold.err" +"$PY" - "$OUT/cold.json" <<'EOF' || fail "L: the cold-home document" "$OUT/cold.json" +import json, sys +db = json.load(open(sys.argv[1]))["data"]["database"] +std = [u for s in db["sets"] if s["name"] == "mcpp:std" for u in s["translation-units"] if "std" in u["provides"]] +assert len(std) == 1, db["sets"] +EOF +written=$(compiled_in_cold_home) +[ -z "$written" ] || fail "L: emit compiled into a cold home: $written" "$OUT/cold.err" +echo "ok: L, the planning pass wrote no object, BMI or link input, and a cold std cache stays cold" + # ── F (against configure-only) and the control leg of B ─────────────────── "$MCPP" build --configure-only > "$OUT/conf.out" 2>&1 || fail "F: configure-only" "$OUT/conf.out" [ "$(tree_digest)" != "$before" ] || fail "B control: configure-only did not change the tree" @@ -288,3 +322,14 @@ for f in e: assert e[f] == w[f], (f, e[f], w[f]) EOF echo "ok: F, the arguments are configure-only's; B control, configure-only writes the project" + +# ── the control leg of L ────────────────────────────────────────────────── +mkdir -p "$OUT/copy" +cp -R "$TMP/hello" "$TMP/devkit" "$OUT/copy/" +rm -rf "$OUT/copy/hello/target" "$OUT/copy/hello/compile_commands.json" +( + export MCPP_HOME="$COLD" MCPP_OFFLINE=1 + cd "$OUT/copy/hello" && "$MCPP" build --configure-only > "$OUT/cold-conf.out" 2>&1 +) || fail "L control: configure-only in the cold home" "$OUT/cold-conf.out" +[ -n "$(compiled_in_cold_home)" ] || fail "L control: configure-only compiled nothing into the home" "$OUT/cold-conf.out" +echo "ok: L control, configure-only compiles the std module into the same home"