diff --git a/.agents/docs/2026-05-21-linux-sysroot-missing-kernel-headers.md b/.agents/docs/2026-05-21-linux-sysroot-missing-kernel-headers.md new file mode 100644 index 0000000..01f5391 --- /dev/null +++ b/.agents/docs/2026-05-21-linux-sysroot-missing-kernel-headers.md @@ -0,0 +1,370 @@ +# Linux sysroot 缺少内核头文件导致 std module 预编译失败 + +## 现象 + +在用户机器上（非 CI），使用 LLVM 或 GCC 工具链执行 `mcpp run` / `mcpp build` 时， +std module 预编译失败： + +``` +/home/speak/.mcpp/registry/subos/default/usr/include/bits/local_lim.h:38:10: +fatal error: 'linux/limits.h' file not found +``` + +GCC 和 Clang 均受影响，问题是系统性的。 + +## 根因 + +### 直接原因：M5.5 sysroot 覆盖逻辑 + +`cli.cppm:1192-1203` 中的 M5.5 逻辑，将 `tc->sysroot` 强制覆盖为 mcpp 自己的 +subos（`~/.mcpp/registry/subos/default`）： + +```cpp +if (!isMuslTc) { + if (auto cfg = get_cfg(); cfg) { + auto mcppSubos = (*cfg)->xlingsHome() / "subos" / "default"; + if (std::filesystem::exists(mcppSubos / "usr" / "include")) { + tc->sysroot = mcppSubos; + } + } +} +``` + +### 触发 commit + +**`063fb6f`** — 将 MCPP_HOME 从 xpkgs 包目录改为 `~/.mcpp/`，使 M5.5 +找到一个存在但不完整的 subos。 + +### CI/e2e 未发现的原因 + +CI 的 subos 由 `xlings self install` 完整初始化（含内核头文件），e2e 测试 +通过 `_inherit_toolchain.sh` 继承宿主完整 subos。 + +--- + +## 设计分析：当前问题的本质 + +### 当前架构的矛盾 + +mcpp 的工具链管理存在一个架构层面的矛盾： + +``` + ┌─────────────────────────┐ + │ xlings 下载 payload │ + │ ~/.xlings/data/xpkgs/ │ + │ (cfg/specs 路径正确) │ + └──────────┬──────────────┘ + │ copy (因 XLINGS_HOME 传播不可靠) + ▼ + ┌─────────────────────────┐ + │ mcpp sandbox 副本 │ + │ ~/.mcpp/registry/xpkgs/ │ + │ (cfg/specs 路径 stale!) │ + └──────────┬──────────────┘ + │ + ┌──────────────┼──────────────┐ + ▼ ▼ ▼ + GCC: specs Clang macOS: Clang Linux: + fixup 修复 --no-default 没有修复 ← BUG + 路径 ✅ -config ✅ 用 subos 补救 ✗ +``` + +**三个平台/工具链的 stale path 问题，用了三种不同的解法：** + +| 工具链 | stale path 来源 | 当前解法 | 状态 | +|--------|-----------------|---------|------| +| GCC (Linux) | specs 文件 | `fixup_gcc_specs()` 重写 specs | ✅ 正确但只修了 specs，没修 `-print-sysroot` | +| Clang (macOS) | clang++.cfg | `--no-default-config` + xcrun | ✅ 但只在 macOS | +| Clang (Linux) | clang++.cfg | M5.5 subos 覆盖 | ❌ subos 不完整 | + +**根本问题：mcpp 复制了 payload 但没有统一处理 stale path。** + +### mcpp 对 xlings 的依赖边界不清 + +当前 mcpp 对 xlings 有三层依赖： + +| 层次 | 内容 | 应该依赖? | +|------|------|-----------| +| 包下载 | xlings 作为包索引 + 下载工具 | ✅ 是 | +| payload 路径 | xpkgs 下具体包的文件结构 | ✅ 是 | +| subos | xlings 的沙箱 sysroot | ❌ 不应该 | + +M5.5 的问题就在于跨越了第三层边界——用 xlings 的内部实现细节（subos） +来补救 mcpp 自身的路径问题。 + +--- + +## 设计方案 + +### 核心原则 + +**mcpp 只把 xlings 当包索引 + 下载工具。工具链的编译环境由 payload 自描述， +mcpp 忠实读取，不替换、不覆盖。** + +### 方案：Payload-first + 统一 stale path 处理 + +#### 1. 工具链 sysroot 来源：只从 payload 获取 + +``` +┌─────────────────────────────────────────────────┐ +│ sysroot 解析优先级 │ +│ │ +│ 1. compiler -print-sysroot (GCC 原生支持) │ +│ → 路径存在则使用 │ +│ │ +│ 2. payload cfg 文件解析 (Clang clang++.cfg) │ +│ → 解析 --sysroot= 行，路径存在则使用 │ +│ │ +│ 3. macOS: xcrun --show-sdk-path │ +│ │ +│ 4. 空 (不传 --sysroot，让编译器用自身默认值) │ +│ │ +│ ✗ 不再有 subos fallback │ +└─────────────────────────────────────────────────┘ +``` + +**改动**： +- 删除 `cli.cppm` M5.5 subos 覆盖代码 +- `probe.cppm` 增加 cfg 文件解析作为第 2 优先级 + +#### 2. 复制 payload 时统一修复 stale path + +当前只有 GCC 做了 specs fixup，Clang 只在 macOS 做了 `--no-default-config`。 +应该统一为：**凡是复制了 payload，就修复其中的路径配置。** + +``` + copy payload 后 + │ + ┌────────┼────────┐ + ▼ ▼ ▼ + GCC specs Clang cfg 其他配置 + │ │ + ▼ ▼ + rewrite_gcc rewrite_cfg + _specs() _paths() + │ │ + ▼ ▼ + 新 sysroot 新 sysroot + 新 rpath 新 -isystem + 新 -L/-rpath +``` + +**具体做法**：在 `package_fetcher.cppm` 复制 payload 后（或在 `cli.cppm` +toolchain install 后），对 Clang cfg 做类似 `fixup_gcc_specs` 的路径重写： + +```cpp +void fixup_clang_cfg(const std::filesystem::path& payloadRoot, + const std::filesystem::path& newSysroot) { + auto cfgPath = payloadRoot / "bin" / "clang++.cfg"; + if (!std::filesystem::exists(cfgPath)) return; + + // 读取 cfg，将旧 sysroot/isystem/rpath 路径替换为 payload 实际位置 + // ... +} +``` + +**但这里有一个关键设计选择：新路径指向哪里？** + +#### 3. 关于 sysroot 本身从哪来 + +工具链需要 C 库头文件（glibc headers + linux kernel headers）。来源有三种： + +| 来源 | 说明 | 优劣 | +|------|------|------| +| 系统 `/usr/include` | 宿主机自带 | 简单，但不可控，不同发行版不同 | +| xlings subos | xlings 管理的沙箱 sysroot | 可控，但 mcpp 需依赖 xlings 内部结构 | +| payload 自带 | 工具链包自带 sysroot（如 musl-gcc） | 最干净，但需要上游包支持 | + +**推荐策略**： + +- **短期**：信任 payload 自身配置的 sysroot 路径。xlings 安装 GCC/LLVM 时 + 已经配置好了 sysroot（指向 xlings 自己的 subos），mcpp 只需忠实读取。 + 如果路径存在且有效，就用它。如果路径无效，不传 `--sysroot`，让编译器 + 用系统默认路径。 + +- **中期**：推动 xlings 上游让 LLVM/GCC 包的 cfg/specs 使用相对路径或 + 可配置路径，避免硬编码绝对路径。这从源头消除 stale path 问题。 + +- **长期**：mcpp 自带轻量 sysroot 管理（类似 Zig 的做法：打包 libc headers + 作为 mcpp 自身的资源），彻底不依赖宿主系统或 xlings 的 sysroot。但这是 + 大工程，不急。 + +--- + +## 修复方案（基于以上设计） + +### Phase 1：修复当前 bug（最小改动） + +#### P1-1：删除 M5.5 subos 覆盖 + +**文件**：`src/cli.cppm:1192-1203` + +**删除**整个代码块。工具链的 sysroot 由 payload 决定，mcpp 不介入。 + +同时删除 `cli.cppm:1001` 的 subos 注释和 `cli.cppm:1178` 的 "glibc subos" 注释。 + +#### P1-2：`probe_sysroot` 增加 cfg 解析 + +**文件**：`src/toolchain/probe.cppm:254-272` + +`-print-sysroot` 失败后（Clang 不支持），解析 payload 中 `clang++.cfg` +的 `--sysroot=` 行： + +```cpp +std::filesystem::path +probe_sysroot(const std::filesystem::path& compilerBin, + const std::string& envPrefix) { + // 1. -print-sysroot (GCC) + auto r = run_capture(std::format("{}{} -print-sysroot {}", + envPrefix, + mcpp::xlings::shq(compilerBin.string()), + mcpp::platform::null_redirect)); + if (r) { + auto s = trim_line(*r); + if (!s.empty() && std::filesystem::exists(s)) return s; + } + + // 2. Parse payload cfg (Clang) + auto cfgPath = compilerBin.parent_path() + / (compilerBin.stem().string() + ".cfg"); + if (std::filesystem::exists(cfgPath)) { + std::ifstream ifs(cfgPath); + std::string line; + while (std::getline(ifs, line)) { + constexpr std::string_view prefix = "--sysroot="; + if (line.starts_with(prefix)) { + auto val = trim_line(std::string(line.substr(prefix.size()))); + if (!val.empty() && std::filesystem::exists(val)) + return val; + } + } + } + + // 3. macOS: xcrun SDK + if (auto sdk = mcpp::platform::macos::sdk_path()) + return *sdk; + return {}; +} +``` + +**Phase 1 效果**： +- Clang：cfg 中的 `--sysroot=~/.xlings/subos/default` 被正确读取， + `tc->sysroot` 不再为空。stdmod.cppm 和 flags.cppm 传递正确的 sysroot。 +- GCC：`-print-sysroot` 正常工作（如果路径存在）；若不存在则 sysroot 为空， + GCC 用默认系统路径（`/usr/include`）。 +- 不再依赖 subos。 + +### Phase 2：统一 Clang stale cfg 处理（消除隐患） + +#### P2-1：`stdmod.cppm` — 所有有 cfg 的 Clang 都走 `--no-default-config` + +**文件**：`src/toolchain/stdmod.cppm:103-116` + +将 macOS 特有的 `--no-default-config` 逻辑泛化为"有 cfg 文件的 Clang"： + +```cpp +std::string sysroot_flag; +if (is_clang(tc)) { + auto cfgPath = tc.binaryPath.parent_path() + / (tc.binaryPath.stem().string() + ".cfg"); + if (std::filesystem::exists(cfgPath)) { + // Bypass cfg (may have stale paths after payload copy). + // Provide correct flags from payload structure directly. + auto llvmRoot = tc.binaryPath.parent_path().parent_path(); + auto libcxxInclude = llvmRoot / "include" / "c++" / "v1"; + sysroot_flag = " --no-default-config"; + sysroot_flag += std::format(" -isystem'{}'", libcxxInclude.string()); + if (!tc.sysroot.empty()) + sysroot_flag += std::format(" --sysroot='{}'", tc.sysroot.string()); + else if (auto sdk = mcpp::platform::macos::sdk_path()) + sysroot_flag += std::format(" --sysroot='{}'", sdk->string()); + } else if (!tc.sysroot.empty()) { + sysroot_flag = std::format(" --sysroot='{}'", tc.sysroot.string()); + } +} else if (!tc.sysroot.empty()) { + sysroot_flag = std::format(" --sysroot='{}'", tc.sysroot.string()); +} +``` + +#### P2-2：`flags.cppm` — 同步修改 + +**文件**：`src/build/flags.cppm:96-111` + +同步 P2-1 的逻辑：将 `is_macos_clang` 条件改为"检测到 cfg 文件存在"。 + +**Phase 2 效果**： +- Linux 和 macOS Clang 走统一路径 +- 不再依赖 cfg 中的路径碰巧有效 +- mcpp 从 payload 结构推导出正确的 `-isystem` 和 `--sysroot` + +### Phase 3（未来）：复制 payload 时重写 cfg 路径 + +在 `package_fetcher.cppm` 或 `cli.cppm` toolchain install 后，添加 +`fixup_clang_cfg()`，类似 `fixup_gcc_specs()` 的做法： + +```cpp +void fixup_clang_cfg(const std::filesystem::path& payloadRoot, + const std::filesystem::path& oldXlingsHome, + const std::filesystem::path& newRegistryHome) { + // 重写 clang++.cfg 中的路径： + // --sysroot= → --sysroot= + // -isystem → -isystem + // -L → -L + // -rpath, → -rpath, +} +``` + +这样即使不用 `--no-default-config`，cfg 路径也是正确的。 +但需要 mcpp 管理自己的 sysroot 内容（确保完整性），所以这是更远期的方向。 + +--- + +## 修改总结 + +| Phase | 修改 | 文件 | 效果 | +|-------|------|------|------| +| P1-1 | 删除 M5.5 | cli.cppm | 去除 subos 依赖 | +| P1-2 | cfg 解析 sysroot | probe.cppm | Clang 获取正确 sysroot | +| P2-1 | 统一 --no-default-config | stdmod.cppm | 消除 stale cfg 隐患 | +| P2-2 | 同步 P2-1 | flags.cppm | 常规编译也用正确路径 | +| P3 | cfg 路径重写 | package_fetcher/cli | 从根源修复 stale path | + +**Phase 1 修复 bug，Phase 2 消除隐患，Phase 3 完善架构。** + +--- + +## 测试补充 + +### 新增 e2e 测试：无 subos 下的 import std + +```bash +#!/usr/bin/env bash +# requires: import-std +# Test that import std works without mcpp's subos sysroot. +# Regression test: M5.5 subos override must not be required. +set -euo pipefail + +TMP=$(mktemp -d) +trap "rm -rf $TMP" EXIT + +export MCPP_HOME="$TMP/mcpp-home" +export MCPP_INHERIT_SUBOS=0 +source "$(dirname "$0")/_inherit_toolchain.sh" + +mkdir -p "$TMP/proj/src" +cd "$TMP/proj" + +cat > mcpp.toml <<'EOF' +[package] +name = "sysroot_test" +version = "0.1.0" +EOF + +cat > src/main.cpp <<'EOF' +import std; +int main() { std::println("sysroot ok"); } +EOF + +"$MCPP" build +"$MCPP" run | grep -q "sysroot ok" +``` diff --git a/.agents/docs/2026-05-21-payload-first-sysroot-design.md b/.agents/docs/2026-05-21-payload-first-sysroot-design.md new file mode 100644 index 0000000..9881f21 --- /dev/null +++ b/.agents/docs/2026-05-21-payload-first-sysroot-design.md @@ -0,0 +1,241 @@ +# 设计方案：Payload-first 工具链环境管理 + +## 背景 + +mcpp 当前通过 `--sysroot` 指向 xlings subos 来为工具链提供 C 库头文件和内核头文件。 +这导致了对 xlings 内部结构（subos）的隐式依赖，且 subos 的完整性不可控。 + +## 设计原则 + +**mcpp 只把 xlings 当包索引 + 资源下载工具。所有编译环境由 mcpp 从 xpkgs payload +细粒度组装，不依赖 xlings 的 subos 或其他内部功能。** + +## 当前问题 + +xlings 中相关的 xpkgs 包： + +| 包名 | 内容 | 路径 | +|------|------|------| +| `xim-x-glibc/2.39` | glibc 头文件 + 运行时库 | `include/` (130 头文件) + `lib64/` (libc.so, crt*.o, ld-linux) | +| `scode-x-linux-headers/5.11.1` | Linux 内核头文件 | `include/linux/`, `include/asm/`, `include/asm-generic/` | +| `xim-x-gcc/16.1.0` | GCC 编译器 | C++ 标准库头文件 + 编译器内建头文件 + 运行时库 | +| `xim-x-llvm/20.1.7` | LLVM/Clang 编译器 | libc++ 头文件 + clang 内建头文件 + 运行时库 | + +这些包已经在 xpkgs 中了，但 mcpp 没有直接利用它们的路径，而是依赖 subos +（subos 实际上是 xlings 从这些包 + 宿主系统组装的一个合并目录）。 + +## 目标设计 + +### 核心思路 + +mcpp 在 detect 阶段从工具链 binary 位置推导出 sibling xpkgs（glibc、linux-headers）， +在 flags 阶段直接用这些 payload 路径组装编译和链接参数，不使用 `--sysroot`。 + +### 路径推导链 + +``` +compiler binary (e.g. ~/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++) + │ + ├── xpkgs_from_compiler() → ~/.mcpp/registry/data/xpkgs/ + │ + ├── find_sibling_tool("glibc") + │ → ~/.mcpp/registry/data/xpkgs/xim-x-glibc/2.39/ + │ ├── include/ → glibc 头文件 (-isystem) + │ └── lib64/ → 运行时库 (-L, -rpath, -B for crt*.o) + │ + └── find_sibling_tool("linux-headers") [优先 scode-x-linux-headers] + → ~/.mcpp/registry/data/xpkgs/scode-x-linux-headers/5.11.1/ + └── include/ → linux/, asm/, asm-generic/ (-isystem) +``` + +### 环境检查 + +mcpp 在 detect 阶段主动检查工具链环境完整性，而不是被动等到编译报错。 + +```cpp +struct SysrootPaths { + std::filesystem::path glibcInclude; // glibc headers + std::filesystem::path linuxInclude; // linux kernel headers + std::filesystem::path glibcLib; // glibc runtime (lib64/) + std::filesystem::path dynamicLinker; // ld-linux-x86-64.so.2 +}; + +// detect 阶段调用： +std::expected +probe_sysroot_paths(const std::filesystem::path& compilerBin); +``` + +检查项： + +| 检查 | 验证文件 | 失败提示 | +|------|---------|---------| +| glibc 头文件 | `glibc/include/features.h` | `glibc xpkg not found; run: mcpp toolchain install gcc` | +| linux 内核头文件 | `linux-headers/include/linux/limits.h` | `linux-headers package missing; will use host /usr/include as fallback` | +| glibc 运行时 | `glibc/lib64/libc.so.6` | `glibc runtime not found` | +| 动态链接器 | `glibc/lib64/ld-linux-x86-64.so.2` | `dynamic linker not found` | + +### 编译 flags 组装 + +#### GCC（当前用 --sysroot，改为 -isystem + -L） + +**修改前**： +``` +g++ -std=c++23 --sysroot= ... +``` + +**修改后**： +``` +g++ -std=c++23 \ + -isystem /include \ + -isystem /include \ + -L/lib64 \ + ... +``` + +注意：GCC 使用相对路径自动找到自己的 C++ 标准库头文件和编译器内建头文件， +不需要额外的 `-isystem`。只需提供 glibc 和 linux-headers 的路径。 + +#### Clang（当前依赖 cfg，改为 --no-default-config + 显式路径） + +**修改前**： +``` +# cfg 中的路径（指向 ~/.xlings/，可能 stale） +clang++ --sysroot= -isystem /include/c++/v1 ... +``` + +**修改后**： +``` +clang++ --no-default-config \ + -stdlib=libc++ \ + -isystem /include/c++/v1 \ + -isystem /include//c++/v1 \ + -isystem /include \ + -isystem /include \ + -fuse-ld=lld \ + --rtlib=compiler-rt \ + --unwindlib=libunwind \ + -L/lib/ \ + -L/lib64 \ + ... +``` + +这里 mcpp 显式提供了 cfg 中所有必要的 flags，不再依赖 cfg 文件。 +Linux 和 macOS 走同一套逻辑（macOS 用 xcrun SDK 替代 glibc/linux-headers）。 + +### Toolchain 数据模型扩展 + +```cpp +struct Toolchain { + // ... 现有字段 ... + + // 替代 sysroot 的细粒度路径 + struct PayloadPaths { + std::filesystem::path glibcInclude; // glibc headers + std::filesystem::path glibcLib; // glibc lib64 + std::filesystem::path linuxInclude; // linux kernel headers + std::filesystem::path dynamicLinker; // ld-linux path + }; + std::optional payloadPaths; // 非空 = 使用 payload 模式 +}; +``` + +当 `payloadPaths` 有值时，flags 组装使用细粒度路径； +当 `payloadPaths` 为空时（系统编译器、用户自定义），回退到 `sysroot` 或不传。 + +### GCC specs fixup 的演进 + +当前 `fixup_gcc_specs()` 重写 specs 中的 dynamic-linker 和 rpath 路径， +指向 glibc xpkg 的 `lib64/`。这已经是 payload-first 的做法。 + +未来可以考虑在 specs fixup 中同时重写 sysroot： +``` +# specs 中添加: +*sysroot: + +``` +这样 GCC 自身就知道正确的 sysroot，不需要命令行 `--sysroot`。 +但这需要更深入理解 specs 格式，作为 Phase 3 推进。 + +### Clang cfg fixup（新增） + +类似 `fixup_gcc_specs()`，在 toolchain install 时重写 `clang++.cfg`： + +```cpp +void fixup_clang_cfg(const std::filesystem::path& payloadRoot, + const PayloadPaths& paths) { + auto cfgPath = payloadRoot / "bin" / "clang++.cfg"; + // 重写: + // --sysroot= → 删除（由 mcpp 显式传递） + // -isystem → -isystem /include/c++/v1 + // -L → -L/lib/ + // -Wl,--dynamic-linker= → -Wl,--dynamic-linker=/lib64/ld-linux + // -Wl,-rpath, → -Wl,-rpath,/lib64 +} +``` + +这样即使不用 `--no-default-config`，cfg 中的路径也是自洽的。 + +## 实施路线 + +### Phase 1（当前 PR #62 已完成） + +- 删除 M5.5 subos 覆盖 +- `probe_sysroot` 增加 cfg 解析 + GCC stale path remap +- macOS Clang 保持 `--no-default-config` + xcrun + +**效果**：修复了 bug，但 GCC sysroot 仍依赖 registry subos，Clang 仍依赖 cfg 中的 +xlings 路径。 + +### Phase 2：Payload-first 路径探测 + +- Toolchain 模型增加 `PayloadPaths` 字段 +- `probe_sysroot_paths()` 通过 `find_sibling_tool()` 定位 glibc 和 linux-headers xpkgs +- 环境检查：detect 阶段验证关键头文件和库文件存在 +- 缺失 linux-headers 时给出明确提示，或自动触发安装 + +**改动文件**： +- `src/toolchain/model.cppm` — PayloadPaths 结构 +- `src/toolchain/probe.cppm` — probe_sysroot_paths() 实现 +- `src/toolchain/detect.cppm` — 调用 probe_sysroot_paths + +### Phase 3：Payload-first flags 组装 + +- `flags.cppm` 使用 PayloadPaths 组装 `-isystem` / `-L` / `-B` 替代 `--sysroot` +- `stdmod.cppm` 同步使用 PayloadPaths +- GCC 和 Clang 走统一的 flags 组装逻辑 +- 去掉对 cfg 文件的运行时依赖 + +**改动文件**： +- `src/build/flags.cppm` — 核心 flags 组装 +- `src/toolchain/stdmod.cppm` — std module 预编译 +- `src/toolchain/clang.cppm` — clang std module build commands + +### Phase 4：cfg/specs fixup 统一 + +- toolchain install 时对 Clang cfg 做 fixup（类似 `fixup_gcc_specs`） +- 使 payload 完全自洽，不依赖 xlings 原始路径 +- 去掉 `--no-default-config`（cfg 本身已正确） + +**改动文件**： +- `src/cli.cppm` — fixup_clang_cfg() + install 流程集成 + +### Phase 5（可选）：依赖声明 + +在 xpkgs 包描述中声明 sysroot 依赖关系： +```toml +[toolchain.gcc] +requires = ["glibc", "linux-headers", "binutils"] +``` +mcpp 在 toolchain install 时自动检查和安装这些依赖包。 + +## 设计对比 + +| 维度 | 当前（subos） | 目标（payload-first） | +|------|-------------|---------------------| +| sysroot 来源 | xlings subos（合并目录） | glibc + linux-headers xpkgs | +| 路径控制 | 被动依赖 xlings 初始化 | mcpp 主动探测和组装 | +| 环境检查 | 无（编译时才报错） | detect 阶段验证 | +| cfg/specs | cfg stale + specs fixup | 两者都 fixup，路径自洽 | +| 跨平台 | macOS/Linux/Windows 各自特殊处理 | 统一的 PayloadPaths 抽象 | +| 错误提示 | `linux/limits.h not found` | `linux-headers package missing` | +| xlings 依赖 | subos + xpkgs + cfg 路径 | 只依赖 xpkgs 文件 | diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 5f3e522..6343b67 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -316,13 +316,4 @@ jobs: "$MCPP" --version echo ":: Self-host smoke PASS" - - name: Fresh user experience (xlings install mcpp → new → run) - continue-on-error: true - run: | - # Test real user flow with xlings-distributed mcpp. - # May fail if xlings mcpp version lacks recent fixes. - TMP=$(mktemp -d) - cd "$TMP" - "$MCPP" new hello - cd hello - "$MCPP" run + # Fresh user experience test moved to ci-fresh-install.yml (PR #63) diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index d8b9aa0..90d5412 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -108,18 +108,7 @@ jobs: "$MCPP_SELF" --version echo ":: Self-host smoke PASS" - - name: Fresh user experience (xlings install mcpp → new → run) - continue-on-error: true - shell: bash - run: | - # Test the real user flow with the xlings-distributed mcpp. - # Currently xlings ships 0.0.17 which lacks Windows fixes. - # This step will auto-pass once xlings updates to 0.0.19+. - TMP=$(mktemp -d) - cd "$TMP" - "$MCPP" new hello - cd hello - "$MCPP" run + # Fresh user experience test moved to ci-fresh-install.yml (PR #63) - name: Package Windows release zip id: package diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a40a881..e42abf9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,30 +134,33 @@ jobs: "$MCPP" build "$MCPP" test - - name: Toolchain install smoke test (mcpp toolchain install llvm) + - name: "Toolchain: GCC — mcpp new → build → run" run: | - # Test the core user flow: install a toolchain, create a project, - # build with it. Uses the freshly-built mcpp (not bootstrap). MCPP=$(realpath "$(find target -type f -name mcpp -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2)") - # Install LLVM toolchain into mcpp's sandbox - "$MCPP" toolchain install llvm 20.1.7 - # Set as default so the build picks it up - "$MCPP" toolchain default llvm@20.1.7 - # Build a hello-world project with the installed toolchain + "$MCPP" toolchain default gcc@16.1.0 TMP=$(mktemp -d) cd "$TMP" - "$MCPP" new hello - cd hello - "$MCPP" build + "$MCPP" new hello_gcc + cd hello_gcc "$MCPP" run - - name: Fresh user experience (xlings install mcpp → new → run) - continue-on-error: true + - name: "Toolchain: musl-gcc — mcpp new → build → run" run: | - # Test real user flow with xlings-distributed mcpp. - # May fail if xlings mcpp version lacks recent fixes. + MCPP=$(realpath "$(find target -type f -name mcpp -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2)") + "$MCPP" toolchain default gcc@15.1.0-musl + TMP=$(mktemp -d) + cd "$TMP" + "$MCPP" new hello_musl + cd hello_musl + "$MCPP" run + + - name: "Toolchain: LLVM — install + mcpp new → build → run" + run: | + MCPP=$(realpath "$(find target -type f -name mcpp -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2)") + "$MCPP" toolchain install llvm 20.1.7 + "$MCPP" toolchain default llvm@20.1.7 TMP=$(mktemp -d) cd "$TMP" - "$MCPP" new hello - cd hello + "$MCPP" new hello_llvm + cd hello_llvm "$MCPP" run diff --git a/src/build/flags.cppm b/src/build/flags.cppm index e7278c7..0f70de0 100644 --- a/src/build/flags.cppm +++ b/src/build/flags.cppm @@ -88,24 +88,72 @@ CompileFlags compute_flags(const BuildPlan& plan) { include_flags += " -I" + escape_path(abs); } - // Sysroot + config override for macOS. - // On macOS, xlings LLVM's clang++.cfg contains hardcoded --sysroot and - // -isystem paths from the original install location. When the package is - // copied to mcpp's sandbox, these paths become stale. We pass - // --no-default-config to ignore the cfg and provide correct paths. + // Sysroot / payload paths. + // + // Payload-first: when PayloadPaths are available (glibc + linux-headers + // xpkgs found), use -isystem for each payload include dir. This avoids + // dependency on xlings subos. + // + // For Clang with a cfg file: use --no-default-config to bypass + // potentially-stale paths, then provide all flags explicitly. + // + // Fallback: if no PayloadPaths, use --sysroot from probe_sysroot(). std::string sysroot_flag; - bool is_macos_clang = mcpp::toolchain::is_clang(plan.toolchain) - && (plan.toolchain.targetTriple.find("apple") != std::string::npos - || plan.toolchain.targetTriple.find("darwin") != std::string::npos); - if (is_macos_clang) { + bool isClangWithCfg = false; + std::filesystem::path cfgPath; + if (mcpp::toolchain::is_clang(plan.toolchain)) { + cfgPath = plan.toolchain.binaryPath.parent_path() + / (plan.toolchain.binaryPath.stem().string() + ".cfg"); + isClangWithCfg = std::filesystem::exists(cfgPath); + } + + if (isClangWithCfg) { + // Clang with cfg: bypass cfg and provide all paths explicitly. auto llvmRoot = plan.toolchain.binaryPath.parent_path().parent_path(); auto libcxxInclude = llvmRoot / "include" / "c++" / "v1"; - sysroot_flag = " --no-default-config"; + sysroot_flag = " --no-default-config -nostdinc++"; + // libc++ headers + sysroot_flag += " -stdlib=libc++"; sysroot_flag += " -isystem" + escape_path(libcxxInclude); - if (auto sdk = mcpp::platform::macos::sdk_path()) + if (!plan.toolchain.targetTriple.empty()) { + auto targetInclude = llvmRoot / "include" + / plan.toolchain.targetTriple / "c++" / "v1"; + if (std::filesystem::exists(targetInclude)) + sysroot_flag += " -isystem" + escape_path(targetInclude); + } + // C library + kernel headers from payload + if (plan.toolchain.payloadPaths) { + auto& pp = *plan.toolchain.payloadPaths; + sysroot_flag += " -isystem" + escape_path(pp.glibcInclude); + if (!pp.linuxInclude.empty()) + sysroot_flag += " -isystem" + escape_path(pp.linuxInclude); + } else if (auto sdk = mcpp::platform::macos::sdk_path()) { sysroot_flag += " --sysroot=" + escape_path(*sdk); + } else if (!plan.toolchain.sysroot.empty()) { + sysroot_flag += " --sysroot=" + escape_path(plan.toolchain.sysroot); + } + // Linker flags that cfg normally provides + sysroot_flag += " -fuse-ld=lld --rtlib=compiler-rt --unwindlib=libunwind"; f.sysroot = sysroot_flag; } else if (!plan.toolchain.sysroot.empty()) { + // GCC (or Clang without cfg): use --sysroot from probe. + // GCC requires --sysroot for include-fixed headers (stdlib.h wrapper). + // Supplement with -isystem for linux kernel headers from payload + // if the probed sysroot is missing them. + sysroot_flag = " --sysroot=" + escape_path(plan.toolchain.sysroot); + if (plan.toolchain.payloadPaths && !plan.toolchain.payloadPaths->linuxInclude.empty()) { + auto sysrootLinux = plan.toolchain.sysroot / "usr" / "include" / "linux" / "limits.h"; + if (!std::filesystem::exists(sysrootLinux)) + sysroot_flag += " -isystem" + escape_path(plan.toolchain.payloadPaths->linuxInclude); + } + f.sysroot = sysroot_flag; + } else if (plan.toolchain.payloadPaths) { + // No sysroot but have payload paths: use -isystem. + auto& pp = *plan.toolchain.payloadPaths; + sysroot_flag += " -isystem" + escape_path(pp.glibcInclude); + if (!pp.linuxInclude.empty()) + sysroot_flag += " -isystem" + escape_path(pp.linuxInclude); + f.sysroot = sysroot_flag; sysroot_flag = " --sysroot=" + escape_path(plan.toolchain.sysroot); f.sysroot = sysroot_flag; } @@ -185,12 +233,23 @@ CompileFlags compute_flags(const BuildPlan& plan) { } } + // For Clang with payload paths: add glibc lib + dynamic linker to link flags. + std::string payload_ld; + if (isClangWithCfg && plan.toolchain.payloadPaths) { + auto& pp = *plan.toolchain.payloadPaths; + payload_ld += " -L" + escape_path(pp.glibcLib); + payload_ld += " -Wl,-rpath," + escape_path(pp.glibcLib); + auto loader = pp.glibcLib / "ld-linux-x86-64.so.2"; + if (std::filesystem::exists(loader)) + payload_ld += " -Wl,--dynamic-linker=" + escape_path(loader); + } + if constexpr (mcpp::platform::is_windows) { f.ld = ""; } else if constexpr (mcpp::platform::needs_explicit_libcxx) { f.ld = std::format("{}{}{} -lc++", full_static, static_stdlib, b_flag); } else { - f.ld = std::format("{}{}{}{}{}", full_static, static_stdlib, sysroot_flag, b_flag, runtime_dirs); + f.ld = std::format("{}{}{}{}{}{}", full_static, static_stdlib, sysroot_flag, b_flag, runtime_dirs, payload_ld); } return f; diff --git a/src/cli.cppm b/src/cli.cppm index 172e124..6800aac 100644 --- a/src/cli.cppm +++ b/src/cli.cppm @@ -765,6 +765,125 @@ void fixup_gcc_specs(const std::filesystem::path& gccPkgRoot, } } +// Rewrite clang++.cfg paths after the LLVM payload has been copied to the +// mcpp sandbox. The cfg was authored by xlings at install time and contains +// absolute paths pointing to ~/.xlings/. We rewrite them to point to the +// actual payload location + sibling xpkgs (glibc, linux-headers). +void fixup_clang_cfg(const std::filesystem::path& payloadRoot, + const std::filesystem::path& glibcLibDir) { + for (auto cfgName : {"clang++.cfg", "clang.cfg"}) { + auto cfgPath = payloadRoot / "bin" / cfgName; + if (!std::filesystem::exists(cfgPath)) continue; + + std::ifstream is(cfgPath); + std::stringstream ss; ss << is.rdbuf(); + std::string content = ss.str(); + is.close(); + + auto llvmRoot = payloadRoot; + auto replace_line_prefix = [&](std::string& s, std::string_view prefix, + const std::string& newValue) { + std::istringstream lines(s); + std::string result, line; + while (std::getline(lines, line)) { + if (line.starts_with(prefix)) { + result += std::string(prefix) + newValue + '\n'; + } else { + result += line + '\n'; + } + } + s = result; + }; + + // Rewrite --sysroot to remove (mcpp provides this explicitly). + // Rewrite -isystem to point to payload's libc++ headers. + // Rewrite -L and -rpath to point to payload's lib dir. + // Rewrite dynamic-linker to use glibc payload's ld-linux. + std::istringstream lines(content); + std::string result, line; + while (std::getline(lines, line)) { + if (line.starts_with("--sysroot=")) { + // Remove — mcpp provides sysroot via payload paths. + continue; + } + if (line.starts_with("-isystem ")) { + auto oldPath = line.substr(9); + if (oldPath.find("include/c++/v1") != std::string::npos) { + auto relative = oldPath.substr(oldPath.find("include/c++/v1")); + result += "-isystem " + (llvmRoot / relative).string() + '\n'; + continue; + } + if (oldPath.find("include/x86_64") != std::string::npos || + oldPath.find("include/aarch64") != std::string::npos) { + // Target-specific libc++ include. + auto includePos = oldPath.find("include/"); + auto relative = oldPath.substr(includePos); + result += "-isystem " + (llvmRoot / relative).string() + '\n'; + continue; + } + } + if (line.starts_with("-L")) { + auto oldPath = line.substr(2); + if (oldPath.find("lib/x86_64") != std::string::npos || + oldPath.find("lib/aarch64") != std::string::npos) { + auto libPos = oldPath.find("lib/"); + auto relative = oldPath.substr(libPos); + result += "-L" + (llvmRoot / relative).string() + '\n'; + continue; + } + } + if (line.starts_with("-Wl,-rpath,")) { + auto oldPath = line.substr(11); + // Rpath for LLVM lib dir + if (oldPath.find("lib/x86_64") != std::string::npos || + oldPath.find("lib/aarch64") != std::string::npos) { + auto libPos = oldPath.find("lib/"); + auto relative = oldPath.substr(libPos); + result += "-Wl,-rpath," + (llvmRoot / relative).string() + '\n'; + continue; + } + // Rpath for subos/glibc — rewrite to glibc payload. + if (!glibcLibDir.empty()) { + auto parentDir = std::filesystem::path(oldPath).parent_path(); + // subos rpath lines like -Wl,-rpath,/lib + if (oldPath.find("subos") != std::string::npos) { + result += "-Wl,-rpath," + glibcLibDir.string() + '\n'; + continue; + } + } + } + if (line.starts_with("-Wl,--dynamic-linker=")) { + // Rewrite to glibc payload's ld-linux. + if (!glibcLibDir.empty()) { + result += "-Wl,--dynamic-linker=" + + (glibcLibDir / "ld-linux-x86-64.so.2").string() + '\n'; + continue; + } + } + if (line.starts_with("-Wl,--enable-new-dtags,-rpath,")) { + if (!glibcLibDir.empty()) { + result += "-Wl,--enable-new-dtags,-rpath," + glibcLibDir.string() + '\n'; + continue; + } + } + if (line.starts_with("-Wl,-rpath-link,")) { + if (!glibcLibDir.empty()) { + result += "-Wl,-rpath-link," + glibcLibDir.string() + '\n'; + continue; + } + } + result += line + '\n'; + } + + // Remove trailing newline + while (!result.empty() && result.back() == '\n') result.pop_back(); + result += '\n'; + + std::ofstream os(cfgPath); + os << result; + } +} + // SemVer resolution: a version spec is a "constraint" (vs. exact literal) if // it starts with one of `^~><=` or contains a comma (multi-part), or is `*` // or empty. Bare `1.2.3` is treated as exact for back-compat with pre-SemVer @@ -998,9 +1117,8 @@ prepare_build(bool print_fingerprint, // ─── Toolchain resolution (docs/21) ──────────────────────────────── // Priority chain: // 1. mcpp.toml [toolchain]. → resolve_xpkg_path → abs path - // 2. $MCPP_HOME/registry/subos//bin/g++ (xlings sandbox subos) - // 3. $CXX env var - // 4. PATH g++ (with warning) + // 2. $CXX env var + // 3. PATH g++ (with warning) std::filesystem::path explicit_compiler; std::optional cfg_opt; auto get_cfg = [&]() -> std::expected { @@ -1174,10 +1292,8 @@ prepare_build(bool print_fingerprint, if (!tc) return std::unexpected(tc.error().message); // For musl-gcc the toolchain is fully self-contained - // (`/x86_64-linux-musl/{include,lib}` is its own sysroot), and - // pointing it at mcpp's glibc subos breaks compilation. Skip the - // sysroot injection in that case — musl-gcc's `-dumpmachine` reports - // `x86_64-linux-musl`, which is also the marker we use elsewhere. + // (`/x86_64-linux-musl/{include,lib}` is its own sysroot). + // musl-gcc's `-dumpmachine` reports `x86_64-linux-musl`. bool isMuslTc = tc->targetTriple.find("-musl") != std::string::npos; // A musl toolchain only really makes sense with static linkage — @@ -1189,18 +1305,9 @@ prepare_build(bool print_fingerprint, m->buildConfig.linkage = "static"; } - // M5.5: prefer mcpp's xlings-managed subos as sysroot — it has glibc - // headers + libs in the conventional layout that GCC expects. The - // -print-sysroot output from a freshly-built GCC often points at - // some build-time path that doesn't exist on the user's machine. - if (!isMuslTc) { - if (auto cfg = get_cfg(); cfg) { - auto mcppSubos = (*cfg)->xlingsHome() / "subos" / "default"; - if (std::filesystem::exists(mcppSubos / "usr" / "include")) { - tc->sysroot = mcppSubos; - } - } - } + // Sysroot comes from the toolchain payload itself (GCC -print-sysroot, + // Clang clang++.cfg). mcpp does not override it — the payload is + // self-describing. See docs: 2026-05-21-linux-sysroot-missing-kernel-headers.md // Resolve dependencies: walk the **transitive** graph from the main // manifest, BFS-style. Each unique `(namespace, shortName)` is fetched @@ -3618,6 +3725,18 @@ int cmd_toolchain(const mcpplibs::cmdline::ParsedArgs& parsed) { std::format("{} {} via mcpp's xlings", spec->compiler, spec->version)); mcpp::fetcher::Fetcher fetcher(*cfg); CliInstallProgress progress; + + // Ensure sysroot dependencies (glibc, linux-headers) are installed. + // These are required for C library + kernel headers during compilation. + // musl-gcc is self-contained and doesn't need these. + if (!spec->isMusl) { + for (auto dep : {"xim:glibc", "xim:linux-headers"}) { + auto depPayload = fetcher.resolve_xpkg_path(dep, /*autoInstall=*/true, &progress); + // Best-effort: linux-headers may not be in the index. + // glibc is usually a dependency of gcc/llvm and already installed. + } + } + auto payload = fetcher.resolve_xpkg_path(pkg.target(), /*autoInstall=*/true, &progress); if (!payload) { mcpp::ui::error(std::format("install failed: {}", payload.error().message)); @@ -3683,6 +3802,29 @@ int cmd_toolchain(const mcpplibs::cmdline::ParsedArgs& parsed) { } } + // For LLVM/Clang: post-install cfg fixup so the clang++.cfg paths + // point to the payload's actual location instead of the xlings + // install-time paths (which become stale after copy). + if (pkg.ximName == "llvm") { + auto glibcRoot = mcpp::xlings::paths::xim_tool_root(xlEnv, "glibc"); + std::filesystem::path glibcLibDir; + if (std::filesystem::exists(glibcRoot)) { + for (auto& v : std::filesystem::directory_iterator(glibcRoot)) { + auto candidate = v.path() / "lib64"; + if (std::filesystem::exists(candidate / "ld-linux-x86-64.so.2")) { + glibcLibDir = candidate; + break; + } + candidate = v.path() / "lib"; + if (std::filesystem::exists(candidate / "ld-linux-x86-64.so.2")) { + glibcLibDir = candidate; + break; + } + } + } + fixup_clang_cfg(payload->root, glibcLibDir); + } + mcpp::ui::status("Installed", std::format("{} → {}", pkg.display_spec(), bin.string())); if (cfg->defaultToolchain.empty()) { diff --git a/src/toolchain/clang.cppm b/src/toolchain/clang.cppm index 52d325f..1d9460f 100644 --- a/src/toolchain/clang.cppm +++ b/src/toolchain/clang.cppm @@ -134,7 +134,7 @@ std::optional find_libcxx_std_module_source( void enrich_toolchain(Toolchain& tc, const std::string& envPrefix) { // Clang targeting MSVC uses MSVC STL, not libc++. - bool msvTarget = tc.targetTriple.find("msvc") != std::string::npos; + bool msvTarget = is_msvc_target(tc); tc.stdlibId = msvTarget ? "msvc-stl" : "libc++"; tc.stdlibVersion = tc.version.empty() ? "unknown" : tc.version; tc.linkRuntimeDirs = mcpp::toolchain::discover_link_runtime_dirs( diff --git a/src/toolchain/detect.cppm b/src/toolchain/detect.cppm index f273cae..f0d9c47 100644 --- a/src/toolchain/detect.cppm +++ b/src/toolchain/detect.cppm @@ -69,6 +69,26 @@ detect(const std::filesystem::path& explicit_compiler) { tc.targetTriple = *triple; } +#if defined(_WIN32) + // On Windows, Clang targeting MSVC auto-detects the MSVC version at + // compile time and bakes it into the module AST. The -dumpmachine triple + // doesn't include this version, so fingerprints don't change when MSVC + // patches (e.g. 19.44.35226 → 35227), causing stale BMI cache hits. + // Query the effective triple which includes the actual MSVC version. + if (tc.compiler == CompilerId::Clang + && is_msvc_target(tc)) { + auto vr = run_capture(std::format( + "{}{} -print-effective-triple 2>NUL", + envPrefix, + mcpp::xlings::shq(tc.binaryPath.string()))); + if (vr) { + auto effective = trim_line(*vr); + if (!effective.empty() && effective != tc.targetTriple) + tc.driverIdent += "\neffective-triple: " + effective; + } + } +#endif + if (tc.compiler == CompilerId::GCC) { mcpp::toolchain::gcc::enrich_toolchain(tc); } else if (tc.compiler == CompilerId::Clang) { @@ -77,6 +97,16 @@ detect(const std::filesystem::path& explicit_compiler) { tc.sysroot = probe_sysroot(tc.binaryPath, envPrefix); + // Probe fine-grained payload paths from sibling xpkgs (glibc, linux-headers). + // When available, flags are assembled from these paths instead of --sysroot. + tc.payloadPaths = probe_payload_paths(tc.binaryPath); + + // For GCC: ensure the probed sysroot has complete headers by symlinking + // missing content (linux kernel headers, glibc) from payload xpkgs. + // This makes mcpp self-sufficient — not dependent on xlings subos init. + if (tc.payloadPaths && !tc.sysroot.empty()) + ensure_sysroot_complete(tc.sysroot, *tc.payloadPaths); + return tc; } diff --git a/src/toolchain/model.cppm b/src/toolchain/model.cppm index cecfacc..58ac208 100644 --- a/src/toolchain/model.cppm +++ b/src/toolchain/model.cppm @@ -8,6 +8,14 @@ export namespace mcpp::toolchain { enum class CompilerId { Unknown, GCC, Clang, MSVC }; +// Fine-grained sysroot paths derived from xpkgs payloads. +// When populated, flags are assembled from these paths instead of --sysroot. +struct PayloadPaths { + std::filesystem::path glibcInclude; // glibc headers (features.h, bits/) + std::filesystem::path glibcLib; // glibc runtime (libc.so, crt*.o, ld-linux) + std::filesystem::path linuxInclude; // linux kernel headers (linux/, asm/) +}; + struct Toolchain { CompilerId compiler = CompilerId::Unknown; std::string version; // "15.1.0" @@ -19,6 +27,7 @@ struct Toolchain { std::filesystem::path stdModuleSource; // bits/std.cc / std.cppm std::filesystem::path stdCompatSource; // bits/std_compat.cc / std.compat.cppm std::filesystem::path sysroot; // -print-sysroot output (or empty) + std::optional payloadPaths; // fine-grained sysroot from xpkgs std::vector compilerRuntimeDirs; // LD_LIBRARY_PATH for private tools std::vector linkRuntimeDirs; // -L/-rpath dirs for produced binaries bool hasImportStd = false; @@ -42,6 +51,7 @@ struct DetectError { std::string message; }; bool is_gcc(const Toolchain& tc); bool is_clang(const Toolchain& tc); bool is_musl_target(const Toolchain& tc); +bool is_msvc_target(const Toolchain& tc); struct BmiTraits { std::string_view bmiDir; // "gcm.cache" | "pcm.cache" @@ -70,6 +80,10 @@ bool is_musl_target(const Toolchain& tc) { return tc.targetTriple.find("-musl") != std::string::npos; } +bool is_msvc_target(const Toolchain& tc) { + return tc.targetTriple.find("msvc") != std::string::npos; +} + BmiTraits bmi_traits(const Toolchain& tc) { if (is_clang(tc)) { return { diff --git a/src/toolchain/probe.cppm b/src/toolchain/probe.cppm index ef25032..55b0d70 100644 --- a/src/toolchain/probe.cppm +++ b/src/toolchain/probe.cppm @@ -47,6 +47,18 @@ std::filesystem::path probe_sysroot(const std::filesystem::path& compilerBin, const std::string& envPrefix); +// Probe fine-grained sysroot paths from sibling xpkgs payloads. +// Returns populated PayloadPaths if glibc xpkg found; linux-headers +// may be empty if not available. +std::optional +probe_payload_paths(const std::filesystem::path& compilerBin); + +// Ensure sysroot directory has complete headers by symlinking from +// payload xpkgs. Called when GCC's probed sysroot exists but may +// be missing linux kernel headers or glibc headers. +void ensure_sysroot_complete(const std::filesystem::path& sysroot, + const PayloadPaths& pp); + } // namespace mcpp::toolchain namespace mcpp::toolchain { @@ -254,6 +266,7 @@ probe_target_triple(const std::filesystem::path& compilerBin, std::filesystem::path probe_sysroot(const std::filesystem::path& compilerBin, const std::string& envPrefix) { + // 1. Ask the compiler directly (works for GCC; Clang often doesn't support it). auto r = run_capture(std::format("{}{} -print-sysroot {}", envPrefix, mcpp::xlings::shq(compilerBin.string()), @@ -261,14 +274,117 @@ probe_sysroot(const std::filesystem::path& compilerBin, if (r) { auto s = trim_line(*r); if (!s.empty() && std::filesystem::exists(s)) return s; + + // GCC bakes the build-time sysroot into the binary. For xlings-built + // GCC this is a path like /.xlings/subos/default that + // doesn't exist on the user's machine. If the reported path ends + // with subos/default, look for the equivalent sysroot relative to + // the compiler's own xpkgs directory (payload-derived). + if (!s.empty() && s.ends_with("subos/default")) { + if (auto xpkgs = mcpp::xlings::paths::xpkgs_from_compiler(compilerBin)) { + // xpkgs is /data/xpkgs → registry = xpkgs/../.. + auto registrySysroot = xpkgs->parent_path().parent_path() + / "subos" / "default"; + if (std::filesystem::exists(registrySysroot / "usr" / "include")) + return registrySysroot; + } + } + } + + // 2. Parse the compiler driver config file (Clang .cfg). + // xlings-installed Clang ships a clang++.cfg alongside the binary + // with --sysroot pointing to the payload's associated sysroot. + { + auto stem = compilerBin.stem().string(); + auto cfgPath = compilerBin.parent_path() / (stem + ".cfg"); + if (std::filesystem::exists(cfgPath)) { + std::ifstream ifs(cfgPath); + std::string line; + while (std::getline(ifs, line)) { + constexpr std::string_view prefix = "--sysroot="; + if (line.starts_with(prefix)) { + auto val = trim_line(std::string(line.substr(prefix.size()))); + if (!val.empty() && std::filesystem::exists(val)) + return val; + } + } + } } - // macOS fallback: use xcrun to discover the SDK path. - // The sysroot is used for regular compilation flags (flags.cppm) but - // skipped for std module precompilation on macOS (stdmod.cppm) to - // avoid breaking SDK internal header dependencies. + + // 3. macOS fallback: use xcrun to discover the SDK path. if (auto sdk = mcpp::platform::macos::sdk_path()) return *sdk; return {}; } +std::optional +probe_payload_paths(const std::filesystem::path& compilerBin) { + namespace paths = mcpp::xlings::paths; + + // Find glibc xpkg (required). + auto glibc = paths::find_sibling_tool(compilerBin, "glibc"); + if (!glibc) return std::nullopt; + + // Glibc layout: /include/ + /lib64/ (or lib/). + auto glibcInclude = *glibc / "include"; + if (!std::filesystem::exists(glibcInclude / "features.h")) + return std::nullopt; + + auto glibcLib = *glibc / "lib64"; + if (!std::filesystem::exists(glibcLib)) + glibcLib = *glibc / "lib"; + if (!std::filesystem::exists(glibcLib)) + return std::nullopt; + + PayloadPaths pp; + pp.glibcInclude = glibcInclude; + pp.glibcLib = glibcLib; + + // Find linux kernel headers (optional — search across index prefixes). + auto linuxHeaders = paths::find_sibling_package(compilerBin, "linux-headers"); + if (linuxHeaders) { + auto linuxInclude = *linuxHeaders / "include"; + if (std::filesystem::exists(linuxInclude / "linux" / "limits.h")) + pp.linuxInclude = linuxInclude; + } + + return pp; +} + +void ensure_sysroot_complete(const std::filesystem::path& sysroot, + const PayloadPaths& pp) { + if (sysroot.empty()) return; + + auto sysrootInclude = sysroot / "usr" / "include"; + if (!std::filesystem::exists(sysrootInclude)) return; + + std::error_code ec; + + // Ensure linux kernel headers are present in sysroot. + // If missing, symlink from linux-headers payload. + if (!pp.linuxInclude.empty()) { + for (auto dir : {"linux", "asm", "asm-generic"}) { + auto target = sysrootInclude / dir; + auto source = pp.linuxInclude / dir; + if (!std::filesystem::exists(target, ec) && std::filesystem::exists(source, ec)) { + std::filesystem::create_directory_symlink(source, target, ec); + } + } + } + + // Ensure glibc headers are present if sysroot is bare. + if (!std::filesystem::exists(sysrootInclude / "features.h", ec)) { + // Symlink individual glibc dirs/files into sysroot. + for (auto& entry : std::filesystem::directory_iterator(pp.glibcInclude, ec)) { + auto target = sysrootInclude / entry.path().filename(); + if (!std::filesystem::exists(target, ec)) { + if (entry.is_directory(ec)) + std::filesystem::create_directory_symlink(entry.path(), target, ec); + else + std::filesystem::create_symlink(entry.path(), target, ec); + } + } + } +} + } // namespace mcpp::toolchain diff --git a/src/toolchain/provider.cppm b/src/toolchain/provider.cppm index 985de48..ad3e4ae 100644 --- a/src/toolchain/provider.cppm +++ b/src/toolchain/provider.cppm @@ -80,9 +80,7 @@ ProviderCapabilities capabilities_for(const Toolchain& tc) { case CompilerId::Clang: { // Clang targeting MSVC uses MSVC STL, not libc++. - // We detect this the same way clang.cppm's enrich_toolchain does: - // by checking the target triple for "msvc". - bool msvc_target = tc.targetTriple.find("msvc") != std::string::npos; + bool msvc_target = is_msvc_target(tc); caps.has_scan_deps = true; // clang-scan-deps lives beside clang++ caps.stdlib_id = msvc_target ? "msvc-stl" : "libc++"; diff --git a/src/toolchain/stdmod.cppm b/src/toolchain/stdmod.cppm index 0220d75..b72ed07 100644 --- a/src/toolchain/stdmod.cppm +++ b/src/toolchain/stdmod.cppm @@ -93,26 +93,57 @@ std::expected ensure_built( sm.objectPath = sm.cacheDir / "std.o"; // Build sysroot + include flags for std module precompilation. - // On macOS, xlings LLVM's clang++.cfg contains hardcoded --sysroot and - // -isystem paths from the original install location. When the LLVM package - // is copied to mcpp's sandbox, these cfg paths become stale (still point - // to the original xlings directory). We override both: - // --sysroot → current active SDK (from xcrun) - // --no-default-config → ignore stale cfg entirely - // -isystem → correct libc++ headers in the sandbox copy + // + // Payload-first: use fine-grained -isystem paths from xpkgs payloads + // when available, falling back to --sysroot. + // + // For Clang with a cfg file: use --no-default-config to bypass + // potentially-stale paths, then provide all flags explicitly. + // Std module precompilation only needs compile flags (no linker flags), + // so --no-default-config is safe here on all platforms. std::string sysroot_flag; - bool is_macos = tc.targetTriple.find("apple") != std::string::npos - || tc.targetTriple.find("darwin") != std::string::npos; - if (is_macos && is_clang(tc)) { - // Ignore the stale clang++.cfg and provide correct flags directly. - auto llvmRoot = tc.binaryPath.parent_path().parent_path(); - auto libcxxInclude = llvmRoot / "include" / "c++" / "v1"; - sysroot_flag = " --no-default-config"; - sysroot_flag += std::format(" -isystem'{}'", libcxxInclude.string()); - if (auto sdk = mcpp::platform::macos::sdk_path()) - sysroot_flag += std::format(" --sysroot='{}'", sdk->string()); + if (is_clang(tc)) { + auto cfgPath = tc.binaryPath.parent_path() + / (tc.binaryPath.stem().string() + ".cfg"); + if (std::filesystem::exists(cfgPath)) { + auto llvmRoot = tc.binaryPath.parent_path().parent_path(); + auto libcxxInclude = llvmRoot / "include" / "c++" / "v1"; + sysroot_flag = " --no-default-config -nostdinc++ -stdlib=libc++"; + sysroot_flag += std::format(" -isystem'{}'", libcxxInclude.string()); + if (!tc.targetTriple.empty()) { + auto targetInclude = llvmRoot / "include" + / tc.targetTriple / "c++" / "v1"; + if (std::filesystem::exists(targetInclude)) + sysroot_flag += std::format(" -isystem'{}'", targetInclude.string()); + } + // C library + kernel headers from payload paths. + if (tc.payloadPaths) { + sysroot_flag += std::format(" -isystem'{}'", tc.payloadPaths->glibcInclude.string()); + if (!tc.payloadPaths->linuxInclude.empty()) + sysroot_flag += std::format(" -isystem'{}'", tc.payloadPaths->linuxInclude.string()); + } else if (auto sdk = mcpp::platform::macos::sdk_path()) { + sysroot_flag += std::format(" --sysroot='{}'", sdk->string()); + } else if (!tc.sysroot.empty()) { + sysroot_flag += std::format(" --sysroot='{}'", tc.sysroot.string()); + } + } else if (!tc.sysroot.empty()) { + sysroot_flag = std::format(" --sysroot='{}'", tc.sysroot.string()); + } } else if (!tc.sysroot.empty()) { + // GCC: use --sysroot (required for include-fixed headers). + // Supplement with -isystem for linux kernel headers from payload + // if the probed sysroot is missing them. sysroot_flag = std::format(" --sysroot='{}'", tc.sysroot.string()); + if (tc.payloadPaths && !tc.payloadPaths->linuxInclude.empty()) { + auto sysrootLinux = tc.sysroot / "usr" / "include" / "linux" / "limits.h"; + if (!std::filesystem::exists(sysrootLinux)) + sysroot_flag += std::format(" -isystem'{}'", tc.payloadPaths->linuxInclude.string()); + } + } else if (tc.payloadPaths) { + // No sysroot: use payload -isystem paths. + sysroot_flag += std::format(" -isystem'{}'", tc.payloadPaths->glibcInclude.string()); + if (!tc.payloadPaths->linuxInclude.empty()) + sysroot_flag += std::format(" -isystem'{}'", tc.payloadPaths->linuxInclude.string()); } bool std_cached = std::filesystem::exists(sm.bmiPath) && std::filesystem::exists(sm.objectPath); diff --git a/src/xlings.cppm b/src/xlings.cppm index 572848e..0d58e5a 100644 --- a/src/xlings.cppm +++ b/src/xlings.cppm @@ -75,6 +75,13 @@ namespace paths { std::string_view tool, std::string_view binaryRelPath); + // Find a sibling package across all index prefixes. + // e.g. find_sibling_package(gcc_bin, "linux-headers") searches for + // xim-x-linux-headers, scode-x-linux-headers, etc. + std::optional + find_sibling_package(const std::filesystem::path& compilerBin, + std::string_view packageName); + // index data root: env.home / "data" std::filesystem::path index_data(const Env& env); @@ -384,6 +391,61 @@ find_sibling_binary(const std::filesystem::path& compilerBin, return std::nullopt; } +std::optional +find_sibling_package(const std::filesystem::path& compilerBin, + std::string_view packageName) { + auto xpkgs = xpkgs_from_compiler(compilerBin); + if (!xpkgs) return std::nullopt; + + // Search across index prefixes: xim-x-, scode-x-, compat-x-, etc. + std::error_code ec; + std::string suffix = std::format("-x-{}", packageName); + for (auto& entry : std::filesystem::directory_iterator(*xpkgs, ec)) { + if (!entry.is_directory(ec)) continue; + auto name = entry.path().filename().string(); + if (!name.ends_with(suffix)) continue; + // Return the first (highest) version dir that has actual content. + for (auto& v : std::filesystem::directory_iterator(entry.path(), ec)) { + if (!v.is_directory(ec)) continue; + // Skip empty packages (only .xim-installed marker) + bool hasContent = false; + for (auto& f : std::filesystem::directory_iterator(v.path(), ec)) { + if (f.path().filename() != ".xim-installed") { + hasContent = true; + break; + } + } + if (hasContent) return v.path(); + } + } + + // Also check ~/.xlings/data/xpkgs/ (xlings global home) as fallback. + const char* home = std::getenv("HOME"); + if (home) { + auto xlingsXpkgs = std::filesystem::path(home) / ".xlings" / "data" / "xpkgs"; + if (xlingsXpkgs != *xpkgs && std::filesystem::exists(xlingsXpkgs, ec)) { + for (auto& entry : std::filesystem::directory_iterator(xlingsXpkgs, ec)) { + if (!entry.is_directory(ec)) continue; + auto name = entry.path().filename().string(); + if (!name.ends_with(suffix)) continue; + for (auto& v : std::filesystem::directory_iterator(entry.path(), ec)) { + if (!v.is_directory(ec)) continue; + bool hasContent = false; + for (auto& f : std::filesystem::directory_iterator(v.path(), ec)) { + if (f.path().filename() != ".xim-installed") { + hasContent = true; + break; + } + } + if (hasContent) return v.path(); + } + } + } + } + + return std::nullopt; +} + std::filesystem::path index_data(const Env& env) { return env.home / "data"; }