From 9e882830b6bfe036712d3abfebd825a9e65ed846 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 09:58:27 +0200 Subject: [PATCH 01/14] docs: design for GitLab ::include directive expansion Co-Authored-By: Claude Opus 4.8 Signed-off-by: Thomas Decaux --- ...6-07-04-gitlab-include-directive-design.md | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-04-gitlab-include-directive-design.md diff --git a/docs/superpowers/specs/2026-07-04-gitlab-include-directive-design.md b/docs/superpowers/specs/2026-07-04-gitlab-include-directive-design.md new file mode 100644 index 0000000..907dfab --- /dev/null +++ b/docs/superpowers/specs/2026-07-04-gitlab-include-directive-design.md @@ -0,0 +1,204 @@ +# GitLab `::include` directive expansion — design + +- **Status:** approved (brainstorm), pending implementation plan +- **Date:** 2026-07-04 +- **Package:** `@ebuildy/docusaurus-plugin-gitlab` + +## Problem + +GitLab Flavored Markdown supports an **include directive** that splices one +document's content into another +([docs](https://docs.gitlab.com/user/markdown/#includes)): + +```text +::include{file=chapter1.md} +::include{file=https://example.org/installation.md} +``` + +Our plugin fetches GitLab READMEs and files at build time and splices them into +Docusaurus pages, but it does not yet understand `::include`. When a fetched +README (or markdown file) contains `::include{file=…}`, the directive is passed +through verbatim and renders as literal text instead of pulling in the +referenced content. + +We must expand `::include` so the referenced file's content is included **raw in +the main markdown** — i.e. spliced as markdown source that then compiles as part +of the host document. + +## Scope + +`::include` is authored **inside GitLab-hosted markdown** (a README or a `.md` +file), not in the author's `.mdx` page. It is therefore a feature of the +**webpack-loader include subsystem** (`src/include/`), which already fetches +GitLab sources and rewrites the host `.mdx` source at load time: + +```text +gitlabIncludeLoader (src/include/loader.ts) + └─ transformIncludes (src/include/transform.ts) // finds {@includeGitlab…} + ├─ fetchReadmeSource / fetchFileSource // raw GitLab source + └─ renderSource (src/include/render-source.ts) // → MDX-safe markdown source +``` + +`renderSource` produces **markdown source text** (not HTML) that is spliced back +into the host `.mdx` and compiled by Docusaurus. That is exactly the "raw in the +main markdown" semantics we want, so `::include` expansion belongs here. + +**In scope:** expansion inside both `{@includeGitlabReadme}` and markdown +`{@includeGitlabFile}` includes (any content for which `isMarkdownSource` is +true). + +**Out of scope:** the JSX-component HTML path (`src/gitlab/markdown.ts` +`renderMarkdown`, used by `` / `` React components). +`::include` is not expanded there. + +## Grammar + +A leaf directive occupying its own line, matching GitLab: + +```text +::include{file=} +``` + +- Only the `file` attribute is honored. Its value may be bare + (`file=chapter1.md`) or quoted (`file="path with space.md"` / single quotes). +- No `ref`, line-range, or `project` extensions (YAGNI). A relative include + inherits the **`ref` of the enclosing include** and targets the **same + project**, matching GitLab's same-repo semantics. +- The directive is recognized only as a standalone block (its own line). A + `::include{…}` appearing inside a fenced/indented/inline code region is left + **literal** and never expanded. + +## Design + +### New module: `src/include/expand.ts` + +Exports `expandFileIncludes(md, o, guard)`: + +- `md: string` — markdown source (host include body, already frontmatter-stripped). +- `o` — resolution context: `{ ctx: GitLabContext; project: string; ref: string; + allowedHosts: string[]; strict: boolean }`. +- `guard` — recursion guard: `{ depth: number; stack: Set }`. +- Returns the markdown with every `::include{…}` directive replaced by the raw + (recursively expanded) content of its target. + +### Hook point: `renderSource` + +In `src/include/render-source.ts`, the `isMarkdownSource` branch calls +`expandFileIncludes` **after `stripFrontmatter`, before `processMarkdownSource`**: + +```ts +if (isMarkdownSource(o.kind, o.path)) { + let body = stripFrontmatter(raw); + body = await expandFileIncludes(body, { + ctx: o.ctx, project: o.project, ref: o.ref, + allowedHosts: o.allowedHosts, strict: o.strict, + }, { depth: 0, stack: new Set([keyFor(o)]) }); + return processMarkdownSource(body, { localizeImage, absolutizeLink }); +} +``` + +Consequences of this ordering: + +- Included content is spliced as **raw text**, then the merged document flows + through the *same* prose transforms (image localization, link absolutization, + MDX escaping) as the host — one coherent document. +- Relative images/links inside included files are resolved the same way as the + host README's (project-root-relative, consistent with the existing + `absolutizeFactory` behavior and its current limitation). + +### Resolution per directive + +For each `::include{file=X}` found in a non-code prose region: + +- **Remote** — `X` matches `^https?://`: parse the URL, check its host against + `allowedHosts`. If not listed → error. If listed → fetch the body via native + `fetch` (text). +- **Relative** — otherwise: `fetchFileSource(ctx, { project: o.project, path: X, + ref: o.ref })`, reusing the existing GitLab client and on-disk cache. + +Fetched markdown is `stripFrontmatter`-ed and **recursively expanded** before +being spliced. + +### Code-fence safety + +Expansion runs only over non-code regions. Reuse the existing +`codeRanges` / `mapProseRegions` approach from `render-source.ts` so a +`::include{…}` inside a code block is preserved verbatim. + +### Recursion guard + +`guard = { depth, stack }`: + +- `depth` increments per nesting level; exceeding `MAX_INCLUDE_DEPTH` (constant, + `8`) throws `"::include exceeded max depth (8)"`. +- `stack` holds the resolution keys currently being expanded, keyed by + `` `${project}@${ref}/-/${path}` `` for GitLab files and by the absolute URL + for remote includes. Re-entering a key already on the stack throws + `"::include cycle detected: "`. Keys are removed as each branch unwinds + (a diamond — the same file included twice in different branches — is allowed; + only true cycles are rejected). + +### Security + +New plugin option `includeAllowedHosts: string[]`, **default `[]`**: + +- Empty ⇒ remote (`http(s)`) includes are rejected; only GitLab-project-file + includes work. +- Hosts are matched exactly (case-insensitive) against the URL's host. +- Threaded through `src/options.ts` (`PluginOptions`, `ResolvedOptions`, Joi + schema, `resolveOptions` default `[]`) → `loader.ts` → `transformIncludes` + (`TransformOptions`) → `renderSource` (`RenderSourceOptions.allowedHosts`). + +Local GitLab-file includes are always permitted; they are already gated by the +configured `host`/`token`. + +### Error handling + +Mirrors `transformIncludes`: + +- `strict: true` (production default) → throw; the error bubbles to + `transformIncludes`'s catch, aborting the build with a + `@ebuildy/docusaurus-plugin-gitlab: …` message. +- `strict: false` → replace the offending directive with an inline marker + `> ⚠️ ::include{file=…} failed — ` and continue. + +Failure cases: unlisted remote host, fetch/network error, missing file, cycle, +depth exceeded, malformed directive. + +## Testing (TDD) + +`src/include/expand.test.ts` (fake/mocked `fetchFileSource` and `fetch`): + +- Relative-file include expands to the fetched file's raw content. +- Remote include from an allowlisted host expands; from a non-allowlisted host + errors (strict) / emits the warning marker (non-strict). +- Nested include (an included file containing `::include`) expands recursively. +- Cycle (A includes B includes A) throws `"cycle detected"`. +- Depth beyond `MAX_INCLUDE_DEPTH` throws. +- `::include{…}` inside a fenced code block is left literal. +- Quoted and bare `file=` values both parse. +- Strict vs non-strict error behavior. + +Wiring tests in `src/include/render-source.test.ts`: + +- `renderSource` expands `::include` for markdown sources and passes + `allowedHosts` through. +- Non-markdown (code) includes are unaffected. + +Update `src/include/transform.test.ts` / `src/options.test.ts` as needed for the +new option threading. Run `npx vitest run` and `npm run typecheck`; run the e2e +build (`test/e2e/build.test.ts`) since the loader pipeline is touched. + +## Documentation + +- README: document `::include{file=…}` support and the `includeAllowedHosts` + option. +- Add an example under `examples/site/` exercising a relative include (and, if + practical, a nested include). + +## Non-goals / future work + +- No `ref`, line-range, or `project` attributes on `::include`. +- No expansion in the JSX-component HTML render path. +- File-relative (as opposed to project-root-relative) image/link resolution is + not addressed here; it inherits the existing behavior and limitation. From 7b692911ead65072dbeea5b9f97200ce071fc747 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 10:42:43 +0200 Subject: [PATCH 02/14] docs: implementation plan for GitLab ::include directive Co-Authored-By: Claude Opus 4.8 Signed-off-by: Thomas Decaux --- .../2026-07-04-gitlab-include-directive.md | 828 ++++++++++++++++++ 1 file changed, 828 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-04-gitlab-include-directive.md diff --git a/docs/superpowers/plans/2026-07-04-gitlab-include-directive.md b/docs/superpowers/plans/2026-07-04-gitlab-include-directive.md new file mode 100644 index 0000000..1bd0c73 --- /dev/null +++ b/docs/superpowers/plans/2026-07-04-gitlab-include-directive.md @@ -0,0 +1,828 @@ +# GitLab `::include` Directive Expansion — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Expand GitLab `::include{file=…}` directives found inside fetched GitLab READMEs/markdown files into the host document as raw markdown source, at build time. + +**Architecture:** `::include` lives inside GitLab markdown pulled in by the webpack-loader include subsystem (`src/include/`). A new `src/include/expand.ts` scans the fetched raw markdown for `::include{file=…}` leaf directives (skipping code regions), resolves each target — a GitLab project file (via the existing `fetchFileSource`) or an allowlisted remote URL (via native `fetch`) — and splices its raw content in place, recursively, with depth + cycle guards. `transformIncludes` (`src/include/transform.ts`) calls the expander on the fetched raw source *before* `renderSource`, so the merged document flows through the existing prose transforms and out-processors as one unit. Hooking in `transform.ts` (not inside `renderSource`) keeps `render-source.ts` free of a back-import and avoids a module cycle. + +**Tech Stack:** TypeScript (ESM, `.js` import extensions), Vitest, unified/remark (only reused indirectly via existing `codeRanges`), native `fetch` (Node 20+), Joi (options), `@gitbeaker/rest`. + +**Note on the spec:** the design doc places the hook "inside `renderSource`". During file-structure review this was moved one level out to `transformIncludes` to avoid a circular import between `expand.ts` and `render-source.ts` (`expand.ts` imports `codeRanges`/`stripFrontmatter` from `render-source.ts`). The behavior is identical — expansion still happens on the raw source before `processMarkdownSource` runs on the merged document. + +--- + +## File Structure + +- **Create** `src/include/expand.ts` — directive scanning, target resolution (GitLab file / remote URL), recursion + guards, error handling. One responsibility: turn raw markdown containing `::include` into raw markdown with those directives expanded. +- **Create** `src/include/expand.test.ts` — unit tests for the above (mocked `fetchFileSource` + `fetch`). +- **Modify** `src/options.ts` — add `includeAllowedHosts` option (interface, Joi, default). +- **Modify** `src/options.test.ts` — cover the new option's default + validation. +- **Modify** `src/include/transform.ts` — add `allowedHosts` to `TransformOptions`; call `expandFileIncludes` on the fetched raw source for markdown includes. +- **Modify** `src/include/transform.test.ts` — cover expansion wiring. +- **Modify** `src/include/loader.ts` — pass `resolved.includeAllowedHosts` into `transformIncludes`. +- **Modify** `README.md` and `examples/gitlab/docs/includes.mdx` — document the feature. + +--- + +## Task 1: Add the `includeAllowedHosts` plugin option + +**Files:** +- Modify: `src/options.ts` +- Test: `src/options.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Add to `src/options.test.ts` inside the `describe("resolveOptions", …)` block: + +```ts +it("defaults includeAllowedHosts to an empty array", () => { + const o = resolveOptions({ host: "https://gitlab.com" }, "production"); + expect(o.includeAllowedHosts).toEqual([]); +}); + +it("passes through a configured includeAllowedHosts list", () => { + const o = resolveOptions( + { host: "https://gitlab.com", includeAllowedHosts: ["example.org"] }, + "production", + ); + expect(o.includeAllowedHosts).toEqual(["example.org"]); +}); + +it("rejects a non-array includeAllowedHosts", () => { + expect(() => + resolveOptions( + { host: "https://gitlab.com", includeAllowedHosts: "example.org" } as any, + "production", + ), + ).toThrow(); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run src/options.test.ts` +Expected: FAIL — `includeAllowedHosts` is `undefined` / not validated. + +- [ ] **Step 3: Implement the option** + +In `src/options.ts`: + +Add to the `PluginOptions` interface (after `stripToc`): + +```ts + /** Hostnames (exact, case-insensitive) allowed as remote `::include{file=https://…}` + * targets inside fetched GitLab markdown. Empty ⇒ remote includes are rejected. + * Default: `[]`. */ + includeAllowedHosts?: string[]; +``` + +Add to the `ResolvedOptions` interface (after `stripToc: boolean;`): + +```ts + includeAllowedHosts: string[]; +``` + +Add to the Joi `schema` object (after `stripToc: …`): + +```ts + includeAllowedHosts: Joi.array().items(Joi.string()).optional(), +``` + +Add to the `resolveOptions` return object (after `stripToc: …`): + +```ts + includeAllowedHosts: opts.includeAllowedHosts ?? [], +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `npx vitest run src/options.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/options.ts src/options.test.ts +git commit -S -m "feat: add includeAllowedHosts plugin option" +``` + +--- + +## Task 2: Directive attribute parser + +**Files:** +- Create: `src/include/expand.ts` +- Test: `src/include/expand.test.ts` + +- [ ] **Step 1: Write the failing test** + +Create `src/include/expand.test.ts`: + +```ts +import { describe, it, expect } from "vitest"; +import { parseIncludeAttrs } from "./expand.js"; + +describe("parseIncludeAttrs", () => { + it("reads a bare file value", () => { + expect(parseIncludeAttrs("file=chapter1.md")).toEqual({ file: "chapter1.md" }); + }); + it("reads a double-quoted file value with spaces", () => { + expect(parseIncludeAttrs('file="a b.md"')).toEqual({ file: "a b.md" }); + }); + it("reads a single-quoted file value", () => { + expect(parseIncludeAttrs("file='c.md'")).toEqual({ file: "c.md" }); + }); + it("reads a URL value", () => { + expect(parseIncludeAttrs("file=https://example.org/x.md")).toEqual({ + file: "https://example.org/x.md", + }); + }); + it("returns empty when file is absent", () => { + expect(parseIncludeAttrs("other=1")).toEqual({}); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: FAIL — cannot import `parseIncludeAttrs` (module missing). + +- [ ] **Step 3: Implement the parser** + +Create `src/include/expand.ts`: + +```ts +import { fetchFileSource } from "../gitlab/fetchers.js"; +import type { GitLabContext } from "../gitlab/fetchers.js"; +import { codeRanges, stripFrontmatter } from "./render-source.js"; + +/** Markdown file extensions whose content is expanded recursively as markdown. */ +const MD_EXT = /\.(?:md|mdx|markdown)$/i; + +/** Extract the `file=` value from a `::include{…}` attribute string (bare or quoted). */ +export function parseIncludeAttrs(attrs: string): { file?: string } { + const m = /(?:^|\s)file\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s}]+))/.exec(attrs); + if (!m) return {}; + return { file: m[1] ?? m[2] ?? m[3] }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/include/expand.ts src/include/expand.test.ts +git commit -S -m "feat: parse ::include directive attributes" +``` + +--- + +## Task 3: Expand relative GitLab-file includes (single level, code-fence safe) + +**Files:** +- Modify: `src/include/expand.ts` +- Test: `src/include/expand.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Append to `src/include/expand.test.ts` (add imports at the top of the file: `import { vi } from "vitest";` and `import { expandFileIncludes, type ExpandContext, type ExpandGuard } from "./expand.js";`): + +```ts +vi.mock("../gitlab/fetchers.js", () => ({ + fetchFileSource: vi.fn(), +})); +import { fetchFileSource } from "../gitlab/fetchers.js"; + +const mockedFetchFileSource = vi.mocked(fetchFileSource); + +function baseCtx(overrides: Partial = {}): ExpandContext { + return { + ctx: {} as any, + project: "group/proj", + ref: "main", + allowedHosts: [], + strict: true, + ...overrides, + }; +} + +function baseGuard(): ExpandGuard { + return { depth: 0, stack: new Set(["group/proj@main/-/README.md"]) }; +} + +describe("expandFileIncludes — relative GitLab files", () => { + beforeEach(() => mockedFetchFileSource.mockReset()); + + it("replaces a directive with the fetched file's raw content", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "Chapter one body.", ref: "main" }); + const out = await expandFileIncludes( + "Intro\n\n::include{file=chapter1.md}\n\nOutro", + baseCtx(), + baseGuard(), + ); + expect(out).toContain("Chapter one body."); + expect(out).not.toContain("::include"); + expect(mockedFetchFileSource).toHaveBeenCalledWith({} as any, { + project: "group/proj", + path: "chapter1.md", + ref: "main", + }); + }); + + it("strips frontmatter from the included file", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "---\ntitle: x\n---\nBody", ref: "main" }); + const out = await expandFileIncludes("::include{file=a.md}", baseCtx(), baseGuard()); + expect(out).toContain("Body"); + expect(out).not.toContain("title: x"); + }); + + it("leaves a directive inside a fenced code block untouched", async () => { + const md = "```\n::include{file=chapter1.md}\n```"; + const out = await expandFileIncludes(md, baseCtx(), baseGuard()); + expect(out).toBe(md); + expect(mockedFetchFileSource).not.toHaveBeenCalled(); + }); + + it("returns the source unchanged when there is no directive", async () => { + const out = await expandFileIncludes("plain text", baseCtx(), baseGuard()); + expect(out).toBe("plain text"); + expect(mockedFetchFileSource).not.toHaveBeenCalled(); + }); +}); +``` + +Also add `import { beforeEach } from "vitest";` to the existing vitest import line. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: FAIL — `expandFileIncludes` / types not exported. + +- [ ] **Step 3: Implement single-level expansion** + +Append to `src/include/expand.ts`: + +```ts +export interface ExpandContext { + ctx: GitLabContext; + project: string; + ref: string; + allowedHosts: string[]; + strict: boolean; +} + +export interface ExpandGuard { + depth: number; + stack: Set; +} + +/** A standalone `::include{…}` leaf directive occupying its own line. */ +const INCLUDE_RE = /^[ \t]*::include\{([^}\n]*)\}[ \t]*$/gm; + +/** Resolve a directive's target to raw text plus a cycle key and markdown flag. */ +async function resolveTarget( + file: string, + o: ExpandContext, +): Promise<{ raw: string; key: string; isMarkdown: boolean }> { + const key = `${o.project}@${o.ref}/-/${file}`; + const src = await fetchFileSource(o.ctx, { project: o.project, path: file, ref: o.ref }); + return { raw: src.raw, key, isMarkdown: MD_EXT.test(file) }; +} + +/** Resolve a single directive to its replacement text, honoring `strict`. */ +async function resolveOne( + full: string, + attrs: string, + o: ExpandContext, + _guard: ExpandGuard, +): Promise { + try { + const { file } = parseIncludeAttrs(attrs); + if (!file) throw new Error("::include missing file= attribute"); + const { raw } = await resolveTarget(file, o); + return stripFrontmatter(raw); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (o.strict) { + throw new Error(`@ebuildy/docusaurus-plugin-gitlab: ${full} failed — ${message}`); + } + return `\n\n> ⚠️ ${full} failed — ${message}\n\n`; + } +} + +/** + * Replace every standalone `::include{file=…}` directive in `md` (outside code + * regions) with the raw content of its target. `guard` carries recursion state. + */ +export async function expandFileIncludes( + md: string, + o: ExpandContext, + guard: ExpandGuard, +): Promise { + const ranges = codeRanges(md); + const inCode = (i: number) => ranges.some(([s, e]) => i >= s && i < e); + + const matches = [...md.matchAll(INCLUDE_RE)].filter((m) => !inCode(m.index ?? 0)); + if (matches.length === 0) return md; + + const replacements = await Promise.all( + matches.map((m) => resolveOne(m[0], m[1], o, guard)), + ); + + let out = ""; + let last = 0; + matches.forEach((m, i) => { + out += md.slice(last, m.index) + replacements[i]; + last = (m.index ?? 0) + m[0].length; + }); + return out + md.slice(last); +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/include/expand.ts src/include/expand.test.ts +git commit -S -m "feat: expand relative GitLab-file ::include directives" +``` + +--- + +## Task 4: Remote URL includes with host allowlist + +**Files:** +- Modify: `src/include/expand.ts` +- Test: `src/include/expand.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Append to `src/include/expand.test.ts`: + +```ts +describe("expandFileIncludes — remote URLs", () => { + beforeEach(() => { + mockedFetchFileSource.mockReset(); + vi.unstubAllGlobals(); + }); + + it("expands a remote include from an allowlisted host", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ ok: true, status: 200, text: async () => "Remote body." })), + ); + const out = await expandFileIncludes( + "::include{file=https://example.org/x.md}", + baseCtx({ allowedHosts: ["example.org"] }), + baseGuard(), + ); + expect(out).toContain("Remote body."); + }); + + it("rejects a remote include from a non-allowlisted host (strict throws)", async () => { + await expect( + expandFileIncludes( + "::include{file=https://evil.test/x.md}", + baseCtx({ allowedHosts: ["example.org"], strict: true }), + baseGuard(), + ), + ).rejects.toThrow(/host not allowed/); + }); + + it("emits a warning marker for a non-allowlisted host when non-strict", async () => { + const out = await expandFileIncludes( + "::include{file=https://evil.test/x.md}", + baseCtx({ allowedHosts: ["example.org"], strict: false }), + baseGuard(), + ); + expect(out).toContain("⚠️"); + expect(out).toContain("host not allowed"); + }); + + it("errors on a non-OK remote response", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ ok: false, status: 404, text: async () => "" })), + ); + await expect( + expandFileIncludes( + "::include{file=https://example.org/missing.md}", + baseCtx({ allowedHosts: ["example.org"], strict: true }), + baseGuard(), + ), + ).rejects.toThrow(/404/); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: FAIL — remote branch not implemented (relative path resolution tries `fetchFileSource` with a URL). + +- [ ] **Step 3: Add the remote branch to `resolveTarget`** + +In `src/include/expand.ts`, replace the body of `resolveTarget` with: + +```ts +async function resolveTarget( + file: string, + o: ExpandContext, +): Promise<{ raw: string; key: string; isMarkdown: boolean }> { + if (/^https?:\/\//i.test(file)) { + const url = new URL(file); + if (!o.allowedHosts.some((h) => h.toLowerCase() === url.host.toLowerCase())) { + throw new Error(`::include host not allowed: ${url.host}`); + } + const res = await fetch(url); + if (!res.ok) throw new Error(`fetch ${url.href} → HTTP ${res.status}`); + return { raw: await res.text(), key: url.href, isMarkdown: MD_EXT.test(url.pathname) }; + } + const key = `${o.project}@${o.ref}/-/${file}`; + const src = await fetchFileSource(o.ctx, { project: o.project, path: file, ref: o.ref }); + return { raw: src.raw, key, isMarkdown: MD_EXT.test(file) }; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/include/expand.ts src/include/expand.test.ts +git commit -S -m "feat: support allowlisted remote ::include URLs" +``` + +--- + +## Task 5: Recursion, cycle detection, and depth guard + +**Files:** +- Modify: `src/include/expand.ts` +- Test: `src/include/expand.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Append to `src/include/expand.test.ts`: + +```ts +describe("expandFileIncludes — recursion + guards", () => { + beforeEach(() => mockedFetchFileSource.mockReset()); + + it("recursively expands includes inside an included markdown file", async () => { + mockedFetchFileSource.mockImplementation(async (_ctx, args: any) => { + if (args.path === "a.md") return { raw: "A\n\n::include{file=b.md}", ref: "main" }; + if (args.path === "b.md") return { raw: "B body", ref: "main" }; + throw new Error(`unexpected ${args.path}`); + }); + const out = await expandFileIncludes("::include{file=a.md}", baseCtx(), baseGuard()); + expect(out).toContain("A"); + expect(out).toContain("B body"); + expect(out).not.toContain("::include"); + }); + + it("detects a direct cycle", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "loop\n\n::include{file=a.md}", ref: "main" }); + await expect( + expandFileIncludes("::include{file=a.md}", baseCtx({ strict: true }), baseGuard()), + ).rejects.toThrow(/cycle detected/); + }); + + it("enforces the max depth", async () => { + // Every file includes a deeper one, never terminating. + mockedFetchFileSource.mockImplementation(async (_ctx, args: any) => ({ + raw: `level\n\n::include{file=${args.path}x.md}`, + ref: "main", + })); + await expect( + expandFileIncludes("::include{file=a.md}", baseCtx({ strict: true }), baseGuard()), + ).rejects.toThrow(/max depth/); + }); + + it("allows the same file included twice in separate branches (diamond)", async () => { + mockedFetchFileSource.mockImplementation(async (_ctx, args: any) => { + if (args.path === "top.md") + return { raw: "::include{file=shared.md}\n\n::include{file=shared.md}", ref: "main" }; + if (args.path === "shared.md") return { raw: "SHARED", ref: "main" }; + throw new Error(`unexpected ${args.path}`); + }); + const out = await expandFileIncludes("::include{file=top.md}", baseCtx(), baseGuard()); + expect(out.match(/SHARED/g)?.length).toBe(2); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: FAIL — recursion not wired; the cycle/depth tests hang or return unexpanded nested directives. + +- [ ] **Step 3: Add the depth guard, cycle check, and recursion** + +In `src/include/expand.ts`: + +Add the constant near `MD_EXT`: + +```ts +/** Maximum nesting depth for recursive `::include` expansion. */ +export const MAX_INCLUDE_DEPTH = 8; +``` + +Replace `resolveOne` with: + +```ts +async function resolveOne( + full: string, + attrs: string, + o: ExpandContext, + guard: ExpandGuard, +): Promise { + try { + const { file } = parseIncludeAttrs(attrs); + if (!file) throw new Error("::include missing file= attribute"); + const { raw, key, isMarkdown } = await resolveTarget(file, o); + if (guard.stack.has(key)) throw new Error(`::include cycle detected: ${key}`); + let body = stripFrontmatter(raw); + if (isMarkdown) { + body = await expandFileIncludes(body, o, { + depth: guard.depth + 1, + stack: new Set(guard.stack).add(key), + }); + } + return body; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (o.strict) { + throw new Error(`@ebuildy/docusaurus-plugin-gitlab: ${full} failed — ${message}`); + } + return `\n\n> ⚠️ ${full} failed — ${message}\n\n`; + } +} +``` + +Add the depth check as the first statement of `expandFileIncludes`: + +```ts +export async function expandFileIncludes( + md: string, + o: ExpandContext, + guard: ExpandGuard, +): Promise { + if (guard.depth > MAX_INCLUDE_DEPTH) { + throw new Error(`::include exceeded max depth (${MAX_INCLUDE_DEPTH})`); + } + const ranges = codeRanges(md); + // …rest unchanged… +``` + +Note: because `resolveOne` catches errors, the depth/cycle errors thrown from a +nested `expandFileIncludes` are surfaced through the nearest enclosing directive +— under `strict` they abort the build; otherwise they become a warning marker. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/include/expand.ts src/include/expand.test.ts +git commit -S -m "feat: recurse ::include with cycle + depth guards" +``` + +--- + +## Task 6: Wire expansion into the include transform pipeline + +**Files:** +- Modify: `src/include/transform.ts` +- Modify: `src/include/loader.ts` +- Test: `src/include/transform.test.ts` + +- [ ] **Step 1: Write the failing test** + +Open `src/include/transform.test.ts` and inspect how it constructs a fake `GitLabContext` and calls `transformIncludes` (follow the existing pattern in that file — it already mocks the client/fetchers). Add a test that a `::include` inside a fetched README is expanded: + +```ts +it("expands ::include directives inside a fetched README", async () => { + // Arrange a context whose README fetch returns markdown containing a directive, + // and whose chapter fetch returns the included body. Use the same fake-context + // helper already used by the other tests in this file. + const ctx = makeCtx({ + files: { + "group/proj@main/-/README.md": "# Title\n\n::include{file=chapter1.md}", + "group/proj@main/-/chapter1.md": "Chapter one body.", + }, + }); + + const out = await transformIncludes( + "{@includeGitlabReadme: group/proj}", + ctx, + { strict: true, allowedHosts: [] }, + ); + + expect(out).toContain("Chapter one body."); + expect(out).not.toContain("::include"); +}); +``` + +Adapt `makeCtx`/helper names to whatever `transform.test.ts` already defines. If the existing helper cannot express per-path file contents, extend it minimally to do so (matching the file-keying `project@ref/-/path` used by `fetchFileSource`). + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `npx vitest run src/include/transform.test.ts` +Expected: FAIL — the README's `::include` is passed through verbatim (still contains `::include`). + +- [ ] **Step 3: Wire `expandFileIncludes` into `transformIncludes`** + +In `src/include/transform.ts`: + +Add the import (next to the other `./` imports): + +```ts +import { expandFileIncludes } from "./expand.js"; +``` + +Add `allowedHosts` to `TransformOptions` (after `outProcessors?: …`): + +```ts + /** Hostnames allowed as remote `::include{file=https://…}` targets. Default: none. */ + allowedHosts?: string[]; +``` + +Inside the `Promise.all(...)` callback in `transformIncludes`, replace: + +```ts + const { raw, ref } = + kind === "readme" + ? await fetchReadmeSource(ctx, { project: spec.project, ref: spec.ref }) + : await fetchFileSource(ctx, { project: spec.project, path: spec.path!, ref: spec.ref }); + let body = await renderSource(raw, { + ctx, + project: spec.project, + ref, + kind, + path: spec.path, + lineRange: spec.lineRange, + }); +``` + +with: + +```ts + const { raw, ref } = + kind === "readme" + ? await fetchReadmeSource(ctx, { project: spec.project, ref: spec.ref }) + : await fetchFileSource(ctx, { project: spec.project, path: spec.path!, ref: spec.ref }); + let expanded = raw; + if (isMarkdownSource(kind, spec.path)) { + expanded = await expandFileIncludes( + raw, + { + ctx, + project: spec.project, + ref, + allowedHosts: options.allowedHosts ?? [], + strict: options.strict, + }, + { + depth: 0, + stack: new Set([`${spec.project}@${ref}/-/${spec.path ?? "README.md"}`]), + }, + ); + } + let body = await renderSource(expanded, { + ctx, + project: spec.project, + ref, + kind, + path: spec.path, + lineRange: spec.lineRange, + }); +``` + +In `src/include/loader.ts`, add `allowedHosts` to the `options` object built for `transformIncludes` (after `stripToc: resolved.stripToc,`): + +```ts + allowedHosts: resolved.includeAllowedHosts, +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `npx vitest run src/include/transform.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/include/transform.ts src/include/loader.ts src/include/transform.test.ts +git commit -S -m "feat: expand ::include inside loader include pipeline" +``` + +--- + +## Task 7: Full test + typecheck + e2e verification + +**Files:** none (verification only) + +- [ ] **Step 1: Run the full unit suite** + +Run: `npx vitest run` +Expected: PASS (all suites, including the new `expand.test.ts` and updated option/transform tests). + +- [ ] **Step 2: Typecheck** + +Run: `npm run typecheck` +Expected: no errors. + +- [ ] **Step 3: Build** + +Run: `npm run build` +Expected: compiles to `dist/` with no errors. + +- [ ] **Step 4: e2e build (loader pipeline is touched)** + +Run: `npx vitest run test/e2e/build.test.ts` +Expected: PASS (~1 min). Existing includes still render; the added feature does not regress the build. + +- [ ] **Step 5: Commit any incidental fixes** + +If steps 1–4 required small fixes, commit them: + +```bash +git add -A +git commit -S -m "test: verify ::include expansion end-to-end" +``` + +If nothing changed, skip this step. + +--- + +## Task 8: Documentation + +**Files:** +- Modify: `README.md` +- Modify: `examples/gitlab/docs/includes.mdx` + +- [ ] **Step 1: Document the feature in README** + +Add a subsection under the includes documentation in `README.md` explaining: + +```markdown +### `::include` directives inside included markdown + +When a fetched GitLab README or markdown file contains a GitLab +[`::include`](https://docs.gitlab.com/user/markdown/#includes) directive, the +plugin expands it at build time, splicing the referenced file in as raw +markdown: + + ::include{file=docs/chapter1.md} + +- Relative paths resolve to a file in the **same GitLab project and ref** as the + enclosing include, fetched through the same cached client. +- Remote URLs (`::include{file=https://…}`) are fetched only when their host is + listed in the `includeAllowedHosts` plugin option (empty by default, so remote + includes are off until you opt in). +- Includes are expanded recursively (max depth 8) with cycle detection. +- A failed include aborts the build in `strict` mode, or renders an inline + warning otherwise. +``` + +- [ ] **Step 2: Note the behavior in the example site's includes page** + +Add a short paragraph to `examples/gitlab/docs/includes.mdx` explaining that any +`::include` directives inside the fetched READMEs are expanded automatically +(no separate placeholder needed). Keep it documentation-only — do not add an +include that depends on upstream content containing `::include`. + +- [ ] **Step 3: Commit** + +```bash +git add README.md examples/gitlab/docs/includes.mdx +git commit -S -m "docs: document ::include directive expansion" +``` + +--- + +## Definition of Done + +- `::include{file=…}` inside fetched README/markdown-file includes expands to the referenced content as raw markdown, in both `{@includeGitlabReadme}` and markdown `{@includeGitlabFile}` includes. +- Relative paths resolve to the same project/ref via `fetchFileSource`; remote URLs work only for allowlisted hosts. +- Recursion works with depth (8) and cycle guards; directives inside code fences stay literal. +- `strict` throws on failure; non-strict emits an inline warning marker. +- `npx vitest run`, `npm run typecheck`, `npm run build`, and the e2e build all pass. +- README and example docs updated. From f86f868a0bc448dfa0902801f957a70cb6753e92 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 10:45:20 +0200 Subject: [PATCH 03/14] feat: add includeAllowedHosts plugin option Add plugin option to specify allowed remote hosts for ::include directives. Defaults to empty array (all remote includes rejected). Co-Authored-By: Claude Opus 4.8 Signed-off-by: Thomas Decaux --- src/options.test.ts | 22 ++++++++++++++++++++++ src/options.ts | 7 +++++++ 2 files changed, 29 insertions(+) diff --git a/src/options.test.ts b/src/options.test.ts index 72e79b5..d200dde 100644 --- a/src/options.test.ts +++ b/src/options.test.ts @@ -89,4 +89,26 @@ describe("resolveOptions", () => { ); expect("outProcessors" in resolved).toBe(false); }); + + it("defaults includeAllowedHosts to an empty array", () => { + const o = resolveOptions({ host: "https://gitlab.com" }, "production"); + expect(o.includeAllowedHosts).toEqual([]); + }); + + it("passes through a configured includeAllowedHosts list", () => { + const o = resolveOptions( + { host: "https://gitlab.com", includeAllowedHosts: ["example.org"] }, + "production", + ); + expect(o.includeAllowedHosts).toEqual(["example.org"]); + }); + + it("rejects a non-array includeAllowedHosts", () => { + expect(() => + resolveOptions( + { host: "https://gitlab.com", includeAllowedHosts: "example.org" } as any, + "production", + ), + ).toThrow(); + }); }); diff --git a/src/options.ts b/src/options.ts index 2daec3b..d6ef135 100644 --- a/src/options.ts +++ b/src/options.ts @@ -23,6 +23,10 @@ export interface PluginOptions { /** Remove a redundant "Table of Contents" section (and any `[[_TOC_]]` marker) * from included markdown — Docusaurus renders its own. Default: `false`. */ stripToc?: boolean; + /** Hostnames (exact, case-insensitive) allowed as remote `::include{file=https://…}` + * targets inside fetched GitLab markdown. Empty ⇒ remote includes are rejected. + * Default: `[]`. */ + includeAllowedHosts?: string[]; /** Extra post-processors applied to the markdown generated from includes, * in order, after the built-in fixes (when enabled). */ outProcessors?: OutProcessor[]; @@ -40,6 +44,7 @@ export interface ResolvedOptions { fixInlineStyles: boolean; convertAlerts: boolean; stripToc: boolean; + includeAllowedHosts: string[]; } const schema = Joi.object({ @@ -60,6 +65,7 @@ const schema = Joi.object({ fixInlineStyles: Joi.boolean().optional(), convertAlerts: Joi.boolean().optional(), stripToc: Joi.boolean().optional(), + includeAllowedHosts: Joi.array().items(Joi.string()).optional(), outProcessors: Joi.array().items(Joi.function()).optional(), }); @@ -83,5 +89,6 @@ export function resolveOptions( fixInlineStyles: opts.fixInlineStyles ?? true, convertAlerts: opts.convertAlerts ?? true, stripToc: opts.stripToc ?? false, + includeAllowedHosts: opts.includeAllowedHosts ?? [], }; } From 78788a9b5ea7cbc0ab33e9ca8e81140d09ec13e8 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 10:47:44 +0200 Subject: [PATCH 04/14] feat: parse ::include directive attributes Co-Authored-By: Claude Opus 4.8 Signed-off-by: Thomas Decaux --- src/include/expand.test.ts | 22 ++++++++++++++++++++++ src/include/expand.ts | 18 ++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 src/include/expand.test.ts create mode 100644 src/include/expand.ts diff --git a/src/include/expand.test.ts b/src/include/expand.test.ts new file mode 100644 index 0000000..18a4f3a --- /dev/null +++ b/src/include/expand.test.ts @@ -0,0 +1,22 @@ +import { describe, it, expect } from "vitest"; +import { parseIncludeAttrs } from "./expand.js"; + +describe("parseIncludeAttrs", () => { + it("reads a bare file value", () => { + expect(parseIncludeAttrs("file=chapter1.md")).toEqual({ file: "chapter1.md" }); + }); + it("reads a double-quoted file value with spaces", () => { + expect(parseIncludeAttrs('file="a b.md"')).toEqual({ file: "a b.md" }); + }); + it("reads a single-quoted file value", () => { + expect(parseIncludeAttrs("file='c.md'")).toEqual({ file: "c.md" }); + }); + it("reads a URL value", () => { + expect(parseIncludeAttrs("file=https://example.org/x.md")).toEqual({ + file: "https://example.org/x.md", + }); + }); + it("returns empty when file is absent", () => { + expect(parseIncludeAttrs("other=1")).toEqual({}); + }); +}); diff --git a/src/include/expand.ts b/src/include/expand.ts new file mode 100644 index 0000000..12398c1 --- /dev/null +++ b/src/include/expand.ts @@ -0,0 +1,18 @@ +// These imports are used by later tasks (Task 3+); kept here for code organization +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import { fetchFileSource } from "../gitlab/fetchers.js"; +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import type { GitLabContext } from "../gitlab/fetchers.js"; +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import { codeRanges, stripFrontmatter } from "./render-source.js"; + +/** Markdown file extensions whose content is expanded recursively as markdown. */ +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const MD_EXT = /\.(?:md|mdx|markdown)$/i; + +/** Extract the `file=` value from a `::include{…}` attribute string (bare or quoted). */ +export function parseIncludeAttrs(attrs: string): { file?: string } { + const m = /(?:^|\s)file\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s}]+))/.exec(attrs); + if (!m) return {}; + return { file: m[1] ?? m[2] ?? m[3] }; +} From 970cc63d2a65fb7409fbf74d6db9ee25bc57a8b4 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 10:50:38 +0200 Subject: [PATCH 05/14] feat: expand relative GitLab-file ::include directives Signed-off-by: Thomas Decaux --- src/include/expand.test.ts | 66 ++++++++++++++++++++++++++++++- src/include/expand.ts | 81 +++++++++++++++++++++++++++++++++++--- 2 files changed, 140 insertions(+), 7 deletions(-) diff --git a/src/include/expand.test.ts b/src/include/expand.test.ts index 18a4f3a..ea71d05 100644 --- a/src/include/expand.test.ts +++ b/src/include/expand.test.ts @@ -1,5 +1,5 @@ -import { describe, it, expect } from "vitest"; -import { parseIncludeAttrs } from "./expand.js"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { parseIncludeAttrs, expandFileIncludes, type ExpandContext, type ExpandGuard } from "./expand.js"; describe("parseIncludeAttrs", () => { it("reads a bare file value", () => { @@ -20,3 +20,65 @@ describe("parseIncludeAttrs", () => { expect(parseIncludeAttrs("other=1")).toEqual({}); }); }); + +vi.mock("../gitlab/fetchers.js", () => ({ + fetchFileSource: vi.fn(), +})); +import { fetchFileSource } from "../gitlab/fetchers.js"; + +const mockedFetchFileSource = vi.mocked(fetchFileSource); + +function baseCtx(overrides: Partial = {}): ExpandContext { + return { + ctx: {} as any, + project: "group/proj", + ref: "main", + allowedHosts: [], + strict: true, + ...overrides, + }; +} + +function baseGuard(): ExpandGuard { + return { depth: 0, stack: new Set(["group/proj@main/-/README.md"]) }; +} + +describe("expandFileIncludes — relative GitLab files", () => { + beforeEach(() => mockedFetchFileSource.mockReset()); + + it("replaces a directive with the fetched file's raw content", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "Chapter one body.", ref: "main" }); + const out = await expandFileIncludes( + "Intro\n\n::include{file=chapter1.md}\n\nOutro", + baseCtx(), + baseGuard(), + ); + expect(out).toContain("Chapter one body."); + expect(out).not.toContain("::include"); + expect(mockedFetchFileSource).toHaveBeenCalledWith({} as any, { + project: "group/proj", + path: "chapter1.md", + ref: "main", + }); + }); + + it("strips frontmatter from the included file", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "---\ntitle: x\n---\nBody", ref: "main" }); + const out = await expandFileIncludes("::include{file=a.md}", baseCtx(), baseGuard()); + expect(out).toContain("Body"); + expect(out).not.toContain("title: x"); + }); + + it("leaves a directive inside a fenced code block untouched", async () => { + const md = "```\n::include{file=chapter1.md}\n```"; + const out = await expandFileIncludes(md, baseCtx(), baseGuard()); + expect(out).toBe(md); + expect(mockedFetchFileSource).not.toHaveBeenCalled(); + }); + + it("returns the source unchanged when there is no directive", async () => { + const out = await expandFileIncludes("plain text", baseCtx(), baseGuard()); + expect(out).toBe("plain text"); + expect(mockedFetchFileSource).not.toHaveBeenCalled(); + }); +}); diff --git a/src/include/expand.ts b/src/include/expand.ts index 12398c1..a7cec2d 100644 --- a/src/include/expand.ts +++ b/src/include/expand.ts @@ -1,13 +1,9 @@ -// These imports are used by later tasks (Task 3+); kept here for code organization -// eslint-disable-next-line @typescript-eslint/no-unused-vars import { fetchFileSource } from "../gitlab/fetchers.js"; -// eslint-disable-next-line @typescript-eslint/no-unused-vars import type { GitLabContext } from "../gitlab/fetchers.js"; -// eslint-disable-next-line @typescript-eslint/no-unused-vars import { codeRanges, stripFrontmatter } from "./render-source.js"; /** Markdown file extensions whose content is expanded recursively as markdown. */ -// eslint-disable-next-line @typescript-eslint/no-unused-vars + const MD_EXT = /\.(?:md|mdx|markdown)$/i; /** Extract the `file=` value from a `::include{…}` attribute string (bare or quoted). */ @@ -16,3 +12,78 @@ export function parseIncludeAttrs(attrs: string): { file?: string } { if (!m) return {}; return { file: m[1] ?? m[2] ?? m[3] }; } + +export interface ExpandContext { + ctx: GitLabContext; + project: string; + ref: string; + allowedHosts: string[]; + strict: boolean; +} + +export interface ExpandGuard { + depth: number; + stack: Set; +} + +/** A standalone `::include{…}` leaf directive occupying its own line. */ +const INCLUDE_RE = /^[ \t]*::include\{([^}\n]*)\}[ \t]*$/gm; + +/** Resolve a directive's target to raw text plus a cycle key and markdown flag. */ +async function resolveTarget( + file: string, + o: ExpandContext, +): Promise<{ raw: string; key: string; isMarkdown: boolean }> { + const key = `${o.project}@${o.ref}/-/${file}`; + const src = await fetchFileSource(o.ctx, { project: o.project, path: file, ref: o.ref }); + return { raw: src.raw, key, isMarkdown: MD_EXT.test(file) }; +} + +/** Resolve a single directive to its replacement text, honoring `strict`. */ +async function resolveOne( + full: string, + attrs: string, + o: ExpandContext, + _guard: ExpandGuard, +): Promise { + try { + const { file } = parseIncludeAttrs(attrs); + if (!file) throw new Error("::include missing file= attribute"); + const { raw } = await resolveTarget(file, o); + return stripFrontmatter(raw); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (o.strict) { + throw new Error(`@ebuildy/docusaurus-plugin-gitlab: ${full} failed — ${message}`); + } + return `\n\n> ⚠️ ${full} failed — ${message}\n\n`; + } +} + +/** + * Replace every standalone `::include{file=…}` directive in `md` (outside code + * regions) with the raw content of its target. `guard` carries recursion state. + */ +export async function expandFileIncludes( + md: string, + o: ExpandContext, + guard: ExpandGuard, +): Promise { + const ranges = codeRanges(md); + const inCode = (i: number) => ranges.some(([s, e]) => i >= s && i < e); + + const matches = [...md.matchAll(INCLUDE_RE)].filter((m) => !inCode(m.index ?? 0)); + if (matches.length === 0) return md; + + const replacements = await Promise.all( + matches.map((m) => resolveOne(m[0], m[1], o, guard)), + ); + + let out = ""; + let last = 0; + matches.forEach((m, i) => { + out += md.slice(last, m.index) + replacements[i]; + last = (m.index ?? 0) + m[0].length; + }); + return out + md.slice(last); +} From 807d6541570fcd805abbac6621c2830189fb5a9f Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 10:52:53 +0200 Subject: [PATCH 06/14] feat: support allowlisted remote ::include URLs Signed-off-by: Thomas Decaux --- src/include/expand.test.ts | 54 ++++++++++++++++++++++++++++++++++++++ src/include/expand.ts | 10 ++++++- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/include/expand.test.ts b/src/include/expand.test.ts index ea71d05..994c020 100644 --- a/src/include/expand.test.ts +++ b/src/include/expand.test.ts @@ -82,3 +82,57 @@ describe("expandFileIncludes — relative GitLab files", () => { expect(mockedFetchFileSource).not.toHaveBeenCalled(); }); }); + +describe("expandFileIncludes — remote URLs", () => { + beforeEach(() => { + mockedFetchFileSource.mockReset(); + vi.unstubAllGlobals(); + }); + + it("expands a remote include from an allowlisted host", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ ok: true, status: 200, text: async () => "Remote body." })), + ); + const out = await expandFileIncludes( + "::include{file=https://example.org/x.md}", + baseCtx({ allowedHosts: ["example.org"] }), + baseGuard(), + ); + expect(out).toContain("Remote body."); + }); + + it("rejects a remote include from a non-allowlisted host (strict throws)", async () => { + await expect( + expandFileIncludes( + "::include{file=https://evil.test/x.md}", + baseCtx({ allowedHosts: ["example.org"], strict: true }), + baseGuard(), + ), + ).rejects.toThrow(/host not allowed/); + }); + + it("emits a warning marker for a non-allowlisted host when non-strict", async () => { + const out = await expandFileIncludes( + "::include{file=https://evil.test/x.md}", + baseCtx({ allowedHosts: ["example.org"], strict: false }), + baseGuard(), + ); + expect(out).toContain("⚠️"); + expect(out).toContain("host not allowed"); + }); + + it("errors on a non-OK remote response", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ ok: false, status: 404, text: async () => "" })), + ); + await expect( + expandFileIncludes( + "::include{file=https://example.org/missing.md}", + baseCtx({ allowedHosts: ["example.org"], strict: true }), + baseGuard(), + ), + ).rejects.toThrow(/404/); + }); +}); diff --git a/src/include/expand.ts b/src/include/expand.ts index a7cec2d..f23a6c5 100644 --- a/src/include/expand.ts +++ b/src/include/expand.ts @@ -3,7 +3,6 @@ import type { GitLabContext } from "../gitlab/fetchers.js"; import { codeRanges, stripFrontmatter } from "./render-source.js"; /** Markdown file extensions whose content is expanded recursively as markdown. */ - const MD_EXT = /\.(?:md|mdx|markdown)$/i; /** Extract the `file=` value from a `::include{…}` attribute string (bare or quoted). */ @@ -34,6 +33,15 @@ async function resolveTarget( file: string, o: ExpandContext, ): Promise<{ raw: string; key: string; isMarkdown: boolean }> { + if (/^https?:\/\//i.test(file)) { + const url = new URL(file); + if (!o.allowedHosts.some((h) => h.toLowerCase() === url.host.toLowerCase())) { + throw new Error(`::include host not allowed: ${url.host}`); + } + const res = await fetch(url); + if (!res.ok) throw new Error(`fetch ${url.href} → HTTP ${res.status}`); + return { raw: await res.text(), key: url.href, isMarkdown: MD_EXT.test(url.pathname) }; + } const key = `${o.project}@${o.ref}/-/${file}`; const src = await fetchFileSource(o.ctx, { project: o.project, path: file, ref: o.ref }); return { raw: src.raw, key, isMarkdown: MD_EXT.test(file) }; From 24ce87513180be12d4dc2dcfb0def4d10d3fc06f Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 10:59:36 +0200 Subject: [PATCH 07/14] feat: recurse ::include with cycle + depth guards Expand nested ::include directives found inside included markdown files, guarded by a max depth cap and a per-branch cycle-detection stack so a diamond-shaped include graph still resolves while a self-referential one fails fast. Signed-off-by: Thomas Decaux --- src/include/expand.test.ts | 47 ++++++++++++++++++++++++++++++++++++++ src/include/expand.ts | 20 +++++++++++++--- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/include/expand.test.ts b/src/include/expand.test.ts index 994c020..50c05bf 100644 --- a/src/include/expand.test.ts +++ b/src/include/expand.test.ts @@ -136,3 +136,50 @@ describe("expandFileIncludes — remote URLs", () => { ).rejects.toThrow(/404/); }); }); + +describe("expandFileIncludes — recursion + guards", () => { + beforeEach(() => { + mockedFetchFileSource.mockReset(); + }); + + it("recursively expands includes inside an included markdown file", async () => { + mockedFetchFileSource.mockImplementation(async (_ctx, args: any) => { + if (args.path === "a.md") return { raw: "A\n\n::include{file=b.md}", ref: "main" }; + if (args.path === "b.md") return { raw: "B body", ref: "main" }; + throw new Error(`unexpected ${args.path}`); + }); + const out = await expandFileIncludes("::include{file=a.md}", baseCtx(), baseGuard()); + expect(out).toContain("A"); + expect(out).toContain("B body"); + expect(out).not.toContain("::include"); + }); + + it("detects a direct cycle", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "loop\n\n::include{file=a.md}", ref: "main" }); + await expect( + expandFileIncludes("::include{file=a.md}", baseCtx({ strict: true }), baseGuard()), + ).rejects.toThrow(/cycle detected/); + }); + + it("enforces the max depth", async () => { + // Every file includes a deeper one, never terminating. + mockedFetchFileSource.mockImplementation(async (_ctx, args: any) => ({ + raw: `level\n\n::include{file=${args.path}x.md}`, + ref: "main", + })); + await expect( + expandFileIncludes("::include{file=a.md}", baseCtx({ strict: true }), baseGuard()), + ).rejects.toThrow(/max depth/); + }); + + it("allows the same file included twice in separate branches (diamond)", async () => { + mockedFetchFileSource.mockImplementation(async (_ctx, args: any) => { + if (args.path === "top.md") + return { raw: "::include{file=shared.md}\n\n::include{file=shared.md}", ref: "main" }; + if (args.path === "shared.md") return { raw: "SHARED", ref: "main" }; + throw new Error(`unexpected ${args.path}`); + }); + const out = await expandFileIncludes("::include{file=top.md}", baseCtx(), baseGuard()); + expect(out.match(/SHARED/g)?.length).toBe(2); + }); +}); diff --git a/src/include/expand.ts b/src/include/expand.ts index f23a6c5..79d5481 100644 --- a/src/include/expand.ts +++ b/src/include/expand.ts @@ -5,6 +5,9 @@ import { codeRanges, stripFrontmatter } from "./render-source.js"; /** Markdown file extensions whose content is expanded recursively as markdown. */ const MD_EXT = /\.(?:md|mdx|markdown)$/i; +/** Maximum nesting depth for recursive `::include` expansion. */ +export const MAX_INCLUDE_DEPTH = 8; + /** Extract the `file=` value from a `::include{…}` attribute string (bare or quoted). */ export function parseIncludeAttrs(attrs: string): { file?: string } { const m = /(?:^|\s)file\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s}]+))/.exec(attrs); @@ -52,13 +55,21 @@ async function resolveOne( full: string, attrs: string, o: ExpandContext, - _guard: ExpandGuard, + guard: ExpandGuard, ): Promise { try { const { file } = parseIncludeAttrs(attrs); if (!file) throw new Error("::include missing file= attribute"); - const { raw } = await resolveTarget(file, o); - return stripFrontmatter(raw); + const { raw, key, isMarkdown } = await resolveTarget(file, o); + if (guard.stack.has(key)) throw new Error(`::include cycle detected: ${key}`); + let body = stripFrontmatter(raw); + if (isMarkdown) { + body = await expandFileIncludes(body, o, { + depth: guard.depth + 1, + stack: new Set(guard.stack).add(key), + }); + } + return body; } catch (err) { const message = err instanceof Error ? err.message : String(err); if (o.strict) { @@ -77,6 +88,9 @@ export async function expandFileIncludes( o: ExpandContext, guard: ExpandGuard, ): Promise { + if (guard.depth > MAX_INCLUDE_DEPTH) { + throw new Error(`::include exceeded max depth (${MAX_INCLUDE_DEPTH})`); + } const ranges = codeRanges(md); const inCode = (i: number) => ranges.some(([s, e]) => i >= s && i < e); From 3f0588c1b51491e8b07b2b25b862968f64656022 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 11:02:43 +0200 Subject: [PATCH 08/14] feat: expand ::include inside loader include pipeline Signed-off-by: Thomas Decaux --- src/include/loader.ts | 1 + src/include/transform.test.ts | 30 +++++++++++++++++++++++++++--- src/include/transform.ts | 22 +++++++++++++++++++++- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/include/loader.ts b/src/include/loader.ts index dec351e..2e57e25 100644 --- a/src/include/loader.ts +++ b/src/include/loader.ts @@ -24,6 +24,7 @@ export default function gitlabIncludeLoader(this: LoaderThis, source: string): v fixInlineStyles: resolved.fixInlineStyles, convertAlerts: resolved.convertAlerts, stripToc: resolved.stripToc, + allowedHosts: resolved.includeAllowedHosts, outProcessors: processorsId ? getOutProcessors(processorsId) : [], }; transformIncludes(source, getContext(resolved), options).then( diff --git a/src/include/transform.test.ts b/src/include/transform.test.ts index c3a9760..9b80290 100644 --- a/src/include/transform.test.ts +++ b/src/include/transform.test.ts @@ -1,13 +1,19 @@ import { describe, it, expect } from "vitest"; import { transformIncludes } from "./transform.js"; -function makeCtx() { +function makeCtx(opts?: { files?: Record }) { const store = new Map(); + const files = opts?.files; return { client: { getProject: async () => ({ default_branch: "main" }), - getFileRaw: async (_p: unknown, path: string) => - path === "README.md" ? "# Title\n\nbody {x}" : "const a = 1;\nconst b = 2;\n", + getFileRaw: async (project: unknown, path: string, ref: string) => { + if (files) { + const key = `${project}@${ref}/-/${path}`; + if (key in files) return files[key]; + } + return path === "README.md" ? "# Title\n\nbody {x}" : "const a = 1;\nconst b = 2;\n"; + }, }, cache: { get: async (k: string) => store.get(k), set: async (k: string, v: unknown) => void store.set(k, v) }, assets: { localize: async (u: string) => `/a/${u}` }, @@ -180,6 +186,24 @@ describe("transformIncludes", () => { expect(out).toContain("x [processed]"); }); + it("expands ::include directives inside a fetched README", async () => { + const ctx = makeCtx({ + files: { + "group/proj@main/-/README.md": "# Title\n\n::include{file=chapter1.md}", + "group/proj@main/-/chapter1.md": "Chapter one body.", + }, + }); + + const out = await transformIncludes( + "{@includeGitlabReadme: group/proj}", + ctx, + { strict: true, allowedHosts: [] } as any, + ); + + expect(out).toContain("Chapter one body."); + expect(out).not.toContain("::include"); + }); + it("does not run processors on a code-file include", async () => { const ctx = { client: { diff --git a/src/include/transform.ts b/src/include/transform.ts index 37eadb6..3dc7b1d 100644 --- a/src/include/transform.ts +++ b/src/include/transform.ts @@ -1,5 +1,6 @@ import type { GitLabContext } from "../gitlab/fetchers.js"; import { fetchFileSource, fetchReadmeSource } from "../gitlab/fetchers.js"; +import { expandFileIncludes } from "./expand.js"; import { parseInclude } from "./grammar.js"; import { applyOutProcessors, @@ -28,6 +29,8 @@ export interface TransformOptions { stripToc?: boolean; /** Extra post-processors applied to the generated markdown of each markdown include. */ outProcessors?: OutProcessor[]; + /** Hostnames allowed as remote `::include{file=https://…}` targets. Default: none. */ + allowedHosts?: string[]; } export async function transformIncludes( @@ -67,7 +70,24 @@ export async function transformIncludes( kind === "readme" ? await fetchReadmeSource(ctx, { project: spec.project, ref: spec.ref }) : await fetchFileSource(ctx, { project: spec.project, path: spec.path!, ref: spec.ref }); - let body = await renderSource(raw, { + let expanded = raw; + if (isMarkdownSource(kind, spec.path)) { + expanded = await expandFileIncludes( + raw, + { + ctx, + project: spec.project, + ref, + allowedHosts: options.allowedHosts ?? [], + strict: options.strict, + }, + { + depth: 0, + stack: new Set([`${spec.project}@${ref}/-/${spec.path ?? "README.md"}`]), + }, + ); + } + let body = await renderSource(expanded, { ctx, project: spec.project, ref, From ee673aa24e4b5243eb12858e92cf7b55fb3b30c4 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 11:04:54 +0200 Subject: [PATCH 09/14] test: tidy ::include test setup (braced beforeEach, drop stray cast) Co-Authored-By: Claude Opus 4.8 Signed-off-by: Thomas Decaux --- src/include/expand.test.ts | 4 +++- src/include/transform.test.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/include/expand.test.ts b/src/include/expand.test.ts index 50c05bf..9ec1708 100644 --- a/src/include/expand.test.ts +++ b/src/include/expand.test.ts @@ -44,7 +44,9 @@ function baseGuard(): ExpandGuard { } describe("expandFileIncludes — relative GitLab files", () => { - beforeEach(() => mockedFetchFileSource.mockReset()); + beforeEach(() => { + mockedFetchFileSource.mockReset(); + }); it("replaces a directive with the fetched file's raw content", async () => { mockedFetchFileSource.mockResolvedValue({ raw: "Chapter one body.", ref: "main" }); diff --git a/src/include/transform.test.ts b/src/include/transform.test.ts index 9b80290..396e5f4 100644 --- a/src/include/transform.test.ts +++ b/src/include/transform.test.ts @@ -197,7 +197,7 @@ describe("transformIncludes", () => { const out = await transformIncludes( "{@includeGitlabReadme: group/proj}", ctx, - { strict: true, allowedHosts: [] } as any, + { strict: true, allowedHosts: [] }, ); expect(out).toContain("Chapter one body."); From 2d18f8938f2739c4be7e23bd0a4eceac4e1fa98c Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 11:07:12 +0200 Subject: [PATCH 10/14] docs: document ::include directive expansion Signed-off-by: Thomas Decaux --- README.md | 21 +++++++++++++++++++++ examples/gitlab/docs/includes.mdx | 8 ++++++++ 2 files changed, 29 insertions(+) diff --git a/README.md b/README.md index 370fd49..b9ff9b9 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,26 @@ Both components render [scoped labels/topics](https://docs.gitlab.com/ee/user/pr (`scope::value`, e.g. `Abilities::Performance`) as a two-part badge — the scope keeps its color and the value gets a dark-gray background. The split is on the last `::`. +### `::include` directives inside included markdown + +When a fetched GitLab README or markdown file contains a GitLab +[`::include`](https://docs.gitlab.com/user/markdown/#includes) directive, the +plugin expands it at build time, splicing the referenced file in as raw +markdown: + +```text +::include{file=docs/chapter1.md} +``` + +- Relative paths resolve to a file in the **same GitLab project and ref** as the + enclosing include, fetched through the same cached client. +- Remote URLs (`::include{file=https://…}`) are fetched only when their host is + listed in the `includeAllowedHosts` plugin option (empty by default, so remote + includes are off until you opt in). +- Includes are expanded recursively (max depth 8) with cycle detection. +- A failed include aborts the build in `strict` mode, or renders an inline + warning otherwise. + ## Plugin options | Option | Type | Default | Description | @@ -250,6 +270,7 @@ color and the value gets a dark-gray background. The split is on the last `::`. | `convertAlerts` | boolean | `true` | Translate GitLab alert blockquotes (`> [!note]`) to Docusaurus admonitions (`:::note`) in included markdown | | `stripToc` | boolean | `false` | Remove a redundant "Table of Contents" section (and `[[_TOC_]]` marker) from included markdown | | `outProcessors` | `Array<(md: string) => string \| Promise>` | `[]` | Extra post-processors for included markdown, run after the built-in fixes | +| `includeAllowedHosts` | `string[]` | `[]` | Hostnames allowed as remote `::include{file=https://…}` targets | The token is read at build time only. Provide it via an environment variable (`GITLAB_TOKEN`) — never commit it. diff --git a/examples/gitlab/docs/includes.mdx b/examples/gitlab/docs/includes.mdx index ce0200a..fd581d0 100644 --- a/examples/gitlab/docs/includes.mdx +++ b/examples/gitlab/docs/includes.mdx @@ -21,3 +21,11 @@ A non-markdown file is wrapped in a fenced code block with the language inferred from its extension; the `#L1-40` range trims it to the first 40 lines: {@includeGitlabFile: gitlab-org/api/client-go/-/gitlab.go#L1-40} + +## Nested `::include` directives + +If a fetched README or file above contained a GitLab `::include{file=…}` +directive, it would be expanded automatically as part of the same fetch — no +extra placeholder needed. Relative targets resolve against the enclosing +file's own project and ref; remote URLs are only followed when their host is +allow-listed via the `includeAllowedHosts` plugin option. From 506a08d69492374cf8ee9f77197e3ba63a0704a9 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 11:13:27 +0200 Subject: [PATCH 11/14] fix: block ::include redirect bypass and non-markdown frontmatter stripping - Remote ::include fetches now use redirect: "manual" and reject 3xx responses explicitly, closing an allowlist-bypass path where an allowlisted host could redirect the build-time request elsewhere. - Frontmatter stripping and recursive expansion now only apply to markdown targets, so non-markdown includes (e.g. .yml) keep a legitimate leading "---" block intact and are spliced in verbatim without being scanned for ::include directives. Signed-off-by: Thomas Decaux --- src/include/expand.test.ts | 51 ++++++++++++++++++++++++++++++++++++++ src/include/expand.ts | 19 +++++++------- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/src/include/expand.test.ts b/src/include/expand.test.ts index 9ec1708..bdce563 100644 --- a/src/include/expand.test.ts +++ b/src/include/expand.test.ts @@ -185,3 +185,54 @@ describe("expandFileIncludes — recursion + guards", () => { expect(out.match(/SHARED/g)?.length).toBe(2); }); }); + +describe("expandFileIncludes — robustness fixes", () => { + beforeEach(() => { + mockedFetchFileSource.mockReset(); + vi.unstubAllGlobals(); + }); + + it("does not strip a leading --- block from a non-markdown include", async () => { + mockedFetchFileSource.mockResolvedValue({ + raw: "---\nname: prod\nport: 8080\n---\nrest: true", + ref: "main", + }); + const out = await expandFileIncludes("::include{file=config/settings.yml}", baseCtx(), baseGuard()); + expect(out).toContain("name: prod"); + expect(out).toContain("port: 8080"); + }); + + it("does not recurse into a non-markdown include's ::include-looking content", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "::include{file=evil.md}", ref: "main" }); + const out = await expandFileIncludes("::include{file=data.txt}", baseCtx(), baseGuard()); + // .txt is not markdown → spliced raw, not expanded + expect(out).toContain("::include{file=evil.md}"); + // fetched exactly once (the .txt), never followed the inner directive + expect(mockedFetchFileSource).toHaveBeenCalledTimes(1); + }); + + it("blocks a remote include that responds with a redirect (allowlist bypass prevention)", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ ok: false, status: 302, text: async () => "" })), + ); + await expect( + expandFileIncludes( + "::include{file=https://example.org/redirect}", + baseCtx({ allowedHosts: ["example.org"], strict: true }), + baseGuard(), + ), + ).rejects.toThrow(/redirect blocked/); + }); + + it("passes redirect:manual to fetch so redirects are not auto-followed", async () => { + const fetchMock = vi.fn(async () => ({ ok: true, status: 200, text: async () => "ok body" })); + vi.stubGlobal("fetch", fetchMock); + await expandFileIncludes( + "::include{file=https://example.org/x.md}", + baseCtx({ allowedHosts: ["example.org"] }), + baseGuard(), + ); + expect(fetchMock).toHaveBeenCalledWith(expect.anything(), { redirect: "manual" }); + }); +}); diff --git a/src/include/expand.ts b/src/include/expand.ts index 79d5481..065a03e 100644 --- a/src/include/expand.ts +++ b/src/include/expand.ts @@ -41,7 +41,10 @@ async function resolveTarget( if (!o.allowedHosts.some((h) => h.toLowerCase() === url.host.toLowerCase())) { throw new Error(`::include host not allowed: ${url.host}`); } - const res = await fetch(url); + const res = await fetch(url, { redirect: "manual" }); + if (res.status >= 300 && res.status < 400) { + throw new Error(`::include remote redirect blocked: ${url.href} → ${res.status}`); + } if (!res.ok) throw new Error(`fetch ${url.href} → HTTP ${res.status}`); return { raw: await res.text(), key: url.href, isMarkdown: MD_EXT.test(url.pathname) }; } @@ -62,14 +65,12 @@ async function resolveOne( if (!file) throw new Error("::include missing file= attribute"); const { raw, key, isMarkdown } = await resolveTarget(file, o); if (guard.stack.has(key)) throw new Error(`::include cycle detected: ${key}`); - let body = stripFrontmatter(raw); - if (isMarkdown) { - body = await expandFileIncludes(body, o, { - depth: guard.depth + 1, - stack: new Set(guard.stack).add(key), - }); - } - return body; + if (!isMarkdown) return raw; + const body = stripFrontmatter(raw); + return expandFileIncludes(body, o, { + depth: guard.depth + 1, + stack: new Set(guard.stack).add(key), + }); } catch (err) { const message = err instanceof Error ? err.message : String(err); if (o.strict) { From 2982922aada4dfd3253b65a1df38bb893bfef543 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 13:11:58 +0200 Subject: [PATCH 12/14] refactor: model ::include expansion as a pre-render source processor Extract ::include expansion into an expandIncludes() factory that returns an OutProcessor-shaped fn, and run it in transform.ts as a pre-render source-processor stage (reusing applyOutProcessors) symmetric to the post-render out-processor chain. Also render non-markdown include targets (e.g. .yaml) as fenced, syntax-highlighted code blocks instead of raw prose, matching {@includeGitlabFile}. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Thomas Decaux --- README.md | 4 ++- src/include/expand.test.ts | 55 ++++++++++++++++++++++++++++++++++++-- src/include/expand.ts | 38 +++++++++++++++++++++++++- src/include/transform.ts | 35 ++++++++++++------------ 4 files changed, 110 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b9ff9b9..cf7167a 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,9 @@ markdown: - Remote URLs (`::include{file=https://…}`) are fetched only when their host is listed in the `includeAllowedHosts` plugin option (empty by default, so remote includes are off until you opt in). -- Includes are expanded recursively (max depth 8) with cycle detection. +- Markdown targets (`.md`/`.mdx`/`.markdown`) are spliced inline and expanded + recursively (max depth 8) with cycle detection. Any other file (e.g. `.yaml`, + `.json`) is inserted as a fenced, syntax-highlighted code block. - A failed include aborts the build in `strict` mode, or renders an inline warning otherwise. diff --git a/src/include/expand.test.ts b/src/include/expand.test.ts index bdce563..81f85f5 100644 --- a/src/include/expand.test.ts +++ b/src/include/expand.test.ts @@ -1,5 +1,11 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; -import { parseIncludeAttrs, expandFileIncludes, type ExpandContext, type ExpandGuard } from "./expand.js"; +import { + parseIncludeAttrs, + expandFileIncludes, + expandIncludes, + type ExpandContext, + type ExpandGuard, +} from "./expand.js"; describe("parseIncludeAttrs", () => { it("reads a bare file value", () => { @@ -205,12 +211,20 @@ describe("expandFileIncludes — robustness fixes", () => { it("does not recurse into a non-markdown include's ::include-looking content", async () => { mockedFetchFileSource.mockResolvedValue({ raw: "::include{file=evil.md}", ref: "main" }); const out = await expandFileIncludes("::include{file=data.txt}", baseCtx(), baseGuard()); - // .txt is not markdown → spliced raw, not expanded + // .txt is not markdown → fenced verbatim, its inner directive never followed expect(out).toContain("::include{file=evil.md}"); // fetched exactly once (the .txt), never followed the inner directive expect(mockedFetchFileSource).toHaveBeenCalledTimes(1); }); + it("wraps a non-markdown include in a fenced code block using its language", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "name: prod\nport: 8080", ref: "main" }); + const out = await expandFileIncludes("::include{file=config/profiles.yaml}", baseCtx(), baseGuard()); + expect(out).toContain("```yaml"); + expect(out).toMatch(/```yaml\n[\s\S]*name: prod[\s\S]*```/); + expect(out).not.toContain("::include"); + }); + it("blocks a remote include that responds with a redirect (allowlist bypass prevention)", async () => { vi.stubGlobal( "fetch", @@ -236,3 +250,40 @@ describe("expandFileIncludes — robustness fixes", () => { expect(fetchMock).toHaveBeenCalledWith(expect.anything(), { redirect: "manual" }); }); }); + +describe("expandIncludes (source processor)", () => { + beforeEach(() => { + mockedFetchFileSource.mockReset(); + }); + + it("returns an OutProcessor-shaped fn that expands includes with its bound ctx", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "Chapter body.", ref: "main" }); + const proc = expandIncludes({ + ctx: {} as any, + project: "group/proj", + ref: "main", + allowedHosts: [], + strict: true, + }); + const out = await proc("::include{file=chapter1.md}"); + expect(out).toContain("Chapter body."); + expect(mockedFetchFileSource).toHaveBeenCalledWith({} as any, { + project: "group/proj", + path: "chapter1.md", + ref: "main", + }); + }); + + it("seeds cycle detection with the host path so a self-referential include is caught", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "loop", ref: "main" }); + const proc = expandIncludes({ + ctx: {} as any, + project: "group/proj", + ref: "main", + path: "docs/page.md", + allowedHosts: [], + strict: true, + }); + await expect(proc("::include{file=docs/page.md}")).rejects.toThrow(/cycle detected/); + }); +}); diff --git a/src/include/expand.ts b/src/include/expand.ts index 065a03e..fa99922 100644 --- a/src/include/expand.ts +++ b/src/include/expand.ts @@ -1,5 +1,7 @@ +import { languageFromPath } from "../gitlab/code.js"; import { fetchFileSource } from "../gitlab/fetchers.js"; import type { GitLabContext } from "../gitlab/fetchers.js"; +import type { OutProcessor } from "./out-processors.js"; import { codeRanges, stripFrontmatter } from "./render-source.js"; /** Markdown file extensions whose content is expanded recursively as markdown. */ @@ -65,7 +67,10 @@ async function resolveOne( if (!file) throw new Error("::include missing file= attribute"); const { raw, key, isMarkdown } = await resolveTarget(file, o); if (guard.stack.has(key)) throw new Error(`::include cycle detected: ${key}`); - if (!isMarkdown) return raw; + // Non-markdown targets are spliced as a fenced, syntax-highlighted code block + // (like `{@includeGitlabFile}` of a code file) rather than as raw prose — raw + // YAML/JSON/etc. would otherwise render as garbled markdown. + if (!isMarkdown) return `\n\`\`\`${languageFromPath(file)}\n${raw}\n\`\`\`\n`; const body = stripFrontmatter(raw); return expandFileIncludes(body, o, { depth: guard.depth + 1, @@ -110,3 +115,34 @@ export async function expandFileIncludes( }); return out + md.slice(last); } + +/** Configuration for the {@link expandIncludes} source processor. */ +export interface ExpandConfig { + ctx: GitLabContext; + project: string; + ref: string; + /** Path of the host include (a markdown file), or undefined for a README. Seeds + * cycle detection so a nested include can't re-enter the host document. */ + path?: string; + allowedHosts: string[]; + strict: boolean; +} + +/** + * Build a pre-render source processor that expands GitLab `::include{file=…}` + * directives in raw fetched markdown, against the given project/ref. It runs + * *before* `renderSource` — the directive's `{…}` braces would be MDX-escaped + * afterward — and reuses the {@link OutProcessor} shape so it composes with the + * post-render processor chain via `applyOutProcessors`. + */ +export function expandIncludes(config: ExpandConfig): OutProcessor { + const o: ExpandContext = { + ctx: config.ctx, + project: config.project, + ref: config.ref, + allowedHosts: config.allowedHosts, + strict: config.strict, + }; + const seed = `${config.project}@${config.ref}/-/${config.path ?? "README.md"}`; + return (md) => expandFileIncludes(md, o, { depth: 0, stack: new Set([seed]) }); +} diff --git a/src/include/transform.ts b/src/include/transform.ts index 3dc7b1d..eb90008 100644 --- a/src/include/transform.ts +++ b/src/include/transform.ts @@ -1,6 +1,6 @@ import type { GitLabContext } from "../gitlab/fetchers.js"; import { fetchFileSource, fetchReadmeSource } from "../gitlab/fetchers.js"; -import { expandFileIncludes } from "./expand.js"; +import { expandIncludes } from "./expand.js"; import { parseInclude } from "./grammar.js"; import { applyOutProcessors, @@ -70,23 +70,22 @@ export async function transformIncludes( kind === "readme" ? await fetchReadmeSource(ctx, { project: spec.project, ref: spec.ref }) : await fetchFileSource(ctx, { project: spec.project, path: spec.path!, ref: spec.ref }); - let expanded = raw; - if (isMarkdownSource(kind, spec.path)) { - expanded = await expandFileIncludes( - raw, - { - ctx, - project: spec.project, - ref, - allowedHosts: options.allowedHosts ?? [], - strict: options.strict, - }, - { - depth: 0, - stack: new Set([`${spec.project}@${ref}/-/${spec.path ?? "README.md"}`]), - }, - ); - } + // Pre-render source processors run on the RAW source before renderSource; + // ::include expansion must happen here (its `{…}` braces would be + // MDX-escaped afterward). Reuses the OutProcessor shape + runner. + const sourceProcessors: OutProcessor[] = isMarkdownSource(kind, spec.path) + ? [ + expandIncludes({ + ctx, + project: spec.project, + ref, + path: spec.path, + allowedHosts: options.allowedHosts ?? [], + strict: options.strict, + }), + ] + : []; + const expanded = await applyOutProcessors(raw, sourceProcessors); let body = await renderSource(expanded, { ctx, project: spec.project, From 363b4e2e621da1f2f140456ad4d2aea39b96bb18 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 13:28:51 +0200 Subject: [PATCH 13/14] feat: add opt-in debug logging for the include pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a `debug` plugin option (default false) that emits build-time traces via @docusaurus/logger: each resolved {@includeGitlab…} placeholder (project/ref/path, byte size) in transform.ts and each ::include directive (target, depth, markdown-vs-fenced) in expand.ts. The logger lives in a small isolated module that lazily imports @docusaurus/logger (an optional peer dependency) only when enabled, with a defensive default-unwrap for the CJS/ESM interop; a no-op logger is used otherwise. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Thomas Decaux --- README.md | 1 + package-lock.json | 22 ++++++++++++++++++++++ package.json | 7 +++++++ src/include/expand.test.ts | 26 ++++++++++++++++++++++++++ src/include/expand.ts | 10 ++++++++++ src/include/loader.ts | 1 + src/include/logger.test.ts | 23 +++++++++++++++++++++++ src/include/logger.ts | 31 +++++++++++++++++++++++++++++++ src/include/transform.ts | 15 +++++++++++++++ src/options.test.ts | 6 ++++++ src/options.ts | 7 +++++++ 11 files changed, 149 insertions(+) create mode 100644 src/include/logger.test.ts create mode 100644 src/include/logger.ts diff --git a/README.md b/README.md index cf7167a..66e7de6 100644 --- a/README.md +++ b/README.md @@ -273,6 +273,7 @@ markdown: | `stripToc` | boolean | `false` | Remove a redundant "Table of Contents" section (and `[[_TOC_]]` marker) from included markdown | | `outProcessors` | `Array<(md: string) => string \| Promise>` | `[]` | Extra post-processors for included markdown, run after the built-in fixes | | `includeAllowedHosts` | `string[]` | `[]` | Hostnames allowed as remote `::include{file=https://…}` targets | +| `debug` | boolean | `false` | Emit build-time debug traces for the include pipeline (resolved placeholders and `::include` directives) via `@docusaurus/logger` | The token is read at build time only. Provide it via an environment variable (`GITLAB_TOKEN`) — never commit it. diff --git a/package-lock.json b/package-lock.json index 8fe9cc9..38607d3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "unist-util-visit-parents": "^6.0.0" }, "devDependencies": { + "@docusaurus/logger": "^3.10.1", "@eslint/js": "^9.39.4", "@testing-library/jest-dom": "^6.4.0", "@testing-library/react": "^16.0.0", @@ -544,6 +545,20 @@ "node": ">=20.19.0" } }, + "node_modules/@docusaurus/logger": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/@docusaurus/logger/-/logger-3.10.1.tgz", + "integrity": "sha512-oPjNFnfJsRCkePVjkGrxWGq4MvJKRQT0r9jOP0eRBTZ7Wr9FAbzdP/Gjs0I2Ss6YRkPoEgygKG112OkE6skvJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=20.0" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.9.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", @@ -8685,6 +8700,13 @@ "json5": "lib/cli.js" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index 3f661a5..dceec62 100644 --- a/package.json +++ b/package.json @@ -50,9 +50,15 @@ "prepare": "husky" }, "peerDependencies": { + "@docusaurus/logger": "^3.0.0", "react": ">=18", "react-dom": ">=18" }, + "peerDependenciesMeta": { + "@docusaurus/logger": { + "optional": true + } + }, "dependencies": { "@gitbeaker/core": "^43.8.0", "@gitbeaker/rest": "^43.8.0", @@ -74,6 +80,7 @@ "unist-util-visit-parents": "^6.0.0" }, "devDependencies": { + "@docusaurus/logger": "^3.10.1", "@eslint/js": "^9.39.4", "@testing-library/jest-dom": "^6.4.0", "@testing-library/react": "^16.0.0", diff --git a/src/include/expand.test.ts b/src/include/expand.test.ts index 81f85f5..d81ca6d 100644 --- a/src/include/expand.test.ts +++ b/src/include/expand.test.ts @@ -251,6 +251,32 @@ describe("expandFileIncludes — robustness fixes", () => { }); }); +describe("expandFileIncludes — debug logging", () => { + beforeEach(() => { + mockedFetchFileSource.mockReset(); + }); + + it("traces each resolved directive through the provided logger", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "Body", ref: "main" }); + const debug = vi.fn(); + const out = await expandFileIncludes( + "::include{file=chapter1.md}", + baseCtx({ log: { debug } }), + baseGuard(), + ); + expect(out).toContain("Body"); + expect(debug).toHaveBeenCalledTimes(1); + expect(debug.mock.calls[0][0]).toContain("::include chapter1.md"); + }); + + it("does nothing when no logger is provided", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "Body", ref: "main" }); + await expect( + expandFileIncludes("::include{file=a.md}", baseCtx(), baseGuard()), + ).resolves.toContain("Body"); + }); +}); + describe("expandIncludes (source processor)", () => { beforeEach(() => { mockedFetchFileSource.mockReset(); diff --git a/src/include/expand.ts b/src/include/expand.ts index fa99922..756af62 100644 --- a/src/include/expand.ts +++ b/src/include/expand.ts @@ -1,6 +1,7 @@ import { languageFromPath } from "../gitlab/code.js"; import { fetchFileSource } from "../gitlab/fetchers.js"; import type { GitLabContext } from "../gitlab/fetchers.js"; +import type { IncludeLogger } from "./logger.js"; import type { OutProcessor } from "./out-processors.js"; import { codeRanges, stripFrontmatter } from "./render-source.js"; @@ -23,6 +24,8 @@ export interface ExpandContext { ref: string; allowedHosts: string[]; strict: boolean; + /** Optional debug logger; when present, each resolved directive is traced. */ + log?: IncludeLogger; } export interface ExpandGuard { @@ -66,6 +69,10 @@ async function resolveOne( const { file } = parseIncludeAttrs(attrs); if (!file) throw new Error("::include missing file= attribute"); const { raw, key, isMarkdown } = await resolveTarget(file, o); + o.log?.debug( + `::include ${file} → ${key} ` + + `[depth ${guard.depth}, ${isMarkdown ? "markdown, expanding" : `code:${languageFromPath(file)}`}, ${raw.length} bytes]`, + ); if (guard.stack.has(key)) throw new Error(`::include cycle detected: ${key}`); // Non-markdown targets are spliced as a fenced, syntax-highlighted code block // (like `{@includeGitlabFile}` of a code file) rather than as raw prose — raw @@ -126,6 +133,8 @@ export interface ExpandConfig { path?: string; allowedHosts: string[]; strict: boolean; + /** Optional debug logger; traces each resolved `::include` directive. */ + log?: IncludeLogger; } /** @@ -142,6 +151,7 @@ export function expandIncludes(config: ExpandConfig): OutProcessor { ref: config.ref, allowedHosts: config.allowedHosts, strict: config.strict, + log: config.log, }; const seed = `${config.project}@${config.ref}/-/${config.path ?? "README.md"}`; return (md) => expandFileIncludes(md, o, { depth: 0, stack: new Set([seed]) }); diff --git a/src/include/loader.ts b/src/include/loader.ts index 2e57e25..0333409 100644 --- a/src/include/loader.ts +++ b/src/include/loader.ts @@ -25,6 +25,7 @@ export default function gitlabIncludeLoader(this: LoaderThis, source: string): v convertAlerts: resolved.convertAlerts, stripToc: resolved.stripToc, allowedHosts: resolved.includeAllowedHosts, + debug: resolved.debug, outProcessors: processorsId ? getOutProcessors(processorsId) : [], }; transformIncludes(source, getContext(resolved), options).then( diff --git a/src/include/logger.test.ts b/src/include/logger.test.ts new file mode 100644 index 0000000..e985a9f --- /dev/null +++ b/src/include/logger.test.ts @@ -0,0 +1,23 @@ +import docusaurusLogger from "@docusaurus/logger"; +import { describe, it, expect, vi } from "vitest"; +import { createIncludeLogger } from "./logger.js"; + +vi.mock("@docusaurus/logger", () => ({ default: { info: vi.fn() } })); + +const info = (docusaurusLogger as unknown as { info: ReturnType }).info; + +describe("createIncludeLogger", () => { + it("returns a no-op logger when disabled (does not touch @docusaurus/logger)", async () => { + info.mockClear(); + const log = await createIncludeLogger(false); + expect(() => log.debug("anything")).not.toThrow(); + expect(info).not.toHaveBeenCalled(); + }); + + it("routes debug messages through @docusaurus/logger with a prefix when enabled", async () => { + info.mockClear(); + const log = await createIncludeLogger(true); + log.debug("hello world"); + expect(info).toHaveBeenCalledWith("[gitlab-include] hello world"); + }); +}); diff --git a/src/include/logger.ts b/src/include/logger.ts new file mode 100644 index 0000000..70975e0 --- /dev/null +++ b/src/include/logger.ts @@ -0,0 +1,31 @@ +/** A minimal debug logger for the build-time include pipeline. */ +export interface IncludeLogger { + debug(message: string): void; +} + +const NOOP: IncludeLogger = { debug() {} }; + +type DocusaurusLogger = { info: (message: string) => void }; + +/** + * Build the include-pipeline debug logger. When `enabled`, traces are printed + * through Docusaurus's own logger (`@docusaurus/logger`) so they match the rest + * of the build output; when disabled, a no-op is returned and `@docusaurus/logger` + * is never loaded (it's an optional peer dependency, imported lazily here). + * + * Docusaurus's logger is CJS and has no debug level, so — via a defensive + * `.default` unwrap for the ESM/CJS interop — traces go through `info` with a + * `[gitlab-include]` prefix. + */ +export async function createIncludeLogger(enabled: boolean): Promise { + if (!enabled) return NOOP; + const imported = (await import("@docusaurus/logger")).default as unknown as DocusaurusLogger & { + default?: DocusaurusLogger; + }; + const logger: DocusaurusLogger = imported.default ?? imported; + return { + debug(message: string) { + logger.info(`[gitlab-include] ${message}`); + }, + }; +} diff --git a/src/include/transform.ts b/src/include/transform.ts index eb90008..10bfef5 100644 --- a/src/include/transform.ts +++ b/src/include/transform.ts @@ -2,6 +2,7 @@ import type { GitLabContext } from "../gitlab/fetchers.js"; import { fetchFileSource, fetchReadmeSource } from "../gitlab/fetchers.js"; import { expandIncludes } from "./expand.js"; import { parseInclude } from "./grammar.js"; +import { createIncludeLogger } from "./logger.js"; import { applyOutProcessors, convertAlerts, @@ -31,6 +32,8 @@ export interface TransformOptions { outProcessors?: OutProcessor[]; /** Hostnames allowed as remote `::include{file=https://…}` targets. Default: none. */ allowedHosts?: string[]; + /** Emit build-time debug traces for the include pipeline. Default: false. */ + debug?: boolean; } export async function transformIncludes( @@ -41,6 +44,8 @@ export async function transformIncludes( const matches = [...source.matchAll(PLACEHOLDER_RE)]; if (matches.length === 0) return source; + const log = await createIncludeLogger(options.debug ?? false); + // The built-in autolink fix is driven by a serializable flag (so it reliably // crosses the webpack loader boundary); user processors come from the registry. const processors: OutProcessor[] = [ @@ -60,16 +65,23 @@ export async function transformIncludes( }); } + log.debug(`found ${matches.length} include placeholder(s), ${seen.size} unique to resolve`); + // Resolve each unique placeholder once (dedupe by full match text). const replacements = new Map(); await Promise.all( [...seen.entries()].map(async ([full, { kind, arg }]) => { try { const spec = parseInclude(kind, arg); + log.debug( + `resolving ${full} → ${kind} ${spec.project}@${spec.ref ?? "(default ref)"}` + + `${spec.path ? `/-/${spec.path}` : ""}${spec.lineRange ? `#L${spec.lineRange}` : ""}`, + ); const { raw, ref } = kind === "readme" ? await fetchReadmeSource(ctx, { project: spec.project, ref: spec.ref }) : await fetchFileSource(ctx, { project: spec.project, path: spec.path!, ref: spec.ref }); + log.debug(`fetched ${spec.project}@${ref}${spec.path ? `/-/${spec.path}` : ""} (${raw.length} bytes)`); // Pre-render source processors run on the RAW source before renderSource; // ::include expansion must happen here (its `{…}` braces would be // MDX-escaped afterward). Reuses the OutProcessor shape + runner. @@ -82,6 +94,7 @@ export async function transformIncludes( path: spec.path, allowedHosts: options.allowedHosts ?? [], strict: options.strict, + log, }), ] : []; @@ -97,9 +110,11 @@ export async function transformIncludes( if (processors.length && isMarkdownSource(kind, spec.path)) { body = await applyOutProcessors(body, processors); } + log.debug(`rendered ${full} → ${body.length} chars of MDX`); replacements.set(full, `\n\n${body}\n\n`); } catch (err) { const message = err instanceof Error ? err.message : String(err); + log.debug(`FAILED ${full} — ${message}`); if (options.strict) { throw new Error(`@ebuildy/docusaurus-plugin-gitlab: ${full} failed — ${message}`); } diff --git a/src/options.test.ts b/src/options.test.ts index d200dde..00a2bb2 100644 --- a/src/options.test.ts +++ b/src/options.test.ts @@ -9,6 +9,12 @@ describe("resolveOptions", () => { expect(o.assetDir).toBe("static/gitlab-assets"); expect(o.assetBaseUrl).toBe("/gitlab-assets"); expect(o.cache).toEqual({ ttl: 3600 }); + expect(o.debug).toBe(false); + }); + + it("passes through debug: true", () => { + const o = resolveOptions({ host: "https://gitlab.com", debug: true }, "production"); + expect(o.debug).toBe(true); }); it("defaults strict to false in development", () => { diff --git a/src/options.ts b/src/options.ts index d6ef135..5bda88e 100644 --- a/src/options.ts +++ b/src/options.ts @@ -27,6 +27,10 @@ export interface PluginOptions { * targets inside fetched GitLab markdown. Empty ⇒ remote includes are rejected. * Default: `[]`. */ includeAllowedHosts?: string[]; + /** Emit build-time debug traces for the include pipeline (each resolved + * placeholder and `::include` directive) via `@docusaurus/logger`. + * Default: `false`. */ + debug?: boolean; /** Extra post-processors applied to the markdown generated from includes, * in order, after the built-in fixes (when enabled). */ outProcessors?: OutProcessor[]; @@ -45,6 +49,7 @@ export interface ResolvedOptions { convertAlerts: boolean; stripToc: boolean; includeAllowedHosts: string[]; + debug: boolean; } const schema = Joi.object({ @@ -66,6 +71,7 @@ const schema = Joi.object({ convertAlerts: Joi.boolean().optional(), stripToc: Joi.boolean().optional(), includeAllowedHosts: Joi.array().items(Joi.string()).optional(), + debug: Joi.boolean().optional(), outProcessors: Joi.array().items(Joi.function()).optional(), }); @@ -90,5 +96,6 @@ export function resolveOptions( convertAlerts: opts.convertAlerts ?? true, stripToc: opts.stripToc ?? false, includeAllowedHosts: opts.includeAllowedHosts ?? [], + debug: opts.debug ?? false, }; } From b948ded283e3ac5e3ad89392f12d0d5d9ea6ba7a Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 4 Jul 2026 15:12:50 +0200 Subject: [PATCH 14/14] feat: expand ::include in code blocks and in the component render path - Expand ::include directives inside fenced/inline code regions, inserting the target verbatim (GitLab 'includes in code blocks') instead of skipping them; prose directives keep markdown-inline/recurse and non-markdown code-fence behavior. - Extend ::include support to the / components: fetchReadme/fetchFile now expand directives on the raw markdown before renderMarkdown, with strict/allowedHosts/debug threaded into GitLabContext.options via buildContext. - Clean up the ad-hoc full-document debug probe; keep the structured traces. - Update spec, README, and tests accordingly. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Thomas Decaux --- README.md | 11 +++++ ...6-07-04-gitlab-include-directive-design.md | 47 +++++++++++++------ examples/gitlab/docs/demo-include.mdx | 2 +- examples/gitlab/docusaurus.config.ts | 1 + src/gitlab/context.ts | 12 ++++- src/gitlab/fetchers.test.ts | 29 ++++++++++++ src/gitlab/fetchers.ts | 46 ++++++++++++++++-- src/include/expand.test.ts | 38 ++++++++++++--- src/include/expand.ts | 41 +++++++++++----- 9 files changed, 190 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 66e7de6..bbdb2dd 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,17 @@ markdown: - Markdown targets (`.md`/`.mdx`/`.markdown`) are spliced inline and expanded recursively (max depth 8) with cycle detection. Any other file (e.g. `.yaml`, `.json`) is inserted as a fenced, syntax-highlighted code block. +- Put the directive **inside a fenced code block** to insert a file's content + verbatim into that block ([GitLab: includes in code blocks](https://docs.gitlab.com/user/markdown/#use-includes-in-code-blocks)): + + ````text + ```yaml + ::include{file=config/profiles.yaml} + ``` + ```` + + The directive is replaced by the file content in place — no extra fence, no + markdown processing. - A failed include aborts the build in `strict` mode, or renders an inline warning otherwise. diff --git a/docs/superpowers/specs/2026-07-04-gitlab-include-directive-design.md b/docs/superpowers/specs/2026-07-04-gitlab-include-directive-design.md index 907dfab..a63e3f8 100644 --- a/docs/superpowers/specs/2026-07-04-gitlab-include-directive-design.md +++ b/docs/superpowers/specs/2026-07-04-gitlab-include-directive-design.md @@ -47,9 +47,12 @@ main markdown" semantics we want, so `::include` expansion belongs here. `{@includeGitlabFile}` includes (any content for which `isMarkdownSource` is true). -**Out of scope:** the JSX-component HTML path (`src/gitlab/markdown.ts` -`renderMarkdown`, used by `` / `` React components). -`::include` is not expanded there. +**Also in scope (added later):** the JSX-component path (`` / +markdown ``). The `fetchReadme` / `fetchFile` fetchers expand +`::include` on the raw markdown **before** `renderMarkdown`, so the components +behave consistently with the loader placeholders. This required threading the +include settings (`strict`, `includeAllowedHosts`, `debug`) into the +`GitLabContext.options` populated by `buildContext`. ## Grammar @@ -64,9 +67,13 @@ A leaf directive occupying its own line, matching GitLab: - No `ref`, line-range, or `project` extensions (YAGNI). A relative include inherits the **`ref` of the enclosing include** and targets the **same project**, matching GitLab's same-repo semantics. -- The directive is recognized only as a standalone block (its own line). A - `::include{…}` appearing inside a fenced/indented/inline code region is left - **literal** and never expanded. +- The directive is recognized as a standalone block (its own line), **including + inside a fenced/indented/inline code region**. Per GitLab + ([Use includes in code blocks](https://docs.gitlab.com/user/markdown/#use-includes-in-code-blocks)), + a directive inside a code region has its target inserted **verbatim** (not + processed as markdown, not wrapped in another fence, not recursed into) so the + file's content appears in the surrounding code block. Outside code regions, + targets are expanded as described below. ## Design @@ -108,7 +115,7 @@ Consequences of this ordering: ### Resolution per directive -For each `::include{file=X}` found in a non-code prose region: +For each `::include{file=X}` the target is fetched: - **Remote** — `X` matches `^https?://`: parse the URL, check its host against `allowedHosts`. If not listed → error. If listed → fetch the body via native @@ -116,14 +123,23 @@ For each `::include{file=X}` found in a non-code prose region: - **Relative** — otherwise: `fetchFileSource(ctx, { project: o.project, path: X, ref: o.ref })`, reusing the existing GitLab client and on-disk cache. -Fetched markdown is `stripFrontmatter`-ed and **recursively expanded** before -being spliced. +How the fetched content is spliced depends on where the directive sits: -### Code-fence safety +- **Inside a code region** (fenced/indented/inline) → inserted **verbatim**: no + frontmatter stripping, no fence wrapping, no recursion. The content lands in + the author's existing code block. +- **In prose, markdown target** (`.md`/`.mdx`/`.markdown`) → `stripFrontmatter`-ed + and **recursively expanded** before being spliced. +- **In prose, non-markdown target** → wrapped in a fenced, syntax-highlighted + code block (`` ``` `` via `languageFromPath`) so raw YAML/JSON/etc. + renders cleanly instead of as garbled markdown. -Expansion runs only over non-code regions. Reuse the existing -`codeRanges` / `mapProseRegions` approach from `render-source.ts` so a -`::include{…}` inside a code block is preserved verbatim. +### Code-region detection + +Each directive's position is classified via `codeRanges` from +`render-source.ts`: a match whose offset falls inside a code range is treated as +"inside a code block" (verbatim insertion above), otherwise as prose. Directives +are **not** skipped — GitLab expands them in both contexts. ### Recursion guard @@ -175,7 +191,9 @@ depth exceeded, malformed directive. - Nested include (an included file containing `::include`) expands recursively. - Cycle (A includes B includes A) throws `"cycle detected"`. - Depth beyond `MAX_INCLUDE_DEPTH` throws. -- `::include{…}` inside a fenced code block is left literal. +- `::include{…}` inside a fenced code block inserts the target **verbatim** (no + extra fence, no processing). +- A non-markdown target in prose is wrapped in a highlighted code block. - Quoted and bare `file=` values both parse. - Strict vs non-strict error behavior. @@ -199,6 +217,5 @@ build (`test/e2e/build.test.ts`) since the loader pipeline is touched. ## Non-goals / future work - No `ref`, line-range, or `project` attributes on `::include`. -- No expansion in the JSX-component HTML render path. - File-relative (as opposed to project-root-relative) image/link resolution is not addressed here; it inherits the existing behavior and limitation. diff --git a/examples/gitlab/docs/demo-include.mdx b/examples/gitlab/docs/demo-include.mdx index b78953f..cab2448 100644 --- a/examples/gitlab/docs/demo-include.mdx +++ b/examples/gitlab/docs/demo-include.mdx @@ -1 +1 @@ -{@includeGitlabReadme: ebuildy/demo-docusaurus} \ No newline at end of file +{@includeGitlabReadme: ebuildy/demo-docusaurus} diff --git a/examples/gitlab/docusaurus.config.ts b/examples/gitlab/docusaurus.config.ts index b823efe..0878431 100644 --- a/examples/gitlab/docusaurus.config.ts +++ b/examples/gitlab/docusaurus.config.ts @@ -14,6 +14,7 @@ const gitlabOptions = { token: process.env.GITLAB_TOKEN, strict: false, stripToc: true, + debug: true, }; // Live example: embeds REAL public content from gitlab.com. diff --git a/src/gitlab/context.ts b/src/gitlab/context.ts index d5ebce5..e8919a1 100644 --- a/src/gitlab/context.ts +++ b/src/gitlab/context.ts @@ -16,5 +16,15 @@ export function buildContext(options: ResolvedOptions): GitLabContext { assetBaseUrl: options.assetBaseUrl, host: options.host, }); - return { client, cache, assets, options: { host: options.host } }; + return { + client, + cache, + assets, + options: { + host: options.host, + strict: options.strict, + allowedHosts: options.includeAllowedHosts, + debug: options.debug, + }, + }; } diff --git a/src/gitlab/fetchers.test.ts b/src/gitlab/fetchers.test.ts index bf25c6d..072dc74 100644 --- a/src/gitlab/fetchers.test.ts +++ b/src/gitlab/fetchers.test.ts @@ -118,6 +118,35 @@ describe("fetchReadme", () => { expect(client.getFileRaw).toHaveBeenCalledWith("g/r", "README.md", "v2"); }); + it("expands ::include directives in the README before rendering", async () => { + const client: any = { + getProject: vi.fn(async () => ({ default_branch: "main" })), + getFileRaw: vi.fn(async (_p: unknown, path: string) => { + if (path === "README.md") return "# Title\n\n::include{file=chapter1.md}\n"; + if (path === "chapter1.md") return "Chapter one body."; + throw new Error(`unexpected ${path}`); + }), + }; + const data = await fetchReadme(ctx(client), { project: "g/r" }); + expect(data.html).toContain("Chapter one body."); + expect(data.html).not.toContain("::include"); + expect(client.getFileRaw).toHaveBeenCalledWith("g/r", "chapter1.md", "main"); + }); + + it("inserts a non-markdown ::include target inside its code fence verbatim", async () => { + const client: any = { + getProject: vi.fn(async () => ({ default_branch: "main" })), + getFileRaw: vi.fn(async (_p: unknown, path: string) => { + if (path === "README.md") return "## Config\n\n```yaml\n::include{file=profiles.yaml}\n```\n"; + if (path === "profiles.yaml") return "name: prod\nport: 8080"; + throw new Error(`unexpected ${path}`); + }), + }; + const data = await fetchReadme(ctx(client), { project: "g/r" }); + expect(data.html).toContain("name: prod"); + expect(data.html).not.toContain("::include"); + }); + it("sidebar mode returns toc entries and assigns heading ids", async () => { const client: any = { getProject: vi.fn(async () => ({ default_branch: "main" })), diff --git a/src/gitlab/fetchers.ts b/src/gitlab/fetchers.ts index 792ea3f..3adb096 100644 --- a/src/gitlab/fetchers.ts +++ b/src/gitlab/fetchers.ts @@ -1,3 +1,8 @@ +// `expandIncludes` lives in the include subsystem, which imports `fetchFileSource` +// from this module — a benign call-time-only cycle (both are hoisted function +// declarations, used only at runtime, never at module-eval time). +import { expandIncludes } from "../include/expand.js"; +import { createIncludeLogger } from "../include/logger.js"; import type { AssetManager } from "./assets"; import type { FileCache } from "./cache"; import type { GitLabClient, PageOptions } from "./client"; @@ -17,7 +22,40 @@ export interface GitLabContext { client: GitLabClient; cache: FileCache; assets: AssetManager; - options: { host: string }; + options: { + host: string; + /** Include-pipeline settings (populated by `buildContext`); optional so test + * fakes can omit them. Defaults applied where read. */ + strict?: boolean; + allowedHosts?: string[]; + debug?: boolean; + }; +} + +/** + * Expand GitLab `::include{file=…}` directives in fetched markdown before it is + * rendered, so the `` / `` components behave like the + * `{@includeGitlab…}` loader path. No-op when the source has no directive. + */ +async function expandDirectives( + ctx: GitLabContext, + project: string, + ref: string, + path: string | undefined, + md: string, +): Promise { + if (!md.includes("::include")) return md; + const log = await createIncludeLogger(ctx.options.debug ?? false); + const expand = expandIncludes({ + ctx, + project, + ref, + path, + allowedHosts: ctx.options.allowedHosts ?? [], + strict: ctx.options.strict ?? true, + log, + }); + return expand(md); } type Attrs = Record; @@ -146,7 +184,8 @@ export async function fetchReadme(ctx: GitLabContext, attrs: Attrs): Promise { const ref = explicitRef ?? (await ctx.client.getProject(attrs.project as string | number)).default_branch; - const md = await ctx.client.getFileRaw(attrs.project as string | number, "README.md", ref); + const rawMd = await ctx.client.getFileRaw(attrs.project as string | number, "README.md", ref); + const md = await expandDirectives(ctx, project, ref, undefined, rawMd); const collectToc: TocEntry[] = []; const html = await renderMarkdown(md, { tocMode, @@ -313,7 +352,8 @@ export async function fetchFile(ctx: GitLabContext, attrs: Attrs): Promise ctx.assets.localize(src, ref, String(project)), }); return { kind: "markdown", html, ref, path } satisfies FileData; diff --git a/src/include/expand.test.ts b/src/include/expand.test.ts index d81ca6d..d84896b 100644 --- a/src/include/expand.test.ts +++ b/src/include/expand.test.ts @@ -77,11 +77,17 @@ describe("expandFileIncludes — relative GitLab files", () => { expect(out).not.toContain("title: x"); }); - it("leaves a directive inside a fenced code block untouched", async () => { - const md = "```\n::include{file=chapter1.md}\n```"; + it("expands a directive inside a fenced code block verbatim (GitLab 'includes in code blocks')", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: 'const s = "hi";\nalert(s);', ref: "main" }); + const md = "```javascript\n::include{file=code.js}\n```"; const out = await expandFileIncludes(md, baseCtx(), baseGuard()); - expect(out).toBe(md); - expect(mockedFetchFileSource).not.toHaveBeenCalled(); + // Target inserted verbatim inside the existing fence — no extra fence, no processing. + expect(out).toBe('```javascript\nconst s = "hi";\nalert(s);\n```'); + expect(mockedFetchFileSource).toHaveBeenCalledWith({} as any, { + project: "group/proj", + path: "code.js", + ref: "main", + }); }); it("returns the source unchanged when there is no directive", async () => { @@ -89,6 +95,27 @@ describe("expandFileIncludes — relative GitLab files", () => { expect(out).toBe("plain text"); expect(mockedFetchFileSource).not.toHaveBeenCalled(); }); + + it("expands a directive embedded in a multi-line document, preserving surrounding lines", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "name: prod\nport: 8080", ref: "main" }); + const md = `## Includes + +\`\`\`yaml +::include{file=profiles.yaml} +\`\`\``; + const out = await expandFileIncludes(md, baseCtx(), baseGuard()); + // Heading + the author's ```yaml fence kept; the directive is replaced by the + // file content inserted verbatim inside that fence (GitLab "includes in code blocks"). + expect(out).toContain("## Includes"); + expect(out).toContain("```yaml"); + expect(out).toContain("name: prod"); + expect(out).not.toContain("::include"); + expect(mockedFetchFileSource).toHaveBeenCalledWith({} as any, { + project: "group/proj", + path: "profiles.yaml", + ref: "main", + }); + }); }); describe("expandFileIncludes — remote URLs", () => { @@ -265,8 +292,7 @@ describe("expandFileIncludes — debug logging", () => { baseGuard(), ); expect(out).toContain("Body"); - expect(debug).toHaveBeenCalledTimes(1); - expect(debug.mock.calls[0][0]).toContain("::include chapter1.md"); + expect(debug.mock.calls.some((c) => String(c[0]).includes("::include chapter1.md"))).toBe(true); }); it("does nothing when no logger is provided", async () => { diff --git a/src/include/expand.ts b/src/include/expand.ts index 756af62..aa24a59 100644 --- a/src/include/expand.ts +++ b/src/include/expand.ts @@ -58,26 +58,35 @@ async function resolveTarget( return { raw: src.raw, key, isMarkdown: MD_EXT.test(file) }; } -/** Resolve a single directive to its replacement text, honoring `strict`. */ +/** + * Resolve a single directive to its replacement text, honoring `strict`. + * `inCodeBlock` marks a directive that sits inside a fenced/inline code region: + * per GitLab ("Use includes in code blocks"), its target is inserted **verbatim** + * — not processed as markdown, not wrapped in another fence, not recursed into. + */ async function resolveOne( full: string, attrs: string, o: ExpandContext, guard: ExpandGuard, + inCodeBlock: boolean, ): Promise { try { const { file } = parseIncludeAttrs(attrs); if (!file) throw new Error("::include missing file= attribute"); const { raw, key, isMarkdown } = await resolveTarget(file, o); o.log?.debug( - `::include ${file} → ${key} ` + - `[depth ${guard.depth}, ${isMarkdown ? "markdown, expanding" : `code:${languageFromPath(file)}`}, ${raw.length} bytes]`, + `::include ${file} → ${key} [depth ${guard.depth}, ` + + `${inCodeBlock ? "in code block, verbatim" : isMarkdown ? "markdown, expanding" : `code:${languageFromPath(file)}`}, ` + + `${raw.length} bytes]`, ); - if (guard.stack.has(key)) throw new Error(`::include cycle detected: ${key}`); - // Non-markdown targets are spliced as a fenced, syntax-highlighted code block - // (like `{@includeGitlabFile}` of a code file) rather than as raw prose — raw - // YAML/JSON/etc. would otherwise render as garbled markdown. + // Inside a code fence: insert the target verbatim (GitLab semantics). + if (inCodeBlock) return raw; + // Non-markdown target in prose: wrap in a fenced, syntax-highlighted block so + // raw YAML/JSON/etc. doesn't render as garbled markdown. if (!isMarkdown) return `\n\`\`\`${languageFromPath(file)}\n${raw}\n\`\`\`\n`; + // Markdown target in prose: strip frontmatter and expand nested includes. + if (guard.stack.has(key)) throw new Error(`::include cycle detected: ${key}`); const body = stripFrontmatter(raw); return expandFileIncludes(body, o, { depth: guard.depth + 1, @@ -93,8 +102,11 @@ async function resolveOne( } /** - * Replace every standalone `::include{file=…}` directive in `md` (outside code - * regions) with the raw content of its target. `guard` carries recursion state. + * Replace every standalone `::include{file=…}` directive in `md` with its target. + * Directives in a fenced/inline code region get their target inserted verbatim + * (GitLab "includes in code blocks"); directives in prose get markdown targets + * expanded recursively and other files wrapped in a highlighted code block. + * `guard` carries recursion state. */ export async function expandFileIncludes( md: string, @@ -107,11 +119,12 @@ export async function expandFileIncludes( const ranges = codeRanges(md); const inCode = (i: number) => ranges.some(([s, e]) => i >= s && i < e); - const matches = [...md.matchAll(INCLUDE_RE)].filter((m) => !inCode(m.index ?? 0)); + const matches = [...md.matchAll(INCLUDE_RE)]; + o.log?.debug(`expandFileIncludes: ${matches.length} directive(s) at depth ${guard.depth}`); if (matches.length === 0) return md; const replacements = await Promise.all( - matches.map((m) => resolveOne(m[0], m[1], o, guard)), + matches.map((m) => resolveOne(m[0], m[1], o, guard, inCode(m.index ?? 0))), ); let out = ""; @@ -153,6 +166,12 @@ export function expandIncludes(config: ExpandConfig): OutProcessor { strict: config.strict, log: config.log, }; + if (config.log) { + config.log.debug?.( + `expandIncludes: ${config.project}@${config.ref}/-/${config.path ?? "README.md"} ` + + `[allowedHosts=${config.allowedHosts.join(",")}]`, + ); + } const seed = `${config.project}@${config.ref}/-/${config.path ?? "README.md"}`; return (md) => expandFileIncludes(md, o, { depth: 0, stack: new Set([seed]) }); }