diff --git a/README.md b/README.md index 370fd49..bbdb2dd 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,39 @@ Both components render [scoped labels/topics](https://docs.gitlab.com/ee/user/pr (`scope::value`, e.g. `Abilities::Performance`) as a two-part badge — the scope keeps its color and the value gets a dark-gray background. The split is on the last `::`. +### `::include` directives inside included markdown + +When a fetched GitLab README or markdown file contains a GitLab +[`::include`](https://docs.gitlab.com/user/markdown/#includes) directive, the +plugin expands it at build time, splicing the referenced file in as raw +markdown: + +```text +::include{file=docs/chapter1.md} +``` + +- Relative paths resolve to a file in the **same GitLab project and ref** as the + enclosing include, fetched through the same cached client. +- Remote URLs (`::include{file=https://…}`) are fetched only when their host is + listed in the `includeAllowedHosts` plugin option (empty by default, so remote + includes are off until you opt in). +- Markdown targets (`.md`/`.mdx`/`.markdown`) are spliced inline and expanded + recursively (max depth 8) with cycle detection. Any other file (e.g. `.yaml`, + `.json`) is inserted as a fenced, syntax-highlighted code block. +- Put the directive **inside a fenced code block** to insert a file's content + verbatim into that block ([GitLab: includes in code blocks](https://docs.gitlab.com/user/markdown/#use-includes-in-code-blocks)): + + ````text + ```yaml + ::include{file=config/profiles.yaml} + ``` + ```` + + The directive is replaced by the file content in place — no extra fence, no + markdown processing. +- A failed include aborts the build in `strict` mode, or renders an inline + warning otherwise. + ## Plugin options | Option | Type | Default | Description | @@ -250,6 +283,8 @@ color and the value gets a dark-gray background. The split is on the last `::`. | `convertAlerts` | boolean | `true` | Translate GitLab alert blockquotes (`> [!note]`) to Docusaurus admonitions (`:::note`) in included markdown | | `stripToc` | boolean | `false` | Remove a redundant "Table of Contents" section (and `[[_TOC_]]` marker) from included markdown | | `outProcessors` | `Array<(md: string) => string \| Promise>` | `[]` | Extra post-processors for included markdown, run after the built-in fixes | +| `includeAllowedHosts` | `string[]` | `[]` | Hostnames allowed as remote `::include{file=https://…}` targets | +| `debug` | boolean | `false` | Emit build-time debug traces for the include pipeline (resolved placeholders and `::include` directives) via `@docusaurus/logger` | The token is read at build time only. Provide it via an environment variable (`GITLAB_TOKEN`) — never commit it. diff --git a/docs/superpowers/plans/2026-07-04-gitlab-include-directive.md b/docs/superpowers/plans/2026-07-04-gitlab-include-directive.md new file mode 100644 index 0000000..1bd0c73 --- /dev/null +++ b/docs/superpowers/plans/2026-07-04-gitlab-include-directive.md @@ -0,0 +1,828 @@ +# GitLab `::include` Directive Expansion — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Expand GitLab `::include{file=…}` directives found inside fetched GitLab READMEs/markdown files into the host document as raw markdown source, at build time. + +**Architecture:** `::include` lives inside GitLab markdown pulled in by the webpack-loader include subsystem (`src/include/`). A new `src/include/expand.ts` scans the fetched raw markdown for `::include{file=…}` leaf directives (skipping code regions), resolves each target — a GitLab project file (via the existing `fetchFileSource`) or an allowlisted remote URL (via native `fetch`) — and splices its raw content in place, recursively, with depth + cycle guards. `transformIncludes` (`src/include/transform.ts`) calls the expander on the fetched raw source *before* `renderSource`, so the merged document flows through the existing prose transforms and out-processors as one unit. Hooking in `transform.ts` (not inside `renderSource`) keeps `render-source.ts` free of a back-import and avoids a module cycle. + +**Tech Stack:** TypeScript (ESM, `.js` import extensions), Vitest, unified/remark (only reused indirectly via existing `codeRanges`), native `fetch` (Node 20+), Joi (options), `@gitbeaker/rest`. + +**Note on the spec:** the design doc places the hook "inside `renderSource`". During file-structure review this was moved one level out to `transformIncludes` to avoid a circular import between `expand.ts` and `render-source.ts` (`expand.ts` imports `codeRanges`/`stripFrontmatter` from `render-source.ts`). The behavior is identical — expansion still happens on the raw source before `processMarkdownSource` runs on the merged document. + +--- + +## File Structure + +- **Create** `src/include/expand.ts` — directive scanning, target resolution (GitLab file / remote URL), recursion + guards, error handling. One responsibility: turn raw markdown containing `::include` into raw markdown with those directives expanded. +- **Create** `src/include/expand.test.ts` — unit tests for the above (mocked `fetchFileSource` + `fetch`). +- **Modify** `src/options.ts` — add `includeAllowedHosts` option (interface, Joi, default). +- **Modify** `src/options.test.ts` — cover the new option's default + validation. +- **Modify** `src/include/transform.ts` — add `allowedHosts` to `TransformOptions`; call `expandFileIncludes` on the fetched raw source for markdown includes. +- **Modify** `src/include/transform.test.ts` — cover expansion wiring. +- **Modify** `src/include/loader.ts` — pass `resolved.includeAllowedHosts` into `transformIncludes`. +- **Modify** `README.md` and `examples/gitlab/docs/includes.mdx` — document the feature. + +--- + +## Task 1: Add the `includeAllowedHosts` plugin option + +**Files:** +- Modify: `src/options.ts` +- Test: `src/options.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Add to `src/options.test.ts` inside the `describe("resolveOptions", …)` block: + +```ts +it("defaults includeAllowedHosts to an empty array", () => { + const o = resolveOptions({ host: "https://gitlab.com" }, "production"); + expect(o.includeAllowedHosts).toEqual([]); +}); + +it("passes through a configured includeAllowedHosts list", () => { + const o = resolveOptions( + { host: "https://gitlab.com", includeAllowedHosts: ["example.org"] }, + "production", + ); + expect(o.includeAllowedHosts).toEqual(["example.org"]); +}); + +it("rejects a non-array includeAllowedHosts", () => { + expect(() => + resolveOptions( + { host: "https://gitlab.com", includeAllowedHosts: "example.org" } as any, + "production", + ), + ).toThrow(); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run src/options.test.ts` +Expected: FAIL — `includeAllowedHosts` is `undefined` / not validated. + +- [ ] **Step 3: Implement the option** + +In `src/options.ts`: + +Add to the `PluginOptions` interface (after `stripToc`): + +```ts + /** Hostnames (exact, case-insensitive) allowed as remote `::include{file=https://…}` + * targets inside fetched GitLab markdown. Empty ⇒ remote includes are rejected. + * Default: `[]`. */ + includeAllowedHosts?: string[]; +``` + +Add to the `ResolvedOptions` interface (after `stripToc: boolean;`): + +```ts + includeAllowedHosts: string[]; +``` + +Add to the Joi `schema` object (after `stripToc: …`): + +```ts + includeAllowedHosts: Joi.array().items(Joi.string()).optional(), +``` + +Add to the `resolveOptions` return object (after `stripToc: …`): + +```ts + includeAllowedHosts: opts.includeAllowedHosts ?? [], +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `npx vitest run src/options.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/options.ts src/options.test.ts +git commit -S -m "feat: add includeAllowedHosts plugin option" +``` + +--- + +## Task 2: Directive attribute parser + +**Files:** +- Create: `src/include/expand.ts` +- Test: `src/include/expand.test.ts` + +- [ ] **Step 1: Write the failing test** + +Create `src/include/expand.test.ts`: + +```ts +import { describe, it, expect } from "vitest"; +import { parseIncludeAttrs } from "./expand.js"; + +describe("parseIncludeAttrs", () => { + it("reads a bare file value", () => { + expect(parseIncludeAttrs("file=chapter1.md")).toEqual({ file: "chapter1.md" }); + }); + it("reads a double-quoted file value with spaces", () => { + expect(parseIncludeAttrs('file="a b.md"')).toEqual({ file: "a b.md" }); + }); + it("reads a single-quoted file value", () => { + expect(parseIncludeAttrs("file='c.md'")).toEqual({ file: "c.md" }); + }); + it("reads a URL value", () => { + expect(parseIncludeAttrs("file=https://example.org/x.md")).toEqual({ + file: "https://example.org/x.md", + }); + }); + it("returns empty when file is absent", () => { + expect(parseIncludeAttrs("other=1")).toEqual({}); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: FAIL — cannot import `parseIncludeAttrs` (module missing). + +- [ ] **Step 3: Implement the parser** + +Create `src/include/expand.ts`: + +```ts +import { fetchFileSource } from "../gitlab/fetchers.js"; +import type { GitLabContext } from "../gitlab/fetchers.js"; +import { codeRanges, stripFrontmatter } from "./render-source.js"; + +/** Markdown file extensions whose content is expanded recursively as markdown. */ +const MD_EXT = /\.(?:md|mdx|markdown)$/i; + +/** Extract the `file=` value from a `::include{…}` attribute string (bare or quoted). */ +export function parseIncludeAttrs(attrs: string): { file?: string } { + const m = /(?:^|\s)file\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s}]+))/.exec(attrs); + if (!m) return {}; + return { file: m[1] ?? m[2] ?? m[3] }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/include/expand.ts src/include/expand.test.ts +git commit -S -m "feat: parse ::include directive attributes" +``` + +--- + +## Task 3: Expand relative GitLab-file includes (single level, code-fence safe) + +**Files:** +- Modify: `src/include/expand.ts` +- Test: `src/include/expand.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Append to `src/include/expand.test.ts` (add imports at the top of the file: `import { vi } from "vitest";` and `import { expandFileIncludes, type ExpandContext, type ExpandGuard } from "./expand.js";`): + +```ts +vi.mock("../gitlab/fetchers.js", () => ({ + fetchFileSource: vi.fn(), +})); +import { fetchFileSource } from "../gitlab/fetchers.js"; + +const mockedFetchFileSource = vi.mocked(fetchFileSource); + +function baseCtx(overrides: Partial = {}): ExpandContext { + return { + ctx: {} as any, + project: "group/proj", + ref: "main", + allowedHosts: [], + strict: true, + ...overrides, + }; +} + +function baseGuard(): ExpandGuard { + return { depth: 0, stack: new Set(["group/proj@main/-/README.md"]) }; +} + +describe("expandFileIncludes — relative GitLab files", () => { + beforeEach(() => mockedFetchFileSource.mockReset()); + + it("replaces a directive with the fetched file's raw content", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "Chapter one body.", ref: "main" }); + const out = await expandFileIncludes( + "Intro\n\n::include{file=chapter1.md}\n\nOutro", + baseCtx(), + baseGuard(), + ); + expect(out).toContain("Chapter one body."); + expect(out).not.toContain("::include"); + expect(mockedFetchFileSource).toHaveBeenCalledWith({} as any, { + project: "group/proj", + path: "chapter1.md", + ref: "main", + }); + }); + + it("strips frontmatter from the included file", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "---\ntitle: x\n---\nBody", ref: "main" }); + const out = await expandFileIncludes("::include{file=a.md}", baseCtx(), baseGuard()); + expect(out).toContain("Body"); + expect(out).not.toContain("title: x"); + }); + + it("leaves a directive inside a fenced code block untouched", async () => { + const md = "```\n::include{file=chapter1.md}\n```"; + const out = await expandFileIncludes(md, baseCtx(), baseGuard()); + expect(out).toBe(md); + expect(mockedFetchFileSource).not.toHaveBeenCalled(); + }); + + it("returns the source unchanged when there is no directive", async () => { + const out = await expandFileIncludes("plain text", baseCtx(), baseGuard()); + expect(out).toBe("plain text"); + expect(mockedFetchFileSource).not.toHaveBeenCalled(); + }); +}); +``` + +Also add `import { beforeEach } from "vitest";` to the existing vitest import line. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: FAIL — `expandFileIncludes` / types not exported. + +- [ ] **Step 3: Implement single-level expansion** + +Append to `src/include/expand.ts`: + +```ts +export interface ExpandContext { + ctx: GitLabContext; + project: string; + ref: string; + allowedHosts: string[]; + strict: boolean; +} + +export interface ExpandGuard { + depth: number; + stack: Set; +} + +/** A standalone `::include{…}` leaf directive occupying its own line. */ +const INCLUDE_RE = /^[ \t]*::include\{([^}\n]*)\}[ \t]*$/gm; + +/** Resolve a directive's target to raw text plus a cycle key and markdown flag. */ +async function resolveTarget( + file: string, + o: ExpandContext, +): Promise<{ raw: string; key: string; isMarkdown: boolean }> { + const key = `${o.project}@${o.ref}/-/${file}`; + const src = await fetchFileSource(o.ctx, { project: o.project, path: file, ref: o.ref }); + return { raw: src.raw, key, isMarkdown: MD_EXT.test(file) }; +} + +/** Resolve a single directive to its replacement text, honoring `strict`. */ +async function resolveOne( + full: string, + attrs: string, + o: ExpandContext, + _guard: ExpandGuard, +): Promise { + try { + const { file } = parseIncludeAttrs(attrs); + if (!file) throw new Error("::include missing file= attribute"); + const { raw } = await resolveTarget(file, o); + return stripFrontmatter(raw); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (o.strict) { + throw new Error(`@ebuildy/docusaurus-plugin-gitlab: ${full} failed — ${message}`); + } + return `\n\n> ⚠️ ${full} failed — ${message}\n\n`; + } +} + +/** + * Replace every standalone `::include{file=…}` directive in `md` (outside code + * regions) with the raw content of its target. `guard` carries recursion state. + */ +export async function expandFileIncludes( + md: string, + o: ExpandContext, + guard: ExpandGuard, +): Promise { + const ranges = codeRanges(md); + const inCode = (i: number) => ranges.some(([s, e]) => i >= s && i < e); + + const matches = [...md.matchAll(INCLUDE_RE)].filter((m) => !inCode(m.index ?? 0)); + if (matches.length === 0) return md; + + const replacements = await Promise.all( + matches.map((m) => resolveOne(m[0], m[1], o, guard)), + ); + + let out = ""; + let last = 0; + matches.forEach((m, i) => { + out += md.slice(last, m.index) + replacements[i]; + last = (m.index ?? 0) + m[0].length; + }); + return out + md.slice(last); +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/include/expand.ts src/include/expand.test.ts +git commit -S -m "feat: expand relative GitLab-file ::include directives" +``` + +--- + +## Task 4: Remote URL includes with host allowlist + +**Files:** +- Modify: `src/include/expand.ts` +- Test: `src/include/expand.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Append to `src/include/expand.test.ts`: + +```ts +describe("expandFileIncludes — remote URLs", () => { + beforeEach(() => { + mockedFetchFileSource.mockReset(); + vi.unstubAllGlobals(); + }); + + it("expands a remote include from an allowlisted host", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ ok: true, status: 200, text: async () => "Remote body." })), + ); + const out = await expandFileIncludes( + "::include{file=https://example.org/x.md}", + baseCtx({ allowedHosts: ["example.org"] }), + baseGuard(), + ); + expect(out).toContain("Remote body."); + }); + + it("rejects a remote include from a non-allowlisted host (strict throws)", async () => { + await expect( + expandFileIncludes( + "::include{file=https://evil.test/x.md}", + baseCtx({ allowedHosts: ["example.org"], strict: true }), + baseGuard(), + ), + ).rejects.toThrow(/host not allowed/); + }); + + it("emits a warning marker for a non-allowlisted host when non-strict", async () => { + const out = await expandFileIncludes( + "::include{file=https://evil.test/x.md}", + baseCtx({ allowedHosts: ["example.org"], strict: false }), + baseGuard(), + ); + expect(out).toContain("⚠️"); + expect(out).toContain("host not allowed"); + }); + + it("errors on a non-OK remote response", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ ok: false, status: 404, text: async () => "" })), + ); + await expect( + expandFileIncludes( + "::include{file=https://example.org/missing.md}", + baseCtx({ allowedHosts: ["example.org"], strict: true }), + baseGuard(), + ), + ).rejects.toThrow(/404/); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: FAIL — remote branch not implemented (relative path resolution tries `fetchFileSource` with a URL). + +- [ ] **Step 3: Add the remote branch to `resolveTarget`** + +In `src/include/expand.ts`, replace the body of `resolveTarget` with: + +```ts +async function resolveTarget( + file: string, + o: ExpandContext, +): Promise<{ raw: string; key: string; isMarkdown: boolean }> { + if (/^https?:\/\//i.test(file)) { + const url = new URL(file); + if (!o.allowedHosts.some((h) => h.toLowerCase() === url.host.toLowerCase())) { + throw new Error(`::include host not allowed: ${url.host}`); + } + const res = await fetch(url); + if (!res.ok) throw new Error(`fetch ${url.href} → HTTP ${res.status}`); + return { raw: await res.text(), key: url.href, isMarkdown: MD_EXT.test(url.pathname) }; + } + const key = `${o.project}@${o.ref}/-/${file}`; + const src = await fetchFileSource(o.ctx, { project: o.project, path: file, ref: o.ref }); + return { raw: src.raw, key, isMarkdown: MD_EXT.test(file) }; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/include/expand.ts src/include/expand.test.ts +git commit -S -m "feat: support allowlisted remote ::include URLs" +``` + +--- + +## Task 5: Recursion, cycle detection, and depth guard + +**Files:** +- Modify: `src/include/expand.ts` +- Test: `src/include/expand.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Append to `src/include/expand.test.ts`: + +```ts +describe("expandFileIncludes — recursion + guards", () => { + beforeEach(() => mockedFetchFileSource.mockReset()); + + it("recursively expands includes inside an included markdown file", async () => { + mockedFetchFileSource.mockImplementation(async (_ctx, args: any) => { + if (args.path === "a.md") return { raw: "A\n\n::include{file=b.md}", ref: "main" }; + if (args.path === "b.md") return { raw: "B body", ref: "main" }; + throw new Error(`unexpected ${args.path}`); + }); + const out = await expandFileIncludes("::include{file=a.md}", baseCtx(), baseGuard()); + expect(out).toContain("A"); + expect(out).toContain("B body"); + expect(out).not.toContain("::include"); + }); + + it("detects a direct cycle", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "loop\n\n::include{file=a.md}", ref: "main" }); + await expect( + expandFileIncludes("::include{file=a.md}", baseCtx({ strict: true }), baseGuard()), + ).rejects.toThrow(/cycle detected/); + }); + + it("enforces the max depth", async () => { + // Every file includes a deeper one, never terminating. + mockedFetchFileSource.mockImplementation(async (_ctx, args: any) => ({ + raw: `level\n\n::include{file=${args.path}x.md}`, + ref: "main", + })); + await expect( + expandFileIncludes("::include{file=a.md}", baseCtx({ strict: true }), baseGuard()), + ).rejects.toThrow(/max depth/); + }); + + it("allows the same file included twice in separate branches (diamond)", async () => { + mockedFetchFileSource.mockImplementation(async (_ctx, args: any) => { + if (args.path === "top.md") + return { raw: "::include{file=shared.md}\n\n::include{file=shared.md}", ref: "main" }; + if (args.path === "shared.md") return { raw: "SHARED", ref: "main" }; + throw new Error(`unexpected ${args.path}`); + }); + const out = await expandFileIncludes("::include{file=top.md}", baseCtx(), baseGuard()); + expect(out.match(/SHARED/g)?.length).toBe(2); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: FAIL — recursion not wired; the cycle/depth tests hang or return unexpanded nested directives. + +- [ ] **Step 3: Add the depth guard, cycle check, and recursion** + +In `src/include/expand.ts`: + +Add the constant near `MD_EXT`: + +```ts +/** Maximum nesting depth for recursive `::include` expansion. */ +export const MAX_INCLUDE_DEPTH = 8; +``` + +Replace `resolveOne` with: + +```ts +async function resolveOne( + full: string, + attrs: string, + o: ExpandContext, + guard: ExpandGuard, +): Promise { + try { + const { file } = parseIncludeAttrs(attrs); + if (!file) throw new Error("::include missing file= attribute"); + const { raw, key, isMarkdown } = await resolveTarget(file, o); + if (guard.stack.has(key)) throw new Error(`::include cycle detected: ${key}`); + let body = stripFrontmatter(raw); + if (isMarkdown) { + body = await expandFileIncludes(body, o, { + depth: guard.depth + 1, + stack: new Set(guard.stack).add(key), + }); + } + return body; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (o.strict) { + throw new Error(`@ebuildy/docusaurus-plugin-gitlab: ${full} failed — ${message}`); + } + return `\n\n> ⚠️ ${full} failed — ${message}\n\n`; + } +} +``` + +Add the depth check as the first statement of `expandFileIncludes`: + +```ts +export async function expandFileIncludes( + md: string, + o: ExpandContext, + guard: ExpandGuard, +): Promise { + if (guard.depth > MAX_INCLUDE_DEPTH) { + throw new Error(`::include exceeded max depth (${MAX_INCLUDE_DEPTH})`); + } + const ranges = codeRanges(md); + // …rest unchanged… +``` + +Note: because `resolveOne` catches errors, the depth/cycle errors thrown from a +nested `expandFileIncludes` are surfaced through the nearest enclosing directive +— under `strict` they abort the build; otherwise they become a warning marker. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `npx vitest run src/include/expand.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/include/expand.ts src/include/expand.test.ts +git commit -S -m "feat: recurse ::include with cycle + depth guards" +``` + +--- + +## Task 6: Wire expansion into the include transform pipeline + +**Files:** +- Modify: `src/include/transform.ts` +- Modify: `src/include/loader.ts` +- Test: `src/include/transform.test.ts` + +- [ ] **Step 1: Write the failing test** + +Open `src/include/transform.test.ts` and inspect how it constructs a fake `GitLabContext` and calls `transformIncludes` (follow the existing pattern in that file — it already mocks the client/fetchers). Add a test that a `::include` inside a fetched README is expanded: + +```ts +it("expands ::include directives inside a fetched README", async () => { + // Arrange a context whose README fetch returns markdown containing a directive, + // and whose chapter fetch returns the included body. Use the same fake-context + // helper already used by the other tests in this file. + const ctx = makeCtx({ + files: { + "group/proj@main/-/README.md": "# Title\n\n::include{file=chapter1.md}", + "group/proj@main/-/chapter1.md": "Chapter one body.", + }, + }); + + const out = await transformIncludes( + "{@includeGitlabReadme: group/proj}", + ctx, + { strict: true, allowedHosts: [] }, + ); + + expect(out).toContain("Chapter one body."); + expect(out).not.toContain("::include"); +}); +``` + +Adapt `makeCtx`/helper names to whatever `transform.test.ts` already defines. If the existing helper cannot express per-path file contents, extend it minimally to do so (matching the file-keying `project@ref/-/path` used by `fetchFileSource`). + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `npx vitest run src/include/transform.test.ts` +Expected: FAIL — the README's `::include` is passed through verbatim (still contains `::include`). + +- [ ] **Step 3: Wire `expandFileIncludes` into `transformIncludes`** + +In `src/include/transform.ts`: + +Add the import (next to the other `./` imports): + +```ts +import { expandFileIncludes } from "./expand.js"; +``` + +Add `allowedHosts` to `TransformOptions` (after `outProcessors?: …`): + +```ts + /** Hostnames allowed as remote `::include{file=https://…}` targets. Default: none. */ + allowedHosts?: string[]; +``` + +Inside the `Promise.all(...)` callback in `transformIncludes`, replace: + +```ts + const { raw, ref } = + kind === "readme" + ? await fetchReadmeSource(ctx, { project: spec.project, ref: spec.ref }) + : await fetchFileSource(ctx, { project: spec.project, path: spec.path!, ref: spec.ref }); + let body = await renderSource(raw, { + ctx, + project: spec.project, + ref, + kind, + path: spec.path, + lineRange: spec.lineRange, + }); +``` + +with: + +```ts + const { raw, ref } = + kind === "readme" + ? await fetchReadmeSource(ctx, { project: spec.project, ref: spec.ref }) + : await fetchFileSource(ctx, { project: spec.project, path: spec.path!, ref: spec.ref }); + let expanded = raw; + if (isMarkdownSource(kind, spec.path)) { + expanded = await expandFileIncludes( + raw, + { + ctx, + project: spec.project, + ref, + allowedHosts: options.allowedHosts ?? [], + strict: options.strict, + }, + { + depth: 0, + stack: new Set([`${spec.project}@${ref}/-/${spec.path ?? "README.md"}`]), + }, + ); + } + let body = await renderSource(expanded, { + ctx, + project: spec.project, + ref, + kind, + path: spec.path, + lineRange: spec.lineRange, + }); +``` + +In `src/include/loader.ts`, add `allowedHosts` to the `options` object built for `transformIncludes` (after `stripToc: resolved.stripToc,`): + +```ts + allowedHosts: resolved.includeAllowedHosts, +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `npx vitest run src/include/transform.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/include/transform.ts src/include/loader.ts src/include/transform.test.ts +git commit -S -m "feat: expand ::include inside loader include pipeline" +``` + +--- + +## Task 7: Full test + typecheck + e2e verification + +**Files:** none (verification only) + +- [ ] **Step 1: Run the full unit suite** + +Run: `npx vitest run` +Expected: PASS (all suites, including the new `expand.test.ts` and updated option/transform tests). + +- [ ] **Step 2: Typecheck** + +Run: `npm run typecheck` +Expected: no errors. + +- [ ] **Step 3: Build** + +Run: `npm run build` +Expected: compiles to `dist/` with no errors. + +- [ ] **Step 4: e2e build (loader pipeline is touched)** + +Run: `npx vitest run test/e2e/build.test.ts` +Expected: PASS (~1 min). Existing includes still render; the added feature does not regress the build. + +- [ ] **Step 5: Commit any incidental fixes** + +If steps 1–4 required small fixes, commit them: + +```bash +git add -A +git commit -S -m "test: verify ::include expansion end-to-end" +``` + +If nothing changed, skip this step. + +--- + +## Task 8: Documentation + +**Files:** +- Modify: `README.md` +- Modify: `examples/gitlab/docs/includes.mdx` + +- [ ] **Step 1: Document the feature in README** + +Add a subsection under the includes documentation in `README.md` explaining: + +```markdown +### `::include` directives inside included markdown + +When a fetched GitLab README or markdown file contains a GitLab +[`::include`](https://docs.gitlab.com/user/markdown/#includes) directive, the +plugin expands it at build time, splicing the referenced file in as raw +markdown: + + ::include{file=docs/chapter1.md} + +- Relative paths resolve to a file in the **same GitLab project and ref** as the + enclosing include, fetched through the same cached client. +- Remote URLs (`::include{file=https://…}`) are fetched only when their host is + listed in the `includeAllowedHosts` plugin option (empty by default, so remote + includes are off until you opt in). +- Includes are expanded recursively (max depth 8) with cycle detection. +- A failed include aborts the build in `strict` mode, or renders an inline + warning otherwise. +``` + +- [ ] **Step 2: Note the behavior in the example site's includes page** + +Add a short paragraph to `examples/gitlab/docs/includes.mdx` explaining that any +`::include` directives inside the fetched READMEs are expanded automatically +(no separate placeholder needed). Keep it documentation-only — do not add an +include that depends on upstream content containing `::include`. + +- [ ] **Step 3: Commit** + +```bash +git add README.md examples/gitlab/docs/includes.mdx +git commit -S -m "docs: document ::include directive expansion" +``` + +--- + +## Definition of Done + +- `::include{file=…}` inside fetched README/markdown-file includes expands to the referenced content as raw markdown, in both `{@includeGitlabReadme}` and markdown `{@includeGitlabFile}` includes. +- Relative paths resolve to the same project/ref via `fetchFileSource`; remote URLs work only for allowlisted hosts. +- Recursion works with depth (8) and cycle guards; directives inside code fences stay literal. +- `strict` throws on failure; non-strict emits an inline warning marker. +- `npx vitest run`, `npm run typecheck`, `npm run build`, and the e2e build all pass. +- README and example docs updated. diff --git a/docs/superpowers/specs/2026-07-04-gitlab-include-directive-design.md b/docs/superpowers/specs/2026-07-04-gitlab-include-directive-design.md new file mode 100644 index 0000000..a63e3f8 --- /dev/null +++ b/docs/superpowers/specs/2026-07-04-gitlab-include-directive-design.md @@ -0,0 +1,221 @@ +# GitLab `::include` directive expansion — design + +- **Status:** approved (brainstorm), pending implementation plan +- **Date:** 2026-07-04 +- **Package:** `@ebuildy/docusaurus-plugin-gitlab` + +## Problem + +GitLab Flavored Markdown supports an **include directive** that splices one +document's content into another +([docs](https://docs.gitlab.com/user/markdown/#includes)): + +```text +::include{file=chapter1.md} +::include{file=https://example.org/installation.md} +``` + +Our plugin fetches GitLab READMEs and files at build time and splices them into +Docusaurus pages, but it does not yet understand `::include`. When a fetched +README (or markdown file) contains `::include{file=…}`, the directive is passed +through verbatim and renders as literal text instead of pulling in the +referenced content. + +We must expand `::include` so the referenced file's content is included **raw in +the main markdown** — i.e. spliced as markdown source that then compiles as part +of the host document. + +## Scope + +`::include` is authored **inside GitLab-hosted markdown** (a README or a `.md` +file), not in the author's `.mdx` page. It is therefore a feature of the +**webpack-loader include subsystem** (`src/include/`), which already fetches +GitLab sources and rewrites the host `.mdx` source at load time: + +```text +gitlabIncludeLoader (src/include/loader.ts) + └─ transformIncludes (src/include/transform.ts) // finds {@includeGitlab…} + ├─ fetchReadmeSource / fetchFileSource // raw GitLab source + └─ renderSource (src/include/render-source.ts) // → MDX-safe markdown source +``` + +`renderSource` produces **markdown source text** (not HTML) that is spliced back +into the host `.mdx` and compiled by Docusaurus. That is exactly the "raw in the +main markdown" semantics we want, so `::include` expansion belongs here. + +**In scope:** expansion inside both `{@includeGitlabReadme}` and markdown +`{@includeGitlabFile}` includes (any content for which `isMarkdownSource` is +true). + +**Also in scope (added later):** the JSX-component path (`` / +markdown ``). The `fetchReadme` / `fetchFile` fetchers expand +`::include` on the raw markdown **before** `renderMarkdown`, so the components +behave consistently with the loader placeholders. This required threading the +include settings (`strict`, `includeAllowedHosts`, `debug`) into the +`GitLabContext.options` populated by `buildContext`. + +## Grammar + +A leaf directive occupying its own line, matching GitLab: + +```text +::include{file=} +``` + +- Only the `file` attribute is honored. Its value may be bare + (`file=chapter1.md`) or quoted (`file="path with space.md"` / single quotes). +- No `ref`, line-range, or `project` extensions (YAGNI). A relative include + inherits the **`ref` of the enclosing include** and targets the **same + project**, matching GitLab's same-repo semantics. +- The directive is recognized as a standalone block (its own line), **including + inside a fenced/indented/inline code region**. Per GitLab + ([Use includes in code blocks](https://docs.gitlab.com/user/markdown/#use-includes-in-code-blocks)), + a directive inside a code region has its target inserted **verbatim** (not + processed as markdown, not wrapped in another fence, not recursed into) so the + file's content appears in the surrounding code block. Outside code regions, + targets are expanded as described below. + +## Design + +### New module: `src/include/expand.ts` + +Exports `expandFileIncludes(md, o, guard)`: + +- `md: string` — markdown source (host include body, already frontmatter-stripped). +- `o` — resolution context: `{ ctx: GitLabContext; project: string; ref: string; + allowedHosts: string[]; strict: boolean }`. +- `guard` — recursion guard: `{ depth: number; stack: Set }`. +- Returns the markdown with every `::include{…}` directive replaced by the raw + (recursively expanded) content of its target. + +### Hook point: `renderSource` + +In `src/include/render-source.ts`, the `isMarkdownSource` branch calls +`expandFileIncludes` **after `stripFrontmatter`, before `processMarkdownSource`**: + +```ts +if (isMarkdownSource(o.kind, o.path)) { + let body = stripFrontmatter(raw); + body = await expandFileIncludes(body, { + ctx: o.ctx, project: o.project, ref: o.ref, + allowedHosts: o.allowedHosts, strict: o.strict, + }, { depth: 0, stack: new Set([keyFor(o)]) }); + return processMarkdownSource(body, { localizeImage, absolutizeLink }); +} +``` + +Consequences of this ordering: + +- Included content is spliced as **raw text**, then the merged document flows + through the *same* prose transforms (image localization, link absolutization, + MDX escaping) as the host — one coherent document. +- Relative images/links inside included files are resolved the same way as the + host README's (project-root-relative, consistent with the existing + `absolutizeFactory` behavior and its current limitation). + +### Resolution per directive + +For each `::include{file=X}` the target is fetched: + +- **Remote** — `X` matches `^https?://`: parse the URL, check its host against + `allowedHosts`. If not listed → error. If listed → fetch the body via native + `fetch` (text). +- **Relative** — otherwise: `fetchFileSource(ctx, { project: o.project, path: X, + ref: o.ref })`, reusing the existing GitLab client and on-disk cache. + +How the fetched content is spliced depends on where the directive sits: + +- **Inside a code region** (fenced/indented/inline) → inserted **verbatim**: no + frontmatter stripping, no fence wrapping, no recursion. The content lands in + the author's existing code block. +- **In prose, markdown target** (`.md`/`.mdx`/`.markdown`) → `stripFrontmatter`-ed + and **recursively expanded** before being spliced. +- **In prose, non-markdown target** → wrapped in a fenced, syntax-highlighted + code block (`` ``` `` via `languageFromPath`) so raw YAML/JSON/etc. + renders cleanly instead of as garbled markdown. + +### Code-region detection + +Each directive's position is classified via `codeRanges` from +`render-source.ts`: a match whose offset falls inside a code range is treated as +"inside a code block" (verbatim insertion above), otherwise as prose. Directives +are **not** skipped — GitLab expands them in both contexts. + +### Recursion guard + +`guard = { depth, stack }`: + +- `depth` increments per nesting level; exceeding `MAX_INCLUDE_DEPTH` (constant, + `8`) throws `"::include exceeded max depth (8)"`. +- `stack` holds the resolution keys currently being expanded, keyed by + `` `${project}@${ref}/-/${path}` `` for GitLab files and by the absolute URL + for remote includes. Re-entering a key already on the stack throws + `"::include cycle detected: "`. Keys are removed as each branch unwinds + (a diamond — the same file included twice in different branches — is allowed; + only true cycles are rejected). + +### Security + +New plugin option `includeAllowedHosts: string[]`, **default `[]`**: + +- Empty ⇒ remote (`http(s)`) includes are rejected; only GitLab-project-file + includes work. +- Hosts are matched exactly (case-insensitive) against the URL's host. +- Threaded through `src/options.ts` (`PluginOptions`, `ResolvedOptions`, Joi + schema, `resolveOptions` default `[]`) → `loader.ts` → `transformIncludes` + (`TransformOptions`) → `renderSource` (`RenderSourceOptions.allowedHosts`). + +Local GitLab-file includes are always permitted; they are already gated by the +configured `host`/`token`. + +### Error handling + +Mirrors `transformIncludes`: + +- `strict: true` (production default) → throw; the error bubbles to + `transformIncludes`'s catch, aborting the build with a + `@ebuildy/docusaurus-plugin-gitlab: …` message. +- `strict: false` → replace the offending directive with an inline marker + `> ⚠️ ::include{file=…} failed — ` and continue. + +Failure cases: unlisted remote host, fetch/network error, missing file, cycle, +depth exceeded, malformed directive. + +## Testing (TDD) + +`src/include/expand.test.ts` (fake/mocked `fetchFileSource` and `fetch`): + +- Relative-file include expands to the fetched file's raw content. +- Remote include from an allowlisted host expands; from a non-allowlisted host + errors (strict) / emits the warning marker (non-strict). +- Nested include (an included file containing `::include`) expands recursively. +- Cycle (A includes B includes A) throws `"cycle detected"`. +- Depth beyond `MAX_INCLUDE_DEPTH` throws. +- `::include{…}` inside a fenced code block inserts the target **verbatim** (no + extra fence, no processing). +- A non-markdown target in prose is wrapped in a highlighted code block. +- Quoted and bare `file=` values both parse. +- Strict vs non-strict error behavior. + +Wiring tests in `src/include/render-source.test.ts`: + +- `renderSource` expands `::include` for markdown sources and passes + `allowedHosts` through. +- Non-markdown (code) includes are unaffected. + +Update `src/include/transform.test.ts` / `src/options.test.ts` as needed for the +new option threading. Run `npx vitest run` and `npm run typecheck`; run the e2e +build (`test/e2e/build.test.ts`) since the loader pipeline is touched. + +## Documentation + +- README: document `::include{file=…}` support and the `includeAllowedHosts` + option. +- Add an example under `examples/site/` exercising a relative include (and, if + practical, a nested include). + +## Non-goals / future work + +- No `ref`, line-range, or `project` attributes on `::include`. +- File-relative (as opposed to project-root-relative) image/link resolution is + not addressed here; it inherits the existing behavior and limitation. diff --git a/examples/gitlab/docs/demo-include.mdx b/examples/gitlab/docs/demo-include.mdx index b78953f..cab2448 100644 --- a/examples/gitlab/docs/demo-include.mdx +++ b/examples/gitlab/docs/demo-include.mdx @@ -1 +1 @@ -{@includeGitlabReadme: ebuildy/demo-docusaurus} \ No newline at end of file +{@includeGitlabReadme: ebuildy/demo-docusaurus} diff --git a/examples/gitlab/docs/includes.mdx b/examples/gitlab/docs/includes.mdx index ce0200a..fd581d0 100644 --- a/examples/gitlab/docs/includes.mdx +++ b/examples/gitlab/docs/includes.mdx @@ -21,3 +21,11 @@ A non-markdown file is wrapped in a fenced code block with the language inferred from its extension; the `#L1-40` range trims it to the first 40 lines: {@includeGitlabFile: gitlab-org/api/client-go/-/gitlab.go#L1-40} + +## Nested `::include` directives + +If a fetched README or file above contained a GitLab `::include{file=…}` +directive, it would be expanded automatically as part of the same fetch — no +extra placeholder needed. Relative targets resolve against the enclosing +file's own project and ref; remote URLs are only followed when their host is +allow-listed via the `includeAllowedHosts` plugin option. diff --git a/examples/gitlab/docusaurus.config.ts b/examples/gitlab/docusaurus.config.ts index b823efe..0878431 100644 --- a/examples/gitlab/docusaurus.config.ts +++ b/examples/gitlab/docusaurus.config.ts @@ -14,6 +14,7 @@ const gitlabOptions = { token: process.env.GITLAB_TOKEN, strict: false, stripToc: true, + debug: true, }; // Live example: embeds REAL public content from gitlab.com. diff --git a/package-lock.json b/package-lock.json index 8fe9cc9..38607d3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "unist-util-visit-parents": "^6.0.0" }, "devDependencies": { + "@docusaurus/logger": "^3.10.1", "@eslint/js": "^9.39.4", "@testing-library/jest-dom": "^6.4.0", "@testing-library/react": "^16.0.0", @@ -544,6 +545,20 @@ "node": ">=20.19.0" } }, + "node_modules/@docusaurus/logger": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/@docusaurus/logger/-/logger-3.10.1.tgz", + "integrity": "sha512-oPjNFnfJsRCkePVjkGrxWGq4MvJKRQT0r9jOP0eRBTZ7Wr9FAbzdP/Gjs0I2Ss6YRkPoEgygKG112OkE6skvJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=20.0" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.9.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", @@ -8685,6 +8700,13 @@ "json5": "lib/cli.js" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index 3f661a5..dceec62 100644 --- a/package.json +++ b/package.json @@ -50,9 +50,15 @@ "prepare": "husky" }, "peerDependencies": { + "@docusaurus/logger": "^3.0.0", "react": ">=18", "react-dom": ">=18" }, + "peerDependenciesMeta": { + "@docusaurus/logger": { + "optional": true + } + }, "dependencies": { "@gitbeaker/core": "^43.8.0", "@gitbeaker/rest": "^43.8.0", @@ -74,6 +80,7 @@ "unist-util-visit-parents": "^6.0.0" }, "devDependencies": { + "@docusaurus/logger": "^3.10.1", "@eslint/js": "^9.39.4", "@testing-library/jest-dom": "^6.4.0", "@testing-library/react": "^16.0.0", diff --git a/src/gitlab/context.ts b/src/gitlab/context.ts index d5ebce5..e8919a1 100644 --- a/src/gitlab/context.ts +++ b/src/gitlab/context.ts @@ -16,5 +16,15 @@ export function buildContext(options: ResolvedOptions): GitLabContext { assetBaseUrl: options.assetBaseUrl, host: options.host, }); - return { client, cache, assets, options: { host: options.host } }; + return { + client, + cache, + assets, + options: { + host: options.host, + strict: options.strict, + allowedHosts: options.includeAllowedHosts, + debug: options.debug, + }, + }; } diff --git a/src/gitlab/fetchers.test.ts b/src/gitlab/fetchers.test.ts index bf25c6d..072dc74 100644 --- a/src/gitlab/fetchers.test.ts +++ b/src/gitlab/fetchers.test.ts @@ -118,6 +118,35 @@ describe("fetchReadme", () => { expect(client.getFileRaw).toHaveBeenCalledWith("g/r", "README.md", "v2"); }); + it("expands ::include directives in the README before rendering", async () => { + const client: any = { + getProject: vi.fn(async () => ({ default_branch: "main" })), + getFileRaw: vi.fn(async (_p: unknown, path: string) => { + if (path === "README.md") return "# Title\n\n::include{file=chapter1.md}\n"; + if (path === "chapter1.md") return "Chapter one body."; + throw new Error(`unexpected ${path}`); + }), + }; + const data = await fetchReadme(ctx(client), { project: "g/r" }); + expect(data.html).toContain("Chapter one body."); + expect(data.html).not.toContain("::include"); + expect(client.getFileRaw).toHaveBeenCalledWith("g/r", "chapter1.md", "main"); + }); + + it("inserts a non-markdown ::include target inside its code fence verbatim", async () => { + const client: any = { + getProject: vi.fn(async () => ({ default_branch: "main" })), + getFileRaw: vi.fn(async (_p: unknown, path: string) => { + if (path === "README.md") return "## Config\n\n```yaml\n::include{file=profiles.yaml}\n```\n"; + if (path === "profiles.yaml") return "name: prod\nport: 8080"; + throw new Error(`unexpected ${path}`); + }), + }; + const data = await fetchReadme(ctx(client), { project: "g/r" }); + expect(data.html).toContain("name: prod"); + expect(data.html).not.toContain("::include"); + }); + it("sidebar mode returns toc entries and assigns heading ids", async () => { const client: any = { getProject: vi.fn(async () => ({ default_branch: "main" })), diff --git a/src/gitlab/fetchers.ts b/src/gitlab/fetchers.ts index 792ea3f..3adb096 100644 --- a/src/gitlab/fetchers.ts +++ b/src/gitlab/fetchers.ts @@ -1,3 +1,8 @@ +// `expandIncludes` lives in the include subsystem, which imports `fetchFileSource` +// from this module — a benign call-time-only cycle (both are hoisted function +// declarations, used only at runtime, never at module-eval time). +import { expandIncludes } from "../include/expand.js"; +import { createIncludeLogger } from "../include/logger.js"; import type { AssetManager } from "./assets"; import type { FileCache } from "./cache"; import type { GitLabClient, PageOptions } from "./client"; @@ -17,7 +22,40 @@ export interface GitLabContext { client: GitLabClient; cache: FileCache; assets: AssetManager; - options: { host: string }; + options: { + host: string; + /** Include-pipeline settings (populated by `buildContext`); optional so test + * fakes can omit them. Defaults applied where read. */ + strict?: boolean; + allowedHosts?: string[]; + debug?: boolean; + }; +} + +/** + * Expand GitLab `::include{file=…}` directives in fetched markdown before it is + * rendered, so the `` / `` components behave like the + * `{@includeGitlab…}` loader path. No-op when the source has no directive. + */ +async function expandDirectives( + ctx: GitLabContext, + project: string, + ref: string, + path: string | undefined, + md: string, +): Promise { + if (!md.includes("::include")) return md; + const log = await createIncludeLogger(ctx.options.debug ?? false); + const expand = expandIncludes({ + ctx, + project, + ref, + path, + allowedHosts: ctx.options.allowedHosts ?? [], + strict: ctx.options.strict ?? true, + log, + }); + return expand(md); } type Attrs = Record; @@ -146,7 +184,8 @@ export async function fetchReadme(ctx: GitLabContext, attrs: Attrs): Promise { const ref = explicitRef ?? (await ctx.client.getProject(attrs.project as string | number)).default_branch; - const md = await ctx.client.getFileRaw(attrs.project as string | number, "README.md", ref); + const rawMd = await ctx.client.getFileRaw(attrs.project as string | number, "README.md", ref); + const md = await expandDirectives(ctx, project, ref, undefined, rawMd); const collectToc: TocEntry[] = []; const html = await renderMarkdown(md, { tocMode, @@ -313,7 +352,8 @@ export async function fetchFile(ctx: GitLabContext, attrs: Attrs): Promise ctx.assets.localize(src, ref, String(project)), }); return { kind: "markdown", html, ref, path } satisfies FileData; diff --git a/src/include/expand.test.ts b/src/include/expand.test.ts new file mode 100644 index 0000000..d84896b --- /dev/null +++ b/src/include/expand.test.ts @@ -0,0 +1,341 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { + parseIncludeAttrs, + expandFileIncludes, + expandIncludes, + type ExpandContext, + type ExpandGuard, +} from "./expand.js"; + +describe("parseIncludeAttrs", () => { + it("reads a bare file value", () => { + expect(parseIncludeAttrs("file=chapter1.md")).toEqual({ file: "chapter1.md" }); + }); + it("reads a double-quoted file value with spaces", () => { + expect(parseIncludeAttrs('file="a b.md"')).toEqual({ file: "a b.md" }); + }); + it("reads a single-quoted file value", () => { + expect(parseIncludeAttrs("file='c.md'")).toEqual({ file: "c.md" }); + }); + it("reads a URL value", () => { + expect(parseIncludeAttrs("file=https://example.org/x.md")).toEqual({ + file: "https://example.org/x.md", + }); + }); + it("returns empty when file is absent", () => { + expect(parseIncludeAttrs("other=1")).toEqual({}); + }); +}); + +vi.mock("../gitlab/fetchers.js", () => ({ + fetchFileSource: vi.fn(), +})); +import { fetchFileSource } from "../gitlab/fetchers.js"; + +const mockedFetchFileSource = vi.mocked(fetchFileSource); + +function baseCtx(overrides: Partial = {}): ExpandContext { + return { + ctx: {} as any, + project: "group/proj", + ref: "main", + allowedHosts: [], + strict: true, + ...overrides, + }; +} + +function baseGuard(): ExpandGuard { + return { depth: 0, stack: new Set(["group/proj@main/-/README.md"]) }; +} + +describe("expandFileIncludes — relative GitLab files", () => { + beforeEach(() => { + mockedFetchFileSource.mockReset(); + }); + + it("replaces a directive with the fetched file's raw content", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "Chapter one body.", ref: "main" }); + const out = await expandFileIncludes( + "Intro\n\n::include{file=chapter1.md}\n\nOutro", + baseCtx(), + baseGuard(), + ); + expect(out).toContain("Chapter one body."); + expect(out).not.toContain("::include"); + expect(mockedFetchFileSource).toHaveBeenCalledWith({} as any, { + project: "group/proj", + path: "chapter1.md", + ref: "main", + }); + }); + + it("strips frontmatter from the included file", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "---\ntitle: x\n---\nBody", ref: "main" }); + const out = await expandFileIncludes("::include{file=a.md}", baseCtx(), baseGuard()); + expect(out).toContain("Body"); + expect(out).not.toContain("title: x"); + }); + + it("expands a directive inside a fenced code block verbatim (GitLab 'includes in code blocks')", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: 'const s = "hi";\nalert(s);', ref: "main" }); + const md = "```javascript\n::include{file=code.js}\n```"; + const out = await expandFileIncludes(md, baseCtx(), baseGuard()); + // Target inserted verbatim inside the existing fence — no extra fence, no processing. + expect(out).toBe('```javascript\nconst s = "hi";\nalert(s);\n```'); + expect(mockedFetchFileSource).toHaveBeenCalledWith({} as any, { + project: "group/proj", + path: "code.js", + ref: "main", + }); + }); + + it("returns the source unchanged when there is no directive", async () => { + const out = await expandFileIncludes("plain text", baseCtx(), baseGuard()); + expect(out).toBe("plain text"); + expect(mockedFetchFileSource).not.toHaveBeenCalled(); + }); + + it("expands a directive embedded in a multi-line document, preserving surrounding lines", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "name: prod\nport: 8080", ref: "main" }); + const md = `## Includes + +\`\`\`yaml +::include{file=profiles.yaml} +\`\`\``; + const out = await expandFileIncludes(md, baseCtx(), baseGuard()); + // Heading + the author's ```yaml fence kept; the directive is replaced by the + // file content inserted verbatim inside that fence (GitLab "includes in code blocks"). + expect(out).toContain("## Includes"); + expect(out).toContain("```yaml"); + expect(out).toContain("name: prod"); + expect(out).not.toContain("::include"); + expect(mockedFetchFileSource).toHaveBeenCalledWith({} as any, { + project: "group/proj", + path: "profiles.yaml", + ref: "main", + }); + }); +}); + +describe("expandFileIncludes — remote URLs", () => { + beforeEach(() => { + mockedFetchFileSource.mockReset(); + vi.unstubAllGlobals(); + }); + + it("expands a remote include from an allowlisted host", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ ok: true, status: 200, text: async () => "Remote body." })), + ); + const out = await expandFileIncludes( + "::include{file=https://example.org/x.md}", + baseCtx({ allowedHosts: ["example.org"] }), + baseGuard(), + ); + expect(out).toContain("Remote body."); + }); + + it("rejects a remote include from a non-allowlisted host (strict throws)", async () => { + await expect( + expandFileIncludes( + "::include{file=https://evil.test/x.md}", + baseCtx({ allowedHosts: ["example.org"], strict: true }), + baseGuard(), + ), + ).rejects.toThrow(/host not allowed/); + }); + + it("emits a warning marker for a non-allowlisted host when non-strict", async () => { + const out = await expandFileIncludes( + "::include{file=https://evil.test/x.md}", + baseCtx({ allowedHosts: ["example.org"], strict: false }), + baseGuard(), + ); + expect(out).toContain("⚠️"); + expect(out).toContain("host not allowed"); + }); + + it("errors on a non-OK remote response", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ ok: false, status: 404, text: async () => "" })), + ); + await expect( + expandFileIncludes( + "::include{file=https://example.org/missing.md}", + baseCtx({ allowedHosts: ["example.org"], strict: true }), + baseGuard(), + ), + ).rejects.toThrow(/404/); + }); +}); + +describe("expandFileIncludes — recursion + guards", () => { + beforeEach(() => { + mockedFetchFileSource.mockReset(); + }); + + it("recursively expands includes inside an included markdown file", async () => { + mockedFetchFileSource.mockImplementation(async (_ctx, args: any) => { + if (args.path === "a.md") return { raw: "A\n\n::include{file=b.md}", ref: "main" }; + if (args.path === "b.md") return { raw: "B body", ref: "main" }; + throw new Error(`unexpected ${args.path}`); + }); + const out = await expandFileIncludes("::include{file=a.md}", baseCtx(), baseGuard()); + expect(out).toContain("A"); + expect(out).toContain("B body"); + expect(out).not.toContain("::include"); + }); + + it("detects a direct cycle", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "loop\n\n::include{file=a.md}", ref: "main" }); + await expect( + expandFileIncludes("::include{file=a.md}", baseCtx({ strict: true }), baseGuard()), + ).rejects.toThrow(/cycle detected/); + }); + + it("enforces the max depth", async () => { + // Every file includes a deeper one, never terminating. + mockedFetchFileSource.mockImplementation(async (_ctx, args: any) => ({ + raw: `level\n\n::include{file=${args.path}x.md}`, + ref: "main", + })); + await expect( + expandFileIncludes("::include{file=a.md}", baseCtx({ strict: true }), baseGuard()), + ).rejects.toThrow(/max depth/); + }); + + it("allows the same file included twice in separate branches (diamond)", async () => { + mockedFetchFileSource.mockImplementation(async (_ctx, args: any) => { + if (args.path === "top.md") + return { raw: "::include{file=shared.md}\n\n::include{file=shared.md}", ref: "main" }; + if (args.path === "shared.md") return { raw: "SHARED", ref: "main" }; + throw new Error(`unexpected ${args.path}`); + }); + const out = await expandFileIncludes("::include{file=top.md}", baseCtx(), baseGuard()); + expect(out.match(/SHARED/g)?.length).toBe(2); + }); +}); + +describe("expandFileIncludes — robustness fixes", () => { + beforeEach(() => { + mockedFetchFileSource.mockReset(); + vi.unstubAllGlobals(); + }); + + it("does not strip a leading --- block from a non-markdown include", async () => { + mockedFetchFileSource.mockResolvedValue({ + raw: "---\nname: prod\nport: 8080\n---\nrest: true", + ref: "main", + }); + const out = await expandFileIncludes("::include{file=config/settings.yml}", baseCtx(), baseGuard()); + expect(out).toContain("name: prod"); + expect(out).toContain("port: 8080"); + }); + + it("does not recurse into a non-markdown include's ::include-looking content", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "::include{file=evil.md}", ref: "main" }); + const out = await expandFileIncludes("::include{file=data.txt}", baseCtx(), baseGuard()); + // .txt is not markdown → fenced verbatim, its inner directive never followed + expect(out).toContain("::include{file=evil.md}"); + // fetched exactly once (the .txt), never followed the inner directive + expect(mockedFetchFileSource).toHaveBeenCalledTimes(1); + }); + + it("wraps a non-markdown include in a fenced code block using its language", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "name: prod\nport: 8080", ref: "main" }); + const out = await expandFileIncludes("::include{file=config/profiles.yaml}", baseCtx(), baseGuard()); + expect(out).toContain("```yaml"); + expect(out).toMatch(/```yaml\n[\s\S]*name: prod[\s\S]*```/); + expect(out).not.toContain("::include"); + }); + + it("blocks a remote include that responds with a redirect (allowlist bypass prevention)", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ ok: false, status: 302, text: async () => "" })), + ); + await expect( + expandFileIncludes( + "::include{file=https://example.org/redirect}", + baseCtx({ allowedHosts: ["example.org"], strict: true }), + baseGuard(), + ), + ).rejects.toThrow(/redirect blocked/); + }); + + it("passes redirect:manual to fetch so redirects are not auto-followed", async () => { + const fetchMock = vi.fn(async () => ({ ok: true, status: 200, text: async () => "ok body" })); + vi.stubGlobal("fetch", fetchMock); + await expandFileIncludes( + "::include{file=https://example.org/x.md}", + baseCtx({ allowedHosts: ["example.org"] }), + baseGuard(), + ); + expect(fetchMock).toHaveBeenCalledWith(expect.anything(), { redirect: "manual" }); + }); +}); + +describe("expandFileIncludes — debug logging", () => { + beforeEach(() => { + mockedFetchFileSource.mockReset(); + }); + + it("traces each resolved directive through the provided logger", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "Body", ref: "main" }); + const debug = vi.fn(); + const out = await expandFileIncludes( + "::include{file=chapter1.md}", + baseCtx({ log: { debug } }), + baseGuard(), + ); + expect(out).toContain("Body"); + expect(debug.mock.calls.some((c) => String(c[0]).includes("::include chapter1.md"))).toBe(true); + }); + + it("does nothing when no logger is provided", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "Body", ref: "main" }); + await expect( + expandFileIncludes("::include{file=a.md}", baseCtx(), baseGuard()), + ).resolves.toContain("Body"); + }); +}); + +describe("expandIncludes (source processor)", () => { + beforeEach(() => { + mockedFetchFileSource.mockReset(); + }); + + it("returns an OutProcessor-shaped fn that expands includes with its bound ctx", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "Chapter body.", ref: "main" }); + const proc = expandIncludes({ + ctx: {} as any, + project: "group/proj", + ref: "main", + allowedHosts: [], + strict: true, + }); + const out = await proc("::include{file=chapter1.md}"); + expect(out).toContain("Chapter body."); + expect(mockedFetchFileSource).toHaveBeenCalledWith({} as any, { + project: "group/proj", + path: "chapter1.md", + ref: "main", + }); + }); + + it("seeds cycle detection with the host path so a self-referential include is caught", async () => { + mockedFetchFileSource.mockResolvedValue({ raw: "loop", ref: "main" }); + const proc = expandIncludes({ + ctx: {} as any, + project: "group/proj", + ref: "main", + path: "docs/page.md", + allowedHosts: [], + strict: true, + }); + await expect(proc("::include{file=docs/page.md}")).rejects.toThrow(/cycle detected/); + }); +}); diff --git a/src/include/expand.ts b/src/include/expand.ts new file mode 100644 index 0000000..aa24a59 --- /dev/null +++ b/src/include/expand.ts @@ -0,0 +1,177 @@ +import { languageFromPath } from "../gitlab/code.js"; +import { fetchFileSource } from "../gitlab/fetchers.js"; +import type { GitLabContext } from "../gitlab/fetchers.js"; +import type { IncludeLogger } from "./logger.js"; +import type { OutProcessor } from "./out-processors.js"; +import { codeRanges, stripFrontmatter } from "./render-source.js"; + +/** Markdown file extensions whose content is expanded recursively as markdown. */ +const MD_EXT = /\.(?:md|mdx|markdown)$/i; + +/** Maximum nesting depth for recursive `::include` expansion. */ +export const MAX_INCLUDE_DEPTH = 8; + +/** Extract the `file=` value from a `::include{…}` attribute string (bare or quoted). */ +export function parseIncludeAttrs(attrs: string): { file?: string } { + const m = /(?:^|\s)file\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s}]+))/.exec(attrs); + if (!m) return {}; + return { file: m[1] ?? m[2] ?? m[3] }; +} + +export interface ExpandContext { + ctx: GitLabContext; + project: string; + ref: string; + allowedHosts: string[]; + strict: boolean; + /** Optional debug logger; when present, each resolved directive is traced. */ + log?: IncludeLogger; +} + +export interface ExpandGuard { + depth: number; + stack: Set; +} + +/** A standalone `::include{…}` leaf directive occupying its own line. */ +const INCLUDE_RE = /^[ \t]*::include\{([^}\n]*)\}[ \t]*$/gm; + +/** Resolve a directive's target to raw text plus a cycle key and markdown flag. */ +async function resolveTarget( + file: string, + o: ExpandContext, +): Promise<{ raw: string; key: string; isMarkdown: boolean }> { + if (/^https?:\/\//i.test(file)) { + const url = new URL(file); + if (!o.allowedHosts.some((h) => h.toLowerCase() === url.host.toLowerCase())) { + throw new Error(`::include host not allowed: ${url.host}`); + } + const res = await fetch(url, { redirect: "manual" }); + if (res.status >= 300 && res.status < 400) { + throw new Error(`::include remote redirect blocked: ${url.href} → ${res.status}`); + } + if (!res.ok) throw new Error(`fetch ${url.href} → HTTP ${res.status}`); + return { raw: await res.text(), key: url.href, isMarkdown: MD_EXT.test(url.pathname) }; + } + const key = `${o.project}@${o.ref}/-/${file}`; + const src = await fetchFileSource(o.ctx, { project: o.project, path: file, ref: o.ref }); + return { raw: src.raw, key, isMarkdown: MD_EXT.test(file) }; +} + +/** + * Resolve a single directive to its replacement text, honoring `strict`. + * `inCodeBlock` marks a directive that sits inside a fenced/inline code region: + * per GitLab ("Use includes in code blocks"), its target is inserted **verbatim** + * — not processed as markdown, not wrapped in another fence, not recursed into. + */ +async function resolveOne( + full: string, + attrs: string, + o: ExpandContext, + guard: ExpandGuard, + inCodeBlock: boolean, +): Promise { + try { + const { file } = parseIncludeAttrs(attrs); + if (!file) throw new Error("::include missing file= attribute"); + const { raw, key, isMarkdown } = await resolveTarget(file, o); + o.log?.debug( + `::include ${file} → ${key} [depth ${guard.depth}, ` + + `${inCodeBlock ? "in code block, verbatim" : isMarkdown ? "markdown, expanding" : `code:${languageFromPath(file)}`}, ` + + `${raw.length} bytes]`, + ); + // Inside a code fence: insert the target verbatim (GitLab semantics). + if (inCodeBlock) return raw; + // Non-markdown target in prose: wrap in a fenced, syntax-highlighted block so + // raw YAML/JSON/etc. doesn't render as garbled markdown. + if (!isMarkdown) return `\n\`\`\`${languageFromPath(file)}\n${raw}\n\`\`\`\n`; + // Markdown target in prose: strip frontmatter and expand nested includes. + if (guard.stack.has(key)) throw new Error(`::include cycle detected: ${key}`); + const body = stripFrontmatter(raw); + return expandFileIncludes(body, o, { + depth: guard.depth + 1, + stack: new Set(guard.stack).add(key), + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (o.strict) { + throw new Error(`@ebuildy/docusaurus-plugin-gitlab: ${full} failed — ${message}`); + } + return `\n\n> ⚠️ ${full} failed — ${message}\n\n`; + } +} + +/** + * Replace every standalone `::include{file=…}` directive in `md` with its target. + * Directives in a fenced/inline code region get their target inserted verbatim + * (GitLab "includes in code blocks"); directives in prose get markdown targets + * expanded recursively and other files wrapped in a highlighted code block. + * `guard` carries recursion state. + */ +export async function expandFileIncludes( + md: string, + o: ExpandContext, + guard: ExpandGuard, +): Promise { + if (guard.depth > MAX_INCLUDE_DEPTH) { + throw new Error(`::include exceeded max depth (${MAX_INCLUDE_DEPTH})`); + } + const ranges = codeRanges(md); + const inCode = (i: number) => ranges.some(([s, e]) => i >= s && i < e); + + const matches = [...md.matchAll(INCLUDE_RE)]; + o.log?.debug(`expandFileIncludes: ${matches.length} directive(s) at depth ${guard.depth}`); + if (matches.length === 0) return md; + + const replacements = await Promise.all( + matches.map((m) => resolveOne(m[0], m[1], o, guard, inCode(m.index ?? 0))), + ); + + let out = ""; + let last = 0; + matches.forEach((m, i) => { + out += md.slice(last, m.index) + replacements[i]; + last = (m.index ?? 0) + m[0].length; + }); + return out + md.slice(last); +} + +/** Configuration for the {@link expandIncludes} source processor. */ +export interface ExpandConfig { + ctx: GitLabContext; + project: string; + ref: string; + /** Path of the host include (a markdown file), or undefined for a README. Seeds + * cycle detection so a nested include can't re-enter the host document. */ + path?: string; + allowedHosts: string[]; + strict: boolean; + /** Optional debug logger; traces each resolved `::include` directive. */ + log?: IncludeLogger; +} + +/** + * Build a pre-render source processor that expands GitLab `::include{file=…}` + * directives in raw fetched markdown, against the given project/ref. It runs + * *before* `renderSource` — the directive's `{…}` braces would be MDX-escaped + * afterward — and reuses the {@link OutProcessor} shape so it composes with the + * post-render processor chain via `applyOutProcessors`. + */ +export function expandIncludes(config: ExpandConfig): OutProcessor { + const o: ExpandContext = { + ctx: config.ctx, + project: config.project, + ref: config.ref, + allowedHosts: config.allowedHosts, + strict: config.strict, + log: config.log, + }; + if (config.log) { + config.log.debug?.( + `expandIncludes: ${config.project}@${config.ref}/-/${config.path ?? "README.md"} ` + + `[allowedHosts=${config.allowedHosts.join(",")}]`, + ); + } + const seed = `${config.project}@${config.ref}/-/${config.path ?? "README.md"}`; + return (md) => expandFileIncludes(md, o, { depth: 0, stack: new Set([seed]) }); +} diff --git a/src/include/loader.ts b/src/include/loader.ts index dec351e..0333409 100644 --- a/src/include/loader.ts +++ b/src/include/loader.ts @@ -24,6 +24,8 @@ export default function gitlabIncludeLoader(this: LoaderThis, source: string): v fixInlineStyles: resolved.fixInlineStyles, convertAlerts: resolved.convertAlerts, stripToc: resolved.stripToc, + allowedHosts: resolved.includeAllowedHosts, + debug: resolved.debug, outProcessors: processorsId ? getOutProcessors(processorsId) : [], }; transformIncludes(source, getContext(resolved), options).then( diff --git a/src/include/logger.test.ts b/src/include/logger.test.ts new file mode 100644 index 0000000..e985a9f --- /dev/null +++ b/src/include/logger.test.ts @@ -0,0 +1,23 @@ +import docusaurusLogger from "@docusaurus/logger"; +import { describe, it, expect, vi } from "vitest"; +import { createIncludeLogger } from "./logger.js"; + +vi.mock("@docusaurus/logger", () => ({ default: { info: vi.fn() } })); + +const info = (docusaurusLogger as unknown as { info: ReturnType }).info; + +describe("createIncludeLogger", () => { + it("returns a no-op logger when disabled (does not touch @docusaurus/logger)", async () => { + info.mockClear(); + const log = await createIncludeLogger(false); + expect(() => log.debug("anything")).not.toThrow(); + expect(info).not.toHaveBeenCalled(); + }); + + it("routes debug messages through @docusaurus/logger with a prefix when enabled", async () => { + info.mockClear(); + const log = await createIncludeLogger(true); + log.debug("hello world"); + expect(info).toHaveBeenCalledWith("[gitlab-include] hello world"); + }); +}); diff --git a/src/include/logger.ts b/src/include/logger.ts new file mode 100644 index 0000000..70975e0 --- /dev/null +++ b/src/include/logger.ts @@ -0,0 +1,31 @@ +/** A minimal debug logger for the build-time include pipeline. */ +export interface IncludeLogger { + debug(message: string): void; +} + +const NOOP: IncludeLogger = { debug() {} }; + +type DocusaurusLogger = { info: (message: string) => void }; + +/** + * Build the include-pipeline debug logger. When `enabled`, traces are printed + * through Docusaurus's own logger (`@docusaurus/logger`) so they match the rest + * of the build output; when disabled, a no-op is returned and `@docusaurus/logger` + * is never loaded (it's an optional peer dependency, imported lazily here). + * + * Docusaurus's logger is CJS and has no debug level, so — via a defensive + * `.default` unwrap for the ESM/CJS interop — traces go through `info` with a + * `[gitlab-include]` prefix. + */ +export async function createIncludeLogger(enabled: boolean): Promise { + if (!enabled) return NOOP; + const imported = (await import("@docusaurus/logger")).default as unknown as DocusaurusLogger & { + default?: DocusaurusLogger; + }; + const logger: DocusaurusLogger = imported.default ?? imported; + return { + debug(message: string) { + logger.info(`[gitlab-include] ${message}`); + }, + }; +} diff --git a/src/include/transform.test.ts b/src/include/transform.test.ts index c3a9760..396e5f4 100644 --- a/src/include/transform.test.ts +++ b/src/include/transform.test.ts @@ -1,13 +1,19 @@ import { describe, it, expect } from "vitest"; import { transformIncludes } from "./transform.js"; -function makeCtx() { +function makeCtx(opts?: { files?: Record }) { const store = new Map(); + const files = opts?.files; return { client: { getProject: async () => ({ default_branch: "main" }), - getFileRaw: async (_p: unknown, path: string) => - path === "README.md" ? "# Title\n\nbody {x}" : "const a = 1;\nconst b = 2;\n", + getFileRaw: async (project: unknown, path: string, ref: string) => { + if (files) { + const key = `${project}@${ref}/-/${path}`; + if (key in files) return files[key]; + } + return path === "README.md" ? "# Title\n\nbody {x}" : "const a = 1;\nconst b = 2;\n"; + }, }, cache: { get: async (k: string) => store.get(k), set: async (k: string, v: unknown) => void store.set(k, v) }, assets: { localize: async (u: string) => `/a/${u}` }, @@ -180,6 +186,24 @@ describe("transformIncludes", () => { expect(out).toContain("x [processed]"); }); + it("expands ::include directives inside a fetched README", async () => { + const ctx = makeCtx({ + files: { + "group/proj@main/-/README.md": "# Title\n\n::include{file=chapter1.md}", + "group/proj@main/-/chapter1.md": "Chapter one body.", + }, + }); + + const out = await transformIncludes( + "{@includeGitlabReadme: group/proj}", + ctx, + { strict: true, allowedHosts: [] }, + ); + + expect(out).toContain("Chapter one body."); + expect(out).not.toContain("::include"); + }); + it("does not run processors on a code-file include", async () => { const ctx = { client: { diff --git a/src/include/transform.ts b/src/include/transform.ts index 37eadb6..10bfef5 100644 --- a/src/include/transform.ts +++ b/src/include/transform.ts @@ -1,6 +1,8 @@ import type { GitLabContext } from "../gitlab/fetchers.js"; import { fetchFileSource, fetchReadmeSource } from "../gitlab/fetchers.js"; +import { expandIncludes } from "./expand.js"; import { parseInclude } from "./grammar.js"; +import { createIncludeLogger } from "./logger.js"; import { applyOutProcessors, convertAlerts, @@ -28,6 +30,10 @@ export interface TransformOptions { stripToc?: boolean; /** Extra post-processors applied to the generated markdown of each markdown include. */ outProcessors?: OutProcessor[]; + /** Hostnames allowed as remote `::include{file=https://…}` targets. Default: none. */ + allowedHosts?: string[]; + /** Emit build-time debug traces for the include pipeline. Default: false. */ + debug?: boolean; } export async function transformIncludes( @@ -38,6 +44,8 @@ export async function transformIncludes( const matches = [...source.matchAll(PLACEHOLDER_RE)]; if (matches.length === 0) return source; + const log = await createIncludeLogger(options.debug ?? false); + // The built-in autolink fix is driven by a serializable flag (so it reliably // crosses the webpack loader boundary); user processors come from the registry. const processors: OutProcessor[] = [ @@ -57,17 +65,41 @@ export async function transformIncludes( }); } + log.debug(`found ${matches.length} include placeholder(s), ${seen.size} unique to resolve`); + // Resolve each unique placeholder once (dedupe by full match text). const replacements = new Map(); await Promise.all( [...seen.entries()].map(async ([full, { kind, arg }]) => { try { const spec = parseInclude(kind, arg); + log.debug( + `resolving ${full} → ${kind} ${spec.project}@${spec.ref ?? "(default ref)"}` + + `${spec.path ? `/-/${spec.path}` : ""}${spec.lineRange ? `#L${spec.lineRange}` : ""}`, + ); const { raw, ref } = kind === "readme" ? await fetchReadmeSource(ctx, { project: spec.project, ref: spec.ref }) : await fetchFileSource(ctx, { project: spec.project, path: spec.path!, ref: spec.ref }); - let body = await renderSource(raw, { + log.debug(`fetched ${spec.project}@${ref}${spec.path ? `/-/${spec.path}` : ""} (${raw.length} bytes)`); + // Pre-render source processors run on the RAW source before renderSource; + // ::include expansion must happen here (its `{…}` braces would be + // MDX-escaped afterward). Reuses the OutProcessor shape + runner. + const sourceProcessors: OutProcessor[] = isMarkdownSource(kind, spec.path) + ? [ + expandIncludes({ + ctx, + project: spec.project, + ref, + path: spec.path, + allowedHosts: options.allowedHosts ?? [], + strict: options.strict, + log, + }), + ] + : []; + const expanded = await applyOutProcessors(raw, sourceProcessors); + let body = await renderSource(expanded, { ctx, project: spec.project, ref, @@ -78,9 +110,11 @@ export async function transformIncludes( if (processors.length && isMarkdownSource(kind, spec.path)) { body = await applyOutProcessors(body, processors); } + log.debug(`rendered ${full} → ${body.length} chars of MDX`); replacements.set(full, `\n\n${body}\n\n`); } catch (err) { const message = err instanceof Error ? err.message : String(err); + log.debug(`FAILED ${full} — ${message}`); if (options.strict) { throw new Error(`@ebuildy/docusaurus-plugin-gitlab: ${full} failed — ${message}`); } diff --git a/src/options.test.ts b/src/options.test.ts index 72e79b5..00a2bb2 100644 --- a/src/options.test.ts +++ b/src/options.test.ts @@ -9,6 +9,12 @@ describe("resolveOptions", () => { expect(o.assetDir).toBe("static/gitlab-assets"); expect(o.assetBaseUrl).toBe("/gitlab-assets"); expect(o.cache).toEqual({ ttl: 3600 }); + expect(o.debug).toBe(false); + }); + + it("passes through debug: true", () => { + const o = resolveOptions({ host: "https://gitlab.com", debug: true }, "production"); + expect(o.debug).toBe(true); }); it("defaults strict to false in development", () => { @@ -89,4 +95,26 @@ describe("resolveOptions", () => { ); expect("outProcessors" in resolved).toBe(false); }); + + it("defaults includeAllowedHosts to an empty array", () => { + const o = resolveOptions({ host: "https://gitlab.com" }, "production"); + expect(o.includeAllowedHosts).toEqual([]); + }); + + it("passes through a configured includeAllowedHosts list", () => { + const o = resolveOptions( + { host: "https://gitlab.com", includeAllowedHosts: ["example.org"] }, + "production", + ); + expect(o.includeAllowedHosts).toEqual(["example.org"]); + }); + + it("rejects a non-array includeAllowedHosts", () => { + expect(() => + resolveOptions( + { host: "https://gitlab.com", includeAllowedHosts: "example.org" } as any, + "production", + ), + ).toThrow(); + }); }); diff --git a/src/options.ts b/src/options.ts index 2daec3b..5bda88e 100644 --- a/src/options.ts +++ b/src/options.ts @@ -23,6 +23,14 @@ export interface PluginOptions { /** Remove a redundant "Table of Contents" section (and any `[[_TOC_]]` marker) * from included markdown — Docusaurus renders its own. Default: `false`. */ stripToc?: boolean; + /** Hostnames (exact, case-insensitive) allowed as remote `::include{file=https://…}` + * targets inside fetched GitLab markdown. Empty ⇒ remote includes are rejected. + * Default: `[]`. */ + includeAllowedHosts?: string[]; + /** Emit build-time debug traces for the include pipeline (each resolved + * placeholder and `::include` directive) via `@docusaurus/logger`. + * Default: `false`. */ + debug?: boolean; /** Extra post-processors applied to the markdown generated from includes, * in order, after the built-in fixes (when enabled). */ outProcessors?: OutProcessor[]; @@ -40,6 +48,8 @@ export interface ResolvedOptions { fixInlineStyles: boolean; convertAlerts: boolean; stripToc: boolean; + includeAllowedHosts: string[]; + debug: boolean; } const schema = Joi.object({ @@ -60,6 +70,8 @@ const schema = Joi.object({ fixInlineStyles: Joi.boolean().optional(), convertAlerts: Joi.boolean().optional(), stripToc: Joi.boolean().optional(), + includeAllowedHosts: Joi.array().items(Joi.string()).optional(), + debug: Joi.boolean().optional(), outProcessors: Joi.array().items(Joi.function()).optional(), }); @@ -83,5 +95,7 @@ export function resolveOptions( fixInlineStyles: opts.fixInlineStyles ?? true, convertAlerts: opts.convertAlerts ?? true, stripToc: opts.stripToc ?? false, + includeAllowedHosts: opts.includeAllowedHosts ?? [], + debug: opts.debug ?? false, }; }