From b2cc7bcb1564e022c428e96054fdcab2df718218 Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Mon, 13 Jul 2026 09:11:36 -0700 Subject: [PATCH 1/3] Sort axe report violations by severity, then standard The axe document reporter previously listed violations in axe-core's own order. Sort them most-important-first: by impact (critical, serious, moderate, minor, then null), then by conformance standard (WCAG A, AA, AAA, Best Practice, obsolete, untagged), then by rule id for deterministic output. Sorting happens in createReportElement, so the fixed overlay, revealjs report slide, dashboard offcanvas, and dashboard rescans all inherit it. The reporter sorts a copy, so the json reporter keeps emitting axe's raw result untouched. The rank helpers are exported for unit tests (and for reuse by the upcoming project scanner), alongside a parseConformanceTags helper now shared with axeConformanceLevel. --- src/resources/formats/html/axe/axe-check.js | 80 ++++++++--- tests/unit/axe-sort.test.ts | 143 ++++++++++++++++++++ 2 files changed, 203 insertions(+), 20 deletions(-) create mode 100644 tests/unit/axe-sort.test.ts diff --git a/src/resources/formats/html/axe/axe-check.js b/src/resources/formats/html/axe/axe-check.js index f62f2fd9b2..808c308ae7 100644 --- a/src/resources/formats/html/axe/axe-check.js +++ b/src/resources/formats/html/axe/axe-check.js @@ -1,24 +1,35 @@ -// Derive a human-readable WCAG conformance label from axe-core's `tags` array. -// Tags encode the version+level (`wcag2a`, `wcag21aa`), the specific success -// criteria (`wcag111` → 1.1.1), and `best-practice` for axe's own -// recommendations that aren't tied to any WCAG success criterion. Returns "" when -// no conformance tags are present so callers can fall back to the impact alone. -export function axeConformanceLevel(tags) { - if (tags.includes("best-practice")) return "Best Practice"; - - // Version+level: wcag2a, wcag2aa, wcag21aa, wcag22aa, ... An `-obsolete` - // suffix (e.g. `wcag2a-obsolete` on the deprecated `duplicate-id` rule) marks - // a criterion that was withdrawn from later WCAG versions, e.g. SC 4.1.1, - // removed in WCAG 2.2. We surface the original level but flag it as obsolete - // so a withdrawn criterion isn't mistaken for a current conformance failure. +// Parse the conformance-related tags from axe-core's `tags` array. Tags encode +// the version+level (`wcag2a`, `wcag21aa`), and `best-practice` for axe's own +// recommendations that aren't tied to any WCAG success criterion. An +// `-obsolete` suffix (e.g. `wcag2a-obsolete` on the deprecated `duplicate-id` +// rule) marks a criterion that was withdrawn from later WCAG versions, e.g. +// SC 4.1.1, removed in WCAG 2.2. Returns: +// { bestPractice: true } for best-practice rules +// { major, minor, level, obsolete } for WCAG version+level tags (level is +// the raw "a"/"aa"/"aaa" string) +// null when no conformance tags are present +function parseConformanceTags(tags) { + if (tags.includes("best-practice")) return { bestPractice: true }; const versionTag = tags.find((t) => /^wcag\d+a+(-obsolete)?$/.test(t)); - // Without a version+level tag there's no conformance level to report, so fall - // back to the impact alone rather than emitting a bare, level-less criterion. - if (!versionTag) return ""; - + if (!versionTag) return null; const [, major, minor, level, obsolete] = versionTag.match(/^wcag(\d)(\d?)(a+)(-obsolete)?$/); - let label = `WCAG ${major}.${minor || "0"} ${level.toUpperCase()}`; + return { major, minor, level, obsolete: !!obsolete }; +} + +// Derive a human-readable WCAG conformance label from axe-core's `tags` array, +// including the specific success criteria (`wcag111` → 1.1.1). Returns "" when +// no conformance tags are present so callers can fall back to the impact alone. +export function axeConformanceLevel(tags) { + const parsed = parseConformanceTags(tags); + // Without conformance tags there's no level to report, so fall back to the + // impact alone rather than emitting a bare, level-less criterion. + if (!parsed) return ""; + if (parsed.bestPractice) return "Best Practice"; + + // We surface an obsolete criterion's original level but flag it as obsolete + // so it isn't mistaken for a current conformance failure. + let label = `WCAG ${parsed.major}.${parsed.minor || "0"} ${parsed.level.toUpperCase()}`; // Success criteria: wcag111 → 1.1.1, wcag1410 → 1.4.10. Principle and // guideline are always single digits; the remainder is the criterion number. @@ -32,7 +43,34 @@ export function axeConformanceLevel(tags) { if (criteria.length) { label += ` (${criteria.join(", ")})`; } - return obsolete ? `Obsolete ${label}` : label; + return parsed.obsolete ? `Obsolete ${label}` : label; +} + +// Rank a violation's axe-core `impact` for sorting: critical=0, serious=1, +// moderate=2, minor=3. A null or unrecognized impact sorts last. +export function impactRank(impact) { + return { critical: 0, serious: 1, moderate: 2, minor: 3 }[impact] ?? 4; +} + +// Rank a violation's conformance standard for sorting: WCAG A=0, AA=1, AAA=2, +// Best Practice=3, obsolete criteria=4, no conformance tags=5. Violations of a +// current WCAG requirement outrank recommendations and withdrawn criteria. +export function standardRank(tags) { + const parsed = parseConformanceTags(tags); + if (!parsed) return 5; + if (parsed.bestPractice) return 3; + if (parsed.obsolete) return 4; + return parsed.level.length - 1; +} + +// Order report violations most-important-first: by impact, then by +// conformance standard, then by rule id for deterministic output. +export function compareViolations(a, b) { + return ( + impactRank(a.impact) - impactRank(b.impact) || + standardRank(a.tags) - standardRank(b.tags) || + a.id.localeCompare(b.id) + ); } class QuartoAxeReporter { @@ -151,7 +189,9 @@ class QuartoAxeDocumentReporter extends QuartoAxeReporter { } createReportElement() { - const violations = this.axeResult.violations; + // Sort a copy so this.axeResult stays untouched: the json reporter (and + // anything else reading axeResult) must see raw axe-core output. + const violations = [...this.axeResult.violations].sort(compareViolations); const reportElement = document.createElement("div"); reportElement.className = "quarto-axe-report"; if (violations.length === 0) { diff --git a/tests/unit/axe-sort.test.ts b/tests/unit/axe-sort.test.ts new file mode 100644 index 0000000000..d20ce945f3 --- /dev/null +++ b/tests/unit/axe-sort.test.ts @@ -0,0 +1,143 @@ +/* + * axe-sort.test.ts + * + * Tests the sort order our axe document reporter applies to violations: + * impact (critical → serious → moderate → minor → null), then conformance + * standard (WCAG A → AA → AAA → Best Practice → obsolete → none), then rule + * id for deterministic output. + * + * These exercise our own ranking and comparison logic, not axe-core's rule + * data: every input is a hand-written violation fixture (the contract axe + * hands us). An axe-core upgrade that re-tags or re-ranks a rule therefore + * cannot break these tests. + * + * Copyright (C) 2020-2025 Posit Software, PBC + */ + +import { unitTest } from "../test.ts"; +import { assertEquals } from "testing/asserts"; +import { + compareViolations, + impactRank, + standardRank, +} from "../../src/resources/formats/html/axe/axe-check.js"; + +unitTest( + "impactRank - orders critical → serious → moderate → minor → null", + // deno-lint-ignore require-await + async () => { + assertEquals(impactRank("critical"), 0); + assertEquals(impactRank("serious"), 1); + assertEquals(impactRank("moderate"), 2); + assertEquals(impactRank("minor"), 3); + assertEquals(impactRank(null), 4); + assertEquals(impactRank("unheard-of"), 4); + }, +); + +unitTest( + "standardRank - orders A → AA → AAA → Best Practice → obsolete → none", + // deno-lint-ignore require-await + async () => { + assertEquals(standardRank(["wcag2a", "wcag111"]), 0); + assertEquals(standardRank(["wcag21aa", "wcag143"]), 1); + assertEquals(standardRank(["wcag2aaa"]), 2); + assertEquals(standardRank(["best-practice", "cat.color"]), 3); + assertEquals(standardRank(["wcag2a-obsolete", "wcag411"]), 4); + assertEquals(standardRank(["cat.color"]), 5); + }, +); + +unitTest( + "standardRank - best-practice takes precedence over any WCAG tags", + // deno-lint-ignore require-await + async () => { + // Mirrors axeConformanceLevel: axe tags best-practice rules with related + // WCAG criteria, but they're still recommendations, not requirements. + assertEquals(standardRank(["best-practice", "wcag2a", "wcag111"]), 3); + }, +); + +unitTest( + "compareViolations - impact ordering, null impact last", + // deno-lint-ignore require-await + async () => { + const violations = [ + { id: "r-minor", impact: "minor", tags: ["wcag2a"] }, + { id: "r-null", impact: null, tags: ["wcag2a"] }, + { id: "r-critical", impact: "critical", tags: ["wcag2a"] }, + { id: "r-moderate", impact: "moderate", tags: ["wcag2a"] }, + { id: "r-serious", impact: "serious", tags: ["wcag2a"] }, + ]; + assertEquals( + violations.sort(compareViolations).map((v) => v.id), + ["r-critical", "r-serious", "r-moderate", "r-minor", "r-null"], + ); + }, +); + +unitTest( + "compareViolations - standard breaks ties within an impact", + // deno-lint-ignore require-await + async () => { + const violations = [ + { id: "r-none", impact: "serious", tags: ["cat.color"] }, + { id: "r-obsolete", impact: "serious", tags: ["wcag2a-obsolete"] }, + { id: "r-bp", impact: "serious", tags: ["best-practice"] }, + { id: "r-aa", impact: "serious", tags: ["wcag21aa", "wcag143"] }, + { id: "r-a", impact: "serious", tags: ["wcag2a", "wcag111"] }, + ]; + assertEquals( + violations.sort(compareViolations).map((v) => v.id), + ["r-a", "r-aa", "r-bp", "r-obsolete", "r-none"], + ); + }, +); + +unitTest( + "compareViolations - impact outranks standard", + // deno-lint-ignore require-await + async () => { + // A critical best-practice violation precedes a minor WCAG A one. + const violations = [ + { id: "r-minor-a", impact: "minor", tags: ["wcag2a"] }, + { id: "r-critical-bp", impact: "critical", tags: ["best-practice"] }, + ]; + assertEquals( + violations.sort(compareViolations).map((v) => v.id), + ["r-critical-bp", "r-minor-a"], + ); + }, +); + +unitTest( + "compareViolations - rule id breaks full ties deterministically", + // deno-lint-ignore require-await + async () => { + const violations = [ + { id: "image-alt", impact: "critical", tags: ["wcag2a"] }, + { id: "area-alt", impact: "critical", tags: ["wcag2a"] }, + { id: "button-name", impact: "critical", tags: ["wcag2a"] }, + ]; + assertEquals( + violations.sort(compareViolations).map((v) => v.id), + ["area-alt", "button-name", "image-alt"], + ); + }, +); + +unitTest( + "compareViolations - sorting a copy leaves the input array untouched", + // deno-lint-ignore require-await + async () => { + // The document reporter sorts a copy so axeResult keeps axe-core's raw + // order for the json reporter. + const violations = [ + { id: "r-minor", impact: "minor", tags: ["wcag2a"] }, + { id: "r-critical", impact: "critical", tags: ["wcag2a"] }, + ]; + const sorted = [...violations].sort(compareViolations); + assertEquals(sorted.map((v) => v.id), ["r-critical", "r-minor"]); + assertEquals(violations.map((v) => v.id), ["r-minor", "r-critical"]); + }, +); From a7694b33fd55c8435cb7eba159afda64e9a99aad Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Mon, 13 Jul 2026 09:12:40 -0700 Subject: [PATCH 2/3] Changelog entry for #14676 --- news/changelog-1.10.md | 1 + 1 file changed, 1 insertion(+) diff --git a/news/changelog-1.10.md b/news/changelog-1.10.md index 98675fbafd..2bb442c9b2 100644 --- a/news/changelog-1.10.md +++ b/news/changelog-1.10.md @@ -19,6 +19,7 @@ All changes included in 1.10: - ([#14602](https://github.com/quarto-dev/quarto-cli/pull/14602), [#14632](https://github.com/quarto-dev/quarto-cli/pull/14632)): Fix ORCID profile link having no accessible name for screen readers in HTML, Reveal.js, and ipynb title-block author metadata. (author: @mcanouil for #14602) - ([#14604](https://github.com/quarto-dev/quarto-cli/issues/14604)): The `axe` accessibility report UI now shows each violation's WCAG conformance level (e.g. `WCAG 2.0 AA (1.4.3)`) or `Best Practice`, derived from the violation's axe-core tags. - ([#14655](https://github.com/quarto-dev/quarto-cli/issues/14655)): Add accessible names to code line-number links so screen readers and accessibility audits no longer report them as empty links. +- ([#14676](https://github.com/quarto-dev/quarto-cli/pull/14676)): The `axe` accessibility report UI now lists violations most-important-first: by impact (critical first), then WCAG conformance level (A before AA, current criteria before Best Practice and obsolete ones), then rule id. `output: json` still emits axe-core's raw, unsorted result. - ([#14677](https://github.com/quarto-dev/quarto-cli/pull/14677)): The `axe` option now uses a copy of axe-core bundled with Quarto instead of loading it from the Skypack CDN in the reader's browser. Accessibility checking now works offline, and viewing a rendered document no longer triggers a request to `cdn.skypack.dev`. The axe-core version is unchanged (4.10.3), so scan results are identical. ## Formats From c6c63bd899d8df2a13db2ebc4bd5d8f48e33e203 Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Mon, 13 Jul 2026 09:39:21 -0700 Subject: [PATCH 3/3] Add Playwright test for axe report severity ordering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fixture triggers image-alt (critical), color-contrast (serious), and heading-order (moderate); axe-core emits these roughly alphabetically by rule id, which is not severity order, so the test fails if the report stops sorting. Standard tie-breaks within one impact level are covered by the unit tests instead — triggering two same-impact rules with different standards is fragile against axe-core upgrades. --- tests/docs/playwright/html/axe-sort-order.qmd | 19 ++++++++++++++ .../tests/axe-accessibility.spec.ts | 26 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 tests/docs/playwright/html/axe-sort-order.qmd diff --git a/tests/docs/playwright/html/axe-sort-order.qmd b/tests/docs/playwright/html/axe-sort-order.qmd new file mode 100644 index 0000000000..7eef26d275 --- /dev/null +++ b/tests/docs/playwright/html/axe-sort-order.qmd @@ -0,0 +1,19 @@ +--- +format: + html: + axe: + output: document +--- + +## Axe report sort order + +Three violations with distinct impacts, chosen so axe-core's own order +(roughly alphabetical by rule id: color-contrast, heading-order, image-alt) +differs from the report's severity order (image-alt, color-contrast, +heading-order). + +This text violates contrast rules: [insufficient contrast.]{style="color: #eee; background: #fff;"} (serious, WCAG 2.0 AA) + + + +#### This heading skips from h2 to h4 (moderate, Best Practice) diff --git a/tests/integration/playwright/tests/axe-accessibility.spec.ts b/tests/integration/playwright/tests/axe-accessibility.spec.ts index 4b83c3165f..88039991d7 100644 --- a/tests/integration/playwright/tests/axe-accessibility.spec.ts +++ b/tests/integration/playwright/tests/axe-accessibility.spec.ts @@ -487,6 +487,32 @@ test.describe('Dashboard axe — re-scan on visibility change', () => { }); }); +test.describe('Axe report — violations sorted by severity (#14676)', () => { + test('html — report lists critical before serious before moderate', async ({ page }) => { + // Fixture triggers image-alt (critical), color-contrast (serious), and + // heading-order (moderate). axe-core emits these roughly alphabetically by + // rule id (color-contrast, heading-order, image-alt), which is not + // severity order, so this fails if the report stops sorting. + await page.goto('/html/axe-sort-order.html', { waitUntil: 'networkidle' }); + + const axeReport = page.locator('.quarto-axe-report'); + await expect(axeReport).toBeVisible({ timeout: 10000 }); + + const descriptions = await axeReport + .locator('.quarto-axe-violation-description').allTextContents(); + const indexOf = (impact: string) => + descriptions.findIndex(d => d.startsWith(impact)); + + for (const impact of ['Critical', 'Serious', 'Moderate']) { + expect(indexOf(impact), + `Expected a ${impact} violation in: ${descriptions.join(' | ')}`) + .toBeGreaterThanOrEqual(0); + } + expect(indexOf('Critical')).toBeLessThan(indexOf('Serious')); + expect(indexOf('Serious')).toBeLessThan(indexOf('Moderate')); + }); +}); + test.describe('Axe — code line-number anchors have accessible names (#14655)', () => { test('html — numbered code block has no axe violations', async ({ page }) => { await page.goto('/html/axe-code-line-numbers.html', { waitUntil: 'networkidle' });