Sort axe report violations by severity, then standard

news/changelog-1.10.md

@@ -19,6 +19,7 @@ All changes included in 1.10:
 - ([#14602](https://github.com/quarto-dev/quarto-cli/pull/14602), [#14632](https://github.com/quarto-dev/quarto-cli/pull/14632)): Fix ORCID profile link having no accessible name for screen readers in HTML, Reveal.js, and ipynb title-block author metadata. (author: @mcanouil for #14602)
 - ([#14604](https://github.com/quarto-dev/quarto-cli/issues/14604)): The `axe` accessibility report UI now shows each violation's WCAG conformance level (e.g. `WCAG 2.0 AA (1.4.3)`) or `Best Practice`, derived from the violation's axe-core tags.
 - ([#14655](https://github.com/quarto-dev/quarto-cli/issues/14655)): Add accessible names to code line-number links so screen readers and accessibility audits no longer report them as empty links.
+- ([#14676](https://github.com/quarto-dev/quarto-cli/pull/14676)): The `axe` accessibility report UI now lists violations most-important-first: by impact (critical first), then WCAG conformance level (A before AA, current criteria before Best Practice and obsolete ones), then rule id. `output: json` still emits axe-core's raw, unsorted result.
 - ([#14677](https://github.com/quarto-dev/quarto-cli/pull/14677)): The `axe` option now uses a copy of axe-core bundled with Quarto instead of loading it from the Skypack CDN in the reader's browser. Accessibility checking now works offline, and viewing a rendered document no longer triggers a request to `cdn.skypack.dev`. The axe-core version is unchanged (4.10.3), so scan results are identical.
 
 ## Formats

src/resources/formats/html/axe/axe-check.js

@@ -1,24 +1,35 @@
-// Derive a human-readable WCAG conformance label from axe-core's `tags` array.
-// Tags encode the version+level (`wcag2a`, `wcag21aa`), the specific success
-// criteria (`wcag111` → 1.1.1), and `best-practice` for axe's own
-// recommendations that aren't tied to any WCAG success criterion. Returns "" when
-// no conformance tags are present so callers can fall back to the impact alone.
-export function axeConformanceLevel(tags) {
-  if (tags.includes("best-practice")) return "Best Practice";
-
-  // Version+level: wcag2a, wcag2aa, wcag21aa, wcag22aa, ... An `-obsolete`
-  // suffix (e.g. `wcag2a-obsolete` on the deprecated `duplicate-id` rule) marks
-  // a criterion that was withdrawn from later WCAG versions, e.g. SC 4.1.1,
-  // removed in WCAG 2.2. We surface the original level but flag it as obsolete
-  // so a withdrawn criterion isn't mistaken for a current conformance failure.
+// Parse the conformance-related tags from axe-core's `tags` array. Tags encode
+// the version+level (`wcag2a`, `wcag21aa`), and `best-practice` for axe's own
+// recommendations that aren't tied to any WCAG success criterion. An
+// `-obsolete` suffix (e.g. `wcag2a-obsolete` on the deprecated `duplicate-id`
+// rule) marks a criterion that was withdrawn from later WCAG versions, e.g.
+// SC 4.1.1, removed in WCAG 2.2. Returns:
+//   { bestPractice: true }            for best-practice rules
+//   { major, minor, level, obsolete } for WCAG version+level tags (level is
+//                                     the raw "a"/"aa"/"aaa" string)
+//   null                              when no conformance tags are present
+function parseConformanceTags(tags) {
+  if (tags.includes("best-practice")) return { bestPractice: true };
   const versionTag = tags.find((t) => /^wcag\d+a+(-obsolete)?$/.test(t));
-  // Without a version+level tag there's no conformance level to report, so fall
-  // back to the impact alone rather than emitting a bare, level-less criterion.
-  if (!versionTag) return "";
-
+  if (!versionTag) return null;
   const [, major, minor, level, obsolete] =
     versionTag.match(/^wcag(\d)(\d?)(a+)(-obsolete)?$/);
-  let label = `WCAG ${major}.${minor || "0"} ${level.toUpperCase()}`;
+  return { major, minor, level, obsolete: !!obsolete };
+}
+
+// Derive a human-readable WCAG conformance label from axe-core's `tags` array,
+// including the specific success criteria (`wcag111` → 1.1.1). Returns "" when
+// no conformance tags are present so callers can fall back to the impact alone.
+export function axeConformanceLevel(tags) {
+  const parsed = parseConformanceTags(tags);
+  // Without conformance tags there's no level to report, so fall back to the
+  // impact alone rather than emitting a bare, level-less criterion.
+  if (!parsed) return "";
+  if (parsed.bestPractice) return "Best Practice";
+
+  // We surface an obsolete criterion's original level but flag it as obsolete
+  // so it isn't mistaken for a current conformance failure.
+  let label = `WCAG ${parsed.major}.${parsed.minor || "0"} ${parsed.level.toUpperCase()}`;
 
   // Success criteria: wcag111 → 1.1.1, wcag1410 → 1.4.10. Principle and
   // guideline are always single digits; the remainder is the criterion number.
@@ -32,7 +43,34 @@ export function axeConformanceLevel(tags) {
   if (criteria.length) {
     label += ` (${criteria.join(", ")})`;
   }
-  return obsolete ? `Obsolete ${label}` : label;
+  return parsed.obsolete ? `Obsolete ${label}` : label;
+}
+
+// Rank a violation's axe-core `impact` for sorting: critical=0, serious=1,
+// moderate=2, minor=3. A null or unrecognized impact sorts last.
+export function impactRank(impact) {
+  return { critical: 0, serious: 1, moderate: 2, minor: 3 }[impact] ?? 4;
+}
+
+// Rank a violation's conformance standard for sorting: WCAG A=0, AA=1, AAA=2,
+// Best Practice=3, obsolete criteria=4, no conformance tags=5. Violations of a
+// current WCAG requirement outrank recommendations and withdrawn criteria.
+export function standardRank(tags) {
+  const parsed = parseConformanceTags(tags);
+  if (!parsed) return 5;
+  if (parsed.bestPractice) return 3;
+  if (parsed.obsolete) return 4;
+  return parsed.level.length - 1;
+}
+
+// Order report violations most-important-first: by impact, then by
+// conformance standard, then by rule id for deterministic output.
+export function compareViolations(a, b) {
+  return (
+    impactRank(a.impact) - impactRank(b.impact) ||
+    standardRank(a.tags) - standardRank(b.tags) ||
+    a.id.localeCompare(b.id)
+  );
 }
 
 class QuartoAxeReporter {
@@ -151,7 +189,9 @@ class QuartoAxeDocumentReporter extends QuartoAxeReporter {
   }
 
   createReportElement() {
-    const violations = this.axeResult.violations;
+    // Sort a copy so this.axeResult stays untouched: the json reporter (and
+    // anything else reading axeResult) must see raw axe-core output.
+    const violations = [...this.axeResult.violations].sort(compareViolations);
     const reportElement = document.createElement("div");
     reportElement.className = "quarto-axe-report";
     if (violations.length === 0) {

tests/docs/playwright/html/axe-sort-order.qmd

@@ -0,0 +1,19 @@
+---
+format:
+  html:
+    axe:
+      output: document
+---
+
+## Axe report sort order
+
+Three violations with distinct impacts, chosen so axe-core's own order
+(roughly alphabetical by rule id: color-contrast, heading-order, image-alt)
+differs from the report's severity order (image-alt, color-contrast,
+heading-order).
+
+This text violates contrast rules: [insufficient contrast.]{style="color: #eee; background: #fff;"} (serious, WCAG 2.0 AA)
+
+<img src="placeholder.png" class="axe-test-img">
+
+#### This heading skips from h2 to h4 (moderate, Best Practice)

tests/integration/playwright/tests/axe-accessibility.spec.ts

@@ -487,6 +487,32 @@ test.describe('Dashboard axe — re-scan on visibility change', () => {
   });
 });
 
+test.describe('Axe report — violations sorted by severity (#14676)', () => {
+  test('html — report lists critical before serious before moderate', async ({ page }) => {
+    // Fixture triggers image-alt (critical), color-contrast (serious), and
+    // heading-order (moderate). axe-core emits these roughly alphabetically by
+    // rule id (color-contrast, heading-order, image-alt), which is not
+    // severity order, so this fails if the report stops sorting.
+    await page.goto('/html/axe-sort-order.html', { waitUntil: 'networkidle' });
+
+    const axeReport = page.locator('.quarto-axe-report');
+    await expect(axeReport).toBeVisible({ timeout: 10000 });
+
+    const descriptions = await axeReport
+      .locator('.quarto-axe-violation-description').allTextContents();
+    const indexOf = (impact: string) =>
+      descriptions.findIndex(d => d.startsWith(impact));
+
+    for (const impact of ['Critical', 'Serious', 'Moderate']) {
+      expect(indexOf(impact),
+        `Expected a ${impact} violation in: ${descriptions.join(' | ')}`)
+        .toBeGreaterThanOrEqual(0);
+    }
+    expect(indexOf('Critical')).toBeLessThan(indexOf('Serious'));
+    expect(indexOf('Serious')).toBeLessThan(indexOf('Moderate'));
+  });
+});
+
 test.describe('Axe — code line-number anchors have accessible names (#14655)', () => {
   test('html — numbered code block has no axe violations', async ({ page }) => {
     await page.goto('/html/axe-code-line-numbers.html', { waitUntil: 'networkidle' });

tests/unit/axe-sort.test.ts

@@ -0,0 +1,143 @@
+/*
+ * axe-sort.test.ts
+ *
+ * Tests the sort order our axe document reporter applies to violations:
+ * impact (critical → serious → moderate → minor → null), then conformance
+ * standard (WCAG A → AA → AAA → Best Practice → obsolete → none), then rule
+ * id for deterministic output.
+ *
+ * These exercise our own ranking and comparison logic, not axe-core's rule
+ * data: every input is a hand-written violation fixture (the contract axe
+ * hands us). An axe-core upgrade that re-tags or re-ranks a rule therefore
+ * cannot break these tests.
+ *
+ * Copyright (C) 2020-2025 Posit Software, PBC
+ */
+
+import { unitTest } from "../test.ts";
+import { assertEquals } from "testing/asserts";
+import {
+  compareViolations,
+  impactRank,
+  standardRank,
+} from "../../src/resources/formats/html/axe/axe-check.js";
+
+unitTest(
+  "impactRank - orders critical → serious → moderate → minor → null",
+  // deno-lint-ignore require-await
+  async () => {
+    assertEquals(impactRank("critical"), 0);
+    assertEquals(impactRank("serious"), 1);
+    assertEquals(impactRank("moderate"), 2);
+    assertEquals(impactRank("minor"), 3);
+    assertEquals(impactRank(null), 4);
+    assertEquals(impactRank("unheard-of"), 4);
+  },
+);
+
+unitTest(
+  "standardRank - orders A → AA → AAA → Best Practice → obsolete → none",
+  // deno-lint-ignore require-await
+  async () => {
+    assertEquals(standardRank(["wcag2a", "wcag111"]), 0);
+    assertEquals(standardRank(["wcag21aa", "wcag143"]), 1);
+    assertEquals(standardRank(["wcag2aaa"]), 2);
+    assertEquals(standardRank(["best-practice", "cat.color"]), 3);
+    assertEquals(standardRank(["wcag2a-obsolete", "wcag411"]), 4);
+    assertEquals(standardRank(["cat.color"]), 5);
+  },
+);
+
+unitTest(
+  "standardRank - best-practice takes precedence over any WCAG tags",
+  // deno-lint-ignore require-await
+  async () => {
+    // Mirrors axeConformanceLevel: axe tags best-practice rules with related
+    // WCAG criteria, but they're still recommendations, not requirements.
+    assertEquals(standardRank(["best-practice", "wcag2a", "wcag111"]), 3);
+  },
+);
+
+unitTest(
+  "compareViolations - impact ordering, null impact last",
+  // deno-lint-ignore require-await
+  async () => {
+    const violations = [
+      { id: "r-minor", impact: "minor", tags: ["wcag2a"] },
+      { id: "r-null", impact: null, tags: ["wcag2a"] },
+      { id: "r-critical", impact: "critical", tags: ["wcag2a"] },
+      { id: "r-moderate", impact: "moderate", tags: ["wcag2a"] },
+      { id: "r-serious", impact: "serious", tags: ["wcag2a"] },
+    ];
+    assertEquals(
+      violations.sort(compareViolations).map((v) => v.id),
+      ["r-critical", "r-serious", "r-moderate", "r-minor", "r-null"],
+    );
+  },
+);
+
+unitTest(
+  "compareViolations - standard breaks ties within an impact",
+  // deno-lint-ignore require-await
+  async () => {
+    const violations = [
+      { id: "r-none", impact: "serious", tags: ["cat.color"] },
+      { id: "r-obsolete", impact: "serious", tags: ["wcag2a-obsolete"] },
+      { id: "r-bp", impact: "serious", tags: ["best-practice"] },
+      { id: "r-aa", impact: "serious", tags: ["wcag21aa", "wcag143"] },
+      { id: "r-a", impact: "serious", tags: ["wcag2a", "wcag111"] },
+    ];
+    assertEquals(
+      violations.sort(compareViolations).map((v) => v.id),
+      ["r-a", "r-aa", "r-bp", "r-obsolete", "r-none"],
+    );
+  },
+);
+
+unitTest(
+  "compareViolations - impact outranks standard",
+  // deno-lint-ignore require-await
+  async () => {
+    // A critical best-practice violation precedes a minor WCAG A one.
+    const violations = [
+      { id: "r-minor-a", impact: "minor", tags: ["wcag2a"] },
+      { id: "r-critical-bp", impact: "critical", tags: ["best-practice"] },
+    ];
+    assertEquals(
+      violations.sort(compareViolations).map((v) => v.id),
+      ["r-critical-bp", "r-minor-a"],
+    );
+  },
+);
+
+unitTest(
+  "compareViolations - rule id breaks full ties deterministically",
+  // deno-lint-ignore require-await
+  async () => {
+    const violations = [
+      { id: "image-alt", impact: "critical", tags: ["wcag2a"] },
+      { id: "area-alt", impact: "critical", tags: ["wcag2a"] },
+      { id: "button-name", impact: "critical", tags: ["wcag2a"] },
+    ];
+    assertEquals(
+      violations.sort(compareViolations).map((v) => v.id),
+      ["area-alt", "button-name", "image-alt"],
+    );
+  },
+);
+
+unitTest(
+  "compareViolations - sorting a copy leaves the input array untouched",
+  // deno-lint-ignore require-await
+  async () => {
+    // The document reporter sorts a copy so axeResult keeps axe-core's raw
+    // order for the json reporter.
+    const violations = [
+      { id: "r-minor", impact: "minor", tags: ["wcag2a"] },
+      { id: "r-critical", impact: "critical", tags: ["wcag2a"] },
+    ];
+    const sorted = [...violations].sort(compareViolations);
+    assertEquals(sorted.map((v) => v.id), ["r-critical", "r-minor"]);
+    assertEquals(violations.map((v) => v.id), ["r-minor", "r-critical"]);
+  },
+);