Skip to content

fix: make Gemini Interactions drift tests resilient to API unavailability #151

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jpr5 merged 1 commit into from
May 3, 2026
Merged

fix: make Gemini Interactions drift tests resilient to API unavailability #151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 50 additions & 4 deletions scripts/drift-report-collector.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,11 +372,57 @@ function collectDriftEntries(results: VitestJsonResult): DriftEntry[] {
}

if (unparseable > 0 && entries.length === 0) {
console.error(
`ERROR: ${unparseable} test failure(s) could not be parsed as drift reports.`,
"This may indicate broken test infrastructure or a changed report format.",
// Collect the unparseable failure messages to classify them
const unparseableMessages: string[] = [];
for (const file of results.testResults) {
for (const assertion of file.assertionResults) {
if (assertion.status !== "failed" || assertion.failureMessages.length === 0) continue;
const fullMessage = assertion.failureMessages.join("\n");
const parsed = parseDriftBlock(fullMessage);
if (!parsed || parsed.diffs.length === 0) {
unparseableMessages.push(fullMessage);
}
}
}

// Distinguish infrastructure errors from broken drift report formats
const infraIndicators = [
/API returned \d{3}/i,
/status \d{3}/i,
/<!DOCTYPE/i,
/<html/i,
/failed to parse JSON/i,
/empty response/i,
/fetch failed/i,
/ECONNREFUSED/i,
/ETIMEDOUT/i,
/ENOTFOUND/i,
/network error/i,
/API unavailable/i,
];
const driftLikeIndicators = [/drift/i, /mismatch/i, /expected.*but/i, /LLMOCK DRIFT/i];

const allInfraErrors = unparseableMessages.every((msg) =>
infraIndicators.some((re) => re.test(msg)),
);
const anyDriftLike = unparseableMessages.some((msg) =>
driftLikeIndicators.some((re) => re.test(msg)),
);
throw new Error(`${unparseable} unparseable test failures with 0 drift entries — investigate`);

if (allInfraErrors && !anyDriftLike) {
console.warn(
`WARNING: ${unparseable} test failure(s) appear to be API/infrastructure errors ` +
`(not drift reports). Continuing with 0 drift entries.`,
);
} else {
console.error(
`ERROR: ${unparseable} test failure(s) could not be parsed as drift reports.`,
"This may indicate broken test infrastructure or a changed report format.",
);
throw new Error(
`${unparseable} unparseable test failures with 0 drift entries — investigate`,
);
}
} else if (unparseable > 0) {
console.warn(
`WARNING: ${unparseable} test failure(s) did not contain parseable drift data (${entries.length} drift entries collected).`,
Expand Down
100 changes: 66 additions & 34 deletions src/__tests__/drift/gemini-interactions.drift.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,22 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Interactions API drift", () => {
it("non-streaming text shape matches", async () => {
const sdkShape = geminiInteractionsResponseShape();

const [realRes, mockRes] = await Promise.all([
geminiInteractionsNonStreaming(config, "Say hello"),
httpPost(`${instance.url}/v1beta/interactions`, {
model: "gemini-2.5-flash",
input: "Say hello",
stream: false,
}),
]);
let realRes;
try {
realRes = await geminiInteractionsNonStreaming(config, "Say hello");
} catch (err) {
console.warn(
"Gemini Interactions API unavailable:",
err instanceof Error ? err.message : String(err),
);
return;
}

const mockRes = await httpPost(`${instance.url}/v1beta/interactions`, {
model: "gemini-2.5-flash",
input: "Say hello",
stream: false,
});

const realShape = extractShape(realRes.body);
const mockShape = extractShape(JSON.parse(mockRes.body));
Expand All @@ -73,17 +81,25 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Interactions API drift", () => {
it("streaming text event sequence and shapes match", async () => {
const sdkEvents = geminiInteractionsStreamEventShapes();

const [realStream, mockStreamRes] = await Promise.all([
geminiInteractionsStreaming(config, "Say hello"),
httpPost(`${instance.url}/v1beta/interactions`, {
model: "gemini-2.5-flash",
input: "Say hello",
stream: true,
}),
]);
let realStream;
try {
realStream = await geminiInteractionsStreaming(config, "Say hello");
} catch (err) {
console.warn(
"Gemini Interactions API unavailable:",
err instanceof Error ? err.message : String(err),
);
return;
}

expect(realStream.rawEvents.length, "Real API returned no SSE events").toBeGreaterThan(0);

const mockStreamRes = await httpPost(`${instance.url}/v1beta/interactions`, {
model: "gemini-2.5-flash",
input: "Say hello",
stream: true,
});

const mockEvents = parseInteractionsSSE(mockStreamRes.body);
expect(mockEvents.length, "Mock returned no SSE events").toBeGreaterThan(0);

Expand Down Expand Up @@ -116,15 +132,23 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Interactions API drift", () => {
},
];

const [realRes, mockRes] = await Promise.all([
geminiInteractionsNonStreaming(config, "Weather in Paris", tools),
httpPost(`${instance.url}/v1beta/interactions`, {
model: "gemini-2.5-flash",
input: "Weather in Paris",
stream: false,
tools,
}),
]);
let realRes;
try {
realRes = await geminiInteractionsNonStreaming(config, "Weather in Paris", tools);
} catch (err) {
console.warn(
"Gemini Interactions API unavailable:",
err instanceof Error ? err.message : String(err),
);
return;
}

const mockRes = await httpPost(`${instance.url}/v1beta/interactions`, {
model: "gemini-2.5-flash",
input: "Weather in Paris",
stream: false,
tools,
});

const realShape = extractShape(realRes.body);
const mockShape = extractShape(JSON.parse(mockRes.body));
Expand Down Expand Up @@ -153,18 +177,26 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Interactions API drift", () => {
},
];

const [realStream, mockStreamRes] = await Promise.all([
geminiInteractionsStreaming(config, "Weather in Paris", tools),
httpPost(`${instance.url}/v1beta/interactions`, {
model: "gemini-2.5-flash",
input: "Weather in Paris",
stream: true,
tools,
}),
]);
let realStream;
try {
realStream = await geminiInteractionsStreaming(config, "Weather in Paris", tools);
} catch (err) {
console.warn(
"Gemini Interactions API unavailable:",
err instanceof Error ? err.message : String(err),
);
return;
}

expect(realStream.rawEvents.length, "Real API returned no SSE events").toBeGreaterThan(0);

const mockStreamRes = await httpPost(`${instance.url}/v1beta/interactions`, {
model: "gemini-2.5-flash",
input: "Weather in Paris",
stream: true,
tools,
});

const mockEvents = parseInteractionsSSE(mockStreamRes.body);
expect(mockEvents.length, "Mock returned no SSE events").toBeGreaterThan(0);

Expand Down
Loading