From 67cab6c47f68c86d51b407d03a2ef73d6d637673 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 1 May 2026 12:31:22 +0100 Subject: [PATCH] fix(cli,core): stop dev workers spinning at 100% CPU after parent CLI disconnect Orphaned trigger-dev-run-worker and trigger-dev-index-worker processes were getting stuck in an uncaughtException feedback loop when the parent CLI closed the IPC channel: a periodic IPC send via process.send would throw ERR_IPC_CHANNEL_CLOSED, which re-entered the same handler that itself called process.send. The loop was amplified by source-map-support's prepareStackTrace running every iteration. - @trigger.dev/core: ZodIpcConnection drops packets when process.connected is false and swallows synchronous send errors, so closed-channel sends no longer throw out of the IPC layer. - trigger.dev (cli-v3): dev-run-worker and dev-index-worker now exit cleanly via process.on("disconnect") instead of being re-parented to init. - trigger.dev (cli-v3): all four worker entry points wrap their uncaughtException process.send calls in safeSend, which checks process.connected and swallows synchronous throws so a closed channel can never re-enter the handler. --- .changeset/dev-worker-disconnect-loop.md | 6 ++ .../src/entryPoints/dev-index-worker.ts | 57 +++++++++++------ .../cli-v3/src/entryPoints/dev-run-worker.ts | 64 ++++++++++++------- .../src/entryPoints/managed-index-worker.ts | 53 ++++++++------- .../src/entryPoints/managed-run-worker.ts | 57 ++++++++++------- packages/core/src/v3/zodIpc.ts | 15 ++++- 6 files changed, 160 insertions(+), 92 deletions(-) create mode 100644 .changeset/dev-worker-disconnect-loop.md diff --git a/.changeset/dev-worker-disconnect-loop.md b/.changeset/dev-worker-disconnect-loop.md new file mode 100644 index 00000000000..cf5afbb2135 --- /dev/null +++ b/.changeset/dev-worker-disconnect-loop.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"trigger.dev": patch +--- + +Fix dev workers spinning at 100% CPU after the parent CLI disconnects. Orphaned `trigger-dev-run-worker` (and indexer) processes were caught in an `uncaughtException` feedback loop: a periodic IPC send via `process.send` would throw `ERR_IPC_CHANNEL_CLOSED` once the parent closed the channel, which re-entered the same handler that itself called `process.send`, scheduled via `setImmediate` and amplified by source-map-support's `prepareStackTrace`. Fixed by (1) silently dropping packets in `ZodIpcConnection` when the channel is disconnected, (2) adding a `process.on("disconnect", ...)` handler in dev workers so they exit cleanly when the CLI closes the IPC channel, and (3) wrapping all `uncaughtException`-path `process.send` calls in a `safeSend` guard that checks `process.connected` and swallows synchronous throws. diff --git a/packages/cli-v3/src/entryPoints/dev-index-worker.ts b/packages/cli-v3/src/entryPoints/dev-index-worker.ts index 0b8ee54b0a1..53b95ad040a 100644 --- a/packages/cli-v3/src/entryPoints/dev-index-worker.ts +++ b/packages/cli-v3/src/entryPoints/dev-index-worker.ts @@ -27,30 +27,45 @@ sourceMapSupport.install({ hookRequire: false, }); +// If the parent CLI closes the IPC channel, exit cleanly instead of being +// re-parented to init and busy-looping on `process.send` against a dead channel. +process.on("disconnect", () => { + process.exit(0); +}); + +function safeSend(message: unknown) { + if (!process.connected || !process.send) { + return; + } + try { + process.send(message); + } catch { + // swallow: a throw here would re-enter this handler and busy-loop the worker + } +} + process.on("uncaughtException", function (error, origin) { if (error instanceof Error) { - process.send && - process.send({ - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { name: error.name, message: error.message, stack: error.stack }, - origin, - }, - version: "v1", - }); + safeSend({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { name: error.name, message: error.message, stack: error.stack }, + origin, + }, + version: "v1", + }); } else { - process.send && - process.send({ - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { - name: "Error", - message: typeof error === "string" ? error : JSON.stringify(error), - }, - origin, + safeSend({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { + name: "Error", + message: typeof error === "string" ? error : JSON.stringify(error), }, - version: "v1", - }); + origin, + }, + version: "v1", + }); } }); @@ -183,7 +198,7 @@ await sendMessageInCatalog( importErrors, }, async (msg) => { - process.send?.(msg); + safeSend(msg); } ).catch((err) => { if (err instanceof ZodSchemaParsedError) { diff --git a/packages/cli-v3/src/entryPoints/dev-run-worker.ts b/packages/cli-v3/src/entryPoints/dev-run-worker.ts index ae9a2667f51..6ffc6afd29b 100644 --- a/packages/cli-v3/src/entryPoints/dev-run-worker.ts +++ b/packages/cli-v3/src/entryPoints/dev-run-worker.ts @@ -77,37 +77,53 @@ sourceMapSupport.install({ hookRequire: false, }); +// If the parent CLI closes the IPC channel (process restart, crash, lost +// handle), exit cleanly instead of being re-parented to init and busy-looping +// on `process.send` that throws against a dead channel. +process.on("disconnect", () => { + process.exit(0); +}); + +function safeSend(message: unknown) { + if (!process.connected || !process.send) { + return; + } + try { + process.send(message); + } catch { + // swallow: a throw here would re-enter this handler and busy-loop the worker + } +} + process.on("uncaughtException", function (error, origin) { logError("Uncaught exception", { error, origin }); if (error instanceof Error) { - process.send && - process.send({ - type: "EVENT", - message: { - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { name: error.name, message: error.message, stack: error.stack }, - origin, - }, - version: "v1", + safeSend({ + type: "EVENT", + message: { + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { name: error.name, message: error.message, stack: error.stack }, + origin, }, - }); + version: "v1", + }, + }); } else { - process.send && - process.send({ - type: "EVENT", - message: { - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { - name: "Error", - message: typeof error === "string" ? error : JSON.stringify(error), - }, - origin, + safeSend({ + type: "EVENT", + message: { + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { + name: "Error", + message: typeof error === "string" ? error : JSON.stringify(error), }, - version: "v1", + origin, }, - }); + version: "v1", + }, + }); } }); diff --git a/packages/cli-v3/src/entryPoints/managed-index-worker.ts b/packages/cli-v3/src/entryPoints/managed-index-worker.ts index efc226b99e0..644673537e3 100644 --- a/packages/cli-v3/src/entryPoints/managed-index-worker.ts +++ b/packages/cli-v3/src/entryPoints/managed-index-worker.ts @@ -27,30 +27,39 @@ sourceMapSupport.install({ hookRequire: false, }); +function safeSend(message: unknown) { + if (!process.connected || !process.send) { + return; + } + try { + process.send(message); + } catch { + // swallow: a throw here would re-enter this handler and busy-loop the worker + } +} + process.on("uncaughtException", function (error, origin) { if (error instanceof Error) { - process.send && - process.send({ - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { name: error.name, message: error.message, stack: error.stack }, - origin, - }, - version: "v1", - }); + safeSend({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { name: error.name, message: error.message, stack: error.stack }, + origin, + }, + version: "v1", + }); } else { - process.send && - process.send({ - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { - name: "Error", - message: typeof error === "string" ? error : JSON.stringify(error), - }, - origin, + safeSend({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { + name: "Error", + message: typeof error === "string" ? error : JSON.stringify(error), }, - version: "v1", - }); + origin, + }, + version: "v1", + }); } }); @@ -191,7 +200,7 @@ await sendMessageInCatalog( importErrors, }, async (msg) => { - process.send?.(msg); + safeSend(msg); } ).catch((err) => { if (err instanceof ZodSchemaParsedError) { @@ -200,7 +209,7 @@ await sendMessageInCatalog( "TASKS_FAILED_TO_PARSE", { zodIssues: err.error.issues, tasks }, async (msg) => { - process.send?.(msg); + safeSend(msg); } ); } else { diff --git a/packages/cli-v3/src/entryPoints/managed-run-worker.ts b/packages/cli-v3/src/entryPoints/managed-run-worker.ts index f8234aa68c4..1d5d2a5f0d6 100644 --- a/packages/cli-v3/src/entryPoints/managed-run-worker.ts +++ b/packages/cli-v3/src/entryPoints/managed-run-worker.ts @@ -77,37 +77,46 @@ sourceMapSupport.install({ hookRequire: false, }); +function safeSend(message: unknown) { + if (!process.connected || !process.send) { + return; + } + try { + process.send(message); + } catch { + // swallow: a throw here would re-enter this handler and busy-loop the worker + } +} + process.on("uncaughtException", function (error, origin) { console.error("Uncaught exception", { error, origin }); if (error instanceof Error) { - process.send && - process.send({ - type: "EVENT", - message: { - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { name: error.name, message: error.message, stack: error.stack }, - origin, - }, - version: "v1", + safeSend({ + type: "EVENT", + message: { + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { name: error.name, message: error.message, stack: error.stack }, + origin, }, - }); + version: "v1", + }, + }); } else { - process.send && - process.send({ - type: "EVENT", - message: { - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { - name: "Error", - message: typeof error === "string" ? error : JSON.stringify(error), - }, - origin, + safeSend({ + type: "EVENT", + message: { + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { + name: "Error", + message: typeof error === "string" ? error : JSON.stringify(error), }, - version: "v1", + origin, }, - }); + version: "v1", + }, + }); } }); diff --git a/packages/core/src/v3/zodIpc.ts b/packages/core/src/v3/zodIpc.ts index 8879c7d0538..f42de5a9ddf 100644 --- a/packages/core/src/v3/zodIpc.ts +++ b/packages/core/src/v3/zodIpc.ts @@ -143,6 +143,7 @@ interface ZodIpcConnectionOptions< process: { send?: (message: any) => any; on?: (event: "message", listener: (message: any) => void) => void; + connected?: boolean; }; handlers?: ZodIpcMessageHandlers; } @@ -257,7 +258,19 @@ export class ZodIpcConnection< } async #sendPacket(packet: Packet) { - await this.opts.process.send?.(packet); + // When the IPC channel is closed (e.g. parent process exited), there is no + // recipient — drop the packet rather than letting `process.send` throw + // ERR_IPC_CHANNEL_CLOSED, which would otherwise propagate as an + // uncaughtException and re-enter any handler that itself calls `process.send`. + if (this.opts.process.connected === false) { + return; + } + + try { + await this.opts.process.send?.(packet); + } catch { + // swallow: channel raced from open to closed between the check and the send + } } async send>(