progress

Author: icecrasher321

apps/docs/content/docs/en/blocks/function.mdx

@@ -198,7 +198,7 @@ const file = <readfile.file>;
 const base64 = await sim.files.readBase64(file);
 ```
 
-`sim.files.readBase64(file)`, `sim.files.readText(file)`, `sim.files.readBase64Chunk(file, { offset, length })`, and `sim.files.readTextChunk(file, { offset, length })` read from server-side execution storage under memory caps. `sim.values.read(ref)` can explicitly read a large execution value reference. These helpers are available only in JavaScript functions without imports. JavaScript with imports, Python, and shell do not support these lazy helpers yet.
+`sim.files.readBase64(file)`, `sim.files.readText(file)`, `sim.files.readBase64Chunk(file, { offset, length })`, and `sim.files.readTextChunk(file, { offset, length })` read from server-side execution storage under memory caps. `sim.values.read(ref)` explicitly reads a large execution value reference, and `sim.values.readArray(ref)` reads a manifest-backed large array. These helpers are available only in JavaScript functions without imports. JavaScript with imports, Python, and shell do not support these lazy helpers yet.
 
 Very large full reads can still fail by design; use chunk helpers or return a file when you need to handle more data.
 
@@ -228,7 +228,7 @@ return { name: file.name, chunk: firstMegabyteBase64 };
 
 Chunk `offset` and `length` are byte-based. For Unicode text, a chunk can split a multi-byte character at the boundary; use text chunks for approximate text processing and prefer smaller structured references when exact parsing matters.
 
-Avoid passing a full large object into a Function block when you only need one field. For example, prefer `<api.data.customerId>` over `<api.data>` when the API response is large. If a JavaScript Function without imports references a large execution value, Sim automatically reads it through `sim.values.read(...)` at runtime under memory caps.
+Avoid passing a full large object into a Function block when you only need one field. For example, prefer `<api.data.customerId>` over `<api.data>` when the API response is large. If a JavaScript Function without imports references a whole large execution value, Sim automatically rewrites it to `sim.values.read(...)` at runtime under memory caps. If the value is a manifest-backed array, Sim rewrites it to `sim.values.readArray(...)` so array variables can stay compact between blocks.
 
 For large generated data, write the result to a file or table with `outputPath`, `outputSandboxPath`, or `outputTable` instead of returning the entire payload inline.
 

apps/docs/content/docs/en/execution/api-deployment.mdx

@@ -232,7 +232,7 @@ Workflow execution responses are capped by platform request and response limits.
 }
 ```
 
-The `version` field is part of the external API contract. Treat the reference as an opaque placeholder for a value that could not be safely embedded in the response. `id`, `key`, and `executionId` are not fetch URLs; `key` points to execution-scoped server storage. Use `selectedOutputs` to request a smaller nested field, reduce the data passed between blocks, or return the data from a Response block when your workflow intentionally owns the HTTP response body. File outputs are metadata-first; request `.base64` only when you need inline file content. JavaScript Function blocks can explicitly read large files or value refs with the `sim.files` and `sim.values` helpers under memory caps.
+The `version` field is part of the external API contract. Treat the reference as an opaque placeholder for a value that could not be safely embedded in the response. `id`, `key`, and `executionId` are not fetch URLs; `key` points to execution-scoped server storage. Use `selectedOutputs` to request a smaller nested field, reduce the data passed between blocks, or return the data from a Response block when your workflow intentionally owns the HTTP response body. File outputs are metadata-first; request `.base64` only when you need inline file content. JavaScript Function blocks can explicitly read large files, value refs, and manifest-backed arrays with the `sim.files` and `sim.values` helpers under memory caps.
 
 ### Asynchronous
 

apps/sim/app/api/function/execute/route.test.ts

@@ -245,7 +245,7 @@ describe('Function Execute API Route', () => {
     it('rejects large refs in runtimes without ref-native helpers', async () => {
       featureFlagsMock.isE2bEnabled = true
       const req = createMockRequest('POST', {
-        code: 'echo "${__blockRef_0}"',
+        code: 'echo "$__blockRef_0"',
         language: 'shell',
         contextVariables: {
           __blockRef_0: {
@@ -268,6 +268,46 @@ describe('Function Execute API Route', () => {
         'Large execution values require the JavaScript isolated-vm runtime'
       )
     })
+
+    it('registers manifest array read broker for isolated-vm execution', async () => {
+      const req = createMockRequest('POST', {
+        code: 'return await sim.values.readArray(__blockRef_0)',
+        language: 'javascript',
+        contextVariables: {
+          __blockRef_0: {
+            __simLargeArrayManifest: true,
+            version: 2,
+            kind: 'array',
+            totalCount: 1,
+            chunkCount: 1,
+            byteSize: 16,
+            chunks: [
+              {
+                ref: {
+                  __simLargeValueRef: true,
+                  version: 1,
+                  id: 'lv_ABCDEFGHIJKL',
+                  kind: 'array',
+                  size: 16,
+                  executionId: 'execution-1',
+                },
+                count: 1,
+                byteSize: 16,
+              },
+            ],
+            preview: [{ id: 1 }],
+          },
+        },
+      })
+
+      const response = await POST(req)
+      const data = await response.json()
+      const [, options] = mockExecuteInIsolatedVM.mock.calls.at(-1) ?? []
+
+      expect(response.status).toBe(200)
+      expect(data.success).toBe(true)
+      expect(options?.brokers).toHaveProperty('sim.values.readArray')
+    })
   })
 
   describe('Template Variable Resolution', () => {

apps/sim/app/api/function/execute/route.ts

@@ -14,6 +14,10 @@ import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { executeInE2B, executeShellInE2B } from '@/lib/execution/e2b'
 import { executeInIsolatedVM, type IsolatedVMBrokerHandler } from '@/lib/execution/isolated-vm'
 import { CodeLanguage, DEFAULT_CODE_LANGUAGE, isValidCodeLanguage } from '@/lib/execution/languages'
+import {
+  isLargeArrayManifest,
+  materializeLargeArrayManifest,
+} from '@/lib/execution/payloads/large-array-manifest'
 import { containsLargeValueRef, isLargeValueRef } from '@/lib/execution/payloads/large-value-ref'
 import {
   MAX_FUNCTION_INLINE_BYTES,
@@ -788,6 +792,21 @@ function createFunctionRuntimeBrokers(
       }
       return value
     },
+    'sim.values.readArray': async (args) => {
+      const record = asRecord(args)
+      const options = asRecord(record.options)
+      const manifest = record.ref
+      if (!isLargeArrayManifest(manifest)) {
+        throw new Error('Expected a large array manifest.')
+      }
+      if (!context.executionId) {
+        throw new Error('Large array manifests require an execution context.')
+      }
+      return materializeLargeArrayManifest(manifest, {
+        ...base,
+        maxBytes: clampInlineBytes(options.maxBytes, MAX_INLINE_MATERIALIZATION_BYTES),
+      })
+    },
   }
 }
 

apps/sim/executor/handlers/variables/variables-handler.test.ts

@@ -3,7 +3,7 @@
  */
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache'
-import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref'
+import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest'
 import { BlockType } from '@/executor/constants'
 import { VariablesBlockHandler } from '@/executor/handlers/variables/variables-handler'
 import type { ExecutionContext } from '@/executor/types'
@@ -81,7 +81,7 @@ describe('VariablesBlockHandler', () => {
     expect(mockUploadFile).not.toHaveBeenCalled()
   })
 
-  it('stores oversized assignments as durable refs in variables and block output', async () => {
+  it('stores oversized array assignments as durable manifests in variables and block output', async () => {
     const handler = new VariablesBlockHandler()
     const ctx = createContext()
     const value = Array.from({ length: 120_000 }, (_, index) => ({
@@ -101,13 +101,14 @@ describe('VariablesBlockHandler', () => {
     })
 
     const storedValue = ctx.workflowVariables?.['var-1'].value
-    expect(isLargeValueRef(storedValue)).toBe(true)
+    expect(isLargeArrayManifest(storedValue)).toBe(true)
     expect(output.issues).toBe(storedValue)
     expect(storedValue).toMatchObject({
-      __simLargeValueRef: true,
+      __simLargeArrayManifest: true,
       kind: 'array',
-      executionId: 'execution-1',
+      totalCount: value.length,
     })
+    expect(storedValue.chunkCount).toBeGreaterThan(1)
     expect(mockUploadFile).toHaveBeenCalledWith(
       expect.objectContaining({
         context: 'execution',
@@ -143,6 +144,60 @@ describe('VariablesBlockHandler', () => {
     expect(mockUploadFile).not.toHaveBeenCalled()
   })
 
+  it('preserves whole large refs before scalar type coercion', async () => {
+    const handler = new VariablesBlockHandler()
+    const ref = {
+      __simLargeValueRef: true,
+      version: 1,
+      id: 'lv_ABCDEFGHIJKL',
+      kind: 'object',
+      size: 12 * 1024 * 1024,
+      executionId: 'execution-1',
+    }
+    const ctx = createContext({
+      workflowVariables: {
+        stringVar: { id: 'stringVar', name: 'stringRef', type: 'string', value: '' },
+        plainVar: { id: 'plainVar', name: 'plainRef', type: 'plain', value: '' },
+        numberVar: { id: 'numberVar', name: 'numberRef', type: 'number', value: 0 },
+        booleanVar: { id: 'booleanVar', name: 'booleanRef', type: 'boolean', value: false },
+      },
+    })
+
+    await handler.execute(ctx, createBlock(), {
+      variables: [
+        {
+          variableId: 'stringVar',
+          variableName: 'stringRef',
+          type: 'string',
+          value: JSON.stringify(ref),
+        },
+        {
+          variableId: 'plainVar',
+          variableName: 'plainRef',
+          type: 'plain',
+          value: JSON.stringify(ref),
+        },
+        {
+          variableId: 'numberVar',
+          variableName: 'numberRef',
+          type: 'number',
+          value: JSON.stringify(ref),
+        },
+        {
+          variableId: 'booleanVar',
+          variableName: 'booleanRef',
+          type: 'boolean',
+          value: JSON.stringify(ref),
+        },
+      ],
+    })
+
+    expect(ctx.workflowVariables?.stringVar.value).toEqual(ref)
+    expect(ctx.workflowVariables?.plainVar.value).toEqual(ref)
+    expect(ctx.workflowVariables?.numberVar.value).toEqual(ref)
+    expect(ctx.workflowVariables?.booleanVar.value).toEqual(ref)
+  })
+
   it('preserves existing variable metadata when compacting reassignment', async () => {
     const handler = new VariablesBlockHandler()
     const ctx = createContext({

apps/sim/executor/handlers/variables/variables-handler.ts

@@ -1,5 +1,7 @@
 import { createLogger } from '@sim/logger'
 import { toError } from '@sim/utils/errors'
+import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest'
+import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref'
 import { compactWorkflowVariableValue } from '@/lib/execution/payloads/serializer'
 import type { BlockOutput } from '@/blocks/types'
 import { BlockType } from '@/executor/constants'
@@ -96,6 +98,11 @@ export class VariablesBlockHandler implements BlockHandler {
   }
 
   private parseValueByType(value: any, type: string, variableName?: string): any {
+    const refValue = this.parseLargeExecutionValue(value)
+    if (refValue !== undefined) {
+      return refValue
+    }
+
     if (value === null || value === undefined || value === '') {
       if (type === 'number') return 0
       if (type === 'boolean') return false
@@ -167,4 +174,21 @@ export class VariablesBlockHandler implements BlockHandler {
 
     return value
   }
+
+  private parseLargeExecutionValue(value: any): any | undefined {
+    if (isLargeValueRef(value) || isLargeArrayManifest(value)) {
+      return value
+    }
+
+    if (typeof value !== 'string' || !value.trim()) {
+      return undefined
+    }
+
+    try {
+      const parsed = JSON.parse(value)
+      return isLargeValueRef(parsed) || isLargeArrayManifest(parsed) ? parsed : undefined
+    } catch {
+      return undefined
+    }
+  }
 }

apps/sim/executor/variables/resolver.test.ts

@@ -95,6 +95,106 @@ describe('VariableResolver function block inputs', () => {
     expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' })
   })
 
+  it('allows Variables block assignments to receive whole large refs', async () => {
+    const producer = createBlock('producer', 'Producer', BlockType.API)
+    const variablesBlock = createBlock('variables', 'Variables', BlockType.VARIABLES, {
+      variables: [
+        {
+          variableId: 'var-1',
+          variableName: 'issues',
+          type: 'array',
+          value: '<Producer.result>',
+        },
+      ],
+    })
+    const workflow: SerializedWorkflow = {
+      version: '1',
+      blocks: [producer, variablesBlock],
+      connections: [],
+      loops: {},
+      parallels: {},
+    }
+    const state = new ExecutionState()
+    const ref = {
+      __simLargeValueRef: true,
+      version: 1,
+      id: 'lv_ABCDEFGHIJKL',
+      kind: 'array',
+      size: 12 * 1024 * 1024,
+      executionId: 'execution-1',
+    }
+    state.setBlockOutput('producer', { result: ref })
+    const ctx = {
+      blockStates: state.getBlockStates(),
+      blockLogs: [],
+      environmentVariables: {},
+      workflowVariables: {},
+      decisions: { router: new Map(), condition: new Map() },
+      loopExecutions: new Map(),
+      executedBlocks: new Set(),
+      activeExecutionPath: new Set(),
+      completedLoops: new Set(),
+      metadata: {},
+    } as ExecutionContext
+
+    const resolver = new VariableResolver(workflow, {}, state)
+    const result = await resolver.resolveInputs(
+      ctx,
+      'variables',
+      variablesBlock.config.params,
+      variablesBlock
+    )
+
+    expect(JSON.parse(result.variables[0].value)).toEqual(ref)
+  })
+
+  it('allows Response block data to preserve whole large refs', async () => {
+    const producer = createBlock('producer', 'Producer', BlockType.API)
+    const responseBlock = createBlock('response', 'Response', BlockType.RESPONSE, {
+      dataMode: 'json',
+      data: '<Producer.result>',
+    })
+    const workflow: SerializedWorkflow = {
+      version: '1',
+      blocks: [producer, responseBlock],
+      connections: [],
+      loops: {},
+      parallels: {},
+    }
+    const state = new ExecutionState()
+    const ref = {
+      __simLargeValueRef: true,
+      version: 1,
+      id: 'lv_ZYXWVUTSRQPO',
+      kind: 'array',
+      size: 12 * 1024 * 1024,
+      executionId: 'execution-1',
+    }
+    state.setBlockOutput('producer', { result: ref })
+    const ctx = {
+      blockStates: state.getBlockStates(),
+      blockLogs: [],
+      environmentVariables: {},
+      workflowVariables: {},
+      decisions: { router: new Map(), condition: new Map() },
+      loopExecutions: new Map(),
+      executedBlocks: new Set(),
+      activeExecutionPath: new Set(),
+      completedLoops: new Set(),
+      metadata: {},
+    } as ExecutionContext
+
+    const resolver = new VariableResolver(workflow, {}, state)
+    const result = await resolver.resolveInputs(
+      ctx,
+      'response',
+      responseBlock.config.params,
+      responseBlock
+    )
+
+    expect(JSON.parse(result.data)).toEqual(ref)
+  })
+
   it('resolves workflow variable object references through context variables', async () => {
     const { block, ctx, resolver } = createResolver('javascript')
     const issues = [{ key: 'SIM-1', summary: 'Small issue' }]