Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions apps/sim/app/api/folders/[id]/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { eq } from 'drizzle-orm'
import { type NextRequest, NextResponse } from 'next/server'
import { z } from 'zod'
import { getSession } from '@/lib/auth'
import { captureServerEvent } from '@/lib/posthog/server'
import { performDeleteFolder } from '@/lib/workflows/orchestration'
import { checkForCircularReference } from '@/lib/workflows/utils'
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
Expand Down Expand Up @@ -156,6 +157,13 @@ export async function DELETE(
return NextResponse.json({ error: result.error }, { status })
}

captureServerEvent(
session.user.id,
'folder_deleted',
{ workspace_id: existingFolder.workspaceId },
{ groups: { workspace: existingFolder.workspaceId } }
)

return NextResponse.json({
success: true,
deletedItems: result.deletedItems,
Expand Down
8 changes: 8 additions & 0 deletions apps/sim/app/api/folders/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { type NextRequest, NextResponse } from 'next/server'
import { z } from 'zod'
import { AuditAction, AuditResourceType, recordAudit } from '@/lib/audit/log'
import { getSession } from '@/lib/auth'
import { captureServerEvent } from '@/lib/posthog/server'
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'

const logger = createLogger('FoldersAPI')
Expand Down Expand Up @@ -145,6 +146,13 @@ export async function POST(request: NextRequest) {

logger.info('Created new folder:', { id, name, workspaceId, parentId })

captureServerEvent(
session.user.id,
'folder_created',
{ workspace_id: workspaceId },
{ groups: { workspace: workspaceId } }
)

recordAudit({
workspaceId,
actorId: session.user.id,
Expand Down
15 changes: 15 additions & 0 deletions apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { z } from 'zod'
import { decryptApiKey } from '@/lib/api-key/crypto'
import { AuditAction, AuditResourceType, recordAudit } from '@/lib/audit/log'
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
import { hasLiveSyncAccess } from '@/lib/billing/core/subscription'
import { generateRequestId } from '@/lib/core/utils/request'
import { deleteDocumentStorageFiles } from '@/lib/knowledge/documents/service'
import { cleanupUnusedTagDefinitions } from '@/lib/knowledge/tags/service'
Expand Down Expand Up @@ -116,6 +117,20 @@ export async function PATCH(request: NextRequest, { params }: RouteParams) {
)
}

if (
parsed.data.syncIntervalMinutes !== undefined &&
parsed.data.syncIntervalMinutes > 0 &&
parsed.data.syncIntervalMinutes < 60
) {
const canUseLiveSync = await hasLiveSyncAccess(auth.userId)
if (!canUseLiveSync) {
return NextResponse.json(
{ error: 'Live sync requires a Max or Enterprise plan' },
{ status: 403 }
)
}
}

if (parsed.data.sourceConfig !== undefined) {
const existingRows = await db
.select()
Expand Down
37 changes: 34 additions & 3 deletions apps/sim/app/api/knowledge/[id]/connectors/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { z } from 'zod'
import { encryptApiKey } from '@/lib/api-key/crypto'
import { AuditAction, AuditResourceType, recordAudit } from '@/lib/audit/log'
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
import { hasLiveSyncAccess } from '@/lib/billing/core/subscription'
import { generateRequestId } from '@/lib/core/utils/request'
import { dispatchSync } from '@/lib/knowledge/connectors/sync-engine'
import { allocateTagSlots } from '@/lib/knowledge/constants'
Expand Down Expand Up @@ -97,6 +98,16 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{

const { connectorType, credentialId, apiKey, sourceConfig, syncIntervalMinutes } = parsed.data

if (syncIntervalMinutes > 0 && syncIntervalMinutes < 60) {
const canUseLiveSync = await hasLiveSyncAccess(auth.userId)
if (!canUseLiveSync) {
return NextResponse.json(
{ error: 'Live sync requires a Max or Enterprise plan' },
{ status: 403 }
)
}
}

const connectorConfig = CONNECTOR_REGISTRY[connectorType]
if (!connectorConfig) {
return NextResponse.json(
Expand Down Expand Up @@ -151,19 +162,39 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
}

const tagSlotMapping: Record<string, string> = {}
let newTagSlots: Record<string, string> = {}

if (connectorConfig.tagDefinitions?.length) {
const disabledIds = new Set((sourceConfig.disabledTagIds as string[] | undefined) ?? [])
const enabledDefs = connectorConfig.tagDefinitions.filter((td) => !disabledIds.has(td.id))

const existingDefs = await db
.select({ tagSlot: knowledgeBaseTagDefinitions.tagSlot })
.select({
tagSlot: knowledgeBaseTagDefinitions.tagSlot,
displayName: knowledgeBaseTagDefinitions.displayName,
fieldType: knowledgeBaseTagDefinitions.fieldType,
})
.from(knowledgeBaseTagDefinitions)
.where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))

const usedSlots = new Set<string>(existingDefs.map((d) => d.tagSlot))
const { mapping, skipped: skippedTags } = allocateTagSlots(enabledDefs, usedSlots)
const existingByName = new Map(
existingDefs.map((d) => [d.displayName, { tagSlot: d.tagSlot, fieldType: d.fieldType }])
)

const defsNeedingSlots: typeof enabledDefs = []
for (const td of enabledDefs) {
const existing = existingByName.get(td.displayName)
if (existing && existing.fieldType === td.fieldType) {
tagSlotMapping[td.id] = existing.tagSlot
} else {
defsNeedingSlots.push(td)
}
}

const { mapping, skipped: skippedTags } = allocateTagSlots(defsNeedingSlots, usedSlots)
Object.assign(tagSlotMapping, mapping)
newTagSlots = mapping

for (const name of skippedTags) {
logger.warn(`[${requestId}] No available slots for "${name}"`)
Expand Down Expand Up @@ -197,7 +228,7 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
throw new Error('Knowledge base not found')
}

for (const [semanticId, slot] of Object.entries(tagSlotMapping)) {
for (const [semanticId, slot] of Object.entries(newTagSlots)) {
const td = connectorConfig.tagDefinitions!.find((d) => d.id === semanticId)!
await createTagDefinition(
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
retryDocumentProcessing,
updateDocument,
} from '@/lib/knowledge/documents/service'
import { captureServerEvent } from '@/lib/posthog/server'
import { checkDocumentAccess, checkDocumentWriteAccess } from '@/app/api/knowledge/utils'

const logger = createLogger('DocumentByIdAPI')
Expand Down Expand Up @@ -285,6 +286,14 @@ export async function DELETE(
request: req,
})

const kbWorkspaceId = accessCheck.knowledgeBase?.workspaceId ?? ''
captureServerEvent(
userId,
'knowledge_base_document_deleted',
{ knowledge_base_id: knowledgeBaseId, workspace_id: kbWorkspaceId },
kbWorkspaceId ? { groups: { workspace: kbWorkspaceId } } : undefined
)

return NextResponse.json({
success: true,
data: result,
Expand Down
103 changes: 43 additions & 60 deletions apps/sim/app/api/knowledge/search/utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
* @vitest-environment node
*/
import { createEnvMock, databaseMock, loggerMock } from '@sim/testing'
import { mockNextFetchResponse } from '@sim/testing/mocks'
import { beforeEach, describe, expect, it, vi } from 'vitest'

vi.mock('drizzle-orm')
Expand All @@ -14,16 +15,6 @@ vi.mock('@/lib/knowledge/documents/utils', () => ({
retryWithExponentialBackoff: (fn: any) => fn(),
}))

vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
data: [{ embedding: [0.1, 0.2, 0.3] }],
}),
})
)

vi.mock('@/lib/core/config/env', () => createEnvMock())

import {
Expand Down Expand Up @@ -178,17 +169,16 @@ describe('Knowledge Search Utils', () => {
OPENAI_API_KEY: 'test-openai-key',
})

const fetchSpy = vi.mocked(fetch)
fetchSpy.mockResolvedValueOnce({
ok: true,
json: async () => ({
mockNextFetchResponse({
json: {
data: [{ embedding: [0.1, 0.2, 0.3] }],
}),
} as any)
usage: { prompt_tokens: 1, total_tokens: 1 },
},
})

const result = await generateSearchEmbedding('test query')

expect(fetchSpy).toHaveBeenCalledWith(
expect(vi.mocked(fetch)).toHaveBeenCalledWith(
'https://test.openai.azure.com/openai/deployments/text-embedding-ada-002/embeddings?api-version=2024-12-01-preview',
expect.objectContaining({
headers: expect.objectContaining({
Expand All @@ -209,17 +199,16 @@ describe('Knowledge Search Utils', () => {
OPENAI_API_KEY: 'test-openai-key',
})

const fetchSpy = vi.mocked(fetch)
fetchSpy.mockResolvedValueOnce({
ok: true,
json: async () => ({
mockNextFetchResponse({
json: {
data: [{ embedding: [0.1, 0.2, 0.3] }],
}),
} as any)
usage: { prompt_tokens: 1, total_tokens: 1 },
},
})

const result = await generateSearchEmbedding('test query')

expect(fetchSpy).toHaveBeenCalledWith(
expect(vi.mocked(fetch)).toHaveBeenCalledWith(
'https://api.openai.com/v1/embeddings',
expect.objectContaining({
headers: expect.objectContaining({
Expand All @@ -243,17 +232,16 @@ describe('Knowledge Search Utils', () => {
OPENAI_API_KEY: 'test-openai-key',
})

const fetchSpy = vi.mocked(fetch)
fetchSpy.mockResolvedValueOnce({
ok: true,
json: async () => ({
mockNextFetchResponse({
json: {
data: [{ embedding: [0.1, 0.2, 0.3] }],
}),
} as any)
usage: { prompt_tokens: 1, total_tokens: 1 },
},
})

await generateSearchEmbedding('test query')

expect(fetchSpy).toHaveBeenCalledWith(
expect(vi.mocked(fetch)).toHaveBeenCalledWith(
expect.stringContaining('api-version='),
expect.any(Object)
)
Expand All @@ -273,17 +261,16 @@ describe('Knowledge Search Utils', () => {
OPENAI_API_KEY: 'test-openai-key',
})

const fetchSpy = vi.mocked(fetch)
fetchSpy.mockResolvedValueOnce({
ok: true,
json: async () => ({
mockNextFetchResponse({
json: {
data: [{ embedding: [0.1, 0.2, 0.3] }],
}),
} as any)
usage: { prompt_tokens: 1, total_tokens: 1 },
},
})

await generateSearchEmbedding('test query', 'text-embedding-3-small')

expect(fetchSpy).toHaveBeenCalledWith(
expect(vi.mocked(fetch)).toHaveBeenCalledWith(
'https://test.openai.azure.com/openai/deployments/custom-embedding-model/embeddings?api-version=2024-12-01-preview',
expect.any(Object)
)
Expand Down Expand Up @@ -311,13 +298,12 @@ describe('Knowledge Search Utils', () => {
KB_OPENAI_MODEL_NAME: 'text-embedding-ada-002',
})

const fetchSpy = vi.mocked(fetch)
fetchSpy.mockResolvedValueOnce({
mockNextFetchResponse({
ok: false,
status: 404,
statusText: 'Not Found',
text: async () => 'Deployment not found',
} as any)
text: 'Deployment not found',
})

await expect(generateSearchEmbedding('test query')).rejects.toThrow('Embedding API failed')

Expand All @@ -332,13 +318,12 @@ describe('Knowledge Search Utils', () => {
OPENAI_API_KEY: 'test-openai-key',
})

const fetchSpy = vi.mocked(fetch)
fetchSpy.mockResolvedValueOnce({
mockNextFetchResponse({
ok: false,
status: 429,
statusText: 'Too Many Requests',
text: async () => 'Rate limit exceeded',
} as any)
text: 'Rate limit exceeded',
})

await expect(generateSearchEmbedding('test query')).rejects.toThrow('Embedding API failed')

Expand All @@ -356,17 +341,16 @@ describe('Knowledge Search Utils', () => {
KB_OPENAI_MODEL_NAME: 'text-embedding-ada-002',
})

const fetchSpy = vi.mocked(fetch)
fetchSpy.mockResolvedValueOnce({
ok: true,
json: async () => ({
mockNextFetchResponse({
json: {
data: [{ embedding: [0.1, 0.2, 0.3] }],
}),
} as any)
usage: { prompt_tokens: 1, total_tokens: 1 },
},
})

await generateSearchEmbedding('test query')

expect(fetchSpy).toHaveBeenCalledWith(
expect(vi.mocked(fetch)).toHaveBeenCalledWith(
expect.any(String),
expect.objectContaining({
body: JSON.stringify({
Expand All @@ -387,17 +371,16 @@ describe('Knowledge Search Utils', () => {
OPENAI_API_KEY: 'test-openai-key',
})

const fetchSpy = vi.mocked(fetch)
fetchSpy.mockResolvedValueOnce({
ok: true,
json: async () => ({
mockNextFetchResponse({
json: {
data: [{ embedding: [0.1, 0.2, 0.3] }],
}),
} as any)
usage: { prompt_tokens: 1, total_tokens: 1 },
},
})

await generateSearchEmbedding('test query', 'text-embedding-3-small')

expect(fetchSpy).toHaveBeenCalledWith(
expect(vi.mocked(fetch)).toHaveBeenCalledWith(
expect.any(String),
expect.objectContaining({
body: JSON.stringify({
Expand Down
Loading
Loading