diff --git a/apps/sim/app/api/files/upload/route.ts b/apps/sim/app/api/files/upload/route.ts index a68fb9da045..c6c65f03407 100644 --- a/apps/sim/app/api/files/upload/route.ts +++ b/apps/sim/app/api/files/upload/route.ts @@ -13,6 +13,7 @@ import { getSession } from '@/lib/auth' import { assertKnownSizeWithinLimit, isPayloadSizeLimitError, + MAX_MULTIPART_OVERHEAD_BYTES, readFileToBufferWithLimit, readFormDataWithLimit, } from '@/lib/core/utils/stream-limits' @@ -32,7 +33,6 @@ import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { createErrorResponse, InvalidRequestError } from '@/app/api/files/utils' const ALLOWED_EXTENSIONS = new Set(SUPPORTED_ATTACHMENT_EXTENSIONS) -const MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024 function validateFileExtension(filename: string): boolean { const extension = filename.split('.').pop()?.toLowerCase() diff --git a/apps/sim/app/api/v1/files/route.ts b/apps/sim/app/api/v1/files/route.ts index 9e573a126f8..350a2080c72 100644 --- a/apps/sim/app/api/v1/files/route.ts +++ b/apps/sim/app/api/v1/files/route.ts @@ -7,6 +7,7 @@ import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { isPayloadSizeLimitError, + MAX_MULTIPART_OVERHEAD_BYTES, readFileToBufferWithLimit, readFormDataWithLimit, } from '@/lib/core/utils/stream-limits' @@ -30,7 +31,6 @@ export const dynamic = 'force-dynamic' export const revalidate = 0 const MAX_FILE_SIZE = 100 * 1024 * 1024 -const MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024 /** GET /api/v1/files — List all files in a workspace. */ export const GET = withRouteHandler(async (request: NextRequest) => { diff --git a/apps/sim/app/api/v1/knowledge/[id]/documents/route.test.ts b/apps/sim/app/api/v1/knowledge/[id]/documents/route.test.ts new file mode 100644 index 00000000000..002204f1425 --- /dev/null +++ b/apps/sim/app/api/v1/knowledge/[id]/documents/route.test.ts @@ -0,0 +1,167 @@ +/** + * Tests for the v1 knowledge document upload route's bounded multipart read. + * + * @vitest-environment node + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockAuthenticateRequest, + mockResolveKnowledgeBase, + mockCheckActorUsageLimits, + mockUploadWorkspaceFile, + mockCreateSingleDocument, + mockProcessDocumentsWithQueue, + mockValidateFileType, +} = vi.hoisted(() => ({ + mockAuthenticateRequest: vi.fn(), + mockResolveKnowledgeBase: vi.fn(), + mockCheckActorUsageLimits: vi.fn(), + mockUploadWorkspaceFile: vi.fn(), + mockCreateSingleDocument: vi.fn(), + mockProcessDocumentsWithQueue: vi.fn(), + mockValidateFileType: vi.fn(), +})) + +vi.mock('@/app/api/v1/middleware', () => ({ + authenticateRequest: mockAuthenticateRequest, +})) + +vi.mock('@/app/api/v1/knowledge/utils', () => ({ + resolveKnowledgeBase: mockResolveKnowledgeBase, + serializeDate: (date: unknown) => (date instanceof Date ? date.toISOString() : date), + handleError: (_requestId: string, error: unknown) => + new Response(JSON.stringify({ error: error instanceof Error ? error.message : 'error' }), { + status: 500, + }), +})) + +vi.mock('@/lib/billing/calculations/usage-monitor', () => ({ + checkActorUsageLimits: mockCheckActorUsageLimits, +})) + +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + uploadWorkspaceFile: mockUploadWorkspaceFile, +})) + +vi.mock('@/lib/uploads/utils/validation', () => ({ + validateFileType: mockValidateFileType, +})) + +vi.mock('@/lib/knowledge/documents/service', () => ({ + createSingleDocument: mockCreateSingleDocument, + getDocuments: vi.fn(), + processDocumentsWithQueue: mockProcessDocumentsWithQueue, +})) + +import { POST } from '@/app/api/v1/knowledge/[id]/documents/route' + +const routeContext = { params: Promise.resolve({ id: 'kb-1' }) } + +function buildFormData(file: File, workspaceId = 'ws-1'): FormData { + const formData = new FormData() + formData.append('workspaceId', workspaceId) + formData.append('file', file) + return formData +} + +/** + * Builds a pull-based stream that emits fixed-size chunks on demand, so the + * size-capped reader's `reader.cancel()` simply stops future `pull` calls + * instead of racing an external (e.g. undici FormData) chunk producer. + */ +function makeChunkedOverLimitBody( + chunkBytes: number, + chunkCount: number +): ReadableStream { + let emitted = 0 + return new ReadableStream({ + pull(controller) { + if (emitted >= chunkCount) { + controller.close() + return + } + emitted++ + controller.enqueue(new Uint8Array(chunkBytes)) + }, + }) +} + +describe('v1 knowledge document upload route', () => { + beforeEach(() => { + vi.clearAllMocks() + mockAuthenticateRequest.mockResolvedValue({ + requestId: 'req-1', + userId: 'user-1', + rateLimit: {}, + }) + mockResolveKnowledgeBase.mockResolvedValue({ kb: { id: 'kb-1', workspaceId: 'ws-1' } }) + mockCheckActorUsageLimits.mockResolvedValue({ isExceeded: false }) + mockValidateFileType.mockReturnValue(null) + mockUploadWorkspaceFile.mockResolvedValue({ + url: 'https://example.com/file.txt', + }) + mockCreateSingleDocument.mockResolvedValue({ + id: 'doc-1', + filename: 'file.txt', + fileSize: 100, + mimeType: 'text/plain', + enabled: true, + uploadedAt: new Date('2026-01-01T00:00:00.000Z'), + }) + mockProcessDocumentsWithQueue.mockResolvedValue(undefined) + }) + + it('rejects a declared content-length above the limit before reading the body', async () => { + const formData = buildFormData(new File(['x'.repeat(10)], 'file.txt', { type: 'text/plain' })) + const req = new NextRequest('http://localhost:3000/api/v1/knowledge/kb-1/documents', { + method: 'POST', + headers: { 'content-length': String(200 * 1024 * 1024) }, + body: formData, + }) + + const response = await POST(req, routeContext) + const data = await response.json() + + expect(response.status).toBe(413) + expect(data.error).toContain('exceeds maximum size') + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + }) + + it('rejects a chunked body without content-length once the streamed size trips the cap', async () => { + const body = makeChunkedOverLimitBody(1024 * 1024, 200) + const req = new NextRequest('http://localhost:3000/api/v1/knowledge/kb-1/documents', { + method: 'POST', + body, + // @ts-expect-error - duplex is required by undici for streamed bodies but missing from NextRequestInit types + duplex: 'half', + }) + expect(req.headers.get('content-length')).toBeNull() + + const response = await POST(req, routeContext) + const data = await response.json() + + expect(response.status).toBe(413) + expect(data.error).toContain('exceeds maximum size') + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + }) + + it('uploads a normal, well-under-limit document successfully', async () => { + const file = new File(['hello world'], 'file.txt', { type: 'text/plain' }) + const formData = buildFormData(file) + const req = new NextRequest('http://localhost:3000/api/v1/knowledge/kb-1/documents', { + method: 'POST', + headers: { 'content-length': '1024' }, + body: formData, + }) + + const response = await POST(req, routeContext) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(mockUploadWorkspaceFile).toHaveBeenCalledTimes(1) + expect(mockCreateSingleDocument).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts b/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts index dafed80b7d2..d0d80070dcd 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts @@ -6,6 +6,11 @@ import { } from '@/lib/api/contracts/v1/knowledge' import { parseRequest } from '@/lib/api/server' import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor' +import { + isPayloadSizeLimitError, + MAX_MULTIPART_OVERHEAD_BYTES, + readFormDataWithLimit, +} from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { createSingleDocument, @@ -96,8 +101,14 @@ export const POST = withRouteHandler( let formData: FormData try { - formData = await request.formData() - } catch { + formData = await readFormDataWithLimit(request, { + maxBytes: MAX_FILE_SIZE + MAX_MULTIPART_OVERHEAD_BYTES, + label: 'knowledge document upload body', + }) + } catch (error) { + if (isPayloadSizeLimitError(error)) { + return NextResponse.json({ error: error.message }, { status: 413 }) + } return NextResponse.json( { error: 'Request body must be valid multipart form data' }, { status: 400 } diff --git a/apps/sim/app/api/workspaces/[id]/files/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/route.test.ts new file mode 100644 index 00000000000..3ce4bc9898e --- /dev/null +++ b/apps/sim/app/api/workspaces/[id]/files/route.test.ts @@ -0,0 +1,143 @@ +/** + * Tests for the workspace files upload route's bounded multipart read. + * + * @vitest-environment node + */ +import { authMockFns, permissionsMock, permissionsMockFns, posthogServerMock } from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockUploadWorkspaceFile, mockGetSharesForResources, mockRecordAudit } = vi.hoisted(() => ({ + mockUploadWorkspaceFile: vi.fn(), + mockGetSharesForResources: vi.fn(), + mockRecordAudit: vi.fn(), +})) + +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + uploadWorkspaceFile: mockUploadWorkspaceFile, + FileConflictError: class FileConflictError extends Error {}, +})) + +vi.mock('@/lib/uploads/shared/types', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + MAX_WORKSPACE_FORMDATA_FILE_SIZE: 1024, + } +}) + +vi.mock('@/lib/public-shares/share-manager', () => ({ + getSharesForResources: mockGetSharesForResources, +})) + +vi.mock('@/lib/posthog/server', () => posthogServerMock) +vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) +vi.mock('@/app/api/workflows/utils', () => ({ + verifyWorkspaceMembership: vi.fn().mockResolvedValue('write'), +})) +vi.mock('@sim/audit', () => ({ + recordAudit: mockRecordAudit, + AuditAction: { FILE_UPLOADED: 'file_uploaded' }, + AuditResourceType: { FILE: 'file' }, +})) + +const WS = '7727ef3f-8cf6-4686-b063-2bb006a10785' + +import { POST } from '@/app/api/workspaces/[id]/files/route' + +const routeContext = { params: Promise.resolve({ id: WS }) } + +function buildFormData(file: File): FormData { + const formData = new FormData() + formData.append('file', file) + return formData +} + +/** + * Builds a pull-based stream that emits fixed-size chunks on demand, so the + * size-capped reader's `reader.cancel()` simply stops future `pull` calls + * instead of racing an external (e.g. undici FormData) chunk producer. + */ +function makeChunkedOverLimitBody( + chunkBytes: number, + chunkCount: number +): ReadableStream { + let emitted = 0 + return new ReadableStream({ + pull(controller) { + if (emitted >= chunkCount) { + controller.close() + return + } + emitted++ + controller.enqueue(new Uint8Array(chunkBytes)) + }, + }) +} + +describe('workspace files upload route', () => { + beforeEach(() => { + vi.clearAllMocks() + authMockFns.mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) + permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') + mockGetSharesForResources.mockResolvedValue(new Map()) + mockUploadWorkspaceFile.mockResolvedValue({ + id: 'file-1', + name: 'file.txt', + url: 'https://example.com/file.txt', + size: 11, + type: 'text/plain', + }) + }) + + it('rejects a declared content-length above the limit before reading the body', async () => { + const formData = buildFormData(new File(['x'.repeat(10)], 'file.txt', { type: 'text/plain' })) + const req = new NextRequest(`http://localhost:3000/api/workspaces/${WS}/files`, { + method: 'POST', + headers: { 'content-length': String(10 * 1024 * 1024) }, + body: formData, + }) + + const response = await POST(req, routeContext) + const data = await response.json() + + expect(response.status).toBe(413) + expect(data.error).toContain('exceeds maximum size') + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + }) + + it('rejects a chunked body without content-length once the streamed size trips the cap', async () => { + const body = makeChunkedOverLimitBody(64 * 1024, 32) + const req = new NextRequest(`http://localhost:3000/api/workspaces/${WS}/files`, { + method: 'POST', + body, + // @ts-expect-error - duplex is required by undici for streamed bodies but missing from NextRequestInit types + duplex: 'half', + }) + expect(req.headers.get('content-length')).toBeNull() + + const response = await POST(req, routeContext) + const data = await response.json() + + expect(response.status).toBe(413) + expect(data.error).toContain('exceeds maximum size') + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + }) + + it('uploads a normal, well-under-limit file successfully', async () => { + const file = new File(['hello world'], 'file.txt', { type: 'text/plain' }) + const formData = buildFormData(file) + const req = new NextRequest(`http://localhost:3000/api/workspaces/${WS}/files`, { + method: 'POST', + headers: { 'content-length': '512' }, + body: formData, + }) + + const response = await POST(req, routeContext) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(mockUploadWorkspaceFile).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/app/api/workspaces/[id]/files/route.ts b/apps/sim/app/api/workspaces/[id]/files/route.ts index b13a8d08b6f..6f20d15a213 100644 --- a/apps/sim/app/api/workspaces/[id]/files/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/route.ts @@ -9,6 +9,11 @@ import { import { getValidationErrorMessage } from '@/lib/api/server' import { getSession } from '@/lib/auth' import { generateRequestId } from '@/lib/core/utils/request' +import { + isPayloadSizeLimitError, + MAX_MULTIPART_OVERHEAD_BYTES, + readFormDataWithLimit, +} from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { captureServerEvent } from '@/lib/posthog/server' import { getSharesForResources } from '@/lib/public-shares/share-manager' @@ -132,7 +137,21 @@ export const POST = withRouteHandler( return NextResponse.json({ error: 'Insufficient permissions' }, { status: 403 }) } - const formData = await request.formData() + let formData: FormData + try { + formData = await readFormDataWithLimit(request, { + maxBytes: MAX_WORKSPACE_FORMDATA_FILE_SIZE + MAX_MULTIPART_OVERHEAD_BYTES, + label: 'workspace file upload body', + }) + } catch (error) { + if (isPayloadSizeLimitError(error)) { + return NextResponse.json({ error: error.message }, { status: 413 }) + } + return NextResponse.json( + { error: 'Request body must be valid multipart form data' }, + { status: 400 } + ) + } const rawFile = formData.get('file') const rawFolderId = formData.get('folderId') const folderId = diff --git a/apps/sim/lib/core/utils/stream-limits.ts b/apps/sim/lib/core/utils/stream-limits.ts index d48fbb7083a..24d5182a9a8 100644 --- a/apps/sim/lib/core/utils/stream-limits.ts +++ b/apps/sim/lib/core/utils/stream-limits.ts @@ -2,6 +2,13 @@ import { toError } from '@sim/utils/errors' export const DEFAULT_MAX_ERROR_BODY_BYTES = 64 * 1024 +/** + * Slack for multipart boundary/header overhead on top of the intended file + * size limit, so legitimate uploads near the limit aren't rejected purely + * for encoding overhead. + */ +export const MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024 + export interface PayloadSizeLimitContext { label: string maxBytes: number