diff --git a/apps/sim/lib/uploads/core/prefix-sweep.ts b/apps/sim/lib/uploads/core/prefix-sweep.ts index 6786bcd8d55..662d65103b9 100644 --- a/apps/sim/lib/uploads/core/prefix-sweep.ts +++ b/apps/sim/lib/uploads/core/prefix-sweep.ts @@ -120,22 +120,17 @@ export async function listStorageKeyPage( ): Promise { signal.throwIfAborted() if (USE_S3_STORAGE) { - const { ListObjectsV2Command } = await import('@aws-sdk/client-s3') - const { getS3Client } = await import('@/lib/uploads/providers/s3/client') + const { listS3ObjectKeyPage } = await import('@/lib/uploads/providers/s3/client') const config = getStorageConfig(context) - const page = await getS3Client().send( - new ListObjectsV2Command({ - Bucket: config.bucket, - Prefix: prefix, - MaxKeys: STORAGE_KEY_PAGE_SIZE, - ...(pageToken ? { ContinuationToken: pageToken } : {}), - }), - { abortSignal: signal } - ) - return keyPage( - (page.Contents ?? []).flatMap((object) => (object.Key ? [object.Key] : [])), - page.IsTruncated ? page.NextContinuationToken : undefined - ) + if (!config.bucket) throw new Error(`Bucket missing for ${context} storage`) + const page = await listS3ObjectKeyPage({ + bucket: config.bucket, + prefix, + maxKeys: STORAGE_KEY_PAGE_SIZE, + continuationToken: pageToken, + signal, + }) + return keyPage(page.keys, page.nextContinuationToken) } if (USE_BLOB_STORAGE) { const { getBlobServiceClient } = await import('@/lib/uploads/providers/blob/client') diff --git a/apps/sim/lib/uploads/providers/s3/client.ts b/apps/sim/lib/uploads/providers/s3/client.ts index f072664d4cc..226e439a9e4 100644 --- a/apps/sim/lib/uploads/providers/s3/client.ts +++ b/apps/sim/lib/uploads/providers/s3/client.ts @@ -7,6 +7,7 @@ import { DeleteObjectsCommand, GetObjectCommand, HeadObjectCommand, + ListObjectsV2Command, ListPartsCommand, PutObjectCommand, S3Client, @@ -413,6 +414,32 @@ export async function deleteManyFromS3( return { failed } } +/** + * Lists one page of object keys under `prefix` in `bucket`, resuming from `continuationToken` + * when given; `nextContinuationToken` is set while the listing has more. + */ +export async function listS3ObjectKeyPage(params: { + bucket: string + prefix: string + maxKeys: number + continuationToken?: string + signal: AbortSignal +}): Promise<{ keys: string[]; nextContinuationToken?: string }> { + const page = await getS3Client().send( + new ListObjectsV2Command({ + Bucket: params.bucket, + Prefix: params.prefix, + MaxKeys: params.maxKeys, + ...(params.continuationToken ? { ContinuationToken: params.continuationToken } : {}), + }), + { abortSignal: params.signal } + ) + const keys = (page.Contents ?? []).flatMap((object) => (object.Key ? [object.Key] : [])) + return page.IsTruncated && page.NextContinuationToken + ? { keys, nextContinuationToken: page.NextContinuationToken } + : { keys } +} + /** * Initiate a multipart upload for S3 */