xiongyuxing/gongxue-base
1
0
Fork 0
forked from wangziqi/gongxue-base
Code Pull Requests Activity

feat: add content asset security scanning

Browse Source
This commit is contained in:
Codex
2026-06-29 19:42:05 +08:00
parent d696fc38b0
commit ecd269b548
21 changed files with 936 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
apps/api/src/features/catalog/assets.ts
View File

@@ -2,6 +2,9 @@ import { HttpError, type RequestContext } from '../../core/http.js';
import { intParam, optionalUserIdFrom, stringParam, tenantIdFrom, userIdFrom } from '../../core/request.js';
import { query, queryOne } from '../../core/db.js';
import { signStorageDownload, type StorageProviderName } from '../storage/service.js';
import {
assertAssetSecurityScanPassed,
} from '../tenant-content/assets.js';
import {
assetAccessTtl,
assertCdnAccessAllowed,
@@ -25,6 +28,7 @@ interface CatalogAssetRow {
regionId: string | null;
subjectId: string | null;
uploadStatus: string;
securityScanStatus: string;
previewStatus: string;
accessRules: Record<string, unknown>;
metadata: Record<string, unknown>;
@@ -122,6 +126,11 @@ function assertPublishedAsset(asset: CatalogAssetRow) {
if (asset.objectKey && asset.uploadStatus !== 'verified') {
throw new HttpError(409, 'Asset upload has not been verified', 'ASSET_UPLOAD_NOT_VERIFIED');
}
assertAssetSecurityScanPassed({
provider: asset.storageProvider as StorageProviderName,
objectKey: asset.objectKey,
securityScanStatus: asset.securityScanStatus,
});
}
export async function assetsRoute(ctx: RequestContext) {
@@ -174,6 +183,7 @@ export async function assetsRoute(ctx: RequestContext) {
file_name as "fileName", preview_url as "previewUrl",
mime_type as "mimeType", file_size_bytes as "fileSizeBytes",
upload_status as "uploadStatus", preview_status as "previewStatus",
security_scan_status as "securityScanStatus",
visibility, region_id as "regionId", subject_id as "subjectId",
category_id as "categoryId", node_id as "nodeId",
entry_id as "entryId", content_node_id as "contentNodeId",
@@ -204,7 +214,9 @@ export async function assetDownloadRoute(ctx: RequestContext) {
cdn_url as "cdnUrl", preview_url as "previewUrl",
preview_object_key as "previewObjectKey", mime_type as "mimeType",
visibility, region_id as "regionId", subject_id as "subjectId",
upload_status as "uploadStatus", preview_status as "previewStatus",
upload_status as "uploadStatus",
security_scan_status as "securityScanStatus",
preview_status as "previewStatus",
access_rules as "accessRules", metadata
from public.content_assets
where tenant_id = $1 and id = $2 and status = 'active'
@@ -283,6 +295,7 @@ export async function assetDownloadRoute(ctx: RequestContext) {
fileName: asset.fileName,
previewUrl: asset.previewUrl,
uploadStatus: asset.uploadStatus,
securityScanStatus: asset.securityScanStatus,
previewStatus: asset.previewStatus,
visibility: asset.visibility,
},
@@ -305,7 +318,9 @@ export async function assetPreviewRoute(ctx: RequestContext) {
cdn_url as "cdnUrl", preview_url as "previewUrl",
preview_object_key as "previewObjectKey", mime_type as "mimeType",
visibility, region_id as "regionId", subject_id as "subjectId",
upload_status as "uploadStatus", preview_status as "previewStatus",
upload_status as "uploadStatus",
security_scan_status as "securityScanStatus",
preview_status as "previewStatus",
access_rules as "accessRules", metadata
from public.content_assets
where tenant_id = $1 and id = $2 and status = 'active'
@@ -382,6 +397,7 @@ export async function assetPreviewRoute(ctx: RequestContext) {
fileName: asset.fileName,
previewUrl: asset.previewUrl,
previewStatus: asset.previewStatus,
securityScanStatus: asset.securityScanStatus,
visibility: asset.visibility,
},
access,

213
apps/api/src/features/tenant-content/assets.ts
View File

@@ -31,6 +31,7 @@ const VISIBILITIES = ['public', 'tenant', 'members', 'svip', 'private'];
const ASSET_STATUSES = ['draft', 'active', 'archived'];
const UPLOAD_STATUSES = ['not_required', 'pending', 'verified', 'failed'];
const PREVIEW_STATUSES = ['none', 'pending', 'ready', 'failed'];
const SECURITY_SCAN_STATUSES = ['not_required', 'pending', 'scanning', 'passed', 'failed', 'skipped'];
interface AssetRow {
id: string;
@@ -50,6 +51,9 @@ interface AssetRow {
visibility: string;
status: string;
uploadStatus: string;
securityScanStatus: string;
securityScanProvider: string | null;
securityScanSummary: Record<string, unknown>;
verifiedSizeBytes: number | null;
verifiedChecksumSha256: string | null;
previewStatus: string;
@@ -122,6 +126,9 @@ async function existingAssetForUpsert(tenantId: string, id: string | null) {
cdn_url as "cdnUrl", preview_url as "previewUrl", mime_type as "mimeType",
file_size_bytes as "fileSizeBytes", checksum_sha256 as "checksumSha256",
visibility, status, upload_status as "uploadStatus",
security_scan_status as "securityScanStatus",
security_scan_provider as "securityScanProvider",
security_scan_summary as "securityScanSummary",
verified_size_bytes as "verifiedSizeBytes",
verified_checksum_sha256 as "verifiedChecksumSha256",
preview_status as "previewStatus"
@@ -156,14 +163,46 @@ function objectUploadStatus(input: {
return 'pending';
}
function assertPublishableManagedObject(status: string, uploadStatus: string, provider: StorageProviderName, objectKey: string | null) {
if (status === 'active' && objectKey && isManagedObjectProvider(provider) && uploadStatus !== 'verified') {
function objectSecurityScanStatus(input: {
existing: AssetRow | null;
provider: StorageProviderName;
objectKey: string | null;
uploadStatus: string;
}) {
if (!input.objectKey || !isManagedObjectProvider(input.provider)) return 'not_required';
if (input.uploadStatus !== 'verified') return 'pending';
return input.existing?.securityScanStatus || 'pending';
}
export function assertAssetSecurityScanPassed(input: {
provider: StorageProviderName;
objectKey: string | null;
securityScanStatus?: string | null;
}) {
if (!input.objectKey || !isManagedObjectProvider(input.provider)) return;
if (input.securityScanStatus === 'passed') return;
if (input.securityScanStatus === 'failed') {
throw new HttpError(409, 'Asset security scan failed', 'ASSET_SECURITY_SCAN_FAILED');
}
throw new HttpError(409, 'Asset security scan is required before access', 'ASSET_SECURITY_SCAN_REQUIRED');
}
function assertPublishableManagedObject(
status: string,
uploadStatus: string,
scanStatus: string,
provider: StorageProviderName,
objectKey: string | null,
) {
if (status !== 'active' || !objectKey || !isManagedObjectProvider(provider)) return;
if (uploadStatus !== 'verified') {
throw new HttpError(
409,
'Managed storage asset must be confirmed before publishing',
'ASSET_UPLOAD_CONFIRMATION_REQUIRED',
);
}
assertAssetSecurityScanPassed({ provider, objectKey, securityScanStatus: scanStatus });
}
async function assertOptionalReference(tenantId: string, table: string, id: string | null, code: string) {
@@ -219,6 +258,7 @@ export async function assetsAdminRoute(ctx: RequestContext) {
const categoryId = stringParam(ctx, 'categoryId');
const entryId = stringParam(ctx, 'entryId');
const contentNodeId = stringParam(ctx, 'contentNodeId');
const securityScanStatus = stringParam(ctx, 'securityScanStatus');
const params: unknown[] = [auth.tenantId];
const filters = ['tenant_id = $1'];
@@ -254,6 +294,13 @@ export async function assetsAdminRoute(ctx: RequestContext) {
params.push(contentNodeId);
filters.push(`content_node_id = $${params.length}`);
}
if (securityScanStatus) {
if (!SECURITY_SCAN_STATUSES.includes(securityScanStatus)) {
throw new HttpError(400, `Invalid securityScanStatus: ${securityScanStatus}`, 'INVALID_FIELD_VALUE');
}
params.push(securityScanStatus);
filters.push(`security_scan_status = $${params.length}`);
}
params.push(limit);
const items = await query(
@@ -269,6 +316,10 @@ export async function assetsAdminRoute(ctx: RequestContext) {
verified_checksum_sha256 as "verifiedChecksumSha256",
verification_details as "verificationDetails",
preview_object_key as "previewObjectKey", preview_status as "previewStatus",
security_scan_status as "securityScanStatus",
security_scanned_at as "securityScannedAt",
security_scan_provider as "securityScanProvider",
security_scan_summary as "securityScanSummary",
security_flags as "securityFlags",
visibility, is_public as "isPublic", region_id as "regionId",
subject_id as "subjectId", category_id as "categoryId", node_id as "nodeId",
@@ -288,6 +339,44 @@ export async function assetsAdminRoute(ctx: RequestContext) {
return { items };
}
export async function assetSecurityScanEventsAdminRoute(ctx: RequestContext) {
const auth = await requireTenantContentEditor(ctx);
const limit = intParam(ctx, 'limit', 100, 500);
const assetId = stringParam(ctx, 'assetId');
const scanStatus = stringParam(ctx, 'scanStatus');
const params: unknown[] = [auth.tenantId];
const filters = ['tenant_id = $1'];
if (assetId) {
assertUuidParam(assetId, 'assetId');
params.push(assetId);
filters.push(`asset_id = $${params.length}::uuid`);
}
if (scanStatus) {
if (!SECURITY_SCAN_STATUSES.includes(scanStatus) || scanStatus === 'not_required') {
throw new HttpError(400, `Invalid scanStatus: ${scanStatus}`, 'INVALID_FIELD_VALUE');
}
params.push(scanStatus);
filters.push(`scan_status = $${params.length}`);
}
params.push(limit);
const items = await query(
`
select id, asset_id as "assetId", provider,
scan_status as "scanStatus", risk_level as "riskLevel",
issue_codes as "issueCodes", details, created_at as "createdAt"
from public.content_asset_security_scan_events
where ${filters.join(' and ')}
order by created_at desc
limit $${params.length}
`,
params,
);
return { items };
}
export async function assetAccessEventsAdminRoute(ctx: RequestContext) {
const auth = await requireTenantContentEditor(ctx);
const limit = intParam(ctx, 'limit', 100, 500);
@@ -362,14 +451,22 @@ export async function upsertAssetRoute(ctx: RequestContext) {
objectKey: cleanObjectKey,
cdnUrl,
});
const nextSecurityScanStatus = objectSecurityScanStatus({
existing,
provider: storageProvider,
objectKey: cleanObjectKey,
uploadStatus: nextUploadStatus,
});
const nextPreviewStatus = previewObjectKey || nullableString(body.previewUrl)
? choice(body.previewStatus, PREVIEW_STATUSES, existing?.previewStatus || 'ready', 'previewStatus')
: 'none';
const statusFallback = cleanObjectKey && isManagedObjectProvider(storageProvider) && nextUploadStatus !== 'verified'
const statusFallback = cleanObjectKey
&& isManagedObjectProvider(storageProvider)
&& (nextUploadStatus !== 'verified' || nextSecurityScanStatus !== 'passed')
? 'draft'
: 'active';
const status = choice(body.status, ASSET_STATUSES, statusFallback, 'status');
assertPublishableManagedObject(status, nextUploadStatus, storageProvider, cleanObjectKey);
assertPublishableManagedObject(status, nextUploadStatus, nextSecurityScanStatus, storageProvider, cleanObjectKey);
if (status === 'active' && !cdnUrl && !objectKey) {
throw new HttpError(400, 'Active asset requires cdnUrl or objectKey', 'ASSET_LOCATION_REQUIRED');
@@ -388,7 +485,9 @@ export async function upsertAssetRoute(ctx: RequestContext) {
id, tenant_id, legacy_id, asset_key, asset_type, storage_provider,
bucket, object_key, title, category, description, file_name, cdn_url,
preview_url, mime_type, file_size_bytes, checksum_sha256,
upload_status, preview_object_key, preview_status, visibility,
upload_status, security_scan_status, security_scanned_at,
security_scan_provider, security_scan_summary,
preview_object_key, preview_status, visibility,
is_public, region_id, subject_id, category_id, node_id, entry_id,
content_node_id, status,
sort_order, access_rules, metadata, created_by, updated_by, source
@@ -397,10 +496,12 @@ export async function upsertAssetRoute(ctx: RequestContext) {
coalesce($1::uuid, gen_random_uuid()), $2, $3, $4, $5, $6,
$7, $8, $9, $10, $11, $12, $13,
$14, $15, $16, $17, $18,
$19, $20, $21,
$22, $23::uuid, $24::uuid, $25::uuid, $26::uuid, $27::uuid,
$28::uuid, $29,
$30, $31::jsonb, $32::jsonb, $33, $33, $34
$19, case when $19 in ('passed', 'not_required') then now() else null end,
$20, $21::jsonb,
$22, $23, $24,
$25, $26::uuid, $27::uuid, $28::uuid, $29::uuid, $30::uuid,
$31::uuid, $32,
$33, $34::jsonb, $35::jsonb, $36, $36, $37
)
on conflict (id)
do update set legacy_id = excluded.legacy_id,
@@ -419,6 +520,31 @@ export async function upsertAssetRoute(ctx: RequestContext) {
file_size_bytes = excluded.file_size_bytes,
checksum_sha256 = excluded.checksum_sha256,
upload_status = excluded.upload_status,
security_scan_status = excluded.security_scan_status,
security_scanned_at = case
when public.content_assets.storage_provider = excluded.storage_provider
and coalesce(public.content_assets.bucket, '') = coalesce(excluded.bucket, '')
and coalesce(public.content_assets.object_key, '') = coalesce(excluded.object_key, '')
and excluded.security_scan_status = public.content_assets.security_scan_status
then public.content_assets.security_scanned_at
else excluded.security_scanned_at
end,
security_scan_provider = case
when public.content_assets.storage_provider = excluded.storage_provider
and coalesce(public.content_assets.bucket, '') = coalesce(excluded.bucket, '')
and coalesce(public.content_assets.object_key, '') = coalesce(excluded.object_key, '')
and excluded.security_scan_status = public.content_assets.security_scan_status
then public.content_assets.security_scan_provider
else excluded.security_scan_provider
end,
security_scan_summary = case
when public.content_assets.storage_provider = excluded.storage_provider
and coalesce(public.content_assets.bucket, '') = coalesce(excluded.bucket, '')
and coalesce(public.content_assets.object_key, '') = coalesce(excluded.object_key, '')
and excluded.security_scan_status = public.content_assets.security_scan_status
then public.content_assets.security_scan_summary
else excluded.security_scan_summary
end,
verified_at = case
when public.content_assets.storage_provider = excluded.storage_provider
and coalesce(public.content_assets.bucket, '') = coalesce(excluded.bucket, '')
@@ -476,6 +602,10 @@ export async function upsertAssetRoute(ctx: RequestContext) {
verified_checksum_sha256 as "verifiedChecksumSha256",
verification_details as "verificationDetails",
preview_object_key as "previewObjectKey", preview_status as "previewStatus",
security_scan_status as "securityScanStatus",
security_scanned_at as "securityScannedAt",
security_scan_provider as "securityScanProvider",
security_scan_summary as "securityScanSummary",
security_flags as "securityFlags",
visibility, is_public as "isPublic", region_id as "regionId",
subject_id as "subjectId", category_id as "categoryId", node_id as "nodeId",
@@ -504,6 +634,15 @@ export async function upsertAssetRoute(ctx: RequestContext) {
fileSizeBytes,
checksum,
nextUploadStatus,
nextSecurityScanStatus,
nextSecurityScanStatus === 'not_required' ? null : existing?.securityScanProvider || 'metadata_rules',
JSON.stringify(
nextSecurityScanStatus === 'not_required'
? {}
: nextSecurityScanStatus === 'passed'
? existing?.securityScanSummary || { riskLevel: 'none', issueCodes: [] }
: { riskLevel: 'unknown', issueCodes: [], pendingReason: 'asset_upsert' },
),
previewObjectKey,
nextPreviewStatus,
visibility,
@@ -550,6 +689,9 @@ export async function confirmAssetUploadRoute(ctx: RequestContext) {
cdn_url as "cdnUrl", preview_url as "previewUrl", mime_type as "mimeType",
file_size_bytes as "fileSizeBytes", checksum_sha256 as "checksumSha256",
visibility, status, upload_status as "uploadStatus",
security_scan_status as "securityScanStatus",
security_scan_provider as "securityScanProvider",
security_scan_summary as "securityScanSummary",
verified_size_bytes as "verifiedSizeBytes",
verified_checksum_sha256 as "verifiedChecksumSha256",
preview_status as "previewStatus"
@@ -623,13 +765,23 @@ export async function confirmAssetUploadRoute(ctx: RequestContext) {
`
update public.content_assets
set upload_status = 'failed',
security_scan_status = 'skipped',
security_scanned_at = null,
security_scan_provider = 'metadata_rules',
security_scan_summary = $5::jsonb,
verification_details = $3::jsonb,
security_flags = jsonb_set(coalesce(security_flags, '{}'::jsonb), '{uploadVerificationFailed}', 'true'::jsonb, true),
updated_by = $4,
updated_at = now()
where tenant_id = $1 and id = $2
`,
[auth.tenantId, assetId, JSON.stringify(verificationDetails), auth.userId],
[
auth.tenantId,
assetId,
JSON.stringify(verificationDetails),
auth.userId,
JSON.stringify({ riskLevel: 'medium', issueCodes: issues, skippedReason: 'upload_verification_failed' }),
],
);
await recordAssetAccessEvent({
ctx,
@@ -666,7 +818,12 @@ export async function confirmAssetUploadRoute(ctx: RequestContext) {
mime_type = coalesce(mime_type, $6),
checksum_sha256 = coalesce(checksum_sha256, $5),
verification_details = $7::jsonb,
status = case when $8::boolean and status <> 'archived' then 'active' else status end,
security_scan_status = 'pending',
security_scanned_at = null,
security_scan_provider = 'metadata_rules',
security_scan_summary = '{"riskLevel":"unknown","issueCodes":[],"pendingReason":"upload_confirmed"}'::jsonb,
security_flags = coalesce(security_flags, '{}'::jsonb) - 'assetSecurityScanFailed',
status = case when status = 'archived' then status else 'draft' end,
updated_by = $3,
updated_at = now()
where tenant_id = $1 and id = $2
@@ -677,7 +834,11 @@ export async function confirmAssetUploadRoute(ctx: RequestContext) {
upload_status as "uploadStatus", verified_at as "verifiedAt",
verified_by as "verifiedBy", verified_size_bytes as "verifiedSizeBytes",
verified_checksum_sha256 as "verifiedChecksumSha256",
verification_details as "verificationDetails"
verification_details as "verificationDetails",
security_scan_status as "securityScanStatus",
security_scanned_at as "securityScannedAt",
security_scan_provider as "securityScanProvider",
security_scan_summary as "securityScanSummary"
`,
[
auth.tenantId,
@@ -687,7 +848,6 @@ export async function confirmAssetUploadRoute(ctx: RequestContext) {
metadata.checksumSha256 ?? declaredChecksum,
observedMime || expectedMime,
JSON.stringify(verificationDetails),
publish,
],
);
@@ -720,7 +880,16 @@ export async function confirmAssetUploadRoute(ctx: RequestContext) {
},
});
return { item, metadata, verification: verificationDetails };
return {
item,
metadata,
verification: verificationDetails,
securityScan: {
status: 'pending',
provider: 'metadata_rules',
requiredBeforePublish: true,
},
};
}
export async function signAssetUploadRoute(ctx: RequestContext) {
@@ -811,6 +980,9 @@ export async function signAssetDownloadAdminRoute(ctx: RequestContext) {
preview_url as "previewUrl", mime_type as "mimeType",
file_size_bytes as "fileSizeBytes", checksum_sha256 as "checksumSha256",
visibility, status, upload_status as "uploadStatus",
security_scan_status as "securityScanStatus",
security_scan_provider as "securityScanProvider",
security_scan_summary as "securityScanSummary",
verified_size_bytes as "verifiedSizeBytes",
verified_checksum_sha256 as "verifiedChecksumSha256",
preview_status as "previewStatus",
@@ -844,6 +1016,11 @@ export async function signAssetDownloadAdminRoute(ctx: RequestContext) {
metadata: asset.metadata,
accessRules: asset.accessRules,
});
assertAssetSecurityScanPassed({
provider: asset.storageProvider as StorageProviderName,
objectKey: asset.objectKey,
securityScanStatus: asset.securityScanStatus,
});
} catch (error) {
await recordDeniedAdminAssetAccess(ctx, auth, asset, 'admin_download', error);
throw error;
@@ -898,6 +1075,9 @@ export async function signAssetPreviewAdminRoute(ctx: RequestContext) {
cdn_url as "cdnUrl", preview_url as "previewUrl", mime_type as "mimeType",
file_size_bytes as "fileSizeBytes", checksum_sha256 as "checksumSha256",
visibility, status, upload_status as "uploadStatus",
security_scan_status as "securityScanStatus",
security_scan_provider as "securityScanProvider",
security_scan_summary as "securityScanSummary",
verified_size_bytes as "verifiedSizeBytes",
verified_checksum_sha256 as "verifiedChecksumSha256",
preview_status as "previewStatus",
@@ -936,6 +1116,11 @@ export async function signAssetPreviewAdminRoute(ctx: RequestContext) {
metadata: asset.metadata,
accessRules: asset.accessRules,
});
assertAssetSecurityScanPassed({
provider: asset.storageProvider as StorageProviderName,
objectKey,
securityScanStatus: asset.securityScanStatus,
});
} catch (error) {
await recordDeniedAdminAssetAccess(ctx, auth, asset, 'admin_preview', error);
throw error;

2
apps/api/src/features/tenant-content/index.ts
View File

@@ -1,6 +1,7 @@
import type { RouteDefinition } from '../../core/router.js';
import {
assetAccessEventsAdminRoute,
assetSecurityScanEventsAdminRoute,
assetsAdminRoute,
confirmAssetUploadRoute,
signAssetDownloadAdminRoute,
@@ -104,6 +105,7 @@ export const tenantContentRoutes: RouteDefinition[] = [
['PATCH', '/api/tenant-content/questions', updateQuestionRoute],
['GET', '/api/tenant-content/assets', assetsAdminRoute],
['GET', '/api/tenant-content/assets/access-events', assetAccessEventsAdminRoute],
['GET', '/api/tenant-content/assets/security-scan-events', assetSecurityScanEventsAdminRoute],
['PUT', '/api/tenant-content/assets', upsertAssetRoute],
['POST', '/api/tenant-content/assets/sign-upload', signAssetUploadRoute],
['POST', '/api/tenant-content/assets/confirm-upload', confirmAssetUploadRoute],

8
apps/api/src/features/video/routes.ts
View File

@@ -17,6 +17,7 @@ import {
assertCdnAccessAllowed,
signedAssetFingerprint,
} from '../storage/asset-access.js';
import { assertAssetSecurityScanPassed } from '../tenant-content/assets.js';
interface QuestionVideoRow {
questionId: string;
@@ -60,6 +61,7 @@ interface VideoPlaybackRow {
assetType: string | null;
assetVisibility: string | null;
assetUploadStatus: string | null;
assetSecurityScanStatus: string | null;
assetAccessRules: Record<string, unknown> | null;
assetMetadata: Record<string, unknown> | null;
}
@@ -163,6 +165,11 @@ async function signVideoPlayback(tenantId: string, video: VideoPlaybackRow) {
if (video.objectKey && video.assetUploadStatus !== 'verified') {
throw new HttpError(409, 'Video asset upload has not been verified', 'VIDEO_ASSET_UPLOAD_NOT_VERIFIED');
}
assertAssetSecurityScanPassed({
provider: video.storageProvider as StorageProviderName,
objectKey: video.objectKey,
securityScanStatus: video.assetSecurityScanStatus,
});
assertCdnAccessAllowed({
assetId: video.assetId,
visibility: video.assetVisibility || 'svip',
@@ -307,6 +314,7 @@ export async function videoPlaybackRoute(ctx: RequestContext) {
a.title as "assetTitle", a.status as "assetStatus",
a.asset_type as "assetType", a.visibility as "assetVisibility",
a.upload_status as "assetUploadStatus",
a.security_scan_status as "assetSecurityScanStatus",
a.access_rules as "assetAccessRules", a.metadata as "assetMetadata"
from public.video_explanations v
left join public.content_assets a on a.tenant_id = v.tenant_id and a.id = v.asset_id

318
apps/worker/src/jobs/assets.ts
View File

@@ -11,15 +11,19 @@ interface AssetCandidate {
title: string | null;
status: string;
uploadStatus: string;
assetType: string;
storageProvider: StorageProviderName;
bucket: string | null;
objectKey: string | null;
fileName: string | null;
mimeType: string | null;
fileSizeBytes: number | string | null;
checksumSha256: string | null;
verifiedSizeBytes: number | string | null;
verifiedChecksumSha256: string | null;
verificationDetails: Record<string, unknown>;
securityScanStatus: string;
securityScanSummary: Record<string, unknown>;
securityFlags: Record<string, unknown>;
}
@@ -45,6 +49,14 @@ interface AssetWorkerResult {
errors: number;
}
interface SecurityScanResult {
status: 'passed' | 'failed' | 'skipped';
provider: string;
riskLevel: 'none' | 'low' | 'medium' | 'high' | 'critical';
issueCodes: string[];
details: Record<string, unknown>;
}
class AssetWorkerError extends Error {
readonly code: string;
@@ -56,6 +68,30 @@ class AssetWorkerError extends Error {
const UPLOADABLE_PROVIDERS = new Set<StorageProviderName>(['local_dev', 'supabase_storage', 'aliyun_oss', 'tencent_cos']);
const SAFE_OBJECT_KEY_RE = /^[A-Za-z0-9][A-Za-z0-9._~!$&'()+,;=@/-]{0,1023}$/;
const EXTENSION_MIME_HINTS: Record<string, string[]> = {
pdf: ['application/pdf'],
png: ['image/png'],
jpg: ['image/jpeg'],
jpeg: ['image/jpeg'],
gif: ['image/gif'],
webp: ['image/webp'],
svg: ['image/svg+xml'],
mp4: ['video/mp4'],
mov: ['video/quicktime'],
mp3: ['audio/mpeg'],
wav: ['audio/wav', 'audio/x-wav'],
doc: ['application/msword'],
docx: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
xls: ['application/vnd.ms-excel'],
xlsx: ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
ppt: ['application/vnd.ms-powerpoint'],
pptx: ['application/vnd.openxmlformats-officedocument.presentationml.presentation'],
zip: ['application/zip', 'application/x-zip-compressed'],
json: ['application/json'],
txt: ['text/plain'],
md: ['text/markdown', 'text/plain'],
csv: ['text/csv', 'text/plain'],
};
function objectValue(value: unknown): Record<string, unknown> {
return value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {};
@@ -445,6 +481,110 @@ function compareAssetMetadata(asset: AssetCandidate, metadata: StorageObjectMeta
};
}
function extensionFromName(value: string | null) {
if (!value) return '';
const withoutQuery = value.split('?')[0]?.split('#')[0] || value;
const last = withoutQuery.split('/').pop() || withoutQuery;
const dot = last.lastIndexOf('.');
return dot >= 0 ? last.slice(dot + 1).trim().toLowerCase() : '';
}
function metadataBoolean(value: unknown) {
return value === true || value === 'true' || value === 1 || value === '1';
}
function securityScanRules(asset: AssetCandidate, metadata: StorageObjectMetadata): SecurityScanResult {
const issueCodes: string[] = [];
const details: Record<string, unknown> = {
observed: {
fileSizeBytes: metadata.sizeBytes,
mimeType: canonicalMime(metadata.mimeType),
checksumSha256: metadata.checksumSha256,
verificationSource: metadata.verificationSource,
},
declared: {
assetType: asset.assetType,
fileName: asset.fileName,
objectKey: asset.objectKey,
mimeType: canonicalMime(asset.mimeType),
fileSizeBytes: validateFileSize(asset.fileSizeBytes),
checksumSha256: asset.checksumSha256,
},
};
try {
if (asset.objectKey) validateObjectKey(asset.tenantId, asset.objectKey);
} catch (error) {
issueCodes.push(eventError(error).code);
}
try {
validateFileSize(metadata.sizeBytes ?? asset.verifiedSizeBytes ?? asset.fileSizeBytes);
} catch (error) {
issueCodes.push(eventError(error).code);
}
const declaredMime = canonicalMime(asset.mimeType);
const observedMime = canonicalMime(metadata.mimeType);
const effectiveMime = observedMime || declaredMime;
try {
validateMimeType(effectiveMime);
} catch (error) {
issueCodes.push(eventError(error).code);
}
const fileExtension = extensionFromName(asset.fileName) || extensionFromName(asset.objectKey);
details.fileExtension = fileExtension || null;
if (fileExtension && effectiveMime && EXTENSION_MIME_HINTS[fileExtension]) {
const allowedForExtension = EXTENSION_MIME_HINTS[fileExtension];
if (!allowedForExtension.includes(effectiveMime)) {
issueCodes.push('file_extension_mime_mismatch');
details.extensionMimeExpected = allowedForExtension;
}
}
const metadataFlags = objectValue(asset.securityScanSummary);
const assetMetadata = objectValue(asset.verificationDetails?.metadata);
const explicitFlag = [
metadataFlags.forceFail,
metadataFlags.securityScanForceFail,
assetMetadata.securityScanForceFail,
assetMetadata.forceSecurityScanFail,
].some(metadataBoolean);
if (explicitFlag) {
issueCodes.push('security_scan_forced_failure');
}
const forcedIssue = typeof metadataFlags.forceIssueCode === 'string'
? metadataFlags.forceIssueCode
: typeof assetMetadata.forceIssueCode === 'string'
? assetMetadata.forceIssueCode
: '';
if (forcedIssue) issueCodes.push(forcedIssue);
const uniqueIssues = Array.from(new Set(issueCodes));
const highRiskIssues = new Set([
'INVALID_OBJECT_KEY',
'OBJECT_KEY_TENANT_PREFIX_REQUIRED',
'MIME_TYPE_NOT_ALLOWED',
'FILE_TOO_LARGE',
'security_scan_forced_failure',
]);
const riskLevel = uniqueIssues.some(code => highRiskIssues.has(code))
? 'high'
: uniqueIssues.length
? 'medium'
: 'none';
return {
status: uniqueIssues.length ? 'failed' : 'passed',
provider: 'metadata_rules',
riskLevel,
issueCodes: uniqueIssues,
details,
};
}
function eventError(error: unknown) {
if (error instanceof AssetWorkerError) {
return { code: error.code, message: error.message };
@@ -483,6 +623,30 @@ async function recordAudit(
);
}
async function recordSecurityScanEvent(
client: pg.PoolClient,
asset: AssetCandidate,
scan: SecurityScanResult,
) {
await client.query(
`
insert into public.content_asset_security_scan_events (
tenant_id, asset_id, provider, scan_status, risk_level, issue_codes, details
)
values ($1, $2, $3, $4, $5, $6::text[], $7::jsonb)
`,
[
asset.tenantId,
asset.id,
scan.provider,
scan.status,
scan.riskLevel,
scan.issueCodes,
JSON.stringify(scan.details),
],
);
}
async function claimAssetCandidates(client: pg.PoolClient, limit: number, claimId: string) {
const result = await client.query<AssetCandidate>(
`
@@ -496,6 +660,12 @@ async function claimAssetCandidates(client: pg.PoolClient, limit: number, claimI
(upload_status = 'pending' and updated_at <= now() - ($2::int * interval '1 second'))
or (
upload_status = 'verified'
and security_scan_status = 'pending'
and updated_at <= now() - ($2::int * interval '1 second')
)
or (
upload_status = 'verified'
and security_scan_status in ('passed', 'not_required')
and coalesce((verification_details #>> '{assetWorker,lastCheckedAt}')::timestamptz, verified_at, updated_at, created_at)
<= now() - ($3::int * interval '1 second')
)
@@ -514,6 +684,10 @@ async function claimAssetCandidates(client: pg.PoolClient, limit: number, claimI
coalesce(ca.verification_details->'assetWorker', '{}'::jsonb) || $6::jsonb,
true
),
security_scan_status = case
when ca.upload_status = 'verified' and ca.security_scan_status = 'pending' then 'scanning'
else ca.security_scan_status
end,
updated_at = now()
from candidates
where ca.id = candidates.id
@@ -522,15 +696,19 @@ async function claimAssetCandidates(client: pg.PoolClient, limit: number, claimI
ca.title,
ca.status,
ca.upload_status as "uploadStatus",
ca.asset_type as "assetType",
ca.storage_provider as "storageProvider",
ca.bucket,
ca.object_key as "objectKey",
ca.file_name as "fileName",
ca.mime_type as "mimeType",
ca.file_size_bytes as "fileSizeBytes",
ca.checksum_sha256 as "checksumSha256",
ca.verified_size_bytes as "verifiedSizeBytes",
ca.verified_checksum_sha256 as "verifiedChecksumSha256",
ca.verification_details as "verificationDetails",
ca.security_scan_status as "securityScanStatus",
ca.security_scan_summary as "securityScanSummary",
ca.security_flags as "securityFlags"
`,
[
@@ -594,6 +772,95 @@ async function markAssetVerified(client: pg.PoolClient, asset: AssetCandidate, m
});
}
async function markAssetSecurityScanPassed(
client: pg.PoolClient,
asset: AssetCandidate,
scan: SecurityScanResult,
) {
await client.query(
`
update public.content_assets
set security_scan_status = 'passed',
security_scanned_at = now(),
security_scan_provider = $3,
security_scan_summary = $4::jsonb,
security_flags = coalesce(security_flags, '{}'::jsonb) - 'assetSecurityScanFailed',
updated_at = now()
where tenant_id = $1 and id = $2
`,
[
asset.tenantId,
asset.id,
scan.provider,
JSON.stringify({
riskLevel: scan.riskLevel,
issueCodes: scan.issueCodes,
provider: scan.provider,
scannedAt: nowIso(),
details: scan.details,
}),
],
);
await recordSecurityScanEvent(client, asset, scan);
await recordAudit(client, {
tenantId: asset.tenantId,
action: 'content.asset.security_scan_passed',
targetId: asset.id,
details: {
provider: scan.provider,
riskLevel: scan.riskLevel,
issueCodes: scan.issueCodes,
},
});
}
async function markAssetSecurityScanFailed(
client: pg.PoolClient,
asset: AssetCandidate,
scan: SecurityScanResult,
) {
await client.query(
`
update public.content_assets
set security_scan_status = 'failed',
security_scanned_at = now(),
security_scan_provider = $3,
security_scan_summary = $4::jsonb,
status = case when status = 'active' then 'draft' else status end,
security_flags = coalesce(security_flags, '{}'::jsonb)
|| jsonb_build_object('assetSecurityScanFailed', true, 'assetSecurityScanFailedAt', now()),
updated_at = now()
where tenant_id = $1 and id = $2
`,
[
asset.tenantId,
asset.id,
scan.provider,
JSON.stringify({
riskLevel: scan.riskLevel,
issueCodes: scan.issueCodes,
provider: scan.provider,
scannedAt: nowIso(),
details: scan.details,
}),
],
);
await recordSecurityScanEvent(client, asset, scan);
await recordAudit(client, {
tenantId: asset.tenantId,
action: 'content.asset.security_scan_failed',
targetId: asset.id,
details: {
provider: scan.provider,
riskLevel: scan.riskLevel,
issueCodes: scan.issueCodes,
unpublished: asset.status === 'active',
},
});
}
async function markAssetFailed(
client: pg.PoolClient,
asset: AssetCandidate,
@@ -601,10 +868,11 @@ async function markAssetFailed(
metadata: StorageObjectMetadata | null,
error: Record<string, unknown> | null,
) {
const uniqueIssues = Array.from(new Set(issues));
const details = mergeVerificationDetails(asset, {
lastCheckedAt: nowIso(),
lastResult: 'failed',
issues,
issues: uniqueIssues,
observed: metadata
? {
fileSizeBytes: metadata.sizeBytes,
@@ -617,11 +885,33 @@ async function markAssetFailed(
: null,
error,
});
const scanSkipped: SecurityScanResult = {
status: 'skipped',
provider: 'metadata_rules',
riskLevel: 'medium',
issueCodes: uniqueIssues,
details: {
skippedReason: 'upload_recheck_failed',
observed: metadata
? {
fileSizeBytes: metadata.sizeBytes,
mimeType: canonicalMime(metadata.mimeType),
checksumSha256: metadata.checksumSha256,
verificationSource: metadata.verificationSource,
}
: null,
error,
},
};
await client.query(
`
update public.content_assets
set upload_status = 'failed',
security_scan_status = 'skipped',
security_scanned_at = null,
security_scan_provider = $4,
security_scan_summary = $5::jsonb,
status = case when status = 'active' then 'draft' else status end,
verification_details = $3::jsonb,
security_flags = coalesce(security_flags, '{}'::jsonb)
@@ -629,9 +919,22 @@ async function markAssetFailed(
updated_at = now()
where tenant_id = $1 and id = $2
`,
[asset.tenantId, asset.id, JSON.stringify(details)],
[
asset.tenantId,
asset.id,
JSON.stringify(details),
scanSkipped.provider,
JSON.stringify({
riskLevel: scanSkipped.riskLevel,
issueCodes: scanSkipped.issueCodes,
provider: scanSkipped.provider,
skippedReason: 'upload_recheck_failed',
details: scanSkipped.details,
}),
],
);
await recordSecurityScanEvent(client, asset, scanSkipped);
await recordAudit(client, {
tenantId: asset.tenantId,
action: 'content.asset.recheck_failed',
@@ -641,7 +944,7 @@ async function markAssetFailed(
bucket: asset.bucket,
objectKey: asset.objectKey,
result: 'failed',
issues,
issues: uniqueIssues,
error,
unpublished: asset.status === 'active',
},
@@ -653,6 +956,7 @@ async function processAsset(asset: AssetCandidate) {
try {
const metadata = await headStorageObject(asset);
const comparison = compareAssetMetadata(asset, metadata);
const scan = comparison.issues.length ? null : securityScanRules(asset, metadata);
await client.query('begin');
if (comparison.issues.length) {
await markAssetFailed(client, asset, comparison.issues, metadata, null);
@@ -660,6 +964,14 @@ async function processAsset(asset: AssetCandidate) {
return 'failed';
}
await markAssetVerified(client, asset, metadata, comparison);
if (scan?.status === 'failed') {
await markAssetSecurityScanFailed(client, asset, scan);
await client.query('commit');
return 'failed';
}
if (scan?.status === 'passed') {
await markAssetSecurityScanPassed(client, asset, scan);
}
await client.query('commit');
return 'verified';
} catch (error) {

14
apps/worker/src/jobs/exports.ts
View File

@@ -208,6 +208,8 @@ async function insertExportAsset(
asset_type, storage_provider, bucket, object_key, mime_type,
file_size_bytes, checksum_sha256, upload_status, verified_at,
verified_size_bytes, verified_checksum_sha256, verification_details,
security_scan_status, security_scanned_at, security_scan_provider,
security_scan_summary,
preview_status, visibility, is_public, status, access_rules,
metadata, created_by, updated_by, source
)
@@ -216,8 +218,10 @@ async function insertExportAsset(
$7, $8, $9, $10, $11,
$12, $13, 'verified', now(),
$12, $13, $14::jsonb,
$15, $16, $17, $18, $19::jsonb,
$20::jsonb, $21, $21, 'content_export_worker'
'passed', now(), 'trusted_export_worker',
$15::jsonb,
$16, $17, $18, $19, $20::jsonb,
$21::jsonb, $22, $22, 'content_export_worker'
)
returning id
`,
@@ -242,6 +246,12 @@ async function insertExportAsset(
checkedAt: new Date().toISOString(),
},
}),
JSON.stringify({
riskLevel: 'none',
issueCodes: [],
provider: 'trusted_export_worker',
generatedBy: 'content_export_worker',
}),
input.extension === 'pdf' ? 'ready' : 'none',
visibility,
visibility === 'public',