xiongyuxing/gongxue-base
1
0
Fork 0
forked from wangziqi/gongxue-base
Code Pull Requests Activity

feat: add content asset security scanning

Browse Source
This commit is contained in:
Codex
2026-06-29 19:42:05 +08:00
parent d696fc38b0
commit ecd269b548
21 changed files with 936 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

318
apps/worker/src/jobs/assets.ts
View File

@@ -11,15 +11,19 @@ interface AssetCandidate {
title: string | null;
status: string;
uploadStatus: string;
assetType: string;
storageProvider: StorageProviderName;
bucket: string | null;
objectKey: string | null;
fileName: string | null;
mimeType: string | null;
fileSizeBytes: number | string | null;
checksumSha256: string | null;
verifiedSizeBytes: number | string | null;
verifiedChecksumSha256: string | null;
verificationDetails: Record<string, unknown>;
securityScanStatus: string;
securityScanSummary: Record<string, unknown>;
securityFlags: Record<string, unknown>;
}
@@ -45,6 +49,14 @@ interface AssetWorkerResult {
errors: number;
}
interface SecurityScanResult {
status: 'passed' | 'failed' | 'skipped';
provider: string;
riskLevel: 'none' | 'low' | 'medium' | 'high' | 'critical';
issueCodes: string[];
details: Record<string, unknown>;
}
class AssetWorkerError extends Error {
readonly code: string;
@@ -56,6 +68,30 @@ class AssetWorkerError extends Error {
const UPLOADABLE_PROVIDERS = new Set<StorageProviderName>(['local_dev', 'supabase_storage', 'aliyun_oss', 'tencent_cos']);
const SAFE_OBJECT_KEY_RE = /^[A-Za-z0-9][A-Za-z0-9._~!$&'()+,;=@/-]{0,1023}$/;
const EXTENSION_MIME_HINTS: Record<string, string[]> = {
pdf: ['application/pdf'],
png: ['image/png'],
jpg: ['image/jpeg'],
jpeg: ['image/jpeg'],
gif: ['image/gif'],
webp: ['image/webp'],
svg: ['image/svg+xml'],
mp4: ['video/mp4'],
mov: ['video/quicktime'],
mp3: ['audio/mpeg'],
wav: ['audio/wav', 'audio/x-wav'],
doc: ['application/msword'],
docx: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
xls: ['application/vnd.ms-excel'],
xlsx: ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
ppt: ['application/vnd.ms-powerpoint'],
pptx: ['application/vnd.openxmlformats-officedocument.presentationml.presentation'],
zip: ['application/zip', 'application/x-zip-compressed'],
json: ['application/json'],
txt: ['text/plain'],
md: ['text/markdown', 'text/plain'],
csv: ['text/csv', 'text/plain'],
};
function objectValue(value: unknown): Record<string, unknown> {
return value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {};
@@ -445,6 +481,110 @@ function compareAssetMetadata(asset: AssetCandidate, metadata: StorageObjectMeta
};
}
function extensionFromName(value: string | null) {
if (!value) return '';
const withoutQuery = value.split('?')[0]?.split('#')[0] || value;
const last = withoutQuery.split('/').pop() || withoutQuery;
const dot = last.lastIndexOf('.');
return dot >= 0 ? last.slice(dot + 1).trim().toLowerCase() : '';
}
function metadataBoolean(value: unknown) {
return value === true || value === 'true' || value === 1 || value === '1';
}
function securityScanRules(asset: AssetCandidate, metadata: StorageObjectMetadata): SecurityScanResult {
const issueCodes: string[] = [];
const details: Record<string, unknown> = {
observed: {
fileSizeBytes: metadata.sizeBytes,
mimeType: canonicalMime(metadata.mimeType),
checksumSha256: metadata.checksumSha256,
verificationSource: metadata.verificationSource,
},
declared: {
assetType: asset.assetType,
fileName: asset.fileName,
objectKey: asset.objectKey,
mimeType: canonicalMime(asset.mimeType),
fileSizeBytes: validateFileSize(asset.fileSizeBytes),
checksumSha256: asset.checksumSha256,
},
};
try {
if (asset.objectKey) validateObjectKey(asset.tenantId, asset.objectKey);
} catch (error) {
issueCodes.push(eventError(error).code);
}
try {
validateFileSize(metadata.sizeBytes ?? asset.verifiedSizeBytes ?? asset.fileSizeBytes);
} catch (error) {
issueCodes.push(eventError(error).code);
}
const declaredMime = canonicalMime(asset.mimeType);
const observedMime = canonicalMime(metadata.mimeType);
const effectiveMime = observedMime || declaredMime;
try {
validateMimeType(effectiveMime);
} catch (error) {
issueCodes.push(eventError(error).code);
}
const fileExtension = extensionFromName(asset.fileName) || extensionFromName(asset.objectKey);
details.fileExtension = fileExtension || null;
if (fileExtension && effectiveMime && EXTENSION_MIME_HINTS[fileExtension]) {
const allowedForExtension = EXTENSION_MIME_HINTS[fileExtension];
if (!allowedForExtension.includes(effectiveMime)) {
issueCodes.push('file_extension_mime_mismatch');
details.extensionMimeExpected = allowedForExtension;
}
}
const metadataFlags = objectValue(asset.securityScanSummary);
const assetMetadata = objectValue(asset.verificationDetails?.metadata);
const explicitFlag = [
metadataFlags.forceFail,
metadataFlags.securityScanForceFail,
assetMetadata.securityScanForceFail,
assetMetadata.forceSecurityScanFail,
].some(metadataBoolean);
if (explicitFlag) {
issueCodes.push('security_scan_forced_failure');
}
const forcedIssue = typeof metadataFlags.forceIssueCode === 'string'
? metadataFlags.forceIssueCode
: typeof assetMetadata.forceIssueCode === 'string'
? assetMetadata.forceIssueCode
: '';
if (forcedIssue) issueCodes.push(forcedIssue);
const uniqueIssues = Array.from(new Set(issueCodes));
const highRiskIssues = new Set([
'INVALID_OBJECT_KEY',
'OBJECT_KEY_TENANT_PREFIX_REQUIRED',
'MIME_TYPE_NOT_ALLOWED',
'FILE_TOO_LARGE',
'security_scan_forced_failure',
]);
const riskLevel = uniqueIssues.some(code => highRiskIssues.has(code))
? 'high'
: uniqueIssues.length
? 'medium'
: 'none';
return {
status: uniqueIssues.length ? 'failed' : 'passed',
provider: 'metadata_rules',
riskLevel,
issueCodes: uniqueIssues,
details,
};
}
function eventError(error: unknown) {
if (error instanceof AssetWorkerError) {
return { code: error.code, message: error.message };
@@ -483,6 +623,30 @@ async function recordAudit(
);
}
async function recordSecurityScanEvent(
client: pg.PoolClient,
asset: AssetCandidate,
scan: SecurityScanResult,
) {
await client.query(
`
insert into public.content_asset_security_scan_events (
tenant_id, asset_id, provider, scan_status, risk_level, issue_codes, details
)
values ($1, $2, $3, $4, $5, $6::text[], $7::jsonb)
`,
[
asset.tenantId,
asset.id,
scan.provider,
scan.status,
scan.riskLevel,
scan.issueCodes,
JSON.stringify(scan.details),
],
);
}
async function claimAssetCandidates(client: pg.PoolClient, limit: number, claimId: string) {
const result = await client.query<AssetCandidate>(
`
@@ -496,6 +660,12 @@ async function claimAssetCandidates(client: pg.PoolClient, limit: number, claimI
(upload_status = 'pending' and updated_at <= now() - ($2::int * interval '1 second'))
or (
upload_status = 'verified'
and security_scan_status = 'pending'
and updated_at <= now() - ($2::int * interval '1 second')
)
or (
upload_status = 'verified'
and security_scan_status in ('passed', 'not_required')
and coalesce((verification_details #>> '{assetWorker,lastCheckedAt}')::timestamptz, verified_at, updated_at, created_at)
<= now() - ($3::int * interval '1 second')
)
@@ -514,6 +684,10 @@ async function claimAssetCandidates(client: pg.PoolClient, limit: number, claimI
coalesce(ca.verification_details->'assetWorker', '{}'::jsonb) || $6::jsonb,
true
),
security_scan_status = case
when ca.upload_status = 'verified' and ca.security_scan_status = 'pending' then 'scanning'
else ca.security_scan_status
end,
updated_at = now()
from candidates
where ca.id = candidates.id
@@ -522,15 +696,19 @@ async function claimAssetCandidates(client: pg.PoolClient, limit: number, claimI
ca.title,
ca.status,
ca.upload_status as "uploadStatus",
ca.asset_type as "assetType",
ca.storage_provider as "storageProvider",
ca.bucket,
ca.object_key as "objectKey",
ca.file_name as "fileName",
ca.mime_type as "mimeType",
ca.file_size_bytes as "fileSizeBytes",
ca.checksum_sha256 as "checksumSha256",
ca.verified_size_bytes as "verifiedSizeBytes",
ca.verified_checksum_sha256 as "verifiedChecksumSha256",
ca.verification_details as "verificationDetails",
ca.security_scan_status as "securityScanStatus",
ca.security_scan_summary as "securityScanSummary",
ca.security_flags as "securityFlags"
`,
[
@@ -594,6 +772,95 @@ async function markAssetVerified(client: pg.PoolClient, asset: AssetCandidate, m
});
}
async function markAssetSecurityScanPassed(
client: pg.PoolClient,
asset: AssetCandidate,
scan: SecurityScanResult,
) {
await client.query(
`
update public.content_assets
set security_scan_status = 'passed',
security_scanned_at = now(),
security_scan_provider = $3,
security_scan_summary = $4::jsonb,
security_flags = coalesce(security_flags, '{}'::jsonb) - 'assetSecurityScanFailed',
updated_at = now()
where tenant_id = $1 and id = $2
`,
[
asset.tenantId,
asset.id,
scan.provider,
JSON.stringify({
riskLevel: scan.riskLevel,
issueCodes: scan.issueCodes,
provider: scan.provider,
scannedAt: nowIso(),
details: scan.details,
}),
],
);
await recordSecurityScanEvent(client, asset, scan);
await recordAudit(client, {
tenantId: asset.tenantId,
action: 'content.asset.security_scan_passed',
targetId: asset.id,
details: {
provider: scan.provider,
riskLevel: scan.riskLevel,
issueCodes: scan.issueCodes,
},
});
}
async function markAssetSecurityScanFailed(
client: pg.PoolClient,
asset: AssetCandidate,
scan: SecurityScanResult,
) {
await client.query(
`
update public.content_assets
set security_scan_status = 'failed',
security_scanned_at = now(),
security_scan_provider = $3,
security_scan_summary = $4::jsonb,
status = case when status = 'active' then 'draft' else status end,
security_flags = coalesce(security_flags, '{}'::jsonb)
|| jsonb_build_object('assetSecurityScanFailed', true, 'assetSecurityScanFailedAt', now()),
updated_at = now()
where tenant_id = $1 and id = $2
`,
[
asset.tenantId,
asset.id,
scan.provider,
JSON.stringify({
riskLevel: scan.riskLevel,
issueCodes: scan.issueCodes,
provider: scan.provider,
scannedAt: nowIso(),
details: scan.details,
}),
],
);
await recordSecurityScanEvent(client, asset, scan);
await recordAudit(client, {
tenantId: asset.tenantId,
action: 'content.asset.security_scan_failed',
targetId: asset.id,
details: {
provider: scan.provider,
riskLevel: scan.riskLevel,
issueCodes: scan.issueCodes,
unpublished: asset.status === 'active',
},
});
}
async function markAssetFailed(
client: pg.PoolClient,
asset: AssetCandidate,
@@ -601,10 +868,11 @@ async function markAssetFailed(
metadata: StorageObjectMetadata | null,
error: Record<string, unknown> | null,
) {
const uniqueIssues = Array.from(new Set(issues));
const details = mergeVerificationDetails(asset, {
lastCheckedAt: nowIso(),
lastResult: 'failed',
issues,
issues: uniqueIssues,
observed: metadata
? {
fileSizeBytes: metadata.sizeBytes,
@@ -617,11 +885,33 @@ async function markAssetFailed(
: null,
error,
});
const scanSkipped: SecurityScanResult = {
status: 'skipped',
provider: 'metadata_rules',
riskLevel: 'medium',
issueCodes: uniqueIssues,
details: {
skippedReason: 'upload_recheck_failed',
observed: metadata
? {
fileSizeBytes: metadata.sizeBytes,
mimeType: canonicalMime(metadata.mimeType),
checksumSha256: metadata.checksumSha256,
verificationSource: metadata.verificationSource,
}
: null,
error,
},
};
await client.query(
`
update public.content_assets
set upload_status = 'failed',
security_scan_status = 'skipped',
security_scanned_at = null,
security_scan_provider = $4,
security_scan_summary = $5::jsonb,
status = case when status = 'active' then 'draft' else status end,
verification_details = $3::jsonb,
security_flags = coalesce(security_flags, '{}'::jsonb)
@@ -629,9 +919,22 @@ async function markAssetFailed(
updated_at = now()
where tenant_id = $1 and id = $2
`,
[asset.tenantId, asset.id, JSON.stringify(details)],
[
asset.tenantId,
asset.id,
JSON.stringify(details),
scanSkipped.provider,
JSON.stringify({
riskLevel: scanSkipped.riskLevel,
issueCodes: scanSkipped.issueCodes,
provider: scanSkipped.provider,
skippedReason: 'upload_recheck_failed',
details: scanSkipped.details,
}),
],
);
await recordSecurityScanEvent(client, asset, scanSkipped);
await recordAudit(client, {
tenantId: asset.tenantId,
action: 'content.asset.recheck_failed',
@@ -641,7 +944,7 @@ async function markAssetFailed(
bucket: asset.bucket,
objectKey: asset.objectKey,
result: 'failed',
issues,
issues: uniqueIssues,
error,
unpublished: asset.status === 'active',
},
@@ -653,6 +956,7 @@ async function processAsset(asset: AssetCandidate) {
try {
const metadata = await headStorageObject(asset);
const comparison = compareAssetMetadata(asset, metadata);
const scan = comparison.issues.length ? null : securityScanRules(asset, metadata);
await client.query('begin');
if (comparison.issues.length) {
await markAssetFailed(client, asset, comparison.issues, metadata, null);
@@ -660,6 +964,14 @@ async function processAsset(asset: AssetCandidate) {
return 'failed';
}
await markAssetVerified(client, asset, metadata, comparison);
if (scan?.status === 'failed') {
await markAssetSecurityScanFailed(client, asset, scan);
await client.query('commit');
return 'failed';
}
if (scan?.status === 'passed') {
await markAssetSecurityScanPassed(client, asset, scan);
}
await client.query('commit');
return 'verified';
} catch (error) {

14
apps/worker/src/jobs/exports.ts
View File

@@ -208,6 +208,8 @@ async function insertExportAsset(
asset_type, storage_provider, bucket, object_key, mime_type,
file_size_bytes, checksum_sha256, upload_status, verified_at,
verified_size_bytes, verified_checksum_sha256, verification_details,
security_scan_status, security_scanned_at, security_scan_provider,
security_scan_summary,
preview_status, visibility, is_public, status, access_rules,
metadata, created_by, updated_by, source
)
@@ -216,8 +218,10 @@ async function insertExportAsset(
$7, $8, $9, $10, $11,
$12, $13, 'verified', now(),
$12, $13, $14::jsonb,
$15, $16, $17, $18, $19::jsonb,
$20::jsonb, $21, $21, 'content_export_worker'
'passed', now(), 'trusted_export_worker',
$15::jsonb,
$16, $17, $18, $19, $20::jsonb,
$21::jsonb, $22, $22, 'content_export_worker'
)
returning id
`,
@@ -242,6 +246,12 @@ async function insertExportAsset(
checkedAt: new Date().toISOString(),
},
}),
JSON.stringify({
riskLevel: 'none',
issueCodes: [],
provider: 'trusted_export_worker',
generatedBy: 'content_export_worker',
}),
input.extension === 'pdf' ? 'ready' : 'none',
visibility,
visibility === 'public',