xiongyuxing/gongxue-base
1
0
Fork 0
forked from wangziqi/gongxue-base
Code Pull Requests Activity

feat: add external asset security scanner

Browse Source
This commit is contained in:
Codex
2026-06-29 20:00:17 +08:00
parent ecd269b548
commit ce538a57c7
17 changed files with 596 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
apps/worker/src/config.ts
View File

@@ -17,6 +17,11 @@ export interface WorkerConfig {
assetMinAgeSeconds: number;
assetRecheckIntervalSeconds: number;
assetRequestTimeoutMs: number;
assetSecurityScanner: string;
assetSecurityScanHttpEndpoint: string;
assetSecurityScanHttpToken: string;
assetSecurityScanHttpTimeoutMs: number;
assetSecurityScanFailOpen: boolean;
importBatchSize: number;
importWorkerId: string;
importBackoffSeconds: number[];
@@ -68,6 +73,11 @@ export const config: WorkerConfig = {
assetMinAgeSeconds: envNumber('WORKER_ASSET_MIN_AGE_SECONDS', 300),
assetRecheckIntervalSeconds: envNumber('WORKER_ASSET_RECHECK_INTERVAL_SECONDS', 60 * 60 * 24),
assetRequestTimeoutMs: envNumber('WORKER_ASSET_REQUEST_TIMEOUT_MS', 10_000),
assetSecurityScanner: envString('WORKER_ASSET_SECURITY_SCANNER', 'metadata_rules'),
assetSecurityScanHttpEndpoint: envString('WORKER_ASSET_SECURITY_SCAN_HTTP_ENDPOINT', ''),
assetSecurityScanHttpToken: envString('WORKER_ASSET_SECURITY_SCAN_HTTP_TOKEN', ''),
assetSecurityScanHttpTimeoutMs: envNumber('WORKER_ASSET_SECURITY_SCAN_HTTP_TIMEOUT_MS', 10_000),
assetSecurityScanFailOpen: envBoolean('WORKER_ASSET_SECURITY_SCAN_FAIL_OPEN', false),
importBatchSize: envNumber('WORKER_IMPORT_BATCH_SIZE', 5),
importWorkerId: envString('WORKER_IMPORT_ID', `imports-${process.pid}`),
importBackoffSeconds: envList('WORKER_IMPORT_BACKOFF_SECONDS', '30,120,600,1800')

226
apps/worker/src/jobs/assets.ts
View File

@@ -493,6 +493,83 @@ function metadataBoolean(value: unknown) {
return value === true || value === 'true' || value === 1 || value === '1';
}
const RISK_LEVEL_ORDER: Record<SecurityScanResult['riskLevel'], number> = {
none: 0,
low: 1,
medium: 2,
high: 3,
critical: 4,
};
function normalizeRiskLevel(value: unknown, fallback: SecurityScanResult['riskLevel'] = 'high'): SecurityScanResult['riskLevel'] {
const normalized = typeof value === 'string' ? value.trim().toLowerCase() : '';
if (normalized === 'none' || normalized === 'low' || normalized === 'medium' || normalized === 'high' || normalized === 'critical') {
return normalized;
}
return fallback;
}
function highestRiskLevel(results: SecurityScanResult[]): SecurityScanResult['riskLevel'] {
return results.reduce<SecurityScanResult['riskLevel']>((highest, result) => (
RISK_LEVEL_ORDER[result.riskLevel] > RISK_LEVEL_ORDER[highest] ? result.riskLevel : highest
), 'none');
}
function normalizeIssueCodes(value: unknown, fallback: string[] = []) {
const rawItems = Array.isArray(value) ? value : fallback;
return Array.from(new Set(
rawItems
.map(item => String(item || '').trim())
.filter(Boolean)
.map(item => item.replace(/[^A-Za-z0-9_.:-]/g, '_').slice(0, 96))
.filter(Boolean),
));
}
function scannerModes() {
return Array.from(new Set(
config.assetSecurityScanner
.split(',')
.map(item => item.trim().toLowerCase())
.filter(Boolean),
));
}
function sanitizeProviderDetails(value: unknown, depth = 0): unknown {
if (depth > 5) return '[max_depth]';
if (value === null || value === undefined) return value;
if (typeof value === 'string') return value.length > 2000 ? `${value.slice(0, 2000)}...` : value;
if (typeof value === 'number' || typeof value === 'boolean') return value;
if (Array.isArray(value)) return value.slice(0, 50).map(item => sanitizeProviderDetails(item, depth + 1));
if (typeof value !== 'object') return String(value);
const sanitized: Record<string, unknown> = {};
for (const [key, child] of Object.entries(value as Record<string, unknown>).slice(0, 100)) {
const normalizedKey = key.toLowerCase().replace(/[-_\s]/g, '');
if (
normalizedKey.includes('secret') ||
normalizedKey.includes('token') ||
normalizedKey.includes('password') ||
normalizedKey.includes('privatekey') ||
normalizedKey.includes('apikey') ||
normalizedKey.includes('authorization')
) {
sanitized[key] = '[redacted]';
continue;
}
sanitized[key] = sanitizeProviderDetails(child, depth + 1);
}
return sanitized;
}
function endpointHost(value: string) {
try {
return new URL(value).host;
} catch {
return 'invalid_endpoint';
}
}
function securityScanRules(asset: AssetCandidate, metadata: StorageObjectMetadata): SecurityScanResult {
const issueCodes: string[] = [];
const details: Record<string, unknown> = {
@@ -585,6 +662,153 @@ function securityScanRules(asset: AssetCandidate, metadata: StorageObjectMetadat
};
}
function mergeSecurityScanResults(results: SecurityScanResult[]): SecurityScanResult {
const uniqueIssueCodes = Array.from(new Set(results.flatMap(result => result.issueCodes)));
const failed = results.find(result => result.status === 'failed');
const skipped = results.find(result => result.status === 'skipped');
return {
status: failed ? 'failed' : skipped ? 'skipped' : 'passed',
provider: results.map(result => result.provider).join('+'),
riskLevel: highestRiskLevel(results),
issueCodes: uniqueIssueCodes,
details: {
results: results.map(result => ({
provider: result.provider,
status: result.status,
riskLevel: result.riskLevel,
issueCodes: result.issueCodes,
details: result.details,
})),
},
};
}
async function externalHttpSecurityScan(
asset: AssetCandidate,
metadata: StorageObjectMetadata,
metadataRulesResult: SecurityScanResult,
): Promise<SecurityScanResult> {
const endpoint = config.assetSecurityScanHttpEndpoint.trim();
const provider = 'http_security_scan';
const unavailableIssue = ['external_security_scan_unavailable'];
const unavailableDetails = (error: unknown) => ({
endpointHost: endpoint ? endpointHost(endpoint) : null,
failOpen: config.assetSecurityScanFailOpen,
error: eventError(error),
});
if (!endpoint) {
return {
status: config.assetSecurityScanFailOpen ? 'passed' : 'failed',
provider,
riskLevel: config.assetSecurityScanFailOpen ? 'low' : 'high',
issueCodes: unavailableIssue,
details: unavailableDetails(new AssetWorkerError('WORKER_ASSET_SECURITY_SCAN_HTTP_ENDPOINT is required', 'ASSET_SECURITY_SCAN_HTTP_ENDPOINT_REQUIRED')),
};
}
const payload = {
assetId: asset.id,
tenantId: asset.tenantId,
assetType: asset.assetType,
storageProvider: asset.storageProvider,
bucket: asset.bucket,
objectKey: asset.objectKey,
fileName: asset.fileName,
mimeType: canonicalMime(metadata.mimeType) || canonicalMime(asset.mimeType),
fileSizeBytes: metadata.sizeBytes ?? normalizeSafeInteger(asset.fileSizeBytes),
checksumSha256: metadata.checksumSha256 || asset.verifiedChecksumSha256 || asset.checksumSha256,
metadata: {
observed: {
fileSizeBytes: metadata.sizeBytes,
mimeType: canonicalMime(metadata.mimeType),
checksumSha256: metadata.checksumSha256,
etag: metadata.etag,
lastModified: metadata.lastModified,
verificationSource: metadata.verificationSource,
},
declared: objectValue(metadataRulesResult.details).declared || {},
},
requestedAt: nowIso(),
};
try {
const response = await fetchWithTimeout(endpoint, {
method: 'POST',
headers: {
'content-type': 'application/json',
accept: 'application/json',
...(config.assetSecurityScanHttpToken ? { authorization: `Bearer ${config.assetSecurityScanHttpToken}` } : {}),
},
body: JSON.stringify(payload),
}, config.assetSecurityScanHttpTimeoutMs);
if (!response.ok) {
throw new AssetWorkerError(`External security scanner returned HTTP ${response.status}`, 'ASSET_SECURITY_SCAN_HTTP_STATUS');
}
const body = await response.json() as Record<string, unknown>;
const status = typeof body.status === 'string' ? body.status.trim().toLowerCase() : '';
if (status !== 'passed' && status !== 'failed') {
throw new AssetWorkerError('External security scanner returned invalid status', 'ASSET_SECURITY_SCAN_INVALID_RESPONSE');
}
const bodyIssueCodes = normalizeIssueCodes(body.issueCodes, status === 'failed' ? ['external_security_scan_failed'] : []);
return {
status,
provider,
riskLevel: normalizeRiskLevel(body.riskLevel, status === 'failed' ? 'high' : 'none'),
issueCodes: bodyIssueCodes,
details: {
endpointHost: endpointHost(endpoint),
httpStatus: response.status,
scannerProvider: typeof body.provider === 'string' ? body.provider.slice(0, 120) : null,
scannerDetails: sanitizeProviderDetails(body.details || {}),
},
};
} catch (error) {
return {
status: config.assetSecurityScanFailOpen ? 'passed' : 'failed',
provider,
riskLevel: config.assetSecurityScanFailOpen ? 'low' : 'high',
issueCodes: unavailableIssue,
details: unavailableDetails(error),
};
}
}
async function runSecurityScans(asset: AssetCandidate, metadata: StorageObjectMetadata): Promise<SecurityScanResult> {
const metadataRulesResult = securityScanRules(asset, metadata);
if (metadataRulesResult.status !== 'passed') {
return metadataRulesResult;
}
const modes = scannerModes();
if (modes.length === 0 || (modes.length === 1 && modes[0] === 'metadata_rules')) {
return metadataRulesResult;
}
const unsupportedModes = modes.filter(mode => mode !== 'metadata_rules' && mode !== 'http');
if (unsupportedModes.length > 0) {
return mergeSecurityScanResults([
metadataRulesResult,
{
status: 'failed',
provider: 'asset_security_scanner_config',
riskLevel: 'high',
issueCodes: ['asset_security_scanner_unsupported'],
details: { configuredModes: unsupportedModes },
},
]);
}
const results = [metadataRulesResult];
if (modes.includes('http')) {
results.push(await externalHttpSecurityScan(asset, metadata, metadataRulesResult));
}
return mergeSecurityScanResults(results);
}
function eventError(error: unknown) {
if (error instanceof AssetWorkerError) {
return { code: error.code, message: error.message };
@@ -956,7 +1180,7 @@ async function processAsset(asset: AssetCandidate) {
try {
const metadata = await headStorageObject(asset);
const comparison = compareAssetMetadata(asset, metadata);
const scan = comparison.issues.length ? null : securityScanRules(asset, metadata);
const scan = comparison.issues.length ? null : await runSecurityScans(asset, metadata);
await client.query('begin');
if (comparison.issues.length) {
await markAssetFailed(client, asset, comparison.issues, metadata, null);