xiongyuxing/gongxue-base
1
0
Fork 0
forked from wangziqi/gongxue-base
Code Pull Requests Activity

feat: add external asset security scanner

Browse Source
This commit is contained in:
Codex
2026-06-29 20:00:17 +08:00
parent ecd269b548
commit ce538a57c7
17 changed files with 596 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

238
scripts/asset-worker-integration-test.js
View File

@@ -1,6 +1,7 @@
import assert from 'node:assert/strict';
import pg from 'pg';
import { spawn } from 'node:child_process';
import http from 'node:http';
const databaseUrl = process.env.DATABASE_URL || 'postgresql://postgres:postgres@127.0.0.1:54322/postgres';
const tenantId = '00000000-0000-0000-0000-000000000001';
@@ -9,12 +10,15 @@ const ids = {
okAsset: '20000000-0000-0000-0000-000000000801',
badAsset: '20000000-0000-0000-0000-000000000802',
scanBadAsset: '20000000-0000-0000-0000-000000000803',
httpPassAsset: '20000000-0000-0000-0000-000000000804',
httpFailAsset: '20000000-0000-0000-0000-000000000805',
httpUnavailableAsset: '20000000-0000-0000-0000-000000000806',
};
const checksumA = 'a'.repeat(64);
const checksumB = 'b'.repeat(64);
async function runWorkerOnce() {
async function runWorkerOnce(extraEnv = {}) {
const child = spawn(process.execPath, ['apps/worker/dist/apps/worker/src/index.js', '--once', '--job', 'assets'], {
cwd: process.cwd(),
env: {
@@ -22,9 +26,15 @@ async function runWorkerOnce() {
DATABASE_URL: databaseUrl,
WORKER_ASSET_BATCH_SIZE: '20',
WORKER_ASSET_MIN_AGE_SECONDS: '0',
WORKER_ASSET_RECHECK_INTERVAL_SECONDS: '0',
WORKER_ASSET_RECHECK_INTERVAL_SECONDS: '86400',
WORKER_ASSET_REQUEST_TIMEOUT_MS: '5000',
WORKER_ASSET_SECURITY_SCANNER: 'metadata_rules',
WORKER_ASSET_SECURITY_SCAN_HTTP_ENDPOINT: '',
WORKER_ASSET_SECURITY_SCAN_HTTP_TOKEN: '',
WORKER_ASSET_SECURITY_SCAN_HTTP_TIMEOUT_MS: '10000',
WORKER_ASSET_SECURITY_SCAN_FAIL_OPEN: 'false',
STORAGE_REQUIRE_TENANT_PREFIX: 'true',
...extraEnv,
},
stdio: ['ignore', 'pipe', 'pipe'],
windowsHide: true,
@@ -42,6 +52,54 @@ async function runWorkerOnce() {
return output;
}
async function startScannerServer() {
const requests = [];
const server = http.createServer((request, response) => {
let body = '';
request.on('data', chunk => {
body += chunk.toString();
});
request.on('end', () => {
const payload = JSON.parse(body || '{}');
requests.push({
authorization: request.headers.authorization,
payload,
});
response.setHeader('content-type', 'application/json');
if (payload.assetId === ids.httpFailAsset) {
response.end(JSON.stringify({
status: 'failed',
riskLevel: 'critical',
issueCodes: ['virus_detected'],
details: {
engine: 'fake-av',
signature: 'EICAR-Test-File',
accessToken: 'must-be-redacted',
},
}));
return;
}
response.end(JSON.stringify({
status: 'passed',
riskLevel: 'none',
issueCodes: [],
details: {
engine: 'fake-av',
scannedBytes: payload.fileSizeBytes,
},
}));
});
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
const address = server.address();
assert.ok(address && typeof address === 'object', 'scanner server should listen on a TCP port');
return {
endpoint: `http://127.0.0.1:${address.port}/scan`,
requests,
close: () => new Promise(resolve => server.close(resolve)),
};
}
function countFromWorkerOutput(output, name) {
const match = output.match(new RegExp(`${name}=(\\d+)`));
return match ? Number(match[1]) : 0;
@@ -53,23 +111,23 @@ async function cleanup(pool) {
delete from public.audit_logs
where tenant_id = $1
and target_type = 'content_asset'
and target_id in ($2, $3, $4)
and target_id = any($2::text[])
`,
[tenantId, ids.okAsset, ids.badAsset, ids.scanBadAsset],
[tenantId, Object.values(ids)],
);
await pool.query(
`
delete from public.content_asset_security_scan_events
where tenant_id = $1 and asset_id in ($2::uuid, $3::uuid, $4::uuid)
where tenant_id = $1 and asset_id = any($2::uuid[])
`,
[tenantId, ids.okAsset, ids.badAsset, ids.scanBadAsset],
[tenantId, Object.values(ids)],
);
await pool.query(
`
delete from public.content_assets
where tenant_id = $1 and id in ($2::uuid, $3::uuid, $4::uuid)
where tenant_id = $1 and id = any($2::uuid[])
`,
[tenantId, ids.okAsset, ids.badAsset, ids.scanBadAsset],
[tenantId, Object.values(ids)],
);
}
@@ -112,6 +170,33 @@ async function seed(pool) {
'pending', 'metadata_rules', '{"securityScanForceFail":true}'::jsonb,
'{}'::jsonb, 'integration-test',
now() - interval '2 days', now() - interval '2 days'
),
(
$10, $2, 'asset-worker-http-pass', '外部扫描正常 PDF', 'pdf', 'local_dev',
'tenant-assets', $11, 'http-pass.pdf', 'application/pdf', 4096, $4,
'tenant', 'active', 'verified', now() - interval '2 days', 4096,
$4, '{"source":"asset-worker-test"}'::jsonb,
'pending', 'metadata_rules', '{}'::jsonb,
'{}'::jsonb, 'integration-test',
now() - interval '2 days', now() - interval '2 days'
),
(
$12, $2, 'asset-worker-http-fail', '外部扫描异常 PDF', 'pdf', 'local_dev',
'tenant-assets', $13, 'http-fail.pdf', 'application/pdf', 4096, $4,
'tenant', 'active', 'verified', now() - interval '2 days', 4096,
$4, '{"source":"asset-worker-test"}'::jsonb,
'pending', 'metadata_rules', '{}'::jsonb,
'{}'::jsonb, 'integration-test',
now() - interval '2 days', now() - interval '2 days'
),
(
$14, $2, 'asset-worker-http-unavailable', '外部扫描不可用 PDF', 'pdf', 'local_dev',
'tenant-assets', $15, 'http-unavailable.pdf', 'application/pdf', 4096, $4,
'tenant', 'active', 'verified', now() - interval '2 days', 4096,
$4, '{"source":"asset-worker-test"}'::jsonb,
'pending', 'metadata_rules', '{}'::jsonb,
'{}'::jsonb, 'integration-test',
now() - interval '2 days', now() - interval '2 days'
)
`,
[
@@ -124,10 +209,43 @@ async function seed(pool) {
checksumB,
ids.scanBadAsset,
`${tenantId}/assets/worker-scan-bad.pdf`,
ids.httpPassAsset,
`${tenantId}/assets/worker-http-pass.pdf`,
ids.httpFailAsset,
`${tenantId}/assets/worker-http-fail.pdf`,
ids.httpUnavailableAsset,
`${tenantId}/assets/worker-http-unavailable.pdf`,
],
);
}
async function resetAssetsForHttp(pool) {
await pool.query(
`
update public.content_assets
set status = 'active',
upload_status = 'verified',
verified_at = now() - interval '2 days',
verification_details = '{"source":"asset-worker-test"}'::jsonb,
security_scan_status = 'pending',
security_scan_provider = 'metadata_rules',
security_scan_summary = '{}'::jsonb,
security_flags = '{}'::jsonb,
updated_at = now() - interval '2 days'
where tenant_id = $1
and id = any($2::uuid[])
`,
[tenantId, [ids.httpPassAsset, ids.httpFailAsset, ids.httpUnavailableAsset]],
);
await pool.query(
`
delete from public.content_asset_security_scan_events
where tenant_id = $1 and asset_id = any($2::uuid[])
`,
[tenantId, [ids.httpPassAsset, ids.httpFailAsset, ids.httpUnavailableAsset]],
);
}
async function main() {
const pool = new pg.Pool({ connectionString: databaseUrl });
let seeded = false;
@@ -139,7 +257,7 @@ async function main() {
seeded = true;
const output = await runWorkerOnce();
assert.ok(countFromWorkerOutput(output, 'verified') >= 1, 'worker should verify at least one asset');
assert.ok(countFromWorkerOutput(output, 'verified') >= 3, 'worker should verify metadata-safe assets');
assert.ok(countFromWorkerOutput(output, 'failed') >= 2, 'worker should fail metadata and security scan assets');
const assets = await pool.query(
@@ -148,10 +266,10 @@ async function main() {
security_scan_status, security_scan_provider, security_scan_summary,
security_flags
from public.content_assets
where tenant_id = $1 and id in ($2::uuid, $3::uuid, $4::uuid)
where tenant_id = $1 and id = any($2::uuid[])
order by id
`,
[tenantId, ids.okAsset, ids.badAsset, ids.scanBadAsset],
[tenantId, [ids.okAsset, ids.badAsset, ids.scanBadAsset, ids.httpPassAsset, ids.httpFailAsset, ids.httpUnavailableAsset]],
);
const okAsset = assets.rows.find(row => row.id === ids.okAsset);
const badAsset = assets.rows.find(row => row.id === ids.badAsset);
@@ -237,6 +355,104 @@ async function main() {
'worker should write failed security scan audit log',
);
await resetAssetsForHttp(pool);
const scanner = await startScannerServer();
try {
const httpOutput = await runWorkerOnce({
WORKER_ASSET_SECURITY_SCANNER: 'metadata_rules,http',
WORKER_ASSET_SECURITY_SCAN_HTTP_ENDPOINT: scanner.endpoint,
WORKER_ASSET_SECURITY_SCAN_HTTP_TOKEN: 'scanner-test-token',
WORKER_ASSET_SECURITY_SCAN_HTTP_TIMEOUT_MS: '5000',
WORKER_ASSET_SECURITY_SCAN_FAIL_OPEN: 'false',
});
assert.ok(countFromWorkerOutput(httpOutput, 'verified') >= 1, 'worker should pass externally clean assets');
assert.ok(countFromWorkerOutput(httpOutput, 'failed') >= 1, 'worker should fail externally rejected assets');
assert.equal(scanner.requests.length, 3, 'external scanner should receive metadata-clean assets only');
assert.ok(
scanner.requests.every(item => item.authorization === 'Bearer scanner-test-token'),
'external scanner should receive bearer token',
);
const httpAssets = await pool.query(
`
select id, status, security_scan_status, security_scan_provider, security_scan_summary, security_flags
from public.content_assets
where tenant_id = $1 and id = any($2::uuid[])
order by id
`,
[tenantId, [ids.httpPassAsset, ids.httpFailAsset, ids.httpUnavailableAsset]],
);
const httpPassAsset = httpAssets.rows.find(row => row.id === ids.httpPassAsset);
const httpFailAsset = httpAssets.rows.find(row => row.id === ids.httpFailAsset);
const httpUnavailableAsset = httpAssets.rows.find(row => row.id === ids.httpUnavailableAsset);
assert.equal(httpPassAsset?.status, 'active', 'external scan pass asset should stay active');
assert.equal(httpPassAsset?.security_scan_status, 'passed', 'external scan pass asset should pass');
assert.equal(
httpPassAsset?.security_scan_provider,
'metadata_rules+http_security_scan',
'external scan pass asset should record merged provider',
);
assert.equal(httpFailAsset?.status, 'draft', 'external scan failed asset should be unpublished');
assert.equal(httpFailAsset?.security_scan_status, 'failed', 'external scan failed asset should fail');
assert.ok(
httpFailAsset?.security_scan_summary?.issueCodes?.includes('virus_detected'),
'external scan failed asset should include provider issue code',
);
const scannerDetails = httpFailAsset?.security_scan_summary?.details?.results?.find(
result => result.provider === 'http_security_scan',
)?.details?.scannerDetails;
assert.equal(scannerDetails?.accessToken, '[redacted]', 'scanner details should redact secret-like keys');
assert.equal(httpUnavailableAsset?.security_scan_status, 'passed', 'available scanner should pass clean third asset');
const httpScanEvents = await pool.query(
`
select asset_id, scan_status, provider, issue_codes, details
from public.content_asset_security_scan_events
where tenant_id = $1 and asset_id = any($2::uuid[])
order by created_at asc
`,
[tenantId, [ids.httpPassAsset, ids.httpFailAsset]],
);
assert.ok(
httpScanEvents.rows.some(row => row.asset_id === ids.httpPassAsset && row.provider === 'metadata_rules+http_security_scan' && row.scan_status === 'passed'),
'worker should write external pass event',
);
assert.ok(
httpScanEvents.rows.some(row => row.asset_id === ids.httpFailAsset && row.issue_codes.includes('virus_detected')),
'worker should write external failure event',
);
} finally {
await scanner.close();
}
await resetAssetsForHttp(pool);
const unavailableOutput = await runWorkerOnce({
WORKER_ASSET_SECURITY_SCANNER: 'metadata_rules,http',
WORKER_ASSET_SECURITY_SCAN_HTTP_ENDPOINT: 'http://127.0.0.1:9/scan',
WORKER_ASSET_SECURITY_SCAN_HTTP_TOKEN: 'scanner-test-token',
WORKER_ASSET_SECURITY_SCAN_HTTP_TIMEOUT_MS: '300',
WORKER_ASSET_SECURITY_SCAN_FAIL_OPEN: 'false',
});
assert.ok(countFromWorkerOutput(unavailableOutput, 'failed') >= 3, 'fail-closed scanner outage should fail assets');
const unavailableAssets = await pool.query(
`
select id, status, security_scan_status, security_scan_summary, security_flags
from public.content_assets
where tenant_id = $1 and id = any($2::uuid[])
`,
[tenantId, [ids.httpPassAsset, ids.httpFailAsset, ids.httpUnavailableAsset]],
);
for (const row of unavailableAssets.rows) {
assert.equal(row.status, 'draft', 'fail-closed scanner outage should unpublish active asset');
assert.equal(row.security_scan_status, 'failed', 'fail-closed scanner outage should fail scan');
assert.ok(
row.security_scan_summary?.issueCodes?.includes('external_security_scan_unavailable'),
'fail-closed scanner outage should record unavailable issue',
);
assert.equal(row.security_flags?.assetSecurityScanFailed, true, 'fail-closed scanner outage should record security flag');
}
console.log('Asset worker integration test complete.');
} catch (error) {
await pool.query('rollback').catch(() => {});

5
scripts/production-readiness-check-test.js
View File

@@ -72,6 +72,11 @@ ALIYUN_OSS_REGION=cn-hangzhou
ALIYUN_OSS_ENDPOINT=https://oss-cn-hangzhou.aliyuncs.com
ALIYUN_OSS_ACCESS_KEY_ID=LTAI_READINESS_TEST_ONLY
ALIYUN_OSS_ACCESS_KEY_SECRET=aliyun-readiness-secret-placeholder
WORKER_ASSET_SECURITY_SCANNER=metadata_rules,http
WORKER_ASSET_SECURITY_SCAN_HTTP_ENDPOINT=https://scanner.gongxue100.com/api/scan
WORKER_ASSET_SECURITY_SCAN_HTTP_TOKEN=s3cure-asset-scanner-token-2026-06-29-stuvwx
WORKER_ASSET_SECURITY_SCAN_HTTP_TIMEOUT_MS=10000
WORKER_ASSET_SECURITY_SCAN_FAIL_OPEN=false
WORKER_CRM_ALLOW_INSECURE_LOCALHOST=false
WORKER_CRM_BATCH_SIZE=20
WORKER_COMMERCE_BATCH_SIZE=20

45
scripts/production-readiness-check.js
View File

@@ -267,6 +267,51 @@ function validateEnv() {
warn('env.storage_octet_stream', 'application/octet-stream is allowed; consider removing it after import migration is stable');
}
const scannerModes = envList('WORKER_ASSET_SECURITY_SCANNER', 'metadata_rules').map(item => item.toLowerCase());
const unsupportedScannerModes = scannerModes.filter(mode => mode !== 'metadata_rules' && mode !== 'http');
if (scannerModes.length === 0) {
block('env.asset_security_scanner.empty', 'WORKER_ASSET_SECURITY_SCANNER must include metadata_rules and a production scanner');
} else if (unsupportedScannerModes.length > 0) {
block('env.asset_security_scanner.unsupported', 'WORKER_ASSET_SECURITY_SCANNER contains unsupported modes', {
modes: unsupportedScannerModes,
});
} else if (!scannerModes.includes('http')) {
block('env.asset_security_scanner.external', 'Production assets require WORKER_ASSET_SECURITY_SCANNER=http or metadata_rules,http');
} else {
pass('env.asset_security_scanner', 'external asset security scanner is enabled', { modes: scannerModes });
}
const scannerEndpoint = env('WORKER_ASSET_SECURITY_SCAN_HTTP_ENDPOINT', '');
if (scannerModes.includes('http')) {
const scannerHost = hostFromUrl(scannerEndpoint);
if (!scannerEndpoint) {
block('env.asset_security_scan_http_endpoint', 'WORKER_ASSET_SECURITY_SCAN_HTTP_ENDPOINT is required when http scanner is enabled');
} else if (!scannerEndpoint.startsWith('https://') || isLocalHost(scannerHost)) {
block('env.asset_security_scan_http_endpoint.unsafe', 'WORKER_ASSET_SECURITY_SCAN_HTTP_ENDPOINT must be a production HTTPS URL');
} else {
pass('env.asset_security_scan_http_endpoint', 'asset security scanner endpoint is HTTPS');
}
if (isUnsafeSecret(env('WORKER_ASSET_SECURITY_SCAN_HTTP_TOKEN', ''), '')) {
block('env.asset_security_scan_http_token', 'WORKER_ASSET_SECURITY_SCAN_HTTP_TOKEN must be a strong shared secret or service token');
} else {
pass('env.asset_security_scan_http_token', 'asset security scanner token looks production-grade');
}
}
const scannerTimeout = envNumber('WORKER_ASSET_SECURITY_SCAN_HTTP_TIMEOUT_MS', 10_000);
if (scannerTimeout <= 0 || scannerTimeout > 60_000) {
block('env.asset_security_scan_http_timeout', 'WORKER_ASSET_SECURITY_SCAN_HTTP_TIMEOUT_MS must be between 1 and 60000');
} else {
pass('env.asset_security_scan_http_timeout', 'asset security scanner timeout is bounded');
}
if (envBool('WORKER_ASSET_SECURITY_SCAN_FAIL_OPEN', false)) {
block('env.asset_security_scan_fail_open', 'WORKER_ASSET_SECURITY_SCAN_FAIL_OPEN must be false in production');
} else {
pass('env.asset_security_scan_fail_open', 'asset security scanning fails closed');
}
if (envBool('WORKER_CRM_ALLOW_INSECURE_LOCALHOST', false)) {
block('env.worker_crm_insecure_localhost', 'WORKER_CRM_ALLOW_INSECURE_LOCALHOST must be false in production');
} else {