feat: add pocketbase dry-run report

This commit is contained in:
Codex
2026-06-29 11:06:31 +08:00
parent e587a0c955
commit f74db433cc
14 changed files with 811 additions and 13 deletions

View File

@@ -6,6 +6,7 @@
"scripts": {
"schema:summary": "tsx src/analyze-schema.ts",
"schema:risk": "tsx src/risk-report.ts",
"import:dry-run": "tsx src/dry-run-report.ts",
"import:json": "tsx src/import-json.ts",
"import:validate": "tsx src/validate-import.ts",
"check": "tsc -p tsconfig.json --noEmit"

View File

@@ -0,0 +1,523 @@
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { fieldsOf, readPocketBaseSchema, type PocketBaseCollection, type PocketBaseField } from './pb-schema.js';
import { loadEnv } from './env.js';
loadEnv();
type JsonRecord = Record<string, unknown> & { id?: unknown };
type Severity = 'blocker' | 'warning' | 'info';
interface DryRunIssue {
severity: Severity;
code: string;
collection?: string;
file?: string;
fieldPath?: string;
message: string;
sample?: string;
count?: number;
}
interface CollectionReport {
collection: string;
file: string;
recordCount: number;
missingIdCount: number;
duplicateIdCount: number;
fieldCount: number;
sampleIds: string[];
supported: boolean;
}
interface DryRunReport {
exportDir: string;
schemaPath?: string;
generatedAt: string;
summary: {
files: number;
collections: number;
records: number;
supportedCollections: number;
unsupportedCollections: number;
blockers: number;
warnings: number;
infos: number;
};
businessCounts: Record<string, number>;
collections: CollectionReport[];
issues: DryRunIssue[];
}
const args = new Set(process.argv.slice(2));
const jsonOutput = args.has('--json');
const failOnWarnings = args.has('--fail-on-warnings') || process.env.PB_DRY_RUN_FAIL_ON_WARNINGS === 'true';
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
const exportDir = path.resolve(repoRoot, process.env.PB_EXPORT_DIR || 'pb_export');
const supportedCollections = new Set([
'announcements',
'app_assets',
'audit_logs',
'badges',
'banners',
'categories',
'code_batches',
'codes',
'coupons',
'coupon_redemptions',
'crm_config',
'crm_webhook_log',
'crm_webhook_queue',
'dashboard_daily_stats',
'exam_dates',
'faqs',
'handbook_chapters',
'handbook_entries',
'handbook_subjects',
'images',
'majors',
'module_nodes',
'orders',
'products',
'question_type_groups',
'question_videos',
'questions',
'recent_practices',
'referral_tracks',
'region_modules',
'regions',
'reports',
'revenue_daily_stats',
'schools',
'scoreline_fields',
'scoreline_majors',
'scoreline_records',
'scoreline_schools',
'settings',
'subject_shares',
'subjects',
'svip_plans',
'timelines',
'user_badges',
'user_word_favorites',
'user_word_progress',
'users',
'video_explanations',
'vocabulary',
'vocabulary_units',
]);
const coreCollections = [
'users',
'questions',
'subjects',
'categories',
'orders',
'svip_plans',
'codes',
'vocabulary_units',
'vocabulary',
'handbook_subjects',
'handbook_chapters',
'handbook_entries',
];
const businessCollections = [
'users',
'questions',
'orders',
'svip_plans',
'codes',
'vocabulary_units',
'vocabulary',
'handbook_subjects',
'handbook_chapters',
'handbook_entries',
'scoreline_schools',
'scoreline_majors',
'scoreline_records',
'video_explanations',
'question_videos',
'referral_tracks',
'coupons',
'badges',
];
const sensitiveKeyPattern =
/(password|token|secret|privatekey|sessionkey|accesskey|appkey|apikey|api_v3_key|notifytoken|aeskey|openid|unionid|wxaccesstoken|wechatsessionkey|smscode|verifycode|verificationcode|captcha)/i;
const highRiskCollections = new Set(['settings', 'crm_config', 'smscodes']);
function issue(issues: DryRunIssue[], nextIssue: DryRunIssue) {
issues.push(nextIssue);
}
function asRecords(input: unknown): JsonRecord[] | null {
if (Array.isArray(input)) return input as JsonRecord[];
if (input && typeof input === 'object') {
const candidate = input as { items?: unknown; records?: unknown };
if (Array.isArray(candidate.items)) return candidate.items as JsonRecord[];
if (Array.isArray(candidate.records)) return candidate.records as JsonRecord[];
}
return null;
}
function collectionNameFromFile(fileName: string) {
return fileName.replace(/\.json$/i, '');
}
function text(value: unknown) {
if (value === null || value === undefined) return '';
return String(value).trim();
}
function readJsonFile(filePath: string) {
return JSON.parse(fs.readFileSync(filePath, 'utf8')) as unknown;
}
function scalarSample(value: unknown) {
if (value === null || value === undefined) return '';
if (typeof value === 'object') return JSON.stringify(value).slice(0, 160);
return String(value).slice(0, 160);
}
function walkSensitiveFields(
issues: DryRunIssue[],
collection: string,
value: unknown,
pathParts: string[] = [],
depth = 0,
) {
if (!value || typeof value !== 'object' || depth > 8) return;
if (Array.isArray(value)) {
value.slice(0, 10).forEach((item, index) => walkSensitiveFields(issues, collection, item, [...pathParts, String(index)], depth + 1));
return;
}
for (const [key, childValue] of Object.entries(value as Record<string, unknown>)) {
const fieldPath = [...pathParts, key].join('.');
if (sensitiveKeyPattern.test(key) && childValue !== null && childValue !== undefined && childValue !== '') {
issue(issues, {
severity: highRiskCollections.has(collection) ? 'blocker' : 'warning',
code: 'sensitive_field_in_export',
collection,
fieldPath,
message: `Legacy export contains sensitive-looking field "${fieldPath}". It must not be copied into public normalized tables.`,
sample: scalarSample(childValue),
});
}
walkSensitiveFields(issues, collection, childValue, [...pathParts, key], depth + 1);
}
}
function fieldValues(record: JsonRecord, field: PocketBaseField) {
const value = record[field.name];
if (value === null || value === undefined || value === '') return [];
if (Array.isArray(value)) return value.map(item => text(item)).filter(Boolean);
return [text(value)].filter(Boolean);
}
function relationTargetName(field: PocketBaseField, byId: Map<string, PocketBaseCollection>) {
if (!field.collectionId) return '';
return byId.get(field.collectionId)?.name || '';
}
function validateRelations(
issues: DryRunIssue[],
collection: PocketBaseCollection,
records: JsonRecord[],
idIndex: Map<string, Set<string>>,
byId: Map<string, PocketBaseCollection>,
) {
const relationFields = fieldsOf(collection).filter(field => field.type === 'relation');
for (const field of relationFields) {
const targetName = relationTargetName(field, byId);
if (!targetName) {
issue(issues, {
severity: 'warning',
code: 'relation_target_schema_missing',
collection: collection.name,
fieldPath: field.name,
message: `Relation field "${field.name}" points to an unknown collection id in pb_schema.json.`,
});
continue;
}
const targetIds = idIndex.get(targetName);
if (!targetIds) {
const hasValues = records.some(record => fieldValues(record, field).length > 0);
if (hasValues) {
issue(issues, {
severity: coreCollections.includes(collection.name) ? 'blocker' : 'warning',
code: 'relation_target_export_missing',
collection: collection.name,
fieldPath: field.name,
message: `Relation field "${field.name}" references "${targetName}", but ${targetName}.json is not present in export.`,
});
}
continue;
}
let unresolved = 0;
let sample = '';
for (const record of records) {
for (const value of fieldValues(record, field)) {
if (!targetIds.has(value)) {
unresolved += 1;
if (!sample) sample = `${text(record.id) || '(missing id)'} -> ${value}`;
}
}
}
if (unresolved > 0) {
issue(issues, {
severity: coreCollections.includes(collection.name) ? 'blocker' : 'warning',
code: 'relation_unresolved',
collection: collection.name,
fieldPath: field.name,
message: `Relation field "${field.name}" has ${unresolved} unresolved reference(s) to "${targetName}".`,
count: unresolved,
sample,
});
}
}
}
function loadSchema(issues: DryRunIssue[]) {
try {
return readPocketBaseSchema();
} catch (error) {
issue(issues, {
severity: 'warning',
code: 'schema_unavailable',
message: `pb_schema.json could not be loaded; relation checks will be skipped. ${error instanceof Error ? error.message : ''}`.trim(),
});
return null;
}
}
function collectFieldNames(records: JsonRecord[]) {
const fields = new Set<string>();
for (const record of records.slice(0, 500)) {
Object.keys(record).forEach(key => fields.add(key));
}
return fields;
}
function buildReport(): DryRunReport {
const issues: DryRunIssue[] = [];
const schema = loadSchema(issues);
const collectionByName = schema?.byName || new Map<string, PocketBaseCollection>();
const files = fs.existsSync(exportDir) ? fs.readdirSync(exportDir).filter(file => file.toLowerCase().endsWith('.json')) : [];
const collections: CollectionReport[] = [];
const recordsByCollection = new Map<string, JsonRecord[]>();
const idIndex = new Map<string, Set<string>>();
if (!fs.existsSync(exportDir)) {
issue(issues, {
severity: 'blocker',
code: 'export_dir_missing',
message: `Export dir not found: ${exportDir}`,
});
}
if (files.length === 0) {
issue(issues, {
severity: 'blocker',
code: 'export_files_missing',
message: `No PocketBase collection JSON files found in ${exportDir}`,
});
}
for (const file of files.sort()) {
const collection = collectionNameFromFile(file);
const filePath = path.join(exportDir, file);
let parsed: unknown;
try {
parsed = readJsonFile(filePath);
} catch (error) {
issue(issues, {
severity: 'blocker',
code: 'json_parse_failed',
collection,
file,
message: `Failed to parse ${file}: ${error instanceof Error ? error.message : String(error)}`,
});
continue;
}
const records = asRecords(parsed);
if (!records) {
issue(issues, {
severity: 'blocker',
code: 'collection_shape_invalid',
collection,
file,
message: `${file} must be an array or an object with items/records array.`,
});
continue;
}
const ids = new Set<string>();
const duplicateIds = new Set<string>();
let missingIdCount = 0;
for (const record of records) {
const id = text(record.id);
if (!id) {
missingIdCount += 1;
continue;
}
if (ids.has(id)) duplicateIds.add(id);
ids.add(id);
}
recordsByCollection.set(collection, records);
idIndex.set(collection, ids);
const fields = collectFieldNames(records);
collections.push({
collection,
file,
recordCount: records.length,
missingIdCount,
duplicateIdCount: duplicateIds.size,
fieldCount: fields.size,
sampleIds: [...ids].slice(0, 5),
supported: supportedCollections.has(collection),
});
if (!supportedCollections.has(collection) && !collection.startsWith('_')) {
issue(issues, {
severity: 'warning',
code: 'unsupported_collection',
collection,
file,
message: `${collection}.json is not currently normalized by import-json.ts. It will be preserved only if a future mapper is added.`,
count: records.length,
});
}
if (missingIdCount > 0) {
issue(issues, {
severity: coreCollections.includes(collection) ? 'blocker' : 'warning',
code: 'record_id_missing',
collection,
file,
message: `${collection} has ${missingIdCount} record(s) without id.`,
count: missingIdCount,
});
}
if (duplicateIds.size > 0) {
issue(issues, {
severity: 'blocker',
code: 'record_id_duplicate',
collection,
file,
message: `${collection} has duplicate legacy id(s).`,
count: duplicateIds.size,
sample: [...duplicateIds].slice(0, 5).join(', '),
});
}
records.slice(0, 200).forEach(record => walkSensitiveFields(issues, collection, record));
}
for (const collection of coreCollections) {
if (!recordsByCollection.has(collection)) {
issue(issues, {
severity: 'warning',
code: 'core_collection_missing',
collection,
message: `${collection}.json is not present. If old production has this collection, export it before migration.`,
});
}
}
if (schema) {
for (const [collectionName, records] of recordsByCollection.entries()) {
const collection = collectionByName.get(collectionName);
if (collection) validateRelations(issues, collection, records, idIndex, schema.byId);
}
}
const businessCounts = Object.fromEntries(
businessCollections.map(collection => [collection, recordsByCollection.get(collection)?.length || 0]),
);
const supportedCount = collections.filter(item => item.supported).length;
const unsupportedCount = collections.length - supportedCount;
const recordCount = collections.reduce((sum, item) => sum + item.recordCount, 0);
const blockers = issues.filter(item => item.severity === 'blocker').length;
const warnings = issues.filter(item => item.severity === 'warning').length;
const infos = issues.filter(item => item.severity === 'info').length;
return {
exportDir,
schemaPath: schema?.absPath,
generatedAt: new Date().toISOString(),
summary: {
files: files.length,
collections: collections.length,
records: recordCount,
supportedCollections: supportedCount,
unsupportedCollections: unsupportedCount,
blockers,
warnings,
infos,
},
businessCounts,
collections: collections.sort((a, b) => a.collection.localeCompare(b.collection)),
issues: issues.sort((a, b) => {
const order: Record<Severity, number> = { blocker: 0, warning: 1, info: 2 };
return order[a.severity] - order[b.severity] || (a.collection || '').localeCompare(b.collection || '') || a.code.localeCompare(b.code);
}),
};
}
function printHuman(report: DryRunReport) {
console.log('PocketBase dry-run migration report');
console.log(`Export dir: ${report.exportDir}`);
if (report.schemaPath) console.log(`Schema: ${report.schemaPath}`);
console.log(
`Summary: ${report.summary.records} records across ${report.summary.collections} collections; ` +
`${report.summary.blockers} blocker(s), ${report.summary.warnings} warning(s)`,
);
console.log('');
console.log('Business counts:');
for (const [name, count] of Object.entries(report.businessCounts)) {
console.log(` ${name.padEnd(24)} ${count}`);
}
console.log('');
console.log('Collections:');
for (const item of report.collections) {
const marker = item.supported ? 'mapped' : 'unmapped';
console.log(
` ${item.collection.padEnd(24)} records=${String(item.recordCount).padStart(6)} ` +
`fields=${String(item.fieldCount).padStart(3)} ${marker}`,
);
}
console.log('');
console.log('Issues:');
if (report.issues.length === 0) {
console.log(' none');
} else {
for (const item of report.issues.slice(0, 200)) {
const location = [item.collection, item.fieldPath].filter(Boolean).join('.') || item.file || 'export';
const count = item.count === undefined ? '' : ` (${item.count})`;
const sample = item.sample ? ` sample=${item.sample}` : '';
console.log(` [${item.severity.toUpperCase()}] ${item.code} ${location}: ${item.message}${count}${sample}`);
}
if (report.issues.length > 200) console.log(` ... ${report.issues.length - 200} more issue(s) omitted`);
}
}
function main() {
const report = buildReport();
if (jsonOutput) console.log(JSON.stringify(report, null, 2));
else printHuman(report);
if (report.summary.blockers > 0 || (failOnWarnings && report.summary.warnings > 0)) {
process.exitCode = 1;
}
}
main();

View File

@@ -0,0 +1,122 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawnSync } from 'node:child_process';
const repoRoot = process.cwd();
const dryRunScript = path.join(repoRoot, 'scripts', 'import-pocketbase', 'src', 'dry-run-report.ts');
function writeJson(filePath, value) {
fs.writeFileSync(filePath, JSON.stringify(value, null, 2), 'utf8');
}
function runDryRun(exportDir) {
const result = spawnSync(process.execPath, ['--import', 'tsx', dryRunScript, '--json'], {
cwd: repoRoot,
encoding: 'utf8',
env: {
...process.env,
PB_EXPORT_DIR: exportDir,
},
});
if (result.error) throw result.error;
const stdout = result.stdout || '';
const jsonStart = stdout.indexOf('{');
const payload = JSON.parse(jsonStart >= 0 ? stdout.slice(jsonStart) : '{}');
return { ...result, payload };
}
function createSafeExport() {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tiku-pb-dry-safe-'));
writeJson(path.join(dir, 'users.json'), [
{ id: 'u1', phone: '13800000000', role: 'student', nickname: '学生A' },
]);
writeJson(path.join(dir, 'subjects.json'), [
{ id: 's1', name: '英语' },
]);
writeJson(path.join(dir, 'categories.json'), [
{ id: 'c1', name: '阅读理解', subjectId: 's1' },
]);
writeJson(path.join(dir, 'questions.json'), [
{ id: 'q1', subjectId: 's1', categoryId: 'c1', type: 'choice', content: '题干', answer: 'A' },
]);
writeJson(path.join(dir, 'orders.json'), [
{ id: 'o1', userId: 'u1', amount: 99, status: 'paid' },
]);
writeJson(path.join(dir, 'svip_plans.json'), [
{ id: 'p1', name: '题库会员', price: 99 },
]);
writeJson(path.join(dir, 'codes.json'), [
{ id: 'code1', code: 'SAFE-001', status: 'unused' },
]);
writeJson(path.join(dir, 'vocabulary_units.json'), [
{ id: 'vu1', name: 'Unit 1' },
]);
writeJson(path.join(dir, 'vocabulary.json'), [
{ id: 'vw1', unitId: 'vu1', word: 'abandon', meaning: '放弃' },
]);
writeJson(path.join(dir, 'handbook_subjects.json'), [
{ id: 'hs1', name: '政治' },
]);
writeJson(path.join(dir, 'handbook_chapters.json'), [
{ id: 'hc1', subjectId: 'hs1', title: '第一章' },
]);
writeJson(path.join(dir, 'handbook_entries.json'), [
{ id: 'he1', chapterId: 'hc1', title: '知识点', content: '内容' },
]);
return dir;
}
function createUnsafeExport() {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tiku-pb-dry-unsafe-'));
writeJson(path.join(dir, 'users.json'), [
{ id: 'u1', phone: '13800000000', password: 'plaintext-should-not-exist' },
{ phone: '13900000000' },
]);
writeJson(path.join(dir, 'questions.json'), [
{ id: 'q1', subjectId: 'missing-subject', content: '题干' },
{ id: 'q1', subjectId: 'missing-subject', content: '重复题' },
]);
writeJson(path.join(dir, 'settings.json'), [
{ id: 'settings1', appSecret: 'secret-in-public-export' },
]);
writeJson(path.join(dir, 'unknown_business.json'), [
{ id: 'x1', name: '尚未映射集合' },
]);
return dir;
}
const safeDir = createSafeExport();
const safe = runDryRun(safeDir);
assert.equal(safe.status, 0, `safe dry-run should pass: ${safe.stdout} ${safe.stderr}`);
assert.equal(safe.payload.summary?.blockers, 0, 'safe dry-run should have no blockers');
assert.ok(safe.payload.summary?.records >= 12, 'safe dry-run should count fixture records');
assert.equal(safe.payload.businessCounts?.questions, 1, 'safe dry-run should report question count');
const unsafeDir = createUnsafeExport();
const unsafe = runDryRun(unsafeDir);
assert.notEqual(unsafe.status, 0, 'unsafe dry-run should fail');
assert.ok(unsafe.payload.summary?.blockers > 0, 'unsafe dry-run should report blockers');
assert.ok(
unsafe.payload.issues?.some(item => item.code === 'record_id_missing' && item.collection === 'users'),
'unsafe dry-run should detect missing user id',
);
assert.ok(
unsafe.payload.issues?.some(item => item.code === 'record_id_duplicate' && item.collection === 'questions'),
'unsafe dry-run should detect duplicate question id',
);
assert.ok(
unsafe.payload.issues?.some(item => item.code === 'sensitive_field_in_export'),
'unsafe dry-run should detect sensitive fields',
);
assert.ok(
unsafe.payload.issues?.some(item => item.code === 'unsupported_collection'),
'unsafe dry-run should warn about unsupported collections',
);
fs.rmSync(safeDir, { recursive: true, force: true });
fs.rmSync(unsafeDir, { recursive: true, force: true });
console.log('[PASS] PocketBase dry-run report');

View File

@@ -87,6 +87,109 @@ async function main() {
try {
await client.query('begin');
await client.query(
`
create temporary table smoke_seed_transient_questions (
id uuid primary key
) on commit drop
`,
);
await client.query(
`
insert into smoke_seed_transient_questions (id)
select distinct q.id
from public.questions q
left join public.question_collection_items ci
on ci.tenant_id = q.tenant_id
and ci.question_id = q.id
where q.tenant_id = $1
and q.id <> all($3::uuid[])
and (
q.primary_collection_id = $2::uuid
or ci.collection_id = $2::uuid
or coalesce(q.legacy_id, '') like any($4::text[])
)
`,
[
tenantId,
ids.questionCollection,
[ids.question, ids.questionTwo, ids.questionThree],
['integration-import-%', 'worker-import-question-%', 'public-sync-source-%', 'content-import:%'],
],
);
await client.query(
`
delete from public.question_collection_items
where tenant_id = $1
and (
question_id in (select id from smoke_seed_transient_questions)
or (
collection_id = $2::uuid
and question_id <> all($3::uuid[])
)
)
`,
[tenantId, ids.questionCollection, [ids.question, ids.questionTwo, ids.questionThree]],
);
await client.query(
`
delete from public.question_versions
where tenant_id = $1
and question_id in (select id from smoke_seed_transient_questions)
`,
[tenantId],
);
await client.query(
`
delete from public.questions
where tenant_id = $1
and id in (select id from smoke_seed_transient_questions)
`,
[tenantId],
);
await client.query(
`
delete from public.content_export_jobs
where tenant_id = $1
and scope_id = $2::uuid
`,
[tenantId, ids.questionCollection],
);
await client.query(
`
delete from public.content_import_jobs
where tenant_id = $1
and (
source_name = any($2::text[])
or source_name like 'worker-import-%'
)
`,
[
tenantId,
[
'large-but-allowed-question-import.json',
'invalid-question-import.json',
'valid-question-import.json',
'questions.csv',
'async-question-import.json',
'invalid-vocabulary-import.json',
'legacy-vocabulary-import.json',
'vocabulary.csv',
'handbook-nested-import.json',
'scoreline-batch-import.json',
'scoreline-mixed-items.json',
'scoreline.xlsx',
'video-batch-import.json',
],
],
);
await client.query(
`
delete from public.report_status_events