forked from wangziqi/gongxue-base
feat: add pocketbase dry-run report
This commit is contained in:
Codex
@@ -6,6 +6,7 @@
|
||||
"scripts": {
|
||||
"schema:summary": "tsx src/analyze-schema.ts",
|
||||
"schema:risk": "tsx src/risk-report.ts",
|
||||
"import:dry-run": "tsx src/dry-run-report.ts",
|
||||
"import:json": "tsx src/import-json.ts",
|
||||
"import:validate": "tsx src/validate-import.ts",
|
||||
"check": "tsc -p tsconfig.json --noEmit"
|
||||
|
||||
523
scripts/import-pocketbase/src/dry-run-report.ts
Normal file
523
scripts/import-pocketbase/src/dry-run-report.ts
Normal file
@@ -0,0 +1,523 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { fieldsOf, readPocketBaseSchema, type PocketBaseCollection, type PocketBaseField } from './pb-schema.js';
|
||||
import { loadEnv } from './env.js';
|
||||
|
||||
loadEnv();
|
||||
|
||||
type JsonRecord = Record<string, unknown> & { id?: unknown };
|
||||
type Severity = 'blocker' | 'warning' | 'info';
|
||||
|
||||
interface DryRunIssue {
|
||||
severity: Severity;
|
||||
code: string;
|
||||
collection?: string;
|
||||
file?: string;
|
||||
fieldPath?: string;
|
||||
message: string;
|
||||
sample?: string;
|
||||
count?: number;
|
||||
}
|
||||
|
||||
interface CollectionReport {
|
||||
collection: string;
|
||||
file: string;
|
||||
recordCount: number;
|
||||
missingIdCount: number;
|
||||
duplicateIdCount: number;
|
||||
fieldCount: number;
|
||||
sampleIds: string[];
|
||||
supported: boolean;
|
||||
}
|
||||
|
||||
interface DryRunReport {
|
||||
exportDir: string;
|
||||
schemaPath?: string;
|
||||
generatedAt: string;
|
||||
summary: {
|
||||
files: number;
|
||||
collections: number;
|
||||
records: number;
|
||||
supportedCollections: number;
|
||||
unsupportedCollections: number;
|
||||
blockers: number;
|
||||
warnings: number;
|
||||
infos: number;
|
||||
};
|
||||
businessCounts: Record<string, number>;
|
||||
collections: CollectionReport[];
|
||||
issues: DryRunIssue[];
|
||||
}
|
||||
|
||||
const args = new Set(process.argv.slice(2));
|
||||
const jsonOutput = args.has('--json');
|
||||
const failOnWarnings = args.has('--fail-on-warnings') || process.env.PB_DRY_RUN_FAIL_ON_WARNINGS === 'true';
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
|
||||
const exportDir = path.resolve(repoRoot, process.env.PB_EXPORT_DIR || 'pb_export');
|
||||
|
||||
const supportedCollections = new Set([
|
||||
'announcements',
|
||||
'app_assets',
|
||||
'audit_logs',
|
||||
'badges',
|
||||
'banners',
|
||||
'categories',
|
||||
'code_batches',
|
||||
'codes',
|
||||
'coupons',
|
||||
'coupon_redemptions',
|
||||
'crm_config',
|
||||
'crm_webhook_log',
|
||||
'crm_webhook_queue',
|
||||
'dashboard_daily_stats',
|
||||
'exam_dates',
|
||||
'faqs',
|
||||
'handbook_chapters',
|
||||
'handbook_entries',
|
||||
'handbook_subjects',
|
||||
'images',
|
||||
'majors',
|
||||
'module_nodes',
|
||||
'orders',
|
||||
'products',
|
||||
'question_type_groups',
|
||||
'question_videos',
|
||||
'questions',
|
||||
'recent_practices',
|
||||
'referral_tracks',
|
||||
'region_modules',
|
||||
'regions',
|
||||
'reports',
|
||||
'revenue_daily_stats',
|
||||
'schools',
|
||||
'scoreline_fields',
|
||||
'scoreline_majors',
|
||||
'scoreline_records',
|
||||
'scoreline_schools',
|
||||
'settings',
|
||||
'subject_shares',
|
||||
'subjects',
|
||||
'svip_plans',
|
||||
'timelines',
|
||||
'user_badges',
|
||||
'user_word_favorites',
|
||||
'user_word_progress',
|
||||
'users',
|
||||
'video_explanations',
|
||||
'vocabulary',
|
||||
'vocabulary_units',
|
||||
]);
|
||||
|
||||
const coreCollections = [
|
||||
'users',
|
||||
'questions',
|
||||
'subjects',
|
||||
'categories',
|
||||
'orders',
|
||||
'svip_plans',
|
||||
'codes',
|
||||
'vocabulary_units',
|
||||
'vocabulary',
|
||||
'handbook_subjects',
|
||||
'handbook_chapters',
|
||||
'handbook_entries',
|
||||
];
|
||||
|
||||
const businessCollections = [
|
||||
'users',
|
||||
'questions',
|
||||
'orders',
|
||||
'svip_plans',
|
||||
'codes',
|
||||
'vocabulary_units',
|
||||
'vocabulary',
|
||||
'handbook_subjects',
|
||||
'handbook_chapters',
|
||||
'handbook_entries',
|
||||
'scoreline_schools',
|
||||
'scoreline_majors',
|
||||
'scoreline_records',
|
||||
'video_explanations',
|
||||
'question_videos',
|
||||
'referral_tracks',
|
||||
'coupons',
|
||||
'badges',
|
||||
];
|
||||
|
||||
const sensitiveKeyPattern =
|
||||
/(password|token|secret|privatekey|sessionkey|accesskey|appkey|apikey|api_v3_key|notifytoken|aeskey|openid|unionid|wxaccesstoken|wechatsessionkey|smscode|verifycode|verificationcode|captcha)/i;
|
||||
const highRiskCollections = new Set(['settings', 'crm_config', 'smscodes']);
|
||||
|
||||
function issue(issues: DryRunIssue[], nextIssue: DryRunIssue) {
|
||||
issues.push(nextIssue);
|
||||
}
|
||||
|
||||
function asRecords(input: unknown): JsonRecord[] | null {
|
||||
if (Array.isArray(input)) return input as JsonRecord[];
|
||||
if (input && typeof input === 'object') {
|
||||
const candidate = input as { items?: unknown; records?: unknown };
|
||||
if (Array.isArray(candidate.items)) return candidate.items as JsonRecord[];
|
||||
if (Array.isArray(candidate.records)) return candidate.records as JsonRecord[];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function collectionNameFromFile(fileName: string) {
|
||||
return fileName.replace(/\.json$/i, '');
|
||||
}
|
||||
|
||||
function text(value: unknown) {
|
||||
if (value === null || value === undefined) return '';
|
||||
return String(value).trim();
|
||||
}
|
||||
|
||||
function readJsonFile(filePath: string) {
|
||||
return JSON.parse(fs.readFileSync(filePath, 'utf8')) as unknown;
|
||||
}
|
||||
|
||||
function scalarSample(value: unknown) {
|
||||
if (value === null || value === undefined) return '';
|
||||
if (typeof value === 'object') return JSON.stringify(value).slice(0, 160);
|
||||
return String(value).slice(0, 160);
|
||||
}
|
||||
|
||||
function walkSensitiveFields(
|
||||
issues: DryRunIssue[],
|
||||
collection: string,
|
||||
value: unknown,
|
||||
pathParts: string[] = [],
|
||||
depth = 0,
|
||||
) {
|
||||
if (!value || typeof value !== 'object' || depth > 8) return;
|
||||
if (Array.isArray(value)) {
|
||||
value.slice(0, 10).forEach((item, index) => walkSensitiveFields(issues, collection, item, [...pathParts, String(index)], depth + 1));
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [key, childValue] of Object.entries(value as Record<string, unknown>)) {
|
||||
const fieldPath = [...pathParts, key].join('.');
|
||||
if (sensitiveKeyPattern.test(key) && childValue !== null && childValue !== undefined && childValue !== '') {
|
||||
issue(issues, {
|
||||
severity: highRiskCollections.has(collection) ? 'blocker' : 'warning',
|
||||
code: 'sensitive_field_in_export',
|
||||
collection,
|
||||
fieldPath,
|
||||
message: `Legacy export contains sensitive-looking field "${fieldPath}". It must not be copied into public normalized tables.`,
|
||||
sample: scalarSample(childValue),
|
||||
});
|
||||
}
|
||||
walkSensitiveFields(issues, collection, childValue, [...pathParts, key], depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
function fieldValues(record: JsonRecord, field: PocketBaseField) {
|
||||
const value = record[field.name];
|
||||
if (value === null || value === undefined || value === '') return [];
|
||||
if (Array.isArray(value)) return value.map(item => text(item)).filter(Boolean);
|
||||
return [text(value)].filter(Boolean);
|
||||
}
|
||||
|
||||
function relationTargetName(field: PocketBaseField, byId: Map<string, PocketBaseCollection>) {
|
||||
if (!field.collectionId) return '';
|
||||
return byId.get(field.collectionId)?.name || '';
|
||||
}
|
||||
|
||||
function validateRelations(
|
||||
issues: DryRunIssue[],
|
||||
collection: PocketBaseCollection,
|
||||
records: JsonRecord[],
|
||||
idIndex: Map<string, Set<string>>,
|
||||
byId: Map<string, PocketBaseCollection>,
|
||||
) {
|
||||
const relationFields = fieldsOf(collection).filter(field => field.type === 'relation');
|
||||
for (const field of relationFields) {
|
||||
const targetName = relationTargetName(field, byId);
|
||||
if (!targetName) {
|
||||
issue(issues, {
|
||||
severity: 'warning',
|
||||
code: 'relation_target_schema_missing',
|
||||
collection: collection.name,
|
||||
fieldPath: field.name,
|
||||
message: `Relation field "${field.name}" points to an unknown collection id in pb_schema.json.`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const targetIds = idIndex.get(targetName);
|
||||
if (!targetIds) {
|
||||
const hasValues = records.some(record => fieldValues(record, field).length > 0);
|
||||
if (hasValues) {
|
||||
issue(issues, {
|
||||
severity: coreCollections.includes(collection.name) ? 'blocker' : 'warning',
|
||||
code: 'relation_target_export_missing',
|
||||
collection: collection.name,
|
||||
fieldPath: field.name,
|
||||
message: `Relation field "${field.name}" references "${targetName}", but ${targetName}.json is not present in export.`,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let unresolved = 0;
|
||||
let sample = '';
|
||||
for (const record of records) {
|
||||
for (const value of fieldValues(record, field)) {
|
||||
if (!targetIds.has(value)) {
|
||||
unresolved += 1;
|
||||
if (!sample) sample = `${text(record.id) || '(missing id)'} -> ${value}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (unresolved > 0) {
|
||||
issue(issues, {
|
||||
severity: coreCollections.includes(collection.name) ? 'blocker' : 'warning',
|
||||
code: 'relation_unresolved',
|
||||
collection: collection.name,
|
||||
fieldPath: field.name,
|
||||
message: `Relation field "${field.name}" has ${unresolved} unresolved reference(s) to "${targetName}".`,
|
||||
count: unresolved,
|
||||
sample,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function loadSchema(issues: DryRunIssue[]) {
|
||||
try {
|
||||
return readPocketBaseSchema();
|
||||
} catch (error) {
|
||||
issue(issues, {
|
||||
severity: 'warning',
|
||||
code: 'schema_unavailable',
|
||||
message: `pb_schema.json could not be loaded; relation checks will be skipped. ${error instanceof Error ? error.message : ''}`.trim(),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function collectFieldNames(records: JsonRecord[]) {
|
||||
const fields = new Set<string>();
|
||||
for (const record of records.slice(0, 500)) {
|
||||
Object.keys(record).forEach(key => fields.add(key));
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
function buildReport(): DryRunReport {
|
||||
const issues: DryRunIssue[] = [];
|
||||
const schema = loadSchema(issues);
|
||||
const collectionByName = schema?.byName || new Map<string, PocketBaseCollection>();
|
||||
const files = fs.existsSync(exportDir) ? fs.readdirSync(exportDir).filter(file => file.toLowerCase().endsWith('.json')) : [];
|
||||
const collections: CollectionReport[] = [];
|
||||
const recordsByCollection = new Map<string, JsonRecord[]>();
|
||||
const idIndex = new Map<string, Set<string>>();
|
||||
|
||||
if (!fs.existsSync(exportDir)) {
|
||||
issue(issues, {
|
||||
severity: 'blocker',
|
||||
code: 'export_dir_missing',
|
||||
message: `Export dir not found: ${exportDir}`,
|
||||
});
|
||||
}
|
||||
|
||||
if (files.length === 0) {
|
||||
issue(issues, {
|
||||
severity: 'blocker',
|
||||
code: 'export_files_missing',
|
||||
message: `No PocketBase collection JSON files found in ${exportDir}`,
|
||||
});
|
||||
}
|
||||
|
||||
for (const file of files.sort()) {
|
||||
const collection = collectionNameFromFile(file);
|
||||
const filePath = path.join(exportDir, file);
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = readJsonFile(filePath);
|
||||
} catch (error) {
|
||||
issue(issues, {
|
||||
severity: 'blocker',
|
||||
code: 'json_parse_failed',
|
||||
collection,
|
||||
file,
|
||||
message: `Failed to parse ${file}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const records = asRecords(parsed);
|
||||
if (!records) {
|
||||
issue(issues, {
|
||||
severity: 'blocker',
|
||||
code: 'collection_shape_invalid',
|
||||
collection,
|
||||
file,
|
||||
message: `${file} must be an array or an object with items/records array.`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const ids = new Set<string>();
|
||||
const duplicateIds = new Set<string>();
|
||||
let missingIdCount = 0;
|
||||
for (const record of records) {
|
||||
const id = text(record.id);
|
||||
if (!id) {
|
||||
missingIdCount += 1;
|
||||
continue;
|
||||
}
|
||||
if (ids.has(id)) duplicateIds.add(id);
|
||||
ids.add(id);
|
||||
}
|
||||
|
||||
recordsByCollection.set(collection, records);
|
||||
idIndex.set(collection, ids);
|
||||
const fields = collectFieldNames(records);
|
||||
collections.push({
|
||||
collection,
|
||||
file,
|
||||
recordCount: records.length,
|
||||
missingIdCount,
|
||||
duplicateIdCount: duplicateIds.size,
|
||||
fieldCount: fields.size,
|
||||
sampleIds: [...ids].slice(0, 5),
|
||||
supported: supportedCollections.has(collection),
|
||||
});
|
||||
|
||||
if (!supportedCollections.has(collection) && !collection.startsWith('_')) {
|
||||
issue(issues, {
|
||||
severity: 'warning',
|
||||
code: 'unsupported_collection',
|
||||
collection,
|
||||
file,
|
||||
message: `${collection}.json is not currently normalized by import-json.ts. It will be preserved only if a future mapper is added.`,
|
||||
count: records.length,
|
||||
});
|
||||
}
|
||||
if (missingIdCount > 0) {
|
||||
issue(issues, {
|
||||
severity: coreCollections.includes(collection) ? 'blocker' : 'warning',
|
||||
code: 'record_id_missing',
|
||||
collection,
|
||||
file,
|
||||
message: `${collection} has ${missingIdCount} record(s) without id.`,
|
||||
count: missingIdCount,
|
||||
});
|
||||
}
|
||||
if (duplicateIds.size > 0) {
|
||||
issue(issues, {
|
||||
severity: 'blocker',
|
||||
code: 'record_id_duplicate',
|
||||
collection,
|
||||
file,
|
||||
message: `${collection} has duplicate legacy id(s).`,
|
||||
count: duplicateIds.size,
|
||||
sample: [...duplicateIds].slice(0, 5).join(', '),
|
||||
});
|
||||
}
|
||||
|
||||
records.slice(0, 200).forEach(record => walkSensitiveFields(issues, collection, record));
|
||||
}
|
||||
|
||||
for (const collection of coreCollections) {
|
||||
if (!recordsByCollection.has(collection)) {
|
||||
issue(issues, {
|
||||
severity: 'warning',
|
||||
code: 'core_collection_missing',
|
||||
collection,
|
||||
message: `${collection}.json is not present. If old production has this collection, export it before migration.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (schema) {
|
||||
for (const [collectionName, records] of recordsByCollection.entries()) {
|
||||
const collection = collectionByName.get(collectionName);
|
||||
if (collection) validateRelations(issues, collection, records, idIndex, schema.byId);
|
||||
}
|
||||
}
|
||||
|
||||
const businessCounts = Object.fromEntries(
|
||||
businessCollections.map(collection => [collection, recordsByCollection.get(collection)?.length || 0]),
|
||||
);
|
||||
|
||||
const supportedCount = collections.filter(item => item.supported).length;
|
||||
const unsupportedCount = collections.length - supportedCount;
|
||||
const recordCount = collections.reduce((sum, item) => sum + item.recordCount, 0);
|
||||
const blockers = issues.filter(item => item.severity === 'blocker').length;
|
||||
const warnings = issues.filter(item => item.severity === 'warning').length;
|
||||
const infos = issues.filter(item => item.severity === 'info').length;
|
||||
|
||||
return {
|
||||
exportDir,
|
||||
schemaPath: schema?.absPath,
|
||||
generatedAt: new Date().toISOString(),
|
||||
summary: {
|
||||
files: files.length,
|
||||
collections: collections.length,
|
||||
records: recordCount,
|
||||
supportedCollections: supportedCount,
|
||||
unsupportedCollections: unsupportedCount,
|
||||
blockers,
|
||||
warnings,
|
||||
infos,
|
||||
},
|
||||
businessCounts,
|
||||
collections: collections.sort((a, b) => a.collection.localeCompare(b.collection)),
|
||||
issues: issues.sort((a, b) => {
|
||||
const order: Record<Severity, number> = { blocker: 0, warning: 1, info: 2 };
|
||||
return order[a.severity] - order[b.severity] || (a.collection || '').localeCompare(b.collection || '') || a.code.localeCompare(b.code);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function printHuman(report: DryRunReport) {
|
||||
console.log('PocketBase dry-run migration report');
|
||||
console.log(`Export dir: ${report.exportDir}`);
|
||||
if (report.schemaPath) console.log(`Schema: ${report.schemaPath}`);
|
||||
console.log(
|
||||
`Summary: ${report.summary.records} records across ${report.summary.collections} collections; ` +
|
||||
`${report.summary.blockers} blocker(s), ${report.summary.warnings} warning(s)`,
|
||||
);
|
||||
console.log('');
|
||||
console.log('Business counts:');
|
||||
for (const [name, count] of Object.entries(report.businessCounts)) {
|
||||
console.log(` ${name.padEnd(24)} ${count}`);
|
||||
}
|
||||
console.log('');
|
||||
console.log('Collections:');
|
||||
for (const item of report.collections) {
|
||||
const marker = item.supported ? 'mapped' : 'unmapped';
|
||||
console.log(
|
||||
` ${item.collection.padEnd(24)} records=${String(item.recordCount).padStart(6)} ` +
|
||||
`fields=${String(item.fieldCount).padStart(3)} ${marker}`,
|
||||
);
|
||||
}
|
||||
console.log('');
|
||||
console.log('Issues:');
|
||||
if (report.issues.length === 0) {
|
||||
console.log(' none');
|
||||
} else {
|
||||
for (const item of report.issues.slice(0, 200)) {
|
||||
const location = [item.collection, item.fieldPath].filter(Boolean).join('.') || item.file || 'export';
|
||||
const count = item.count === undefined ? '' : ` (${item.count})`;
|
||||
const sample = item.sample ? ` sample=${item.sample}` : '';
|
||||
console.log(` [${item.severity.toUpperCase()}] ${item.code} ${location}: ${item.message}${count}${sample}`);
|
||||
}
|
||||
if (report.issues.length > 200) console.log(` ... ${report.issues.length - 200} more issue(s) omitted`);
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
const report = buildReport();
|
||||
if (jsonOutput) console.log(JSON.stringify(report, null, 2));
|
||||
else printHuman(report);
|
||||
|
||||
if (report.summary.blockers > 0 || (failOnWarnings && report.summary.warnings > 0)) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user