import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { fieldsOf, readPocketBaseSchema, type PocketBaseCollection, type PocketBaseField } from './pb-schema.js'; import { loadEnv } from './env.js'; loadEnv(); type JsonRecord = Record & { id?: unknown }; type Severity = 'blocker' | 'warning' | 'info'; type MigrationProfile = 'development' | 'production'; interface DryRunIssue { severity: Severity; code: string; collection?: string; file?: string; fieldPath?: string; message: string; sample?: string; count?: number; } interface CollectionReport { collection: string; file: string; recordCount: number; missingIdCount: number; duplicateIdCount: number; fieldCount: number; sampleIds: string[]; supported: boolean; } interface DryRunReport { exportDir: string; schemaPath?: string; generatedAt: string; migrationProfile: MigrationProfile; summary: { files: number; collections: number; records: number; supportedCollections: number; unsupportedCollections: number; blockers: number; warnings: number; infos: number; }; businessCounts: Record; migrationReadiness: { requiredCollections: Array<{ collection: string; present: boolean; recordCount: number; minimumRecords: number; }>; criticalFieldCoverage: Array<{ collection: string; field: string; present: boolean; missingCount: number; presentCount: number; total: number; missingRatio: number; requiredRatio: number; }>; }; collections: CollectionReport[]; issues: DryRunIssue[]; } const rawArgs = process.argv.slice(2); const args = new Set(rawArgs); const jsonOutput = args.has('--json'); const failOnWarnings = args.has('--fail-on-warnings') || process.env.PB_DRY_RUN_FAIL_ON_WARNINGS === 'true'; const profileArg = rawArgs.find(arg => arg.startsWith('--profile='))?.split('=')[1]?.trim(); const migrationProfileInput = (profileArg || process.env.PB_DRY_RUN_PROFILE || 'development').trim().toLowerCase(); const validMigrationProfiles = new Set(['development', 'production']); const migrationProfile: MigrationProfile = migrationProfileInput === 'production' ? 'production' : 'development'; const migrationProfileInvalid = !validMigrationProfiles.has(migrationProfileInput); const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..'); const exportDir = path.resolve(repoRoot, process.env.PB_EXPORT_DIR || 'pb_export'); const nonCollectionJsonFiles = new Set([ 'pb_schema.sqlite.json', 'sqlite-export-manifest.json', 'storage-manifest.json', ]); const supportedCollections = new Set([ 'announcements', 'app_assets', 'audit_logs', 'badges', 'banners', 'categories', 'code_batches', 'codes', 'coupons', 'coupon_redemptions', 'crm_config', 'crm_webhook_log', 'crm_webhook_queue', 'dashboard_daily_stats', 'exam_dates', 'faqs', 'handbook_chapters', 'handbook_entries', 'handbook_subjects', 'images', 'majors', 'module_nodes', 'orders', 'products', 'question_type_groups', 'question_videos', 'questions', 'recent_practices', 'referral_tracks', 'region_modules', 'regions', 'reports', 'revenue_daily_stats', 'schools', 'scoreline_fields', 'scoreline_majors', 'scoreline_records', 'scoreline_schools', 'settings', 'subject_shares', 'subjects', 'svip_plans', 'timelines', 'user_badges', 'user_word_favorites', 'user_word_progress', 'users', 'video_explanations', 'vocabulary', 'vocabulary_units', ]); const coreCollections = [ 'users', 'questions', 'subjects', 'categories', 'orders', 'svip_plans', 'codes', 'vocabulary_units', 'vocabulary', 'handbook_subjects', 'handbook_chapters', 'handbook_entries', ]; const businessCollections = [ 'users', 'questions', 'orders', 'svip_plans', 'codes', 'vocabulary_units', 'vocabulary', 'handbook_subjects', 'handbook_chapters', 'handbook_entries', 'scoreline_schools', 'scoreline_majors', 'scoreline_records', 'video_explanations', 'question_videos', 'referral_tracks', 'coupons', 'badges', ]; const productionRequiredCollections = [ { collection: 'users', minimumRecords: 1 }, { collection: 'questions', minimumRecords: 1 }, { collection: 'subjects', minimumRecords: 1 }, { collection: 'categories', minimumRecords: 1 }, { collection: 'orders', minimumRecords: 1 }, { collection: 'svip_plans', minimumRecords: 1 }, { collection: 'codes', minimumRecords: 1 }, { collection: 'vocabulary_units', minimumRecords: 1 }, { collection: 'vocabulary', minimumRecords: 1 }, { collection: 'handbook_subjects', minimumRecords: 1 }, { collection: 'handbook_chapters', minimumRecords: 1 }, { collection: 'handbook_entries', minimumRecords: 1 }, ]; const criticalFieldRules = [ { collection: 'users', field: 'id', requiredRatio: 1 }, { collection: 'users', field: 'phone', requiredRatio: 0.8 }, { collection: 'questions', field: 'id', requiredRatio: 1 }, { collection: 'questions', field: 'subjectId', requiredRatio: 0.95 }, { collection: 'questions', field: 'categoryId', requiredRatio: 0.8, alternatives: ['nodeId'] }, { collection: 'questions', field: 'content', requiredRatio: 0.95, alternatives: ['question', 'title', 'stem'] }, { collection: 'orders', field: 'userId', requiredRatio: 0.98 }, { collection: 'orders', field: 'planId', requiredRatio: 0.8 }, { collection: 'orders', field: 'status', requiredRatio: 0.95 }, { collection: 'codes', field: 'code', requiredRatio: 0.98 }, { collection: 'vocabulary', field: 'unitId', requiredRatio: 0.95, alternatives: ['unit'] }, { collection: 'vocabulary', field: 'word', requiredRatio: 0.98 }, { collection: 'handbook_chapters', field: 'subjectId', requiredRatio: 0.95 }, { collection: 'handbook_entries', field: 'chapterId', requiredRatio: 0.95 }, { collection: 'scoreline_records', field: 'schoolId', requiredRatio: 0.8 }, { collection: 'question_videos', field: 'questionId', requiredRatio: 0.95 }, { collection: 'question_videos', field: 'videoId', requiredRatio: 0.95 }, { collection: 'user_word_progress', field: 'userId', requiredRatio: 0.95 }, { collection: 'user_word_favorites', field: 'userId', requiredRatio: 0.95 }, ]; const sensitiveKeyPattern = /(password|token|secret|privatekey|sessionkey|accesskey|appkey|apikey|api_v3_key|notifytoken|aeskey|openid|unionid|wxaccesstoken|wechatsessionkey|smscode|verifycode|verificationcode|captcha)/i; const highRiskCollections = new Set(['settings', 'crm_config', 'smscodes']); function issue(issues: DryRunIssue[], nextIssue: DryRunIssue) { issues.push(nextIssue); } function asRecords(input: unknown): JsonRecord[] | null { if (Array.isArray(input)) return input as JsonRecord[]; if (input && typeof input === 'object') { const candidate = input as { items?: unknown; records?: unknown }; if (Array.isArray(candidate.items)) return candidate.items as JsonRecord[]; if (Array.isArray(candidate.records)) return candidate.records as JsonRecord[]; } return null; } function collectionNameFromFile(fileName: string) { return fileName.replace(/\.json$/i, ''); } function text(value: unknown) { if (value === null || value === undefined) return ''; return String(value).trim(); } function hasValue(record: JsonRecord, field: string, alternatives: string[] = []) { return [field, ...alternatives].some(key => text(record[key])); } function readJsonFile(filePath: string) { return JSON.parse(fs.readFileSync(filePath, 'utf8')) as unknown; } function scalarSample(value: unknown) { if (value === null || value === undefined) return ''; if (typeof value === 'object') return JSON.stringify(value).slice(0, 160); return String(value).slice(0, 160); } function walkSensitiveFields( issues: DryRunIssue[], collection: string, value: unknown, pathParts: string[] = [], depth = 0, ) { if (!value || typeof value !== 'object' || depth > 8) return; if (Array.isArray(value)) { value.slice(0, 10).forEach((item, index) => walkSensitiveFields(issues, collection, item, [...pathParts, String(index)], depth + 1)); return; } for (const [key, childValue] of Object.entries(value as Record)) { const fieldPath = [...pathParts, key].join('.'); if (sensitiveKeyPattern.test(key) && childValue !== null && childValue !== undefined && childValue !== '') { issue(issues, { severity: highRiskCollections.has(collection) ? 'blocker' : 'warning', code: 'sensitive_field_in_export', collection, fieldPath, message: `Legacy export contains sensitive-looking field "${fieldPath}". It must not be copied into public normalized tables.`, sample: scalarSample(childValue), }); } walkSensitiveFields(issues, collection, childValue, [...pathParts, key], depth + 1); } } function fieldValues(record: JsonRecord, field: PocketBaseField) { const value = record[field.name]; if (value === null || value === undefined || value === '') return []; if (Array.isArray(value)) return value.map(item => text(item)).filter(Boolean); return [text(value)].filter(Boolean); } function relationTargetName(field: PocketBaseField, byId: Map) { if (!field.collectionId) return ''; return byId.get(field.collectionId)?.name || ''; } function validateRelations( issues: DryRunIssue[], collection: PocketBaseCollection, records: JsonRecord[], idIndex: Map>, byId: Map, ) { const relationFields = fieldsOf(collection).filter(field => field.type === 'relation'); for (const field of relationFields) { const targetName = relationTargetName(field, byId); if (!targetName) { issue(issues, { severity: 'warning', code: 'relation_target_schema_missing', collection: collection.name, fieldPath: field.name, message: `Relation field "${field.name}" points to an unknown collection id in pb_schema.json.`, }); continue; } const targetIds = idIndex.get(targetName); if (!targetIds) { const hasValues = records.some(record => fieldValues(record, field).length > 0); if (hasValues) { issue(issues, { severity: coreCollections.includes(collection.name) ? 'blocker' : 'warning', code: 'relation_target_export_missing', collection: collection.name, fieldPath: field.name, message: `Relation field "${field.name}" references "${targetName}", but ${targetName}.json is not present in export.`, }); } continue; } let unresolved = 0; let sample = ''; for (const record of records) { for (const value of fieldValues(record, field)) { if (!targetIds.has(value)) { if (collection.name === 'questions' && field.name === 'categoryId' && idIndex.get('module_nodes')?.has(value)) { continue; } unresolved += 1; if (!sample) sample = `${text(record.id) || '(missing id)'} -> ${value}`; } } } if (unresolved > 0) { issue(issues, { severity: coreCollections.includes(collection.name) ? 'blocker' : 'warning', code: 'relation_unresolved', collection: collection.name, fieldPath: field.name, message: `Relation field "${field.name}" has ${unresolved} unresolved reference(s) to "${targetName}".`, count: unresolved, sample, }); } } } function buildReadiness( issues: DryRunIssue[], recordsByCollection: Map, ): DryRunReport['migrationReadiness'] { const requiredCollections = productionRequiredCollections.map(rule => { const count = recordsByCollection.get(rule.collection)?.length || 0; const present = count >= rule.minimumRecords; if (migrationProfile === 'production' && !present) { issue(issues, { severity: 'blocker', code: 'production_required_collection_missing', collection: rule.collection, message: `${rule.collection}.json is required for production migration and must contain at least ${rule.minimumRecords} record(s).`, count, }); } return { collection: rule.collection, present, recordCount: count, minimumRecords: rule.minimumRecords, }; }); const criticalFieldCoverage = criticalFieldRules.flatMap(rule => { const records = recordsByCollection.get(rule.collection); if (!records || records.length === 0) return []; let missingCount = 0; for (const record of records) { if (!hasValue(record, rule.field, rule.alternatives)) missingCount += 1; } const total = records.length; const presentCount = total - missingCount; const presentRatio = total === 0 ? 1 : presentCount / total; const missingRatio = total === 0 ? 0 : missingCount / total; if (presentRatio < rule.requiredRatio) { issue(issues, { severity: migrationProfile === 'production' ? 'blocker' : 'warning', code: 'critical_field_coverage_low', collection: rule.collection, fieldPath: rule.field, message: `${rule.collection}.${rule.field} coverage is ${(presentRatio * 100).toFixed(1)}%, ` + `below required ${(rule.requiredRatio * 100).toFixed(1)}% for reliable migration.`, count: missingCount, }); } return [{ collection: rule.collection, field: rule.field, present: presentRatio >= rule.requiredRatio, missingCount, presentCount, total, missingRatio, requiredRatio: rule.requiredRatio, }]; }); return { requiredCollections, criticalFieldCoverage }; } function loadSchema(issues: DryRunIssue[]) { try { return readPocketBaseSchema(); } catch (error) { issue(issues, { severity: 'warning', code: 'schema_unavailable', message: `pb_schema.json could not be loaded; relation checks will be skipped. ${error instanceof Error ? error.message : ''}`.trim(), }); return null; } } function collectFieldNames(records: JsonRecord[]) { const fields = new Set(); for (const record of records.slice(0, 500)) { Object.keys(record).forEach(key => fields.add(key)); } return fields; } function buildReport(): DryRunReport { const issues: DryRunIssue[] = []; if (migrationProfileInvalid) { issue(issues, { severity: 'blocker', code: 'migration_profile_invalid', message: `Invalid migration profile "${migrationProfileInput}". Use "development" or "production".`, }); } const schema = loadSchema(issues); const collectionByName = schema?.byName || new Map(); const files = fs.existsSync(exportDir) ? fs .readdirSync(exportDir) .filter(file => file.toLowerCase().endsWith('.json')) .filter(file => !nonCollectionJsonFiles.has(file.toLowerCase())) : []; const collections: CollectionReport[] = []; const recordsByCollection = new Map(); const idIndex = new Map>(); if (!fs.existsSync(exportDir)) { issue(issues, { severity: 'blocker', code: 'export_dir_missing', message: `Export dir not found: ${exportDir}`, }); } if (files.length === 0) { issue(issues, { severity: 'blocker', code: 'export_files_missing', message: `No PocketBase collection JSON files found in ${exportDir}`, }); } for (const file of files.sort()) { const collection = collectionNameFromFile(file); const filePath = path.join(exportDir, file); let parsed: unknown; try { parsed = readJsonFile(filePath); } catch (error) { issue(issues, { severity: 'blocker', code: 'json_parse_failed', collection, file, message: `Failed to parse ${file}: ${error instanceof Error ? error.message : String(error)}`, }); continue; } const records = asRecords(parsed); if (!records) { issue(issues, { severity: 'blocker', code: 'collection_shape_invalid', collection, file, message: `${file} must be an array or an object with items/records array.`, }); continue; } const ids = new Set(); const duplicateIds = new Set(); let missingIdCount = 0; for (const record of records) { const id = text(record.id); if (!id) { missingIdCount += 1; continue; } if (ids.has(id)) duplicateIds.add(id); ids.add(id); } recordsByCollection.set(collection, records); idIndex.set(collection, ids); const fields = collectFieldNames(records); collections.push({ collection, file, recordCount: records.length, missingIdCount, duplicateIdCount: duplicateIds.size, fieldCount: fields.size, sampleIds: [...ids].slice(0, 5), supported: supportedCollections.has(collection), }); if (!supportedCollections.has(collection) && !collection.startsWith('_')) { issue(issues, { severity: 'warning', code: 'unsupported_collection', collection, file, message: `${collection}.json is not currently normalized by import-json.ts. It will be preserved only if a future mapper is added.`, count: records.length, }); } if (missingIdCount > 0) { issue(issues, { severity: coreCollections.includes(collection) ? 'blocker' : 'warning', code: 'record_id_missing', collection, file, message: `${collection} has ${missingIdCount} record(s) without id.`, count: missingIdCount, }); } if (duplicateIds.size > 0) { issue(issues, { severity: 'blocker', code: 'record_id_duplicate', collection, file, message: `${collection} has duplicate legacy id(s).`, count: duplicateIds.size, sample: [...duplicateIds].slice(0, 5).join(', '), }); } records.slice(0, 200).forEach(record => walkSensitiveFields(issues, collection, record)); } for (const collection of coreCollections) { if (!recordsByCollection.has(collection)) { issue(issues, { severity: 'warning', code: 'core_collection_missing', collection, message: `${collection}.json is not present. If old production has this collection, export it before migration.`, }); } } if (schema) { for (const [collectionName, records] of recordsByCollection.entries()) { const collection = collectionByName.get(collectionName); if (collection) validateRelations(issues, collection, records, idIndex, schema.byId); } } const migrationReadiness = buildReadiness(issues, recordsByCollection); const businessCounts = Object.fromEntries( businessCollections.map(collection => [collection, recordsByCollection.get(collection)?.length || 0]), ); const supportedCount = collections.filter(item => item.supported).length; const unsupportedCount = collections.length - supportedCount; const recordCount = collections.reduce((sum, item) => sum + item.recordCount, 0); const blockers = issues.filter(item => item.severity === 'blocker').length; const warnings = issues.filter(item => item.severity === 'warning').length; const infos = issues.filter(item => item.severity === 'info').length; return { exportDir, schemaPath: schema?.absPath, generatedAt: new Date().toISOString(), migrationProfile, summary: { files: files.length, collections: collections.length, records: recordCount, supportedCollections: supportedCount, unsupportedCollections: unsupportedCount, blockers, warnings, infos, }, businessCounts, migrationReadiness, collections: collections.sort((a, b) => a.collection.localeCompare(b.collection)), issues: issues.sort((a, b) => { const order: Record = { blocker: 0, warning: 1, info: 2 }; return order[a.severity] - order[b.severity] || (a.collection || '').localeCompare(b.collection || '') || a.code.localeCompare(b.code); }), }; } function printHuman(report: DryRunReport) { console.log('PocketBase dry-run migration report'); console.log(`Export dir: ${report.exportDir}`); if (report.schemaPath) console.log(`Schema: ${report.schemaPath}`); console.log(`Migration profile: ${report.migrationProfile}`); console.log( `Summary: ${report.summary.records} records across ${report.summary.collections} collections; ` + `${report.summary.blockers} blocker(s), ${report.summary.warnings} warning(s)`, ); console.log(''); console.log('Migration readiness:'); const missingRequired = report.migrationReadiness.requiredCollections.filter(item => !item.present); const weakFields = report.migrationReadiness.criticalFieldCoverage.filter(item => !item.present); console.log(` required collections missing/too small: ${missingRequired.length}`); console.log(` critical field coverage warnings: ${weakFields.length}`); for (const item of missingRequired.slice(0, 20)) { console.log(` ${item.collection}: records=${item.recordCount}, minimum=${item.minimumRecords}`); } for (const item of weakFields.slice(0, 20)) { console.log( ` ${item.collection}.${item.field}: present=${item.presentCount}/${item.total}, ` + `required=${(item.requiredRatio * 100).toFixed(1)}%`, ); } console.log(''); console.log('Business counts:'); for (const [name, count] of Object.entries(report.businessCounts)) { console.log(` ${name.padEnd(24)} ${count}`); } console.log(''); console.log('Collections:'); for (const item of report.collections) { const marker = item.supported ? 'mapped' : 'unmapped'; console.log( ` ${item.collection.padEnd(24)} records=${String(item.recordCount).padStart(6)} ` + `fields=${String(item.fieldCount).padStart(3)} ${marker}`, ); } console.log(''); console.log('Issues:'); if (report.issues.length === 0) { console.log(' none'); } else { for (const item of report.issues.slice(0, 200)) { const location = [item.collection, item.fieldPath].filter(Boolean).join('.') || item.file || 'export'; const count = item.count === undefined ? '' : ` (${item.count})`; const sample = item.sample ? ` sample=${item.sample}` : ''; console.log(` [${item.severity.toUpperCase()}] ${item.code} ${location}: ${item.message}${count}${sample}`); } if (report.issues.length > 200) console.log(` ... ${report.issues.length - 200} more issue(s) omitted`); } } function main() { const report = buildReport(); if (jsonOutput) console.log(JSON.stringify(report, null, 2)); else printHuman(report); if (report.summary.blockers > 0 || (failOnWarnings && report.summary.warnings > 0)) { process.exitCode = 1; } } main();