forked from wangziqi/gongxue-base
test: harden pocketbase dry run readiness
This commit is contained in:
Codex
@@ -8,6 +8,7 @@ loadEnv();
|
||||
|
||||
type JsonRecord = Record<string, unknown> & { id?: unknown };
|
||||
type Severity = 'blocker' | 'warning' | 'info';
|
||||
type MigrationProfile = 'development' | 'production';
|
||||
|
||||
interface DryRunIssue {
|
||||
severity: Severity;
|
||||
@@ -35,6 +36,7 @@ interface DryRunReport {
|
||||
exportDir: string;
|
||||
schemaPath?: string;
|
||||
generatedAt: string;
|
||||
migrationProfile: MigrationProfile;
|
||||
summary: {
|
||||
files: number;
|
||||
collections: number;
|
||||
@@ -46,13 +48,37 @@ interface DryRunReport {
|
||||
infos: number;
|
||||
};
|
||||
businessCounts: Record<string, number>;
|
||||
migrationReadiness: {
|
||||
requiredCollections: Array<{
|
||||
collection: string;
|
||||
present: boolean;
|
||||
recordCount: number;
|
||||
minimumRecords: number;
|
||||
}>;
|
||||
criticalFieldCoverage: Array<{
|
||||
collection: string;
|
||||
field: string;
|
||||
present: boolean;
|
||||
missingCount: number;
|
||||
presentCount: number;
|
||||
total: number;
|
||||
missingRatio: number;
|
||||
requiredRatio: number;
|
||||
}>;
|
||||
};
|
||||
collections: CollectionReport[];
|
||||
issues: DryRunIssue[];
|
||||
}
|
||||
|
||||
const args = new Set(process.argv.slice(2));
|
||||
const rawArgs = process.argv.slice(2);
|
||||
const args = new Set(rawArgs);
|
||||
const jsonOutput = args.has('--json');
|
||||
const failOnWarnings = args.has('--fail-on-warnings') || process.env.PB_DRY_RUN_FAIL_ON_WARNINGS === 'true';
|
||||
const profileArg = rawArgs.find(arg => arg.startsWith('--profile='))?.split('=')[1]?.trim();
|
||||
const migrationProfileInput = (profileArg || process.env.PB_DRY_RUN_PROFILE || 'development').trim().toLowerCase();
|
||||
const validMigrationProfiles = new Set(['development', 'production']);
|
||||
const migrationProfile: MigrationProfile = migrationProfileInput === 'production' ? 'production' : 'development';
|
||||
const migrationProfileInvalid = !validMigrationProfiles.has(migrationProfileInput);
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
|
||||
const exportDir = path.resolve(repoRoot, process.env.PB_EXPORT_DIR || 'pb_export');
|
||||
|
||||
@@ -145,6 +171,43 @@ const businessCollections = [
|
||||
'badges',
|
||||
];
|
||||
|
||||
const productionRequiredCollections = [
|
||||
{ collection: 'users', minimumRecords: 1 },
|
||||
{ collection: 'questions', minimumRecords: 1 },
|
||||
{ collection: 'subjects', minimumRecords: 1 },
|
||||
{ collection: 'categories', minimumRecords: 1 },
|
||||
{ collection: 'orders', minimumRecords: 1 },
|
||||
{ collection: 'svip_plans', minimumRecords: 1 },
|
||||
{ collection: 'codes', minimumRecords: 1 },
|
||||
{ collection: 'vocabulary_units', minimumRecords: 1 },
|
||||
{ collection: 'vocabulary', minimumRecords: 1 },
|
||||
{ collection: 'handbook_subjects', minimumRecords: 1 },
|
||||
{ collection: 'handbook_chapters', minimumRecords: 1 },
|
||||
{ collection: 'handbook_entries', minimumRecords: 1 },
|
||||
];
|
||||
|
||||
const criticalFieldRules = [
|
||||
{ collection: 'users', field: 'id', requiredRatio: 1 },
|
||||
{ collection: 'users', field: 'phone', requiredRatio: 0.8 },
|
||||
{ collection: 'questions', field: 'id', requiredRatio: 1 },
|
||||
{ collection: 'questions', field: 'subjectId', requiredRatio: 0.95 },
|
||||
{ collection: 'questions', field: 'categoryId', requiredRatio: 0.8 },
|
||||
{ collection: 'questions', field: 'content', requiredRatio: 0.95, alternatives: ['question', 'title', 'stem'] },
|
||||
{ collection: 'orders', field: 'userId', requiredRatio: 0.98 },
|
||||
{ collection: 'orders', field: 'planId', requiredRatio: 0.8 },
|
||||
{ collection: 'orders', field: 'status', requiredRatio: 0.95 },
|
||||
{ collection: 'codes', field: 'code', requiredRatio: 0.98 },
|
||||
{ collection: 'vocabulary', field: 'unitId', requiredRatio: 0.95 },
|
||||
{ collection: 'vocabulary', field: 'word', requiredRatio: 0.98 },
|
||||
{ collection: 'handbook_chapters', field: 'subjectId', requiredRatio: 0.95 },
|
||||
{ collection: 'handbook_entries', field: 'chapterId', requiredRatio: 0.95 },
|
||||
{ collection: 'scoreline_records', field: 'schoolId', requiredRatio: 0.8 },
|
||||
{ collection: 'question_videos', field: 'questionId', requiredRatio: 0.95 },
|
||||
{ collection: 'question_videos', field: 'videoId', requiredRatio: 0.95 },
|
||||
{ collection: 'user_word_progress', field: 'userId', requiredRatio: 0.95 },
|
||||
{ collection: 'user_word_favorites', field: 'userId', requiredRatio: 0.95 },
|
||||
];
|
||||
|
||||
const sensitiveKeyPattern =
|
||||
/(password|token|secret|privatekey|sessionkey|accesskey|appkey|apikey|api_v3_key|notifytoken|aeskey|openid|unionid|wxaccesstoken|wechatsessionkey|smscode|verifycode|verificationcode|captcha)/i;
|
||||
const highRiskCollections = new Set(['settings', 'crm_config', 'smscodes']);
|
||||
@@ -172,6 +235,10 @@ function text(value: unknown) {
|
||||
return String(value).trim();
|
||||
}
|
||||
|
||||
function hasValue(record: JsonRecord, field: string, alternatives: string[] = []) {
|
||||
return [field, ...alternatives].some(key => text(record[key]));
|
||||
}
|
||||
|
||||
function readJsonFile(filePath: string) {
|
||||
return JSON.parse(fs.readFileSync(filePath, 'utf8')) as unknown;
|
||||
}
|
||||
@@ -284,6 +351,68 @@ function validateRelations(
|
||||
}
|
||||
}
|
||||
|
||||
function buildReadiness(
|
||||
issues: DryRunIssue[],
|
||||
recordsByCollection: Map<string, JsonRecord[]>,
|
||||
): DryRunReport['migrationReadiness'] {
|
||||
const requiredCollections = productionRequiredCollections.map(rule => {
|
||||
const count = recordsByCollection.get(rule.collection)?.length || 0;
|
||||
const present = count >= rule.minimumRecords;
|
||||
if (migrationProfile === 'production' && !present) {
|
||||
issue(issues, {
|
||||
severity: 'blocker',
|
||||
code: 'production_required_collection_missing',
|
||||
collection: rule.collection,
|
||||
message: `${rule.collection}.json is required for production migration and must contain at least ${rule.minimumRecords} record(s).`,
|
||||
count,
|
||||
});
|
||||
}
|
||||
return {
|
||||
collection: rule.collection,
|
||||
present,
|
||||
recordCount: count,
|
||||
minimumRecords: rule.minimumRecords,
|
||||
};
|
||||
});
|
||||
|
||||
const criticalFieldCoverage = criticalFieldRules.flatMap(rule => {
|
||||
const records = recordsByCollection.get(rule.collection);
|
||||
if (!records || records.length === 0) return [];
|
||||
let missingCount = 0;
|
||||
for (const record of records) {
|
||||
if (!hasValue(record, rule.field, rule.alternatives)) missingCount += 1;
|
||||
}
|
||||
const total = records.length;
|
||||
const presentCount = total - missingCount;
|
||||
const presentRatio = total === 0 ? 1 : presentCount / total;
|
||||
const missingRatio = total === 0 ? 0 : missingCount / total;
|
||||
if (presentRatio < rule.requiredRatio) {
|
||||
issue(issues, {
|
||||
severity: migrationProfile === 'production' ? 'blocker' : 'warning',
|
||||
code: 'critical_field_coverage_low',
|
||||
collection: rule.collection,
|
||||
fieldPath: rule.field,
|
||||
message:
|
||||
`${rule.collection}.${rule.field} coverage is ${(presentRatio * 100).toFixed(1)}%, ` +
|
||||
`below required ${(rule.requiredRatio * 100).toFixed(1)}% for reliable migration.`,
|
||||
count: missingCount,
|
||||
});
|
||||
}
|
||||
return [{
|
||||
collection: rule.collection,
|
||||
field: rule.field,
|
||||
present: presentRatio >= rule.requiredRatio,
|
||||
missingCount,
|
||||
presentCount,
|
||||
total,
|
||||
missingRatio,
|
||||
requiredRatio: rule.requiredRatio,
|
||||
}];
|
||||
});
|
||||
|
||||
return { requiredCollections, criticalFieldCoverage };
|
||||
}
|
||||
|
||||
function loadSchema(issues: DryRunIssue[]) {
|
||||
try {
|
||||
return readPocketBaseSchema();
|
||||
@@ -307,6 +436,13 @@ function collectFieldNames(records: JsonRecord[]) {
|
||||
|
||||
function buildReport(): DryRunReport {
|
||||
const issues: DryRunIssue[] = [];
|
||||
if (migrationProfileInvalid) {
|
||||
issue(issues, {
|
||||
severity: 'blocker',
|
||||
code: 'migration_profile_invalid',
|
||||
message: `Invalid migration profile "${migrationProfileInput}". Use "development" or "production".`,
|
||||
});
|
||||
}
|
||||
const schema = loadSchema(issues);
|
||||
const collectionByName = schema?.byName || new Map<string, PocketBaseCollection>();
|
||||
const files = fs.existsSync(exportDir) ? fs.readdirSync(exportDir).filter(file => file.toLowerCase().endsWith('.json')) : [];
|
||||
@@ -439,6 +575,8 @@ function buildReport(): DryRunReport {
|
||||
}
|
||||
}
|
||||
|
||||
const migrationReadiness = buildReadiness(issues, recordsByCollection);
|
||||
|
||||
const businessCounts = Object.fromEntries(
|
||||
businessCollections.map(collection => [collection, recordsByCollection.get(collection)?.length || 0]),
|
||||
);
|
||||
@@ -454,6 +592,7 @@ function buildReport(): DryRunReport {
|
||||
exportDir,
|
||||
schemaPath: schema?.absPath,
|
||||
generatedAt: new Date().toISOString(),
|
||||
migrationProfile,
|
||||
summary: {
|
||||
files: files.length,
|
||||
collections: collections.length,
|
||||
@@ -465,6 +604,7 @@ function buildReport(): DryRunReport {
|
||||
infos,
|
||||
},
|
||||
businessCounts,
|
||||
migrationReadiness,
|
||||
collections: collections.sort((a, b) => a.collection.localeCompare(b.collection)),
|
||||
issues: issues.sort((a, b) => {
|
||||
const order: Record<Severity, number> = { blocker: 0, warning: 1, info: 2 };
|
||||
@@ -477,11 +617,27 @@ function printHuman(report: DryRunReport) {
|
||||
console.log('PocketBase dry-run migration report');
|
||||
console.log(`Export dir: ${report.exportDir}`);
|
||||
if (report.schemaPath) console.log(`Schema: ${report.schemaPath}`);
|
||||
console.log(`Migration profile: ${report.migrationProfile}`);
|
||||
console.log(
|
||||
`Summary: ${report.summary.records} records across ${report.summary.collections} collections; ` +
|
||||
`${report.summary.blockers} blocker(s), ${report.summary.warnings} warning(s)`,
|
||||
);
|
||||
console.log('');
|
||||
console.log('Migration readiness:');
|
||||
const missingRequired = report.migrationReadiness.requiredCollections.filter(item => !item.present);
|
||||
const weakFields = report.migrationReadiness.criticalFieldCoverage.filter(item => !item.present);
|
||||
console.log(` required collections missing/too small: ${missingRequired.length}`);
|
||||
console.log(` critical field coverage warnings: ${weakFields.length}`);
|
||||
for (const item of missingRequired.slice(0, 20)) {
|
||||
console.log(` ${item.collection}: records=${item.recordCount}, minimum=${item.minimumRecords}`);
|
||||
}
|
||||
for (const item of weakFields.slice(0, 20)) {
|
||||
console.log(
|
||||
` ${item.collection}.${item.field}: present=${item.presentCount}/${item.total}, ` +
|
||||
`required=${(item.requiredRatio * 100).toFixed(1)}%`,
|
||||
);
|
||||
}
|
||||
console.log('');
|
||||
console.log('Business counts:');
|
||||
for (const [name, count] of Object.entries(report.businessCounts)) {
|
||||
console.log(` ${name.padEnd(24)} ${count}`);
|
||||
|
||||
Reference in New Issue
Block a user