xiongyuxing/gongxue-base
1
0
Fork 0
forked from wangziqi/gongxue-base
Code Pull Requests Activity

test: harden pocketbase dry run readiness

Browse Source
This commit is contained in:
Codex
2026-06-30 02:23:17 +08:00
parent 7eaf26fa40
commit 52cef9f13d
7 changed files with 223 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
scripts/import-pocketbase/src/dry-run-report.ts
View File

@@ -8,6 +8,7 @@ loadEnv();
type JsonRecord = Record<string, unknown> & { id?: unknown };
type Severity = 'blocker' | 'warning' | 'info';
type MigrationProfile = 'development' | 'production';
interface DryRunIssue {
severity: Severity;
@@ -35,6 +36,7 @@ interface DryRunReport {
exportDir: string;
schemaPath?: string;
generatedAt: string;
migrationProfile: MigrationProfile;
summary: {
files: number;
collections: number;
@@ -46,13 +48,37 @@ interface DryRunReport {
infos: number;
};
businessCounts: Record<string, number>;
migrationReadiness: {
requiredCollections: Array<{
collection: string;
present: boolean;
recordCount: number;
minimumRecords: number;
}>;
criticalFieldCoverage: Array<{
collection: string;
field: string;
present: boolean;
missingCount: number;
presentCount: number;
total: number;
missingRatio: number;
requiredRatio: number;
}>;
};
collections: CollectionReport[];
issues: DryRunIssue[];
}
const args = new Set(process.argv.slice(2));
const rawArgs = process.argv.slice(2);
const args = new Set(rawArgs);
const jsonOutput = args.has('--json');
const failOnWarnings = args.has('--fail-on-warnings') || process.env.PB_DRY_RUN_FAIL_ON_WARNINGS === 'true';
const profileArg = rawArgs.find(arg => arg.startsWith('--profile='))?.split('=')[1]?.trim();
const migrationProfileInput = (profileArg || process.env.PB_DRY_RUN_PROFILE || 'development').trim().toLowerCase();
const validMigrationProfiles = new Set(['development', 'production']);
const migrationProfile: MigrationProfile = migrationProfileInput === 'production' ? 'production' : 'development';
const migrationProfileInvalid = !validMigrationProfiles.has(migrationProfileInput);
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
const exportDir = path.resolve(repoRoot, process.env.PB_EXPORT_DIR || 'pb_export');
@@ -145,6 +171,43 @@ const businessCollections = [
'badges',
];
const productionRequiredCollections = [
{ collection: 'users', minimumRecords: 1 },
{ collection: 'questions', minimumRecords: 1 },
{ collection: 'subjects', minimumRecords: 1 },
{ collection: 'categories', minimumRecords: 1 },
{ collection: 'orders', minimumRecords: 1 },
{ collection: 'svip_plans', minimumRecords: 1 },
{ collection: 'codes', minimumRecords: 1 },
{ collection: 'vocabulary_units', minimumRecords: 1 },
{ collection: 'vocabulary', minimumRecords: 1 },
{ collection: 'handbook_subjects', minimumRecords: 1 },
{ collection: 'handbook_chapters', minimumRecords: 1 },
{ collection: 'handbook_entries', minimumRecords: 1 },
];
const criticalFieldRules = [
{ collection: 'users', field: 'id', requiredRatio: 1 },
{ collection: 'users', field: 'phone', requiredRatio: 0.8 },
{ collection: 'questions', field: 'id', requiredRatio: 1 },
{ collection: 'questions', field: 'subjectId', requiredRatio: 0.95 },
{ collection: 'questions', field: 'categoryId', requiredRatio: 0.8 },
{ collection: 'questions', field: 'content', requiredRatio: 0.95, alternatives: ['question', 'title', 'stem'] },
{ collection: 'orders', field: 'userId', requiredRatio: 0.98 },
{ collection: 'orders', field: 'planId', requiredRatio: 0.8 },
{ collection: 'orders', field: 'status', requiredRatio: 0.95 },
{ collection: 'codes', field: 'code', requiredRatio: 0.98 },
{ collection: 'vocabulary', field: 'unitId', requiredRatio: 0.95 },
{ collection: 'vocabulary', field: 'word', requiredRatio: 0.98 },
{ collection: 'handbook_chapters', field: 'subjectId', requiredRatio: 0.95 },
{ collection: 'handbook_entries', field: 'chapterId', requiredRatio: 0.95 },
{ collection: 'scoreline_records', field: 'schoolId', requiredRatio: 0.8 },
{ collection: 'question_videos', field: 'questionId', requiredRatio: 0.95 },
{ collection: 'question_videos', field: 'videoId', requiredRatio: 0.95 },
{ collection: 'user_word_progress', field: 'userId', requiredRatio: 0.95 },
{ collection: 'user_word_favorites', field: 'userId', requiredRatio: 0.95 },
];
const sensitiveKeyPattern =
/(password|token|secret|privatekey|sessionkey|accesskey|appkey|apikey|api_v3_key|notifytoken|aeskey|openid|unionid|wxaccesstoken|wechatsessionkey|smscode|verifycode|verificationcode|captcha)/i;
const highRiskCollections = new Set(['settings', 'crm_config', 'smscodes']);
@@ -172,6 +235,10 @@ function text(value: unknown) {
return String(value).trim();
}
function hasValue(record: JsonRecord, field: string, alternatives: string[] = []) {
return [field, ...alternatives].some(key => text(record[key]));
}
function readJsonFile(filePath: string) {
return JSON.parse(fs.readFileSync(filePath, 'utf8')) as unknown;
}
@@ -284,6 +351,68 @@ function validateRelations(
}
}
function buildReadiness(
issues: DryRunIssue[],
recordsByCollection: Map<string, JsonRecord[]>,
): DryRunReport['migrationReadiness'] {
const requiredCollections = productionRequiredCollections.map(rule => {
const count = recordsByCollection.get(rule.collection)?.length || 0;
const present = count >= rule.minimumRecords;
if (migrationProfile === 'production' && !present) {
issue(issues, {
severity: 'blocker',
code: 'production_required_collection_missing',
collection: rule.collection,
message: `${rule.collection}.json is required for production migration and must contain at least ${rule.minimumRecords} record(s).`,
count,
});
}
return {
collection: rule.collection,
present,
recordCount: count,
minimumRecords: rule.minimumRecords,
};
});
const criticalFieldCoverage = criticalFieldRules.flatMap(rule => {
const records = recordsByCollection.get(rule.collection);
if (!records || records.length === 0) return [];
let missingCount = 0;
for (const record of records) {
if (!hasValue(record, rule.field, rule.alternatives)) missingCount += 1;
}
const total = records.length;
const presentCount = total - missingCount;
const presentRatio = total === 0 ? 1 : presentCount / total;
const missingRatio = total === 0 ? 0 : missingCount / total;
if (presentRatio < rule.requiredRatio) {
issue(issues, {
severity: migrationProfile === 'production' ? 'blocker' : 'warning',
code: 'critical_field_coverage_low',
collection: rule.collection,
fieldPath: rule.field,
message:
`${rule.collection}.${rule.field} coverage is ${(presentRatio * 100).toFixed(1)}%, ` +
`below required ${(rule.requiredRatio * 100).toFixed(1)}% for reliable migration.`,
count: missingCount,
});
}
return [{
collection: rule.collection,
field: rule.field,
present: presentRatio >= rule.requiredRatio,
missingCount,
presentCount,
total,
missingRatio,
requiredRatio: rule.requiredRatio,
}];
});
return { requiredCollections, criticalFieldCoverage };
}
function loadSchema(issues: DryRunIssue[]) {
try {
return readPocketBaseSchema();
@@ -307,6 +436,13 @@ function collectFieldNames(records: JsonRecord[]) {
function buildReport(): DryRunReport {
const issues: DryRunIssue[] = [];
if (migrationProfileInvalid) {
issue(issues, {
severity: 'blocker',
code: 'migration_profile_invalid',
message: `Invalid migration profile "${migrationProfileInput}". Use "development" or "production".`,
});
}
const schema = loadSchema(issues);
const collectionByName = schema?.byName || new Map<string, PocketBaseCollection>();
const files = fs.existsSync(exportDir) ? fs.readdirSync(exportDir).filter(file => file.toLowerCase().endsWith('.json')) : [];
@@ -439,6 +575,8 @@ function buildReport(): DryRunReport {
}
}
const migrationReadiness = buildReadiness(issues, recordsByCollection);
const businessCounts = Object.fromEntries(
businessCollections.map(collection => [collection, recordsByCollection.get(collection)?.length || 0]),
);
@@ -454,6 +592,7 @@ function buildReport(): DryRunReport {
exportDir,
schemaPath: schema?.absPath,
generatedAt: new Date().toISOString(),
migrationProfile,
summary: {
files: files.length,
collections: collections.length,
@@ -465,6 +604,7 @@ function buildReport(): DryRunReport {
infos,
},
businessCounts,
migrationReadiness,
collections: collections.sort((a, b) => a.collection.localeCompare(b.collection)),
issues: issues.sort((a, b) => {
const order: Record<Severity, number> = { blocker: 0, warning: 1, info: 2 };
@@ -477,11 +617,27 @@ function printHuman(report: DryRunReport) {
console.log('PocketBase dry-run migration report');
console.log(`Export dir: ${report.exportDir}`);
if (report.schemaPath) console.log(`Schema: ${report.schemaPath}`);
console.log(`Migration profile: ${report.migrationProfile}`);
console.log(
`Summary: ${report.summary.records} records across ${report.summary.collections} collections; ` +
`${report.summary.blockers} blocker(s), ${report.summary.warnings} warning(s)`,
);
console.log('');
console.log('Migration readiness:');
const missingRequired = report.migrationReadiness.requiredCollections.filter(item => !item.present);
const weakFields = report.migrationReadiness.criticalFieldCoverage.filter(item => !item.present);
console.log(` required collections missing/too small: ${missingRequired.length}`);
console.log(` critical field coverage warnings: ${weakFields.length}`);
for (const item of missingRequired.slice(0, 20)) {
console.log(` ${item.collection}: records=${item.recordCount}, minimum=${item.minimumRecords}`);
}
for (const item of weakFields.slice(0, 20)) {
console.log(
` ${item.collection}.${item.field}: present=${item.presentCount}/${item.total}, ` +
`required=${(item.requiredRatio * 100).toFixed(1)}%`,
);
}
console.log('');
console.log('Business counts:');
for (const [name, count] of Object.entries(report.businessCounts)) {
console.log(` ${name.padEnd(24)} ${count}`);

40
scripts/pb-dry-run-report-test.js
View File

@@ -44,7 +44,7 @@ function createSafeExport() {
{ id: 'q1', subjectId: 's1', categoryId: 'c1', type: 'choice', content: '题干', answer: 'A' },
]);
writeJson(path.join(dir, 'orders.json'), [
{ id: 'o1', userId: 'u1', amount: 99, status: 'paid' },
{ id: 'o1', userId: 'u1', planId: 'p1', amount: 99, status: 'paid' },
]);
writeJson(path.join(dir, 'svip_plans.json'), [
{ id: 'p1', name: '题库会员', price: 99 },
@@ -105,12 +105,36 @@ function createBrokenRelationExport() {
return dir;
}
function createProductionIncompleteExport() {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tiku-pb-dry-prod-incomplete-'));
writeJson(path.join(dir, 'users.json'), [
{ id: 'u1', role: 'student', nickname: '缺手机号学生' },
]);
writeJson(path.join(dir, 'questions.json'), [
{ id: 'q1', categoryId: 'c1', type: 'choice', content: '缺科目题' },
]);
return dir;
}
const safeDir = createSafeExport();
const safe = runDryRun(safeDir);
assert.equal(safe.status, 0, `safe dry-run should pass: ${safe.stdout} ${safe.stderr}`);
assert.equal(safe.payload.summary?.blockers, 0, 'safe dry-run should have no blockers');
assert.ok(safe.payload.summary?.records >= 12, 'safe dry-run should count fixture records');
assert.equal(safe.payload.businessCounts?.questions, 1, 'safe dry-run should report question count');
assert.equal(safe.payload.migrationProfile, 'development', 'default dry-run profile should be development');
assert.ok(Array.isArray(safe.payload.migrationReadiness?.requiredCollections), 'dry-run should report migration readiness');
const safeProduction = runDryRun(safeDir, { args: ['--profile=production'] });
assert.equal(safeProduction.status, 0, `complete production dry-run should pass: ${safeProduction.stdout} ${safeProduction.stderr}`);
assert.equal(safeProduction.payload.summary?.blockers, 0, 'complete production dry-run should have no blockers');
const invalidProfile = runDryRun(safeDir, { args: ['--profile=prodution'] });
assert.notEqual(invalidProfile.status, 0, 'invalid migration profile should fail closed');
assert.ok(
invalidProfile.payload.issues?.some(item => item.code === 'migration_profile_invalid'),
'invalid migration profile should be reported as a blocker',
);
const unsafeDir = createUnsafeExport();
const unsafe = runDryRun(unsafeDir);
@@ -152,9 +176,23 @@ assert.ok(
'broken relation dry-run should detect unresolved question subject relation',
);
const productionIncompleteDir = createProductionIncompleteExport();
const productionIncomplete = runDryRun(productionIncompleteDir, { args: ['--profile=production'] });
assert.notEqual(productionIncomplete.status, 0, 'production profile should fail incomplete export');
assert.ok(
productionIncomplete.payload.issues?.some(item => item.code === 'production_required_collection_missing'),
'production profile should block missing required collections',
);
assert.ok(
productionIncomplete.payload.issues?.some(item => item.code === 'critical_field_coverage_low' && item.collection === 'questions'),
'production profile should block critical field coverage gaps',
);
assert.equal(productionIncomplete.payload.migrationProfile, 'production', 'production profile should be reflected in report');
fs.rmSync(safeDir, { recursive: true, force: true });
fs.rmSync(unsafeDir, { recursive: true, force: true });
fs.rmSync(warningOnlyDir, { recursive: true, force: true });
fs.rmSync(brokenRelationDir, { recursive: true, force: true });
fs.rmSync(productionIncompleteDir, { recursive: true, force: true });
console.log('[PASS] PocketBase dry-run report');