gongxue-base/scripts/import-pocketbase/src/dry-run-report.ts

import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { fieldsOf, readPocketBaseSchema, type PocketBaseCollection, type PocketBaseField } from './pb-schema.js';
import { loadEnv } from './env.js';

loadEnv();

type JsonRecord = Record<string, unknown> & { id?: unknown };
type Severity = 'blocker' | 'warning' | 'info';
type MigrationProfile = 'development' | 'production';

interface DryRunIssue {
  severity: Severity;
  code: string;
  collection?: string;
  file?: string;
  fieldPath?: string;
  message: string;
  sample?: string;
  count?: number;
}

interface CollectionReport {
  collection: string;
  file: string;
  recordCount: number;
  missingIdCount: number;
  duplicateIdCount: number;
  fieldCount: number;
  sampleIds: string[];
  supported: boolean;
}

interface DryRunReport {
  exportDir: string;
  schemaPath?: string;
  generatedAt: string;
  migrationProfile: MigrationProfile;
  summary: {
    files: number;
    collections: number;
    records: number;
    supportedCollections: number;
    unsupportedCollections: number;
    blockers: number;
    warnings: number;
    infos: number;
  };
  businessCounts: Record<string, number>;
  migrationReadiness: {
    requiredCollections: Array<{
      collection: string;
      present: boolean;
      recordCount: number;
      minimumRecords: number;
    }>;
    criticalFieldCoverage: Array<{
      collection: string;
      field: string;
      present: boolean;
      missingCount: number;
      presentCount: number;
      total: number;
      missingRatio: number;
      requiredRatio: number;
    }>;
  };
  collections: CollectionReport[];
  issues: DryRunIssue[];
}

const rawArgs = process.argv.slice(2);
const args = new Set(rawArgs);
const jsonOutput = args.has('--json');
const failOnWarnings = args.has('--fail-on-warnings') || process.env.PB_DRY_RUN_FAIL_ON_WARNINGS === 'true';
const profileArg = rawArgs.find(arg => arg.startsWith('--profile='))?.split('=')[1]?.trim();
const migrationProfileInput = (profileArg || process.env.PB_DRY_RUN_PROFILE || 'development').trim().toLowerCase();
const validMigrationProfiles = new Set(['development', 'production']);
const migrationProfile: MigrationProfile = migrationProfileInput === 'production' ? 'production' : 'development';
const migrationProfileInvalid = !validMigrationProfiles.has(migrationProfileInput);
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
const exportDir = path.resolve(repoRoot, process.env.PB_EXPORT_DIR || 'pb_export');
const nonCollectionJsonFiles = new Set([
  'pb_schema.sqlite.json',
  'sqlite-export-manifest.json',
  'storage-manifest.json',
]);

const supportedCollections = new Set([
  'announcements',
  'app_assets',
  'audit_logs',
  'badges',
  'banners',
  'categories',
  'code_batches',
  'codes',
  'coupons',
  'coupon_redemptions',
  'crm_config',
  'crm_webhook_log',
  'crm_webhook_queue',
  'dashboard_daily_stats',
  'exam_dates',
  'faqs',
  'handbook_chapters',
  'handbook_entries',
  'handbook_subjects',
  'images',
  'majors',
  'module_nodes',
  'orders',
  'products',
  'question_type_groups',
  'question_videos',
  'questions',
  'recent_practices',
  'referral_tracks',
  'region_modules',
  'regions',
  'reports',
  'revenue_daily_stats',
  'schools',
  'scoreline_fields',
  'scoreline_majors',
  'scoreline_records',
  'scoreline_schools',
  'settings',
  'subject_shares',
  'subjects',
  'svip_plans',
  'timelines',
  'user_badges',
  'user_word_favorites',
  'user_word_progress',
  'users',
  'video_explanations',
  'vocabulary',
  'vocabulary_units',
]);

const coreCollections = [
  'users',
  'questions',
  'subjects',
  'categories',
  'orders',
  'svip_plans',
  'codes',
  'vocabulary_units',
  'vocabulary',
  'handbook_subjects',
  'handbook_chapters',
  'handbook_entries',
];

const businessCollections = [
  'users',
  'questions',
  'orders',
  'svip_plans',
  'codes',
  'vocabulary_units',
  'vocabulary',
  'handbook_subjects',
  'handbook_chapters',
  'handbook_entries',
  'scoreline_schools',
  'scoreline_majors',
  'scoreline_records',
  'video_explanations',
  'question_videos',
  'referral_tracks',
  'coupons',
  'badges',
];

const productionRequiredCollections = [
  { collection: 'users', minimumRecords: 1 },
  { collection: 'questions', minimumRecords: 1 },
  { collection: 'subjects', minimumRecords: 1 },
  { collection: 'categories', minimumRecords: 1 },
  { collection: 'orders', minimumRecords: 1 },
  { collection: 'svip_plans', minimumRecords: 1 },
  { collection: 'codes', minimumRecords: 1 },
  { collection: 'vocabulary_units', minimumRecords: 1 },
  { collection: 'vocabulary', minimumRecords: 1 },
  { collection: 'handbook_subjects', minimumRecords: 1 },
  { collection: 'handbook_chapters', minimumRecords: 1 },
  { collection: 'handbook_entries', minimumRecords: 1 },
];

const criticalFieldRules = [
  { collection: 'users', field: 'id', requiredRatio: 1 },
  { collection: 'users', field: 'phone', requiredRatio: 0.8 },
  { collection: 'questions', field: 'id', requiredRatio: 1 },
  { collection: 'questions', field: 'subjectId', requiredRatio: 0.95 },
  { collection: 'questions', field: 'categoryId', requiredRatio: 0.8, alternatives: ['nodeId'] },
  { collection: 'questions', field: 'content', requiredRatio: 0.95, alternatives: ['question', 'title', 'stem'] },
  { collection: 'orders', field: 'userId', requiredRatio: 0.98 },
  { collection: 'orders', field: 'planId', requiredRatio: 0.8 },
  { collection: 'orders', field: 'status', requiredRatio: 0.95 },
  { collection: 'codes', field: 'code', requiredRatio: 0.98 },
  { collection: 'vocabulary', field: 'unitId', requiredRatio: 0.95, alternatives: ['unit'] },
  { collection: 'vocabulary', field: 'word', requiredRatio: 0.98 },
  { collection: 'handbook_chapters', field: 'subjectId', requiredRatio: 0.95 },
  { collection: 'handbook_entries', field: 'chapterId', requiredRatio: 0.95 },
  { collection: 'scoreline_records', field: 'schoolId', requiredRatio: 0.8 },
  { collection: 'question_videos', field: 'questionId', requiredRatio: 0.95 },
  { collection: 'question_videos', field: 'videoId', requiredRatio: 0.95 },
  { collection: 'user_word_progress', field: 'userId', requiredRatio: 0.95 },
  { collection: 'user_word_favorites', field: 'userId', requiredRatio: 0.95 },
];

const sensitiveKeyPattern =
  /(password|token|secret|privatekey|sessionkey|accesskey|appkey|apikey|api_v3_key|notifytoken|aeskey|openid|unionid|wxaccesstoken|wechatsessionkey|smscode|verifycode|verificationcode|captcha)/i;
const highRiskCollections = new Set(['settings', 'crm_config', 'smscodes']);

function issue(issues: DryRunIssue[], nextIssue: DryRunIssue) {
  issues.push(nextIssue);
}

function asRecords(input: unknown): JsonRecord[] | null {
  if (Array.isArray(input)) return input as JsonRecord[];
  if (input && typeof input === 'object') {
    const candidate = input as { items?: unknown; records?: unknown };
    if (Array.isArray(candidate.items)) return candidate.items as JsonRecord[];
    if (Array.isArray(candidate.records)) return candidate.records as JsonRecord[];
  }
  return null;
}

function collectionNameFromFile(fileName: string) {
  return fileName.replace(/\.json$/i, '');
}

function text(value: unknown) {
  if (value === null || value === undefined) return '';
  return String(value).trim();
}

function hasValue(record: JsonRecord, field: string, alternatives: string[] = []) {
  return [field, ...alternatives].some(key => text(record[key]));
}

function readJsonFile(filePath: string) {
  return JSON.parse(fs.readFileSync(filePath, 'utf8')) as unknown;
}

function scalarSample(value: unknown) {
  if (value === null || value === undefined) return '';
  if (typeof value === 'object') return JSON.stringify(value).slice(0, 160);
  return String(value).slice(0, 160);
}

function walkSensitiveFields(
  issues: DryRunIssue[],
  collection: string,
  value: unknown,
  pathParts: string[] = [],
  depth = 0,
) {
  if (!value || typeof value !== 'object' || depth > 8) return;
  if (Array.isArray(value)) {
    value.slice(0, 10).forEach((item, index) => walkSensitiveFields(issues, collection, item, [...pathParts, String(index)], depth + 1));
    return;
  }

  for (const [key, childValue] of Object.entries(value as Record<string, unknown>)) {
    const fieldPath = [...pathParts, key].join('.');
    if (sensitiveKeyPattern.test(key) && childValue !== null && childValue !== undefined && childValue !== '') {
      issue(issues, {
        severity: highRiskCollections.has(collection) ? 'blocker' : 'warning',
        code: 'sensitive_field_in_export',
        collection,
        fieldPath,
        message: `Legacy export contains sensitive-looking field "${fieldPath}". It must not be copied into public normalized tables.`,
        sample: scalarSample(childValue),
      });
    }
    walkSensitiveFields(issues, collection, childValue, [...pathParts, key], depth + 1);
  }
}

function fieldValues(record: JsonRecord, field: PocketBaseField) {
  const value = record[field.name];
  if (value === null || value === undefined || value === '') return [];
  if (Array.isArray(value)) return value.map(item => text(item)).filter(Boolean);
  return [text(value)].filter(Boolean);
}

function relationTargetName(field: PocketBaseField, byId: Map<string, PocketBaseCollection>) {
  if (!field.collectionId) return '';
  return byId.get(field.collectionId)?.name || '';
}

function validateRelations(
  issues: DryRunIssue[],
  collection: PocketBaseCollection,
  records: JsonRecord[],
  idIndex: Map<string, Set<string>>,
  byId: Map<string, PocketBaseCollection>,
) {
  const relationFields = fieldsOf(collection).filter(field => field.type === 'relation');
  for (const field of relationFields) {
    const targetName = relationTargetName(field, byId);
    if (!targetName) {
      issue(issues, {
        severity: 'warning',
        code: 'relation_target_schema_missing',
        collection: collection.name,
        fieldPath: field.name,
        message: `Relation field "${field.name}" points to an unknown collection id in pb_schema.json.`,
      });
      continue;
    }

    const targetIds = idIndex.get(targetName);
    if (!targetIds) {
      const hasValues = records.some(record => fieldValues(record, field).length > 0);
      if (hasValues) {
        issue(issues, {
          severity: coreCollections.includes(collection.name) ? 'blocker' : 'warning',
          code: 'relation_target_export_missing',
          collection: collection.name,
          fieldPath: field.name,
          message: `Relation field "${field.name}" references "${targetName}", but ${targetName}.json is not present in export.`,
        });
      }
      continue;
    }

    let unresolved = 0;
    let sample = '';
    for (const record of records) {
      for (const value of fieldValues(record, field)) {
        if (!targetIds.has(value)) {
          if (collection.name === 'questions' && field.name === 'categoryId' && idIndex.get('module_nodes')?.has(value)) {
            continue;
          }
          unresolved += 1;
          if (!sample) sample = `${text(record.id) || '(missing id)'} -> ${value}`;
        }
      }
    }

    if (unresolved > 0) {
      issue(issues, {
        severity: coreCollections.includes(collection.name) ? 'blocker' : 'warning',
        code: 'relation_unresolved',
        collection: collection.name,
        fieldPath: field.name,
        message: `Relation field "${field.name}" has ${unresolved} unresolved reference(s) to "${targetName}".`,
        count: unresolved,
        sample,
      });
    }
  }
}

function buildReadiness(
  issues: DryRunIssue[],
  recordsByCollection: Map<string, JsonRecord[]>,
): DryRunReport['migrationReadiness'] {
  const requiredCollections = productionRequiredCollections.map(rule => {
    const count = recordsByCollection.get(rule.collection)?.length || 0;
    const present = count >= rule.minimumRecords;
    if (migrationProfile === 'production' && !present) {
      issue(issues, {
        severity: 'blocker',
        code: 'production_required_collection_missing',
        collection: rule.collection,
        message: `${rule.collection}.json is required for production migration and must contain at least ${rule.minimumRecords} record(s).`,
        count,
      });
    }
    return {
      collection: rule.collection,
      present,
      recordCount: count,
      minimumRecords: rule.minimumRecords,
    };
  });

  const criticalFieldCoverage = criticalFieldRules.flatMap(rule => {
    const records = recordsByCollection.get(rule.collection);
    if (!records || records.length === 0) return [];
    let missingCount = 0;
    for (const record of records) {
      if (!hasValue(record, rule.field, rule.alternatives)) missingCount += 1;
    }
    const total = records.length;
    const presentCount = total - missingCount;
    const presentRatio = total === 0 ? 1 : presentCount / total;
    const missingRatio = total === 0 ? 0 : missingCount / total;
    if (presentRatio < rule.requiredRatio) {
      issue(issues, {
        severity: migrationProfile === 'production' ? 'blocker' : 'warning',
        code: 'critical_field_coverage_low',
        collection: rule.collection,
        fieldPath: rule.field,
        message:
          `${rule.collection}.${rule.field} coverage is ${(presentRatio * 100).toFixed(1)}%, ` +
          `below required ${(rule.requiredRatio * 100).toFixed(1)}% for reliable migration.`,
        count: missingCount,
      });
    }
    return [{
      collection: rule.collection,
      field: rule.field,
      present: presentRatio >= rule.requiredRatio,
      missingCount,
      presentCount,
      total,
      missingRatio,
      requiredRatio: rule.requiredRatio,
    }];
  });

  return { requiredCollections, criticalFieldCoverage };
}

function loadSchema(issues: DryRunIssue[]) {
  try {
    return readPocketBaseSchema();
  } catch (error) {
    issue(issues, {
      severity: 'warning',
      code: 'schema_unavailable',
      message: `pb_schema.json could not be loaded; relation checks will be skipped. ${error instanceof Error ? error.message : ''}`.trim(),
    });
    return null;
  }
}

function collectFieldNames(records: JsonRecord[]) {
  const fields = new Set<string>();
  for (const record of records.slice(0, 500)) {
    Object.keys(record).forEach(key => fields.add(key));
  }
  return fields;
}

function buildReport(): DryRunReport {
  const issues: DryRunIssue[] = [];
  if (migrationProfileInvalid) {
    issue(issues, {
      severity: 'blocker',
      code: 'migration_profile_invalid',
      message: `Invalid migration profile "${migrationProfileInput}". Use "development" or "production".`,
    });
  }
  const schema = loadSchema(issues);
  const collectionByName = schema?.byName || new Map<string, PocketBaseCollection>();
  const files = fs.existsSync(exportDir)
    ? fs
      .readdirSync(exportDir)
      .filter(file => file.toLowerCase().endsWith('.json'))
      .filter(file => !nonCollectionJsonFiles.has(file.toLowerCase()))
    : [];
  const collections: CollectionReport[] = [];
  const recordsByCollection = new Map<string, JsonRecord[]>();
  const idIndex = new Map<string, Set<string>>();

  if (!fs.existsSync(exportDir)) {
    issue(issues, {
      severity: 'blocker',
      code: 'export_dir_missing',
      message: `Export dir not found: ${exportDir}`,
    });
  }

  if (files.length === 0) {
    issue(issues, {
      severity: 'blocker',
      code: 'export_files_missing',
      message: `No PocketBase collection JSON files found in ${exportDir}`,
    });
  }

  for (const file of files.sort()) {
    const collection = collectionNameFromFile(file);
    const filePath = path.join(exportDir, file);
    let parsed: unknown;
    try {
      parsed = readJsonFile(filePath);
    } catch (error) {
      issue(issues, {
        severity: 'blocker',
        code: 'json_parse_failed',
        collection,
        file,
        message: `Failed to parse ${file}: ${error instanceof Error ? error.message : String(error)}`,
      });
      continue;
    }

    const records = asRecords(parsed);
    if (!records) {
      issue(issues, {
        severity: 'blocker',
        code: 'collection_shape_invalid',
        collection,
        file,
        message: `${file} must be an array or an object with items/records array.`,
      });
      continue;
    }

    const ids = new Set<string>();
    const duplicateIds = new Set<string>();
    let missingIdCount = 0;
    for (const record of records) {
      const id = text(record.id);
      if (!id) {
        missingIdCount += 1;
        continue;
      }
      if (ids.has(id)) duplicateIds.add(id);
      ids.add(id);
    }

    recordsByCollection.set(collection, records);
    idIndex.set(collection, ids);
    const fields = collectFieldNames(records);
    collections.push({
      collection,
      file,
      recordCount: records.length,
      missingIdCount,
      duplicateIdCount: duplicateIds.size,
      fieldCount: fields.size,
      sampleIds: [...ids].slice(0, 5),
      supported: supportedCollections.has(collection),
    });

    if (!supportedCollections.has(collection) && !collection.startsWith('_')) {
      issue(issues, {
        severity: 'warning',
        code: 'unsupported_collection',
        collection,
        file,
        message: `${collection}.json is not currently normalized by import-json.ts. It will be preserved only if a future mapper is added.`,
        count: records.length,
      });
    }
    if (missingIdCount > 0) {
      issue(issues, {
        severity: coreCollections.includes(collection) ? 'blocker' : 'warning',
        code: 'record_id_missing',
        collection,
        file,
        message: `${collection} has ${missingIdCount} record(s) without id.`,
        count: missingIdCount,
      });
    }
    if (duplicateIds.size > 0) {
      issue(issues, {
        severity: 'blocker',
        code: 'record_id_duplicate',
        collection,
        file,
        message: `${collection} has duplicate legacy id(s).`,
        count: duplicateIds.size,
        sample: [...duplicateIds].slice(0, 5).join(', '),
      });
    }

    records.slice(0, 200).forEach(record => walkSensitiveFields(issues, collection, record));
  }

  for (const collection of coreCollections) {
    if (!recordsByCollection.has(collection)) {
      issue(issues, {
        severity: 'warning',
        code: 'core_collection_missing',
        collection,
        message: `${collection}.json is not present. If old production has this collection, export it before migration.`,
      });
    }
  }

  if (schema) {
    for (const [collectionName, records] of recordsByCollection.entries()) {
      const collection = collectionByName.get(collectionName);
      if (collection) validateRelations(issues, collection, records, idIndex, schema.byId);
    }
  }

  const migrationReadiness = buildReadiness(issues, recordsByCollection);

  const businessCounts = Object.fromEntries(
    businessCollections.map(collection => [collection, recordsByCollection.get(collection)?.length || 0]),
  );

  const supportedCount = collections.filter(item => item.supported).length;
  const unsupportedCount = collections.length - supportedCount;
  const recordCount = collections.reduce((sum, item) => sum + item.recordCount, 0);
  const blockers = issues.filter(item => item.severity === 'blocker').length;
  const warnings = issues.filter(item => item.severity === 'warning').length;
  const infos = issues.filter(item => item.severity === 'info').length;

  return {
    exportDir,
    schemaPath: schema?.absPath,
    generatedAt: new Date().toISOString(),
    migrationProfile,
    summary: {
      files: files.length,
      collections: collections.length,
      records: recordCount,
      supportedCollections: supportedCount,
      unsupportedCollections: unsupportedCount,
      blockers,
      warnings,
      infos,
    },
    businessCounts,
    migrationReadiness,
    collections: collections.sort((a, b) => a.collection.localeCompare(b.collection)),
    issues: issues.sort((a, b) => {
      const order: Record<Severity, number> = { blocker: 0, warning: 1, info: 2 };
      return order[a.severity] - order[b.severity] || (a.collection || '').localeCompare(b.collection || '') || a.code.localeCompare(b.code);
    }),
  };
}

function printHuman(report: DryRunReport) {
  console.log('PocketBase dry-run migration report');
  console.log(`Export dir: ${report.exportDir}`);
  if (report.schemaPath) console.log(`Schema: ${report.schemaPath}`);
  console.log(`Migration profile: ${report.migrationProfile}`);
  console.log(
    `Summary: ${report.summary.records} records across ${report.summary.collections} collections; ` +
      `${report.summary.blockers} blocker(s), ${report.summary.warnings} warning(s)`,
  );
  console.log('');
  console.log('Migration readiness:');
  const missingRequired = report.migrationReadiness.requiredCollections.filter(item => !item.present);
  const weakFields = report.migrationReadiness.criticalFieldCoverage.filter(item => !item.present);
  console.log(`  required collections missing/too small: ${missingRequired.length}`);
  console.log(`  critical field coverage warnings: ${weakFields.length}`);
  for (const item of missingRequired.slice(0, 20)) {
    console.log(`    ${item.collection}: records=${item.recordCount}, minimum=${item.minimumRecords}`);
  }
  for (const item of weakFields.slice(0, 20)) {
    console.log(
      `    ${item.collection}.${item.field}: present=${item.presentCount}/${item.total}, ` +
        `required=${(item.requiredRatio * 100).toFixed(1)}%`,
    );
  }
  console.log('');
  console.log('Business counts:');
  for (const [name, count] of Object.entries(report.businessCounts)) {
    console.log(`  ${name.padEnd(24)} ${count}`);
  }
  console.log('');
  console.log('Collections:');
  for (const item of report.collections) {
    const marker = item.supported ? 'mapped' : 'unmapped';
    console.log(
      `  ${item.collection.padEnd(24)} records=${String(item.recordCount).padStart(6)} ` +
        `fields=${String(item.fieldCount).padStart(3)} ${marker}`,
    );
  }
  console.log('');
  console.log('Issues:');
  if (report.issues.length === 0) {
    console.log('  none');
  } else {
    for (const item of report.issues.slice(0, 200)) {
      const location = [item.collection, item.fieldPath].filter(Boolean).join('.') || item.file || 'export';
      const count = item.count === undefined ? '' : ` (${item.count})`;
      const sample = item.sample ? ` sample=${item.sample}` : '';
      console.log(`  [${item.severity.toUpperCase()}] ${item.code} ${location}: ${item.message}${count}${sample}`);
    }
    if (report.issues.length > 200) console.log(`  ... ${report.issues.length - 200} more issue(s) omitted`);
  }
}

function main() {
  const report = buildReport();
  if (jsonOutput) console.log(JSON.stringify(report, null, 2));
  else printHuman(report);

  if (report.summary.blockers > 0 || (failOnWarnings && report.summary.warnings > 0)) {
    process.exitCode = 1;
  }
}

main();