From 2510d736a44d7dab95127a1a8e2db424e27d3b8a Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 30 Jun 2026 09:30:00 +0800 Subject: [PATCH] feat: add pocketbase sqlite exporter --- .gitignore | 8 + README.md | 28 +- docs/refactor/next-development-todo.md | 6 +- .../pocketbase-real-data-migration-runbook.md | 54 ++- package.json | 2 + scripts/import-pocketbase/package.json | 1 + .../import-pocketbase/src/dry-run-report.ts | 19 +- .../import-pocketbase/src/export-sqlite.ts | 113 +++++ .../import-pocketbase/src/sqlite-exporter.py | 436 ++++++++++++++++++ scripts/pb-sqlite-export-test.js | 185 ++++++++ 10 files changed, 841 insertions(+), 11 deletions(-) create mode 100644 scripts/import-pocketbase/src/export-sqlite.ts create mode 100644 scripts/import-pocketbase/src/sqlite-exporter.py create mode 100644 scripts/pb-sqlite-export-test.js diff --git a/.gitignore b/.gitignore index d3b16c71..40798586 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,14 @@ scripts/satellite/sync-config.json stash_comparison.txt stash_diff.txt .local-storage/ +pb_export/ +pb_export_*/ +migration-dry-run-report*.json +sqlite-export-manifest*.json +storage-manifest*.json + +# ✅ 真实数据迁移报告（可能含业务敏感信息） +docs/refactor/migration-reports/ # ✅ 生产上线验收证据（可能包含内部域名、抽样说明或敏感运维信息） docs/refactor/production-launch-evidence.json diff --git a/README.md b/README.md index a07810ac..a10564d5 100644 --- a/README.md +++ b/README.md @@ -404,7 +404,26 @@ docs/refactor/production-launch-evidence.template.json ## PocketBase 迁移 Dry-Run -把旧 PocketBase 导出的集合 JSON 放到仓库根目录 `pb_export/` 后，先执行不写数据库的静态 dry-run： +如果旧数据源是 SQLite，先从旧 PocketBase 数据目录只读导出业务 collection。默认读取 `F:\project\参考\旧题库数据库文件`，输出到已被 `.gitignore` 覆盖的 `pb_export/`： + +```powershell +$env:PB_SQLITE_DIR="F:\project\参考\旧题库数据库文件" +$env:PB_EXPORT_DIR="F:\project\pb_export" +npm run pb:export:sqlite +``` + +导出会生成： + +```text +pb_export/*.json 各 PocketBase collection 的普通业务 JSON +pb_export/sqlite-export-manifest.json +pb_export/storage-manifest.json +pb_export/pb_schema.sqlite.json +``` + +默认导出会移除 `password/token/secret/sessionKey/openid/unionid` 等敏感身份或密钥字段，只在 manifest 中记录脱敏字段数量；手机号等迁移必需字段会保留。不要把 `pb_export/`、manifest 或真实迁移报告提交到 Git。 + +把旧 PocketBase 导出的集合 JSON 放到仓库根目录 `pb_export/` 后，执行不写数据库的静态 dry-run： ```bash npm run pb:import:dry-run @@ -420,6 +439,13 @@ dry-run 会检查导出目录、JSON 形态、核心集合缺失、重复/缺失 默认 dry-run 使用 `development` profile；正式迁移、预生产验收和 CI 应使用 `production` profile。生产 profile 会额外输出 `migrationReadiness`，检查用户、题目、科目、分类、订单、套餐、激活码、单词和知识手册等必需集合，以及用户手机号、题目归属、订单套餐、激活码、单词和手册归属等关键字段覆盖率。`--profile` 只接受 `development` 或 `production`，拼写错误会按 blocker 失败。 +当前真实 SQLite 基线已经跑通只读导出：58 个业务 collection、248555 条记录、9 个 storage 原始资源文件。production dry-run 目前仍有 2 个真实数据 blocker：30 个订单缺用户、22 个知识手册章节缺所属手册；还有 `user_answer_records`、`mock_exam_configs`、`referral_qrcodes`、`commission_settings` 等 mapper gap，详见： + +```text +docs/refactor/pocketbase-real-data-migration-runbook.md +docs/refactor/next-development-todo.md +``` + 真实生产数据迁移不要只看命令是否能跑完，需要按迁移验收 runbook 执行 dry-run、正式导入演练、导入后校验、业务抽样、Taro 联调、冻结切换和回滚准备： ```text diff --git a/docs/refactor/next-development-todo.md b/docs/refactor/next-development-todo.md index 74a4e9a1..78c5e8cc 100644 --- a/docs/refactor/next-development-todo.md +++ b/docs/refactor/next-development-todo.md @@ -74,7 +74,11 @@ 3. 真实导入 dry-run - 从 PocketBase 导出现有用户、题库、单词、知识手册、分数线、订单、权益数据。 - - 已补 `npm run pb:import:dry-run` 静态迁移报告工具、`--profile=production` 生产迁移门禁、关键集合/关键字段覆盖率检查、strict warning 门禁测试和真实数据迁移验收 runbook；拿到真实导出后先跑 `npm run pb:import:dry-run -- --profile=production --json --fail-on-warnings`，再跑 `pb:import:json`、`pb:import:validate` 和业务抽样。 + - 已补 `npm run pb:export:sqlite` 只读 SQLite 导出工具，默认从 `F:\project\参考\旧题库数据库文件\data.db` 导出到 `.gitignore` 覆盖的 `pb_export/`，并生成 `sqlite-export-manifest.json`、`storage-manifest.json` 和脱敏统计。 + - 已补 `npm run pb:import:dry-run` 静态迁移报告工具、`--profile=production` 生产迁移门禁、关键集合/关键字段覆盖率检查、strict warning 门禁测试和真实数据迁移验收 runbook；真实 SQLite 已导出 58 个业务 collection、248555 条记录，并跑过 production dry-run。 + - 当前真实 dry-run 剩余 blocker：30 个订单缺 `userId`，其中 7 个 paid；22 个知识手册章节缺 `subjectId`。正式导入前必须补自动修复/隔离 mapper 和人工复核报告。 + - 当前真实 dry-run 剩余 mapper gap：`user_answer_records` 85442 条、`mock_exam_configs` 48 条、`referral_qrcodes` 79 条、`commission_settings` 1 条。它们分别对应学习历史/错题归因、全真模拟蓝图、推广码/小程序码、分佣设置，后续要进入标准化导入。 + - 处理完 blocker 后再跑 `npm run pb:import:dry-run -- --profile=production --json --fail-on-warnings`，再跑 `pb:import:json`、`pb:import:validate` 和业务抽样。 - 对题目 JSON、单词、知识手册、分数线、视频走后端 preview/import API 做二次验证。 4. 部署配置 diff --git a/docs/refactor/pocketbase-real-data-migration-runbook.md b/docs/refactor/pocketbase-real-data-migration-runbook.md index d96e91a2..d2bab6c2 100644 --- a/docs/refactor/pocketbase-real-data-migration-runbook.md +++ b/docs/refactor/pocketbase-real-data-migration-runbook.md @@ -56,7 +56,7 @@ F:\project\参考\旧题库数据库文件 storage/ ``` -`data.db` 和 `auxiliary.db` 必须按只读源处理，不能直接在旧库上执行修复 SQL。后续应由本地转换脚本把 PocketBase SQLite 集合导出为规范 JSON，再进入下面的 dry-run/import 管线；`storage/` 中的附件、题图、PDF、视频封面等资源要同步生成资源迁移清单，最终进入 `content_assets` 和对象存储，不允许把旧本地路径或长效 URL 直接写给前端。 +`data.db` 和 `auxiliary.db` 必须按只读源处理，不能直接在旧库上执行修复 SQL。当前已补 `npm run pb:export:sqlite`，会用只读 SQLite 连接把 PocketBase collection 导出为规范 JSON，再进入下面的 dry-run/import 管线；`storage/` 中的附件、题图、PDF、视频封面等资源会同步生成资源迁移清单，最终进入 `content_assets` 和对象存储，不允许把旧本地路径或长效 URL 直接写给前端。 把 PocketBase 导出的集合 JSON 放到仓库根目录： @@ -93,20 +93,61 @@ $env:PB_EXPORT_DIR="F:\migration\pb_export_20260630" npm run pb:import:dry-run ``` -如果输入源仍是 SQLite，而不是 JSON 目录，下一阶段需要先补 `scripts/import-pocketbase` 的 SQLite 只读导出命令，建议输出到仓库外或 `.gitignore` 覆盖的 `pb_export/`： +如果输入源仍是 SQLite，而不是 JSON 目录，先执行 `scripts/import-pocketbase` 的 SQLite 只读导出命令，建议输出到仓库外或 `.gitignore` 覆盖的 `pb_export/`： ```powershell $env:PB_SQLITE_DIR="F:\project\参考\旧题库数据库文件" $env:PB_EXPORT_DIR="F:\project\pb_export" -# 规划命令：npm run pb:export:sqlite +npm run pb:export:sqlite ``` -导出脚本要求： +导出脚本行为： - 使用只读 SQLite 连接，禁止修改 `data.db`、`auxiliary.db` 和 `storage/`。 - 每个 PocketBase collection 输出一个 JSON 文件，并保留旧 `id`、`created`、`updated`、relation 字段和文件字段。 -- 对 `users`、订单、支付、openid/unionid、手机号等敏感字段默认脱敏写 dry-run 报告；正式导入只在后端 mapper 中进入受控身份/订单/审计表。 -- 资源文件字段只输出相对路径、hash、size、mime 和旧 collection/record/file 三元组，后续由对象存储迁移步骤转为 `content_assets`。 +- `password`、`tokenKey`、`secret`、`wechatSessionKey`、短信验证码等密钥类字段默认不会写入普通 JSON；`openid/unionid` 默认也会移除，只在 `sqlite-export-manifest.json` 里记录字段和数量。后续如果确实要迁移第三方登录身份，必须走单独的加密身份迁移链路，不混入普通 `pb_export`。 +- 手机号、邮箱、业务旧 ID、订单、题目、学习记录等迁移必需字段会保留，用于自营 ToC 租户的数据归属和售后追溯。 +- 资源清单输出到 `storage-manifest.json`，包含相对路径、hash、size、mime 和旧 collection/record/file 三元组，后续由对象存储迁移步骤转为 `content_assets`。 +- `auxiliary.db` 默认只写 `_logs` 的数量和 level 摘要，不导出完整日志正文，避免把 UA、请求参数或历史 token 放进迁移数据包。 + +可选环境变量： + +```powershell +$env:PB_SQLITE_DATA_DB="F:\project\参考\旧题库数据库文件\data.db" +$env:PB_SQLITE_AUX_DB="F:\project\参考\旧题库数据库文件\auxiliary.db" +$env:PB_SQLITE_STORAGE_DIR="F:\project\参考\旧题库数据库文件\storage" +$env:PB_SQLITE_COLLECTIONS="users,regions,subjects,categories,questions" +$env:PB_SQLITE_BATCH_SIZE="2000" +``` + +正常迁移不要设置 `PB_SQLITE_INCLUDE_SENSITIVE_IDENTITIES=true`。这个开关只允许在加密迁移工作区临时使用，并且导出的身份文件不得提交 Git。 + +当前真实库只读导出基线： + +```text +data.db: 58 个业务 collection，248555 条记录 +questions: 74102 +users: 3670 +orders: 636 +user_answer_records: 85442 +vocabulary: 3500 +handbook_entries: 2676 +storage-manifest: 9 个原始资源文件 +auxiliary.db: _logs 121715 条，仅摘要 +``` + +最近一次 production dry-run 仍有 2 个 blocker，需要在正式导入前处理： + +- `orders.userId` 覆盖率 95.3%，30 个订单缺用户，其中 7 个为已支付订单。处理策略应优先从支付通知、手机号、订单号或人工售后记录找回用户；找不回的已支付订单必须进入人工异常台账，不允许静默开权益。 +- `handbook_chapters.subjectId` 覆盖率 94.4%，22 个章节缺所属手册。处理策略应按章节标题和条目归属归并到正确 `handbook_subjects`，无法归并的放入迁移隔离手册并标记待人工复核。 + +production dry-run 还有这些 warning，属于后续 mapper 或运营处理项： + +- `user_answer_records` 85442 条尚未规范化到新练习历史/答题记录模型。 +- `mock_exam_configs` 48 条尚未规范化到 `practice_blueprints`。 +- `referral_qrcodes` 79 条尚未规范化到新推广码/小程序码模型。 +- `commission_settings` 1 条需要迁移到新分佣设置。 +- `dashboard_cache`、空 `customer_messages/smscodes/tenant_config` 可不作为正式数据源，必要时只保留审计摘要。 ## 阶段 1：静态 Dry-Run @@ -150,6 +191,7 @@ dry-run 会检查： - 用户、题目、订单、SVIP、激活码、单词、手册、分数线、视频等业务数量。 - `production` profile 会额外检查生产迁移必需集合：`users`、`questions`、`subjects`、`categories`、`orders`、`svip_plans`、`codes`、`vocabulary_units`、`vocabulary`、`handbook_subjects`、`handbook_chapters`、`handbook_entries`。 - `migrationReadiness.criticalFieldCoverage` 会统计关键字段覆盖率，例如 `users.phone`、`questions.subjectId/categoryId/content`、`orders.userId/planId/status`、`codes.code`、单词和手册的归属字段；生产模式下低于阈值会变成 blocker。 +- 对 SQLite 导出，`questions.categoryId` 会把 `nodeId` 视为新架构有效归属，`vocabulary.unitId` 会兼容旧字段 `unit`。 准入标准： diff --git a/package.json b/package.json index 4b269399..1736d2aa 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "test:auth:remote-smoke": "node scripts/remote-auth-jwt-smoke-test.js", "test:launch-gate": "node scripts/production-launch-gate-test.js", "test:pb:dry-run": "node scripts/pb-dry-run-report-test.js", + "test:pb:sqlite-export": "node scripts/pb-sqlite-export-test.js", "readiness:production": "node scripts/production-readiness-check.js --skip-db", "readiness:production:db": "node scripts/production-readiness-check.js --check-db", "launch:gate": "node scripts/production-launch-gate.js", @@ -65,6 +66,7 @@ "build:taro:h5:platform": "npm --workspace @tiku-saas/taro run build:h5:platform", "pb:schema:summary": "npm --workspace @tiku-saas/import-pocketbase run schema:summary", "pb:schema:risk": "npm --workspace @tiku-saas/import-pocketbase run schema:risk", + "pb:export:sqlite": "npm --workspace @tiku-saas/import-pocketbase run export:sqlite --", "pb:import:dry-run": "npm --workspace @tiku-saas/import-pocketbase run import:dry-run --", "pb:import:json": "npm --workspace @tiku-saas/import-pocketbase run import:json", "pb:import:validate": "npm --workspace @tiku-saas/import-pocketbase run import:validate" diff --git a/scripts/import-pocketbase/package.json b/scripts/import-pocketbase/package.json index ed62a892..5c6a9a26 100644 --- a/scripts/import-pocketbase/package.json +++ b/scripts/import-pocketbase/package.json @@ -6,6 +6,7 @@ "scripts": { "schema:summary": "tsx src/analyze-schema.ts", "schema:risk": "tsx src/risk-report.ts", + "export:sqlite": "tsx src/export-sqlite.ts", "import:dry-run": "tsx src/dry-run-report.ts", "import:json": "tsx src/import-json.ts", "import:validate": "tsx src/validate-import.ts", diff --git a/scripts/import-pocketbase/src/dry-run-report.ts b/scripts/import-pocketbase/src/dry-run-report.ts index 2ba31594..349b7cdc 100644 --- a/scripts/import-pocketbase/src/dry-run-report.ts +++ b/scripts/import-pocketbase/src/dry-run-report.ts @@ -81,6 +81,11 @@ const migrationProfile: MigrationProfile = migrationProfileInput === 'production const migrationProfileInvalid = !validMigrationProfiles.has(migrationProfileInput); const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..'); const exportDir = path.resolve(repoRoot, process.env.PB_EXPORT_DIR || 'pb_export'); +const nonCollectionJsonFiles = new Set([ + 'pb_schema.sqlite.json', + 'sqlite-export-manifest.json', + 'storage-manifest.json', +]); const supportedCollections = new Set([ 'announcements', @@ -191,13 +196,13 @@ const criticalFieldRules = [ { collection: 'users', field: 'phone', requiredRatio: 0.8 }, { collection: 'questions', field: 'id', requiredRatio: 1 }, { collection: 'questions', field: 'subjectId', requiredRatio: 0.95 }, - { collection: 'questions', field: 'categoryId', requiredRatio: 0.8 }, + { collection: 'questions', field: 'categoryId', requiredRatio: 0.8, alternatives: ['nodeId'] }, { collection: 'questions', field: 'content', requiredRatio: 0.95, alternatives: ['question', 'title', 'stem'] }, { collection: 'orders', field: 'userId', requiredRatio: 0.98 }, { collection: 'orders', field: 'planId', requiredRatio: 0.8 }, { collection: 'orders', field: 'status', requiredRatio: 0.95 }, { collection: 'codes', field: 'code', requiredRatio: 0.98 }, - { collection: 'vocabulary', field: 'unitId', requiredRatio: 0.95 }, + { collection: 'vocabulary', field: 'unitId', requiredRatio: 0.95, alternatives: ['unit'] }, { collection: 'vocabulary', field: 'word', requiredRatio: 0.98 }, { collection: 'handbook_chapters', field: 'subjectId', requiredRatio: 0.95 }, { collection: 'handbook_entries', field: 'chapterId', requiredRatio: 0.95 }, @@ -331,6 +336,9 @@ function validateRelations( for (const record of records) { for (const value of fieldValues(record, field)) { if (!targetIds.has(value)) { + if (collection.name === 'questions' && field.name === 'categoryId' && idIndex.get('module_nodes')?.has(value)) { + continue; + } unresolved += 1; if (!sample) sample = `${text(record.id) || '(missing id)'} -> ${value}`; } @@ -445,7 +453,12 @@ function buildReport(): DryRunReport { } const schema = loadSchema(issues); const collectionByName = schema?.byName || new Map(); - const files = fs.existsSync(exportDir) ? fs.readdirSync(exportDir).filter(file => file.toLowerCase().endsWith('.json')) : []; + const files = fs.existsSync(exportDir) + ? fs + .readdirSync(exportDir) + .filter(file => file.toLowerCase().endsWith('.json')) + .filter(file => !nonCollectionJsonFiles.has(file.toLowerCase())) + : []; const collections: CollectionReport[] = []; const recordsByCollection = new Map(); const idIndex = new Map>(); diff --git a/scripts/import-pocketbase/src/export-sqlite.ts b/scripts/import-pocketbase/src/export-sqlite.ts new file mode 100644 index 00000000..f61e0326 --- /dev/null +++ b/scripts/import-pocketbase/src/export-sqlite.ts @@ -0,0 +1,113 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { loadEnv } from './env.js'; + +loadEnv(); + +const moduleDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(moduleDir, '../../..'); + +function argValue(name: string) { + const prefix = `${name}=`; + return process.argv.slice(2).find(arg => arg.startsWith(prefix))?.slice(prefix.length); +} + +function hasArg(name: string) { + return process.argv.slice(2).includes(name); +} + +function resolvePath(input: string, fallbackBase = process.cwd()) { + return path.resolve(fallbackBase, input); +} + +function findPython() { + const candidates = [ + process.env.PYTHON, + process.env.PYTHON_BIN, + 'python', + 'py', + 'python3', + ].filter(Boolean) as string[]; + + for (const candidate of candidates) { + const result = spawnSync(candidate, ['--version'], { encoding: 'utf8' }); + if (result.status === 0) return candidate; + } + throw new Error('Python 3 is required for SQLite export. Set PYTHON or install Python 3.'); +} + +const sqliteDir = resolvePath( + argValue('--sqlite-dir') || process.env.PB_SQLITE_DIR || path.join(repoRoot, '参考', '旧题库数据库文件'), +); +const dataDb = resolvePath(argValue('--data-db') || process.env.PB_SQLITE_DATA_DB || path.join(sqliteDir, 'data.db')); +const auxDb = resolvePath(argValue('--aux-db') || process.env.PB_SQLITE_AUX_DB || path.join(sqliteDir, 'auxiliary.db')); +const storageDir = resolvePath( + argValue('--storage-dir') || process.env.PB_SQLITE_STORAGE_DIR || path.join(sqliteDir, 'storage'), +); +const exportDir = resolvePath(argValue('--export-dir') || process.env.PB_EXPORT_DIR || path.join(repoRoot, 'pb_export')); +const collections = argValue('--collections') || process.env.PB_SQLITE_COLLECTIONS || ''; +const batchSize = argValue('--batch-size') || process.env.PB_SQLITE_BATCH_SIZE || '1000'; +const includeSystem = hasArg('--include-system') || process.env.PB_SQLITE_INCLUDE_SYSTEM === 'true'; +const includeSensitiveIdentities = + hasArg('--include-sensitive-identities') || process.env.PB_SQLITE_INCLUDE_SENSITIVE_IDENTITIES === 'true'; +const includeAuxLogs = hasArg('--include-aux-logs') || process.env.PB_SQLITE_INCLUDE_AUX_LOGS === 'true'; + +if (!fs.existsSync(dataDb)) { + throw new Error(`PocketBase SQLite data.db not found: ${dataDb}`); +} + +if (!includeSensitiveIdentities) { + console.warn('[pb:export:sqlite] openid/unionid/session-like fields are redacted in JSON export by default.'); +} +console.warn(`[pb:export:sqlite] reading SQLite source in read-only mode: ${dataDb}`); +console.warn(`[pb:export:sqlite] writing collection JSON to: ${exportDir}`); + +const python = findPython(); +const script = path.join(moduleDir, 'sqlite-exporter.py'); +const args = [ + script, + '--data-db', + dataDb, + '--export-dir', + exportDir, + '--batch-size', + batchSize, +]; + +if (fs.existsSync(auxDb)) args.push('--aux-db', auxDb); +if (fs.existsSync(storageDir)) args.push('--storage-dir', storageDir); +if (collections) args.push('--collections', collections); +if (includeSystem) args.push('--include-system'); +if (includeSensitiveIdentities) args.push('--include-sensitive-identities'); +if (includeAuxLogs) args.push('--include-aux-logs'); + +const result = spawnSync(python, args, { + cwd: repoRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], +}); + +if (result.stderr) process.stderr.write(result.stderr); +if (result.stdout) { + try { + const payload = JSON.parse(result.stdout); + console.log( + [ + '[pb:export:sqlite] export completed', + `collections=${payload.collectionCount}`, + `records=${payload.recordCount}`, + `manifest=${path.join(exportDir, 'sqlite-export-manifest.json')}`, + payload.storage ? `storageFiles=${payload.storage.fileCount}` : 'storageFiles=0', + ].join(' '), + ); + } catch { + process.stdout.write(result.stdout); + } +} + +if (result.error) throw result.error; +if (result.status !== 0) { + process.exit(result.status || 1); +} diff --git a/scripts/import-pocketbase/src/sqlite-exporter.py b/scripts/import-pocketbase/src/sqlite-exporter.py new file mode 100644 index 00000000..4b87d198 --- /dev/null +++ b/scripts/import-pocketbase/src/sqlite-exporter.py @@ -0,0 +1,436 @@ +#!/usr/bin/env python3 +"""Read-only PocketBase SQLite exporter. + +This script is intentionally small and dependency-free so it works on Windows +developer machines without compiling a Node SQLite native module. The TypeScript +wrapper owns CLI ergonomics; this script owns the SQLite cursor work. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import mimetypes +import os +import pathlib +import sqlite3 +import sys +from datetime import datetime, timezone +from typing import Any + + +SYSTEM_COLLECTIONS = { + "_authOrigins", + "_externalAuths", + "_mfas", + "_otps", + "_params", + "_superusers", +} + +SYSTEM_TABLE_PREFIXES = ("sqlite_",) + +SENSITIVE_KEY_PARTS = ( + "password", + "token", + "secret", + "privatekey", + "sessionkey", + "accesskey", + "appkey", + "apikey", + "api_v3_key", + "notifytoken", + "aeskey", + "wxaccesstoken", + "wechatsessionkey", + "smscode", + "verifycode", + "verificationcode", + "captcha", +) + +OPTIONAL_IDENTITY_KEY_PARTS = ( + "openid", + "unionid", +) + +JSONISH_STARTS = ("{", "[") +SKIP_FIELD = object() + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Export PocketBase SQLite collections to JSON.") + parser.add_argument("--data-db", required=True, help="Path to PocketBase data.db") + parser.add_argument("--aux-db", default="", help="Optional path to PocketBase auxiliary.db") + parser.add_argument("--storage-dir", default="", help="Optional path to PocketBase storage directory") + parser.add_argument("--export-dir", required=True, help="Output directory for collection JSON files") + parser.add_argument("--collections", default="", help="Comma-separated collection allowlist") + parser.add_argument("--include-system", action="store_true", help="Include PocketBase system collections") + parser.add_argument( + "--include-sensitive-identities", + action="store_true", + help="Keep openid/unionid fields in collection JSON. Use only in an encrypted migration workspace.", + ) + parser.add_argument( + "--include-aux-logs", + action="store_true", + help="Export auxiliary _logs summary and optional samples. Logs are not exported as full records by default.", + ) + parser.add_argument("--batch-size", type=int, default=1000, help="SQLite fetch batch size") + return parser.parse_args() + + +def connect_readonly(db_path: pathlib.Path) -> sqlite3.Connection: + if not db_path.exists(): + raise FileNotFoundError(f"SQLite database does not exist: {db_path}") + uri = db_path.resolve().as_posix() + con = sqlite3.connect(f"file:{uri}?mode=ro", uri=True) + con.row_factory = sqlite3.Row + return con + + +def json_loads(value: Any, fallback: Any) -> Any: + if value is None: + return fallback + if isinstance(value, (dict, list)): + return value + if isinstance(value, (bytes, bytearray)): + try: + value = value.decode("utf-8") + except UnicodeDecodeError: + return fallback + if not isinstance(value, str): + return fallback + text = value.strip() + if not text or not text.startswith(JSONISH_STARTS): + return fallback + try: + return json.loads(text) + except json.JSONDecodeError: + return fallback + + +def normalize_sqlite_value(value: Any) -> Any: + if isinstance(value, bytes): + return {"__sqliteBlobSha256": hashlib.sha256(value).hexdigest(), "__sqliteBlobSize": len(value)} + if isinstance(value, str): + stripped = value.strip() + if stripped.startswith(JSONISH_STARTS): + try: + return json.loads(stripped) + except json.JSONDecodeError: + return value + return value + + +def key_has_part(key: str, parts: tuple[str, ...]) -> bool: + lowered = key.lower().replace("_", "").replace("-", "") + return any(part in lowered for part in parts) + + +def sanitize_value(value: Any, path: list[str], include_sensitive_identities: bool, redactions: list[dict[str, str]]) -> Any: + key = path[-1] if path else "" + is_secret = key_has_part(key, SENSITIVE_KEY_PARTS) + is_optional_identity = key_has_part(key, OPTIONAL_IDENTITY_KEY_PARTS) + + if (is_secret or (is_optional_identity and not include_sensitive_identities)) and value not in (None, ""): + redactions.append( + { + "fieldPath": ".".join(path), + "reason": "secret" if is_secret else "optional_identity", + } + ) + return SKIP_FIELD + + if isinstance(value, dict): + clean: dict[str, Any] = {} + for child_key, child_value in value.items(): + sanitized = sanitize_value(child_value, [*path, child_key], include_sensitive_identities, redactions) + if sanitized is not SKIP_FIELD: + clean[child_key] = sanitized + return clean + if isinstance(value, list): + clean_list = [] + for index, child_value in enumerate(value): + sanitized = sanitize_value(child_value, [*path, str(index)], include_sensitive_identities, redactions) + if sanitized is not SKIP_FIELD: + clean_list.append(sanitized) + return clean_list + return value + + +def read_collections(con: sqlite3.Connection) -> list[dict[str, Any]]: + row = con.execute("select name from sqlite_master where type='table' and name='_collections'").fetchone() + if not row: + return [] + rows = con.execute('select * from "_collections" order by name').fetchall() + collections: list[dict[str, Any]] = [] + for item in rows: + raw = dict(item) + raw["system"] = bool(raw.get("system")) + raw["fields"] = json_loads(raw.get("fields"), []) + raw["indexes"] = json_loads(raw.get("indexes"), []) + raw["options"] = json_loads(raw.get("options"), {}) + collections.append(raw) + return collections + + +def sqlite_tables(con: sqlite3.Connection) -> set[str]: + return { + row["name"] + for row in con.execute("select name from sqlite_master where type='table'").fetchall() + if not any(str(row["name"]).startswith(prefix) for prefix in SYSTEM_TABLE_PREFIXES) + } + + +def collection_should_export(collection: dict[str, Any], allowlist: set[str] | None, include_system: bool, tables: set[str]) -> bool: + name = str(collection.get("name") or "") + if not name or name not in tables: + return False + if allowlist is not None and name not in allowlist: + return False + if not include_system and (name in SYSTEM_COLLECTIONS or bool(collection.get("system")) or name.startswith("_")): + return False + return True + + +def table_count(con: sqlite3.Connection, table: str) -> int: + return int(con.execute(f'select count(*) as c from "{table}"').fetchone()["c"]) + + +def write_collection( + con: sqlite3.Connection, + collection: dict[str, Any], + out_path: pathlib.Path, + batch_size: int, + include_sensitive_identities: bool, +) -> dict[str, Any]: + name = str(collection["name"]) + count = table_count(con, name) + field_names = [row["name"] for row in con.execute(f'pragma table_info("{name}")').fetchall()] + file_fields = [field.get("name") for field in collection.get("fields", []) if field.get("type") == "file"] + redaction_counts: dict[str, int] = {} + + out_path.parent.mkdir(parents=True, exist_ok=True) + with out_path.open("w", encoding="utf-8", newline="\n") as handle: + handle.write("[\n") + cursor = con.execute(f'select * from "{name}" order by id') + written = 0 + while True: + rows = cursor.fetchmany(max(1, batch_size)) + if not rows: + break + for row in rows: + record = {field: normalize_sqlite_value(row[field]) for field in field_names} + redactions: list[dict[str, str]] = [] + safe_record = sanitize_value(record, [], include_sensitive_identities, redactions) + for redaction in redactions: + key = redaction["fieldPath"] + redaction_counts[key] = redaction_counts.get(key, 0) + 1 + if written: + handle.write(",\n") + handle.write(json.dumps(safe_record, ensure_ascii=False, separators=(",", ":"))) + written += 1 + handle.write("\n]\n") + + return { + "name": name, + "collectionId": collection.get("id"), + "type": collection.get("type"), + "system": bool(collection.get("system")), + "recordCount": count, + "fieldCount": len(field_names), + "fields": field_names, + "fileFields": [field for field in file_fields if field], + "outputFile": out_path.name, + "redactedFields": [{"fieldPath": key, "count": value} for key, value in sorted(redaction_counts.items())], + } + + +def build_storage_manifest(storage_dir: pathlib.Path, collections: list[dict[str, Any]]) -> dict[str, Any] | None: + if not storage_dir or not storage_dir.exists(): + return None + + collection_by_id = {str(item.get("id")): str(item.get("name")) for item in collections} + files: list[dict[str, Any]] = [] + total_size = 0 + extension_counts: dict[str, int] = {} + collection_counts: dict[str, int] = {} + + for file_path in sorted(storage_dir.rglob("*")): + if not file_path.is_file(): + continue + relative = file_path.relative_to(storage_dir).as_posix() + if relative == ".DS_Store" or relative.endswith(".attrs") or "/thumbs_" in relative or relative.startswith("thumbs_"): + continue + parts = pathlib.PurePosixPath(relative).parts + collection_id = parts[0] if len(parts) >= 3 else "" + record_id = parts[1] if len(parts) >= 3 else "" + file_name = parts[-1] + size = file_path.stat().st_size + mime, _ = mimetypes.guess_type(file_name) + ext = file_path.suffix.lower() or "" + total_size += size + extension_counts[ext] = extension_counts.get(ext, 0) + 1 + collection_name = collection_by_id.get(collection_id, collection_id or "") + collection_counts[collection_name] = collection_counts.get(collection_name, 0) + 1 + files.append( + { + "collectionId": collection_id, + "collectionName": collection_name, + "recordId": record_id, + "fileName": file_name, + "relativePath": relative, + "size": size, + "mimeType": mime or "application/octet-stream", + "sha256": sha256_file(file_path), + } + ) + + return { + "storageDir": str(storage_dir), + "fileCount": len(files), + "totalSize": total_size, + "extensionCounts": dict(sorted(extension_counts.items())), + "collectionCounts": dict(sorted(collection_counts.items())), + "files": files, + } + + +def sha256_file(file_path: pathlib.Path) -> str: + digest = hashlib.sha256() + with file_path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def auxiliary_summary(aux_db: pathlib.Path, include_logs: bool) -> dict[str, Any] | None: + if not aux_db or not aux_db.exists(): + return None + con = connect_readonly(aux_db) + try: + tables = sqlite_tables(con) + summary: dict[str, Any] = {"path": str(aux_db), "tables": []} + for table in sorted(tables): + item: dict[str, Any] = {"name": table, "recordCount": table_count(con, table)} + if table == "_logs": + levels = con.execute('select level, count(*) as count from "_logs" group by level order by level').fetchall() + item["levelCounts"] = {str(row["level"]): int(row["count"]) for row in levels} + if include_logs: + samples = con.execute('select level, message, created from "_logs" order by created desc limit 25').fetchall() + item["recentSamples"] = [dict(sample) for sample in samples] + summary["tables"].append(item) + return summary + finally: + con.close() + + +def main() -> int: + args = parse_args() + data_db = pathlib.Path(args.data_db) + aux_db = pathlib.Path(args.aux_db) if args.aux_db else None + storage_dir = pathlib.Path(args.storage_dir) if args.storage_dir else None + export_dir = pathlib.Path(args.export_dir) + export_dir.mkdir(parents=True, exist_ok=True) + + allowlist = {item.strip() for item in args.collections.split(",") if item.strip()} or None + if args.batch_size < 1: + raise ValueError("--batch-size must be greater than 0") + + con = connect_readonly(data_db) + try: + collections = read_collections(con) + tables = sqlite_tables(con) + selected = [ + collection + for collection in collections + if collection_should_export(collection, allowlist, args.include_system, tables) + ] + + collection_reports = [] + for collection in selected: + collection_reports.append( + write_collection( + con, + collection, + export_dir / f"{collection['name']}.json", + args.batch_size, + args.include_sensitive_identities, + ) + ) + + schema_payload = { + "source": "pocketbase-sqlite", + "generatedAt": utc_now_iso(), + "collections": collections, + } + (export_dir / "pb_schema.sqlite.json").write_text( + json.dumps(schema_payload, ensure_ascii=False, indent=2), + encoding="utf-8", + newline="\n", + ) + + storage_manifest = build_storage_manifest(storage_dir, collections) if storage_dir else None + if storage_manifest: + (export_dir / "storage-manifest.json").write_text( + json.dumps(storage_manifest, ensure_ascii=False, indent=2), + encoding="utf-8", + newline="\n", + ) + + aux = auxiliary_summary(aux_db, args.include_aux_logs) if aux_db else None + + manifest = { + "source": "pocketbase-sqlite", + "generatedAt": utc_now_iso(), + "dataDb": str(data_db), + "auxDb": str(aux_db) if aux_db else None, + "storageDir": str(storage_dir) if storage_dir else None, + "exportDir": str(export_dir), + "includeSystemCollections": bool(args.include_system), + "includeSensitiveIdentities": bool(args.include_sensitive_identities), + "collectionCount": len(collection_reports), + "recordCount": sum(int(item["recordCount"]) for item in collection_reports), + "collections": collection_reports, + "skippedCollections": [ + {"name": collection.get("name"), "system": bool(collection.get("system")), "type": collection.get("type")} + for collection in collections + if collection not in selected + ], + "storage": None + if storage_manifest is None + else { + "manifestFile": "storage-manifest.json", + "fileCount": storage_manifest["fileCount"], + "totalSize": storage_manifest["totalSize"], + "collectionCounts": storage_manifest["collectionCounts"], + "extensionCounts": storage_manifest["extensionCounts"], + }, + "auxiliary": aux, + } + + (export_dir / "sqlite-export-manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2), + encoding="utf-8", + newline="\n", + ) + + print(json.dumps(manifest, ensure_ascii=False)) + return 0 + finally: + con.close() + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: # noqa: BLE001 - CLI failure should be explicit. + print(f"[sqlite-exporter] {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/pb-sqlite-export-test.js b/scripts/pb-sqlite-export-test.js new file mode 100644 index 00000000..e8a8f657 --- /dev/null +++ b/scripts/pb-sqlite-export-test.js @@ -0,0 +1,185 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const repoRoot = process.cwd(); +const exportScript = path.join(repoRoot, 'scripts', 'import-pocketbase', 'src', 'export-sqlite.ts'); + +function findPython() { + for (const candidate of [process.env.PYTHON, process.env.PYTHON_BIN, 'python', 'py', 'python3'].filter(Boolean)) { + const result = spawnSync(candidate, ['--version'], { encoding: 'utf8' }); + if (result.status === 0) return candidate; + } + throw new Error('Python 3 is required for pb-sqlite-export-test'); +} + +function writeFixture(sqliteDir, storageDir) { + const python = findPython(); + const script = ` +import json +import pathlib +import sqlite3 + +sqlite_dir = pathlib.Path(r'''${sqliteDir}''') +storage_dir = pathlib.Path(r'''${storageDir}''') +data_db = sqlite_dir / 'data.db' +aux_db = sqlite_dir / 'auxiliary.db' + +db = sqlite3.connect(data_db) +db.executescript(""" +create table "_collections" ( + id text primary key, + system boolean, + type text, + name text, + fields text, + indexes text, + listRule text, + viewRule text, + createRule text, + updateRule text, + deleteRule text, + options text, + created text, + updated text +); +create table "users" ( + id text primary key, + created text, + updated text, + phone text, + password text, + tokenKey text, + wechatOpenId text, + wechatUnionId text, + stats text +); +create table "questions" ( + id text primary key, + created text, + updated text, + content text, + options text, + media text +); +create table "_superusers" ( + id text primary key, + email text, + password text +); +""") + +insert_collection = """ +insert into "_collections" ( + id, system, type, name, fields, indexes, listRule, viewRule, createRule, updateRule, deleteRule, options, created, updated +) values (?, ?, ?, ?, ?, '[]', '', '', '', '', '', '{}', '2026-01-01 00:00:00Z', '2026-01-01 00:00:00Z') +""" +db.execute(insert_collection, ('_pb_users_auth_', 0, 'auth', 'users', json.dumps([ + {'name': 'id', 'type': 'text'}, + {'name': 'password', 'type': 'password'}, + {'name': 'tokenKey', 'type': 'text'}, + {'name': 'phone', 'type': 'text'}, + {'name': 'wechatOpenId', 'type': 'text'}, + {'name': 'wechatUnionId', 'type': 'text'}, + {'name': 'stats', 'type': 'json'}, +]))) +db.execute(insert_collection, ('pbc_questions', 0, 'base', 'questions', json.dumps([ + {'name': 'id', 'type': 'text'}, + {'name': 'content', 'type': 'text'}, + {'name': 'options', 'type': 'json'}, + {'name': 'media', 'type': 'file', 'maxSelect': 1}, +]))) +db.execute(insert_collection, ('pbc_superusers', 1, 'auth', '_superusers', json.dumps([ + {'name': 'password', 'type': 'password'}, +]))) +db.execute( + 'insert into users (id, created, updated, phone, password, tokenKey, wechatOpenId, wechatUnionId, stats) values (?, ?, ?, ?, ?, ?, ?, ?, ?)', + ('u1', '2026-01-01 00:00:00Z', '2026-01-01 00:00:00Z', '13800000000', 'hash-secret', 'token-secret', 'wx-openid', 'wx-unionid', json.dumps({'answered': 3})), +) +db.execute( + 'insert into questions (id, created, updated, content, options, media) values (?, ?, ?, ?, ?, ?)', + ('q1', '2026-01-01 00:00:00Z', '2026-01-01 00:00:00Z', '题干', json.dumps(['A', 'B']), 'question.png'), +) +db.execute('insert into _superusers (id, email, password) values (?, ?, ?)', ('su1', 'root@example.com', 'secret')) +db.commit() +db.close() + +aux = sqlite3.connect(aux_db) +aux.executescript(""" +create table "_logs" ( + id text primary key, + level integer not null, + message text not null, + data text not null, + created text not null +); +""") +aux.execute( + 'insert into "_logs" (id, level, message, data, created) values (?, ?, ?, ?, ?)', + ('l1', 1, 'GET /api/collections/users/records/u1', json.dumps({'token': 'must-not-export'}), '2026-01-01 00:00:00Z'), +) +aux.commit() +aux.close() + +media_dir = storage_dir / 'pbc_questions' / 'q1' +media_dir.mkdir(parents=True, exist_ok=True) +(media_dir / 'question.png').write_bytes(b'fixture-image') +(media_dir / 'question.png.attrs').write_text('{}', encoding='utf-8') +`; + + const result = spawnSync(python, ['-c', script], { encoding: 'utf8' }); + if (result.error) throw result.error; + assert.equal(result.status, 0, `fixture setup should pass\nstdout=${result.stdout}\nstderr=${result.stderr}`); +} + +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'tiku-pb-sqlite-export-')); +const sqliteDir = path.join(root, 'pb'); +const exportDir = path.join(root, 'export'); +const storageDir = path.join(sqliteDir, 'storage'); +fs.mkdirSync(storageDir, { recursive: true }); +writeFixture(sqliteDir, storageDir); + +const result = spawnSync(process.execPath, ['--import', 'tsx', exportScript], { + cwd: repoRoot, + encoding: 'utf8', + env: { + ...process.env, + PB_SQLITE_DIR: sqliteDir, + PB_EXPORT_DIR: exportDir, + }, +}); + +assert.equal(result.status, 0, `export should pass\nstdout=${result.stdout}\nstderr=${result.stderr}`); + +const users = JSON.parse(fs.readFileSync(path.join(exportDir, 'users.json'), 'utf8')); +assert.equal(users.length, 1, 'users collection should export records'); +assert.equal(users[0].phone, '13800000000', 'phone should be kept for migration matching'); +assert.equal('password' in users[0], false, 'password should be removed from normal JSON export'); +assert.equal('tokenKey' in users[0], false, 'token should be removed from normal JSON export'); +assert.equal('wechatOpenId' in users[0], false, 'openid should be removed by default'); +assert.deepEqual(users[0].stats, { answered: 3 }, 'JSON fields should be parsed'); + +const questions = JSON.parse(fs.readFileSync(path.join(exportDir, 'questions.json'), 'utf8')); +assert.deepEqual(questions[0].options, ['A', 'B'], 'question JSON fields should be parsed'); + +assert.ok(!fs.existsSync(path.join(exportDir, '_superusers.json')), 'system collections should not export by default'); + +const manifest = JSON.parse(fs.readFileSync(path.join(exportDir, 'sqlite-export-manifest.json'), 'utf8')); +assert.equal(manifest.collectionCount, 2, 'manifest should include exported collection count'); +assert.equal(manifest.recordCount, 2, 'manifest should include exported record count'); +assert.ok( + manifest.collections.find(item => item.name === 'users')?.redactedFields.some(item => item.fieldPath === 'password'), + 'manifest should report redacted user password field', +); +assert.equal(manifest.storage.fileCount, 1, 'manifest should summarize storage files'); +assert.equal(manifest.auxiliary.tables[0].name, '_logs', 'manifest should summarize auxiliary logs'); + +const storageManifest = JSON.parse(fs.readFileSync(path.join(exportDir, 'storage-manifest.json'), 'utf8')); +assert.equal(storageManifest.files[0].collectionName, 'questions', 'storage manifest should map collection id to name'); +assert.ok(storageManifest.files[0].sha256, 'storage manifest should include file hash'); + +fs.rmSync(root, { recursive: true, force: true }); + +console.log('[PASS] PocketBase SQLite export');