diff --git a/.gitignore b/.gitignore index 40798586..4472349f 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,9 @@ storage-manifest*.json # ✅ 真实数据迁移报告（可能含业务敏感信息） docs/refactor/migration-reports/ +# ✅ 本地压测报告（可能含内部规模、接口耗时和错误样本） +docs/refactor/performance-reports/ + # ✅ 生产上线验收证据（可能包含内部域名、抽样说明或敏感运维信息） docs/refactor/production-launch-evidence.json docs/refactor/launch-artifacts/ diff --git a/README.md b/README.md index 001cfcb2..6c3397cd 100644 --- a/README.md +++ b/README.md @@ -439,7 +439,7 @@ dry-run 会检查导出目录、JSON 形态、核心集合缺失、重复/缺失 默认 dry-run 使用 `development` profile；正式迁移、预生产验收和 CI 应使用 `production` profile。生产 profile 会额外输出 `migrationReadiness`，检查用户、题目、科目、分类、订单、套餐、激活码、单词和知识手册等必需集合，以及用户手机号、题目归属、订单套餐、激活码、单词和手册归属等关键字段覆盖率。`--profile` 只接受 `development` 或 `production`，拼写错误会按 blocker 失败。 -当前真实 SQLite 基线已经跑通只读导出和干净库全量导入：58 个业务 collection、248555 条记录、9 个 storage 原始资源文件；本地 `npx supabase db reset` 后执行 `npm run pb:import:json` 用时约 8 分 58 秒，`npm run pb:import:validate` 结果为 0 failures、3 warnings。`user_answer_records`、`mock_exam_configs`、`referral_qrcodes`、`commission_settings` 已进入标准化导入，导入后核心计数包括 3670 用户、74102 题目、85199 条旧答题记录、38205 条错题、636 订单、447 权益、48 个模考蓝图、79 个推广码和 1 条租户分佣设置。 +当前真实 SQLite 基线已经跑通只读导出和干净库全量导入：58 个业务 collection、248555 条记录、9 个 storage 原始资源文件；本地 `npx supabase db reset` 后执行 `npm run pb:import:json` 最近用时约 10 分 11 秒，`npm run pb:import:validate` 结果为 0 failures、3 warnings。`user_answer_records`、`mock_exam_configs`、`referral_qrcodes`、`commission_settings` 已进入标准化导入，导入后核心计数包括 3670 用户、74102 题目、85199 条旧答题记录、38205 条错题、636 订单、447 权益、79 个推广码和 1 条租户分佣设置。导入器现在还会从旧 `region_modules/module_nodes/subjects/categories/questions.nodeId` 生成新架构 `content_entries/content_nodes/question_collections/practice_blueprints`，最新导入计数为 11 个题库入口、2830 个内容节点、1597 个题目合集、82106 条合集题目关系、3102 个顺序/随机练习蓝图；所有已发布旧题都会写入 `entry_id/content_node_id/primary_collection_id`，供 Taro 前端按新模型直接消费。 production dry-run 目前仍有 2 个真实数据 blocker：30 个订单缺用户、22 个知识手册章节缺所属手册；导入器会把它们隔离到财务复核/迁移待复核手册并写入 `pb_import_issues`，其中最新导入 run 的 critical issue 剩余 29 个：7 个已支付订单缺用户、22 个手册章节缺所属手册。正式切换前仍必须人工确认，详见： @@ -454,6 +454,22 @@ docs/refactor/next-development-todo.md docs/refactor/pocketbase-real-data-migration-runbook.md ``` +### API 压测和 4 核 16G 评估 + +默认本地烟测会构建 API 并自动启动临时端口，使用当前 `DATABASE_URL` 的真实数据做只读混合请求： + +```powershell +$env:DATABASE_URL="postgresql://postgres:postgres@127.0.0.1:54322/postgres" +npm run perf:api:local +``` + +报告输出到已忽略的 `docs/refactor/performance-reports/`。4 核 16G 云服务器应按压测 runbook 跑 6/30/50/100 阶梯并发，并结合 PostgreSQL 调参文档观察慢 SQL、连接数、锁等待和 P95/P99： + +```text +docs/refactor/postgresql-4c16g-tuning.md +docs/refactor/performance-benchmark-runbook.md +``` + 正式切换前建议使用 production 严格模式： ```bash diff --git a/docs/refactor/README.md b/docs/refactor/README.md index de2b235b..8a8ac92f 100644 --- a/docs/refactor/README.md +++ b/docs/refactor/README.md @@ -28,6 +28,8 @@ - `docs/refactor/supabase-frontend-access-strategy.md`：Supabase 官方推荐能力与本项目业务 API 边界。 - `docs/refactor/taro-frontend-integration.md`：Taro/H5/小程序启动、请求封装、页面/API 映射。 - `docs/refactor/taro-h5-deployment.md`：Taro H5 三域名部署、运行时配置、Nginx、CSP、缓存和 CORS 边界。 +- `docs/refactor/postgresql-4c16g-tuning.md`：4 核 16G 自托管 PostgreSQL 起步调参、观察 SQL 和回滚方式。 +- `docs/refactor/performance-benchmark-runbook.md`：本地/云端 API 压测、4 核 16G 阶梯并发矩阵和报告归档方式。 - `docs/refactor/multitenant-auth-security-contract.md`：多租户隔离、鉴权、权限和资源安全红线。 - `docs/refactor/production-launch-evidence.template.json`：生产上线证据模板；真实证据填入本地 `production-launch-evidence.json` 后运行 `npm run launch:gate`。 @@ -35,6 +37,6 @@ 1. 先按 `docs/refactor/multitenant-auth-security-contract.md` 的 P0 清单补齐鉴权、生产配置和租户隔离测试，并在上云前运行 `npm run readiness:production` / `npm run readiness:production:db`。 2. 新建 `apps/taro`，按 `docs/refactor/taro-frontend-integration.md` 优先接租户解析、首页、题库练习、背单词、知识手册、个人中心。 -3. 导出 PocketBase 真实数据到 `pb_export/*.json`，执行 `npm run pb:import:json` 和 `npm run pb:import:validate`。 -4. 为对象存储、分数线、视频、Excel/CSV 补齐 provider/导入能力，并复用 `content_import_jobs` 管线。 +3. 继续按 `docs/refactor/pocketbase-real-data-migration-runbook.md` 处理真实旧数据人工复核项；当前真实导入已能生成新架构题库入口、分类、合集和练习蓝图。 +4. 上云前按 `docs/refactor/postgresql-4c16g-tuning.md` 准备 4 核 16G PostgreSQL 起步参数，再按 `docs/refactor/performance-benchmark-runbook.md` 跑 smoke、30/50 并发和少量写入压测。 5. 接真实短信、微信/QQ 登录、微信支付/支付宝，继续增强 CRM worker 和其它异步任务，进入商用验收。 diff --git a/docs/refactor/backend-capability-status.md b/docs/refactor/backend-capability-status.md index 48432fd1..756373b0 100644 --- a/docs/refactor/backend-capability-status.md +++ b/docs/refactor/backend-capability-status.md @@ -172,7 +172,7 @@ | --- | --- | --- | | PocketBase schema/导出分析 | 可联调 | `scripts/import-pocketbase` 支持 schema summary/risk、`npm run pb:import:dry-run` 导出目录静态迁移报告 | | PocketBase JSON dry-run | 可联调 | 不写数据库，检查导出目录、JSON 形态、核心集合、旧 ID、敏感字段、schema relation、未映射集合和关键业务计数；`--profile=production` 会额外检查生产迁移必需集合和关键字段覆盖率，正式切换建议配合 `--fail-on-warnings` | -| PocketBase SQLite 标准化导入 | 可联调 | 已支持真实 SQLite 只读导出后的核心集合标准化，新增覆盖 `user_answer_records -> answer_records/wrong_questions`、`mock_exam_configs -> practice_blueprints`、`referral_qrcodes -> referral_qrcodes/referral_codes`、`commission_settings -> tenant_commission_settings`；干净本地 Supabase 全量真实导入 248555 条记录约 8 分 58 秒，`pb:import:validate` 为 0 failures、3 warnings；缺用户订单会进入财务复核且不自动开权益，缺归属手册章节会进入“迁移待复核手册”并写 `pb_import_issues`；正式切换前仍需人工处理 7 个已支付缺用户订单和 22 个待复核手册章节 | +| PocketBase SQLite 标准化导入 | 可联调 | 已支持真实 SQLite 只读导出后的核心集合标准化，新增覆盖 `user_answer_records -> answer_records/wrong_questions`、`mock_exam_configs -> practice_blueprints`、`referral_qrcodes -> referral_qrcodes/referral_codes`、`commission_settings -> tenant_commission_settings`；干净本地 Supabase 全量真实导入 248555 条记录最近约 10 分 11 秒，`pb:import:validate` 为 0 failures、3 warnings；导入后会生成 11 个题库入口、2830 个内容节点、1597 个题目合集、82106 条合集题目关系和 3102 个顺序/随机练习蓝图，并回填全部已发布旧题的 `entry_id/content_node_id/primary_collection_id`；缺用户订单会进入财务复核且不自动开权益，缺归属手册章节会进入“迁移待复核手册”并写 `pb_import_issues`；正式切换前仍需人工处理 7 个已支付缺用户订单和 22 个待复核手册章节 | | 题目 JSON preview/import | 可联调 | 后端负责规范化、issue、幂等、审计 | | 公共题库采纳、手动同步和自动同步 | 可联调 | 平台授权后，租户可采纳公共题库并复制已发布题目快照；同步 API 和 `public-banks` worker 支持新增/更新题目、重新校验授权、跨租户拒绝、审计记录、租户内容通知和租户自改冲突保护；冲突处理 API 已支持单条/批量采纳平台版本和保留租户本地版本；worker 失败会写 `public_question_bank_sync_failed` 租户通知并保留稳定错误码，恢复成功会自动关闭失败通知；平台可通过 `/api/platform-admin/question-bank-sync-status` 查看跨租户同步运营状态；已覆盖跨租户、重复采纳、采纳后组卷、同步新增题、通知隔离/已读/自动 resolved、冲突不覆盖、单条/批量冲突处理、worker 自动同步/失败通知/恢复关闭、starter 单地区不可见/不可采纳第二地区题库、pro 全国套餐可见第二地区题库等测试 | | 单词 JSON preview/import | 可联调 | 兼容旧模板 | @@ -231,4 +231,4 @@ npm run pb:import:validate npm run check:importer ``` -结果：`pb:import:json` 用时约 8 分 58 秒，`pb:import:validate` 为 0 failures、3 warnings。`npm run pb:import:dry-run -- --profile=production --json` 仍按预期返回非 0，因为真实旧数据还有 30 个订单缺 `userId` 和 22 个手册章节缺 `subjectId` 两个上线前 blocker。 +结果：`pb:import:json` 用时约 10 分 11 秒，`pb:import:validate` 为 0 failures、3 warnings。新题库导航计数：11 个 `content_entries`、2830 个 `content_nodes`、1597 个 `question_collections`、82106 条 `question_collection_items`、3102 个 `practice_blueprints`，已发布旧题缺入口/节点/合集数为 0。`npm run pb:import:dry-run -- --profile=production --json` 仍按预期返回非 0，因为真实旧数据还有 30 个订单缺 `userId` 和 22 个手册章节缺 `subjectId` 两个上线前 blocker。 diff --git a/docs/refactor/next-development-todo.md b/docs/refactor/next-development-todo.md index 9eae840b..cacaa419 100644 --- a/docs/refactor/next-development-todo.md +++ b/docs/refactor/next-development-todo.md @@ -78,7 +78,7 @@ - 已补 `npm run pb:import:dry-run` 静态迁移报告工具、`--profile=production` 生产迁移门禁、关键集合/关键字段覆盖率检查、strict warning 门禁测试和真实数据迁移验收 runbook；真实 SQLite 已导出 58 个业务 collection、248555 条记录，并跑过 production dry-run。 - 当前真实 dry-run 剩余 blocker：30 个订单缺 `userId`，其中 7 个 paid；22 个知识手册章节缺 `subjectId`。导入器已补隔离策略：缺用户订单进入财务复核且不自动开权益，缺归属手册章节进入“迁移待复核手册”。正式切换前仍必须人工找回/确认这些记录。 - 已补真实 mapper gap：`user_answer_records` 85442 条标准化到 `answer_records/wrong_questions`，`mock_exam_configs` 48 条标准化到 `practice_blueprints`，`referral_qrcodes` 79 条标准化到 `referral_qrcodes/referral_codes`，`commission_settings` 1 条标准化到 `tenant_commission_settings`。 - - 真实 `pb:import:json` 已在干净本地 Supabase 上跑通，248555 条真实记录导入约 8 分 58 秒；`pb:import:validate` 为 0 failures、3 warnings。核心落库计数：3670 用户、74102 题目、85199 旧答题记录、38205 错题、636 订单、447 权益、48 个模考蓝图、79 个推广码、1 条租户分佣设置。 + - 真实 `pb:import:json` 已在干净本地 Supabase 上跑通，248555 条真实记录最近导入约 10 分 11 秒；`pb:import:validate` 为 0 failures、3 warnings。核心落库计数：3670 用户、74102 题目、85199 旧答题记录、38205 错题、636 订单、447 权益、79 个推广码、1 条租户分佣设置；同时生成 11 个题库入口、2830 个内容节点、1597 个题目合集、82106 条合集题目关系、3102 个顺序/随机练习蓝图，已发布旧题缺入口/节点/合集数为 0。 - 最新导入 run 剩余 29 个 critical import issues：7 个已支付订单缺用户、22 个手册章节缺所属手册，均为正式切换前人工复核项；旧答题记录空用户或旧用户已删除已降为 warning。 - 处理完 blocker 后再跑 `npm run pb:import:dry-run -- --profile=production --json --fail-on-warnings`，再跑 `pb:import:json`、`pb:import:validate` 和业务抽样。 - 对题目 JSON、单词、知识手册、分数线、视频走后端 preview/import API 做二次验证。 @@ -90,6 +90,7 @@ - 已补 `npm run launch:gate` 生产上线证据门禁和 `docs/refactor/production-launch-evidence.template.json` 模板；最终切换前必须把 readiness、远程 Auth、RLS、生产 dry-run、导入校验、API/worker/Taro、运行时审计、`@codex-security`、备份/回滚/真实抽样/生产 provider 等证据填入本地 `production-launch-evidence.json` 并通过门禁。 - 确认数据库迁移流程、备份恢复、日志、告警。 - 准备 API 容器部署和 Supabase 云端/自托管连接方案。 + - 已补 `npm run perf:api:local` 和 `docs/refactor/performance-benchmark-runbook.md`，可在本地或云端对真实迁移数据做只读混合压测；4 核 16G 正式容量报告需上云后按 6/30/50/100 阶梯并发复跑并归档到本地上线证据。 ### P1 商用功能完善 @@ -188,7 +189,7 @@ 4. 运维 - 后台操作审计报表。 - 定时备份、恢复演练。 - - 性能压测、慢 SQL、索引审查。 + - 性能压测、慢 SQL、索引审查；当前已有 API 压测脚本和 4 核 16G runbook，待云服务器部署后输出正式容量报告。 ## Taro 前端开发 TODO diff --git a/docs/refactor/performance-benchmark-runbook.md b/docs/refactor/performance-benchmark-runbook.md new file mode 100644 index 00000000..66b711c6 --- /dev/null +++ b/docs/refactor/performance-benchmark-runbook.md @@ -0,0 +1,154 @@ +# API 压测与 4 核 16G 容量评估 Runbook + +更新时间：2026-06-30 + +这份 runbook 用于在本地 Docker/Supabase 或未来 4 核 16G 云服务器上，对 SaaS 题库 API 做可重复压测。目标不是一次性给出永久容量承诺，而是建立一套可以随着题库、用户量、SQL 和硬件变化持续复跑的基线。 + +## 工具 + +压测脚本： + +```bash +npm run perf:api:local +``` + +脚本会： + +- 自动从 `DATABASE_URL` 指向的 PostgreSQL 中发现一个活跃租户、学生、题库入口、分类节点、合集、练习蓝图、单词单元和知识手册。 +- 默认启动本地 API 服务，并使用真实 API 请求做混合读路径压测。 +- 默认不创建练习 session，不写业务数据。 +- 输出 JSON 和 Markdown 报告到 `docs/refactor/performance-reports/`。该目录已被 `.gitignore` 忽略，不应提交。 + +## 前置条件 + +本地真实迁移库压测： + +```powershell +npx supabase db reset +$env:DATABASE_URL="postgresql://postgres:postgres@127.0.0.1:54322/postgres" +npm run pb:import:json +npm run pb:import:validate +npm run perf:api:local +``` + +烟测 seed 压测： + +```powershell +$env:DATABASE_URL="postgresql://postgres:postgres@127.0.0.1:54322/postgres" +npm run db:smoke-seed +npm run perf:api:local +``` + +如果 API 已经在运行： + +```powershell +$env:PERF_START_SERVER="false" +$env:PERF_API_BASE="http://127.0.0.1:8787" +npm run perf:api:local +``` + +## 环境变量 + +| 变量 | 默认值 | 说明 | +| --- | --- | --- | +| `DATABASE_URL` | `postgresql://postgres:postgres@127.0.0.1:54322/postgres` | 压测使用的数据源 | +| `PERF_API_BASE` | 空 | 已运行 API 地址；为空时默认启动本地 API | +| `PERF_START_SERVER` | 根据 `PERF_API_BASE` 推断 | 是否由脚本启动 API | +| `PERF_API_PORT` | 随机端口 | 脚本启动 API 时使用的端口 | +| `PERF_TENANT_CODE` | `master` | 优先选择的租户 slug | +| `PERF_DURATION_SECONDS` | `15` | 压测持续时间 | +| `PERF_CONCURRENCY` | `6` | 并发 worker 数 | +| `PERF_RAMP_SECONDS` | `3` | 并发爬坡时间 | +| `PERF_QUESTION_LIMIT` | `20` | 单次合集题目请求数量 | +| `PERF_INCLUDE_WRITES` | `false` | 是否加入创建练习 session 写请求 | +| `PERF_OUTPUT_DIR` | `docs/refactor/performance-reports` | 报告输出目录 | +| `PERF_REQUEST_TIMEOUT_MS` | `15000` | 单请求超时 | + +## 默认工作负载 + +默认混合读路径包含： + +- `/health` +- `/api/tenant/resolve` +- `/api/catalog/regions` +- `/api/catalog/content-entries` +- `/api/catalog/content-nodes?mode=flat` +- `/api/catalog/question-collections` +- `/api/catalog/question-collections/questions` +- `/api/catalog/practice-blueprints` +- `/api/learning/stats` +- `/api/learning/trend` +- `/api/learning/leaderboard` +- `/api/catalog/vocabulary-words` +- `/api/learning/vocabulary/stats` +- `/api/catalog/handbook-chapters` +- `/api/catalog/handbook-entries?includeContent=true` + +写入路径默认关闭。只有在专门的测试库或可丢弃预生产库中，才建议打开： + +```powershell +$env:PERF_INCLUDE_WRITES="true" +npm run perf:api:local +``` + +开启后会加入低权重 `POST /api/learning/practice-sessions`，用于观察组卷、免费额度、权益校验和 session 写入开销。 + +## 4 核 16G 阶梯压测建议 + +在云服务器上先按 `docs/refactor/postgresql-4c16g-tuning.md` 配置 shared-host 起步值，再跑以下矩阵。每轮之间间隔 2 到 5 分钟，观察 CPU、内存、磁盘 I/O、连接数和慢 SQL。 + +| 场景 | 并发 | 时长 | 写入 | 用途 | +| --- | ---: | ---: | --- | --- | +| smoke | 6 | 30s | 否 | 确认部署和数据可访问 | +| baseline-10 | 10 | 2min | 否 | 学生正常浏览/刷题入口 | +| baseline-30 | 30 | 5min | 否 | 中小租户晚高峰 | +| baseline-50 | 50 | 5min | 否 | 单机读路径压力观察 | +| mixed-30 | 30 | 5min | 是 | 加入少量创建练习 session | +| mixed-50 | 50 | 5min | 是 | 观察写入、锁和连接池 | +| spike-100 | 100 | 2min | 否 | 短峰值和缓存命中观察 | + +PowerShell 示例： + +```powershell +$env:DATABASE_URL="postgresql://postgres:***@127.0.0.1:5432/postgres" +$env:PERF_API_BASE="https://api.example.com" +$env:PERF_START_SERVER="false" +$env:PERF_DURATION_SECONDS="300" +$env:PERF_CONCURRENCY="30" +$env:PERF_RAMP_SECONDS="30" +$env:PERF_INCLUDE_WRITES="false" +npm run perf:api:local +``` + +## 初步验收线 + +上云测试早期建议先用保守指标： + +- 只读 mixed catalog 工作负载：P95 小于 300ms，P99 小于 800ms，错误率小于 0.1%。 +- 加入少量 session 写入：P95 小于 500ms，P99 小于 1200ms，错误率小于 0.5%。 +- 数据库无连接耗尽、无 OOM、无长时间 `idle in transaction`。 +- `pg_stat_database` 中 `xact_rollback` 不应快速增长。 +- `pg_stat_activity` 不应长期堆积锁等待或 I/O wait。 + +这些不是最终 SLA。正式 SLA 要结合真实云服务器、真实 CDN、真实对象存储、真实支付回调和前端 Web Vitals 重新制定。 + +## 问题定位 + +如果 P95 或 P99 明显升高，按下面顺序排查： + +1. 查看报告的接口明细，先定位慢接口。 +2. 打开 PostgreSQL 慢 SQL 日志，或在生产启用 `pg_stat_statements`。 +3. 对慢接口涉及 SQL 执行 `explain (analyze, buffers)`。 +4. 检查 API/worker 连接池是否过大导致数据库活跃连接超过 4 核可承受范围。 +5. 检查 `content_nodes path`、`question_collection_items`、`answer_records`、`practice_sessions`、`vocabulary_progress` 等大表索引。 +6. 对 dashboard、排行榜、销售转化、分佣报表这类聚合接口，优先做日/小时预聚合，而不是无限放大 SQL 超时。 + +## 结果归档 + +本地生成的报告目录不入 Git。正式上云验收时，可以把代表性结果摘要写入 `production-launch-evidence.json` 的本地证据文件，再运行： + +```bash +npm run launch:gate +``` + +证据中只记录报告路径、并发矩阵、P95/P99、错误率和结论，不保存真实 token、支付密钥、用户隐私或完整响应。 diff --git a/docs/refactor/pocketbase-real-data-migration-runbook.md b/docs/refactor/pocketbase-real-data-migration-runbook.md index 7f29a8da..466c8869 100644 --- a/docs/refactor/pocketbase-real-data-migration-runbook.md +++ b/docs/refactor/pocketbase-real-data-migration-runbook.md @@ -148,6 +148,15 @@ auxiliary.db: _logs 121715 条，仅摘要 - `referral_qrcodes` 79 条会导入到 `referral_qrcodes`，并同步补 `referral_codes`。 - `commission_settings` 1 条会导入到 `tenant_commission_settings`。 +当前导入器还会把旧题库的树形导航整理成新 SaaS 题库导航： + +- `region_modules` 生成 `content_entries(entry_type='question_practice')`。 +- `module_nodes`、`subjects`、`categories` 生成 `content_nodes`，保留任意深度分类和考试意向标记。 +- `questions.nodeId` 会优先生成 `module_node:*:direct` 题目合集；旧库真实数据中 74102 道题都有 `nodeId`，因此这是迁移后的第一归属。 +- `categoryId` 只覆盖 27665 道题，作为补充分类合集处理，不再作为唯一前端入口。 +- 每个有效合集自动生成顺序刷题和随机刷题 `practice_blueprints`，旧 `mock_exam_configs` 继续生成全真模拟蓝图。 +- 每道已发布旧题都会回填 `entry_id`、`content_node_id`、`primary_collection_id`，方便 Taro 前端直接按 `content_entries/content_nodes/question_collections/practice_blueprints` 对接。 + 导入器对两个真实 blocker 采用“隔离 + 审计”策略，不会静默丢数据： - 缺用户订单仍导入 `orders/payments/order_items`，`raw_payload.migration.reviewRequired=true`、`entitlementBlocked=true`，并写入 `pb_import_issues`。缺用户订单不会自动开通权益。 @@ -163,7 +172,7 @@ production dry-run 还有这些 warning，属于运营处理项： ```text 环境：本地 Supabase，先执行 npx supabase db reset 命令：npm run pb:import:json -耗时：约 8 分 58 秒 +耗时：约 10 分 11 秒 导入后校验：npm run pb:import:validate => 0 failures, 3 warnings 核心计数： @@ -173,9 +182,20 @@ answer_records: 85199 wrong_questions: 38205 orders: 636 entitlements: 447 -practice_blueprints(mode=mock_exam): 48 referral_qrcodes: 79 tenant_commission_settings: 1 + +新题库导航计数： +content_entries: 11 +content_nodes: 2830 +question_collections: 1597 +question_collection_items: 82106 +practice_blueprints: 3102 +questions_missing_entry/node/collection: 0 +node_direct_collections: 1518 +hidden_review_entries: 1 +orphan_review_collections: 5 +orphan_review_questions: 103 ``` 最新导入 run 的 `pb_import_issues` 仍有 29 个 critical，需要正式切换前人工复核： @@ -185,11 +205,16 @@ tenant_commission_settings: 1 导入后校验的 3 个 warnings 当前含义： -- 2533 道题保留 `legacy_category_id` 但无法解析到旧 `categories/module_nodes`，原始 `categoryId` 与 `nodeId` 相同，且旧导出中已没有对应分类或节点。不能由导入器硬绑到错误分类，需后续按科目/题型人工归档或在内容后台批量挂载。 +- 2533 道题保留 `legacy_category_id` 但无法解析到旧 `categories/module_nodes`。其中能落到明确 `nodeId` 或公开科目的题会进入对应合集；确实断裂且不可公开归类的 103 道题进入隐藏的“迁移待复核题库”，不会出现在学生端目录。 - 5298 条旧答题记录保留 `legacy_question_id` 但无法解析到题目，说明旧学习日志引用了已删除/未导出的题目；这些记录不会进入错题本有效组卷。 - 最新 run 存在 29 个 critical import issues，即上面的付费订单和手册章节人工复核项。 -性能注意：真实 `pb:import:json` 已能在 9 分钟左右完成 248555 条真实记录导入。后续若题库规模继续增长，仍建议继续批量化 `questions/question_versions` 并补阶段耗时统计、失败恢复和断点重跑策略。 +旧 `nodeId` 断裂的题目分两类处理： + +- 103 道题：科目可解析，但旧节点和科目入口都无法恢复，进入隐藏入口 `migration_review:orphan_questions`、inactive 节点和 draft 合集，只给租户后台复核。 +- 129 道题：旧节点断裂，但科目/入口仍可公开解析，挂到对应公开科目 fallback 合集，学生端可正常看到。 + +性能注意：真实 `pb:import:json` 已能在约 10 分钟完成 248555 条真实记录导入并生成新导航。后续若题库规模继续增长，仍建议继续批量化 `questions/question_versions`、记录阶段耗时、增加失败恢复和断点重跑策略，并按 `docs/refactor/performance-benchmark-runbook.md` 做导入后 API 压测。 ## 阶段 1：静态 Dry-Run diff --git a/docs/refactor/postgresql-4c16g-tuning.md b/docs/refactor/postgresql-4c16g-tuning.md new file mode 100644 index 00000000..e4c25efe --- /dev/null +++ b/docs/refactor/postgresql-4c16g-tuning.md @@ -0,0 +1,230 @@ +# PostgreSQL 4 核 16G 生产调参基线 + +更新时间：2026-06-30 + +这份文档用于后续把 Supabase/PostgreSQL 自托管到 4 核 16G 云服务器时做生产起步配置。目标是先给题库 SaaS 一个安全、可回滚、可观测的基线，而不是追求一次性压满硬件。 + +参考来源： + +- postgresqlco.nf Tuning Guide：https://postgresqlco.nf/tuning-guide +- PostgreSQL 官方 Resource Consumption：https://www.postgresql.org/docs/current/runtime-config-resource.html +- PostgreSQL 官方 Query Planning：https://www.postgresql.org/docs/current/runtime-config-query.html +- PostgreSQL 官方 WAL/Checkpoint：https://www.postgresql.org/docs/current/runtime-config-wal.html +- PostgreSQL 官方 Connections：https://www.postgresql.org/docs/current/runtime-config-connection.html + +说明：postgresqlco.nf 的页面可作为参数分类和调参入口参考；具体参数语义、重启要求和风险以 PostgreSQL 官方文档为准。 + +## 适用前提 + +- 单台 4 vCPU、16 GB RAM 云服务器。 +- 同机可能运行 Supabase 服务、API、worker、Nginx、日志/监控 agent。 +- 数据库以题库读多写少、后台批量导入、支付/订单小事务、学习记录持续写入为主。 +- 生产必须使用 SSD 云盘，数据库数据盘和备份盘分开更好。 + +如果 PostgreSQL 是唯一重负载服务，可以使用下方 `dedicated-db` 值；如果 API/worker/Nginx 也在同机，先使用 `shared-host` 值。 + +## 推荐起步值 + +| 参数 | shared-host 起步值 | dedicated-db 起步值 | 说明 | +| --- | --- | --- | --- | +| `max_connections` | `80` | `120` | 4 核机器不宜靠大量直连抗并发；API 侧连接池和 Supabase pooler 更重要 | +| `shared_buffers` | `3GB` | `4GB` | 官方建议专用库可从约 25% RAM 起步；同机多服务要给 OS cache 和应用留空间 | +| `effective_cache_size` | `10GB` | `12GB` | 这是优化器估算值，不实际占内存 | +| `work_mem` | `16MB` | `16MB` | 每个排序/哈希操作都可能分配一次，不按连接数简单相乘 | +| `maintenance_work_mem` | `512MB` | `768MB` | 导入、建索引、VACUUM 可受益；注意 autovacuum worker 并发 | +| `autovacuum_work_mem` | `256MB` | `256MB` | 避免多个 autovacuum 同时吃掉过多内存 | +| `wal_buffers` | `16MB` | `16MB` | 默认自动通常够用；16MB 是常见稳妥上限 | +| `min_wal_size` | `1GB` | `2GB` | 给批量导入和写入波峰预留 WAL | +| `max_wal_size` | `6GB` | `8GB` | 减少频繁 checkpoint；值越大，崩溃恢复时间可能更长 | +| `checkpoint_timeout` | `10min` | `15min` | 降低 checkpoint 频率；配合 `max_wal_size` 观察恢复窗口 | +| `checkpoint_completion_target` | `0.9` | `0.9` | 平滑 checkpoint I/O | +| `effective_io_concurrency` | `100` | `100` | SSD 云盘起步值；机械盘不适用 | +| `random_page_cost` | `1.1` | `1.1` | SSD 上鼓励合理索引扫描；上线后用 EXPLAIN 复核 | +| `jit` | `off` | `off` | 题库 API 多为短查询，先避免 JIT 带来的计划开销 | +| `log_min_duration_statement` | `500ms` | `500ms` | 上线初期捕捉慢 SQL；稳定后可调到 `1000ms` | +| `idle_in_transaction_session_timeout` | `60s` | `60s` | 防止后台或脚本长事务占锁 | +| `statement_timeout` | `30s` | `30s` | API 请求不应长期占用数据库；导入脚本用会话级覆盖 | +| `lock_timeout` | `5s` | `5s` | 防止普通请求长时间等锁 | + +## 应用连接池边界 + +4 核机器的关键不是把 `max_connections` 拉大，而是控制同时活跃 SQL 的数量。 + +建议： + +- `apps/api` 主进程连接池：`DB_POOL_MAX=10` 到 `20`。 +- `apps/worker` 每类 worker 连接池：`DB_POOL_MAX=4` 到 `8`。 +- 导入、导出、对账、公共题库同步等 worker 不要全部高并发同时跑。 +- H5/Taro 前端永远不直连数据库，只走 Supabase Auth 或 `apps/api`。 +- 生产如使用 Supabase pooler，应限制事务池大小，避免 API 扩容后数据库连接被打满。 + +## ALTER SYSTEM 示例 + +上线前先保存当前值： + +```sql +select name, setting, unit, source, pending_restart +from pg_settings +where name in ( + 'max_connections', + 'shared_buffers', + 'effective_cache_size', + 'work_mem', + 'maintenance_work_mem', + 'autovacuum_work_mem', + 'wal_buffers', + 'min_wal_size', + 'max_wal_size', + 'checkpoint_timeout', + 'checkpoint_completion_target', + 'effective_io_concurrency', + 'random_page_cost', + 'jit', + 'log_min_duration_statement', + 'idle_in_transaction_session_timeout', + 'statement_timeout', + 'lock_timeout' +) +order by name; +``` + +同机部署推荐先执行： + +```sql +alter system set max_connections = '80'; +alter system set shared_buffers = '3GB'; +alter system set effective_cache_size = '10GB'; +alter system set work_mem = '16MB'; +alter system set maintenance_work_mem = '512MB'; +alter system set autovacuum_work_mem = '256MB'; +alter system set wal_buffers = '16MB'; +alter system set min_wal_size = '1GB'; +alter system set max_wal_size = '6GB'; +alter system set checkpoint_timeout = '10min'; +alter system set checkpoint_completion_target = '0.9'; +alter system set effective_io_concurrency = '100'; +alter system set random_page_cost = '1.1'; +alter system set jit = 'off'; +alter system set log_min_duration_statement = '500ms'; +alter system set idle_in_transaction_session_timeout = '60s'; +alter system set statement_timeout = '30s'; +alter system set lock_timeout = '5s'; +select pg_reload_conf(); +``` + +以下参数需要重启 PostgreSQL 才会生效：`max_connections`、`shared_buffers`、`wal_buffers`。执行后用下面语句确认： + +```sql +select name, setting, unit, pending_restart +from pg_settings +where pending_restart = true +order by name; +``` + +## 导入脚本会话级覆盖 + +真实迁移导入、批量导入、建索引可能超过普通 API 的 `statement_timeout`。不要为了导入放大生产全局超时，应该在导入连接上局部设置： + +```sql +set statement_timeout = '30min'; +set lock_timeout = '30s'; +set maintenance_work_mem = '1GB'; +``` + +正式切换仍以 `pb:import:dry-run --profile=production`、`pb:import:json`、`pb:import:validate` 和业务抽样为准。 + +## 观察和验收 + +上线后至少观察 24 到 72 小时： + +```sql +select state, wait_event_type, wait_event, count(*)::int +from pg_stat_activity +where datname = current_database() +group by state, wait_event_type, wait_event +order by count desc; +``` + +```sql +select datname, xact_commit, xact_rollback, blks_read, blks_hit, + round(blks_hit * 100.0 / nullif(blks_hit + blks_read, 0), 2) as cache_hit_ratio +from pg_stat_database +where datname = current_database(); +``` + +```sql +select checkpoints_timed, checkpoints_req, checkpoint_write_time, checkpoint_sync_time +from pg_stat_bgwriter; +``` + +```sql +select pid, usename, now() - xact_start as xact_age, state, query +from pg_stat_activity +where xact_start is not null +order by xact_age desc +limit 20; +``` + +验收标准： + +- API P95 延迟稳定，没有大量 `ClientRead/ClientWrite`、锁等待或 I/O wait。 +- `pb:import:validate` 无 failure。 +- `checkpoints_req` 不应持续快速增长；如增长明显，优先调大 `max_wal_size` 或检查写入峰值。 +- 没有 OOM、没有频繁连接耗尽、没有长时间 idle in transaction。 +- 慢 SQL 日志能对应到具体接口、worker 或迁移脚本。 + +## 压测闭环 + +调参后必须跑 API 压测，不要只凭参数表判断容量。当前仓库提供本地压测脚本： + +```bash +npm run perf:api:local +``` + +默认压测只读路径，会自动发现真实迁移后的租户、学生、题库入口、合集、蓝图、单词和手册数据，并输出报告到已忽略的 `docs/refactor/performance-reports/`。详细环境变量、4 核 16G 阶梯并发矩阵和报告归档方式见： + +- `docs/refactor/performance-benchmark-runbook.md` + +4 核 16G 首次上云建议至少跑： + +- `smoke`：6 并发 30 秒，只读。 +- `baseline-30`：30 并发 5 分钟，只读。 +- `baseline-50`：50 并发 5 分钟，只读。 +- `mixed-30`：30 并发 5 分钟，开启少量练习 session 写入。 + +若 P95/P99 明显升高，优先结合慢 SQL、`pg_stat_activity` 和报告中的接口明细定位，再考虑索引、预聚合、连接池和 worker 调度；不要直接继续放大 `max_connections`。 + +## 回滚方式 + +如果调参后出现内存压力、启动失败或查询计划异常： + +```sql +alter system reset shared_buffers; +alter system reset effective_cache_size; +alter system reset work_mem; +alter system reset maintenance_work_mem; +alter system reset autovacuum_work_mem; +alter system reset wal_buffers; +alter system reset min_wal_size; +alter system reset max_wal_size; +alter system reset checkpoint_timeout; +alter system reset checkpoint_completion_target; +alter system reset effective_io_concurrency; +alter system reset random_page_cost; +alter system reset jit; +alter system reset log_min_duration_statement; +alter system reset idle_in_transaction_session_timeout; +alter system reset statement_timeout; +alter system reset lock_timeout; +select pg_reload_conf(); +``` + +若 `shared_buffers`、`wal_buffers`、`max_connections` 已改，需要重启后才算完全回滚。 + +## 后续优化方向 + +- 上线后启用 `pg_stat_statements`，按总耗时和平均耗时定位慢 SQL。 +- 为 `answer_records`、`practice_sessions`、`content_asset_access_events`、`video_play_events` 等大表规划按月或按租户分区。 +- 对租户后台 dashboard、排行榜、销售转化、分佣报表做日/小时预聚合。 +- 对真实 `pb:import:json` 增加阶段耗时统计，继续批量化 `questions/question_versions`。 +- 如果三套 H5、API、worker、Supabase 全部同机，优先加连接池和 worker 调度，而不是继续放大 PostgreSQL 内存参数。 diff --git a/package.json b/package.json index 1736d2aa..7a403247 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "smoke:core-api": "node scripts/smoke-core-api.js", "smoke:auth:remote": "node scripts/remote-auth-jwt-smoke.js", "test:api": "npm run db:smoke-seed && npm run build:api && node scripts/api-integration-test.js --start-server", + "perf:api:local": "npm run build:api && node scripts/api-performance-benchmark.js", "test:worker:crm": "npm run db:smoke-seed && npm run build:worker && node scripts/crm-worker-integration-test.js", "test:worker:commerce": "npm run db:smoke-seed && npm run build:worker && node scripts/commerce-worker-integration-test.js", "test:worker:platform-billing": "npm run db:smoke-seed && npm run build:worker && node scripts/platform-billing-worker-integration-test.js", diff --git a/scripts/api-performance-benchmark.js b/scripts/api-performance-benchmark.js new file mode 100644 index 00000000..b5b15962 --- /dev/null +++ b/scripts/api-performance-benchmark.js @@ -0,0 +1,721 @@ +import { spawn } from 'node:child_process'; +import fs from 'node:fs/promises'; +import net from 'node:net'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import pg from 'pg'; + +const DEFAULT_DATABASE_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; +const DATABASE_URL = process.env.DATABASE_URL || DEFAULT_DATABASE_URL; +const OUTPUT_DIR = process.env.PERF_OUTPUT_DIR || 'docs/refactor/performance-reports'; +const API_BASE_ENV = process.env.PERF_API_BASE || process.env.API_BASE || ''; +const START_SERVER = envBool('PERF_START_SERVER', !API_BASE_ENV); +const DURATION_SECONDS = envNumber('PERF_DURATION_SECONDS', 15); +const CONCURRENCY = envNumber('PERF_CONCURRENCY', 6); +const RAMP_SECONDS = envNumber('PERF_RAMP_SECONDS', 3, { allowZero: true }); +const INCLUDE_WRITES = envBool('PERF_INCLUDE_WRITES', false); +const START_PORT = Number(process.env.PERF_API_PORT || 0) || 0; +const TENANT_CODE = process.env.PERF_TENANT_CODE || 'master'; +const QUESTION_LIMIT = envNumber('PERF_QUESTION_LIMIT', 20); +const MAX_ERRORS_TO_KEEP = envNumber('PERF_MAX_ERRORS', 20); +const DEFAULT_TIMEOUT_MS = envNumber('PERF_REQUEST_TIMEOUT_MS', 15_000); + +let apiBase = API_BASE_ENV || ''; +let serverProcess = null; +let serverLogs = ''; + +function envBool(key, fallback) { + const value = process.env[key]; + if (value === undefined || value === '') return fallback; + return ['1', 'true', 'yes', 'on'].includes(value.toLowerCase()); +} + +function envNumber(key, fallback, options = {}) { + const raw = process.env[key]; + if (!raw) return fallback; + const value = Number(raw); + if (!Number.isFinite(value)) return fallback; + if (value > 0) return value; + if (options.allowZero && value === 0) return 0; + return fallback; +} + +function getFreePort() { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.on('error', reject); + server.listen(START_PORT, '127.0.0.1', () => { + const address = server.address(); + server.close(() => resolve(address.port)); + }); + }); +} + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +function shanghaiTimestampForFile(date = new Date()) { + const parts = Object.fromEntries( + new Intl.DateTimeFormat('en-CA', { + timeZone: 'Asia/Shanghai', + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hour12: false, + }).formatToParts(date).map(part => [part.type, part.value]), + ); + return `${parts.year}${parts.month}${parts.day}-${parts.hour}${parts.minute}${parts.second}`; +} + +function buildUrl(baseUrl, endpoint) { + const target = new URL(endpoint.path, baseUrl); + for (const [key, value] of Object.entries(endpoint.query || {})) { + if (value !== undefined && value !== null && value !== '') { + target.searchParams.set(key, String(value)); + } + } + return target; +} + +async function requestJson(endpoint, context, timeoutMs = DEFAULT_TIMEOUT_MS) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const started = performance.now(); + const tenantId = context.tenantId || context.tenant?.id; + const userId = context.userId || context.user?.id; + try { + const response = await fetch(buildUrl(apiBase, endpoint), { + method: endpoint.method || 'GET', + signal: controller.signal, + headers: { + 'content-type': 'application/json', + ...(endpoint.tenantHeader === false || !tenantId ? {} : { 'x-tenant-id': tenantId }), + ...(endpoint.userHeader === false || !userId ? {} : { 'x-user-id': userId }), + ...(endpoint.headers || {}), + }, + body: endpoint.body ? JSON.stringify(endpoint.body(context)) : undefined, + }); + const elapsedMs = performance.now() - started; + const payload = await response.json().catch(() => ({})); + if (!response.ok) { + return { + ok: false, + status: response.status, + elapsedMs, + error: payload?.code || payload?.error || payload?.message || response.statusText, + message: payload?.error || payload?.message || response.statusText, + }; + } + return { ok: true, status: response.status, elapsedMs, bytes: JSON.stringify(payload).length }; + } catch (error) { + return { + ok: false, + status: 0, + elapsedMs: performance.now() - started, + error: error?.name === 'AbortError' ? 'REQUEST_TIMEOUT' : error?.message || String(error), + }; + } finally { + clearTimeout(timeout); + } +} + +async function waitForHealth(timeoutMs = 20_000) { + const started = Date.now(); + let lastError = null; + while (Date.now() - started < timeoutMs) { + try { + const response = await fetch(new URL('/health', apiBase)); + const payload = await response.json().catch(() => ({})); + if (response.ok && payload.ok) return; + } catch (error) { + lastError = error; + } + await sleep(250); + } + throw new Error(`API server did not become healthy. ${lastError?.message || ''}\n${serverLogs}`); +} + +async function startServerIfNeeded() { + if (!START_SERVER) return; + const port = await getFreePort(); + apiBase = `http://127.0.0.1:${port}`; + serverProcess = spawn(process.execPath, ['apps/api/dist/apps/api/src/server.js'], { + cwd: process.cwd(), + env: { + ...process.env, + PORT: String(port), + DATABASE_URL, + MAX_JSON_BODY_BYTES: process.env.MAX_JSON_BODY_BYTES || '1048576', + MAX_IMPORT_JSON_BODY_BYTES: process.env.MAX_IMPORT_JSON_BODY_BYTES || '10485760', + }, + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + serverProcess.stdout.on('data', chunk => { + serverLogs += chunk.toString(); + }); + serverProcess.stderr.on('data', chunk => { + serverLogs += chunk.toString(); + }); + await waitForHealth(); +} + +function stopServer() { + if (serverProcess && !serverProcess.killed) { + serverProcess.kill(); + } +} + +async function one(pool, sql, params = []) { + const result = await pool.query(sql, params); + return result.rows[0] || null; +} + +async function many(pool, sql, params = []) { + const result = await pool.query(sql, params); + return result.rows; +} + +async function discoverBenchmarkContext() { + const pool = new pg.Pool({ connectionString: DATABASE_URL, max: 4 }); + try { + const tenant = await one( + pool, + ` + select id, slug, name + from public.tenants + where status = 'active' + order by case when slug = $1 then 0 else 1 end, slug asc + limit 1 + `, + [TENANT_CODE], + ); + if (!tenant) throw new Error('No active tenant found. Run smoke seed or PocketBase import first.'); + + const user = await one( + pool, + ` + select pu.id, coalesce(pu.name, pu.username, pu.phone, pu.legacy_id, pu.id::text) as name + from public.tenant_memberships tm + join public.platform_users pu on pu.id = tm.user_id + where tm.tenant_id = $1 and tm.status = 'active' and tm.role = 'student' + order by pu.created_at asc + limit 1 + `, + [tenant.id], + ); + if (!user) throw new Error(`No active student user found for tenant ${tenant.slug}.`); + + const entry = await one( + pool, + ` + select id, name + from public.content_entries + where tenant_id = $1 + and entry_type = 'question_practice' + and is_active = true + and visibility <> 'hidden' + order by sort_order asc, created_at asc + limit 1 + `, + [tenant.id], + ); + + const node = entry + ? await one( + pool, + ` + select id, name + from public.content_nodes + where tenant_id = $1 and entry_id = $2 and is_active = true + order by depth asc, sort_order asc, created_at asc + limit 1 + `, + [tenant.id, entry.id], + ) + : null; + + const collection = await one( + pool, + ` + select id, name, question_count as "questionCount", entry_id as "entryId", node_id as "nodeId" + from public.question_collections + where tenant_id = $1 and status = 'active' and question_count > 0 + order by question_count desc, sort_order asc, created_at asc + limit 1 + `, + [tenant.id], + ); + + const blueprint = collection + ? await one( + pool, + ` + select id, name, mode + from public.practice_blueprints + where tenant_id = $1 and status = 'active' and collection_id = $2 + order by case mode when 'sequential' then 0 when 'random' then 1 else 2 end, sort_order asc + limit 1 + `, + [tenant.id, collection.id], + ) + : null; + + const vocabularyUnit = await one( + pool, + ` + select id, name + from public.vocabulary_units + where tenant_id = $1 and is_active = true + order by sort_order asc, created_at asc + limit 1 + `, + [tenant.id], + ); + + const handbookSubject = await one( + pool, + ` + select id, name + from public.handbook_subjects + where tenant_id = $1 and is_active = true + order by sort_order asc, created_at asc + limit 1 + `, + [tenant.id], + ); + + const handbookChapter = handbookSubject + ? await one( + pool, + ` + select id, name + from public.handbook_chapters + where tenant_id = $1 and subject_id = $2 and is_active = true + order by sort_order asc, created_at asc + limit 1 + `, + [tenant.id, handbookSubject.id], + ) + : null; + + const dbStats = { + before: await captureDbStats(pool), + tableCounts: await many( + pool, + ` + select label, count_value::bigint::text as value + from ( + select 'questions' as label, count(*) as count_value from public.questions where tenant_id = $1 + union all select 'content_entries', count(*) from public.content_entries where tenant_id = $1 + union all select 'content_nodes', count(*) from public.content_nodes where tenant_id = $1 + union all select 'question_collections', count(*) from public.question_collections where tenant_id = $1 + union all select 'practice_blueprints', count(*) from public.practice_blueprints where tenant_id = $1 + union all select 'vocabulary_words', count(*) from public.vocabulary_words where tenant_id = $1 + union all select 'handbook_entries', count(*) from public.handbook_entries where tenant_id = $1 + ) counts + order by label asc + `, + [tenant.id], + ), + }; + + return { + tenant, + user, + entry, + node, + collection, + blueprint, + vocabularyUnit, + handbookSubject, + handbookChapter, + dbStats, + }; + } finally { + await pool.end(); + } +} + +async function captureDbStats(pool) { + const database = await one( + pool, + ` + select datname, xact_commit::bigint::text as "xactCommit", + xact_rollback::bigint::text as "xactRollback", + blks_read::bigint::text as "blocksRead", + blks_hit::bigint::text as "blocksHit", + tup_returned::bigint::text as "tuplesReturned", + tup_fetched::bigint::text as "tuplesFetched", + tup_inserted::bigint::text as "tuplesInserted", + tup_updated::bigint::text as "tuplesUpdated", + tup_deleted::bigint::text as "tuplesDeleted", + deadlocks::bigint::text as deadlocks + from pg_stat_database + where datname = current_database() + `, + ); + const activity = await many( + pool, + ` + select state, wait_event_type as "waitEventType", wait_event as "waitEvent", count(*)::int + from pg_stat_activity + where datname = current_database() + group by state, wait_event_type, wait_event + order by count desc + limit 20 + `, + ); + return { database, activity }; +} + +function createEndpointCatalog(context) { + const endpoints = [ + { name: 'health', weight: 6, method: 'GET', path: '/health', tenantHeader: false, userHeader: false }, + { name: 'tenant.resolve', weight: 6, method: 'GET', path: '/api/tenant/resolve', query: { tenantCode: context.tenant.slug }, tenantHeader: false, userHeader: false }, + { name: 'catalog.regions', weight: 4, method: 'GET', path: '/api/catalog/regions' }, + { name: 'catalog.content_entries', weight: 10, method: 'GET', path: '/api/catalog/content-entries', query: { entryType: 'question_practice' } }, + { name: 'learning.stats', weight: 5, method: 'GET', path: '/api/learning/stats' }, + { name: 'learning.trend', weight: 4, method: 'GET', path: '/api/learning/trend', query: { days: 14 } }, + { name: 'leaderboard.questions_7d', weight: 3, method: 'GET', path: '/api/learning/leaderboard', query: { metric: 'questions', period: '7d', limit: 20 } }, + ]; + + if (context.entry) { + endpoints.push( + { name: 'catalog.content_nodes.flat', weight: 10, method: 'GET', path: '/api/catalog/content-nodes', query: { entryId: context.entry.id, mode: 'flat' } }, + { name: 'catalog.question_collections.entry', weight: 8, method: 'GET', path: '/api/catalog/question-collections', query: { entryId: context.entry.id, limit: 50 } }, + ); + } + if (context.node) { + endpoints.push( + { name: 'catalog.question_collections.node', weight: 5, method: 'GET', path: '/api/catalog/question-collections', query: { nodeId: context.node.id, limit: 50 } }, + ); + } + if (context.collection) { + endpoints.push( + { name: 'catalog.collection_questions', weight: 12, method: 'GET', path: '/api/catalog/question-collections/questions', query: { collectionId: context.collection.id, limit: QUESTION_LIMIT } }, + { name: 'catalog.practice_blueprints', weight: 7, method: 'GET', path: '/api/catalog/practice-blueprints', query: { collectionId: context.collection.id, limit: 20 } }, + ); + } + if (context.vocabularyUnit) { + endpoints.push( + { name: 'catalog.vocabulary_words', weight: 5, method: 'GET', path: '/api/catalog/vocabulary-words', query: { unitId: context.vocabularyUnit.id, limit: 80 } }, + { name: 'learning.vocabulary.stats', weight: 3, method: 'GET', path: '/api/learning/vocabulary/stats' }, + ); + } + if (context.handbookSubject) { + endpoints.push( + { name: 'catalog.handbook_chapters', weight: 4, method: 'GET', path: '/api/catalog/handbook-chapters', query: { subjectId: context.handbookSubject.id } }, + ); + } + if (context.handbookChapter) { + endpoints.push( + { name: 'catalog.handbook_entries', weight: 5, method: 'GET', path: '/api/catalog/handbook-entries', query: { chapterId: context.handbookChapter.id, includeContent: 'true' } }, + ); + } + if (INCLUDE_WRITES && context.blueprint && context.collection) { + endpoints.push({ + name: 'learning.practice_session.create', + weight: 1, + method: 'POST', + path: '/api/learning/practice-sessions', + body: () => ({ + blueprintId: context.blueprint.id, + collectionId: context.collection.id, + mode: context.blueprint.mode || 'sequential', + questionLimit: Math.min(5, QUESTION_LIMIT), + }), + }); + } + + return expandWeightedEndpoints(endpoints); +} + +function expandWeightedEndpoints(endpoints) { + const weighted = []; + for (const endpoint of endpoints) { + for (let i = 0; i < endpoint.weight; i += 1) weighted.push(endpoint); + } + if (!weighted.length) throw new Error('No benchmark endpoints could be built from discovered data.'); + return weighted; +} + +function pickEndpoint(weightedEndpoints) { + return weightedEndpoints[Math.floor(Math.random() * weightedEndpoints.length)]; +} + +function percentile(sortedValues, p) { + if (!sortedValues.length) return 0; + const index = Math.ceil((p / 100) * sortedValues.length) - 1; + return sortedValues[Math.min(Math.max(index, 0), sortedValues.length - 1)]; +} + +function summarizeSamples(samples) { + if (!samples.length) { + return { + requests: 0, + ok: 0, + errors: 0, + errorRate: 0, + throughputRps: 0, + okThroughputRps: 0, + latencyAll: { p50Ms: 0, p90Ms: 0, p95Ms: 0, p99Ms: 0, maxMs: 0 }, + latencyOk: { p50Ms: 0, p90Ms: 0, p95Ms: 0, p99Ms: 0, maxMs: 0 }, + endpoints: [], + }; + } + + const okSamples = samples.filter(sample => sample.ok); + const allLatencies = samples.map(sample => sample.elapsedMs).sort((a, b) => a - b); + const okLatencies = okSamples.map(sample => sample.elapsedMs).sort((a, b) => a - b); + const byEndpoint = new Map(); + for (const sample of samples) { + const bucket = byEndpoint.get(sample.name) || []; + bucket.push(sample); + byEndpoint.set(sample.name, bucket); + } + + const endpointSummaries = [...byEndpoint.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, endpointSamples]) => { + const ok = endpointSamples.filter(sample => sample.ok); + const latencies = ok.map(sample => sample.elapsedMs).sort((a, b) => a - b); + return { + name, + requests: endpointSamples.length, + ok: ok.length, + errors: endpointSamples.length - ok.length, + p50Ms: round(percentile(latencies, 50)), + p90Ms: round(percentile(latencies, 90)), + p95Ms: round(percentile(latencies, 95)), + p99Ms: round(percentile(latencies, 99)), + maxMs: round(latencies.at(-1) || 0), + }; + }); + + const durationMs = Math.max(...samples.map(sample => sample.finishedAt), 0) - Math.min(...samples.map(sample => sample.startedAt), 0); + const durationSeconds = durationMs > 0 ? durationMs / 1000 : DURATION_SECONDS; + + return { + requests: samples.length, + ok: okSamples.length, + errors: samples.length - okSamples.length, + errorRate: samples.length ? round((samples.length - okSamples.length) / samples.length) : 0, + throughputRps: round(samples.length / durationSeconds), + okThroughputRps: round(okSamples.length / durationSeconds), + latencyAll: { + p50Ms: round(percentile(allLatencies, 50)), + p90Ms: round(percentile(allLatencies, 90)), + p95Ms: round(percentile(allLatencies, 95)), + p99Ms: round(percentile(allLatencies, 99)), + maxMs: round(allLatencies.at(-1) || 0), + }, + latencyOk: { + p50Ms: round(percentile(okLatencies, 50)), + p90Ms: round(percentile(okLatencies, 90)), + p95Ms: round(percentile(okLatencies, 95)), + p99Ms: round(percentile(okLatencies, 99)), + maxMs: round(okLatencies.at(-1) || 0), + }, + endpoints: endpointSummaries, + }; +} + +function round(value, digits = 2) { + return Number(Number(value || 0).toFixed(digits)); +} + +async function workerLoop(workerId, weightedEndpoints, context, stopAt, samples, errors) { + const rampDelay = RAMP_SECONDS > 0 ? (workerId / Math.max(CONCURRENCY, 1)) * RAMP_SECONDS * 1000 : 0; + if (rampDelay > 0) await sleep(rampDelay); + while (performance.now() < stopAt) { + const endpoint = pickEndpoint(weightedEndpoints); + const startedAt = performance.now(); + const result = await requestJson(endpoint, context); + const finishedAt = performance.now(); + samples.push({ + name: endpoint.name, + ok: result.ok, + status: result.status, + elapsedMs: result.elapsedMs, + startedAt, + finishedAt, + }); + if (!result.ok && errors.length < MAX_ERRORS_TO_KEEP) { + errors.push({ + endpoint: endpoint.name, + status: result.status, + error: result.error, + message: result.message, + elapsedMs: round(result.elapsedMs), + }); + } + } +} + +async function runBenchmark(context) { + const weightedEndpoints = createEndpointCatalog(context); + const endpointNames = [...new Set(weightedEndpoints.map(endpoint => endpoint.name))].sort(); + const samples = []; + const errors = []; + const startedAt = new Date(); + const startPerf = performance.now(); + const stopAt = startPerf + DURATION_SECONDS * 1000; + await Promise.all( + Array.from({ length: CONCURRENCY }, (_, index) => workerLoop(index, weightedEndpoints, context, stopAt, samples, errors)), + ); + const finishedAt = new Date(); + + const pool = new pg.Pool({ connectionString: DATABASE_URL, max: 2 }); + let afterStats = null; + try { + afterStats = await captureDbStats(pool); + } finally { + await pool.end(); + } + + return { + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + apiBase, + config: { + durationSeconds: DURATION_SECONDS, + concurrency: CONCURRENCY, + rampSeconds: RAMP_SECONDS, + includeWrites: INCLUDE_WRITES, + questionLimit: QUESTION_LIMIT, + timeoutMs: DEFAULT_TIMEOUT_MS, + }, + target: { + tenant: context.tenant, + user: context.user, + entry: context.entry, + node: context.node, + collection: context.collection, + blueprint: context.blueprint, + vocabularyUnit: context.vocabularyUnit, + handbookSubject: context.handbookSubject, + handbookChapter: context.handbookChapter, + endpointNames, + }, + db: { + before: context.dbStats.before, + after: afterStats, + tableCounts: context.dbStats.tableCounts, + }, + summary: summarizeSamples(samples), + errors, + }; +} + +function markdownReport(report) { + const lines = []; + lines.push('# API 本地压测报告'); + lines.push(''); + lines.push(`生成时间：${new Date(report.finishedAt).toLocaleString('zh-CN', { timeZone: 'Asia/Shanghai' })}`); + lines.push(''); + lines.push('## 配置'); + lines.push(''); + lines.push(`- API：${report.apiBase}`); + lines.push(`- 租户：${report.target.tenant.slug} / ${report.target.tenant.name}`); + lines.push(`- 并发：${report.config.concurrency}`); + lines.push(`- 持续时间：${report.config.durationSeconds}s`); + lines.push(`- Ramp：${report.config.rampSeconds}s`); + lines.push(`- 写入压测：${report.config.includeWrites ? '开启' : '关闭'}`); + lines.push(`- 请求超时：${report.config.timeoutMs}ms`); + lines.push(''); + lines.push('## 总览'); + lines.push(''); + lines.push('| 指标 | 数值 |'); + lines.push('| --- | ---: |'); + lines.push(`| 请求数 | ${report.summary.requests} |`); + lines.push(`| 成功数 | ${report.summary.ok} |`); + lines.push(`| 错误数 | ${report.summary.errors} |`); + lines.push(`| 错误率 | ${(report.summary.errorRate * 100).toFixed(2)}% |`); + lines.push(`| 吞吐 | ${report.summary.throughputRps} req/s |`); + lines.push(`| 成功吞吐 | ${report.summary.okThroughputRps} req/s |`); + lines.push(`| P50 | ${report.summary.latencyOk.p50Ms} ms |`); + lines.push(`| P90 | ${report.summary.latencyOk.p90Ms} ms |`); + lines.push(`| P95 | ${report.summary.latencyOk.p95Ms} ms |`); + lines.push(`| P99 | ${report.summary.latencyOk.p99Ms} ms |`); + lines.push(`| Max | ${report.summary.latencyOk.maxMs} ms |`); + lines.push(''); + lines.push('## 接口明细'); + lines.push(''); + lines.push('| 接口 | 请求 | 成功 | 错误 | P50 ms | P90 ms | P95 ms | P99 ms | Max ms |'); + lines.push('| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |'); + for (const item of report.summary.endpoints) { + lines.push(`| ${item.name} | ${item.requests} | ${item.ok} | ${item.errors} | ${item.p50Ms} | ${item.p90Ms} | ${item.p95Ms} | ${item.p99Ms} | ${item.maxMs} |`); + } + lines.push(''); + lines.push('## 数据规模'); + lines.push(''); + lines.push('| 表 | 当前租户记录数 |'); + lines.push('| --- | ---: |'); + for (const item of report.db.tableCounts) { + lines.push(`| ${item.label} | ${item.value} |`); + } + lines.push(''); + if (report.errors.length) { + lines.push('## 错误样本'); + lines.push(''); + lines.push('| 接口 | 状态 | 错误 | 耗时 ms |'); + lines.push('| --- | ---: | --- | ---: |'); + for (const error of report.errors) { + const errorText = [error.error, error.message && error.message !== error.error ? error.message : ''].filter(Boolean).join(': '); + lines.push(`| ${error.endpoint} | ${error.status} | ${String(errorText).replaceAll('|', '\\|')} | ${error.elapsedMs} |`); + } + lines.push(''); + } + lines.push('## 说明'); + lines.push(''); + lines.push('- 默认压测是只读混合工作负载，适合在真实迁移库上做烟测。'); + lines.push('- 结果只能代表当前本机 Docker、API 进程和数据库状态；4 核 16G 云服务器需要按 runbook 跑阶梯并发。'); + lines.push('- 若 P95 明显升高，下一步应结合 PostgreSQL 慢 SQL、`pg_stat_activity` 和 API 日志定位。'); + lines.push(''); + return `${lines.join('\n')}\n`; +} + +async function writeReport(report) { + await fs.mkdir(OUTPUT_DIR, { recursive: true }); + const stamp = shanghaiTimestampForFile(new Date(report.finishedAt)); + const jsonPath = path.join(OUTPUT_DIR, `api-benchmark-${stamp}.json`); + const mdPath = path.join(OUTPUT_DIR, `api-benchmark-${stamp}.md`); + await fs.writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8'); + await fs.writeFile(mdPath, markdownReport(report), 'utf8'); + return { jsonPath, mdPath }; +} + +async function main() { + try { + if (!apiBase && !START_SERVER) { + throw new Error('PERF_API_BASE is required when PERF_START_SERVER=false.'); + } + const context = await discoverBenchmarkContext(); + await startServerIfNeeded(); + if (!apiBase) throw new Error('API base URL was not resolved.'); + console.log(`[perf] target api=${apiBase} tenant=${context.tenant.slug} user=${context.user.id}`); + console.log(`[perf] duration=${DURATION_SECONDS}s concurrency=${CONCURRENCY} includeWrites=${INCLUDE_WRITES}`); + const report = await runBenchmark(context); + const files = await writeReport(report); + console.log(`[perf] requests=${report.summary.requests} ok=${report.summary.ok} errors=${report.summary.errors} rps=${report.summary.throughputRps} p95=${report.summary.latencyOk.p95Ms}ms`); + console.log(`[perf] wrote ${files.jsonPath}`); + console.log(`[perf] wrote ${files.mdPath}`); + if (report.summary.errors > 0) { + process.exitCode = 1; + } + } finally { + stopServer(); + } +} + +main().catch(error => { + console.error(error); + if (serverLogs) console.error(serverLogs); + stopServer(); + process.exitCode = 1; +}); diff --git a/scripts/import-pocketbase/src/import-json.ts b/scripts/import-pocketbase/src/import-json.ts index d89b7eab..3de70100 100644 --- a/scripts/import-pocketbase/src/import-json.ts +++ b/scripts/import-pocketbase/src/import-json.ts @@ -1031,6 +1031,1075 @@ async function normalizeQuestions(records: JsonRecord[]) { } } +async function buildPracticeNavigation(runId: string) { + const client = await pool.connect(); + try { + await client.query('begin'); + + await client.query( + ` + insert into public.content_entries ( + tenant_id, region_id, legacy_id, entry_key, name, entry_type, + icon, route, description, visibility, access_rules, layout_config, + sort_order, is_active + ) + select rm.tenant_id, rm.region_id, 'region_module:' || rm.legacy_id, + 'legacy-question-practice:' || rm.legacy_id, + rm.name, 'question_practice', + rm.icon, coalesce(rm.route, '/subjects'), rm.description, + 'public', '{}'::jsonb, + jsonb_build_object( + 'source', 'pocketbase.region_modules', + 'legacyModuleId', rm.legacy_id, + 'legacyType', rm.type, + 'route', rm.route + ), + rm.sort_order, rm.is_active + from public.region_modules rm + where rm.tenant_id = $1 + and rm.legacy_id is not null + and ( + rm.type = 'subjects' + or coalesce(rm.route, '') like '/subjects%' + or exists ( + select 1 from public.subjects s + where s.tenant_id = rm.tenant_id and s.module_id = rm.id + ) + ) + on conflict (tenant_id, entry_key) + do update set region_id = excluded.region_id, + legacy_id = excluded.legacy_id, + name = excluded.name, + entry_type = excluded.entry_type, + icon = excluded.icon, + route = excluded.route, + description = excluded.description, + visibility = excluded.visibility, + access_rules = excluded.access_rules, + layout_config = excluded.layout_config, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.content_entries ( + tenant_id, region_id, legacy_id, entry_key, name, entry_type, + icon, route, description, visibility, access_rules, layout_config, + sort_order, is_active + ) + select $1::uuid, null, 'migration_review:orphan_questions', + 'migration-review:orphan-questions', + '迁移待复核题库', 'question_practice', + null, '/subjects', '旧题库中科目或节点归属断裂的题目，需由租户后台重新归类后再发布。', + 'hidden', + jsonb_build_object('requiresReview', true), + jsonb_build_object('source', 'pocketbase.migration_review.orphan_questions'), + 999999, true + where exists ( + select 1 + from public.questions q + join public.subjects s on s.id = q.subject_id and s.tenant_id = q.tenant_id + where q.tenant_id = $1 + and q.legacy_id is not null + and q.status = 'published' + and q.node_id is null + and q.category_id is null + and q.legacy_category_id is null + and q.legacy_node_id is not null + and not exists ( + select 1 + from public.content_nodes subject_node + where subject_node.tenant_id = s.tenant_id + and subject_node.legacy_id = 'subject:' || s.legacy_id + ) + ) + on conflict (tenant_id, entry_key) + do update set legacy_id = excluded.legacy_id, + name = excluded.name, + entry_type = excluded.entry_type, + route = excluded.route, + description = excluded.description, + visibility = excluded.visibility, + access_rules = excluded.access_rules, + layout_config = excluded.layout_config, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.content_nodes ( + tenant_id, entry_id, region_id, parent_id, legacy_id, node_key, + name, node_type, marker_type, marker_config, path, depth, + sort_order, is_active, is_selectable, is_leaf, access_rules, metadata + ) + select mn.tenant_id, ce.id, mn.region_id, null, + 'module_node:' || mn.legacy_id, 'module_node:' || mn.legacy_id, + mn.name, + case + when mn.type in ('category','subject','chapter','paper','school','major','custom') then mn.type + else 'custom' + end, + case + when mn.type = 'school' then 'school' + when mn.type = 'major' then 'major' + when mn.type = 'subject' then 'subject' + else null + end, + jsonb_build_object( + 'source', 'pocketbase.module_nodes', + 'legacyModuleNodeId', mn.legacy_id, + 'legacyModuleId', rm.legacy_id, + 'legacyType', mn.type + ), + null, 0, mn.sort_order, mn.is_active, true, true, '{}'::jsonb, + jsonb_build_object('source', 'pocketbase.module_nodes') + from public.module_nodes mn + join public.region_modules rm on rm.id = mn.module_id and rm.tenant_id = mn.tenant_id + join public.content_entries ce + on ce.tenant_id = rm.tenant_id + and ce.legacy_id = 'region_module:' || rm.legacy_id + where mn.tenant_id = $1 + and mn.legacy_id is not null + on conflict (tenant_id, legacy_id) + do update set entry_id = excluded.entry_id, + region_id = excluded.region_id, + name = excluded.name, + node_type = excluded.node_type, + marker_type = excluded.marker_type, + marker_config = excluded.marker_config, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + is_selectable = excluded.is_selectable, + access_rules = excluded.access_rules, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + update public.content_nodes child + set parent_id = parent.id, + updated_at = now() + from public.module_nodes mn + join public.content_nodes parent + on parent.tenant_id = mn.tenant_id + and parent.legacy_id = 'module_node:' || mn.legacy_parent_id + where child.tenant_id = $1 + and mn.tenant_id = child.tenant_id + and child.legacy_id = 'module_node:' || mn.legacy_id + and mn.legacy_parent_id is not null + `, + [tenantId], + ); + + await client.query( + ` + insert into public.content_nodes ( + tenant_id, entry_id, region_id, parent_id, legacy_id, node_key, + name, node_type, marker_type, marker_config, path, depth, + sort_order, is_active, is_selectable, is_leaf, access_rules, metadata + ) + select s.tenant_id, ce.id, s.region_id, parent_node.id, + 'subject:' || s.legacy_id, 'subject:' || s.legacy_id, + s.name, 'subject', 'subject', + jsonb_build_object( + 'source', 'pocketbase.subjects', + 'subjectId', s.id, + 'legacySubjectId', s.legacy_id, + 'subjectType', s.type + ), + null, 0, s.sort_order, s.is_active, true, false, '{}'::jsonb, + jsonb_build_object('source', 'pocketbase.subjects') + from public.subjects s + join public.region_modules rm on rm.id = s.module_id and rm.tenant_id = s.tenant_id + join public.content_entries ce + on ce.tenant_id = rm.tenant_id + and ce.legacy_id = 'region_module:' || rm.legacy_id + left join public.module_nodes mn on mn.id = s.node_id and mn.tenant_id = s.tenant_id + left join public.content_nodes parent_node + on parent_node.tenant_id = s.tenant_id + and parent_node.legacy_id = 'module_node:' || mn.legacy_id + where s.tenant_id = $1 + and s.legacy_id is not null + on conflict (tenant_id, legacy_id) + do update set entry_id = excluded.entry_id, + region_id = excluded.region_id, + parent_id = excluded.parent_id, + name = excluded.name, + node_type = excluded.node_type, + marker_type = excluded.marker_type, + marker_config = excluded.marker_config, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + is_selectable = excluded.is_selectable, + access_rules = excluded.access_rules, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.content_nodes ( + tenant_id, entry_id, region_id, parent_id, legacy_id, node_key, + name, node_type, marker_type, marker_config, path, depth, + sort_order, is_active, is_selectable, is_leaf, access_rules, metadata + ) + select c.tenant_id, subject_node.entry_id, s.region_id, subject_node.id, + 'category:' || c.legacy_id, 'category:' || c.legacy_id, + c.name, + case when c.category_type = 'paper' then 'paper' else 'chapter' end, + null, + jsonb_build_object( + 'source', 'pocketbase.categories', + 'categoryId', c.id, + 'legacyCategoryId', c.legacy_id, + 'categoryType', c.category_type, + 'subjectId', s.id, + 'legacySubjectId', s.legacy_id + ), + null, 0, c.sort_order, c.is_active, true, true, '{}'::jsonb, + jsonb_build_object('source', 'pocketbase.categories') + from public.categories c + join public.subjects s on s.id = c.subject_id and s.tenant_id = c.tenant_id + join public.content_nodes subject_node + on subject_node.tenant_id = s.tenant_id + and subject_node.legacy_id = 'subject:' || s.legacy_id + where c.tenant_id = $1 + and c.legacy_id is not null + on conflict (tenant_id, legacy_id) + do update set entry_id = excluded.entry_id, + region_id = excluded.region_id, + parent_id = excluded.parent_id, + name = excluded.name, + node_type = excluded.node_type, + marker_type = excluded.marker_type, + marker_config = excluded.marker_config, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + is_selectable = excluded.is_selectable, + access_rules = excluded.access_rules, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.content_nodes ( + tenant_id, entry_id, region_id, parent_id, legacy_id, node_key, + name, node_type, marker_type, marker_config, path, depth, + sort_order, is_active, is_selectable, is_leaf, access_rules, metadata + ) + select q.tenant_id, subject_node.entry_id, s.region_id, subject_node.id, + 'migration_review:unresolved_category:' || s.legacy_id, + 'migration_review:unresolved_category:' || s.legacy_id, + '迁移待复核题目', 'custom', 'custom', + jsonb_build_object( + 'source', 'pocketbase.migration_review.unresolved_category', + 'subjectId', s.id, + 'legacySubjectId', s.legacy_id, + 'questionCount', count(*), + 'reason', 'legacy_category_or_node_missing' + ), + null, 0, 999999, false, false, true, + jsonb_build_object('requiresReview', true), + jsonb_build_object( + 'source', 'pocketbase.migration_review.unresolved_category', + 'reviewRequired', true, + 'questionCount', count(*) + ) + from public.questions q + join public.subjects s on s.id = q.subject_id and s.tenant_id = q.tenant_id + join public.content_nodes subject_node + on subject_node.tenant_id = s.tenant_id + and subject_node.legacy_id = 'subject:' || s.legacy_id + where q.tenant_id = $1 + and q.category_id is null + and q.legacy_category_id is not null + group by q.tenant_id, subject_node.entry_id, s.region_id, subject_node.id, s.id, s.legacy_id + on conflict (tenant_id, legacy_id) + do update set entry_id = excluded.entry_id, + region_id = excluded.region_id, + parent_id = excluded.parent_id, + marker_config = excluded.marker_config, + access_rules = excluded.access_rules, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.content_nodes ( + tenant_id, entry_id, region_id, parent_id, legacy_id, node_key, + name, node_type, marker_type, marker_config, path, depth, + sort_order, is_active, is_selectable, is_leaf, access_rules, metadata + ) + select q.tenant_id, review_entry.id, s.region_id, null, + 'migration_review:orphan_subject:' || s.legacy_id, + 'migration_review:orphan_subject:' || s.legacy_id, + s.name || ' · 迁移待复核题目', + 'subject', 'subject', + jsonb_build_object( + 'source', 'pocketbase.migration_review.orphan_subject', + 'subjectId', s.id, + 'legacySubjectId', s.legacy_id, + 'questionCount', count(*), + 'reason', 'legacy_node_or_subject_module_missing' + ), + null, 0, s.sort_order, false, false, true, + jsonb_build_object('requiresReview', true), + jsonb_build_object( + 'source', 'pocketbase.migration_review.orphan_subject', + 'reviewRequired', true, + 'questionCount', count(*) + ) + from public.questions q + join public.subjects s on s.id = q.subject_id and s.tenant_id = q.tenant_id + join public.content_entries review_entry + on review_entry.tenant_id = q.tenant_id + and review_entry.legacy_id = 'migration_review:orphan_questions' + where q.tenant_id = $1 + and q.legacy_id is not null + and q.status = 'published' + and q.node_id is null + and q.category_id is null + and q.legacy_category_id is null + and q.legacy_node_id is not null + and s.legacy_id is not null + and not exists ( + select 1 + from public.content_nodes subject_node + where subject_node.tenant_id = s.tenant_id + and subject_node.legacy_id = 'subject:' || s.legacy_id + ) + group by q.tenant_id, review_entry.id, s.region_id, s.id, s.legacy_id, s.name, s.sort_order + on conflict (tenant_id, legacy_id) + do update set entry_id = excluded.entry_id, + region_id = excluded.region_id, + name = excluded.name, + node_type = excluded.node_type, + marker_type = excluded.marker_type, + marker_config = excluded.marker_config, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + is_selectable = excluded.is_selectable, + access_rules = excluded.access_rules, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + with recursive tree as ( + select id, parent_id, + ('n_' || replace(id::text, '-', ''))::ltree as path, + 0 as depth + from public.content_nodes + where tenant_id = $1 and parent_id is null + + union all + + select child.id, child.parent_id, + (tree.path::text || '.n_' || replace(child.id::text, '-', ''))::ltree as path, + tree.depth + 1 as depth + from public.content_nodes child + join tree on tree.id = child.parent_id + where child.tenant_id = $1 + ) + update public.content_nodes node + set path = tree.path, + depth = tree.depth, + updated_at = now() + from tree + where node.id = tree.id + and node.tenant_id = $1 + `, + [tenantId], + ); + + await client.query( + ` + update public.content_nodes node + set is_leaf = not exists ( + select 1 + from public.content_nodes child + where child.tenant_id = node.tenant_id + and child.parent_id = node.id + ), + updated_at = now() + where node.tenant_id = $1 + `, + [tenantId], + ); + + await client.query( + ` + insert into public.question_collections ( + tenant_id, region_id, entry_id, node_id, subject_id, legacy_id, + name, collection_type, source_type, filters, question_count, + status, sort_order, access_rules, metadata + ) + select q.tenant_id, s.region_id, review_node.entry_id, review_node.id, + s.id, + 'migration_review:orphan_subject:' || s.legacy_id || ':all', + s.name || ' · 迁移待复核题目', + 'manual', 'manual_questions', + jsonb_build_object( + 'subjectId', s.id, + 'legacySubjectId', s.legacy_id, + 'legacyNodeIds', to_jsonb(array_remove(array_agg(distinct q.legacy_node_id), null)), + 'requiresReview', true + ), + 0, + 'draft', 999999, + jsonb_build_object('requiresReview', true), + jsonb_build_object( + 'source', 'pocketbase.migration_review.orphan_subject', + 'reviewRequired', true, + 'reason', 'legacy_node_or_subject_module_missing' + ) + from public.questions q + join public.subjects s on s.id = q.subject_id and s.tenant_id = q.tenant_id + join public.content_nodes review_node + on review_node.tenant_id = s.tenant_id + and review_node.legacy_id = 'migration_review:orphan_subject:' || s.legacy_id + where q.tenant_id = $1 + and q.legacy_id is not null + and q.status = 'published' + and q.node_id is null + and q.category_id is null + and q.legacy_category_id is null + and q.legacy_node_id is not null + group by q.tenant_id, s.region_id, review_node.entry_id, review_node.id, s.id, s.legacy_id, s.name + on conflict (tenant_id, legacy_id) + do update set region_id = excluded.region_id, + entry_id = excluded.entry_id, + node_id = excluded.node_id, + subject_id = excluded.subject_id, + name = excluded.name, + collection_type = excluded.collection_type, + source_type = excluded.source_type, + filters = excluded.filters, + status = excluded.status, + sort_order = excluded.sort_order, + access_rules = excluded.access_rules, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.question_collections ( + tenant_id, region_id, entry_id, node_id, subject_id, legacy_id, + name, collection_type, source_type, filters, question_count, + status, sort_order, access_rules, metadata + ) + select s.tenant_id, s.region_id, subject_node.entry_id, subject_node.id, s.id, + 'subject:' || s.legacy_id || ':all', + s.name || ' · 全部题目', + 'dynamic', 'subject', + jsonb_build_object( + 'subjectId', s.id, + 'legacySubjectId', s.legacy_id, + 'includeMigrationReview', false + ), + 0, + case when s.is_active then 'active' else 'draft' end, + s.sort_order, '{}'::jsonb, + jsonb_build_object('source', 'pocketbase.subjects', 'scope', 'subject_all') + from public.subjects s + join public.content_nodes subject_node + on subject_node.tenant_id = s.tenant_id + and subject_node.legacy_id = 'subject:' || s.legacy_id + where s.tenant_id = $1 + and s.legacy_id is not null + on conflict (tenant_id, legacy_id) + do update set region_id = excluded.region_id, + entry_id = excluded.entry_id, + node_id = excluded.node_id, + subject_id = excluded.subject_id, + name = excluded.name, + collection_type = excluded.collection_type, + source_type = excluded.source_type, + filters = excluded.filters, + status = excluded.status, + sort_order = excluded.sort_order, + access_rules = excluded.access_rules, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.question_collections ( + tenant_id, region_id, entry_id, node_id, subject_id, category_id, + legacy_id, name, collection_type, source_type, filters, question_count, + status, sort_order, access_rules, metadata + ) + select c.tenant_id, s.region_id, category_node.entry_id, category_node.id, + s.id, c.id, + 'category:' || c.legacy_id || ':all', + c.name, + case when c.category_type = 'paper' then 'paper' else 'chapter' end, + 'category', + jsonb_build_object( + 'categoryId', c.id, + 'legacyCategoryId', c.legacy_id, + 'subjectId', s.id, + 'legacySubjectId', s.legacy_id + ), + 0, + case when c.is_active then 'active' else 'draft' end, + c.sort_order, '{}'::jsonb, + jsonb_build_object('source', 'pocketbase.categories', 'scope', 'category') + from public.categories c + join public.subjects s on s.id = c.subject_id and s.tenant_id = c.tenant_id + join public.content_nodes category_node + on category_node.tenant_id = c.tenant_id + and category_node.legacy_id = 'category:' || c.legacy_id + where c.tenant_id = $1 + and c.legacy_id is not null + on conflict (tenant_id, legacy_id) + do update set region_id = excluded.region_id, + entry_id = excluded.entry_id, + node_id = excluded.node_id, + subject_id = excluded.subject_id, + category_id = excluded.category_id, + name = excluded.name, + collection_type = excluded.collection_type, + source_type = excluded.source_type, + filters = excluded.filters, + status = excluded.status, + sort_order = excluded.sort_order, + access_rules = excluded.access_rules, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + with direct_node_stats as ( + select mn.tenant_id, cn.region_id, cn.entry_id, cn.id as content_node_id, + mn.id as module_node_id, mn.legacy_id, mn.name, mn.type, + mn.sort_order, mn.is_active, + case + when count(distinct q.subject_id) filter (where q.subject_id is not null) = 1 + then min(q.subject_id::text)::uuid + else null + end as subject_id, + count(q.id)::integer as question_count + from public.module_nodes mn + join public.content_nodes cn + on cn.tenant_id = mn.tenant_id + and cn.legacy_id = 'module_node:' || mn.legacy_id + join public.questions q + on q.tenant_id = mn.tenant_id + and q.node_id = mn.id + and q.status = 'published' + where mn.tenant_id = $1 + and mn.legacy_id is not null + group by mn.tenant_id, cn.region_id, cn.entry_id, cn.id, + mn.id, mn.legacy_id, mn.name, mn.type, mn.sort_order, mn.is_active + ) + insert into public.question_collections ( + tenant_id, region_id, entry_id, node_id, subject_id, legacy_id, + name, collection_type, source_type, filters, question_count, + status, sort_order, access_rules, metadata + ) + select tenant_id, region_id, entry_id, content_node_id, subject_id, + 'module_node:' || legacy_id || ':direct', + name, + case + when type = 'paper' then 'paper' + when type = 'chapter' then 'chapter' + else 'dynamic' + end, + 'filters', + jsonb_build_object( + 'contentNodeId', content_node_id, + 'legacyNodeId', legacy_id, + 'moduleNodeId', module_node_id, + 'directOnly', true + ), + question_count, + case when is_active then 'active' else 'draft' end, + sort_order, '{}'::jsonb, + jsonb_build_object( + 'source', 'pocketbase.module_nodes', + 'scope', 'node_direct', + 'legacyNodeId', legacy_id, + 'questionCount', question_count + ) + from direct_node_stats + on conflict (tenant_id, legacy_id) + do update set region_id = excluded.region_id, + entry_id = excluded.entry_id, + node_id = excluded.node_id, + subject_id = excluded.subject_id, + name = excluded.name, + collection_type = excluded.collection_type, + source_type = excluded.source_type, + filters = excluded.filters, + question_count = excluded.question_count, + status = excluded.status, + sort_order = excluded.sort_order, + access_rules = excluded.access_rules, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.question_collections ( + tenant_id, region_id, entry_id, node_id, subject_id, legacy_id, + name, collection_type, source_type, filters, question_count, + status, sort_order, access_rules, metadata + ) + select q.tenant_id, s.region_id, review_node.entry_id, review_node.id, + s.id, + 'migration_review:unresolved_category:' || s.legacy_id || ':all', + s.name || ' · 迁移待复核题目', + 'manual', 'manual_questions', + jsonb_build_object( + 'subjectId', s.id, + 'legacySubjectId', s.legacy_id, + 'requiresReview', true + ), + 0, + 'draft', 999999, + jsonb_build_object('requiresReview', true), + jsonb_build_object( + 'source', 'pocketbase.migration_review.unresolved_category', + 'reviewRequired', true, + 'reason', 'legacy_category_or_node_missing' + ) + from public.questions q + join public.subjects s on s.id = q.subject_id and s.tenant_id = q.tenant_id + join public.content_nodes review_node + on review_node.tenant_id = s.tenant_id + and review_node.legacy_id = 'migration_review:unresolved_category:' || s.legacy_id + where q.tenant_id = $1 + and q.category_id is null + and q.node_id is null + and q.legacy_category_id is not null + group by q.tenant_id, s.region_id, review_node.entry_id, review_node.id, s.id, s.legacy_id, s.name + on conflict (tenant_id, legacy_id) + do update set region_id = excluded.region_id, + entry_id = excluded.entry_id, + node_id = excluded.node_id, + subject_id = excluded.subject_id, + name = excluded.name, + collection_type = excluded.collection_type, + source_type = excluded.source_type, + filters = excluded.filters, + status = excluded.status, + sort_order = excluded.sort_order, + access_rules = excluded.access_rules, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + update public.questions q + set entry_id = qc.entry_id, + content_node_id = qc.node_id, + primary_collection_id = qc.id, + updated_at = now() + from public.subjects s + join public.question_collections qc + on qc.tenant_id = s.tenant_id + and qc.legacy_id = 'migration_review:orphan_subject:' || s.legacy_id || ':all' + where q.tenant_id = $1 + and s.tenant_id = q.tenant_id + and s.id = q.subject_id + and q.primary_collection_id is null + and q.legacy_id is not null + and q.status = 'published' + and q.node_id is null + and q.category_id is null + and q.legacy_category_id is null + and q.legacy_node_id is not null + `, + [tenantId], + ); + + await client.query( + ` + update public.questions q + set entry_id = qc.entry_id, + content_node_id = qc.node_id, + primary_collection_id = qc.id, + updated_at = now() + from public.module_nodes mn + join public.content_nodes cn + on cn.tenant_id = mn.tenant_id + and cn.legacy_id = 'module_node:' || mn.legacy_id + join public.question_collections qc + on qc.tenant_id = mn.tenant_id + and qc.node_id = cn.id + and qc.legacy_id = 'module_node:' || mn.legacy_id || ':direct' + where q.tenant_id = $1 + and mn.tenant_id = q.tenant_id + and q.node_id = mn.id + and q.status = 'published' + `, + [tenantId], + ); + + await client.query( + ` + update public.questions q + set entry_id = qc.entry_id, + content_node_id = qc.node_id, + primary_collection_id = qc.id, + updated_at = now() + from public.question_collections qc + where q.tenant_id = $1 + and qc.tenant_id = q.tenant_id + and qc.category_id = q.category_id + and q.category_id is not null + and qc.metadata->>'source' = 'pocketbase.categories' + and (q.primary_collection_id is null or q.node_id is null) + `, + [tenantId], + ); + + await client.query( + ` + update public.questions q + set entry_id = qc.entry_id, + content_node_id = qc.node_id, + primary_collection_id = qc.id, + updated_at = now() + from public.question_collections qc + where q.tenant_id = $1 + and qc.tenant_id = q.tenant_id + and qc.subject_id = q.subject_id + and q.category_id is null + and q.node_id is null + and q.legacy_category_id is not null + and qc.metadata->>'source' = 'pocketbase.migration_review.unresolved_category' + and q.primary_collection_id is null + `, + [tenantId], + ); + + await client.query( + ` + update public.questions q + set entry_id = qc.entry_id, + content_node_id = qc.node_id, + primary_collection_id = qc.id, + updated_at = now() + from public.question_collections qc + where q.tenant_id = $1 + and qc.tenant_id = q.tenant_id + and qc.subject_id = q.subject_id + and q.primary_collection_id is null + and qc.metadata->>'source' = 'pocketbase.subjects' + and qc.metadata->>'scope' = 'subject_all' + `, + [tenantId], + ); + + await client.query( + ` + insert into public.question_collection_items ( + tenant_id, collection_id, question_id, sort_order, metadata + ) + select q.tenant_id, qc.id, q.id, + row_number() over (partition by qc.id order by q.created_at asc, q.id asc)::integer - 1, + jsonb_build_object( + 'source', 'pocketbase.navigation.orphan_subject_review', + 'legacyNodeId', q.legacy_node_id + ) + from public.questions q + join public.subjects s on s.id = q.subject_id and s.tenant_id = q.tenant_id + join public.question_collections qc + on qc.tenant_id = q.tenant_id + and qc.legacy_id = 'migration_review:orphan_subject:' || s.legacy_id || ':all' + and qc.metadata->>'source' = 'pocketbase.migration_review.orphan_subject' + where q.tenant_id = $1 + and q.node_id is null + and q.category_id is null + and q.legacy_category_id is null + and q.legacy_node_id is not null + and q.status = 'published' + on conflict (tenant_id, collection_id, question_id) + do update set sort_order = excluded.sort_order, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.question_collection_items ( + tenant_id, collection_id, question_id, sort_order, metadata + ) + select q.tenant_id, qc.id, q.id, + row_number() over (partition by qc.id order by q.created_at asc, q.id asc)::integer - 1, + jsonb_build_object( + 'source', 'pocketbase.navigation.module_node', + 'legacyNodeId', mn.legacy_id + ) + from public.questions q + join public.module_nodes mn + on mn.tenant_id = q.tenant_id + and mn.id = q.node_id + join public.question_collections qc + on qc.tenant_id = q.tenant_id + and qc.legacy_id = 'module_node:' || mn.legacy_id || ':direct' + and qc.metadata->>'source' = 'pocketbase.module_nodes' + where q.tenant_id = $1 + and q.status = 'published' + on conflict (tenant_id, collection_id, question_id) + do update set sort_order = excluded.sort_order, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.question_collection_items ( + tenant_id, collection_id, question_id, sort_order, metadata + ) + select q.tenant_id, qc.id, q.id, + row_number() over (partition by qc.id order by q.created_at asc, q.id asc)::integer - 1, + jsonb_build_object('source', 'pocketbase.navigation.category') + from public.questions q + join public.question_collections qc + on qc.tenant_id = q.tenant_id + and qc.category_id = q.category_id + and qc.metadata->>'source' = 'pocketbase.categories' + where q.tenant_id = $1 + and q.category_id is not null + and q.status = 'published' + on conflict (tenant_id, collection_id, question_id) + do update set sort_order = excluded.sort_order, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.question_collection_items ( + tenant_id, collection_id, question_id, sort_order, metadata + ) + select q.tenant_id, qc.id, q.id, + row_number() over (partition by qc.id order by q.created_at asc, q.id asc)::integer - 1, + jsonb_build_object('source', 'pocketbase.navigation.subject_all') + from public.questions q + join public.question_collections qc + on qc.tenant_id = q.tenant_id + and qc.subject_id = q.subject_id + and qc.metadata->>'source' = 'pocketbase.subjects' + and qc.metadata->>'scope' = 'subject_all' + where q.tenant_id = $1 + and q.status = 'published' + and (q.category_id is not null or q.legacy_category_id is null or q.node_id is not null) + on conflict (tenant_id, collection_id, question_id) + do update set sort_order = excluded.sort_order, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.question_collection_items ( + tenant_id, collection_id, question_id, sort_order, metadata + ) + select q.tenant_id, qc.id, q.id, + row_number() over (partition by qc.id order by q.created_at asc, q.id asc)::integer - 1, + jsonb_build_object( + 'source', 'pocketbase.navigation.migration_review', + 'legacyCategoryId', q.legacy_category_id + ) + from public.questions q + join public.question_collections qc + on qc.tenant_id = q.tenant_id + and qc.subject_id = q.subject_id + and qc.metadata->>'source' = 'pocketbase.migration_review.unresolved_category' + where q.tenant_id = $1 + and q.category_id is null + and q.node_id is null + and q.legacy_category_id is not null + and q.status = 'published' + on conflict (tenant_id, collection_id, question_id) + do update set sort_order = excluded.sort_order, + metadata = excluded.metadata, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + update public.question_collections qc + set question_count = coalesce(counts.count, 0), + updated_at = now() + from ( + select collection_id, count(*)::integer as count + from public.question_collection_items + where tenant_id = $1 + group by collection_id + ) counts + where qc.tenant_id = $1 + and qc.id = counts.collection_id + `, + [tenantId], + ); + + await client.query( + ` + update public.question_collections qc + set question_count = 0, + updated_at = now() + where qc.tenant_id = $1 + and not exists ( + select 1 + from public.question_collection_items item + where item.tenant_id = qc.tenant_id + and item.collection_id = qc.id + ) + `, + [tenantId], + ); + + await client.query( + ` + insert into public.practice_blueprints ( + tenant_id, region_id, entry_id, node_id, collection_id, legacy_id, + name, mode, assembly_type, question_limit, rules, status, + sort_order, access_rules + ) + select qc.tenant_id, qc.region_id, qc.entry_id, qc.node_id, qc.id, + qc.legacy_id || ':sequential', + qc.name || ' · 顺序刷题', + 'sequential', 'collection', null, + jsonb_build_object('source', 'pocketbase.navigation', 'collectionLegacyId', qc.legacy_id), + qc.status, qc.sort_order, qc.access_rules + from public.question_collections qc + where qc.tenant_id = $1 + and qc.question_count > 0 + and qc.legacy_id is not null + on conflict (tenant_id, legacy_id) + do update set region_id = excluded.region_id, + entry_id = excluded.entry_id, + node_id = excluded.node_id, + collection_id = excluded.collection_id, + name = excluded.name, + mode = excluded.mode, + assembly_type = excluded.assembly_type, + question_limit = excluded.question_limit, + rules = excluded.rules, + status = excluded.status, + sort_order = excluded.sort_order, + access_rules = excluded.access_rules, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.practice_blueprints ( + tenant_id, region_id, entry_id, node_id, collection_id, legacy_id, + name, mode, assembly_type, question_limit, rules, status, + sort_order, access_rules + ) + select qc.tenant_id, qc.region_id, qc.entry_id, qc.node_id, qc.id, + qc.legacy_id || ':random', + qc.name || ' · 随机刷题', + 'random', 'collection', least(20, qc.question_count), + jsonb_build_object('source', 'pocketbase.navigation', 'collectionLegacyId', qc.legacy_id), + qc.status, qc.sort_order + 1, qc.access_rules + from public.question_collections qc + where qc.tenant_id = $1 + and qc.question_count > 0 + and qc.legacy_id is not null + on conflict (tenant_id, legacy_id) + do update set region_id = excluded.region_id, + entry_id = excluded.entry_id, + node_id = excluded.node_id, + collection_id = excluded.collection_id, + name = excluded.name, + mode = excluded.mode, + assembly_type = excluded.assembly_type, + question_limit = excluded.question_limit, + rules = excluded.rules, + status = excluded.status, + sort_order = excluded.sort_order, + access_rules = excluded.access_rules, + updated_at = now() + `, + [tenantId], + ); + + await client.query( + ` + insert into public.pb_import_issues ( + run_id, tenant_id, collection_name, legacy_id, severity, issue_code, message + ) + values ($1,$2,$3,$4,$5,$6,$7) + `, + [ + runId, + tenantId, + 'questions', + '__migration_navigation__', + 'info', + 'practice_navigation_built', + 'Legacy question practice entries, nodes, collections and blueprints were generated for the new SaaS navigation model.', + ], + ); + + await client.query('commit'); + } catch (error) { + await client.query('rollback'); + throw error; + } finally { + client.release(); + } + + console.log('practice_navigation: generated content entries, nodes, collections and blueprints'); +} + async function normalizeUserAnswerRecords(runId: string, records: JsonRecord[]) { const userIdsByLegacy = await bulkUserIdsByLegacy(records.map(record => record.userId)); const questionIdsByLegacy = await bulkLegacyIds('questions', records.map(record => record.questionId)); @@ -3503,6 +4572,7 @@ async function normalizeAll(runId: string, collections: CollectionMap) { await runNormalizer(runId, collections, 'question_type_groups', normalizeQuestionTypeGroups); await runNormalizer(runId, collections, 'subject_shares', normalizeSubjectShares); await runNormalizer(runId, collections, 'questions', normalizeQuestions); + if ((collections.questions || []).length > 0) await buildPracticeNavigation(runId); await runNormalizer(runId, collections, 'users', normalizeUsers); await normalizeUserEntitlementsAndStats(runId, collections.users || []); diff --git a/scripts/import-pocketbase/src/validate-import.ts b/scripts/import-pocketbase/src/validate-import.ts index a209efba..df686a12 100644 --- a/scripts/import-pocketbase/src/validate-import.ts +++ b/scripts/import-pocketbase/src/validate-import.ts @@ -274,6 +274,232 @@ async function validationChecks(): Promise { ), ); + checks.push( + result( + 'legacy_practice_navigation_missing_entries', + await scalar( + ` + select case + when exists ( + select 1 from public.questions + where tenant_id = $1 and legacy_id is not null + ) + and not exists ( + select 1 from public.content_entries + where tenant_id = $1 + and entry_type = 'question_practice' + and legacy_id is not null + ) + then 1 else 0 end + `, + [tenantId], + ), + 'Imported legacy questions exist but no question_practice content entries were generated.', + 'Legacy question practice content entries are present when imported questions exist.', + ), + ); + + checks.push( + result( + 'legacy_practice_navigation_missing_nodes', + await scalar( + ` + select case + when exists ( + select 1 from public.questions + where tenant_id = $1 and legacy_id is not null + ) + and not exists ( + select 1 from public.content_nodes + where tenant_id = $1 + and legacy_id is not null + and metadata->>'source' in ( + 'pocketbase.module_nodes', + 'pocketbase.subjects', + 'pocketbase.categories', + 'pocketbase.migration_review.unresolved_category' + ) + ) + then 1 else 0 end + `, + [tenantId], + ), + 'Imported legacy questions exist but no content nodes were generated.', + 'Legacy content nodes are present when imported questions exist.', + ), + ); + + checks.push( + result( + 'legacy_practice_collections_missing_items', + await scalar( + ` + select count(*) + from public.question_collections qc + where qc.tenant_id = $1 + and qc.legacy_id is not null + and qc.status = 'active' + and qc.question_count = 0 + and exists ( + select 1 + from public.questions q + left join public.content_nodes cn + on cn.tenant_id = q.tenant_id + and cn.id = q.content_node_id + where q.tenant_id = qc.tenant_id + and q.status = 'published' + and ( + q.content_node_id = qc.node_id + or q.category_id = qc.category_id + or (qc.category_id is null and q.subject_id = qc.subject_id) + ) + ) + `, + [tenantId], + ), + 'Active legacy question collections exist without collection items even though matching questions exist.', + 'Active legacy question collections have item counts when matching questions exist.', + ), + ); + + checks.push( + result( + 'legacy_questions_without_navigation', + await scalar( + ` + select count(*) + from public.questions + where tenant_id = $1 + and legacy_id is not null + and status = 'published' + and ( + entry_id is null + or content_node_id is null + or primary_collection_id is null + ) + `, + [tenantId], + ), + 'Some imported published questions are not attached to entry/node/collection navigation.', + 'Imported published questions are attached to entry/node/collection navigation.', + ), + ); + + checks.push( + result( + 'legacy_unresolved_categories_not_isolated', + await scalar( + ` + select count(*) + from public.questions q + left join public.question_collections qc + on qc.tenant_id = q.tenant_id + and qc.id = q.primary_collection_id + left join public.content_nodes cn + on cn.tenant_id = q.tenant_id + and cn.id = q.content_node_id + where q.tenant_id = $1 + and q.legacy_id is not null + and q.legacy_category_id is not null + and q.category_id is null + and q.node_id is null + and ( + qc.id is null + or qc.status <> 'draft' + or qc.access_rules->>'requiresReview' <> 'true' + or cn.id is null + or cn.is_active <> false + or cn.access_rules->>'requiresReview' <> 'true' + ) + `, + [tenantId], + ), + 'Questions with unresolved legacy categories are not isolated into draft review collections/inactive nodes.', + 'Questions with unresolved legacy categories are isolated for tenant-admin review.', + ), + ); + + checks.push( + result( + 'legacy_orphan_node_questions_not_isolated', + await scalar( + ` + select count(*) + from public.questions q + left join public.question_collections qc + on qc.tenant_id = q.tenant_id + and qc.id = q.primary_collection_id + left join public.content_nodes cn + on cn.tenant_id = q.tenant_id + and cn.id = q.content_node_id + left join public.content_entries ce + on ce.tenant_id = q.tenant_id + and ce.id = q.entry_id + where q.tenant_id = $1 + and q.legacy_id is not null + and q.legacy_node_id is not null + and q.node_id is null + and q.category_id is null + and q.legacy_category_id is null + and not ( + qc.status = 'draft' + and qc.access_rules->>'requiresReview' = 'true' + and qc.metadata->>'source' = 'pocketbase.migration_review.orphan_subject' + and cn.is_active = false + and cn.access_rules->>'requiresReview' = 'true' + and ce.visibility = 'hidden' + and ce.access_rules->>'requiresReview' = 'true' + ) + and not ( + qc.status = 'active' + and qc.metadata->>'source' = 'pocketbase.subjects' + and qc.metadata->>'scope' = 'subject_all' + and cn.node_type = 'subject' + and cn.marker_type = 'subject' + and cn.is_active = true + and ce.visibility <> 'hidden' + ) + `, + [tenantId], + ), + 'Questions with unresolved legacy nodeId are neither isolated into hidden review navigation nor attached to a public subject fallback.', + 'Questions with unresolved legacy nodeId are isolated or attached to a public subject fallback.', + ), + ); + + checks.push( + result( + 'legacy_node_questions_wrong_navigation', + await scalar( + ` + select count(*) + from public.questions q + join public.module_nodes mn + on mn.tenant_id = q.tenant_id + and mn.id = q.node_id + left join public.content_nodes cn + on cn.tenant_id = q.tenant_id + and cn.legacy_id = 'module_node:' || mn.legacy_id + left join public.question_collections qc + on qc.tenant_id = q.tenant_id + and qc.legacy_id = 'module_node:' || mn.legacy_id || ':direct' + where q.tenant_id = $1 + and q.legacy_id is not null + and q.node_id is not null + and ( + cn.id is null + or q.content_node_id is distinct from cn.id + or qc.id is null + or q.primary_collection_id is distinct from qc.id + ) + `, + [tenantId], + ), + 'Some imported questions with legacy nodeId are not attached to the matching content node/direct collection.', + 'Imported questions with legacy nodeId are attached to matching content nodes and direct collections.', + ), + ); + checks.push( result( 'legacy_mock_blueprints_incomplete',