diff --git a/README.md b/README.md index b267632f..cd593159 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ - Supabase/PostgreSQL 多租户数据库 schema、RLS、索引、触发器。 - `apps/api` 独立业务 API,后续供 H5、Taro 小程序、管理后台统一调用。 - 租户后台能力:品牌、域名、公开设置、支付账户、登录配置、私密密钥掩码、活动内容、激活码、优惠券、成员权限、审计日志。 -- 租户内容能力:可配置题库入口、任意深度分类树、考试意向标记、题目集合、顺序/随机/全真模拟蓝图、题目录入/更新、视频绑定、分数线、单词、知识手册、资料资源台账、题目 JSON 批量导入。 +- 租户内容能力:可配置题库入口、任意深度分类树、考试意向标记、题目集合、顺序/随机/全真模拟蓝图、题目录入/更新、视频绑定、分数线、单词、知识手册、资料资源台账、题目/单词/知识手册 JSON 批量导入。 - 学生端能力:题库入口、分类树、题目集合、顺序/随机/模考 session 组卷快照、答题、错题本、收藏夹、背单词进度、个人中心、分数线、题目视频、订单、权益、激活码兑换、资料下载。 - 平台后台能力:租户管理、SaaS 套餐、订阅、账单、服务费收款、用量记录。 - 销售/代理/CRM 增长链路:邀请码、扫码/分享事件、首绑客资保护、销售统计、团队关系、CRM 配置和队列。 @@ -25,13 +25,15 @@ - 正式 Supabase Auth/JWT 鉴权还没替换迁移期请求头。 - 真实短信、微信登录、QQ 登录、微信支付、支付宝等 provider adapter 还没接完。 - 真实 OSS/COS/Supabase Storage 上传下载签名还没接完。 -- Excel/CSV 导入、单词/手册/分数线/视频批量导入和异步 worker 还没完成。 +- Excel/CSV 导入、分数线/视频批量导入和异步 worker 还没完成。 - Taro 跨端前端还没开始 scaffold。 更完整的进度看这些文档: - `docs/refactor/implementation-status.md` - `docs/refactor/backend-progress.md` +- `docs/refactor/content-import-contract.md` +- `docs/refactor/next-development-todo.md` - `docs/refactor/blueprint-coverage.md` - `docs/refactor/api-structure.md` @@ -142,7 +144,7 @@ apps/api/src/features/ - 商户密钥、短信密钥、OAuth app secret 等必须进入 `app_private.tenant_secrets`,或后续生产 KMS/Vault。 - 资料、PDF、视频等资源必须先进入 `content_assets` 台账,再由 API 校验权限并下发签名 URL。 - 题库入口和分类使用 `content_entries/content_nodes`;题目列表和练习规则使用 `question_collections/practice_blueprints`,前端不要再把旧树字段当成唯一业务结构。 -- 批量导入必须先写 `content_import_jobs/items/issues`,保留原始 payload、规范化 payload、逐行问题和审计记录。 +- 批量导入必须先写 `content_import_jobs/items/issues`,保留原始 payload、规范化 payload、逐行问题和审计记录。题目、单词、知识手册导入已走这套后台校验管线,前端只做预检查和预览展示。 - 支付 webhook 必须先设计幂等键和验签流程,再进入生产使用。 ## 最近一次验证 @@ -162,7 +164,7 @@ npm run test:api 优先继续补: 1. 真实对象存储 adapter:阿里云 OSS、腾讯云 COS 或 Supabase Storage。 -2. Excel/CSV 以及单词、知识手册、分数线、视频批量导入。 +2. Excel/CSV 以及分数线、视频批量导入;把现有 JSON 导入升级为可排队异步执行。 3. Supabase Auth/JWT 正式鉴权和生产 RLS 验证。 4. 微信/QQ 登录、短信、微信支付、支付宝支付 adapter。 5. Taro 前端 scaffold,让 H5 和小程序共用同一套 API。 diff --git a/apps/api/src/features/catalog/routes.ts b/apps/api/src/features/catalog/routes.ts index ba8f314b..c7ddd689 100644 --- a/apps/api/src/features/catalog/routes.ts +++ b/apps/api/src/features/catalog/routes.ts @@ -321,6 +321,7 @@ export async function vocabularyUnitsRoute(ctx: RequestContext) { const items = await query( ` select id, legacy_id as "legacyId", region_id as "regionId", + entry_id as "entryId", content_node_id as "contentNodeId", name, description, word_count as "wordCount", sort_order as "order", is_active as "isActive", created_at as "createdAt", updated_at as "updatedAt" @@ -349,6 +350,7 @@ export async function vocabularyWordsRoute(ctx: RequestContext) { const items = await query( ` select id, legacy_id as "legacyId", unit_id as "unitId", + entry_id as "entryId", content_node_id as "contentNodeId", word, phonetic, meaning, example, example_translation as "exampleTranslation", difficulty, tags, sort_order as "order", @@ -378,6 +380,7 @@ export async function handbookSubjectsRoute(ctx: RequestContext) { const items = await query( ` select id, legacy_id as "legacyId", region_id as "regionId", + entry_id as "entryId", content_node_id as "contentNodeId", name, type, icon, color, description, sort_order as "order", is_active as "isActive", metadata, created_at as "createdAt", updated_at as "updatedAt" @@ -404,6 +407,7 @@ export async function handbookChaptersRoute(ctx: RequestContext) { const items = await query( ` select id, legacy_id as "legacyId", subject_id as "subjectId", + entry_id as "entryId", content_node_id as "contentNodeId", name, description, sort_order as "order", is_active as "isActive", created_at as "createdAt", updated_at as "updatedAt" @@ -431,6 +435,7 @@ export async function handbookEntriesRoute(ctx: RequestContext) { const items = await query( ` select id, legacy_id as "legacyId", chapter_id as "chapterId", + entry_id as "entryId", content_node_id as "contentNodeId", title, summary, ${includeContent ? 'content' : 'null::text as content'}, tags, sort_order as "order", is_active as "isActive", created_at as "createdAt", updated_at as "updatedAt" diff --git a/apps/api/src/features/tenant-content/imports.ts b/apps/api/src/features/tenant-content/imports.ts index 01440b22..550a469d 100644 --- a/apps/api/src/features/tenant-content/imports.ts +++ b/apps/api/src/features/tenant-content/imports.ts @@ -1,10 +1,10 @@ -import { createHash } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import type pg from 'pg'; import { HttpError, type RequestContext } from '../../core/http.js'; import { intParam, readJsonBody, requiredString, stringParam } from '../../core/request.js'; import { query, queryOne, transaction } from '../../core/db.js'; import { requireTenantContentEditor, type TenantContentAuth } from './auth.js'; -import { boolValue, intValue, nullableString } from './utils.js'; +import { boolValue, nullableString } from './utils.js'; type JsonObject = Record; @@ -37,7 +37,82 @@ interface NormalizedQuestion { sourceHash: string; } -interface PreviewResult { +interface NormalizedVocabularyWord { + legacyId: string | null; + word: string; + phonetic: string | null; + meaning: string; + example: string | null; + exampleTranslation: string | null; + difficulty: number | null; + tags: string[]; + order: number; + isActive: boolean; + sourceHash: string; + metadata: JsonObject; +} + +interface NormalizedVocabularyUnit { + legacyId: string | null; + name: string; + description: string | null; + order: number; + wordCount: number | null; + isActive: boolean; + words: NormalizedVocabularyWord[]; + sourceHash: string; + metadata: JsonObject; +} + +interface NormalizedHandbookEntry { + legacyId: string | null; + title: string; + summary: string | null; + content: string; + tags: string[]; + order: number; + isActive: boolean; + sourceHash: string; + metadata: JsonObject; +} + +interface NormalizedHandbookSection { + legacyId: string | null; + name: string | null; + description: string | null; + order: number; + isActive: boolean; + entries: NormalizedHandbookEntry[]; + sourceHash: string; + metadata: JsonObject; +} + +interface NormalizedHandbookChapter { + legacyId: string | null; + name: string; + description: string | null; + order: number; + isActive: boolean; + sections: NormalizedHandbookSection[]; + sourceHash: string; + metadata: JsonObject; +} + +interface NormalizedHandbookSubject { + legacyId: string | null; + name: string; + type: string | null; + icon: string | null; + color: string | null; + description: string | null; + order: number; + isActive: boolean; + chapters: NormalizedHandbookChapter[]; + sourceHash: string; + metadata: JsonObject; +} + +interface PreviewResult { job: { id: string; status: string; @@ -50,7 +125,7 @@ interface PreviewResult { rowNo: number; status: 'valid' | 'invalid'; externalId: string | null; - normalized: NormalizedQuestion | null; + normalized: T | null; issues: ImportIssue[]; }>; issues: ImportIssue[]; @@ -120,10 +195,65 @@ function parseQuestionItems(body: JsonObject) { return parsed; } +function parseJsonMaybe(value: unknown, code: string) { + if (typeof value !== 'string') return value; + try { + return JSON.parse(value); + } catch { + throw new HttpError(400, 'payload must be valid JSON', code); + } +} + +function asArray(value: unknown) { + return Array.isArray(value) ? value : []; +} + function contentHash(value: unknown) { return createHash('sha256').update(JSON.stringify(value)).digest('hex'); } +function boolishValue(value: unknown, fallback: boolean) { + if (typeof value === 'boolean') return value; + if (typeof value === 'string') { + const normalized = value.trim().toLowerCase(); + if (['true', '1', 'yes', 'y'].includes(normalized)) return true; + if (['false', '0', 'no', 'n'].includes(normalized)) return false; + } + return fallback; +} + +function integerValue(value: unknown, fallback: number) { + const numberValue = Number(value ?? fallback); + return Number.isFinite(numberValue) ? Math.trunc(numberValue) : fallback; +} + +function nullableDifficulty(value: unknown, issues: ImportIssue[], rowNo: number, fieldPath: string) { + if (value === undefined || value === null || value === '') return null; + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + issues.push({ + rowNo, + severity: 'error', + code: 'INVALID_DIFFICULTY', + fieldPath, + message: 'difficulty must be a number from 1 to 5', + }); + return null; + } + const difficulty = Math.trunc(parsed); + if (difficulty < 1 || difficulty > 5) { + issues.push({ + rowNo, + severity: 'warning', + code: 'DIFFICULTY_OUT_OF_RANGE', + fieldPath, + message: 'difficulty is outside the recommended 1-5 range and was clamped', + details: { original: value }, + }); + } + return Math.min(5, Math.max(1, difficulty)); +} + function normalizeDifficulty(value: unknown, issues: ImportIssue[], rowNo: number) { if (value === undefined || value === null || value === '') return 1; const parsed = Number(value); @@ -597,6 +727,646 @@ async function createQuestionPreviewJob(auth: TenantContentAuth, body: JsonObjec }); } +interface GenericImportTarget { + regionId: string | null; + entryId: string | null; + contentNodeId: string | null; +} + +interface NormalizedImportItem { + rowNo: number; + status: 'valid' | 'invalid'; + externalId: string | null; + source: unknown; + normalized: T | null; + issues: ImportIssue[]; +} + +function parseVocabularyItems(body: JsonObject) { + const source = parseJsonMaybe( + body.items ?? body.units ?? body.vocabularyUnits ?? body.vocabulary_units ?? body.payload ?? body, + 'INVALID_VOCABULARY_IMPORT_PAYLOAD', + ); + const payload = objectValue(source); + + let units = Array.isArray(source) ? source : asArray( + payload.units ?? + payload.vocabularyUnits ?? + payload.vocabulary_units ?? + payload['vocabulary_units_示例数据'], + ); + const flatWords = asArray( + payload.words ?? + payload.vocabularyWords ?? + payload.vocabulary_words ?? + payload.vocabulary ?? + payload['vocabulary_示例数据'], + ); + + if (units.length === 0 && flatWords.length > 0) { + units = [{ + legacyId: stringValue(payload.unitLegacyId ?? payload.unitId ?? payload.legacyId) || null, + name: stringValue(payload.unitName ?? payload.name) || '默认单词单元', + description: stringValue(payload.description) || null, + words: flatWords, + }]; + } else if (flatWords.length > 0) { + const unitBuckets = new Map(); + const unitLookup = new Map(); + units.forEach((unit, index) => { + const raw = objectValue(unit); + for (const key of [ + raw.id, + raw.legacyId, + raw.legacy_id, + raw.externalId, + raw.external_id, + raw.name, + ]) { + const value = stringValue(key); + if (value) unitLookup.set(value, index); + } + }); + + flatWords.forEach(word => { + const raw = objectValue(word); + const ref = stringValue(raw.unitId ?? raw.unit_id ?? raw.unitLegacyId ?? raw.unit_legacy_id ?? raw.unitName ?? raw.unit); + const index = unitLookup.get(ref); + if (index !== undefined) { + const bucket = unitBuckets.get(index) || []; + bucket.push(word); + unitBuckets.set(index, bucket); + } + }); + + units = units.map((unit, index) => ({ + ...objectValue(unit), + words: [...asArray(objectValue(unit).words), ...(unitBuckets.get(index) || [])], + })); + } + + if (units.length === 0) { + throw new HttpError(400, 'Vocabulary import payload must contain at least one unit', 'EMPTY_IMPORT_PAYLOAD'); + } + if (units.length > 500) { + throw new HttpError(400, 'A single vocabulary import job can contain at most 500 units', 'IMPORT_TOO_LARGE'); + } + return { units, flatWords }; +} + +function normalizeVocabularyWord(raw: unknown, rowNo: number, index: number, issues: ImportIssue[]) { + const source = objectValue(raw); + const fieldPrefix = `words[${index}]`; + const word = stringValue(source.word ?? source.name); + const meaning = stringValue(source.meaning ?? source.translation ?? source.definition); + + if (!word) { + issues.push({ + rowNo, + severity: 'error', + code: 'VOCABULARY_WORD_REQUIRED', + fieldPath: `${fieldPrefix}.word`, + message: 'vocabulary word is required', + }); + } + if (!meaning) { + issues.push({ + rowNo, + severity: 'error', + code: 'VOCABULARY_MEANING_REQUIRED', + fieldPath: `${fieldPrefix}.meaning`, + message: 'vocabulary meaning is required', + }); + } + + const normalizedWithoutHash = { + legacyId: stringValue(source.legacyId ?? source.legacy_id ?? source.externalId ?? source.external_id ?? source.id) || null, + word, + phonetic: stringValue(source.phonetic ?? source.pronunciation) || null, + meaning, + example: stringValue(source.example) || null, + exampleTranslation: stringValue(source.exampleTranslation ?? source.example_translation) || null, + difficulty: nullableDifficulty(source.difficulty, issues, rowNo, `${fieldPrefix}.difficulty`), + tags: stringArrayValue(source.tags), + order: integerValue(source.order ?? source.sortOrder ?? source.sort_order, index + 1), + isActive: boolishValue(source.isActive ?? source.is_active, true), + metadata: objectValue(source.metadata), + }; + + return { + ...normalizedWithoutHash, + sourceHash: contentHash(normalizedWithoutHash), + } satisfies NormalizedVocabularyWord; +} + +function normalizeVocabularyUnit(raw: unknown, rowNo: number, orphanWordsCount: number) { + const issues: ImportIssue[] = []; + const source = objectValue(raw); + const name = stringValue(source.name ?? source.title); + if (!name) { + issues.push({ + rowNo, + severity: 'error', + code: 'VOCABULARY_UNIT_NAME_REQUIRED', + fieldPath: 'name', + message: 'vocabulary unit name is required', + }); + } + + const words = asArray(source.words).map((word, index) => normalizeVocabularyWord(word, rowNo, index, issues)); + if (words.length === 0) { + issues.push({ + rowNo, + severity: 'warning', + code: 'VOCABULARY_UNIT_EMPTY', + fieldPath: 'words', + message: 'vocabulary unit has no words', + }); + } + if (orphanWordsCount > 0 && rowNo === 1) { + issues.push({ + rowNo, + severity: 'warning', + code: 'VOCABULARY_ORPHAN_WORDS_IGNORED', + fieldPath: 'words', + message: 'some flat vocabulary rows could not be matched to a unit and were ignored', + details: { orphanWordsCount }, + }); + } + + const normalizedWithoutHash = { + legacyId: stringValue(source.legacyId ?? source.legacy_id ?? source.externalId ?? source.external_id ?? source.id) || null, + name, + description: stringValue(source.description) || null, + order: integerValue(source.order ?? source.sortOrder ?? source.sort_order, rowNo), + wordCount: source.wordCount === undefined && source.word_count === undefined ? words.length : integerValue(source.wordCount ?? source.word_count, words.length), + isActive: boolishValue(source.isActive ?? source.is_active, true), + words, + metadata: objectValue(source.metadata), + }; + + return { + normalized: { + ...normalizedWithoutHash, + sourceHash: contentHash(normalizedWithoutHash), + } satisfies NormalizedVocabularyUnit, + issues, + }; +} + +function createVocabularyNormalizedItems(body: JsonObject): NormalizedImportItem[] { + const { units, flatWords } = parseVocabularyItems(body); + const attachedWordCount = units.reduce((sum, unit) => sum + asArray(objectValue(unit).words).length, 0); + const orphanWordsCount = Math.max(0, flatWords.length - attachedWordCount); + + return units.map((raw, index) => { + const rowNo = index + 1; + const { normalized, issues } = normalizeVocabularyUnit(raw, rowNo, orphanWordsCount); + return { + rowNo, + status: issues.some(issue => issue.severity === 'error') ? 'invalid' : 'valid', + externalId: normalized.legacyId || null, + source: raw, + normalized, + issues, + }; + }); +} + +function parseHandbookItems(body: JsonObject) { + const source = parseJsonMaybe( + body.items ?? body.subjects ?? body.handbooks ?? body.books ?? body.payload ?? body, + 'INVALID_HANDBOOK_IMPORT_PAYLOAD', + ); + const payload = objectValue(source); + + let subjects = Array.isArray(source) ? source : asArray( + payload.subjects ?? + payload.handbooks ?? + payload.books ?? + payload.handbookSubjects ?? + payload.handbook_subjects ?? + payload['handbook_categories_示例数据'], + ); + + if (subjects.length === 0 && payload.book && typeof payload.book === 'object') { + subjects = [payload.book]; + } + + const flatEntries = asArray( + payload.entries ?? + payload.handbookEntries ?? + payload.handbook_entries ?? + payload['handbook_entries_示例数据'], + ); + + if (subjects.length === 0 && flatEntries.length > 0) { + subjects = [{ + name: stringValue(payload.subjectName ?? payload.categoryName ?? payload.name) || '默认知识手册', + chapters: [{ + name: stringValue(payload.chapterName) || '默认章节', + sections: [{ name: stringValue(payload.sectionName) || null, entries: flatEntries }], + }], + }]; + } else if (flatEntries.length > 0) { + const subjectLookup = new Map(); + subjects.forEach((subject, index) => { + const raw = objectValue(subject); + for (const key of [raw.id, raw.legacyId, raw.legacy_id, raw.externalId, raw.external_id, raw.name]) { + const value = stringValue(key); + if (value) subjectLookup.set(value, index); + } + }); + + const buckets = new Map(); + flatEntries.forEach(entry => { + const raw = objectValue(entry); + const ref = stringValue(raw.subjectId ?? raw.subject_id ?? raw.categoryId ?? raw.category_id ?? raw.subjectName ?? raw.categoryName ?? raw.category); + const index = subjectLookup.get(ref); + if (index !== undefined) { + const bucket = buckets.get(index) || []; + bucket.push(entry); + buckets.set(index, bucket); + } + }); + + subjects = subjects.map((subject, index) => { + const extraEntries = buckets.get(index) || []; + if (extraEntries.length === 0) return subject; + const raw = objectValue(subject); + return { + ...raw, + chapters: [ + ...asArray(raw.chapters), + { + name: '默认章节', + sections: [{ name: null, entries: extraEntries }], + }, + ], + }; + }); + } + + if (subjects.length === 0) { + throw new HttpError(400, 'Handbook import payload must contain at least one subject/book', 'EMPTY_IMPORT_PAYLOAD'); + } + if (subjects.length > 300) { + throw new HttpError(400, 'A single handbook import job can contain at most 300 subjects/books', 'IMPORT_TOO_LARGE'); + } + return subjects; +} + +function normalizeHandbookEntry(raw: unknown, rowNo: number, fieldPrefix: string, fallbackOrder: number, issues: ImportIssue[]) { + const source = objectValue(raw); + const title = stringValue(source.title ?? source.name); + const content = stringValue(source.content ?? source.body); + if (!title) { + issues.push({ + rowNo, + severity: 'error', + code: 'HANDBOOK_ENTRY_TITLE_REQUIRED', + fieldPath: `${fieldPrefix}.title`, + message: 'handbook entry title is required', + }); + } + if (!content) { + issues.push({ + rowNo, + severity: 'error', + code: 'HANDBOOK_ENTRY_CONTENT_REQUIRED', + fieldPath: `${fieldPrefix}.content`, + message: 'handbook entry content is required', + }); + } + + const normalizedWithoutHash = { + legacyId: stringValue(source.legacyId ?? source.legacy_id ?? source.externalId ?? source.external_id ?? source.id) || null, + title, + summary: stringValue(source.summary) || null, + content, + tags: stringArrayValue(source.tags), + order: integerValue(source.order ?? source.sortOrder ?? source.sort_order, fallbackOrder), + isActive: boolishValue(source.isActive ?? source.is_active, true), + metadata: { + ...objectValue(source.metadata), + hasNext: boolishValue(source.hasNext ?? source.has_next, false), + nextEntryId: stringValue(source.nextEntryId ?? source.next_entry_id) || null, + }, + }; + + return { + ...normalizedWithoutHash, + sourceHash: contentHash(normalizedWithoutHash), + } satisfies NormalizedHandbookEntry; +} + +function normalizeHandbookSection(raw: unknown, rowNo: number, fieldPrefix: string, fallbackOrder: number, issues: ImportIssue[]) { + const source = objectValue(raw); + const entries = asArray(source.entries ?? source.handbookEntries ?? source.handbook_entries) + .map((entry, index) => normalizeHandbookEntry(entry, rowNo, `${fieldPrefix}.entries[${index}]`, index + 1, issues)); + + const normalizedWithoutHash = { + legacyId: stringValue(source.legacyId ?? source.legacy_id ?? source.externalId ?? source.external_id ?? source.id) || null, + name: stringValue(source.name ?? source.title) || null, + description: stringValue(source.description) || null, + order: integerValue(source.order ?? source.sortOrder ?? source.sort_order, fallbackOrder), + isActive: boolishValue(source.isActive ?? source.is_active, true), + entries, + metadata: objectValue(source.metadata), + }; + + return { + ...normalizedWithoutHash, + sourceHash: contentHash(normalizedWithoutHash), + } satisfies NormalizedHandbookSection; +} + +function normalizeHandbookChapter(raw: unknown, rowNo: number, fieldPrefix: string, fallbackOrder: number, issues: ImportIssue[]) { + const source = objectValue(raw); + const name = stringValue(source.name ?? source.title); + if (!name) { + issues.push({ + rowNo, + severity: 'error', + code: 'HANDBOOK_CHAPTER_NAME_REQUIRED', + fieldPath: `${fieldPrefix}.name`, + message: 'handbook chapter name is required', + }); + } + + const directEntries = asArray(source.entries ?? source.handbookEntries ?? source.handbook_entries); + const rawSections = asArray(source.sections ?? source.handbookSections ?? source.handbook_sections); + const sections = [ + ...rawSections, + ...(directEntries.length ? [{ name: null, entries: directEntries }] : []), + ].map((section, index) => normalizeHandbookSection(section, rowNo, `${fieldPrefix}.sections[${index}]`, index + 1, issues)); + + if (sections.length === 0) { + issues.push({ + rowNo, + severity: 'warning', + code: 'HANDBOOK_CHAPTER_EMPTY', + fieldPath: `${fieldPrefix}.sections`, + message: 'handbook chapter has no sections or entries', + }); + } + + const normalizedWithoutHash = { + legacyId: stringValue(source.legacyId ?? source.legacy_id ?? source.externalId ?? source.external_id ?? source.id) || null, + name, + description: stringValue(source.description) || null, + order: integerValue(source.order ?? source.sortOrder ?? source.sort_order, fallbackOrder), + isActive: boolishValue(source.isActive ?? source.is_active, true), + sections, + metadata: objectValue(source.metadata), + }; + + return { + ...normalizedWithoutHash, + sourceHash: contentHash(normalizedWithoutHash), + } satisfies NormalizedHandbookChapter; +} + +function normalizeHandbookSubject(raw: unknown, rowNo: number) { + const issues: ImportIssue[] = []; + const source = objectValue(raw); + const name = stringValue(source.name ?? source.title); + if (!name) { + issues.push({ + rowNo, + severity: 'error', + code: 'HANDBOOK_SUBJECT_NAME_REQUIRED', + fieldPath: 'name', + message: 'handbook subject/book name is required', + }); + } + + const chapters = asArray(source.chapters ?? source.handbookChapters ?? source.handbook_chapters) + .map((chapter, index) => normalizeHandbookChapter(chapter, rowNo, `chapters[${index}]`, index + 1, issues)); + if (chapters.length === 0) { + issues.push({ + rowNo, + severity: 'warning', + code: 'HANDBOOK_SUBJECT_EMPTY', + fieldPath: 'chapters', + message: 'handbook subject/book has no chapters', + }); + } + + const normalizedWithoutHash = { + legacyId: stringValue(source.legacyId ?? source.legacy_id ?? source.externalId ?? source.external_id ?? source.id) || null, + name, + type: stringValue(source.type) || null, + icon: stringValue(source.icon) || null, + color: stringValue(source.color) || null, + description: stringValue(source.description) || null, + order: integerValue(source.order ?? source.sortOrder ?? source.sort_order, rowNo), + isActive: boolishValue(source.isActive ?? source.is_active, true), + chapters, + metadata: objectValue(source.metadata), + }; + + return { + normalized: { + ...normalizedWithoutHash, + sourceHash: contentHash(normalizedWithoutHash), + } satisfies NormalizedHandbookSubject, + issues, + }; +} + +function createHandbookNormalizedItems(body: JsonObject): NormalizedImportItem[] { + return parseHandbookItems(body).map((raw, index) => { + const rowNo = index + 1; + const { normalized, issues } = normalizeHandbookSubject(raw, rowNo); + return { + rowNo, + status: issues.some(issue => issue.severity === 'error') ? 'invalid' : 'valid', + externalId: normalized.legacyId || null, + source: raw, + normalized, + issues, + }; + }); +} + +async function assertGenericTargetReferences( + client: pg.PoolClient, + auth: TenantContentAuth, + body: JsonObject, + expectedEntryType: 'vocabulary' | 'handbook', +): Promise { + const regionId = nullableString(body.regionId); + let entryId = nullableString(body.entryId); + const contentNodeId = nullableString(body.contentNodeId); + + if (regionId) { + const region = await client.query('select id from public.regions where tenant_id = $1 and id = $2 limit 1', [auth.tenantId, regionId]); + if (!region.rows[0]) throw new HttpError(400, 'regionId is not in this tenant', 'REGION_NOT_FOUND'); + } + + if (entryId) { + const entry = await client.query<{ id: string; entry_type: string }>( + 'select id, entry_type from public.content_entries where tenant_id = $1 and id = $2 limit 1', + [auth.tenantId, entryId], + ); + if (!entry.rows[0]) throw new HttpError(400, 'entryId is not in this tenant', 'ENTRY_NOT_FOUND'); + if (entry.rows[0].entry_type !== expectedEntryType) { + throw new HttpError(400, `entryId must be a ${expectedEntryType} content entry`, 'ENTRY_TYPE_MISMATCH'); + } + } + + if (contentNodeId) { + const node = await client.query<{ id: string; entry_id: string }>( + 'select id, entry_id from public.content_nodes where tenant_id = $1 and id = $2 limit 1', + [auth.tenantId, contentNodeId], + ); + if (!node.rows[0]) throw new HttpError(400, 'contentNodeId is not in this tenant', 'CONTENT_NODE_NOT_FOUND'); + if (entryId && node.rows[0].entry_id !== entryId) { + throw new HttpError(400, 'contentNodeId is not under entryId', 'CONTENT_NODE_ENTRY_MISMATCH'); + } + entryId = entryId || node.rows[0].entry_id; + } + + return { regionId, entryId, contentNodeId }; +} + +async function createGenericPreviewJob( + auth: TenantContentAuth, + body: JsonObject, + importType: 'vocabulary' | 'handbook', + targetType: string, + normalizedItems: NormalizedImportItem[], +): Promise> { + const sourceFormat = stringValue(body.sourceFormat) || 'json'; + const sourceName = stringValue(body.sourceName) || null; + if (sourceFormat !== 'json') { + throw new HttpError(400, 'Only json sourceFormat is supported by the synchronous API for now', 'IMPORT_FORMAT_NOT_SUPPORTED'); + } + + return transaction(async client => { + const target = await assertGenericTargetReferences(client, auth, body, importType); + const issues = normalizedItems.flatMap(item => item.issues); + const errorCount = issues.filter(issue => issue.severity === 'error').length; + const warningCount = issues.filter(issue => issue.severity === 'warning').length; + const validCount = normalizedItems.filter(item => item.status === 'valid').length; + + const jobResult = await client.query( + ` + insert into public.content_import_jobs ( + tenant_id, created_by, import_type, source_format, status, + source_name, source_hash, target_region_id, target_entry_id, + target_content_node_id, dry_run, total_count, valid_count, + error_count, warning_count, summary, raw_payload, normalized_payload + ) + values ( + $1, $2, $3, $4, 'preview', + $5, $6, $7::uuid, $8::uuid, + $9::uuid, true, $10, $11, + $12, $13, $14::jsonb, $15::jsonb, $16::jsonb + ) + returning id, status, total_count as "totalCount", valid_count as "validCount", + error_count as "errorCount", warning_count as "warningCount" + `, + [ + auth.tenantId, + auth.userId, + importType, + sourceFormat, + sourceName, + contentHash(normalizedItems.map(item => item.source)), + target.regionId, + target.entryId, + target.contentNodeId, + normalizedItems.length, + validCount, + errorCount, + warningCount, + JSON.stringify({ target, generatedAt: new Date().toISOString() }), + JSON.stringify(normalizedItems.map(item => item.source)), + JSON.stringify(normalizedItems.map(item => item.normalized).filter(Boolean)), + ], + ); + + const job = jobResult.rows[0]; + const responseItems: PreviewResult['items'] = []; + for (const item of normalizedItems) { + const itemResult = await client.query( + ` + insert into public.content_import_items ( + tenant_id, job_id, row_no, external_id, status, target_type, + source_payload, normalized_payload, content_hash, issues_count + ) + values ($1, $2, $3, $4, $5, $6, $7::jsonb, $8::jsonb, $9, $10) + returning id + `, + [ + auth.tenantId, + job.id, + item.rowNo, + item.externalId, + item.status, + targetType, + JSON.stringify(item.source), + JSON.stringify(item.normalized || {}), + item.normalized && typeof item.normalized === 'object' && 'sourceHash' in item.normalized + ? String((item.normalized as { sourceHash: string }).sourceHash) + : contentHash(item.source), + item.issues.length, + ], + ); + + const itemId = itemResult.rows[0].id; + for (const issue of item.issues) { + await client.query( + ` + insert into public.content_import_issues ( + tenant_id, job_id, item_id, row_no, severity, code, + field_path, message, details + ) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb) + `, + [ + auth.tenantId, + job.id, + itemId, + issue.rowNo, + issue.severity, + issue.code, + issue.fieldPath, + issue.message, + JSON.stringify(issue.details || {}), + ], + ); + } + + responseItems.push({ + rowNo: item.rowNo, + status: item.status, + externalId: item.externalId, + normalized: item.normalized, + issues: item.issues, + }); + } + + await client.query( + ` + insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details) + values ($1, $2, $3, 'content_import_job', $4, $5::jsonb) + `, + [ + auth.tenantId, + auth.userId, + `content.import.${importType}.previewed`, + job.id, + JSON.stringify({ importType, total: normalizedItems.length, errorCount, warningCount }), + ], + ); + + return { job, items: responseItems, issues }; + }); +} + async function loadPreviewJob(client: pg.PoolClient, auth: TenantContentAuth, jobId: string) { const result = await client.query<{ id: string; @@ -637,6 +1407,158 @@ async function loadPreviewJob(client: pg.PoolClient, auth: TenantContentAuth, jo return job; } +async function loadGenericPreviewJob( + client: pg.PoolClient, + auth: TenantContentAuth, + jobId: string, + importType: 'vocabulary' | 'handbook', +) { + const result = await client.query<{ + id: string; + status: string; + total_count: number; + valid_count: number; + error_count: number; + warning_count: number; + target_region_id: string | null; + target_entry_id: string | null; + target_content_node_id: string | null; + }>( + ` + select id, status, total_count, valid_count, error_count, warning_count, + target_region_id, target_entry_id, target_content_node_id + from public.content_import_jobs + where tenant_id = $1 and id = $2 and import_type = $3 + limit 1 + for update + `, + [auth.tenantId, jobId, importType], + ); + + const job = result.rows[0]; + if (!job) throw new HttpError(404, 'Import job not found', 'IMPORT_JOB_NOT_FOUND'); + if (['importing', 'failed'].includes(job.status)) { + throw new HttpError(409, `Import job is ${job.status}`, 'IMPORT_JOB_NOT_READY'); + } + return job; +} + +function ltreeLabel(id: string) { + return `n_${id.replace(/-/g, '')}`; +} + +function nodeKeyFor(prefix: string, legacyId: string) { + return `${prefix}:${legacyId}`.slice(0, 240); +} + +async function upsertImportContentNode( + client: pg.PoolClient, + auth: TenantContentAuth, + params: { + entryId: string | null; + regionId: string | null; + parentId: string | null; + nodeKey: string; + name: string; + nodeType: 'category' | 'subject' | 'chapter' | 'custom'; + order: number; + isActive: boolean; + isLeaf: boolean; + metadata: JsonObject; + }, +) { + if (!params.entryId) return null; + + const existing = await client.query<{ id: string }>( + ` + select id + from public.content_nodes + where tenant_id = $1 and entry_id = $2 and node_key = $3 + limit 1 + for update + `, + [auth.tenantId, params.entryId, params.nodeKey], + ); + const nodeId = existing.rows[0]?.id || randomUUID(); + + let parentPath: string | null = null; + let parentDepth = -1; + if (params.parentId) { + const parent = await client.query<{ path: string; depth: number }>( + ` + select path::text as path, depth + from public.content_nodes + where tenant_id = $1 and entry_id = $2 and id = $3 + limit 1 + `, + [auth.tenantId, params.entryId, params.parentId], + ); + if (!parent.rows[0]) { + throw new HttpError(400, 'parent content node is not in this entry', 'PARENT_NODE_NOT_FOUND'); + } + parentPath = parent.rows[0].path; + parentDepth = Number(parent.rows[0].depth || 0); + } + + const path = parentPath ? `${parentPath}.${ltreeLabel(nodeId)}` : ltreeLabel(nodeId); + const depth = parentDepth + 1; + const node = await client.query<{ id: string }>( + ` + insert into public.content_nodes ( + id, tenant_id, entry_id, region_id, parent_id, legacy_id, node_key, + name, node_type, marker_type, marker_config, path, depth, sort_order, + is_active, is_selectable, is_leaf, metadata, created_by, updated_by + ) + values ( + $1, $2, $3, $4::uuid, $5::uuid, null, $6, + $7, $8, null, '{}'::jsonb, $9::ltree, $10, $11, + $12, true, $13, $14::jsonb, $15, $15 + ) + on conflict (id) + do update set region_id = excluded.region_id, + parent_id = excluded.parent_id, + node_key = excluded.node_key, + name = excluded.name, + node_type = excluded.node_type, + path = excluded.path, + depth = excluded.depth, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + is_leaf = excluded.is_leaf, + metadata = excluded.metadata, + updated_by = excluded.updated_by, + updated_at = now() + returning id + `, + [ + nodeId, + auth.tenantId, + params.entryId, + params.regionId, + params.parentId, + params.nodeKey, + params.name, + params.nodeType, + path, + depth, + params.order, + params.isActive, + params.isLeaf, + JSON.stringify(params.metadata), + auth.userId, + ], + ); + + if (params.parentId) { + await client.query( + 'update public.content_nodes set is_leaf = false, updated_at = now() where tenant_id = $1 and id = $2', + [auth.tenantId, params.parentId], + ); + } + + return node.rows[0].id; +} + async function currentVersionHash(client: pg.PoolClient, questionId: string) { const result = await client.query<{ source_hash: string | null }>( ` @@ -848,12 +1770,560 @@ async function importOneQuestion( return status as 'inserted' | 'updated'; } +async function importOneVocabularyUnit( + client: pg.PoolClient, + auth: TenantContentAuth, + job: { + id: string; + target_region_id: string | null; + target_entry_id: string | null; + target_content_node_id: string | null; + }, + item: { + id: string; + row_no: number; + normalized_payload: NormalizedVocabularyUnit; + }, +) { + const normalized = item.normalized_payload; + const unitLegacyId = normalized.legacyId || `content-import:${job.id}:unit:${item.row_no}`; + const unitNodeId = await upsertImportContentNode(client, auth, { + entryId: job.target_entry_id, + regionId: job.target_region_id, + parentId: job.target_content_node_id, + nodeKey: nodeKeyFor('vocabulary-unit', unitLegacyId), + name: normalized.name, + nodeType: 'category', + order: normalized.order, + isActive: normalized.isActive, + isLeaf: true, + metadata: { source: 'content_import', importJobId: job.id, legacyId: unitLegacyId }, + }); + + const existing = await client.query<{ id: string; source_hash: string | null }>( + 'select id, source_hash from public.vocabulary_units where tenant_id = $1 and legacy_id = $2 limit 1 for update', + [auth.tenantId, unitLegacyId], + ); + const existingUnit = existing.rows[0]; + const unit = await client.query<{ id: string }>( + ` + insert into public.vocabulary_units ( + tenant_id, region_id, entry_id, content_node_id, legacy_id, name, + description, word_count, sort_order, is_active, source_hash, metadata + ) + values ($1, $2::uuid, $3::uuid, $4::uuid, $5, $6, $7, $8, $9, $10, $11, $12::jsonb) + on conflict (tenant_id, legacy_id) + do update set region_id = excluded.region_id, + entry_id = excluded.entry_id, + content_node_id = excluded.content_node_id, + name = excluded.name, + description = excluded.description, + word_count = excluded.word_count, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + source_hash = excluded.source_hash, + metadata = excluded.metadata, + updated_at = now() + returning id + `, + [ + auth.tenantId, + job.target_region_id, + job.target_entry_id, + unitNodeId, + unitLegacyId, + normalized.name, + normalized.description, + normalized.wordCount ?? normalized.words.length, + normalized.order, + normalized.isActive, + normalized.sourceHash, + JSON.stringify({ ...normalized.metadata, source: 'content_import', importJobId: job.id }), + ], + ); + + let insertedCount = existingUnit ? 0 : 1; + let updatedCount = existingUnit && existingUnit.source_hash !== normalized.sourceHash ? 1 : 0; + let skippedCount = existingUnit && existingUnit.source_hash === normalized.sourceHash ? 1 : 0; + + for (const [index, word] of normalized.words.entries()) { + const wordLegacyId = word.legacyId || `${unitLegacyId}:word:${word.word.toLowerCase()}:${index + 1}`; + const wordExisting = await client.query<{ id: string; source_hash: string | null }>( + 'select id, source_hash from public.vocabulary_words where tenant_id = $1 and legacy_id = $2 limit 1 for update', + [auth.tenantId, wordLegacyId], + ); + const existingWord = wordExisting.rows[0]; + await client.query( + ` + insert into public.vocabulary_words ( + tenant_id, unit_id, entry_id, content_node_id, legacy_id, word, + phonetic, meaning, example, example_translation, difficulty, + tags, sort_order, is_active, source_hash, metadata + ) + values ( + $1, $2, $3::uuid, $4::uuid, $5, $6, + $7, $8, $9, $10, $11, + $12::jsonb, $13, $14, $15, $16::jsonb + ) + on conflict (tenant_id, legacy_id) + do update set unit_id = excluded.unit_id, + entry_id = excluded.entry_id, + content_node_id = excluded.content_node_id, + word = excluded.word, + phonetic = excluded.phonetic, + meaning = excluded.meaning, + example = excluded.example, + example_translation = excluded.example_translation, + difficulty = excluded.difficulty, + tags = excluded.tags, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + source_hash = excluded.source_hash, + metadata = excluded.metadata, + updated_at = now() + `, + [ + auth.tenantId, + unit.rows[0].id, + job.target_entry_id, + unitNodeId, + wordLegacyId, + word.word, + word.phonetic, + word.meaning, + word.example, + word.exampleTranslation, + word.difficulty, + JSON.stringify(word.tags), + word.order, + word.isActive, + word.sourceHash, + JSON.stringify({ ...word.metadata, source: 'content_import', importJobId: job.id }), + ], + ); + + if (!existingWord) insertedCount += 1; + else if (existingWord.source_hash === word.sourceHash) skippedCount += 1; + else updatedCount += 1; + } + + await client.query( + ` + update public.vocabulary_units + set word_count = ( + select count(*) + from public.vocabulary_words + where tenant_id = $1 and unit_id = $2 and is_active = true + ), + updated_at = now() + where tenant_id = $1 and id = $2 + `, + [auth.tenantId, unit.rows[0].id], + ); + + const itemStatus = insertedCount > 0 ? 'inserted' : updatedCount > 0 ? 'updated' : 'skipped'; + await client.query( + ` + update public.content_import_items + set status = $3, target_id = $4, updated_at = now() + where tenant_id = $1 and id = $2 + `, + [auth.tenantId, item.id, itemStatus, unit.rows[0].id], + ); + + return { insertedCount, updatedCount, skippedCount }; +} + +async function importOneHandbookSubject( + client: pg.PoolClient, + auth: TenantContentAuth, + job: { + id: string; + target_region_id: string | null; + target_entry_id: string | null; + target_content_node_id: string | null; + }, + item: { + id: string; + row_no: number; + normalized_payload: NormalizedHandbookSubject; + }, +) { + const normalized = item.normalized_payload; + const subjectLegacyId = normalized.legacyId || `content-import:${job.id}:handbook:${item.row_no}`; + const subjectNodeId = await upsertImportContentNode(client, auth, { + entryId: job.target_entry_id, + regionId: job.target_region_id, + parentId: job.target_content_node_id, + nodeKey: nodeKeyFor('handbook-subject', subjectLegacyId), + name: normalized.name, + nodeType: 'subject', + order: normalized.order, + isActive: normalized.isActive, + isLeaf: normalized.chapters.length === 0, + metadata: { source: 'content_import', importJobId: job.id, legacyId: subjectLegacyId }, + }); + + const existing = await client.query<{ id: string; source_hash: string | null }>( + 'select id, source_hash from public.handbook_subjects where tenant_id = $1 and legacy_id = $2 limit 1 for update', + [auth.tenantId, subjectLegacyId], + ); + const existingSubject = existing.rows[0]; + const subject = await client.query<{ id: string }>( + ` + insert into public.handbook_subjects ( + tenant_id, region_id, entry_id, content_node_id, legacy_id, name, + type, icon, color, description, sort_order, is_active, metadata, source_hash + ) + values ( + $1, $2::uuid, $3::uuid, $4::uuid, $5, $6, + $7, $8, $9, $10, $11, $12, $13::jsonb, $14 + ) + on conflict (tenant_id, legacy_id) + do update set region_id = excluded.region_id, + entry_id = excluded.entry_id, + content_node_id = excluded.content_node_id, + name = excluded.name, + type = excluded.type, + icon = excluded.icon, + color = excluded.color, + description = excluded.description, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + metadata = excluded.metadata, + source_hash = excluded.source_hash, + updated_at = now() + returning id + `, + [ + auth.tenantId, + job.target_region_id, + job.target_entry_id, + subjectNodeId, + subjectLegacyId, + normalized.name, + normalized.type, + normalized.icon, + normalized.color, + normalized.description, + normalized.order, + normalized.isActive, + JSON.stringify({ ...normalized.metadata, source: 'content_import', importJobId: job.id }), + normalized.sourceHash, + ], + ); + + let insertedCount = existingSubject ? 0 : 1; + let updatedCount = existingSubject && existingSubject.source_hash !== normalized.sourceHash ? 1 : 0; + let skippedCount = existingSubject && existingSubject.source_hash === normalized.sourceHash ? 1 : 0; + + for (const [chapterIndex, chapter] of normalized.chapters.entries()) { + const chapterLegacyId = chapter.legacyId || `${subjectLegacyId}:chapter:${chapterIndex + 1}`; + const chapterNodeId = await upsertImportContentNode(client, auth, { + entryId: job.target_entry_id, + regionId: job.target_region_id, + parentId: subjectNodeId, + nodeKey: nodeKeyFor('handbook-chapter', chapterLegacyId), + name: chapter.name, + nodeType: 'chapter', + order: chapter.order, + isActive: chapter.isActive, + isLeaf: chapter.sections.length === 0, + metadata: { source: 'content_import', importJobId: job.id, legacyId: chapterLegacyId }, + }); + const chapterExisting = await client.query<{ id: string; source_hash: string | null }>( + 'select id, source_hash from public.handbook_chapters where tenant_id = $1 and legacy_id = $2 limit 1 for update', + [auth.tenantId, chapterLegacyId], + ); + const chapterRow = await client.query<{ id: string }>( + ` + insert into public.handbook_chapters ( + tenant_id, subject_id, entry_id, content_node_id, legacy_id, name, + description, sort_order, is_active, source_hash, metadata + ) + values ($1, $2, $3::uuid, $4::uuid, $5, $6, $7, $8, $9, $10, $11::jsonb) + on conflict (tenant_id, legacy_id) + do update set subject_id = excluded.subject_id, + entry_id = excluded.entry_id, + content_node_id = excluded.content_node_id, + name = excluded.name, + description = excluded.description, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + source_hash = excluded.source_hash, + metadata = excluded.metadata, + updated_at = now() + returning id + `, + [ + auth.tenantId, + subject.rows[0].id, + job.target_entry_id, + chapterNodeId, + chapterLegacyId, + chapter.name, + chapter.description, + chapter.order, + chapter.isActive, + chapter.sourceHash, + JSON.stringify({ ...chapter.metadata, source: 'content_import', importJobId: job.id }), + ], + ); + if (!chapterExisting.rows[0]) insertedCount += 1; + else if (chapterExisting.rows[0].source_hash === chapter.sourceHash) skippedCount += 1; + else updatedCount += 1; + + for (const [sectionIndex, section] of chapter.sections.entries()) { + const sectionLegacyId = section.legacyId || `${chapterLegacyId}:section:${sectionIndex + 1}`; + const sectionNodeId = section.name + ? await upsertImportContentNode(client, auth, { + entryId: job.target_entry_id, + regionId: job.target_region_id, + parentId: chapterNodeId, + nodeKey: nodeKeyFor('handbook-section', sectionLegacyId), + name: section.name, + nodeType: 'custom', + order: section.order, + isActive: section.isActive, + isLeaf: true, + metadata: { + ...section.metadata, + source: 'content_import', + importJobId: job.id, + legacyId: sectionLegacyId, + kind: 'handbook_section', + description: section.description, + }, + }) + : chapterNodeId; + + for (const [entryIndex, entry] of section.entries.entries()) { + const entryLegacyId = entry.legacyId || `${sectionLegacyId}:entry:${entryIndex + 1}`; + const entryExisting = await client.query<{ id: string; source_hash: string | null }>( + 'select id, source_hash from public.handbook_entries where tenant_id = $1 and legacy_id = $2 limit 1 for update', + [auth.tenantId, entryLegacyId], + ); + await client.query( + ` + insert into public.handbook_entries ( + tenant_id, chapter_id, entry_id, content_node_id, legacy_id, title, + summary, content, tags, sort_order, is_active, source_hash, metadata + ) + values ( + $1, $2, $3::uuid, $4::uuid, $5, $6, + $7, $8, $9::jsonb, $10, $11, $12, $13::jsonb + ) + on conflict (tenant_id, legacy_id) + do update set chapter_id = excluded.chapter_id, + entry_id = excluded.entry_id, + content_node_id = excluded.content_node_id, + title = excluded.title, + summary = excluded.summary, + content = excluded.content, + tags = excluded.tags, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + source_hash = excluded.source_hash, + metadata = excluded.metadata, + updated_at = now() + `, + [ + auth.tenantId, + chapterRow.rows[0].id, + job.target_entry_id, + sectionNodeId, + entryLegacyId, + entry.title, + entry.summary, + entry.content, + JSON.stringify(entry.tags), + entry.order, + entry.isActive, + entry.sourceHash, + JSON.stringify({ + ...entry.metadata, + source: 'content_import', + importJobId: job.id, + sectionLegacyId, + sectionName: section.name, + }), + ], + ); + if (!entryExisting.rows[0]) insertedCount += 1; + else if (entryExisting.rows[0].source_hash === entry.sourceHash) skippedCount += 1; + else updatedCount += 1; + } + } + } + + const itemStatus = insertedCount > 0 ? 'inserted' : updatedCount > 0 ? 'updated' : 'skipped'; + await client.query( + ` + update public.content_import_items + set status = $3, target_id = $4, updated_at = now() + where tenant_id = $1 and id = $2 + `, + [auth.tenantId, item.id, itemStatus, subject.rows[0].id], + ); + + return { insertedCount, updatedCount, skippedCount }; +} + +async function runGenericImport( + auth: TenantContentAuth, + body: JsonObject, + importType: 'vocabulary' | 'handbook', + createPreview: () => Promise>, + importOne: ( + client: pg.PoolClient, + auth: TenantContentAuth, + job: { + id: string; + target_region_id: string | null; + target_entry_id: string | null; + target_content_node_id: string | null; + }, + item: { + id: string; + row_no: number; + normalized_payload: T; + }, + ) => Promise<{ insertedCount: number; updatedCount: number; skippedCount: number }>, +) { + const allowPartial = boolValue(body.allowPartial, false); + const jobId = nullableString(body.previewJobId) || nullableString(body.jobId); + const createdPreview = jobId ? null : await createPreview(); + const finalJobId = jobId || createdPreview?.job.id || ''; + + const result = await transaction(async client => { + const job = await loadGenericPreviewJob(client, auth, finalJobId, importType); + if (job.status === 'completed' || job.status === 'completed_with_errors') { + return { + jobId: job.id, + status: job.status, + idempotent: true, + insertedCount: 0, + updatedCount: 0, + skippedCount: 0, + }; + } + + if (job.error_count > 0 && !allowPartial) { + await client.query( + ` + update public.content_import_jobs + set status = 'rejected', error_message = 'Preview contains validation errors', updated_at = now() + where tenant_id = $1 and id = $2 + `, + [auth.tenantId, job.id], + ); + throw new HttpError(409, 'Preview contains validation errors. Fix issues or set allowPartial=true.', 'IMPORT_HAS_ERRORS'); + } + + await client.query( + ` + update public.content_import_jobs + set status = 'importing', dry_run = false, started_at = coalesce(started_at, now()), updated_at = now() + where tenant_id = $1 and id = $2 + `, + [auth.tenantId, job.id], + ); + + const itemResult = await client.query<{ + id: string; + row_no: number; + normalized_payload: T; + }>( + ` + select id, row_no, normalized_payload + from public.content_import_items + where tenant_id = $1 and job_id = $2 and status = 'valid' + order by row_no asc + for update + `, + [auth.tenantId, job.id], + ); + + let insertedCount = 0; + let updatedCount = 0; + let skippedCount = 0; + for (const item of itemResult.rows) { + const status = await importOne(client, auth, job, item); + insertedCount += status.insertedCount; + updatedCount += status.updatedCount; + skippedCount += status.skippedCount; + } + + const finalStatus = job.error_count > 0 ? 'completed_with_errors' : 'completed'; + await client.query( + ` + update public.content_import_jobs + set status = $3, + inserted_count = $4, + updated_count = $5, + skipped_count = $6, + summary = coalesce(summary, '{}'::jsonb) || $7::jsonb, + finished_at = now(), + updated_at = now() + where tenant_id = $1 and id = $2 + `, + [ + auth.tenantId, + job.id, + finalStatus, + insertedCount, + updatedCount, + skippedCount, + JSON.stringify({ insertedCount, updatedCount, skippedCount, importedAt: new Date().toISOString() }), + ], + ); + + await client.query( + ` + insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details) + values ($1, $2, $3, 'content_import_job', $4, $5::jsonb) + `, + [ + auth.tenantId, + auth.userId, + `content.import.${importType}.completed`, + job.id, + JSON.stringify({ insertedCount, updatedCount, skippedCount, allowPartial }), + ], + ); + + return { + jobId: job.id, + status: finalStatus, + insertedCount, + updatedCount, + skippedCount, + errorCount: job.error_count, + warningCount: job.warning_count, + }; + }); + + return { item: result, preview: createdPreview }; +} + export async function previewQuestionsImportRoute(ctx: RequestContext) { const auth = await requireTenantContentEditor(ctx); const body = await readJsonBody(ctx); return createQuestionPreviewJob(auth, body); } +export async function previewVocabularyImportRoute(ctx: RequestContext) { + const auth = await requireTenantContentEditor(ctx); + const body = await readJsonBody(ctx); + return createGenericPreviewJob(auth, body, 'vocabulary', 'vocabulary_unit', createVocabularyNormalizedItems(body)); +} + +export async function previewHandbookImportRoute(ctx: RequestContext) { + const auth = await requireTenantContentEditor(ctx); + const body = await readJsonBody(ctx); + return createGenericPreviewJob(auth, body, 'handbook', 'handbook_subject', createHandbookNormalizedItems(body)); +} + export async function importQuestionsRoute(ctx: RequestContext) { const auth = await requireTenantContentEditor(ctx); const body = await readJsonBody(ctx); @@ -969,6 +2439,30 @@ export async function importQuestionsRoute(ctx: RequestContext) { return { item: result, preview: createdPreview }; } +export async function importVocabularyRoute(ctx: RequestContext) { + const auth = await requireTenantContentEditor(ctx); + const body = await readJsonBody(ctx); + return runGenericImport( + auth, + body, + 'vocabulary', + () => createGenericPreviewJob(auth, body, 'vocabulary', 'vocabulary_unit', createVocabularyNormalizedItems(body)), + importOneVocabularyUnit, + ); +} + +export async function importHandbookRoute(ctx: RequestContext) { + const auth = await requireTenantContentEditor(ctx); + const body = await readJsonBody(ctx); + return runGenericImport( + auth, + body, + 'handbook', + () => createGenericPreviewJob(auth, body, 'handbook', 'handbook_subject', createHandbookNormalizedItems(body)), + importOneHandbookSubject, + ); +} + export async function importJobsRoute(ctx: RequestContext) { const auth = await requireTenantContentEditor(ctx); const limit = intParam(ctx, 'limit', 50, 200); diff --git a/apps/api/src/features/tenant-content/index.ts b/apps/api/src/features/tenant-content/index.ts index ff56ad3f..4e5e971f 100644 --- a/apps/api/src/features/tenant-content/index.ts +++ b/apps/api/src/features/tenant-content/index.ts @@ -6,10 +6,14 @@ import { upsertAssetRoute, } from './assets.js'; import { + importHandbookRoute, importIssuesRoute, importJobsRoute, importQuestionsRoute, + importVocabularyRoute, + previewHandbookImportRoute, previewQuestionsImportRoute, + previewVocabularyImportRoute, } from './imports.js'; import { contentEntriesAdminRoute, @@ -66,6 +70,10 @@ export const tenantContentRoutes: RouteDefinition[] = [ ['POST', '/api/tenant-content/assets/sign-download', signAssetDownloadAdminRoute], ['POST', '/api/tenant-content/imports/preview/questions', previewQuestionsImportRoute], ['POST', '/api/tenant-content/imports/questions', importQuestionsRoute], + ['POST', '/api/tenant-content/imports/preview/vocabulary', previewVocabularyImportRoute], + ['POST', '/api/tenant-content/imports/vocabulary', importVocabularyRoute], + ['POST', '/api/tenant-content/imports/preview/handbook', previewHandbookImportRoute], + ['POST', '/api/tenant-content/imports/handbook', importHandbookRoute], ['GET', '/api/tenant-content/imports', importJobsRoute], ['GET', '/api/tenant-content/imports/issues', importIssuesRoute], ['GET', '/api/tenant-content/videos', videosAdminRoute], diff --git a/apps/api/src/features/tenant-content/routes.ts b/apps/api/src/features/tenant-content/routes.ts index bf74aaee..34d199ff 100644 --- a/apps/api/src/features/tenant-content/routes.ts +++ b/apps/api/src/features/tenant-content/routes.ts @@ -1,4 +1,5 @@ import { HttpError, type RequestContext } from '../../core/http.js'; +import type pg from 'pg'; import { intParam, optionalString, readJsonBody, requiredString, stringParam } from '../../core/request.js'; import { query, queryOne, transaction } from '../../core/db.js'; import { requireTenantContentEditor } from './auth.js'; @@ -20,6 +21,41 @@ async function assertOptionalReference( } } +async function resolveOptionalContentNavigation( + client: pg.PoolClient, + tenantId: string, + entryId: string | null, + contentNodeId: string | null, + expectedEntryType: 'vocabulary' | 'handbook', +) { + let resolvedEntryId = entryId; + + if (resolvedEntryId) { + const entry = await client.query<{ id: string; entry_type: string }>( + 'select id, entry_type from public.content_entries where tenant_id = $1 and id = $2 limit 1', + [tenantId, resolvedEntryId], + ); + if (!entry.rows[0]) throw new HttpError(400, 'content entry reference is not in this tenant', 'ENTRY_NOT_FOUND'); + if (entry.rows[0].entry_type !== expectedEntryType) { + throw new HttpError(400, `content entry must be ${expectedEntryType}`, 'ENTRY_TYPE_MISMATCH'); + } + } + + if (contentNodeId) { + const node = await client.query<{ id: string; entry_id: string }>( + 'select id, entry_id from public.content_nodes where tenant_id = $1 and id = $2 limit 1', + [tenantId, contentNodeId], + ); + if (!node.rows[0]) throw new HttpError(400, 'content node reference is not in this tenant', 'CONTENT_NODE_NOT_FOUND'); + if (resolvedEntryId && node.rows[0].entry_id !== resolvedEntryId) { + throw new HttpError(400, 'content node is not under the selected entry', 'CONTENT_NODE_ENTRY_MISMATCH'); + } + resolvedEntryId = resolvedEntryId || node.rows[0].entry_id; + } + + return { entryId: resolvedEntryId, contentNodeId }; +} + async function syncPrimaryCollectionItem( client: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> }, tenantId: string, @@ -652,7 +688,8 @@ export async function vocabularyUnitsAdminRoute(ctx: RequestContext) { const auth = await requireTenantContentEditor(ctx); const items = await query( ` - select id, region_id as "regionId", name, description, word_count as "wordCount", + select id, region_id as "regionId", entry_id as "entryId", + content_node_id as "contentNodeId", name, description, word_count as "wordCount", sort_order as "order", is_active as "isActive", created_at as "createdAt", updated_at as "updatedAt" from public.vocabulary_units where tenant_id = $1 @@ -666,35 +703,53 @@ export async function vocabularyUnitsAdminRoute(ctx: RequestContext) { export async function upsertVocabularyUnitRoute(ctx: RequestContext) { const auth = await requireTenantContentEditor(ctx); const body = await readJsonBody(ctx); - const item = await queryOne( - ` - insert into public.vocabulary_units ( - id, tenant_id, region_id, legacy_id, name, description, word_count, sort_order, is_active - ) - values (coalesce($1::uuid, gen_random_uuid()), $2, $3::uuid, $4, $5, $6, $7, $8, $9) - on conflict (id) - do update set region_id = excluded.region_id, - name = excluded.name, - description = excluded.description, - word_count = excluded.word_count, - sort_order = excluded.sort_order, - is_active = excluded.is_active, - updated_at = now() - returning id, region_id as "regionId", name, description, word_count as "wordCount", - sort_order as "order", is_active as "isActive", updated_at as "updatedAt" - `, - [ - nullableString(body.id), + const item = await transaction(async client => { + const navigation = await resolveOptionalContentNavigation( + client, auth.tenantId, - nullableString(body.regionId), - nullableString(body.legacyId), - requiredString(body, 'name'), - nullableString(body.description), - body.wordCount === undefined ? null : intValue(body.wordCount, 0), - intValue(body.order, 0), - boolValue(body.isActive, true), - ], - ); + nullableString(body.entryId), + nullableString(body.contentNodeId), + 'vocabulary', + ); + const result = await client.query( + ` + insert into public.vocabulary_units ( + id, tenant_id, region_id, entry_id, content_node_id, legacy_id, + name, description, word_count, sort_order, is_active, metadata + ) + values (coalesce($1::uuid, gen_random_uuid()), $2, $3::uuid, $4::uuid, $5::uuid, $6, $7, $8, $9, $10, $11, $12::jsonb) + on conflict (id) + do update set region_id = excluded.region_id, + entry_id = excluded.entry_id, + content_node_id = excluded.content_node_id, + name = excluded.name, + description = excluded.description, + word_count = excluded.word_count, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + metadata = excluded.metadata, + updated_at = now() + returning id, region_id as "regionId", entry_id as "entryId", + content_node_id as "contentNodeId", name, description, word_count as "wordCount", + sort_order as "order", is_active as "isActive", metadata, updated_at as "updatedAt" + `, + [ + nullableString(body.id), + auth.tenantId, + nullableString(body.regionId), + navigation.entryId, + navigation.contentNodeId, + nullableString(body.legacyId), + requiredString(body, 'name'), + nullableString(body.description), + body.wordCount === undefined ? null : intValue(body.wordCount, 0), + intValue(body.order, 0), + boolValue(body.isActive, true), + jsonObjectValue(body.metadata), + ], + ); + return result.rows[0]; + }); return { item }; } @@ -704,7 +759,8 @@ export async function vocabularyWordsAdminRoute(ctx: RequestContext) { const limit = intParam(ctx, 'limit', 500, 2000); const items = await query( ` - select id, unit_id as "unitId", word, phonetic, meaning, example, + select id, unit_id as "unitId", entry_id as "entryId", + content_node_id as "contentNodeId", word, phonetic, meaning, example, example_translation as "exampleTranslation", difficulty, tags, sort_order as "order", is_active as "isActive", created_at as "createdAt", updated_at as "updatedAt" from public.vocabulary_words @@ -720,46 +776,76 @@ export async function vocabularyWordsAdminRoute(ctx: RequestContext) { export async function upsertVocabularyWordRoute(ctx: RequestContext) { const auth = await requireTenantContentEditor(ctx); const body = await readJsonBody(ctx); - const item = await queryOne( - ` - insert into public.vocabulary_words ( - id, tenant_id, unit_id, legacy_id, word, phonetic, meaning, - example, example_translation, difficulty, tags, sort_order, is_active - ) - values (coalesce($1::uuid, gen_random_uuid()), $2, $3::uuid, $4, $5, $6, $7, $8, $9, $10, $11::jsonb, $12, $13) - on conflict (id) - do update set unit_id = excluded.unit_id, - word = excluded.word, - phonetic = excluded.phonetic, - meaning = excluded.meaning, - example = excluded.example, - example_translation = excluded.example_translation, - difficulty = excluded.difficulty, - tags = excluded.tags, - sort_order = excluded.sort_order, - is_active = excluded.is_active, - updated_at = now() - returning id, unit_id as "unitId", word, phonetic, meaning, - example, example_translation as "exampleTranslation", - difficulty, tags, sort_order as "order", is_active as "isActive", - updated_at as "updatedAt" - `, - [ - nullableString(body.id), + const item = await transaction(async client => { + const unitId = nullableString(body.unitId); + let inheritedEntryId: string | null = null; + let inheritedContentNodeId: string | null = null; + if (unitId) { + const unit = await client.query<{ id: string; entry_id: string | null; content_node_id: string | null }>( + 'select id, entry_id, content_node_id from public.vocabulary_units where tenant_id = $1 and id = $2 limit 1', + [auth.tenantId, unitId], + ); + if (!unit.rows[0]) throw new HttpError(400, 'unitId is not in this tenant', 'VOCABULARY_UNIT_NOT_FOUND'); + inheritedEntryId = unit.rows[0].entry_id; + inheritedContentNodeId = unit.rows[0].content_node_id; + } + const navigation = await resolveOptionalContentNavigation( + client, auth.tenantId, - nullableString(body.unitId), - nullableString(body.legacyId), - requiredString(body, 'word'), - nullableString(body.phonetic), - nullableString(body.meaning), - nullableString(body.example), - nullableString(body.exampleTranslation), - body.difficulty === undefined ? null : intValue(body.difficulty, 1), - jsonArrayValue(body.tags), - intValue(body.order, 0), - boolValue(body.isActive, true), - ], - ); + nullableString(body.entryId) || inheritedEntryId, + nullableString(body.contentNodeId) || inheritedContentNodeId, + 'vocabulary', + ); + const result = await client.query( + ` + insert into public.vocabulary_words ( + id, tenant_id, unit_id, entry_id, content_node_id, legacy_id, word, + phonetic, meaning, example, example_translation, difficulty, tags, + sort_order, is_active, metadata + ) + values (coalesce($1::uuid, gen_random_uuid()), $2, $3::uuid, $4::uuid, $5::uuid, $6, $7, $8, $9, $10, $11, $12, $13::jsonb, $14, $15, $16::jsonb) + on conflict (id) + do update set unit_id = excluded.unit_id, + entry_id = excluded.entry_id, + content_node_id = excluded.content_node_id, + word = excluded.word, + phonetic = excluded.phonetic, + meaning = excluded.meaning, + example = excluded.example, + example_translation = excluded.example_translation, + difficulty = excluded.difficulty, + tags = excluded.tags, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + metadata = excluded.metadata, + updated_at = now() + returning id, unit_id as "unitId", entry_id as "entryId", + content_node_id as "contentNodeId", word, phonetic, meaning, + example, example_translation as "exampleTranslation", + difficulty, tags, sort_order as "order", is_active as "isActive", + metadata, updated_at as "updatedAt" + `, + [ + nullableString(body.id), + auth.tenantId, + unitId, + navigation.entryId, + navigation.contentNodeId, + nullableString(body.legacyId), + requiredString(body, 'word'), + nullableString(body.phonetic), + nullableString(body.meaning), + nullableString(body.example), + nullableString(body.exampleTranslation), + body.difficulty === undefined ? null : intValue(body.difficulty, 1), + jsonArrayValue(body.tags), + intValue(body.order, 0), + boolValue(body.isActive, true), + jsonObjectValue(body.metadata), + ], + ); + return result.rows[0]; + }); return { item }; } @@ -767,7 +853,8 @@ export async function handbookSubjectsAdminRoute(ctx: RequestContext) { const auth = await requireTenantContentEditor(ctx); const items = await query( ` - select id, region_id as "regionId", name, type, icon, color, + select id, region_id as "regionId", entry_id as "entryId", + content_node_id as "contentNodeId", name, type, icon, color, description, sort_order as "order", is_active as "isActive", metadata from public.handbook_subjects where tenant_id = $1 @@ -781,43 +868,58 @@ export async function handbookSubjectsAdminRoute(ctx: RequestContext) { export async function upsertHandbookSubjectRoute(ctx: RequestContext) { const auth = await requireTenantContentEditor(ctx); const body = await readJsonBody(ctx); - const item = await queryOne( - ` - insert into public.handbook_subjects ( - id, tenant_id, region_id, legacy_id, name, type, icon, color, - description, sort_order, is_active, metadata - ) - values (coalesce($1::uuid, gen_random_uuid()), $2, $3::uuid, $4, $5, $6, $7, $8, $9, $10, $11, $12::jsonb) - on conflict (id) - do update set region_id = excluded.region_id, - name = excluded.name, - type = excluded.type, - icon = excluded.icon, - color = excluded.color, - description = excluded.description, - sort_order = excluded.sort_order, - is_active = excluded.is_active, - metadata = excluded.metadata, - updated_at = now() - returning id, region_id as "regionId", name, type, icon, color, - description, sort_order as "order", is_active as "isActive", - metadata, updated_at as "updatedAt" - `, - [ - nullableString(body.id), + const item = await transaction(async client => { + const navigation = await resolveOptionalContentNavigation( + client, auth.tenantId, - nullableString(body.regionId), - nullableString(body.legacyId), - requiredString(body, 'name'), - nullableString(body.type), - nullableString(body.icon), - nullableString(body.color), - nullableString(body.description), - intValue(body.order, 0), - boolValue(body.isActive, true), - jsonObjectValue(body.metadata), - ], - ); + nullableString(body.entryId), + nullableString(body.contentNodeId), + 'handbook', + ); + const result = await client.query( + ` + insert into public.handbook_subjects ( + id, tenant_id, region_id, entry_id, content_node_id, legacy_id, + name, type, icon, color, description, sort_order, is_active, metadata + ) + values (coalesce($1::uuid, gen_random_uuid()), $2, $3::uuid, $4::uuid, $5::uuid, $6, $7, $8, $9, $10, $11, $12, $13, $14::jsonb) + on conflict (id) + do update set region_id = excluded.region_id, + entry_id = excluded.entry_id, + content_node_id = excluded.content_node_id, + name = excluded.name, + type = excluded.type, + icon = excluded.icon, + color = excluded.color, + description = excluded.description, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + metadata = excluded.metadata, + updated_at = now() + returning id, region_id as "regionId", entry_id as "entryId", + content_node_id as "contentNodeId", name, type, icon, color, + description, sort_order as "order", is_active as "isActive", + metadata, updated_at as "updatedAt" + `, + [ + nullableString(body.id), + auth.tenantId, + nullableString(body.regionId), + navigation.entryId, + navigation.contentNodeId, + nullableString(body.legacyId), + requiredString(body, 'name'), + nullableString(body.type), + nullableString(body.icon), + nullableString(body.color), + nullableString(body.description), + intValue(body.order, 0), + boolValue(body.isActive, true), + jsonObjectValue(body.metadata), + ], + ); + return result.rows[0]; + }); return { item }; } @@ -826,7 +928,8 @@ export async function handbookChaptersAdminRoute(ctx: RequestContext) { const subjectId = stringParam(ctx, 'subjectId'); const items = await query( ` - select id, subject_id as "subjectId", name, description, + select id, subject_id as "subjectId", entry_id as "entryId", + content_node_id as "contentNodeId", name, description, sort_order as "order", is_active as "isActive" from public.handbook_chapters where tenant_id = $1 and ($2::uuid is null or subject_id = $2::uuid) @@ -840,33 +943,57 @@ export async function handbookChaptersAdminRoute(ctx: RequestContext) { export async function upsertHandbookChapterRoute(ctx: RequestContext) { const auth = await requireTenantContentEditor(ctx); const body = await readJsonBody(ctx); - const item = await queryOne( - ` - insert into public.handbook_chapters ( - id, tenant_id, subject_id, legacy_id, name, description, sort_order, is_active - ) - values (coalesce($1::uuid, gen_random_uuid()), $2, $3::uuid, $4, $5, $6, $7, $8) - on conflict (id) - do update set subject_id = excluded.subject_id, - name = excluded.name, - description = excluded.description, - sort_order = excluded.sort_order, - is_active = excluded.is_active, - updated_at = now() - returning id, subject_id as "subjectId", name, description, - sort_order as "order", is_active as "isActive", updated_at as "updatedAt" - `, - [ - nullableString(body.id), + const item = await transaction(async client => { + const subjectId = requiredString(body, 'subjectId'); + const subject = await client.query<{ id: string; entry_id: string | null; content_node_id: string | null }>( + 'select id, entry_id, content_node_id from public.handbook_subjects where tenant_id = $1 and id = $2 limit 1', + [auth.tenantId, subjectId], + ); + if (!subject.rows[0]) throw new HttpError(400, 'subjectId is not in this tenant', 'HANDBOOK_SUBJECT_NOT_FOUND'); + const navigation = await resolveOptionalContentNavigation( + client, auth.tenantId, - requiredString(body, 'subjectId'), - nullableString(body.legacyId), - requiredString(body, 'name'), - nullableString(body.description), - intValue(body.order, 0), - boolValue(body.isActive, true), - ], - ); + nullableString(body.entryId) || subject.rows[0].entry_id, + nullableString(body.contentNodeId) || subject.rows[0].content_node_id, + 'handbook', + ); + const result = await client.query( + ` + insert into public.handbook_chapters ( + id, tenant_id, subject_id, entry_id, content_node_id, legacy_id, + name, description, sort_order, is_active, metadata + ) + values (coalesce($1::uuid, gen_random_uuid()), $2, $3::uuid, $4::uuid, $5::uuid, $6, $7, $8, $9, $10, $11::jsonb) + on conflict (id) + do update set subject_id = excluded.subject_id, + entry_id = excluded.entry_id, + content_node_id = excluded.content_node_id, + name = excluded.name, + description = excluded.description, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + metadata = excluded.metadata, + updated_at = now() + returning id, subject_id as "subjectId", entry_id as "entryId", + content_node_id as "contentNodeId", name, description, + sort_order as "order", is_active as "isActive", metadata, updated_at as "updatedAt" + `, + [ + nullableString(body.id), + auth.tenantId, + subjectId, + navigation.entryId, + navigation.contentNodeId, + nullableString(body.legacyId), + requiredString(body, 'name'), + nullableString(body.description), + intValue(body.order, 0), + boolValue(body.isActive, true), + jsonObjectValue(body.metadata), + ], + ); + return result.rows[0]; + }); return { item }; } @@ -875,7 +1002,8 @@ export async function handbookEntriesAdminRoute(ctx: RequestContext) { const chapterId = stringParam(ctx, 'chapterId'); const items = await query( ` - select id, chapter_id as "chapterId", title, summary, content, + select id, chapter_id as "chapterId", entry_id as "entryId", + content_node_id as "contentNodeId", title, summary, content, tags, sort_order as "order", is_active as "isActive" from public.handbook_entries where tenant_id = $1 and ($2::uuid is null or chapter_id = $2::uuid) @@ -889,37 +1017,60 @@ export async function handbookEntriesAdminRoute(ctx: RequestContext) { export async function upsertHandbookEntryRoute(ctx: RequestContext) { const auth = await requireTenantContentEditor(ctx); const body = await readJsonBody(ctx); - const item = await queryOne( - ` - insert into public.handbook_entries ( - id, tenant_id, chapter_id, legacy_id, title, summary, content, - tags, sort_order, is_active - ) - values (coalesce($1::uuid, gen_random_uuid()), $2, $3::uuid, $4, $5, $6, $7, $8::jsonb, $9, $10) - on conflict (id) - do update set chapter_id = excluded.chapter_id, - title = excluded.title, - summary = excluded.summary, - content = excluded.content, - tags = excluded.tags, - sort_order = excluded.sort_order, - is_active = excluded.is_active, - updated_at = now() - returning id, chapter_id as "chapterId", title, summary, content, - tags, sort_order as "order", is_active as "isActive", updated_at as "updatedAt" - `, - [ - nullableString(body.id), + const item = await transaction(async client => { + const chapterId = requiredString(body, 'chapterId'); + const chapter = await client.query<{ id: string; entry_id: string | null; content_node_id: string | null }>( + 'select id, entry_id, content_node_id from public.handbook_chapters where tenant_id = $1 and id = $2 limit 1', + [auth.tenantId, chapterId], + ); + if (!chapter.rows[0]) throw new HttpError(400, 'chapterId is not in this tenant', 'HANDBOOK_CHAPTER_NOT_FOUND'); + const navigation = await resolveOptionalContentNavigation( + client, auth.tenantId, - requiredString(body, 'chapterId'), - nullableString(body.legacyId), - requiredString(body, 'title'), - nullableString(body.summary), - nullableString(body.content), - jsonArrayValue(body.tags), - intValue(body.order, 0), - boolValue(body.isActive, true), - ], - ); + nullableString(body.entryId) || chapter.rows[0].entry_id, + nullableString(body.contentNodeId) || chapter.rows[0].content_node_id, + 'handbook', + ); + const result = await client.query( + ` + insert into public.handbook_entries ( + id, tenant_id, chapter_id, entry_id, content_node_id, legacy_id, title, + summary, content, tags, sort_order, is_active, metadata + ) + values (coalesce($1::uuid, gen_random_uuid()), $2, $3::uuid, $4::uuid, $5::uuid, $6, $7, $8, $9, $10::jsonb, $11, $12, $13::jsonb) + on conflict (id) + do update set chapter_id = excluded.chapter_id, + entry_id = excluded.entry_id, + content_node_id = excluded.content_node_id, + title = excluded.title, + summary = excluded.summary, + content = excluded.content, + tags = excluded.tags, + sort_order = excluded.sort_order, + is_active = excluded.is_active, + metadata = excluded.metadata, + updated_at = now() + returning id, chapter_id as "chapterId", entry_id as "entryId", + content_node_id as "contentNodeId", title, summary, content, + tags, sort_order as "order", is_active as "isActive", metadata, updated_at as "updatedAt" + `, + [ + nullableString(body.id), + auth.tenantId, + chapterId, + navigation.entryId, + navigation.contentNodeId, + nullableString(body.legacyId), + requiredString(body, 'title'), + nullableString(body.summary), + nullableString(body.content), + jsonArrayValue(body.tags), + intValue(body.order, 0), + boolValue(body.isActive, true), + jsonObjectValue(body.metadata), + ], + ); + return result.rows[0]; + }); return { item }; } diff --git a/docs/refactor/README.md b/docs/refactor/README.md index 82aa5e32..ccb359b2 100644 --- a/docs/refactor/README.md +++ b/docs/refactor/README.md @@ -18,11 +18,13 @@ - `scripts/import-pocketbase`:PocketBase schema/数据导入工具。 - `docker-compose.api.yml`、`apps/api/Dockerfile`:本地 Docker API 运行入口。 - `docs/refactor/architecture.md`:新重构目录边界和工程规范。 +- `docs/refactor/content-import-contract.md`:题目、单词、知识手册导入契约,明确后端校验、旧格式转换和前端职责。 +- `docs/refactor/next-development-todo.md`:后端剩余缺口、Taro 前端接入顺序、上云测试前待办。 下一步优先级: 1. 导出 PocketBase 真实数据到 `pb_export/*.json`。 2. 执行 `npm run pb:import:json` 和 `npm run pb:import:validate`。 3. 按学生端页面逐步从 PocketBase SDK 切换到 `src/services/supabaseApi.ts`。 -4. 为订单、支付、权益开通补齐 API 写入流程和 webhook 幂等处理。 +4. 为分数线、视频、Excel/CSV 补齐批量导入,并复用 `content_import_jobs` 管线。 5. 新建 Taro 学生端时复用同一套租户解析和业务 API,不另起一套后端。 diff --git a/docs/refactor/backend-progress.md b/docs/refactor/backend-progress.md index 86c2f990..68a66f85 100644 --- a/docs/refactor/backend-progress.md +++ b/docs/refactor/backend-progress.md @@ -15,7 +15,7 @@ - `referral`:销售/代理邀请码、首绑客资保护、销售统计、团队关系、CRM 队列。 - `platform-admin`:平台方租户管理、SaaS 套餐、订阅、账单、服务费收款、使用量。 - `tenant-admin`:租户资料、品牌、公开设置、域名、支付账户、登录 provider、私密密钥掩码、活动内容、激活码批次、优惠券、成员管理、权限矩阵、审计查询。 - - `tenant-content`:租户后台内容入口、任意深度分类树、考试意向标记、题目集合、练习蓝图、题目、视频、分数线、单词、知识手册、资料资源、题目 JSON 导入维护。 + - `tenant-content`:租户后台内容入口、任意深度分类树、考试意向标记、题目集合、练习蓝图、题目、视频、分数线、单词、知识手册、资料资源、题目/单词/知识手册 JSON 导入维护。 - `tenant`:域名/租户解析。 - `src/services/supabaseApi.ts` 已加入新 API 客户端方法,供旧 Web 逐步替换和后续 Taro 复用。 - 已新增 `npm run db:smoke-seed`,用于 `supabase:reset` 后恢复最小烟测数据。 @@ -93,6 +93,10 @@ POST /api/tenant-content/assets/sign-upload POST /api/tenant-content/assets/sign-download POST /api/tenant-content/imports/preview/questions POST /api/tenant-content/imports/questions +POST /api/tenant-content/imports/preview/vocabulary +POST /api/tenant-content/imports/vocabulary +POST /api/tenant-content/imports/preview/handbook +POST /api/tenant-content/imports/handbook GET /api/tenant-content/imports GET /api/tenant-content/imports/issues PUT /api/tenant-content/videos @@ -173,11 +177,11 @@ GET /api/tenant-admin/audit-logs - CRM 当前完成配置、密钥入私密表、客资入队和队列查询;真实 webhook 发送、重试、签名在后续 `apps/worker` 中实现。 - 内容资源当前完成台账、租户后台维护、上传/下载签名占位和学生端 SVIP 下载权限;真实对象存储签名、PDF 预览渲染和防盗链在 provider/worker 中实现。 - 题库内容导航当前以 `content_entries/content_nodes` 为主模型,可表达“入口 -> 多级分类 -> 院校/专业/学科/销售意向标记”;题目集合和练习方式由 `question_collections/practice_blueprints` 管理,练习 session 会保存当次题目 ID 快照。 -- 题目批量导入当前支持 JSON 数组预览、逐行 issue、job/item 台账、执行导入、幂等跳过,并可落到新内容入口、分类节点和题目集合;Excel/CSV、单词/手册/分数线导入会复用同一套 `content_import_jobs` 管线。 +- 批量导入当前支持题目、单词、知识手册 JSON 预览、逐行 issue、job/item 台账、执行导入、幂等跳过,并可落到新内容入口和分类节点。旧单词模板的 `vocabulary_units_示例数据` / `vocabulary_示例数据`、知识手册的书籍/章节/小节/知识点嵌套结构都由后端规范化。Excel/CSV、分数线/视频导入会继续复用同一套 `content_import_jobs` 管线。 ## 下一步 -1. 完善内容导入和文件上传:Excel/CSV、单词、手册、分数线、视频导入,真实 OSS/COS/Supabase Storage 签名。 +1. 完善内容导入和文件上传:Excel/CSV、分数线、视频导入,真实 OSS/COS/Supabase Storage 签名。 2. 接入真实短信 provider:阿里云/腾讯云,密钥放 `app_private.tenant_secrets` 或生产 Vault。 3. 接入真实 OAuth provider:微信网页、微信小程序、QQ,并处理旧 PocketBase 身份映射。 4. 增加真实支付 provider:XPay、微信支付、支付宝,并完善 webhook 幂等。 diff --git a/docs/refactor/blueprint-coverage.md b/docs/refactor/blueprint-coverage.md index c567ad05..dd6ab496 100644 --- a/docs/refactor/blueprint-coverage.md +++ b/docs/refactor/blueprint-coverage.md @@ -18,10 +18,10 @@ | 平台超级管理员 | 部分完成 | 租户管理、SaaS 套餐、订阅、账单、服务费收款、用量记录 | 公共题库披露策略、地区/全国套餐权限、平台侧主题模板库、平台审计 | | 租户品牌和域名 | 基础完成 | 品牌、Logo、主题 JSON、公开资源、域名、租户公开配置 | 三套默认主题、主题可视化编辑、图标/图片上传 | | 租户成员权限 | 基础完成 | owner/admin/operator/teacher/sales/agent/student,权限矩阵,成员启停,审计查询 | 前端权限 UI、自定义角色模板、菜单级可见配置 | -| 题库内容维护 | 基础完成 | 内容入口、任意深度分类树、院校/专业/学科/销售意向标记、题目集合、顺序/随机/全真模拟练习蓝图、题目录入/更新、题目 JSON 预览/导入、视频绑定、分数线、单词、知识手册后台 API | Excel/CSV 批量导入、公题库采纳/复制/授权、可视化拖拽排序前端 | +| 题库内容维护 | 基础完成 | 内容入口、任意深度分类树、院校/专业/学科/销售意向标记、题目集合、顺序/随机/全真模拟练习蓝图、题目录入/更新、题目 JSON 预览/导入、视频绑定、分数线、单词、知识手册后台 API | Excel/CSV 批量导入、分数线/视频导入、公题库采纳/复制/授权、可视化拖拽排序前端 | | 学生刷题 | 基础完成 | 内容入口、分类树、题目集合、顺序刷题、随机刷题、全真模拟 session 题目快照、答题、错题本、收藏夹 | 完整模考交卷评分报告、专项练习策略、错题复习计划、题型统计深度分析 | -| 背单词 | 基础完成 | 单词单元、单词、进度、收藏、统计 | 复习算法、每日计划、排行榜 | -| 知识手册 | 基础完成 | 科目、章节、条目只读与后台维护 | 富文本资源、版本管理、附件/PDF 关联 | +| 背单词 | 基础完成 | 单词单元、单词、进度、收藏、统计、旧模板/新模板 JSON 预览导入、内容导航绑定 | 复习算法、每日计划、排行榜、Excel 导入 | +| 知识手册 | 基础完成 | 科目、章节、条目只读与后台维护、书籍/章节/小节/知识点嵌套 JSON 预览导入、内容导航绑定 | 富文本资源、版本管理、附件/PDF 关联、Excel/Markdown 批量解析 | | 分数线 | 基础完成 | 字段、院校、专业、记录、趋势、年份 | 复杂动态筛选、批量导入、AI 择校数据上下文 | | 视频解析会员 | 部分完成 | 题目视频、批量查询、后台绑定 | SVIP 权限、播放次数扣减、签名 URL、防盗链、水印、播放统计 | | 资料下载/PDF | 基础完成 | `content_assets` 资源台账、后台资源管理、上传/下载签名占位、学生端列表、SVIP 下载权限 | 真实 OSS/COS/Supabase Storage 签名、PDF 预览渲染、防盗链、资料前端管理页 | @@ -36,7 +36,7 @@ ## 接下来优先级 -1. 完善内容导入和对象存储:Excel/CSV、单词/手册/分数线/视频导入,真实 OSS/COS/Supabase Storage 签名。 +1. 完善内容导入和对象存储:Excel/CSV、分数线/视频导入,真实 OSS/COS/Supabase Storage 签名,JSON 导入异步化。 2. 公共题库/地区题库授权:平台题库向租户披露、租户采纳、按 SaaS 套餐限制地区。 3. 完整模考与学习统计:交卷、评分报告、练习历史、正确率趋势、错题复习计划。 4. 视频会员控制:视频资源签名 URL、防盗链、水印、播放次数和会员权益。 diff --git a/docs/refactor/content-import-contract.md b/docs/refactor/content-import-contract.md new file mode 100644 index 00000000..0143a9c0 --- /dev/null +++ b/docs/refactor/content-import-contract.md @@ -0,0 +1,158 @@ +# 内容导入契约 + +更新时间:2026-06-21 + +## 结论 + +内容导入的最终规范化、校验、租户隔离、幂等和审计必须由后端负责。 + +前端只负责: + +- 上传或粘贴 JSON/Excel/CSV。 +- 做轻量格式预检查,减少明显错误。 +- 展示后端 preview 返回的 `job/items/issues`。 +- 让运营人员修正数据后再确认导入。 + +迁移脚本只负责: + +- 从 PocketBase 导出或旧 JSON 中抽取数据。 +- 转换成新架构推荐格式。 +- 调用后端 preview/import API。 + +不建议迁移脚本直接绕过后端写业务表,除非是一次性内控迁移,并且必须额外跑导入后校验。 + +## 已实现导入 API + +```text +POST /api/tenant-content/imports/preview/questions +POST /api/tenant-content/imports/questions + +POST /api/tenant-content/imports/preview/vocabulary +POST /api/tenant-content/imports/vocabulary + +POST /api/tenant-content/imports/preview/handbook +POST /api/tenant-content/imports/handbook + +GET /api/tenant-content/imports +GET /api/tenant-content/imports/issues +``` + +所有导入都会写入: + +- `content_import_jobs` +- `content_import_items` +- `content_import_issues` +- `audit_logs` + +## 单词导入 + +推荐新格式: + +```json +{ + "regionId": "uuid", + "entryId": "uuid", + "contentNodeId": "uuid", + "units": [ + { + "legacyId": "unit-1", + "name": "Unit 1 - 高频核心词", + "description": "专升本考试高频词汇", + "order": 1, + "words": [ + { + "legacyId": "word-abandon", + "word": "abandon", + "phonetic": "/əˈbændən/", + "meaning": "v. 放弃,抛弃", + "example": "He had to abandon his car in the snow.", + "exampleTranslation": "他不得不把车丢弃在雪地里。", + "difficulty": 3, + "tags": ["高频词"], + "order": 1 + } + ] + } + ] +} +``` + +兼容旧模板: + +- `vocabulary_units_示例数据` +- `vocabulary_示例数据` + +后端会把旧模板归一化为 `vocabulary_units/vocabulary_words`,并可自动挂到 `content_entries/content_nodes`。 + +## 知识手册导入 + +推荐新格式: + +```json +{ + "regionId": "uuid", + "entryId": "uuid", + "contentNodeId": "uuid", + "subjects": [ + { + "legacyId": "handbook-chinese", + "name": "大学语文", + "type": "guide", + "chapters": [ + { + "legacyId": "chapter-outline", + "name": "一、语文考纲", + "sections": [ + { + "legacyId": "section-outline", + "name": "考纲解读", + "entries": [ + { + "legacyId": "entry-outline", + "title": "2024年天津专升本语文考试大纲", + "summary": "全面解读语文考试要求", + "content": "Markdown 内容", + "tags": ["考纲"] + } + ] + } + ] + } + ] + } + ] +} +``` + +映射规则: + +- 手册入口:`content_entries.entry_type = handbook` +- 书籍/科目:`handbook_subjects`,并生成或绑定一个 `content_nodes` +- 章节:`handbook_chapters`,并生成章节节点 +- 小节:默认进入 `content_nodes`,作为知识点的目录节点 +- 知识点:`handbook_entries`,通过 `content_node_id` 归属到小节或章节 + +## 题目导入 + +题目继续兼容旧题库 JSON 数组格式,并支持 Markdown、KaTeX、图片、表格、阅读理解子题等字段。导入时可以传: + +- `subjectId` +- `categoryId` +- `entryId` +- `contentNodeId` +- `collectionId` + +这样题目会同时落到旧兼容表和新内容导航/题目集合。 + +## 幂等规则 + +- 优先使用 `legacyId` 作为跨迁移稳定标识。 +- 没有 `legacyId` 时,后端按导入 job 和行号生成内部标识。 +- 相同 `legacyId` 再导入会更新。 +- 内容 hash 未变化时标记为 `skipped`。 + +## 下一步 + +- 增加 Excel/CSV 解析入口,但解析后仍进入同一套 preview/import 管线。 +- 增加分数线、视频导入。 +- 增加异步 worker,处理大批量导入、重试和导入后校验。 diff --git a/docs/refactor/implementation-status.md b/docs/refactor/implementation-status.md index cfa8c7dc..662ef8d7 100644 --- a/docs/refactor/implementation-status.md +++ b/docs/refactor/implementation-status.md @@ -4,7 +4,7 @@ ## 当前结论 -当前重构已经完成了 Supabase/PostgreSQL 多租户底座、核心业务表、PocketBase 数据导入器雏形、学生端核心 API、租户后台 API、平台后台 SaaS 账务 API、内容资产/题目 JSON 批量导入基础闭环、题库入口/任意深度分类/题目集合/练习蓝图/组卷快照基础闭环,以及本地 Docker/API 构建验证。 +当前重构已经完成了 Supabase/PostgreSQL 多租户底座、核心业务表、PocketBase 数据导入器雏形、学生端核心 API、租户后台 API、平台后台 SaaS 账务 API、内容资产/题目/单词/知识手册 JSON 批量导入基础闭环、题库入口/任意深度分类/题目集合/练习蓝图/组卷快照基础闭环,以及本地 Docker/API 构建验证。 但这还不是完整商用交付状态,也不能说旧项目核心功能已经全部重构完成。现在更准确的状态是:后端商用架构骨架已经立住,核心业务正在按模块补齐。部分功能已经有可调用 API,部分功能只有数据模型和导入映射,部分功能还没有前端/自动化测试闭环。 @@ -28,8 +28,8 @@ | 错题本 | 已建 `wrong_questions` | 已支持旧错题归一化 | 错题列表、答题自动入错题、移出错题已实现 | 仅烟测 | 基础功能已实现,复习计划和统计未完成 | | 收藏夹 | 已建 `favorite_questions` | 已支持旧收藏归一化 | 收藏/取消收藏、收藏列表已实现 | 仅烟测 | 基础功能已实现 | | 用户订阅/题库会员/SVIP | 已建 `orders`、`payments`、`entitlements`、`svip_plans`、激活码 | 已映射旧 SVIP/会员权益 | 下单、手动支付确认、激活码兑换、权益查询已实现 | 仅烟测 | 业务骨架可跑,真实微信/支付宝支付和 webhook 未完成 | -| 背单词 | 已建单词单元、单词、进度、收藏表 | 已支持内容和部分用户状态映射 | 单元/单词只读、进度、收藏、统计、租户后台单词维护 API 已实现 | 核心 API 集成测试 | 学生端基础学习状态和后台单词维护已实现,复习算法和后台统计待完善 | -| 知识手册 | 已建手册科目、章节、条目 | 已支持内容导入 | 只读 API、租户后台手册科目/章节/条目维护 API 已实现 | 核心 API 集成测试 | 学生端阅读和后台维护基础可用,富文本资源/版本管理待补 | +| 背单词 | 已建单词单元、单词、进度、收藏表,并可绑定 `content_entries/content_nodes` | 已支持内容和部分用户状态映射 | 单元/单词只读、进度、收藏、统计、租户后台单词维护 API、旧模板/新模板 JSON 预览导入已实现 | 核心 API 集成测试含导入断言 | 学生端基础学习状态、后台维护和批量 JSON 导入已实现,复习算法和后台统计待完善 | +| 知识手册 | 已建手册科目、章节、条目,并可绑定 `content_entries/content_nodes` | 已支持内容导入 | 只读 API、租户后台手册科目/章节/条目维护 API、嵌套 JSON 预览导入已实现 | 核心 API 集成测试含导入断言 | 学生端阅读、后台维护和批量 JSON 导入基础可用,富文本资源/版本管理待补 | | 分数线 | 已建院校、专业、字段、记录表 | 已支持导入映射 | 字段、院校、专业、记录、趋势、年份、租户后台维护 API 已实现 | 核心 API 集成测试 | 查询和后台维护基础闭环已实现,复杂动态筛选/批量导入待补 | | 题目视频讲解 | 已建 `video_explanations`、`question_videos` | 已支持导入映射 | 单题视频、批量预加载、通用视频搜索、租户后台视频创建绑定 API 已实现 | 核心 API 集成测试 | 播放数据和后台绑定链路已实现,会员权限、签名 URL、播放统计待补 | | 资料下载/PDF | 已扩展 `content_assets`,新增资源台账和导入任务表 | 旧 `app_assets/images` 兼容导入 | 租户后台资源管理、上传/下载签名占位、学生端资料列表/下载权限已实现 | 核心 API 集成测试含 SVIP 资料下载 | 资料资源基础闭环可跑,真实 OSS/COS 签名、PDF 预览渲染、资料下载前端待补 | @@ -134,6 +134,10 @@ tenant-content: POST /api/tenant-content/assets/sign-download POST /api/tenant-content/imports/preview/questions POST /api/tenant-content/imports/questions + POST /api/tenant-content/imports/preview/vocabulary + POST /api/tenant-content/imports/vocabulary + POST /api/tenant-content/imports/preview/handbook + POST /api/tenant-content/imports/handbook GET /api/tenant-content/imports GET /api/tenant-content/imports/issues GET /api/tenant-content/videos @@ -237,7 +241,7 @@ platform-admin: 1. 正式鉴权:迁移期 `x-tenant-id`、`x-user-id`、`x-platform-admin-key` 要替换为 Supabase Auth/JWT/服务端 session,并逐表验证 RLS。 2. 国内能力接入:短信、微信登录、微信小程序登录、QQ 登录、微信支付、支付宝支付的租户级配置入口已具备,但真实 provider adapter、回调验签和 webhook 幂等仍需实现。 3. 核心缺口 API:学生端个人中心、分数线、题目视频详情、背单词进度/收藏已补基础 API;下一步重点是后台维护、权限、统计和真实业务验收。 -4. 后台能力:题库录入、JSON 批量导入、资源台账、视频绑定、知识手册维护、分数线维护、品牌/商户/登录/活动/兑换码配置、销售客资、CRM 队列、成员权限、审计查询已补 API;Excel 导入、真实对象存储签名和前端操作台待补。 +4. 后台能力:题库录入、题目/单词/知识手册 JSON 批量导入、资源台账、视频绑定、知识手册维护、分数线维护、品牌/商户/登录/活动/兑换码配置、销售客资、CRM 队列、成员权限、审计查询已补 API;Excel 导入、分数线/视频导入、真实对象存储签名和前端操作台待补。 5. 自动化测试:已建立核心 API、租户隔离、权限矩阵、后台维护、资源/导入集成测试;仍需真实数据导入回归、支付幂等、前端端到端测试。 6. Taro 前端:建立 `apps/taro` 或等价跨端应用,把 H5 和小程序统一走同一套 API client。 7. 运维交付:生产环境变量、备份恢复、日志监控、异常告警、数据库迁移流程、灰度发布、回滚预案。 @@ -246,7 +250,7 @@ platform-admin: 为了先把旧项目核心业务补齐,再进入支付/短信等商用关键模块,建议按下面顺序继续: -1. 完善内容导入和文件上传:Excel/CSV、单词、手册、分数线、视频导入,接真实 OSS/COS/Supabase Storage 签名。 +1. 完善内容导入和文件上传:Excel/CSV、分数线、视频导入,接真实 OSS/COS/Supabase Storage 签名,并把 JSON 导入扩展为异步 worker。 2. 补地区/公共题库披露策略、租户套餐地区限制、主题模板系统。 3. 补学习统计:练习历史、正确率趋势、错题复习计划、单词复习算法。 4. 补视频商用控制:SVIP 权限、签名 URL、防盗链、水印、播放次数扣减。 diff --git a/docs/refactor/next-development-todo.md b/docs/refactor/next-development-todo.md new file mode 100644 index 00000000..b262d843 --- /dev/null +++ b/docs/refactor/next-development-todo.md @@ -0,0 +1,170 @@ +# 后续开发 TODO + +更新时间:2026-06-22 + +## 当前后端基线 + +后端已经完成可本地验证的商用 SaaS 骨架: + +- Supabase/PostgreSQL 多租户 schema、RLS、索引、触发器。 +- Node.js API 分层:`core/features`。 +- 学生端核心 API:题库、练习、答题、错题、收藏、背单词、知识手册、分数线、视频、资料、订单、权益、个人中心。 +- 租户后台 API:品牌、域名、设置、支付账户、登录 provider、私密密钥、活动、激活码、优惠券、成员权限、审计、内容管理。 +- 平台后台 API:租户、SaaS 套餐、订阅、账单、服务费收款、用量。 +- 销售/代理/CRM 增长链路:邀请码、扫码事件、首绑保护、团队、统计、CRM 队列。 +- 内容导航:`content_entries/content_nodes` 支持任意深度入口和分类。 +- 练习组卷:`question_collections/practice_blueprints` 支持顺序、随机、全真模拟快照。 +- 内容导入:题目、单词、知识手册 JSON 预览、校验、导入、幂等、审计。 +- 本地验证:`npm run check:refactor` 已通过。 + +## 后端待补功能 + +### P0 上云测试前必须补齐 + +1. 生产鉴权 + - 用 Supabase Auth/JWT 或服务端 session 替换迁移期 `x-tenant-id`、`x-user-id`、`x-platform-admin-key`。 + - 校验平台管理员、租户管理员、运营、教师、销售、代理、学生的访问边界。 + - 做一轮真实 JWT + RLS 回归测试。 + +2. 对象存储 + - 接阿里云 OSS、腾讯云 COS 或 Supabase Storage。 + - 完成上传签名、下载签名、PDF 预览地址、视频播放签名。 + - `content_assets` 继续作为资源台账,不允许前端绕过台账直接访问私有资源。 + +3. 真实导入 dry-run + - 从 PocketBase 导出现有用户、题库、单词、知识手册、分数线、订单、权益数据。 + - 跑 `scripts/import-pocketbase`,生成迁移报告。 + - 对题目 JSON、单词、知识手册走后端 preview/import API 做二次验证。 + +4. 部署配置 + - 整理生产 `.env` 模板。 + - 确认数据库迁移流程、备份恢复、日志、告警。 + - 准备 API 容器部署和 Supabase 云端/自托管连接方案。 + +### P1 商用功能完善 + +1. 支付 + - 微信支付、支付宝、XPay 或实际使用的支付网关 adapter。 + - webhook 验签、幂等、退款、支付补偿任务。 + - 租户自有商户收款和平台代收/服务商模式。 + +2. 国内登录和短信 + - 阿里云短信、腾讯云短信 adapter。 + - 微信小程序登录、微信网页登录、QQ 登录。 + - 旧 PocketBase 用户账号和新身份体系的映射/补绑。 + +3. 导入体系扩展 + - Excel/CSV 导入。 + - 分数线批量导入。 + - 视频批量导入和题目视频批量绑定。 + - 大批量导入异步 worker、重试、导入后校验。 + +4. 公共题库和租户授权 + - 平台公共题库/地区题库。 + - 按 SaaS 套餐限制地区、科目、题库范围。 + - 租户采纳、复制、授权、版本同步策略。 + +5. 视频会员控制 + - 视频 SVIP 权限、播放次数扣减。 + - 防盗链、水印、播放日志、播放统计。 + - 单题视频和通用知识视频混合推荐。 + +6. 学习统计 + - 练习历史、正确率趋势、题型分布、错题复习计划。 + - 单词复习算法、每日计划、排行榜。 + - 模考交卷、评分报告、错题解析汇总。 + +7. 数据看板 + - 收益、注册趋势、答题次数、收入趋势、题型分布、科目数量、题目总量。 + - 套餐销量、运营动态、24h 活跃度、激活码使用情况。 + - 销售/代理转化、分佣结算、客资跟进效果。 + +8. AI 择校推荐 + - 地区考试数据上下文。 + - 学生输入 schema。 + - AI 返回 JSON schema。 + - 报告渲染和 PDF 生成。 + +### P2 运营体验和企业交付 + +1. 自定义角色 + - 租户内角色模板。 + - 菜单可见、模块可见、字段级权限。 + - 权限变更审计。 + +2. 主题系统 + - 平台默认三套主题。 + - 租户自定义主色、Logo、图标、启动图、小程序分享图。 + - 主题预览和发布。 + +3. CRM worker + - 钉钉、飞书、企业微信机器人 adapter。 + - 轮询/定向分配。 + - 推送失败重试和签名。 + +4. 运维 + - 后台操作审计报表。 + - 定时备份、恢复演练。 + - 性能压测、慢 SQL、索引审查。 + +## Taro 前端开发 TODO + +### 架构目标 + +- 建议新建 `apps/taro`,不要继续在旧 React Web 上堆兼容。 +- H5 和小程序共用同一套业务 API client。 +- 租户通过域名、小程序配置或启动参数解析。 +- 页面主题、品牌、功能开关都从后端租户配置读取。 + +### 第一批页面 + +1. 租户启动与首页 + - 调 `/api/tenant/resolve` + - 读取品牌、主题、Banner、公告、功能开关 + - 展示题库入口、背单词、知识手册、分数线、资料、会员 + +2. 登录 + - 迁移期可先接短信 mock 或临时登录 + - 生产接微信小程序登录、短信登录、QQ/微信网页登录 + +3. 题库 + - `content_entries/content_nodes` + - `question_collections/practice_blueprints` + - 顺序刷题、随机刷题、全真模拟 + - 答题、解析、错题、收藏、视频解析入口 + +4. 背单词 + - 单元列表、单词列表 + - 学习状态、收藏、统计 + - 后续补复习算法 + +5. 知识手册 + - 手册入口、章节、小节、知识点阅读 + - Markdown/公式/图片基础渲染 + +6. 分数线 + - 院校、专业、年份、动态字段筛选 + - 趋势图 + +7. 资料下载 + - PDF 列表、权限提示、预览、下载 + +8. 个人中心 + - 会员权益、订单、激活码兑换 + - 错题本、收藏夹、学习统计 + +### 前端接入原则 + +- 不在前端实现最终权限判断,前端只做 UI 可见性控制。 +- 不在前端直接拼接私有资源 URL,统一请求后端签名。 +- 不把旧 PocketBase 字段当成长期模型,优先使用新 API 返回的 `entryId/contentNodeId/collectionId/blueprintId`。 +- 小程序码、支付、登录等平台能力统一走 provider adapter,不在页面里硬编码租户密钥。 + +## 推荐下一步顺序 + +1. 先把当前后端代码推到 Gitea。 +2. 云服务器部署 Supabase/PostgreSQL 和 API,跑 `check:refactor` 的远程等价测试。 +3. 导出现有 PocketBase 数据,做完整 dry-run 迁移。 +4. 开始 `apps/taro`,先接租户解析、首页、题库、背单词、知识手册。 +5. 并行补对象存储、真实登录、支付 adapter。 +6. 前后端联调通过后,再做支付、权限、数据导入、资料下载、视频播放的商用验收。 diff --git a/scripts/api-integration-test.js b/scripts/api-integration-test.js index 47d086dc..050bc704 100644 --- a/scripts/api-integration-test.js +++ b/scripts/api-integration-test.js @@ -802,6 +802,231 @@ async function testTenantContentAssetsAndImports() { 'catalog should expose imported question through the new collection binding', ); + const vocabEntry = await request('/api/tenant-content/content-entries', { + userId: TENANT_ADMIN_USER_ID, + method: 'PUT', + body: { + entryKey: 'integration-vocabulary', + regionId: ids.region, + name: '集成测试背单词入口', + entryType: 'vocabulary', + route: '/vocabulary', + order: 21, + }, + }); + const vocabRoot = await request('/api/tenant-content/content-nodes', { + userId: TENANT_ADMIN_USER_ID, + method: 'PUT', + body: { + entryId: vocabEntry.item.id, + regionId: ids.region, + nodeKey: 'integration-vocabulary-root', + name: '英语核心词', + nodeType: 'category', + isLeaf: false, + }, + }); + + const deniedVocabularyPreview = await request('/api/tenant-content/imports/preview/vocabulary', { + method: 'POST', + body: { + units: [{ name: '学生不能导入单词', words: [{ word: 'deny', meaning: '拒绝' }] }], + }, + expectStatus: 403, + }); + assert.equal(deniedVocabularyPreview.code, 'TENANT_CONTENT_EDITOR_REQUIRED', 'student should not preview vocabulary import'); + + const invalidVocabularyPreview = await request('/api/tenant-content/imports/preview/vocabulary', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { + sourceName: 'invalid-vocabulary-import.json', + entryId: vocabEntry.item.id, + contentNodeId: vocabRoot.item.id, + vocabulary_units_示例数据: [ + { legacyId: 'integration-vocab-invalid-unit', name: '旧格式错误单词单元', order: 1 }, + ], + vocabulary_示例数据: [ + { unitId: 'integration-vocab-invalid-unit', word: '', meaning: '', order: 1 }, + ], + }, + }); + assert.ok(invalidVocabularyPreview.issues?.some(issue => issue.code === 'VOCABULARY_WORD_REQUIRED'), 'invalid vocabulary preview should validate word'); + assert.ok(invalidVocabularyPreview.issues?.some(issue => issue.code === 'VOCABULARY_MEANING_REQUIRED'), 'invalid vocabulary preview should validate meaning'); + + const rejectedVocabularyImport = await request('/api/tenant-content/imports/vocabulary', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { previewJobId: invalidVocabularyPreview.job.id }, + expectStatus: 409, + }); + assert.equal(rejectedVocabularyImport.code, 'IMPORT_HAS_ERRORS', 'invalid vocabulary import should be rejected'); + + const vocabularyPreview = await request('/api/tenant-content/imports/preview/vocabulary', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { + sourceName: 'legacy-vocabulary-import.json', + regionId: ids.region, + entryId: vocabEntry.item.id, + contentNodeId: vocabRoot.item.id, + vocabulary_units_示例数据: [ + { + legacyId: 'integration-vocab-unit-001', + name: 'Unit 1 - 高频核心词', + description: '旧模板字段导入验证', + order: 1, + isActive: true, + }, + ], + vocabulary_示例数据: [ + { + legacyId: 'integration-vocab-word-abandon', + unitId: 'integration-vocab-unit-001', + word: 'abandon', + phonetic: '/əˈbændən/', + meaning: 'v. 放弃,抛弃', + example: 'He had to abandon his car in the snow.', + exampleTranslation: '他不得不把车丢弃在雪地里。', + difficulty: 3, + tags: ['高频词', '考纲核心'], + order: 1, + isActive: true, + }, + ], + }, + }); + assert.equal(vocabularyPreview.job?.errorCount, 0, 'valid vocabulary preview should have no errors'); + + const vocabularyImport = await request('/api/tenant-content/imports/vocabulary', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { previewJobId: vocabularyPreview.job.id }, + }); + assert.equal(vocabularyImport.item?.status, 'completed', 'valid vocabulary import should complete'); + assert.ok( + (vocabularyImport.item?.insertedCount || 0) + (vocabularyImport.item?.updatedCount || 0) + (vocabularyImport.item?.skippedCount || 0) >= 2, + 'vocabulary import should process unit and word idempotently', + ); + + const vocabularyUnits = await request('/api/catalog/vocabulary-units', { + query: { regionId: ids.region }, + }); + const importedVocabularyUnit = vocabularyUnits.items?.find(item => item.legacyId === 'integration-vocab-unit-001'); + assert.ok(importedVocabularyUnit, 'catalog should expose imported vocabulary unit'); + assert.equal(importedVocabularyUnit.entryId, vocabEntry.item.id, 'vocabulary unit should bind content entry'); + + const vocabularyWords = await request('/api/catalog/vocabulary-words', { + query: { unitId: importedVocabularyUnit.id }, + }); + assert.ok(vocabularyWords.items?.some(item => item.word === 'abandon' && item.contentNodeId), 'catalog should expose imported vocabulary word with node binding'); + + const handbookEntry = await request('/api/tenant-content/content-entries', { + userId: TENANT_ADMIN_USER_ID, + method: 'PUT', + body: { + entryKey: 'integration-handbook', + regionId: ids.region, + name: '集成测试知识手册入口', + entryType: 'handbook', + route: '/handbook', + order: 22, + }, + }); + const handbookRoot = await request('/api/tenant-content/content-nodes', { + userId: TENANT_ADMIN_USER_ID, + method: 'PUT', + body: { + entryId: handbookEntry.item.id, + regionId: ids.region, + nodeKey: 'integration-handbook-root', + name: '文化课', + nodeType: 'category', + isLeaf: false, + }, + }); + + const handbookPreview = await request('/api/tenant-content/imports/preview/handbook', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { + sourceName: 'handbook-nested-import.json', + regionId: ids.region, + entryId: handbookEntry.item.id, + contentNodeId: handbookRoot.item.id, + subjects: [ + { + legacyId: 'integration-handbook-chinese', + name: '大学语文', + type: 'guide', + icon: 'book-open', + color: '#10B981', + chapters: [ + { + legacyId: 'integration-handbook-chapter-outline', + name: '一、语文考纲', + sections: [ + { + legacyId: 'integration-handbook-section-outline', + name: '考纲解读', + entries: [ + { + legacyId: 'integration-handbook-entry-outline', + title: '2024年天津专升本语文考试大纲', + summary: '全面解读语文考试要求和考点分布', + content: '一、考试性质\n\n天津市高职升本科招生统一考试是选拔性考试。', + tags: ['考纲', '必读'], + order: 1, + }, + ], + }, + ], + }, + ], + }, + ], + }, + }); + assert.equal(handbookPreview.job?.errorCount, 0, 'valid handbook preview should have no errors'); + + const handbookImport = await request('/api/tenant-content/imports/handbook', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { previewJobId: handbookPreview.job.id }, + }); + assert.equal(handbookImport.item?.status, 'completed', 'valid handbook import should complete'); + assert.ok( + (handbookImport.item?.insertedCount || 0) + (handbookImport.item?.updatedCount || 0) + (handbookImport.item?.skippedCount || 0) >= 3, + 'handbook import should process subject, chapter, and entry idempotently', + ); + + const handbookSubjects = await request('/api/catalog/handbook-subjects', { + query: { regionId: ids.region }, + }); + const importedHandbookSubject = handbookSubjects.items?.find(item => item.legacyId === 'integration-handbook-chinese'); + assert.ok(importedHandbookSubject, 'catalog should expose imported handbook subject'); + assert.equal(importedHandbookSubject.entryId, handbookEntry.item.id, 'handbook subject should bind content entry'); + + const handbookChapters = await request('/api/catalog/handbook-chapters', { + query: { subjectId: importedHandbookSubject.id }, + }); + const importedHandbookChapter = handbookChapters.items?.find(item => item.legacyId === 'integration-handbook-chapter-outline'); + assert.ok(importedHandbookChapter, 'catalog should expose imported handbook chapter'); + + const handbookEntries = await request('/api/catalog/handbook-entries', { + query: { chapterId: importedHandbookChapter.id, includeContent: true }, + }); + assert.ok( + handbookEntries.items?.some(item => item.legacyId === 'integration-handbook-entry-outline' && item.content?.includes('选拔性考试')), + 'catalog should expose imported handbook entry content', + ); + + const importJobs = await request('/api/tenant-content/imports', { + userId: TENANT_ADMIN_USER_ID, + query: { importType: 'vocabulary', limit: 10 }, + }); + assert.ok(importJobs.items?.some(item => item.id === vocabularyPreview.job.id && item.status === 'completed'), 'import job list should include completed vocabulary job'); + const partnerImports = await request('/api/tenant-content/imports', { tenantId: PARTNER_TENANT_ID, userId: TENANT_ADMIN_USER_ID, diff --git a/supabase/migrations/202606210009_content_import_vocabulary_handbook.sql b/supabase/migrations/202606210009_content_import_vocabulary_handbook.sql new file mode 100644 index 00000000..34b483c7 --- /dev/null +++ b/supabase/migrations/202606210009_content_import_vocabulary_handbook.sql @@ -0,0 +1,39 @@ +alter table public.vocabulary_units + add column if not exists entry_id uuid references public.content_entries(id) on delete set null, + add column if not exists content_node_id uuid references public.content_nodes(id) on delete set null, + add column if not exists source_hash text, + add column if not exists metadata jsonb not null default '{}'::jsonb; + +alter table public.vocabulary_words + add column if not exists entry_id uuid references public.content_entries(id) on delete set null, + add column if not exists content_node_id uuid references public.content_nodes(id) on delete set null, + add column if not exists source_hash text, + add column if not exists metadata jsonb not null default '{}'::jsonb; + +alter table public.handbook_subjects + add column if not exists entry_id uuid references public.content_entries(id) on delete set null, + add column if not exists content_node_id uuid references public.content_nodes(id) on delete set null, + add column if not exists source_hash text; + +alter table public.handbook_chapters + add column if not exists entry_id uuid references public.content_entries(id) on delete set null, + add column if not exists content_node_id uuid references public.content_nodes(id) on delete set null, + add column if not exists source_hash text, + add column if not exists metadata jsonb not null default '{}'::jsonb; + +alter table public.handbook_entries + add column if not exists entry_id uuid references public.content_entries(id) on delete set null, + add column if not exists content_node_id uuid references public.content_nodes(id) on delete set null, + add column if not exists source_hash text, + add column if not exists metadata jsonb not null default '{}'::jsonb; + +create index if not exists idx_vocabulary_units_navigation + on public.vocabulary_units(tenant_id, entry_id, content_node_id, region_id, is_active, sort_order); +create index if not exists idx_vocabulary_words_navigation + on public.vocabulary_words(tenant_id, entry_id, content_node_id, unit_id, is_active, sort_order); +create index if not exists idx_handbook_subjects_navigation + on public.handbook_subjects(tenant_id, entry_id, content_node_id, region_id, is_active, sort_order); +create index if not exists idx_handbook_chapters_navigation + on public.handbook_chapters(tenant_id, entry_id, content_node_id, subject_id, is_active, sort_order); +create index if not exists idx_handbook_entries_navigation + on public.handbook_entries(tenant_id, entry_id, content_node_id, chapter_id, is_active, sort_order);