forked from wangziqi/gongxue-base
feat: add spreadsheet async imports
This commit is contained in:
15
README.md
15
README.md
@@ -13,7 +13,7 @@
|
||||
- Supabase/PostgreSQL 多租户数据库 schema、RLS、索引、触发器。
|
||||
- `apps/api` 独立业务 API,后续供 H5、Taro 小程序、管理后台统一调用;已支持 Supabase Auth JWT 和迁移期 `tk_` session 双入口。
|
||||
- 租户后台能力:品牌、域名、公开设置、支付账户、登录配置、私密密钥掩码、活动内容、考试日期、题目反馈处理、激活码、优惠券、勋章管理/发放、成员权限、自定义角色模板、班级/教师/学生范围权限、学生批量导入、批量分班、学生备注、跟进任务、审计日志。
|
||||
- 租户内容能力:可配置题库入口、任意深度分类树、考试意向标记、题目集合、顺序/随机/全真模拟蓝图、题目录入/更新、视频绑定、分数线、单词、知识手册、资料资源台账、题目/单词/知识手册/分数线/视频 JSON 批量导入。
|
||||
- 租户内容能力:可配置题库入口、任意深度分类树、考试意向标记、题目集合、顺序/随机/全真模拟蓝图、题目录入/更新、视频绑定、分数线、单词、知识手册、资料资源台账、题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 批量导入。
|
||||
- 学生端能力:题库入口、分类树、题目集合、顺序/随机/模考 session 组卷快照、答题、错题本、收藏夹、背单词进度、个人中心、勋章、考试倒计时、签到积分、题目反馈、排行榜、分数线、题目视频、订单详情/状态轮询、优惠券领取/抵扣、权益、激活码预检查/兑换、资料下载。
|
||||
- 平台后台能力:租户管理、SaaS 套餐、订阅、账单、服务费收款、用量记录。
|
||||
- 销售/代理/CRM 增长链路:邀请码、扫码/分享事件、首绑客资保护、销售统计、团队关系、CRM 配置和队列。
|
||||
@@ -28,7 +28,7 @@
|
||||
- Supabase Auth/JWT、租户角色模板、班级/教师/学生范围权限已可联调;生产前还要做真实云端 Auth/JWKS 回归和 RLS 深测。
|
||||
- 阿里云/腾讯云短信、微信小程序登录、微信支付、支付宝主链路、微信/支付宝发起退款/查询确认/退款通知、支付/退款补偿 worker 已完成本地适配;微信网页登录、QQ 登录、手机号换绑、完整资金流水对账和真实生产账号联调还没接完。
|
||||
- OSS/COS/Supabase Storage 上传下载签名 provider 已接入;上传后校验、PDF/图片预览和资源复检 worker 已完成,CDN 防盗链、杀毒扫描和视频动态水印还没完成。
|
||||
- Excel/CSV 导入和大批量异步导入 worker 还没完成;分数线/视频 JSON 批量导入已可联调。
|
||||
- Excel/CSV 导入解析已完成并复用 `content_import_jobs/items/issues` 管线;大批量异步导入 worker 基础已接入,支持 queued job 消费、重试和审计;导入后复检和模板下载/字段映射 UI 还没完成。
|
||||
- 勋章管理/手动发放已可联调;自动发放规则、积分活动联动、分佣真实打款、结算导出、发票/凭证、CRM 轮询/定向分配、富卡片模板、失败告警和销售转化看板还没完成。
|
||||
- Taro 跨端前端还没开始 scaffold。
|
||||
- 根目录已清理为新 Supabase SaaS monorepo 编排层;旧 PocketBase/React 项目和旧构建产物仅保留在 `参考/` 目录作为迁移参考,不进入 Git 提交。
|
||||
@@ -107,6 +107,12 @@ npm --workspace @tiku-saas/worker run commerce:once
|
||||
npm --workspace @tiku-saas/worker run assets:once
|
||||
```
|
||||
|
||||
单次运行内容导入 worker:
|
||||
|
||||
```bash
|
||||
npm --workspace @tiku-saas/worker run imports:once
|
||||
```
|
||||
|
||||
默认本地数据库:
|
||||
|
||||
```text
|
||||
@@ -147,6 +153,7 @@ npm run test:api
|
||||
npm run test:worker:crm
|
||||
npm run test:worker:commerce
|
||||
npm run test:worker:assets
|
||||
npm run test:worker:imports
|
||||
```
|
||||
|
||||
## API 模块
|
||||
@@ -186,7 +193,7 @@ API 身份上下文:
|
||||
- 商户密钥、短信密钥、OAuth app secret 等必须进入 `app_private.tenant_secrets`,或后续生产 KMS/Vault。
|
||||
- 资料、PDF、视频等资源必须先进入 `content_assets` 台账,再由 API 校验权限并下发签名 URL;生产环境应定时运行 assets worker 复检对象元数据,异常资源会被标记 failed 并退回 draft。
|
||||
- 题库入口和分类使用 `content_entries/content_nodes`;题目列表和练习规则使用 `question_collections/practice_blueprints`,前端不要再把旧树字段当成唯一业务结构。
|
||||
- 批量导入必须先写 `content_import_jobs/items/issues`,保留原始 payload、规范化 payload、逐行问题和审计记录。题目、单词、知识手册、分数线和视频 JSON 导入已走这套后台校验管线,前端只做预检查和预览展示。
|
||||
- 批量导入必须先写 `content_import_jobs/items/issues`,保留原始 payload、规范化 payload、逐行问题和审计记录。题目、单词、知识手册、分数线和视频 JSON/CSV/Excel 导入已走这套后台校验管线;大批量任务可提交 `executionMode=async`,由 imports worker 消费,前端只轮询 job 状态和展示 issues。
|
||||
- 支付 webhook 必须先设计幂等键和验签流程,再进入生产使用;生产环境还应定时运行 commerce worker 兜底供应商漏通知和处理中退款。
|
||||
|
||||
## 最近一次验证
|
||||
@@ -206,5 +213,5 @@ npm run check:refactor
|
||||
1. 真实云端 Auth/JWKS 回归、RLS 深测和生产环境配置验收。
|
||||
2. Taro 前端 scaffold,让 H5 和小程序共用同一套 API。
|
||||
3. 对象存储 CDN 防盗链、杀毒扫描、视频动态水印和生命周期策略。
|
||||
4. Excel/CSV 导入;把现有 JSON 导入升级为可排队异步执行。
|
||||
4. 导入后复检、题库导出 PDF/Word/JSON、模板下载和字段映射 UI。
|
||||
5. 微信网页/QQ 登录、完整资金流水对账、公共题库版本同步 worker、积分活动深化,以及排行榜防刷/预聚合。
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
"dependencies": {
|
||||
"@supabase/storage-js": "^2.108.2",
|
||||
"ali-oss": "^6.23.0",
|
||||
"exceljs": "^4.4.0",
|
||||
"jose": "^6.2.3",
|
||||
"pg": "^8.16.3"
|
||||
},
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import type pg from 'pg';
|
||||
import { createPool, query as runQuery, queryOne as runQueryOne } from '../../../../packages/db/src/index.js';
|
||||
import { config } from './config.js';
|
||||
import { DEFAULT_DATABASE_URL } from '../../../../packages/config/src/index.js';
|
||||
|
||||
export const pool = createPool({
|
||||
connectionString: config.databaseUrl,
|
||||
connectionString: process.env.DATABASE_URL || DEFAULT_DATABASE_URL,
|
||||
max: 10,
|
||||
});
|
||||
|
||||
|
||||
9
apps/api/src/core/errors.ts
Normal file
9
apps/api/src/core/errors.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
export class HttpError extends Error {
|
||||
constructor(
|
||||
public readonly statusCode: number,
|
||||
message: string,
|
||||
public readonly code = 'HTTP_ERROR',
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
import type { IncomingMessage, ServerResponse } from 'node:http';
|
||||
import { URL } from 'node:url';
|
||||
import { config } from './config.js';
|
||||
import { HttpError } from './errors.js';
|
||||
|
||||
export { HttpError } from './errors.js';
|
||||
|
||||
export interface RequestContext {
|
||||
req: IncomingMessage;
|
||||
@@ -10,16 +13,6 @@ export interface RequestContext {
|
||||
|
||||
export type Handler = (ctx: RequestContext) => Promise<unknown>;
|
||||
|
||||
export class HttpError extends Error {
|
||||
constructor(
|
||||
public readonly statusCode: number,
|
||||
message: string,
|
||||
public readonly code = 'HTTP_ERROR',
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export function sendJson(res: ServerResponse, statusCode: number, body: unknown) {
|
||||
res.statusCode = statusCode;
|
||||
res.setHeader('content-type', 'application/json; charset=utf-8');
|
||||
|
||||
739
apps/api/src/features/tenant-content/import-spreadsheet.ts
Normal file
739
apps/api/src/features/tenant-content/import-spreadsheet.ts
Normal file
@@ -0,0 +1,739 @@
|
||||
import { Buffer } from 'node:buffer';
|
||||
import ExcelJS from 'exceljs';
|
||||
import { HttpError } from '../../core/http.js';
|
||||
|
||||
type JsonObject = Record<string, unknown>;
|
||||
type SpreadsheetImportType = 'questions' | 'vocabulary' | 'handbook' | 'scoreline' | 'videos';
|
||||
type SupportedSpreadsheetFormat = 'json' | 'csv' | 'excel';
|
||||
|
||||
interface ParsedSpreadsheet {
|
||||
rows: JsonObject[];
|
||||
sheets: Record<string, JsonObject[]>;
|
||||
metadata: JsonObject;
|
||||
}
|
||||
|
||||
const MAX_SPREADSHEET_ROWS = 5000;
|
||||
const MAX_SPREADSHEET_COLUMNS = 160;
|
||||
const MAX_SPREADSHEET_CELL_CHARS = 100_000;
|
||||
const MAX_EXCEL_FILE_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
const QUESTION_OPTION_KEYS = ['optionA', 'optionB', 'optionC', 'optionD', 'optionE', 'optionF', 'optionG', 'optionH'];
|
||||
|
||||
function objectValue(value: unknown): JsonObject {
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? value as JsonObject : {};
|
||||
}
|
||||
|
||||
function stringValue(value: unknown) {
|
||||
return typeof value === 'string' && value.trim() ? value.trim() : '';
|
||||
}
|
||||
|
||||
function normalizedKey(value: string) {
|
||||
return value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[\s_\-./\\::()()[\]【】]/g, '');
|
||||
}
|
||||
|
||||
function sourceFormatValue(value: unknown): SupportedSpreadsheetFormat {
|
||||
const raw = stringValue(value).toLowerCase();
|
||||
if (!raw || raw === 'json') return 'json';
|
||||
if (raw === 'csv') return 'csv';
|
||||
if (['excel', 'xlsx'].includes(raw)) return 'excel';
|
||||
throw new HttpError(400, 'sourceFormat must be json, csv, or excel', 'IMPORT_FORMAT_NOT_SUPPORTED');
|
||||
}
|
||||
|
||||
function assertRowLimit(count: number) {
|
||||
if (count > MAX_SPREADSHEET_ROWS) {
|
||||
throw new HttpError(400, `Spreadsheet import can contain at most ${MAX_SPREADSHEET_ROWS} rows`, 'IMPORT_TOO_LARGE');
|
||||
}
|
||||
}
|
||||
|
||||
function decodeBase64File(body: JsonObject) {
|
||||
const raw = stringValue(body.fileBase64 ?? body.excelBase64 ?? body.csvBase64);
|
||||
if (!raw) return null;
|
||||
const base64 = raw.includes(',') ? raw.slice(raw.indexOf(',') + 1) : raw;
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
buffer = Buffer.from(base64, 'base64');
|
||||
} catch {
|
||||
throw new HttpError(400, 'fileBase64 must be a valid base64 string', 'IMPORT_FILE_BASE64_INVALID');
|
||||
}
|
||||
if (!buffer.length) {
|
||||
throw new HttpError(400, 'fileBase64 is empty', 'IMPORT_FILE_EMPTY');
|
||||
}
|
||||
if (buffer.length > MAX_EXCEL_FILE_BYTES) {
|
||||
throw new HttpError(400, `Import file can be at most ${MAX_EXCEL_FILE_BYTES} bytes`, 'IMPORT_FILE_TOO_LARGE');
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
|
||||
function primitiveCellValue(value: unknown): unknown {
|
||||
if (value === undefined || value === null) return undefined;
|
||||
if (value instanceof Date) return value.toISOString();
|
||||
if (typeof value === 'number' || typeof value === 'boolean') return value;
|
||||
if (typeof value === 'string') {
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > MAX_SPREADSHEET_CELL_CHARS ? trimmed.slice(0, MAX_SPREADSHEET_CELL_CHARS) : trimmed;
|
||||
}
|
||||
if (typeof value === 'object') {
|
||||
const cell = value as {
|
||||
text?: string;
|
||||
result?: unknown;
|
||||
formula?: string;
|
||||
richText?: Array<{ text?: string }>;
|
||||
hyperlink?: string;
|
||||
};
|
||||
if (cell.result !== undefined) return primitiveCellValue(cell.result);
|
||||
if (Array.isArray(cell.richText)) return cell.richText.map(part => part.text || '').join('');
|
||||
if (cell.text !== undefined) return primitiveCellValue(cell.text);
|
||||
if (cell.hyperlink !== undefined) return primitiveCellValue(cell.hyperlink);
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function parseJsonLike(value: unknown): unknown {
|
||||
if (typeof value !== 'string') return value;
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) return undefined;
|
||||
if (
|
||||
(trimmed.startsWith('[') && trimmed.endsWith(']')) ||
|
||||
(trimmed.startsWith('{') && trimmed.endsWith('}'))
|
||||
) {
|
||||
try {
|
||||
return JSON.parse(trimmed);
|
||||
} catch {
|
||||
return trimmed;
|
||||
}
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function splitList(value: unknown) {
|
||||
const parsed = parseJsonLike(value);
|
||||
if (Array.isArray(parsed)) return parsed.map(item => String(item).trim()).filter(Boolean);
|
||||
if (parsed === undefined || parsed === null) return [];
|
||||
return String(parsed)
|
||||
.split(/[|,,;;\n]/)
|
||||
.map(item => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function parseBooleanLike(value: unknown): unknown {
|
||||
if (typeof value === 'boolean') return value;
|
||||
if (typeof value !== 'string') return value;
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (['true', '1', 'yes', 'y', '是', '启用', '有效'].includes(normalized)) return true;
|
||||
if (['false', '0', 'no', 'n', '否', '禁用', '无效'].includes(normalized)) return false;
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseNumericLike(value: unknown): unknown {
|
||||
if (typeof value !== 'string') return value;
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) return undefined;
|
||||
if (!/^-?\d+(\.\d+)?$/.test(trimmed)) return value;
|
||||
const parsed = Number(trimmed);
|
||||
return Number.isFinite(parsed) ? parsed : value;
|
||||
}
|
||||
|
||||
function coerceValue(key: string, value: unknown) {
|
||||
if (value === undefined || value === null || value === '') return undefined;
|
||||
if ([
|
||||
'options',
|
||||
'tags',
|
||||
'correctOptionIndices',
|
||||
'knowledgeTags',
|
||||
].includes(key)) {
|
||||
return splitList(value);
|
||||
}
|
||||
if (['subQuestions', 'examMarkers', 'metadata', 'fieldValues', 'bindings'].includes(key)) {
|
||||
return parseJsonLike(value);
|
||||
}
|
||||
if (['isActive', 'isHot', 'isFilter', 'isRequired', 'isVisible', 'isTrend', 'hasRestriction', 'isGeneral'].includes(key)) {
|
||||
return parseBooleanLike(value);
|
||||
}
|
||||
if ([
|
||||
'difficulty',
|
||||
'order',
|
||||
'sortOrder',
|
||||
'duration',
|
||||
'durationSeconds',
|
||||
'freePreviewSeconds',
|
||||
'year',
|
||||
'wordCount',
|
||||
].includes(key)) {
|
||||
return parseNumericLike(value);
|
||||
}
|
||||
return parseJsonLike(value);
|
||||
}
|
||||
|
||||
function commonHeaderAlias(header: string) {
|
||||
const key = normalizedKey(header);
|
||||
const aliases: Record<string, string> = {
|
||||
id: 'legacyId',
|
||||
legacyid: 'legacyId',
|
||||
legacy_id: 'legacyId',
|
||||
externalid: 'legacyId',
|
||||
external_id: 'legacyId',
|
||||
旧id: 'legacyId',
|
||||
原id: 'legacyId',
|
||||
外部id: 'legacyId',
|
||||
排序: 'order',
|
||||
sort: 'order',
|
||||
sortorder: 'order',
|
||||
sort_order: 'order',
|
||||
是否启用: 'isActive',
|
||||
状态: 'isActive',
|
||||
标签: 'tags',
|
||||
tags: 'tags',
|
||||
metadata: 'metadata',
|
||||
元数据: 'metadata',
|
||||
};
|
||||
return aliases[key] || null;
|
||||
}
|
||||
|
||||
function importHeaderAlias(header: string, importType: SpreadsheetImportType) {
|
||||
const key = normalizedKey(header);
|
||||
const common = commonHeaderAlias(header);
|
||||
const aliases: Record<SpreadsheetImportType, Record<string, string>> = {
|
||||
questions: {
|
||||
type: 'type',
|
||||
questiontype: 'type',
|
||||
题型: 'type',
|
||||
typelabel: 'typeLabel',
|
||||
题型名称: 'typeLabel',
|
||||
content: 'content',
|
||||
title: 'content',
|
||||
题目: 'content',
|
||||
题干: 'content',
|
||||
问题: 'content',
|
||||
选项: 'options',
|
||||
options: 'options',
|
||||
a: 'optionA',
|
||||
选项a: 'optionA',
|
||||
optiona: 'optionA',
|
||||
b: 'optionB',
|
||||
选项b: 'optionB',
|
||||
optionb: 'optionB',
|
||||
c: 'optionC',
|
||||
选项c: 'optionC',
|
||||
optionc: 'optionC',
|
||||
d: 'optionD',
|
||||
选项d: 'optionD',
|
||||
optiond: 'optionD',
|
||||
e: 'optionE',
|
||||
选项e: 'optionE',
|
||||
optione: 'optionE',
|
||||
f: 'optionF',
|
||||
选项f: 'optionF',
|
||||
optionf: 'optionF',
|
||||
answer: 'answer',
|
||||
correctanswer: 'answer',
|
||||
correct: 'answer',
|
||||
答案: 'answer',
|
||||
正确答案: 'answer',
|
||||
正确选项: 'answer',
|
||||
correctoptionindices: 'correctOptionIndices',
|
||||
correct_option_indices: 'correctOptionIndices',
|
||||
explanation: 'explanation',
|
||||
解析: 'explanation',
|
||||
答案解析: 'explanation',
|
||||
difficulty: 'difficulty',
|
||||
难度: 'difficulty',
|
||||
mediaurl: 'mediaUrl',
|
||||
图片: 'mediaUrl',
|
||||
subquestions: 'subQuestions',
|
||||
子题: 'subQuestions',
|
||||
exammarkers: 'examMarkers',
|
||||
考试标记: 'examMarkers',
|
||||
},
|
||||
vocabulary: {
|
||||
unitlegacyid: 'unitLegacyId',
|
||||
unitid: 'unitLegacyId',
|
||||
单元id: 'unitLegacyId',
|
||||
unitname: 'unitName',
|
||||
unit: 'unitName',
|
||||
单元: 'unitName',
|
||||
单元名称: 'unitName',
|
||||
unitdescription: 'unitDescription',
|
||||
单元描述: 'unitDescription',
|
||||
unitorder: 'unitOrder',
|
||||
单元排序: 'unitOrder',
|
||||
wordlegacyid: 'wordLegacyId',
|
||||
wordid: 'wordLegacyId',
|
||||
单词id: 'wordLegacyId',
|
||||
word: 'word',
|
||||
单词: 'word',
|
||||
phonetic: 'phonetic',
|
||||
pronunciation: 'phonetic',
|
||||
音标: 'phonetic',
|
||||
meaning: 'meaning',
|
||||
translation: 'meaning',
|
||||
definition: 'meaning',
|
||||
释义: 'meaning',
|
||||
中文: 'meaning',
|
||||
意思: 'meaning',
|
||||
example: 'example',
|
||||
例句: 'example',
|
||||
exampletranslation: 'exampleTranslation',
|
||||
例句翻译: 'exampleTranslation',
|
||||
difficulty: 'difficulty',
|
||||
难度: 'difficulty',
|
||||
},
|
||||
handbook: {
|
||||
subjectlegacyid: 'subjectLegacyId',
|
||||
subjectid: 'subjectLegacyId',
|
||||
科目id: 'subjectLegacyId',
|
||||
subjectname: 'subjectName',
|
||||
subject: 'subjectName',
|
||||
科目: 'subjectName',
|
||||
手册: 'subjectName',
|
||||
chapterlegacyid: 'chapterLegacyId',
|
||||
chapterid: 'chapterLegacyId',
|
||||
章节id: 'chapterLegacyId',
|
||||
chaptername: 'chapterName',
|
||||
chapter: 'chapterName',
|
||||
章节: 'chapterName',
|
||||
sectionlegacyid: 'sectionLegacyId',
|
||||
sectionid: 'sectionLegacyId',
|
||||
小节id: 'sectionLegacyId',
|
||||
sectionname: 'sectionName',
|
||||
section: 'sectionName',
|
||||
小节: 'sectionName',
|
||||
entrylegacyid: 'entryLegacyId',
|
||||
entryid: 'entryLegacyId',
|
||||
知识点id: 'entryLegacyId',
|
||||
title: 'title',
|
||||
标题: 'title',
|
||||
知识点: 'title',
|
||||
content: 'content',
|
||||
正文: 'content',
|
||||
内容: 'content',
|
||||
summary: 'summary',
|
||||
摘要: 'summary',
|
||||
},
|
||||
scoreline: {
|
||||
kind: 'kind',
|
||||
itemtype: 'kind',
|
||||
类型: 'kind',
|
||||
fieldkey: 'fieldKey',
|
||||
字段key: 'fieldKey',
|
||||
fieldname: 'fieldName',
|
||||
字段名: 'fieldName',
|
||||
fieldtype: 'fieldType',
|
||||
字段类型: 'fieldType',
|
||||
unit: 'unit',
|
||||
单位: 'unit',
|
||||
schoolid: 'schoolId',
|
||||
院校id: 'schoolId',
|
||||
schoollegacyid: 'schoolLegacyId',
|
||||
schoolname: 'schoolName',
|
||||
school: 'schoolName',
|
||||
学校: 'schoolName',
|
||||
院校: 'schoolName',
|
||||
majorid: 'majorId',
|
||||
专业id: 'majorId',
|
||||
majorlegacyid: 'majorLegacyId',
|
||||
majorname: 'majorName',
|
||||
major: 'majorName',
|
||||
专业: 'majorName',
|
||||
year: 'year',
|
||||
年份: 'year',
|
||||
fieldvalues: 'fieldValues',
|
||||
动态字段: 'fieldValues',
|
||||
最低分: 'minScore',
|
||||
录取分: 'minScore',
|
||||
计划数: 'planCount',
|
||||
招生计划: 'planCount',
|
||||
},
|
||||
videos: {
|
||||
title: 'title',
|
||||
name: 'title',
|
||||
视频标题: 'title',
|
||||
标题: 'title',
|
||||
description: 'description',
|
||||
描述: 'description',
|
||||
videourl: 'videoUrl',
|
||||
url: 'videoUrl',
|
||||
视频地址: 'videoUrl',
|
||||
thumbnailurl: 'thumbnailUrl',
|
||||
封面: 'thumbnailUrl',
|
||||
duration: 'durationSeconds',
|
||||
durationseconds: 'durationSeconds',
|
||||
时长: 'durationSeconds',
|
||||
subjectid: 'subjectId',
|
||||
科目id: 'subjectId',
|
||||
questionid: 'questionId',
|
||||
题目id: 'questionId',
|
||||
legacyquestionid: 'legacyQuestionId',
|
||||
旧题目id: 'legacyQuestionId',
|
||||
assetid: 'assetId',
|
||||
资源id: 'assetId',
|
||||
accessmode: 'accessMode',
|
||||
访问模式: 'accessMode',
|
||||
knowledgetags: 'knowledgeTags',
|
||||
知识点标签: 'knowledgeTags',
|
||||
bindings: 'bindings',
|
||||
绑定题目: 'bindings',
|
||||
},
|
||||
};
|
||||
return aliases[importType][key] || common || header.trim();
|
||||
}
|
||||
|
||||
function normalizeSpreadsheetRow(raw: JsonObject, importType: SpreadsheetImportType) {
|
||||
const row: JsonObject = {};
|
||||
for (const [header, value] of Object.entries(raw)) {
|
||||
const key = importHeaderAlias(header, importType);
|
||||
let coerced = coerceValue(key, value);
|
||||
if (importType === 'scoreline' && key === header.trim()) {
|
||||
coerced = parseNumericLike(coerced);
|
||||
}
|
||||
if (coerced !== undefined && coerced !== '') row[key] = coerced;
|
||||
}
|
||||
|
||||
if (importType === 'questions') {
|
||||
const options = QUESTION_OPTION_KEYS.map(key => row[key]).filter(value => value !== undefined && value !== '');
|
||||
if (!row.options && options.length) row.options = options;
|
||||
if (!row.correctOptionIndices && row.answer) {
|
||||
row.correctOptionIndices = parseAnswerIndices(row.answer, Array.isArray(row.options) ? row.options.length : 0);
|
||||
}
|
||||
}
|
||||
|
||||
if (importType === 'vocabulary') {
|
||||
if (row.wordLegacyId && !row.legacyId) row.legacyId = row.wordLegacyId;
|
||||
if (row.unitOrder && !row.order && !row.word) row.order = row.unitOrder;
|
||||
}
|
||||
|
||||
if (importType === 'handbook') {
|
||||
if (row.entryLegacyId && !row.legacyId) row.legacyId = row.entryLegacyId;
|
||||
}
|
||||
|
||||
return row;
|
||||
}
|
||||
|
||||
function parseAnswerIndices(value: unknown, optionCount: number) {
|
||||
const parts = splitList(value);
|
||||
const indices: number[] = [];
|
||||
for (const part of parts) {
|
||||
const upper = String(part).trim().toUpperCase();
|
||||
if (/^[A-H]$/.test(upper)) {
|
||||
indices.push(upper.charCodeAt(0) - 65);
|
||||
continue;
|
||||
}
|
||||
const parsed = Number(upper);
|
||||
if (Number.isFinite(parsed)) {
|
||||
const integer = Math.trunc(parsed);
|
||||
indices.push(integer === 0 ? 0 : optionCount > 0 && integer <= optionCount ? integer - 1 : integer);
|
||||
}
|
||||
}
|
||||
return [...new Set(indices)];
|
||||
}
|
||||
|
||||
function parseCsv(text: string, delimiter: string) {
|
||||
const rows: string[][] = [];
|
||||
let row: string[] = [];
|
||||
let current = '';
|
||||
let quoted = false;
|
||||
for (let index = 0; index < text.length; index += 1) {
|
||||
const char = text[index];
|
||||
const next = text[index + 1];
|
||||
if (quoted) {
|
||||
if (char === '"' && next === '"') {
|
||||
current += '"';
|
||||
index += 1;
|
||||
} else if (char === '"') {
|
||||
quoted = false;
|
||||
} else {
|
||||
current += char;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (char === '"') {
|
||||
quoted = true;
|
||||
} else if (char === delimiter) {
|
||||
row.push(current);
|
||||
current = '';
|
||||
} else if (char === '\n') {
|
||||
row.push(current);
|
||||
rows.push(row);
|
||||
row = [];
|
||||
current = '';
|
||||
} else if (char !== '\r') {
|
||||
current += char;
|
||||
}
|
||||
}
|
||||
row.push(current);
|
||||
rows.push(row);
|
||||
return rows.filter(item => item.some(cell => cell.trim()));
|
||||
}
|
||||
|
||||
function autoDelimiter(text: string) {
|
||||
const sample = text.slice(0, 4096);
|
||||
const candidates = [',', '\t', ';'];
|
||||
return candidates
|
||||
.map(delimiter => ({ delimiter, count: (sample.match(new RegExp(delimiter === '\t' ? '\\t' : delimiter, 'g')) || []).length }))
|
||||
.sort((left, right) => right.count - left.count)[0]?.delimiter || ',';
|
||||
}
|
||||
|
||||
function rowsFromMatrix(matrix: string[][], importType: SpreadsheetImportType) {
|
||||
if (matrix.length < 2) return [];
|
||||
const headers = matrix[0].map((header, index) => stringValue(header) || `column${index + 1}`);
|
||||
if (headers.length > MAX_SPREADSHEET_COLUMNS) {
|
||||
throw new HttpError(400, `Spreadsheet can contain at most ${MAX_SPREADSHEET_COLUMNS} columns`, 'IMPORT_TOO_MANY_COLUMNS');
|
||||
}
|
||||
const rows = matrix.slice(1).map(line => {
|
||||
const row: JsonObject = {};
|
||||
headers.forEach((header, index) => {
|
||||
const value = stringValue(line[index]);
|
||||
if (value) row[header] = value;
|
||||
});
|
||||
return normalizeSpreadsheetRow(row, importType);
|
||||
}).filter(row => Object.keys(row).length > 0);
|
||||
assertRowLimit(rows.length);
|
||||
return rows;
|
||||
}
|
||||
|
||||
function parseCsvBody(body: JsonObject, importType: SpreadsheetImportType): ParsedSpreadsheet {
|
||||
const buffer = decodeBase64File(body);
|
||||
const text = stringValue(body.csvText ?? body.text ?? body.fileContent ?? body.payload) || buffer?.toString('utf8') || '';
|
||||
if (!text.trim()) {
|
||||
throw new HttpError(400, 'CSV import requires csvText, fileContent, payload, or fileBase64', 'CSV_CONTENT_REQUIRED');
|
||||
}
|
||||
const delimiter = stringValue(body.delimiter) || autoDelimiter(text);
|
||||
const rows = rowsFromMatrix(parseCsv(text, delimiter), importType);
|
||||
return {
|
||||
rows,
|
||||
sheets: { csv: rows },
|
||||
metadata: {
|
||||
parser: 'csv',
|
||||
delimiter: delimiter === '\t' ? 'tab' : delimiter,
|
||||
rowCount: rows.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function worksheetRows(worksheet: ExcelJS.Worksheet, importType: SpreadsheetImportType) {
|
||||
const matrix: string[][] = [];
|
||||
worksheet.eachRow({ includeEmpty: false }, row => {
|
||||
const values: string[] = [];
|
||||
const max = Math.min(row.cellCount, MAX_SPREADSHEET_COLUMNS);
|
||||
for (let col = 1; col <= max; col += 1) {
|
||||
const value = primitiveCellValue(row.getCell(col).value);
|
||||
values.push(value === undefined || value === null ? '' : String(value));
|
||||
}
|
||||
if (values.some(value => value.trim())) matrix.push(values);
|
||||
});
|
||||
return rowsFromMatrix(matrix, importType);
|
||||
}
|
||||
|
||||
async function parseExcelBody(body: JsonObject, importType: SpreadsheetImportType): Promise<ParsedSpreadsheet> {
|
||||
const buffer = decodeBase64File(body);
|
||||
if (!buffer) {
|
||||
throw new HttpError(400, 'Excel import requires fileBase64', 'EXCEL_FILE_REQUIRED');
|
||||
}
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
try {
|
||||
await workbook.xlsx.load(buffer as unknown as Parameters<typeof workbook.xlsx.load>[0]);
|
||||
} catch {
|
||||
throw new HttpError(400, 'Excel file could not be parsed as .xlsx', 'EXCEL_PARSE_FAILED');
|
||||
}
|
||||
|
||||
const sheetName = stringValue(body.sheetName);
|
||||
const sheetIndex = Number(body.sheetIndex ?? 1);
|
||||
let selected: ExcelJS.Worksheet | undefined;
|
||||
if (sheetName) selected = workbook.getWorksheet(sheetName);
|
||||
else if (Number.isFinite(sheetIndex) && sheetIndex > 0) selected = workbook.getWorksheet(Math.trunc(sheetIndex));
|
||||
selected = selected || workbook.worksheets.find(sheet => sheet.actualRowCount > 0);
|
||||
if (!selected) {
|
||||
throw new HttpError(400, 'Excel file has no non-empty worksheet', 'EXCEL_SHEET_EMPTY');
|
||||
}
|
||||
|
||||
const sheets: Record<string, JsonObject[]> = {};
|
||||
let totalRows = 0;
|
||||
for (const sheet of workbook.worksheets) {
|
||||
if (sheet.actualRowCount === 0) continue;
|
||||
const rows = worksheetRows(sheet, importType);
|
||||
sheets[normalizedKey(sheet.name)] = rows;
|
||||
totalRows += rows.length;
|
||||
}
|
||||
assertRowLimit(totalRows);
|
||||
|
||||
const selectedRows = sheets[normalizedKey(selected.name)] || [];
|
||||
return {
|
||||
rows: selectedRows,
|
||||
sheets,
|
||||
metadata: {
|
||||
parser: 'exceljs',
|
||||
workbookSheetCount: workbook.worksheets.length,
|
||||
selectedSheet: selected.name,
|
||||
rowCount: selectedRows.length,
|
||||
totalParsedRows: totalRows,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function sheetRows(parsed: ParsedSpreadsheet, names: string[]) {
|
||||
for (const name of names) {
|
||||
const rows = parsed.sheets[normalizedKey(name)];
|
||||
if (rows?.length) return rows;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function stableLegacyId(value: unknown, fallback: string) {
|
||||
return stringValue(value) || fallback;
|
||||
}
|
||||
|
||||
function buildVocabularyPayload(parsed: ParsedSpreadsheet) {
|
||||
const explicitUnitRows = sheetRows(parsed, ['units', 'vocabulary_units', '单元', '单词单元']);
|
||||
const explicitWordRows = sheetRows(parsed, ['words', 'vocabulary', 'vocabulary_words', '单词', '词汇']);
|
||||
const rows = explicitWordRows.length || explicitUnitRows.length ? [...explicitUnitRows, ...explicitWordRows] : parsed.rows;
|
||||
const units = new Map<string, JsonObject & { words: JsonObject[] }>();
|
||||
|
||||
for (const row of rows) {
|
||||
const rowType = String(row.rowType ?? row.kind ?? row.type ?? '').toLowerCase();
|
||||
const hasWord = Boolean(row.word);
|
||||
const unitName = stringValue(row.unitName ?? row.unit) || '默认单词单元';
|
||||
const unitKey = stableLegacyId(row.unitLegacyId ?? row.unitId, unitName);
|
||||
const unit = units.get(unitKey) || {
|
||||
legacyId: unitKey,
|
||||
name: unitName,
|
||||
description: row.unitDescription ?? row.description ?? null,
|
||||
order: row.unitOrder ?? row.order ?? units.size + 1,
|
||||
isActive: row.isActive ?? true,
|
||||
words: [],
|
||||
};
|
||||
if (rowType.includes('unit') && !hasWord) {
|
||||
unit.name = unitName;
|
||||
unit.description = row.unitDescription ?? row.description ?? unit.description;
|
||||
unit.order = row.unitOrder ?? row.order ?? unit.order;
|
||||
unit.isActive = row.isActive ?? unit.isActive;
|
||||
}
|
||||
if (hasWord) {
|
||||
const word = { ...row };
|
||||
word.legacyId = row.wordLegacyId ?? row.legacyId ?? null;
|
||||
delete word.unitLegacyId;
|
||||
delete word.unitId;
|
||||
delete word.unitName;
|
||||
delete word.unitDescription;
|
||||
delete word.unitOrder;
|
||||
delete word.wordLegacyId;
|
||||
unit.words.push(word);
|
||||
}
|
||||
units.set(unitKey, unit);
|
||||
}
|
||||
|
||||
return { units: Array.from(units.values()) };
|
||||
}
|
||||
|
||||
function getOrCreateNested<T extends JsonObject>(
|
||||
map: Map<string, T>,
|
||||
key: string,
|
||||
create: () => T,
|
||||
) {
|
||||
const existing = map.get(key);
|
||||
if (existing) return existing;
|
||||
const created = create();
|
||||
map.set(key, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
function buildHandbookPayload(parsed: ParsedSpreadsheet) {
|
||||
const rows = sheetRows(parsed, ['entries', 'handbook', '知识点', '手册']) || parsed.rows;
|
||||
const subjectMap = new Map<string, JsonObject & { chapters: Array<JsonObject & { sections: Array<JsonObject & { entries: JsonObject[] }> }> }>();
|
||||
|
||||
for (const row of rows.length ? rows : parsed.rows) {
|
||||
const subjectName = stringValue(row.subjectName ?? row.subject) || '默认知识手册';
|
||||
const subjectKey = stableLegacyId(row.subjectLegacyId ?? row.subjectId, subjectName);
|
||||
const subject = getOrCreateNested(subjectMap, subjectKey, () => ({
|
||||
legacyId: subjectKey,
|
||||
name: subjectName,
|
||||
type: row.subjectType ?? row.type ?? null,
|
||||
icon: row.icon ?? null,
|
||||
color: row.color ?? null,
|
||||
order: subjectMap.size + 1,
|
||||
chapters: [],
|
||||
}));
|
||||
|
||||
const chapterName = stringValue(row.chapterName ?? row.chapter) || '默认章节';
|
||||
const chapterKey = stableLegacyId(row.chapterLegacyId ?? row.chapterId, `${subjectKey}:${chapterName}`);
|
||||
let chapter = subject.chapters.find(item => item.legacyId === chapterKey);
|
||||
if (!chapter) {
|
||||
chapter = { legacyId: chapterKey, name: chapterName, order: subject.chapters.length + 1, sections: [] };
|
||||
subject.chapters.push(chapter);
|
||||
}
|
||||
|
||||
const sectionName = stringValue(row.sectionName ?? row.section) || '';
|
||||
const sectionKey = stableLegacyId(row.sectionLegacyId ?? row.sectionId, `${chapterKey}:${sectionName || 'default'}`);
|
||||
let section = chapter.sections.find(item => item.legacyId === sectionKey);
|
||||
if (!section) {
|
||||
section = { legacyId: sectionKey, name: sectionName || null, order: chapter.sections.length + 1, entries: [] };
|
||||
chapter.sections.push(section);
|
||||
}
|
||||
|
||||
if (row.title || row.content) {
|
||||
section.entries.push({
|
||||
legacyId: row.entryLegacyId ?? row.legacyId ?? null,
|
||||
title: row.title ?? '未命名知识点',
|
||||
summary: row.summary ?? null,
|
||||
content: row.content ?? '',
|
||||
tags: row.tags ?? [],
|
||||
order: row.order ?? section.entries.length + 1,
|
||||
isActive: row.isActive ?? true,
|
||||
metadata: row.metadata ?? {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { subjects: Array.from(subjectMap.values()) };
|
||||
}
|
||||
|
||||
function buildScorelinePayload(parsed: ParsedSpreadsheet) {
|
||||
const buckets = {
|
||||
fields: sheetRows(parsed, ['fields', 'scoreline_fields', '字段']),
|
||||
schools: sheetRows(parsed, ['schools', 'scoreline_schools', '院校', '学校']),
|
||||
majors: sheetRows(parsed, ['majors', 'scoreline_majors', '专业']),
|
||||
records: sheetRows(parsed, ['records', 'scoreline_records', '分数线', '记录']),
|
||||
};
|
||||
if (Object.values(buckets).some(rows => rows.length > 0)) return buckets;
|
||||
return { items: parsed.rows };
|
||||
}
|
||||
|
||||
function buildVideosPayload(parsed: ParsedSpreadsheet) {
|
||||
const rows = sheetRows(parsed, ['videos', 'video_explanations', '视频']) || parsed.rows;
|
||||
return { videos: rows.length ? rows : parsed.rows };
|
||||
}
|
||||
|
||||
export async function expandSpreadsheetImportBody(body: JsonObject, importType: SpreadsheetImportType): Promise<JsonObject> {
|
||||
const sourceFormat = sourceFormatValue(body.sourceFormat);
|
||||
if (sourceFormat === 'json') return body;
|
||||
|
||||
const parsed = sourceFormat === 'csv'
|
||||
? parseCsvBody(body, importType)
|
||||
: await parseExcelBody(body, importType);
|
||||
|
||||
if (!parsed.rows.length && !Object.values(parsed.sheets).some(rows => rows.length > 0)) {
|
||||
throw new HttpError(400, 'Spreadsheet import payload must contain at least one data row', 'EMPTY_IMPORT_PAYLOAD');
|
||||
}
|
||||
|
||||
const next: JsonObject = {
|
||||
...body,
|
||||
sourceFormat,
|
||||
spreadsheet: parsed.metadata,
|
||||
};
|
||||
delete next.csvText;
|
||||
delete next.text;
|
||||
delete next.fileContent;
|
||||
delete next.fileBase64;
|
||||
delete next.excelBase64;
|
||||
delete next.csvBase64;
|
||||
delete next.payload;
|
||||
|
||||
if (importType === 'questions') next.items = parsed.rows;
|
||||
if (importType === 'vocabulary') Object.assign(next, buildVocabularyPayload(parsed));
|
||||
if (importType === 'handbook') Object.assign(next, buildHandbookPayload(parsed));
|
||||
if (importType === 'scoreline') Object.assign(next, buildScorelinePayload(parsed));
|
||||
if (importType === 'videos') Object.assign(next, buildVideosPayload(parsed));
|
||||
|
||||
return next;
|
||||
}
|
||||
@@ -1,11 +1,9 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import type pg from 'pg';
|
||||
import { config } from '../../core/config.js';
|
||||
import { HttpError, type RequestContext } from '../../core/http.js';
|
||||
import { intParam, readJsonBody, requiredString, stringParam } from '../../core/request.js';
|
||||
import { HttpError } from '../../core/errors.js';
|
||||
import type { RequestContext } from '../../core/http.js';
|
||||
import { query, queryOne, transaction } from '../../core/db.js';
|
||||
import { requireTenantContentEditor, type TenantContentAuth } from './auth.js';
|
||||
import { boolValue, nullableString } from './utils.js';
|
||||
import type { TenantContentAuth } from './auth.js';
|
||||
|
||||
type JsonObject = Record<string, unknown>;
|
||||
|
||||
@@ -114,8 +112,10 @@ interface NormalizedHandbookSubject {
|
||||
}
|
||||
|
||||
type ContentImportType = 'vocabulary' | 'handbook' | 'scoreline' | 'videos';
|
||||
export type ExecutableContentImportType = 'questions' | ContentImportType;
|
||||
type EntryBoundImportType = 'vocabulary' | 'handbook';
|
||||
type GenericTargetEntryType = EntryBoundImportType | null;
|
||||
type SourceImportFormat = 'json' | 'csv' | 'excel';
|
||||
|
||||
type ScorelineImportKind = 'field' | 'school' | 'major' | 'record';
|
||||
|
||||
@@ -227,6 +227,19 @@ function stringValue(value: unknown) {
|
||||
return typeof value === 'string' && value.trim() ? value.trim() : '';
|
||||
}
|
||||
|
||||
function sourceImportFormat(value: unknown): SourceImportFormat {
|
||||
const normalized = stringValue(value).toLowerCase();
|
||||
if (!normalized || normalized === 'json') return 'json';
|
||||
if (normalized === 'csv') return 'csv';
|
||||
if (['excel', 'xlsx'].includes(normalized)) return 'excel';
|
||||
throw new HttpError(400, 'sourceFormat must be json, csv, or excel', 'IMPORT_FORMAT_NOT_SUPPORTED');
|
||||
}
|
||||
|
||||
function isAsyncExecution(value: unknown) {
|
||||
const normalized = stringValue(value).toLowerCase();
|
||||
return ['async', 'queued', 'queue'].includes(normalized);
|
||||
}
|
||||
|
||||
function stringArrayValue(value: unknown) {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.map(item => String(item).trim()).filter(Boolean);
|
||||
@@ -291,6 +304,40 @@ function boolishValue(value: unknown, fallback: boolean) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function boolValue(value: unknown, fallback: boolean) {
|
||||
return typeof value === 'boolean' ? value : fallback;
|
||||
}
|
||||
|
||||
function nullableString(value: unknown) {
|
||||
return typeof value === 'string' && value.trim() ? value.trim() : null;
|
||||
}
|
||||
|
||||
function requiredString(body: JsonObject, key: string) {
|
||||
const value = body[key];
|
||||
if (typeof value !== 'string' || !value.trim()) {
|
||||
throw new HttpError(400, `${key} is required`, 'REQUIRED_FIELD');
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
async function routeDeps() {
|
||||
const [{ config }, request, auth, spreadsheet] = await Promise.all([
|
||||
import('../../core/config.js'),
|
||||
import('../../core/request.js'),
|
||||
import('./auth.js'),
|
||||
import('./import-spreadsheet.js'),
|
||||
]);
|
||||
return {
|
||||
config,
|
||||
intParam: request.intParam,
|
||||
readJsonBody: request.readJsonBody,
|
||||
requiredString: request.requiredString,
|
||||
stringParam: request.stringParam,
|
||||
requireTenantContentEditor: auth.requireTenantContentEditor,
|
||||
expandSpreadsheetImportBody: spreadsheet.expandSpreadsheetImportBody,
|
||||
};
|
||||
}
|
||||
|
||||
function integerValue(value: unknown, fallback: number) {
|
||||
const numberValue = Number(value ?? fallback);
|
||||
return Number.isFinite(numberValue) ? Math.trunc(numberValue) : fallback;
|
||||
@@ -647,13 +694,9 @@ async function assertTargetReferences(client: pg.PoolClient, auth: TenantContent
|
||||
|
||||
async function createQuestionPreviewJob(auth: TenantContentAuth, body: JsonObject): Promise<PreviewResult> {
|
||||
const rawItems = parseQuestionItems(body);
|
||||
const sourceFormat = stringValue(body.sourceFormat) || 'json';
|
||||
const sourceFormat = sourceImportFormat(body.sourceFormat);
|
||||
const sourceName = stringValue(body.sourceName) || null;
|
||||
|
||||
if (sourceFormat !== 'json') {
|
||||
throw new HttpError(400, 'Only json sourceFormat is supported by the synchronous API for now', 'IMPORT_FORMAT_NOT_SUPPORTED');
|
||||
}
|
||||
|
||||
return transaction(async client => {
|
||||
const target = await assertTargetReferences(client, auth, body);
|
||||
const normalizedItems = rawItems.map((raw, index) => {
|
||||
@@ -684,7 +727,7 @@ async function createQuestionPreviewJob(auth: TenantContentAuth, body: JsonObjec
|
||||
target_category_id, target_node_id, target_question_bank_id,
|
||||
target_entry_id, target_content_node_id, target_collection_id,
|
||||
dry_run, total_count, valid_count, error_count, warning_count,
|
||||
summary, raw_payload, normalized_payload
|
||||
summary, raw_payload, normalized_payload, parser_metadata
|
||||
)
|
||||
values (
|
||||
$1, $2, 'questions', $3, 'preview',
|
||||
@@ -692,7 +735,7 @@ async function createQuestionPreviewJob(auth: TenantContentAuth, body: JsonObjec
|
||||
$8::uuid, $9::uuid, $10::uuid,
|
||||
$11::uuid, $12::uuid, $13::uuid,
|
||||
true, $14, $15, $16, $17,
|
||||
$18::jsonb, $19::jsonb, $20::jsonb
|
||||
$18::jsonb, $19::jsonb, $20::jsonb, $21::jsonb
|
||||
)
|
||||
returning id, status, total_count as "totalCount", valid_count as "validCount",
|
||||
error_count as "errorCount", warning_count as "warningCount"
|
||||
@@ -718,6 +761,7 @@ async function createQuestionPreviewJob(auth: TenantContentAuth, body: JsonObjec
|
||||
JSON.stringify({ target, generatedAt: new Date().toISOString() }),
|
||||
rawPayload,
|
||||
normalizedPayload,
|
||||
JSON.stringify(objectValue(body.spreadsheet)),
|
||||
],
|
||||
);
|
||||
|
||||
@@ -1659,11 +1703,8 @@ async function createGenericPreviewJob<T>(
|
||||
normalizedItems: NormalizedImportItem<T>[],
|
||||
expectedEntryType: GenericTargetEntryType = importType === 'vocabulary' || importType === 'handbook' ? importType : null,
|
||||
): Promise<PreviewResult<T>> {
|
||||
const sourceFormat = stringValue(body.sourceFormat) || 'json';
|
||||
const sourceFormat = sourceImportFormat(body.sourceFormat);
|
||||
const sourceName = stringValue(body.sourceName) || null;
|
||||
if (sourceFormat !== 'json') {
|
||||
throw new HttpError(400, 'Only json sourceFormat is supported by the synchronous API for now', 'IMPORT_FORMAT_NOT_SUPPORTED');
|
||||
}
|
||||
|
||||
return transaction(async client => {
|
||||
const target = await assertGenericTargetReferences(client, auth, body, expectedEntryType);
|
||||
@@ -1678,13 +1719,13 @@ async function createGenericPreviewJob<T>(
|
||||
tenant_id, created_by, import_type, source_format, status,
|
||||
source_name, source_hash, target_region_id, target_entry_id,
|
||||
target_content_node_id, dry_run, total_count, valid_count,
|
||||
error_count, warning_count, summary, raw_payload, normalized_payload
|
||||
error_count, warning_count, summary, raw_payload, normalized_payload, parser_metadata
|
||||
)
|
||||
values (
|
||||
$1, $2, $3, $4, 'preview',
|
||||
$5, $6, $7::uuid, $8::uuid,
|
||||
$9::uuid, true, $10, $11,
|
||||
$12, $13, $14::jsonb, $15::jsonb, $16::jsonb
|
||||
$12, $13, $14::jsonb, $15::jsonb, $16::jsonb, $17::jsonb
|
||||
)
|
||||
returning id, status, total_count as "totalCount", valid_count as "validCount",
|
||||
error_count as "errorCount", warning_count as "warningCount"
|
||||
@@ -1706,6 +1747,7 @@ async function createGenericPreviewJob<T>(
|
||||
JSON.stringify({ target, generatedAt: new Date().toISOString() }),
|
||||
JSON.stringify(normalizedItems.map(item => item.source)),
|
||||
JSON.stringify(normalizedItems.map(item => item.normalized).filter(Boolean)),
|
||||
JSON.stringify(objectValue(body.spreadsheet)),
|
||||
],
|
||||
);
|
||||
|
||||
@@ -3052,196 +3094,31 @@ async function importOneVideo(
|
||||
return { insertedCount, updatedCount, skippedCount };
|
||||
}
|
||||
|
||||
async function runGenericImport<T>(
|
||||
interface ContentImportExecutionOptions {
|
||||
jobId: string;
|
||||
importType: ExecutableContentImportType;
|
||||
allowPartial?: boolean;
|
||||
allowQueuedJob?: boolean;
|
||||
}
|
||||
|
||||
interface ContentImportExecutionResult {
|
||||
jobId: string;
|
||||
status: string;
|
||||
idempotent?: boolean;
|
||||
insertedCount: number;
|
||||
updatedCount: number;
|
||||
skippedCount: number;
|
||||
errorCount?: number;
|
||||
warningCount?: number;
|
||||
}
|
||||
|
||||
async function executeQuestionsImportJob(
|
||||
auth: TenantContentAuth,
|
||||
body: JsonObject,
|
||||
importType: ContentImportType,
|
||||
createPreview: () => Promise<PreviewResult<T>>,
|
||||
importOne: (
|
||||
client: pg.PoolClient,
|
||||
auth: TenantContentAuth,
|
||||
job: {
|
||||
id: string;
|
||||
target_region_id: string | null;
|
||||
target_entry_id: string | null;
|
||||
target_content_node_id: string | null;
|
||||
},
|
||||
item: {
|
||||
id: string;
|
||||
row_no: number;
|
||||
normalized_payload: T;
|
||||
},
|
||||
) => Promise<{ insertedCount: number; updatedCount: number; skippedCount: number }>,
|
||||
input: Omit<ContentImportExecutionOptions, 'importType'>,
|
||||
) {
|
||||
const allowPartial = boolValue(body.allowPartial, false);
|
||||
const jobId = nullableString(body.previewJobId) || nullableString(body.jobId);
|
||||
const createdPreview = jobId ? null : await createPreview();
|
||||
const finalJobId = jobId || createdPreview?.job.id || '';
|
||||
|
||||
const result = await transaction(async client => {
|
||||
const job = await loadGenericPreviewJob(client, auth, finalJobId, importType);
|
||||
if (job.status === 'completed' || job.status === 'completed_with_errors') {
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: job.status,
|
||||
idempotent: true,
|
||||
insertedCount: 0,
|
||||
updatedCount: 0,
|
||||
skippedCount: 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (job.error_count > 0 && !allowPartial) {
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = 'rejected', error_message = 'Preview contains validation errors', updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[auth.tenantId, job.id],
|
||||
);
|
||||
throw new HttpError(409, 'Preview contains validation errors. Fix issues or set allowPartial=true.', 'IMPORT_HAS_ERRORS');
|
||||
}
|
||||
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = 'importing', dry_run = false, started_at = coalesce(started_at, now()), updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[auth.tenantId, job.id],
|
||||
);
|
||||
|
||||
const itemResult = await client.query<{
|
||||
id: string;
|
||||
row_no: number;
|
||||
normalized_payload: T;
|
||||
}>(
|
||||
`
|
||||
select id, row_no, normalized_payload
|
||||
from public.content_import_items
|
||||
where tenant_id = $1 and job_id = $2 and status = 'valid'
|
||||
order by
|
||||
case
|
||||
when $3 = 'scoreline' then
|
||||
case normalized_payload ->> 'kind'
|
||||
when 'field' then 1
|
||||
when 'school' then 2
|
||||
when 'major' then 3
|
||||
when 'record' then 4
|
||||
else 9
|
||||
end
|
||||
else 1
|
||||
end,
|
||||
row_no asc
|
||||
for update
|
||||
`,
|
||||
[auth.tenantId, job.id, importType],
|
||||
);
|
||||
|
||||
let insertedCount = 0;
|
||||
let updatedCount = 0;
|
||||
let skippedCount = 0;
|
||||
for (const item of itemResult.rows) {
|
||||
const status = await importOne(client, auth, job, item);
|
||||
insertedCount += status.insertedCount;
|
||||
updatedCount += status.updatedCount;
|
||||
skippedCount += status.skippedCount;
|
||||
}
|
||||
|
||||
const finalStatus = job.error_count > 0 ? 'completed_with_errors' : 'completed';
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = $3,
|
||||
inserted_count = $4,
|
||||
updated_count = $5,
|
||||
skipped_count = $6,
|
||||
summary = coalesce(summary, '{}'::jsonb) || $7::jsonb,
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[
|
||||
auth.tenantId,
|
||||
job.id,
|
||||
finalStatus,
|
||||
insertedCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
JSON.stringify({ insertedCount, updatedCount, skippedCount, importedAt: new Date().toISOString() }),
|
||||
],
|
||||
);
|
||||
|
||||
await client.query(
|
||||
`
|
||||
insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details)
|
||||
values ($1, $2, $3, 'content_import_job', $4, $5::jsonb)
|
||||
`,
|
||||
[
|
||||
auth.tenantId,
|
||||
auth.userId,
|
||||
`content.import.${importType}.completed`,
|
||||
job.id,
|
||||
JSON.stringify({ insertedCount, updatedCount, skippedCount, allowPartial }),
|
||||
],
|
||||
);
|
||||
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: finalStatus,
|
||||
insertedCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
errorCount: job.error_count,
|
||||
warningCount: job.warning_count,
|
||||
};
|
||||
});
|
||||
|
||||
return { item: result, preview: createdPreview };
|
||||
}
|
||||
|
||||
export async function previewQuestionsImportRoute(ctx: RequestContext) {
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
return createQuestionPreviewJob(auth, body);
|
||||
}
|
||||
|
||||
export async function previewVocabularyImportRoute(ctx: RequestContext) {
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
return createGenericPreviewJob(auth, body, 'vocabulary', 'vocabulary_unit', createVocabularyNormalizedItems(body));
|
||||
}
|
||||
|
||||
export async function previewHandbookImportRoute(ctx: RequestContext) {
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
return createGenericPreviewJob(auth, body, 'handbook', 'handbook_subject', createHandbookNormalizedItems(body));
|
||||
}
|
||||
|
||||
export async function previewScorelineImportRoute(ctx: RequestContext) {
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
return createGenericPreviewJob(auth, body, 'scoreline', 'scoreline_item', createScorelineNormalizedItems(body), null);
|
||||
}
|
||||
|
||||
export async function previewVideosImportRoute(ctx: RequestContext) {
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
return createGenericPreviewJob(auth, body, 'videos', 'video_explanation', createVideoNormalizedItems(body), null);
|
||||
}
|
||||
|
||||
export async function importQuestionsRoute(ctx: RequestContext) {
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const allowPartial = boolValue(body.allowPartial, false);
|
||||
const jobId = nullableString(body.previewJobId) || nullableString(body.jobId);
|
||||
|
||||
const createdPreview = jobId ? null : await createQuestionPreviewJob(auth, body);
|
||||
const finalJobId = jobId || createdPreview?.job.id || '';
|
||||
|
||||
const result = await transaction(async client => {
|
||||
const job = await loadPreviewJob(client, auth, finalJobId);
|
||||
const allowPartial = input.allowPartial === true;
|
||||
return transaction(async client => {
|
||||
const job = await loadPreviewJob(client, auth, input.jobId);
|
||||
|
||||
if (job.status === 'completed' || job.status === 'completed_with_errors') {
|
||||
return {
|
||||
@@ -3253,12 +3130,19 @@ export async function importQuestionsRoute(ctx: RequestContext) {
|
||||
skippedCount: 0,
|
||||
};
|
||||
}
|
||||
if (job.status === 'pending' && input.allowQueuedJob !== true) {
|
||||
throw new HttpError(409, 'Import job is queued for async processing', 'IMPORT_JOB_QUEUED');
|
||||
}
|
||||
|
||||
if (job.error_count > 0 && !allowPartial) {
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = 'rejected', error_message = 'Preview contains validation errors', updated_at = now()
|
||||
set status = 'rejected',
|
||||
error_message = 'Preview contains validation errors',
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[auth.tenantId, job.id],
|
||||
@@ -3310,6 +3194,8 @@ export async function importQuestionsRoute(ctx: RequestContext) {
|
||||
skipped_count = $6,
|
||||
summary = coalesce(summary, '{}'::jsonb) || $7::jsonb,
|
||||
finished_at = now(),
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
@@ -3342,13 +3228,389 @@ export async function importQuestionsRoute(ctx: RequestContext) {
|
||||
warningCount: job.warning_count,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function executeGenericImportJob<T>(
|
||||
auth: TenantContentAuth,
|
||||
input: Omit<ContentImportExecutionOptions, 'importType'> & { importType: ContentImportType },
|
||||
importOne: (
|
||||
client: pg.PoolClient,
|
||||
auth: TenantContentAuth,
|
||||
job: {
|
||||
id: string;
|
||||
target_region_id: string | null;
|
||||
target_entry_id: string | null;
|
||||
target_content_node_id: string | null;
|
||||
},
|
||||
item: {
|
||||
id: string;
|
||||
row_no: number;
|
||||
normalized_payload: T;
|
||||
},
|
||||
) => Promise<{ insertedCount: number; updatedCount: number; skippedCount: number }>,
|
||||
) {
|
||||
const allowPartial = input.allowPartial === true;
|
||||
return transaction(async client => {
|
||||
const job = await loadGenericPreviewJob(client, auth, input.jobId, input.importType);
|
||||
if (job.status === 'completed' || job.status === 'completed_with_errors') {
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: job.status,
|
||||
idempotent: true,
|
||||
insertedCount: 0,
|
||||
updatedCount: 0,
|
||||
skippedCount: 0,
|
||||
};
|
||||
}
|
||||
if (job.status === 'pending' && input.allowQueuedJob !== true) {
|
||||
throw new HttpError(409, 'Import job is queued for async processing', 'IMPORT_JOB_QUEUED');
|
||||
}
|
||||
|
||||
if (job.error_count > 0 && !allowPartial) {
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = 'rejected',
|
||||
error_message = 'Preview contains validation errors',
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[auth.tenantId, job.id],
|
||||
);
|
||||
throw new HttpError(409, 'Preview contains validation errors. Fix issues or set allowPartial=true.', 'IMPORT_HAS_ERRORS');
|
||||
}
|
||||
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = 'importing', dry_run = false, started_at = coalesce(started_at, now()), updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[auth.tenantId, job.id],
|
||||
);
|
||||
|
||||
const itemResult = await client.query<{
|
||||
id: string;
|
||||
row_no: number;
|
||||
normalized_payload: T;
|
||||
}>(
|
||||
`
|
||||
select id, row_no, normalized_payload
|
||||
from public.content_import_items
|
||||
where tenant_id = $1 and job_id = $2 and status = 'valid'
|
||||
order by
|
||||
case
|
||||
when $3 = 'scoreline' then
|
||||
case normalized_payload ->> 'kind'
|
||||
when 'field' then 1
|
||||
when 'school' then 2
|
||||
when 'major' then 3
|
||||
when 'record' then 4
|
||||
else 9
|
||||
end
|
||||
else 1
|
||||
end,
|
||||
row_no asc
|
||||
for update
|
||||
`,
|
||||
[auth.tenantId, job.id, input.importType],
|
||||
);
|
||||
|
||||
let insertedCount = 0;
|
||||
let updatedCount = 0;
|
||||
let skippedCount = 0;
|
||||
for (const item of itemResult.rows) {
|
||||
const status = await importOne(client, auth, job, item);
|
||||
insertedCount += status.insertedCount;
|
||||
updatedCount += status.updatedCount;
|
||||
skippedCount += status.skippedCount;
|
||||
}
|
||||
|
||||
const finalStatus = job.error_count > 0 ? 'completed_with_errors' : 'completed';
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = $3,
|
||||
inserted_count = $4,
|
||||
updated_count = $5,
|
||||
skipped_count = $6,
|
||||
summary = coalesce(summary, '{}'::jsonb) || $7::jsonb,
|
||||
finished_at = now(),
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[
|
||||
auth.tenantId,
|
||||
job.id,
|
||||
finalStatus,
|
||||
insertedCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
JSON.stringify({ insertedCount, updatedCount, skippedCount, importedAt: new Date().toISOString() }),
|
||||
],
|
||||
);
|
||||
|
||||
await client.query(
|
||||
`
|
||||
insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details)
|
||||
values ($1, $2, $3, 'content_import_job', $4, $5::jsonb)
|
||||
`,
|
||||
[
|
||||
auth.tenantId,
|
||||
auth.userId,
|
||||
`content.import.${input.importType}.completed`,
|
||||
job.id,
|
||||
JSON.stringify({ insertedCount, updatedCount, skippedCount, allowPartial }),
|
||||
],
|
||||
);
|
||||
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: finalStatus,
|
||||
insertedCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
errorCount: job.error_count,
|
||||
warningCount: job.warning_count,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function executeContentImportJob(
|
||||
auth: TenantContentAuth,
|
||||
input: ContentImportExecutionOptions,
|
||||
): Promise<ContentImportExecutionResult> {
|
||||
if (input.importType === 'questions') {
|
||||
return executeQuestionsImportJob(auth, input);
|
||||
}
|
||||
if (input.importType === 'vocabulary') {
|
||||
return executeGenericImportJob(auth, { ...input, importType: 'vocabulary' }, importOneVocabularyUnit);
|
||||
}
|
||||
if (input.importType === 'handbook') {
|
||||
return executeGenericImportJob(auth, { ...input, importType: 'handbook' }, importOneHandbookSubject);
|
||||
}
|
||||
if (input.importType === 'scoreline') {
|
||||
return executeGenericImportJob(auth, { ...input, importType: 'scoreline' }, importOneScorelineItem);
|
||||
}
|
||||
if (input.importType === 'videos') {
|
||||
return executeGenericImportJob(auth, { ...input, importType: 'videos' }, importOneVideo);
|
||||
}
|
||||
throw new HttpError(400, 'Unsupported import type', 'IMPORT_TYPE_NOT_SUPPORTED');
|
||||
}
|
||||
|
||||
async function queueContentImportJob(
|
||||
auth: TenantContentAuth,
|
||||
input: ContentImportExecutionOptions,
|
||||
) {
|
||||
const allowPartial = input.allowPartial === true;
|
||||
return transaction(async client => {
|
||||
const result = await client.query<{
|
||||
id: string;
|
||||
status: string;
|
||||
error_count: number;
|
||||
warning_count: number;
|
||||
attempt_count: number;
|
||||
max_attempts: number;
|
||||
}>(
|
||||
`
|
||||
select id, status, error_count, warning_count, attempt_count, max_attempts
|
||||
from public.content_import_jobs
|
||||
where tenant_id = $1 and id = $2 and import_type = $3
|
||||
limit 1
|
||||
for update
|
||||
`,
|
||||
[auth.tenantId, input.jobId, input.importType],
|
||||
);
|
||||
const job = result.rows[0];
|
||||
if (!job) throw new HttpError(404, 'Import job not found', 'IMPORT_JOB_NOT_FOUND');
|
||||
if (job.status === 'completed' || job.status === 'completed_with_errors') {
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: job.status,
|
||||
idempotent: true,
|
||||
executionMode: 'async',
|
||||
errorCount: job.error_count,
|
||||
warningCount: job.warning_count,
|
||||
};
|
||||
}
|
||||
if (job.status === 'pending') {
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: job.status,
|
||||
idempotent: true,
|
||||
executionMode: 'async',
|
||||
errorCount: job.error_count,
|
||||
warningCount: job.warning_count,
|
||||
};
|
||||
}
|
||||
if (job.status === 'importing') {
|
||||
throw new HttpError(409, 'Import job is already importing', 'IMPORT_JOB_NOT_READY');
|
||||
}
|
||||
if (['failed', 'rejected'].includes(job.status)) {
|
||||
throw new HttpError(409, `Import job is ${job.status}`, 'IMPORT_JOB_NOT_READY');
|
||||
}
|
||||
if (job.error_count > 0 && !allowPartial) {
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = 'rejected',
|
||||
error_message = 'Preview contains validation errors',
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[auth.tenantId, job.id],
|
||||
);
|
||||
throw new HttpError(409, 'Preview contains validation errors. Fix issues or set allowPartial=true.', 'IMPORT_HAS_ERRORS');
|
||||
}
|
||||
|
||||
const queuedAt = new Date().toISOString();
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set execution_mode = 'async',
|
||||
status = 'pending',
|
||||
dry_run = false,
|
||||
queued_at = coalesce(queued_at, now()),
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
next_attempt_at = now(),
|
||||
summary = coalesce(summary, '{}'::jsonb) || $3::jsonb,
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[
|
||||
auth.tenantId,
|
||||
job.id,
|
||||
JSON.stringify({
|
||||
importOptions: { allowPartial },
|
||||
queuedAt,
|
||||
}),
|
||||
],
|
||||
);
|
||||
|
||||
await client.query(
|
||||
`
|
||||
insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details)
|
||||
values ($1, $2, $3, 'content_import_job', $4, $5::jsonb)
|
||||
`,
|
||||
[
|
||||
auth.tenantId,
|
||||
auth.userId,
|
||||
`content.import.${input.importType}.queued`,
|
||||
job.id,
|
||||
JSON.stringify({ allowPartial, executionMode: 'async', queuedAt }),
|
||||
],
|
||||
);
|
||||
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: 'pending',
|
||||
executionMode: 'async',
|
||||
errorCount: job.error_count,
|
||||
warningCount: job.warning_count,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function runGenericImport<T>(
|
||||
auth: TenantContentAuth,
|
||||
body: JsonObject,
|
||||
importType: ContentImportType,
|
||||
createPreview: () => Promise<PreviewResult<T>>,
|
||||
importOne: (
|
||||
client: pg.PoolClient,
|
||||
auth: TenantContentAuth,
|
||||
job: {
|
||||
id: string;
|
||||
target_region_id: string | null;
|
||||
target_entry_id: string | null;
|
||||
target_content_node_id: string | null;
|
||||
},
|
||||
item: {
|
||||
id: string;
|
||||
row_no: number;
|
||||
normalized_payload: T;
|
||||
},
|
||||
) => Promise<{ insertedCount: number; updatedCount: number; skippedCount: number }>,
|
||||
) {
|
||||
const allowPartial = boolValue(body.allowPartial, false);
|
||||
const jobId = nullableString(body.previewJobId) || nullableString(body.jobId);
|
||||
const createdPreview = jobId ? null : await createPreview();
|
||||
const finalJobId = jobId || createdPreview?.job.id || '';
|
||||
const result = isAsyncExecution(body.executionMode ?? body.execution_mode)
|
||||
? await queueContentImportJob(auth, { jobId: finalJobId, importType, allowPartial })
|
||||
: await executeGenericImportJob(auth, { jobId: finalJobId, importType, allowPartial }, importOne);
|
||||
|
||||
return { item: result, preview: createdPreview };
|
||||
}
|
||||
|
||||
export async function previewQuestionsImportRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'questions');
|
||||
return createQuestionPreviewJob(auth, body);
|
||||
}
|
||||
|
||||
export async function previewVocabularyImportRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'vocabulary');
|
||||
return createGenericPreviewJob(auth, body, 'vocabulary', 'vocabulary_unit', createVocabularyNormalizedItems(body));
|
||||
}
|
||||
|
||||
export async function previewHandbookImportRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'handbook');
|
||||
return createGenericPreviewJob(auth, body, 'handbook', 'handbook_subject', createHandbookNormalizedItems(body));
|
||||
}
|
||||
|
||||
export async function previewScorelineImportRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'scoreline');
|
||||
return createGenericPreviewJob(auth, body, 'scoreline', 'scoreline_item', createScorelineNormalizedItems(body), null);
|
||||
}
|
||||
|
||||
export async function previewVideosImportRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'videos');
|
||||
return createGenericPreviewJob(auth, body, 'videos', 'video_explanation', createVideoNormalizedItems(body), null);
|
||||
}
|
||||
|
||||
export async function importQuestionsRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'questions');
|
||||
const allowPartial = boolValue(body.allowPartial, false);
|
||||
const jobId = nullableString(body.previewJobId) || nullableString(body.jobId);
|
||||
|
||||
const createdPreview = jobId ? null : await createQuestionPreviewJob(auth, body);
|
||||
const finalJobId = jobId || createdPreview?.job.id || '';
|
||||
|
||||
const result = isAsyncExecution(body.executionMode ?? body.execution_mode)
|
||||
? await queueContentImportJob(auth, { jobId: finalJobId, importType: 'questions', allowPartial })
|
||||
: await executeContentImportJob(auth, { jobId: finalJobId, importType: 'questions', allowPartial });
|
||||
|
||||
return { item: result, preview: createdPreview };
|
||||
}
|
||||
|
||||
export async function importVocabularyRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'vocabulary');
|
||||
return runGenericImport(
|
||||
auth,
|
||||
body,
|
||||
@@ -3359,8 +3621,10 @@ export async function importVocabularyRoute(ctx: RequestContext) {
|
||||
}
|
||||
|
||||
export async function importHandbookRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'handbook');
|
||||
return runGenericImport(
|
||||
auth,
|
||||
body,
|
||||
@@ -3371,8 +3635,10 @@ export async function importHandbookRoute(ctx: RequestContext) {
|
||||
}
|
||||
|
||||
export async function importScorelineRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'scoreline');
|
||||
return runGenericImport(
|
||||
auth,
|
||||
body,
|
||||
@@ -3383,8 +3649,10 @@ export async function importScorelineRoute(ctx: RequestContext) {
|
||||
}
|
||||
|
||||
export async function importVideosRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'videos');
|
||||
return runGenericImport(
|
||||
auth,
|
||||
body,
|
||||
@@ -3395,6 +3663,7 @@ export async function importVideosRoute(ctx: RequestContext) {
|
||||
}
|
||||
|
||||
export async function importJobsRoute(ctx: RequestContext) {
|
||||
const { intParam, requireTenantContentEditor, stringParam } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const limit = intParam(ctx, 'limit', 50, 200);
|
||||
const importType = stringParam(ctx, 'importType');
|
||||
@@ -3423,6 +3692,7 @@ export async function importJobsRoute(ctx: RequestContext) {
|
||||
valid_count as "validCount", error_count as "errorCount",
|
||||
warning_count as "warningCount", inserted_count as "insertedCount",
|
||||
updated_count as "updatedCount", skipped_count as "skippedCount",
|
||||
execution_mode as "executionMode", parser_metadata as "parserMetadata",
|
||||
summary, error_message as "errorMessage",
|
||||
started_at as "startedAt", finished_at as "finishedAt",
|
||||
created_by as "createdBy", created_at as "createdAt", updated_at as "updatedAt"
|
||||
@@ -3438,6 +3708,7 @@ export async function importJobsRoute(ctx: RequestContext) {
|
||||
}
|
||||
|
||||
export async function importIssuesRoute(ctx: RequestContext) {
|
||||
const { intParam, requireTenantContentEditor, requiredString, stringParam } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const jobId = requiredString({ jobId: stringParam(ctx, 'jobId') }, 'jobId');
|
||||
const limit = intParam(ctx, 'limit', 500, 2000);
|
||||
|
||||
@@ -10,7 +10,8 @@
|
||||
"check": "tsc -p tsconfig.json --noEmit",
|
||||
"crm:once": "tsx src/index.ts --once --job crm",
|
||||
"commerce:once": "tsx src/index.ts --once --job commerce",
|
||||
"assets:once": "tsx src/index.ts --once --job assets"
|
||||
"assets:once": "tsx src/index.ts --once --job assets",
|
||||
"imports:once": "tsx src/index.ts --once --job imports"
|
||||
},
|
||||
"dependencies": {
|
||||
"@supabase/storage-js": "^2.108.2",
|
||||
|
||||
@@ -17,6 +17,9 @@ export interface WorkerConfig {
|
||||
assetMinAgeSeconds: number;
|
||||
assetRecheckIntervalSeconds: number;
|
||||
assetRequestTimeoutMs: number;
|
||||
importBatchSize: number;
|
||||
importWorkerId: string;
|
||||
importBackoffSeconds: number[];
|
||||
storageMaxUploadBytes: number;
|
||||
storageAllowedMimePrefixes: string[];
|
||||
storageAllowedMimeTypes: string[];
|
||||
@@ -53,6 +56,11 @@ export const config: WorkerConfig = {
|
||||
assetMinAgeSeconds: envNumber('WORKER_ASSET_MIN_AGE_SECONDS', 300),
|
||||
assetRecheckIntervalSeconds: envNumber('WORKER_ASSET_RECHECK_INTERVAL_SECONDS', 60 * 60 * 24),
|
||||
assetRequestTimeoutMs: envNumber('WORKER_ASSET_REQUEST_TIMEOUT_MS', 10_000),
|
||||
importBatchSize: envNumber('WORKER_IMPORT_BATCH_SIZE', 5),
|
||||
importWorkerId: envString('WORKER_IMPORT_ID', `imports-${process.pid}`),
|
||||
importBackoffSeconds: envList('WORKER_IMPORT_BACKOFF_SECONDS', '30,120,600,1800')
|
||||
.map((value: string) => Number(value))
|
||||
.filter((value: number) => Number.isFinite(value) && value > 0),
|
||||
storageMaxUploadBytes: envNumber('STORAGE_MAX_UPLOAD_BYTES', 1024 * 1024 * 500),
|
||||
storageAllowedMimePrefixes: envList('STORAGE_ALLOWED_MIME_PREFIXES', 'image/,video/,audio/'),
|
||||
storageAllowedMimeTypes: envList(
|
||||
|
||||
@@ -4,6 +4,8 @@ import { processCrmBatch } from './jobs/crm.js';
|
||||
import { processCommerceBatch } from './jobs/commerce.js';
|
||||
import { processAssetBatch } from './jobs/assets.js';
|
||||
|
||||
const extraClosers = new Set<() => Promise<void>>();
|
||||
|
||||
function hasArg(name: string) {
|
||||
return process.argv.includes(name);
|
||||
}
|
||||
@@ -37,6 +39,17 @@ async function runOnce() {
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (job === 'imports') {
|
||||
const { closeImportExecutorPool, processImportBatch } = await import('./jobs/imports.js');
|
||||
extraClosers.add(closeImportExecutorPool);
|
||||
const result = await processImportBatch();
|
||||
console.log(
|
||||
`[worker] imports batch processed=${result.processed}`
|
||||
+ ` completed=${result.completed} completedWithErrors=${result.completedWithErrors}`
|
||||
+ ` failed=${result.failed} retrying=${result.retrying} skipped=${result.skipped}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
throw new Error(`Unsupported worker job: ${job}`);
|
||||
}
|
||||
|
||||
@@ -66,5 +79,8 @@ try {
|
||||
await runOnce();
|
||||
}
|
||||
} finally {
|
||||
for (const closeExtra of extraClosers) {
|
||||
await closeExtra();
|
||||
}
|
||||
await closePool();
|
||||
}
|
||||
|
||||
221
apps/worker/src/jobs/imports.ts
Normal file
221
apps/worker/src/jobs/imports.ts
Normal file
@@ -0,0 +1,221 @@
|
||||
import { pool } from '../db.js';
|
||||
import { config } from '../config.js';
|
||||
import { executeContentImportJob, type ExecutableContentImportType } from '../../../api/src/features/tenant-content/imports.js';
|
||||
import { closePool as closeApiImportPool } from '../../../api/src/core/db.js';
|
||||
|
||||
interface ImportJobRow {
|
||||
id: string;
|
||||
tenantId: string;
|
||||
createdBy: string | null;
|
||||
importType: ExecutableContentImportType;
|
||||
status: string;
|
||||
attemptCount: number;
|
||||
maxAttempts: number;
|
||||
summary: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface ImportWorkerResult {
|
||||
processed: number;
|
||||
completed: number;
|
||||
completedWithErrors: number;
|
||||
failed: number;
|
||||
retrying: number;
|
||||
skipped: number;
|
||||
}
|
||||
|
||||
function objectValue(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function boolValue(value: unknown, fallback: boolean) {
|
||||
return typeof value === 'boolean' ? value : fallback;
|
||||
}
|
||||
|
||||
function numberValue(value: unknown, fallback: number) {
|
||||
const parsed = Number(value ?? fallback);
|
||||
return Number.isFinite(parsed) ? parsed : fallback;
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function errorCode(error: unknown) {
|
||||
return typeof error === 'object' && error !== null && 'code' in error
|
||||
? String((error as { code?: unknown }).code || 'IMPORT_WORKER_ERROR')
|
||||
: 'IMPORT_WORKER_ERROR';
|
||||
}
|
||||
|
||||
function truncate(value: unknown, max = 1900) {
|
||||
return String(value ?? '').slice(0, max);
|
||||
}
|
||||
|
||||
function backoffSeconds(attemptCount: number) {
|
||||
const backoffs = config.importBackoffSeconds.length ? config.importBackoffSeconds : [30, 120, 600, 1800];
|
||||
return backoffs[Math.min(Math.max(0, attemptCount - 1), backoffs.length - 1)];
|
||||
}
|
||||
|
||||
function importOptions(summary: Record<string, unknown>) {
|
||||
const options = objectValue(summary.importOptions);
|
||||
return {
|
||||
allowPartial: boolValue(options.allowPartial, false),
|
||||
};
|
||||
}
|
||||
|
||||
async function claimImportJobs() {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('begin');
|
||||
const result = await client.query<ImportJobRow>(
|
||||
`
|
||||
select id,
|
||||
tenant_id as "tenantId",
|
||||
created_by as "createdBy",
|
||||
import_type as "importType",
|
||||
status,
|
||||
attempt_count as "attemptCount",
|
||||
max_attempts as "maxAttempts",
|
||||
summary
|
||||
from public.content_import_jobs
|
||||
where execution_mode = 'async'
|
||||
and status = 'pending'
|
||||
and attempt_count < max_attempts
|
||||
and (next_attempt_at is null or next_attempt_at <= now())
|
||||
order by created_at asc
|
||||
limit $1
|
||||
for update skip locked
|
||||
`,
|
||||
[config.importBatchSize],
|
||||
);
|
||||
|
||||
const ids = result.rows.map(row => row.id);
|
||||
if (ids.length > 0) {
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set locked_at = now(),
|
||||
locked_by = $2,
|
||||
attempt_count = attempt_count + 1,
|
||||
updated_at = now()
|
||||
where id = any($1::uuid[])
|
||||
`,
|
||||
[ids, config.importWorkerId],
|
||||
);
|
||||
}
|
||||
await client.query('commit');
|
||||
return result.rows;
|
||||
} catch (error) {
|
||||
await client.query('rollback');
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function markImportFailed(job: ImportJobRow, error: unknown) {
|
||||
const nextAttempt = job.attemptCount + 1;
|
||||
const willRetry = nextAttempt < job.maxAttempts;
|
||||
const status = willRetry ? 'pending' : 'failed';
|
||||
await pool.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = $3,
|
||||
error_message = $4,
|
||||
summary = coalesce(summary, '{}'::jsonb) || $5::jsonb,
|
||||
next_attempt_at = case when $6::boolean then now() + make_interval(secs => $7::integer) else null end,
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
finished_at = case when $3 = 'failed' then now() else finished_at end,
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[
|
||||
job.tenantId,
|
||||
job.id,
|
||||
status,
|
||||
truncate(errorMessage(error)),
|
||||
JSON.stringify({
|
||||
lastWorkerError: {
|
||||
code: errorCode(error),
|
||||
message: truncate(errorMessage(error)),
|
||||
workerId: config.importWorkerId,
|
||||
failedAt: new Date().toISOString(),
|
||||
nextAttempt,
|
||||
maxAttempts: job.maxAttempts,
|
||||
willRetry,
|
||||
},
|
||||
}),
|
||||
willRetry,
|
||||
backoffSeconds(nextAttempt),
|
||||
],
|
||||
);
|
||||
|
||||
await pool.query(
|
||||
`
|
||||
insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details)
|
||||
values ($1, $2, $3, 'content_import_job', $4, $5::jsonb)
|
||||
`,
|
||||
[
|
||||
job.tenantId,
|
||||
job.createdBy,
|
||||
willRetry ? `content.import.${job.importType}.retry_scheduled` : `content.import.${job.importType}.failed`,
|
||||
job.id,
|
||||
JSON.stringify({
|
||||
code: errorCode(error),
|
||||
message: truncate(errorMessage(error)),
|
||||
workerId: config.importWorkerId,
|
||||
nextAttempt,
|
||||
maxAttempts: job.maxAttempts,
|
||||
}),
|
||||
],
|
||||
);
|
||||
|
||||
return willRetry ? 'retrying' : 'failed';
|
||||
}
|
||||
|
||||
export async function processImportBatch(): Promise<ImportWorkerResult> {
|
||||
const jobs = await claimImportJobs();
|
||||
const result: ImportWorkerResult = {
|
||||
processed: jobs.length,
|
||||
completed: 0,
|
||||
completedWithErrors: 0,
|
||||
failed: 0,
|
||||
retrying: 0,
|
||||
skipped: 0,
|
||||
};
|
||||
|
||||
for (const job of jobs) {
|
||||
try {
|
||||
const execution = await executeContentImportJob(
|
||||
{
|
||||
tenantId: job.tenantId,
|
||||
userId: job.createdBy || job.tenantId,
|
||||
role: 'system_worker',
|
||||
permissions: { 'content:*': true },
|
||||
templatePermissions: {},
|
||||
},
|
||||
{
|
||||
jobId: job.id,
|
||||
importType: job.importType,
|
||||
allowPartial: importOptions(job.summary).allowPartial,
|
||||
allowQueuedJob: true,
|
||||
},
|
||||
);
|
||||
|
||||
if (execution.idempotent) result.skipped += 1;
|
||||
else if (execution.status === 'completed_with_errors') result.completedWithErrors += 1;
|
||||
else if (execution.status === 'completed') result.completed += 1;
|
||||
else result.skipped += 1;
|
||||
} catch (error) {
|
||||
const state = await markImportFailed(job, error);
|
||||
if (state === 'retrying') result.retrying += 1;
|
||||
else result.failed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function closeImportExecutorPool() {
|
||||
await closeApiImportPool();
|
||||
}
|
||||
@@ -74,4 +74,4 @@ types.ts 仅本领域使用的类型
|
||||
- `learning` 创建练习 session 时必须保存 `question_ids` 快照,避免随机刷题和模考过程中题目集合变化导致答题记录无法复盘。
|
||||
- 排行榜必须由后端按租户、地区、班级和可信用户上下文聚合,前端不能自行扫描答题记录、积分流水或单词进度后排名;后续高流量场景再通过 worker/materialized view 做日榜、周榜和防刷。
|
||||
- 资料、PDF、视频等对象存储资源必须先进入 `content_assets` 台账,再通过 API/Edge Function 做权限校验和签名 URL 下发;前端不能直接拼 OSS/COS/Supabase Storage 私有地址。
|
||||
- 批量导入必须先写 `content_import_jobs/items/issues`,保留原始 payload、规范化 payload、逐行问题和审计记录;同步 API 当前支持题目 JSON,Excel/CSV 和其它内容类型应接入同一管线。
|
||||
- 批量导入必须先写 `content_import_jobs/items/issues`,保留原始 payload、规范化 payload、逐行问题和审计记录;题目、单词、知识手册、分数线和视频 JSON/CSV/Excel 都应进入同一管线,复杂大批量导入通过 `executionMode=async` 交给 imports worker 消费。
|
||||
|
||||
@@ -163,8 +163,8 @@
|
||||
| 知识手册 JSON preview/import | 可联调 | 支持书籍/章节/小节/知识点归一化 |
|
||||
| 分数线 JSON preview/import | 可联调 | 支持 `fields/schools/majors/records` 分桶或 `items` 列表,后端校验租户地区和院校/专业引用 |
|
||||
| 视频 JSON preview/import | 可联调 | 支持 `videos/items`,后端校验题目、科目、资源引用,导入后写入 `question_videos` |
|
||||
| Excel/CSV 导入 | 待补齐 | 应复用 `content_import_jobs` 管线 |
|
||||
| 大批量异步导入 | 待补齐 | 需要 `apps/worker` |
|
||||
| Excel/CSV 导入 | 可联调 | 题目、单词、知识手册、分数线、视频已支持 CSV 和 `.xlsx` 解析,解析后复用 `content_import_jobs/items/issues` 管线并保留 `parser_metadata` |
|
||||
| 大批量异步导入 | 可联调 | `executionMode=async` 会将 preview job 置为 `pending`;`apps/worker --job imports` 抢占 queued job,复用 API 导入 executor,支持重试、清锁和审计;导入后复检待补 |
|
||||
| 公共题库版本同步 | 待补齐 | 当前采纳为快照复制;后续需 worker 做增量同步、冲突处理、版本升级通知和租户自改保护 |
|
||||
|
||||
## 当前验证
|
||||
@@ -174,6 +174,7 @@
|
||||
```bash
|
||||
npm audit
|
||||
npm run check:refactor
|
||||
npm run test:worker:imports
|
||||
```
|
||||
|
||||
`check:refactor` 包含:
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
- 销售/代理/CRM 已经有邀请码、扫码/分享事件、首绑客资保护、团队关系、统计、CRM 配置和入队能力。
|
||||
- 旧题库 JSON、单词模板、知识手册嵌套模板、分数线 JSON 和视频绑定 JSON 已经进入后端 preview/import 管线,由后端负责规范化、校验、幂等、审计和租户隔离。
|
||||
|
||||
因此,后端现在已经具备进入 Taro 前端第一阶段联调的基础。需要注意的是,它还不是完整生产交付状态,真实云端鉴权、对象存储生产安全、支付/短信生产账号、微信网页登录/QQ 登录、真实数据 dry-run 迁移和大批量异步导入仍需要继续补齐或联调。
|
||||
因此,后端现在已经具备进入 Taro 前端第一阶段联调的基础。需要注意的是,它还不是完整生产交付状态,真实云端鉴权、对象存储生产安全、支付/短信生产账号、微信网页登录/QQ 登录、真实数据 dry-run 迁移、导入后复检和模板字段映射仍需要继续补齐或联调。
|
||||
|
||||
## 后端模块进度
|
||||
|
||||
@@ -23,16 +23,16 @@
|
||||
| 多租户底座 | 可联调 | 租户、域名、品牌、设置、RLS 基础、审计、Supabase JWT/API 身份映射 | 真实云端 Auth/JWKS 回归、生产 RLS 深测 |
|
||||
| 平台后台 | 基础完成 | 租户、套餐、订阅、账单、服务费、用量、公共题库授权 | 自动计费、平台审计、公共题库版本同步 |
|
||||
| 租户后台 | 可联调 | 品牌、域名、支付账户、登录配置、密钥掩码、活动、兑换码、优惠券、勋章管理/发放、成员权限、角色模板、菜单/模块/字段权限配置 API、班级/教师/学生范围权限 | 前端权限 UI、更细的数据范围组合 |
|
||||
| 题库与练习 | 可联调 | 内容入口、任意深度分类、题目集合、顺序/随机/全真模拟蓝图、组卷快照、答题、错题、收藏、模考报告、排行榜、公共题库采纳快照 | 专项策略、公共题库版本同步、Excel 导入、排行榜防刷/预聚合 |
|
||||
| 背单词 | 可联调 | 单元、单词、进度、收藏、统计、每日计划、JSON 导入、排行榜 | Excel 导入、更细复习参数 |
|
||||
| 知识手册 | 可联调 | 科目、章节、条目、Markdown 内容、嵌套 JSON 导入 | 富文本资源、版本管理、附件/PDF 关联 |
|
||||
| 分数线 | 可联调 | 院校、专业、动态字段、记录、年份、趋势、后台维护、JSON 导入 | 复杂筛选、AI 择校上下文 |
|
||||
| 视频解析 | 可联调 | 单题视频、批量查询、后台视频绑定、JSON 导入、会员播放权限、播放次数扣减、签名 URL 和播放日志 | 深度防盗链、动态水印、播放统计 |
|
||||
| 题库与练习 | 可联调 | 内容入口、任意深度分类、题目集合、顺序/随机/全真模拟蓝图、组卷快照、答题、错题、收藏、模考报告、排行榜、公共题库采纳快照 | 专项策略、公共题库版本同步、排行榜防刷/预聚合 |
|
||||
| 背单词 | 可联调 | 单元、单词、进度、收藏、统计、每日计划、JSON/CSV/Excel 导入、排行榜 | 更细复习参数 |
|
||||
| 知识手册 | 可联调 | 科目、章节、条目、Markdown 内容、嵌套 JSON/CSV/Excel 导入 | 富文本资源、版本管理、附件/PDF 关联 |
|
||||
| 分数线 | 可联调 | 院校、专业、动态字段、记录、年份、趋势、后台维护、JSON/CSV/Excel 导入 | 复杂筛选、AI 择校上下文 |
|
||||
| 视频解析 | 可联调 | 单题视频、批量查询、后台视频绑定、JSON/CSV/Excel 导入、会员播放权限、播放次数扣减、签名 URL 和播放日志 | 深度防盗链、动态水印、播放统计 |
|
||||
| 资料下载 | 部分完成 | 资源台账、SVIP 权限校验、`local_dev`/阿里云 OSS/腾讯 COS/Supabase Storage 上传下载签名、上传确认、PDF/图片预览签名、assets worker 复检异常下架 | PDF 渲染、CDN 防盗链、杀毒扫描、视频水印 |
|
||||
| 会员与订单 | 可联调 | 下单、订单详情/状态轮询、优惠券领取/抵扣、零元订单自动开通、手工确认权限保护、激活码预检查/兑换、微信支付、支付宝、微信/支付宝发起退款、微信/支付宝退款查询确认、微信/支付宝退款通知 webhook、支付/退款补偿 worker、权益发放 | 完整资金流水对账、异常订单运营台 |
|
||||
| 登录认证 | 可联调 | 短信 mock、阿里云/腾讯云短信 adapter、迁移期 session、Supabase Auth JWT、微信小程序登录、OAuth 配置表 | 微信网页登录、QQ 登录、手机号换绑、真实生产账号联调 |
|
||||
| 销售/代理/CRM | 基础完成 | 邀请码、首绑保护、团队关系、销售统计、CRM 入队 | 小程序码真实生成、分佣结算、钉钉/飞书/企微 worker |
|
||||
| 内容导入 | 可联调 | 题目、单词、知识手册、分数线、视频 JSON preview/import、issue、job、审计、幂等 | Excel/CSV 解析、大批量异步 worker、导入后复检 |
|
||||
| 内容导入 | 可联调 | 题目、单词、知识手册、分数线、视频 JSON/CSV/Excel preview/import、issue、job、审计、幂等、`executionMode=async` 和 imports worker | 导入后复检、模板下载/字段映射 |
|
||||
| 数据看板 | 可联调 | 租户 dashboard 聚合接口,收益、注册、学习、内容、激活码、反馈、趋势、24h 活跃、套餐销量和运营动态 | 预聚合 worker、缓存、慢 SQL 监控和销售转化看板 |
|
||||
| AI 择校推荐 | 未开始 | 暂无 | 数据上下文、AI JSON schema、报告渲染、PDF 生成 |
|
||||
| Taro 前端 | 未开始 | 旧 Web 已有新 API 适配雏形 | `apps/taro`、跨端 API client、H5/小程序页面和端到端测试 |
|
||||
@@ -87,7 +87,7 @@
|
||||
- XPay 或其它实际支付网关 adapter。
|
||||
- 阿里云/腾讯云短信、微信小程序登录、微信网页登录、QQ 登录。
|
||||
- 公共题库/地区题库版本同步,租户按 SaaS 套餐购买地区、科目和题库范围的更细计费策略。
|
||||
- Excel/CSV 解析、大批量异步导入和导入后复检。
|
||||
- 导入后复检和导入模板/字段映射 UI;大批量导入已支持 `executionMode=async`,前端按 job 状态轮询。
|
||||
- 视频深度防盗链、动态水印和播放统计。
|
||||
- 数据看板 API:收益、注册趋势、答题次数、收入趋势、题型分布、题目总量、套餐销量、24h 活跃。
|
||||
|
||||
|
||||
@@ -32,6 +32,8 @@
|
||||
- 已新增 commerce worker 和 `npm run test:worker:commerce`,用于补偿查询微信/支付宝支付、处理中退款和漏通知场景;支付成功会幂等更新订单/支付并开通权益,退款成功会幂等更新退款/订单/支付并在全额退款时撤销订单权益,测试覆盖密钥不泄露和重复执行不重复开通。
|
||||
- 已新增 assets worker 和 `npm run test:worker:assets`,用于复检 `content_assets` 托管对象元数据;正常资源会写入复检证据,异常资源会自动下架为 `draft`、标记 `upload_status=failed`,并记录审计与安全标记。
|
||||
- 已新增分数线/视频 JSON 导入接口和集成测试:分数线支持字段、院校、专业、记录、动态字段值;视频支持视频元数据、资源引用、访问模式和题目绑定,全部复用 `content_import_jobs/items/issues` 管线。
|
||||
- 已新增 CSV/Excel 导入解析层:题目、单词、知识手册、分数线和视频都可通过 `sourceFormat=csv/excel` 先解析为规范 payload,再进入同一套 preview/import、逐行 issue、幂等和审计管线;分数线支持多 Sheet Excel。
|
||||
- 已新增 imports worker 和 `npm run test:worker:imports`:确认导入时可传 `executionMode=async`,API 将 preview job 排队为 `pending`,worker 复用同一导入 executor 消费任务,支持抢占锁、attempt/backoff、失败重试、清锁和审计。
|
||||
|
||||
## 已验证接口
|
||||
|
||||
@@ -251,11 +253,11 @@ GET /api/tenant-admin/audit-logs
|
||||
- 内容资源当前完成台账、租户后台维护、学生端 SVIP 下载权限,以及 `local_dev`、阿里云 OSS、腾讯 COS、Supabase Storage 的上传/下载签名 provider;上传确认和 assets worker 已支持对象元数据校验/复检。PDF 预览渲染、防盗链、水印和安全扫描仍需继续补。
|
||||
- 题库内容导航当前以 `content_entries/content_nodes` 为主模型,可表达“入口 -> 多级分类 -> 院校/专业/学科/销售意向标记”;题目集合和练习方式由 `question_collections/practice_blueprints` 管理,练习 session 会保存当次题目 ID 快照。
|
||||
- 练习访问控制由 `content_entries/content_nodes/question_collections/practice_blueprints` 的 `accessRules` 合并决定;普通用户消耗 `practice_daily_usage`,事件写入 `practice_access_events`,SVIP/staff 不消耗免费额度。
|
||||
- 批量导入当前支持题目、单词、知识手册、分数线、视频 JSON 预览、逐行 issue、job/item 台账、执行导入、幂等跳过,并可落到新内容入口、分类节点、分数线表或题目视频绑定。旧单词模板的 `vocabulary_units_示例数据` / `vocabulary_示例数据`、知识手册的书籍/章节/小节/知识点嵌套结构都由后端规范化。Excel/CSV 和大批量异步导入会继续复用同一套 `content_import_jobs` 管线。
|
||||
- 批量导入当前支持题目、单词、知识手册、分数线、视频 JSON/CSV/Excel 预览、逐行 issue、job/item 台账、同步执行或 `executionMode=async` 异步执行、幂等跳过,并可落到新内容入口、分类节点、分数线表或题目视频绑定。旧单词模板的 `vocabulary_units_示例数据` / `vocabulary_示例数据`、知识手册的书籍/章节/小节/知识点嵌套结构都由后端规范化。导入后复检、模板下载和字段映射 UI 后续补齐。
|
||||
|
||||
## 下一步
|
||||
|
||||
1. 完善内容导入和文件上传:Excel/CSV 解析、大批量异步导入、导入后复检,PDF 预览渲染、防盗链、杀毒扫描和视频水印。
|
||||
1. 完善内容导入和文件上传:导入后复检、模板下载/字段映射 UI,PDF 预览渲染、防盗链、杀毒扫描和视频水印。
|
||||
2. 接入真实短信 provider:阿里云/腾讯云,密钥放 `app_private.tenant_secrets` 或生产 Vault。
|
||||
3. 接入真实 OAuth provider:微信网页、微信小程序、QQ,并处理旧 PocketBase 身份映射。
|
||||
4. 补完整资金流水对账、异常订单运营台和优惠券核销报表;支付/退款补偿、退款查询确认和退款通知主链路已完成。
|
||||
@@ -269,5 +271,6 @@ npm run test:api
|
||||
npm run test:worker:crm
|
||||
npm run test:worker:commerce
|
||||
npm run test:worker:assets
|
||||
npm run test:worker:imports
|
||||
npm run check:refactor
|
||||
```
|
||||
|
||||
@@ -18,9 +18,9 @@
|
||||
| 平台超级管理员 | 部分完成 | 租户管理、SaaS 套餐、订阅、账单、服务费收款、用量记录 | 公共题库披露策略、地区/全国套餐权限、平台侧主题模板库、平台审计 |
|
||||
| 租户品牌和域名 | 基础完成 | 品牌、Logo、主题 JSON、公开资源、域名、租户公开配置 | 三套默认主题、主题可视化编辑、图标/图片上传 |
|
||||
| 租户成员权限 | 可联调 | owner/admin/operator/teacher/sales/agent/student,权限矩阵,成员启停,角色模板、菜单/模块/字段权限、班级/学生范围权限和审计查询 | 前端权限 UI、更细的数据范围组合 |
|
||||
| 题库内容维护 | 可联调 | 内容入口、任意深度分类树、院校/专业/学科/销售意向标记、题目集合、顺序/随机/全真模拟练习蓝图、题目录入/更新、题目/单词/知识手册/分数线/视频 JSON 预览导入、视频绑定、分数线、单词、知识手册后台 API、公共题库授权和采纳快照 | Excel/CSV 批量导入、公共题库版本同步、可视化拖拽排序前端 |
|
||||
| 题库内容维护 | 可联调 | 内容入口、任意深度分类树、院校/专业/学科/销售意向标记、题目集合、顺序/随机/全真模拟练习蓝图、题目录入/更新、题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 预览导入、`executionMode=async` 导入 worker、视频绑定、分数线、单词、知识手册后台 API、公共题库授权和采纳快照 | 导入后复检、模板/字段映射、公共题库版本同步、可视化拖拽排序前端 |
|
||||
| 学生刷题 | 基础完成 | 内容入口、分类树、题目集合、顺序刷题、随机刷题、全真模拟 session 题目快照、答题、错题本、收藏夹、模考交卷评分报告、错题复习计划、排行榜 | 专项练习策略、题型统计深度分析、排行榜防刷/预聚合 |
|
||||
| 背单词 | 基础完成 | 单词单元、单词、进度、收藏、统计、每日复习计划、旧模板/新模板 JSON 预览导入、内容导航绑定、排行榜 | Excel 导入、更细复习参数 |
|
||||
| 背单词 | 基础完成 | 单词单元、单词、进度、收藏、统计、每日复习计划、旧模板/新模板 JSON/CSV/Excel 预览导入、内容导航绑定、排行榜 | 更细复习参数 |
|
||||
| 知识手册 | 基础完成 | 科目、章节、条目只读与后台维护、书籍/章节/小节/知识点嵌套 JSON 预览导入、内容导航绑定 | 富文本资源、版本管理、附件/PDF 关联、Excel/Markdown 批量解析 |
|
||||
| 分数线 | 可联调 | 字段、院校、专业、记录、趋势、年份、JSON 批量导入 | 复杂动态筛选、AI 择校数据上下文 |
|
||||
| 视频解析会员 | 可联调 | 题目视频、批量查询、后台绑定、视频 JSON 导入、SVIP 权限、播放次数扣减、签名 URL、播放日志 | 深度防盗链、动态水印、播放统计 |
|
||||
@@ -36,7 +36,7 @@
|
||||
|
||||
## 接下来优先级
|
||||
|
||||
1. 完善内容导入和对象存储:Excel/CSV 解析、大批量异步导入、导入后复检、CDN 防盗链、杀毒扫描和视频水印。
|
||||
1. 完善内容导入和对象存储:导入后复检、模板下载/字段映射 UI、CDN 防盗链、杀毒扫描和视频水印。
|
||||
2. 公共题库/地区题库授权:已完成披露和采纳快照;继续补版本同步、租户自改冲突处理和按 SaaS 套餐限制地区。
|
||||
3. 学习统计增强:排行榜防刷/预聚合、断点续练、专项练习策略和更细题型分析。
|
||||
4. 视频会员控制:深度防盗链、水印和播放统计。
|
||||
|
||||
@@ -50,6 +50,59 @@ GET /api/tenant-content/imports/issues
|
||||
- `content_import_issues`
|
||||
- `audit_logs`
|
||||
|
||||
## 格式支持
|
||||
|
||||
当前题目、单词、知识手册、分数线和视频导入均支持:
|
||||
|
||||
- `sourceFormat=json`:直接提交规范 JSON 或兼容旧模板。
|
||||
- `sourceFormat=csv`:提交 `csvText`、`fileContent`、`payload` 或 `fileBase64`,后端按表头归一化。
|
||||
- `sourceFormat=excel`:提交 `.xlsx` 的 `fileBase64`,可选 `sheetName` 或 `sheetIndex`;多 Sheet 会被解析进同一个 `content_import_jobs`。
|
||||
- 同步导入默认在 API 请求内完成;大批量导入可在确认导入时传 `executionMode=async`,API 会把 job 置为 `pending`,由 imports worker 后台消费。
|
||||
|
||||
表格导入安全边界:
|
||||
|
||||
- 单文件最大 8MB。
|
||||
- 单次最多解析 5000 行、160 列。
|
||||
- 单个单元格最多保留 100000 字符。
|
||||
- 后端只保存解析后的原始行和规范化 payload,不把 `fileBase64` 长期写入数据库。
|
||||
- 任务列表会返回 `sourceFormat`、`executionMode` 和 `parserMetadata`,前端可展示解析器、Sheet、行数等信息。
|
||||
|
||||
异步导入调用方式:
|
||||
|
||||
```json
|
||||
{
|
||||
"previewJobId": "uuid",
|
||||
"executionMode": "async",
|
||||
"allowPartial": false
|
||||
}
|
||||
```
|
||||
|
||||
异步导入状态流:
|
||||
|
||||
```text
|
||||
preview -> pending -> importing -> completed
|
||||
preview -> pending -> importing -> completed_with_errors
|
||||
preview -> pending -> failed
|
||||
```
|
||||
|
||||
worker 命令:
|
||||
|
||||
```bash
|
||||
npm --workspace @tiku-saas/worker run imports:once
|
||||
```
|
||||
|
||||
前端提交异步导入后不要重复同步执行同一 job;只需要轮询 `GET /api/tenant-content/imports` 并用 `GET /api/tenant-content/imports/issues` 展示问题行。worker 会按 `attempt_count/max_attempts` 记录重试,失败时写入 `errorMessage` 和审计日志。
|
||||
|
||||
推荐 CSV/Excel 表头:
|
||||
|
||||
| 类型 | 常用表头 |
|
||||
| --- | --- |
|
||||
| 题目 | `legacyId`、`题型/type`、`题干/content`、`选项A`-`选项H`、`答案`、`解析`、`难度`、`标签` |
|
||||
| 单词 | `unitLegacyId`、`unitName`、`wordLegacyId`、`word`、`phonetic`、`meaning`、`example`、`difficulty`、`tags` |
|
||||
| 知识手册 | `subjectLegacyId`、`subjectName`、`chapterLegacyId`、`chapterName`、`sectionName`、`entryLegacyId`、`title`、`content`、`summary`、`tags` |
|
||||
| 分数线 | 多 Sheet 推荐 `fields`、`schools`、`majors`、`records`;记录 Sheet 可包含任意动态字段列 |
|
||||
| 视频 | `legacyId`、`title`、`videoUrl`、`thumbnailUrl`、`durationSeconds`、`subjectId`、`questionId` 或 `legacyQuestionId`、`accessMode` |
|
||||
|
||||
## 单词导入
|
||||
|
||||
推荐新格式:
|
||||
@@ -262,5 +315,5 @@ GET /api/tenant-content/imports/issues
|
||||
|
||||
## 下一步
|
||||
|
||||
- 增加 Excel/CSV 解析入口,但解析后仍进入同一套 preview/import 管线。
|
||||
- 增加异步 worker,处理大批量导入、重试和导入后校验。
|
||||
- 增加导入后校验/复检,确认导入行数、目标表记录、题目集合绑定和视频绑定一致。
|
||||
- 增加导入模板下载接口和后台可视化字段映射器。
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# 前端交接索引
|
||||
|
||||
更新时间:2026-06-28
|
||||
更新时间:2026-06-29
|
||||
|
||||
这份文件是给 Taro/H5/小程序前端同事的入口。当前仓库的前端重构建议从 `apps/taro` 新建工程开始,不再把旧 React/Vite 前端搬回根目录继续开发。
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
- 不要把“Supabase 支持前端 Data API”误解为“本项目所有业务表都由 Taro 直写”;订单、支付、权益、租户后台、导入、CRM、私有资源必须走 RPC、`apps/api`、Edge Function 或 worker 这类后端命令层。
|
||||
- 真实短信、微信登录、QQ 登录、微信支付、支付宝支付 provider 还未正式接完。
|
||||
- 对象存储已完成签名 provider、上传后校验、PDF/图片预览和资源复检 worker,但 CDN 防盗链、视频水印和杀毒扫描还要补。
|
||||
- 分数线/视频 JSON 导入已可联调;大批量 Excel/CSV 解析和异步导入 worker 还未完成。
|
||||
- 题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 导入已可联调;大批量导入可传 `executionMode=async` 交给 imports worker,前端还要补上传预览、排队轮询、问题行展示、模板下载和字段映射 UI。
|
||||
- 数据看板、分佣结算和勋章手动发放基础 API 已可联调;勋章自动发放、分佣真实打款/导出/凭证、AI 择校、主题模板市场等仍是后续商用增强项。
|
||||
|
||||
## 前后端协作建议
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
## 当前结论
|
||||
|
||||
当前重构已经完成了 Supabase/PostgreSQL 多租户底座、核心业务表、PocketBase 数据导入器雏形、学生端核心 API、租户后台 API、平台后台 SaaS 账务 API、内容资产/题目/单词/知识手册/分数线/视频 JSON 批量导入基础闭环、题库入口/任意深度分类/题目集合/练习蓝图/组卷快照基础闭环,以及本地 Docker/API 构建验证。
|
||||
当前重构已经完成了 Supabase/PostgreSQL 多租户底座、核心业务表、PocketBase 数据导入器雏形、学生端核心 API、租户后台 API、平台后台 SaaS 账务 API、内容资产/题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 批量导入基础闭环、异步导入 worker、题库入口/任意深度分类/题目集合/练习蓝图/组卷快照基础闭环,以及本地 Docker/API 构建验证。
|
||||
|
||||
但这还不是完整商用交付状态,也不能说旧项目核心功能已经全部重构完成。现在更准确的状态是:后端商用架构骨架已经立住,核心业务正在按模块补齐。部分功能已经有可调用 API,部分功能只有数据模型和导入映射,部分功能还没有前端/自动化测试闭环。
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
| 模块 | 数据模型 | PocketBase 导入 | API | 自动化测试 | 当前状态 |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 多租户隔离 | 已建 `tenants`、`tenant_domains`、`tenant_branding`、`tenant_settings`、RLS 基础 | 部分支持 | 租户解析、品牌、域名、支付账户、登录 provider、平台建租户已实现 | 核心 API 集成测试含租户隔离断言 | 基础可用,正式 JWT/RLS 权限闭环未完成 |
|
||||
| 刷题题库 | 已建题库、题目、题目版本、内容入口、任意深度分类树、考试意向标记、题目集合、练习蓝图、导入任务台账、公共题库授权/采纳表 | 已支持核心映射,JSON 导入可落到新入口/节点/集合 | 题目列表、内容入口、分类树、集合题目、顺序/随机/全真模拟 session、答题提交、租户后台题目录入/更新、JSON 预览/导入、平台公共题库授权、租户采纳快照已实现 | 核心 API 集成测试含导航、组卷、导入、公共题库授权和采纳后组卷断言 | 新题库导航和组卷基础闭环可跑,公共题库采纳快照可联调;Excel 导入、公共题库全量/增量版本同步仍需补齐 |
|
||||
| 刷题题库 | 已建题库、题目、题目版本、内容入口、任意深度分类树、考试意向标记、题目集合、练习蓝图、导入任务台账、公共题库授权/采纳表 | 已支持核心映射,JSON/CSV/Excel 导入可落到新入口/节点/集合 | 题目列表、内容入口、分类树、集合题目、顺序/随机/全真模拟 session、答题提交、租户后台题目录入/更新、JSON/CSV/Excel 预览/导入、异步导入 worker、平台公共题库授权、租户采纳快照已实现 | 核心 API 集成测试含导航、组卷、导入、公共题库授权和采纳后组卷断言 | 新题库导航和组卷基础闭环可跑,公共题库采纳快照可联调;公共题库全量/增量版本同步、导入后复检仍需补齐 |
|
||||
| 错题本 | 已建 `wrong_questions` | 已支持旧错题归一化 | 错题列表、答题自动入错题、移出错题已实现 | 仅烟测 | 基础功能已实现,复习计划和统计未完成 |
|
||||
| 收藏夹 | 已建 `favorite_questions` | 已支持旧收藏归一化 | 收藏/取消收藏、收藏列表已实现 | 仅烟测 | 基础功能已实现 |
|
||||
| 用户订阅/题库会员/SVIP | 已建 `orders`、`payments`、`entitlements`、`svip_plans`、激活码 | 已映射旧 SVIP/会员权益 | 下单、订单详情/状态轮询、手工支付确认权限保护、微信/支付宝支付、微信/支付宝发起退款、微信/支付宝退款查询确认、微信/支付宝退款通知 webhook、激活码预检查/兑换、优惠券抵扣、零元订单自动开通、权益查询已实现 | API 集成测试 | 商城主链路可联调,对账、支付补偿和异常订单自动处理待补 |
|
||||
@@ -36,7 +36,7 @@
|
||||
| 个人中心 | 已建 `student_profiles`、会员权益、订单、练习记录、`badges/user_badges` | 已支持部分用户资料和勋章导入 | 个人资料、目标院校/专业、会员状态、最近练习、统计聚合、签到积分、题目反馈、考试倒计时、勋章 API 已实现 | API 集成测试 | 学生端基础个人中心已实现,账号绑定/换绑、学习报告可视化和更细任务系统待补 |
|
||||
| 活动/优惠 | 已建优惠券、激活码、激活码批次、banner、FAQ、公告、勋章等基础表 | 部分支持 | banner/FAQ/公告只读与租户后台维护、激活码预检查/兑换、激活码批次、批量生成激活码、优惠券维护、前台领取/下单抵扣、勋章维护和手动发放已实现 | 核心 API 集成测试 | 基础运营后台可用,勋章自动发放、复杂活动规则、营销自动化、核销报表待补 |
|
||||
| 销售/代理客资追踪 | 已建推荐码、首绑客资、团队关系、小程序码缓存、CRM 队列 | 旧 `referral_tracks` 已有映射基础 | 邀请码、扫码/分享事件、首绑保护、销售统计、客资明细、手动补绑、团队关系、CRM 配置/队列、CRM worker 推送已实现 | 核心 API 集成测试、CRM worker 集成测试 | 增长链路基础可用,真实微信小程序码、CRM 分配策略、富卡片和销售转化看板待补 |
|
||||
| 租户后台 | 已建品牌、域名、设置、支付账户、登录 provider、私密密钥表、成员、审计日志、资源台账、导入台账、内容导航台账 | 不适用 | 概览、品牌、设置、域名、支付账户、登录配置、密钥掩码、活动内容、兑换码/优惠券、成员管理、权限矩阵、审计查询、内容入口/分类树/题目集合/练习蓝图维护、资源管理、题目/单词/知识手册/分数线/视频 JSON 导入已实现 | 核心 API 集成测试含角色/权限/租户隔离/密钥不泄露/导航/组卷/资源与导入断言 | 租户配置与运营闭环可用,前端权限 UI、Excel/CSV 导入和大批量异步导入待补 |
|
||||
| 租户后台 | 已建品牌、域名、设置、支付账户、登录 provider、私密密钥表、成员、审计日志、资源台账、导入台账、内容导航台账 | 不适用 | 概览、品牌、设置、域名、支付账户、登录配置、密钥掩码、活动内容、兑换码/优惠券、成员管理、权限矩阵、审计查询、内容入口/分类树/题目集合/练习蓝图维护、资源管理、题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 同步/异步导入已实现 | 核心 API 集成测试含角色/权限/租户隔离/密钥不泄露/导航/组卷/资源与导入断言 | 租户配置与运营闭环可用,前端权限 UI、导入后复检和字段映射操作台待补 |
|
||||
| 平台后台 | 已建 SaaS 套餐、订阅、账单、服务费、用量 | 不适用 | 租户管理、账单、收款确认、用量记录、平台管理员 Supabase JWT 鉴权已实现 | API 集成测试 | 平台收费链路骨架可用,平台审计报表/自动计费待补 |
|
||||
| 登录认证 | 已建短信验证码、会话、OAuth provider 配置表,并支持 `auth_user_id` 映射 | 旧用户映射已预留 | 短信 mock 登录、迁移期 session、Supabase JWT 验签映射、微信小程序登录主链路已实现 | API 集成测试 | H5 Supabase Auth 可联调;真实短信/微信网页/QQ 登录生产联调待补 |
|
||||
| 数据导入 | 已建立 importer、risk report、validate | 已覆盖多类旧集合 | 命令行导入/校验 | `pb:import:validate` | 基础工具可用,需用真实完整数据做多轮 dry-run |
|
||||
@@ -268,7 +268,7 @@ platform-admin:
|
||||
1. 正式鉴权:API 已支持 Supabase Auth JWT;生产前继续做真实云端 Auth/JWKS 回归、RLS 深测,并关闭 `x-user-id`、`x-platform-admin-key` 兼容入口。
|
||||
2. 国内能力接入:短信、微信小程序登录、微信支付、支付宝支付、发起退款、退款查询确认和退款通知 webhook 的租户级配置入口与本地 provider 验证已具备;微信网页登录、QQ 登录、真实生产账号联调、对账和支付补偿仍需实现。
|
||||
3. 核心缺口 API:学生端个人中心、分数线、题目视频详情、背单词进度/收藏、签到积分、题目反馈和勋章已补基础 API;下一步重点是账号绑定、学习报告可视化、后台统计和真实业务验收。
|
||||
4. 后台能力:题库录入、题目/单词/知识手册/分数线/视频 JSON 批量导入、资源台账、视频绑定、知识手册维护、分数线维护、品牌/商户/登录/活动/兑换码配置、销售客资、CRM 队列、成员权限、审计查询已补 API;Excel/CSV 导入、大批量异步导入和前端操作台待补。
|
||||
4. 后台能力:题库录入、题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 同步/异步批量导入、资源台账、视频绑定、知识手册维护、分数线维护、品牌/商户/登录/活动/兑换码配置、销售客资、CRM 队列、成员权限、审计查询已补 API;导入后复检、模板下载/字段映射和前端操作台待补。
|
||||
5. 自动化测试:已建立核心 API、租户隔离、权限矩阵、后台维护、资源/导入、微信/支付宝支付 webhook、优惠券/激活码/订单状态集成测试;仍需真实数据导入回归、退款对账和前端端到端测试。
|
||||
6. Taro 前端:建立 `apps/taro` 或等价跨端应用,把 H5 和小程序统一走同一套 API client。
|
||||
7. 运维交付:生产环境变量、备份恢复、日志监控、异常告警、数据库迁移流程、灰度发布、回滚预案。
|
||||
@@ -277,7 +277,7 @@ platform-admin:
|
||||
|
||||
为了先把旧项目核心业务补齐,再进入支付/短信等商用关键模块,建议按下面顺序继续:
|
||||
|
||||
1. 完善内容导入和文件上传:Excel/CSV 解析、大批量异步导入、导入后复检、CDN 防盗链、杀毒扫描,并把 JSON 导入扩展为异步 worker。
|
||||
1. 完善内容导入和文件上传:导入后复检、模板下载/字段映射 UI、CDN 防盗链、杀毒扫描。
|
||||
2. 补公共题库版本同步 worker、租户套餐地区/科目/题库范围限制、主题模板系统。
|
||||
3. 补学习统计增强:排行榜防刷/预聚合、断点续练、专项练习策略和更细题型分析。
|
||||
4. 补视频商用控制:深度防盗链、动态水印和播放统计。
|
||||
|
||||
@@ -58,7 +58,7 @@
|
||||
| 优惠券 | 已覆盖 | 后台配置、前台领取、同用户同券幂等、下单抵扣、全额优惠自动开通权益已有;复杂活动规则和核销报表待补 |
|
||||
| 激活码 | 已覆盖 | 批次、生成、预检查、兑换、自用码拒绝、地区校验主链路已有 |
|
||||
| 勋章管理 | 部分覆盖 | 后台勋章维护、手动发放、重复发放幂等、学生端勋章展示和权限隔离已覆盖;自动发放规则、积分活动联动和前端运营 UI 待补 |
|
||||
| 题库录入 | 已覆盖 | 单题创建/更新、题目/单词/知识手册/分数线/视频 JSON 导入、集合/蓝图已有;Excel/CSV 和异步大批量导入待补 |
|
||||
| 题库录入 | 已覆盖 | 单题创建/更新、题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 同步/异步导入、集合/蓝图已有;导入后复检、模板下载和字段映射 UI 待补 |
|
||||
| 题库导出 PDF/Word/JSON | 未覆盖 | 旧前端有导出组件;新后端需决定服务端导出、导出水印和权限审计 |
|
||||
| 题型分组/模拟卷配置 | 部分覆盖 | question_type_groups 表和 blueprint 有基础;后台配置体验待补 |
|
||||
| 背单词维护 | 已覆盖 | 单元/单词 CRUD 和导入已有 |
|
||||
@@ -98,7 +98,7 @@
|
||||
1. 排行榜增强:刷题、模考、背单词、积分排行榜主接口已有;还需防刷、日/周榜预聚合、运营后台排名看板。
|
||||
2. 账号设置完整流:头像上传、绑定/更换手机号、微信/QQ 账号合并、密码/邮箱能力。
|
||||
3. 题库导出:PDF/Word/JSON 导出、水印、导出审计和权限控制。
|
||||
4. 导入扩展:分数线/视频 JSON 导入已补齐;仍缺 Excel/CSV 解析、大批量异步 worker 和导入后复检。
|
||||
4. 导入扩展:题目/单词/知识手册/分数线/视频已支持 JSON、CSV 和 Excel 预览导入,并可用 `executionMode=async` 进入 imports worker;仍缺导入后复检和模板下载/字段映射 UI。
|
||||
5. 公共题库商业化:平台公共/地区题库授权和租户快照采纳已完成基础闭环;还需版本同步、租户自改冲突处理和运营后台 UI。
|
||||
6. CRM/销售结算:CRM worker、分佣规则、结算单、审核和打款状态基础闭环已完成;仍缺轮询/定向分配、打款导出、凭证和销售结算看板。
|
||||
7. 题目反馈增强:处理通知、消息提醒、问题聚合统计和内容修复闭环。
|
||||
@@ -117,7 +117,7 @@
|
||||
|
||||
1. 微信/支付宝支付和 webhook 幂等。
|
||||
2. 对象存储 PDF 预览、视频深度防盗链、动态水印。
|
||||
3. Excel/CSV 解析、大批量异步导入和导入后复检。
|
||||
3. 导入后复检、模板下载和字段映射 UI。
|
||||
4. 数据看板预聚合 worker、销售/代理转化看板和分佣结算。
|
||||
5. 公共题库版本同步、租户采纳后的更新策略和同步 worker。
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
- 练习组卷:`question_collections/practice_blueprints` 支持顺序、随机、全真模拟快照。
|
||||
- 模考报告与学习统计:`practice_session_reports/practice_session_report_sections` 支持交卷、评分、题型/小节统计、错题解析汇总和历史查询;`/api/learning/stats`、`trend`、`practice-sessions/history`、`wrong-questions/review-plan` 可支撑个人中心和学习报告基础页。
|
||||
- 练习访问控制:`practice_daily_usage/practice_access_events` 支持免费每日额度、SVIP 范围校验、SVIP-only 内容拦截和答题 session 快照保护。
|
||||
- 内容导入:题目、单词、知识手册、分数线、视频 JSON 预览、校验、导入、幂等、审计。
|
||||
- 内容导入:题目、单词、知识手册、分数线、视频 JSON/CSV/Excel 预览、校验、导入、幂等、审计。
|
||||
- 租户组织范围:班级、班级成员、教师/班主任/助教/学生分组,教师按负责班级查看学生,字段权限可脱敏学生手机号。
|
||||
- 学生运营管理:学生批量 upsert、禁用/恢复、批量分班、备注、跟进任务已完成接口和集成测试;后续补批量 CRM 推送和自动学习督导。
|
||||
- 旧题库运营缺口已补一批:考试日期/倒计时、题目反馈/纠错处理、每日签到积分和积分流水、学习排行榜已完成接口和集成测试。
|
||||
@@ -75,9 +75,9 @@
|
||||
- 旧 PocketBase 用户账号和新身份体系的映射/补绑。
|
||||
|
||||
3. 导入体系扩展
|
||||
- Excel/CSV 导入。
|
||||
- 分数线/视频导入已完成 JSON preview/import;继续补 Excel/CSV 到规范 JSON 的解析适配。
|
||||
- 大批量导入异步 worker、重试、导入后校验。
|
||||
- 已完成题目、单词、知识手册、分数线、视频的 CSV/Excel 到规范 JSON 解析适配。
|
||||
- 继续补大批量导入异步 worker、重试、导入后校验。
|
||||
- 继续补模板下载、字段映射 UI 和导入前抽样校验。
|
||||
|
||||
4. 公共题库和租户授权
|
||||
- 已完成平台公共题库/地区题库的基础授权、租户采纳和题目快照复制。
|
||||
@@ -206,5 +206,5 @@
|
||||
2. 云服务器部署 Supabase/PostgreSQL 和 API,配置对象存储生产环境变量,跑 `check:refactor` 的远程等价测试。
|
||||
3. 导出现有 PocketBase 数据,做完整 dry-run 迁移。
|
||||
4. 开始 `apps/taro`,先接租户解析、首页、题库、背单词、知识手册。
|
||||
5. 并行补对象存储、真实登录、完整资金流水对账和公共题库版本同步 worker。
|
||||
5. 并行补对象存储、真实登录、完整资金流水对账、题库导出和公共题库版本同步 worker。
|
||||
6. 前后端联调通过后,再做支付、权限、数据导入、资料下载、视频播放的商用验收。
|
||||
|
||||
@@ -713,7 +713,7 @@ content_entries
|
||||
|
||||
## 租户内容导入对接
|
||||
|
||||
租户后台导入统一使用 preview -> issues -> import 流程,前端不要直接写 Supabase 表或绕过 `apps/api`。
|
||||
租户后台导入统一使用 preview -> issues -> import 流程,前端不要直接写 Supabase 表或绕过 `apps/api`。当前 JSON、CSV 和 Excel 都进入同一套后端规范化、逐行 issue、幂等和审计管线。
|
||||
|
||||
当前可联调:
|
||||
|
||||
@@ -734,14 +734,63 @@ GET /api/tenant-content/imports/issues
|
||||
|
||||
前端流程:
|
||||
|
||||
1. 上传或粘贴 JSON,先调用对应 preview。
|
||||
1. 上传或粘贴 JSON/CSV/Excel,先调用对应 preview。
|
||||
2. 展示 `job.totalCount/validCount/errorCount/warningCount`。
|
||||
3. 展示逐行 `issues`,错误行必须让运营修正;如果后端允许 `allowPartial`,也要二次确认。
|
||||
4. 用户确认后调用 import;导入完成后刷新内容列表、分数线列表或题目视频列表。
|
||||
3. 展示 `job.sourceFormat`、`job.parserMetadata`、逐行 `issues`,错误行必须让运营修正;如果后端允许 `allowPartial`,也要二次确认。
|
||||
4. 小批量确认后直接调用 import;大批量确认时传 `executionMode=async` 排队,前端轮询 job 状态。
|
||||
5. 导入完成后刷新内容列表、分数线列表或题目视频列表。
|
||||
|
||||
CSV 请求示例:
|
||||
|
||||
```json
|
||||
{
|
||||
"sourceFormat": "csv",
|
||||
"sourceName": "questions.csv",
|
||||
"csvText": "legacyId,题型,题干,选项A,选项B,答案\nq1,choice,题干,A,B,B",
|
||||
"subjectId": "...",
|
||||
"categoryId": "...",
|
||||
"entryId": "...",
|
||||
"contentNodeId": "...",
|
||||
"collectionId": "..."
|
||||
}
|
||||
```
|
||||
|
||||
Excel 请求示例:
|
||||
|
||||
```json
|
||||
{
|
||||
"sourceFormat": "excel",
|
||||
"sourceName": "scoreline.xlsx",
|
||||
"fileBase64": "<xlsx base64>",
|
||||
"sheetName": "records",
|
||||
"regionId": "..."
|
||||
}
|
||||
```
|
||||
|
||||
异步确认导入示例:
|
||||
|
||||
```json
|
||||
{
|
||||
"previewJobId": "uuid",
|
||||
"executionMode": "async",
|
||||
"allowPartial": false
|
||||
}
|
||||
```
|
||||
|
||||
异步导入状态:
|
||||
|
||||
```text
|
||||
pending/importing:展示处理中,不允许重复同步执行同一 job。
|
||||
completed:刷新目标内容列表。
|
||||
completed_with_errors:刷新成功内容,并提示查看 issues。
|
||||
failed/rejected:展示 errorMessage 和 issues,允许运营修正后重新 preview。
|
||||
```
|
||||
|
||||
前端文件限制应与后端一致:单文件最大 8MB,最多 5000 行、160 列。后端不会保存原始 `fileBase64`,但前端仍不要把含隐私的导入文件写入长期缓存。
|
||||
|
||||
分数线导入前端注意:
|
||||
|
||||
- 后端支持 `fields/schools/majors/records` 分桶,也支持 `items` 列表。
|
||||
- 后端支持 `fields/schools/majors/records` 分桶,也支持 `items` 列表;Excel 可用 `fields`、`schools`、`majors`、`records` 多 Sheet。
|
||||
- 页面筛选字段仍以 `/api/scoreline/fields` 为准,不要从导入 JSON 临时生成筛选 UI。
|
||||
- `record` 至少需要 `schoolId`、`schoolLegacyId` 或 `schoolName`,否则 preview 会返回 issue。
|
||||
|
||||
@@ -1081,7 +1130,7 @@ ACTIVATION_CODE_REGION_MISMATCH
|
||||
- 角色模板:`GET/PUT /api/tenant-admin/role-templates`、`POST /api/tenant-admin/role-templates/disable`
|
||||
- 内容入口/分类树/题目集合/练习蓝图
|
||||
- 题目/单词/知识手册/分数线/视频维护
|
||||
- 题目/单词/知识手册/分数线/视频 JSON 导入 preview/import/issues
|
||||
- 题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 导入 preview/import/issues
|
||||
- Banner/FAQ/公告/激活码/优惠券
|
||||
- 勋章:`GET/PUT /api/tenant-admin/badges`、`GET/POST /api/tenant-admin/badge-grants`
|
||||
- 考试日期:`GET/PUT /api/tenant-admin/exam-dates`
|
||||
|
||||
826
package-lock.json
generated
826
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -36,6 +36,7 @@
|
||||
"test:worker:crm": "npm run db:smoke-seed && npm run build:worker && node scripts/crm-worker-integration-test.js",
|
||||
"test:worker:commerce": "npm run db:smoke-seed && npm run build:worker && node scripts/commerce-worker-integration-test.js",
|
||||
"test:worker:assets": "npm run db:smoke-seed && npm run build:worker && node scripts/asset-worker-integration-test.js",
|
||||
"test:worker:imports": "npm run db:smoke-seed && npm run build:worker && node scripts/import-worker-integration-test.js",
|
||||
"test:api:remote": "node scripts/api-integration-test.js",
|
||||
"pb:schema:summary": "npm --workspace @tiku-saas/import-pocketbase run schema:summary",
|
||||
"pb:schema:risk": "npm --workspace @tiku-saas/import-pocketbase run schema:risk",
|
||||
|
||||
@@ -4,6 +4,7 @@ import { spawn } from 'node:child_process';
|
||||
import http from 'node:http';
|
||||
import net from 'node:net';
|
||||
import { SignJWT } from 'jose';
|
||||
import ExcelJS from 'exceljs';
|
||||
|
||||
const DEFAULT_DATABASE_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres';
|
||||
const MAIN_TENANT_ID = process.env.TENANT_ID || '00000000-0000-0000-0000-000000000001';
|
||||
@@ -2794,6 +2795,102 @@ async function testTenantContentAssetsAndImports() {
|
||||
'catalog should expose imported question through the new collection binding',
|
||||
);
|
||||
|
||||
const questionCsv = [
|
||||
['legacyId', '题型', '题干', '选项A', '选项B', '选项C', '选项D', '答案', '解析', '难度', '标签'].join(','),
|
||||
['integration-import-csv-choice-001', 'choice', 'CSV导入题:多租户数据隔离应主要依赖什么?', '前端隐藏菜单', '后端权限和RLS', '浏览器缓存', '静态页面', 'B', '最终权限以后端和RLS为准。', '2', 'csv|import'].join(','),
|
||||
].join('\n');
|
||||
const csvQuestionPreview = await request('/api/tenant-content/imports/preview/questions', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
method: 'POST',
|
||||
body: {
|
||||
sourceFormat: 'csv',
|
||||
sourceName: 'questions.csv',
|
||||
csvText: questionCsv,
|
||||
subjectId: ids.subject,
|
||||
categoryId: ids.category,
|
||||
regionId: ids.region,
|
||||
entryId: ids.contentEntry,
|
||||
contentNodeId: ids.contentNodeSchoolTarget,
|
||||
collectionId: ids.questionCollection,
|
||||
},
|
||||
});
|
||||
assert.equal(csvQuestionPreview.job?.errorCount, 0, 'CSV question preview should have no errors');
|
||||
assert.equal(csvQuestionPreview.job?.validCount, 1, 'CSV question preview should normalize one row');
|
||||
assert.equal(csvQuestionPreview.items?.[0]?.normalized?.correctOptionIndices?.[0], 1, 'CSV answer B should map to option index 1');
|
||||
|
||||
const csvQuestionImport = await request('/api/tenant-content/imports/questions', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
method: 'POST',
|
||||
body: { previewJobId: csvQuestionPreview.job.id },
|
||||
});
|
||||
assert.equal(csvQuestionImport.item?.status, 'completed', 'CSV question import should complete');
|
||||
|
||||
const csvQuestionJobs = await request('/api/tenant-content/imports', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
query: { importType: 'questions', limit: 20 },
|
||||
});
|
||||
assert.ok(
|
||||
csvQuestionJobs.items?.some(item => item.id === csvQuestionPreview.job.id && item.sourceFormat === 'csv'),
|
||||
'CSV question import job should retain sourceFormat',
|
||||
);
|
||||
|
||||
const importedCsvQuestions = await request('/api/catalog/questions', {
|
||||
query: { collectionId: ids.questionCollection, limit: 150 },
|
||||
});
|
||||
assert.ok(
|
||||
importedCsvQuestions.items?.some(item => item.content === 'CSV导入题:多租户数据隔离应主要依赖什么?'),
|
||||
'catalog should expose imported CSV question',
|
||||
);
|
||||
|
||||
const asyncQuestionPreview = await request('/api/tenant-content/imports/preview/questions', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
method: 'POST',
|
||||
body: {
|
||||
sourceName: 'async-question-import.json',
|
||||
subjectId: ids.subject,
|
||||
categoryId: ids.category,
|
||||
regionId: ids.region,
|
||||
entryId: ids.contentEntry,
|
||||
contentNodeId: ids.contentNodeSchoolTarget,
|
||||
collectionId: ids.questionCollection,
|
||||
items: [
|
||||
{
|
||||
legacyId: 'integration-import-async-choice-001',
|
||||
type: 'choice',
|
||||
content: '异步导入排队题:大批量导入应该交给谁执行?',
|
||||
options: ['前端循环写表', '导入 worker', '用户刷新页面', '浏览器缓存'],
|
||||
correctOptionIndices: [1],
|
||||
explanation: '大批量导入由后端 worker 消费,避免请求超时和规则漂移。',
|
||||
difficulty: 2,
|
||||
tags: ['async-import'],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
assert.equal(asyncQuestionPreview.job?.errorCount, 0, 'async preview should have no errors');
|
||||
const asyncQuestionQueued = await request('/api/tenant-content/imports/questions', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
method: 'POST',
|
||||
body: { previewJobId: asyncQuestionPreview.job.id, executionMode: 'async' },
|
||||
});
|
||||
assert.equal(asyncQuestionQueued.item?.status, 'pending', 'async import should queue the job');
|
||||
assert.equal(asyncQuestionQueued.item?.executionMode, 'async', 'async import response should expose execution mode');
|
||||
|
||||
const queuedAgain = await request('/api/tenant-content/imports/questions', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
method: 'POST',
|
||||
body: { previewJobId: asyncQuestionPreview.job.id, executionMode: 'async' },
|
||||
});
|
||||
assert.equal(queuedAgain.item?.idempotent, true, 'queueing the same async job should be idempotent');
|
||||
|
||||
const queuedSyncExecution = await request('/api/tenant-content/imports/questions', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
method: 'POST',
|
||||
body: { previewJobId: asyncQuestionPreview.job.id },
|
||||
expectStatus: 409,
|
||||
});
|
||||
assert.equal(queuedSyncExecution.code, 'IMPORT_JOB_QUEUED', 'queued import job should not be executed synchronously');
|
||||
|
||||
const vocabEntry = await request('/api/tenant-content/content-entries', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
method: 'PUT',
|
||||
@@ -2913,6 +3010,42 @@ async function testTenantContentAssetsAndImports() {
|
||||
});
|
||||
assert.ok(vocabularyWords.items?.some(item => item.word === 'abandon' && item.contentNodeId), 'catalog should expose imported vocabulary word with node binding');
|
||||
|
||||
const vocabularyCsv = [
|
||||
['unitLegacyId', 'unitName', 'wordLegacyId', 'word', 'phonetic', 'meaning', 'example', 'difficulty', 'tags'].join(','),
|
||||
['integration-vocab-csv-unit-001', 'CSV 高频词单元', 'integration-vocab-csv-word-scale', 'scale', '/skeɪl/', 'n. 规模;等级', 'SaaS platforms must scale safely.', '2', 'csv|高频'].join(','),
|
||||
].join('\n');
|
||||
const vocabularyCsvPreview = await request('/api/tenant-content/imports/preview/vocabulary', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
method: 'POST',
|
||||
body: {
|
||||
sourceFormat: 'csv',
|
||||
sourceName: 'vocabulary.csv',
|
||||
csvText: vocabularyCsv,
|
||||
regionId: ids.region,
|
||||
entryId: vocabEntry.item.id,
|
||||
contentNodeId: vocabRoot.item.id,
|
||||
},
|
||||
});
|
||||
assert.equal(vocabularyCsvPreview.job?.errorCount, 0, 'CSV vocabulary preview should have no errors');
|
||||
assert.equal(vocabularyCsvPreview.items?.[0]?.normalized?.words?.[0]?.word, 'scale', 'CSV vocabulary should group words under units');
|
||||
|
||||
const vocabularyCsvImport = await request('/api/tenant-content/imports/vocabulary', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
method: 'POST',
|
||||
body: { previewJobId: vocabularyCsvPreview.job.id },
|
||||
});
|
||||
assert.equal(vocabularyCsvImport.item?.status, 'completed', 'CSV vocabulary import should complete');
|
||||
|
||||
const vocabularyUnitsAfterCsv = await request('/api/catalog/vocabulary-units', {
|
||||
query: { regionId: ids.region },
|
||||
});
|
||||
const csvVocabularyUnit = vocabularyUnitsAfterCsv.items?.find(item => item.legacyId === 'integration-vocab-csv-unit-001');
|
||||
assert.ok(csvVocabularyUnit, 'catalog should expose CSV imported vocabulary unit');
|
||||
const csvVocabularyWords = await request('/api/catalog/vocabulary-words', {
|
||||
query: { unitId: csvVocabularyUnit.id },
|
||||
});
|
||||
assert.ok(csvVocabularyWords.items?.some(item => item.word === 'scale'), 'catalog should expose CSV imported vocabulary word');
|
||||
|
||||
const handbookEntry = await request('/api/tenant-content/content-entries', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
method: 'PUT',
|
||||
@@ -3142,6 +3275,52 @@ async function testTenantContentAssetsAndImports() {
|
||||
'mixed scoreline import should create record after school and major',
|
||||
);
|
||||
|
||||
const scorelineWorkbook = new ExcelJS.Workbook();
|
||||
scorelineWorkbook.addWorksheet('fields').addRows([
|
||||
['legacyId', 'fieldKey', 'fieldName', 'fieldType', 'isTrend'],
|
||||
['integration-scoreline-xlsx-field-min', 'xlsxMinScore', 'Excel最低分', 'number', '是'],
|
||||
]);
|
||||
scorelineWorkbook.addWorksheet('schools').addRows([
|
||||
['legacyId', 'schoolName', 'isHot'],
|
||||
['integration-scoreline-xlsx-school', 'Excel导入学院', 'true'],
|
||||
]);
|
||||
scorelineWorkbook.addWorksheet('majors').addRows([
|
||||
['legacyId', 'schoolLegacyId', 'majorName'],
|
||||
['integration-scoreline-xlsx-major', 'integration-scoreline-xlsx-school', 'Excel专业'],
|
||||
]);
|
||||
scorelineWorkbook.addWorksheet('records').addRows([
|
||||
['legacyId', 'schoolLegacyId', 'majorLegacyId', 'year', 'xlsxMinScore'],
|
||||
['integration-scoreline-xlsx-record-2027', 'integration-scoreline-xlsx-school', 'integration-scoreline-xlsx-major', 2027, 233],
|
||||
]);
|
||||
const scorelineExcelBuffer = await scorelineWorkbook.xlsx.writeBuffer();
|
||||
const scorelineExcelPreview = await request('/api/tenant-content/imports/preview/scoreline', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
method: 'POST',
|
||||
body: {
|
||||
sourceFormat: 'excel',
|
||||
sourceName: 'scoreline.xlsx',
|
||||
fileBase64: Buffer.from(scorelineExcelBuffer).toString('base64'),
|
||||
regionId: ids.region,
|
||||
},
|
||||
});
|
||||
assert.equal(scorelineExcelPreview.job?.errorCount, 0, 'Excel scoreline preview should have no errors');
|
||||
assert.equal(scorelineExcelPreview.job?.validCount, 4, 'Excel scoreline preview should parse multi-sheet workbook');
|
||||
|
||||
const scorelineExcelImport = await request('/api/tenant-content/imports/scoreline', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
method: 'POST',
|
||||
body: { previewJobId: scorelineExcelPreview.job.id },
|
||||
});
|
||||
assert.equal(scorelineExcelImport.item?.status, 'completed', 'Excel scoreline import should complete');
|
||||
|
||||
const excelScorelineRecords = await request('/api/scoreline/records', {
|
||||
query: { regionId: ids.region, year: 2027, pageSize: 50 },
|
||||
});
|
||||
assert.ok(
|
||||
excelScorelineRecords.items?.some(item => item.schoolName === 'Excel导入学院' && item.fieldValues?.xlsxMinScore === 233),
|
||||
'public scoreline records should expose imported Excel record',
|
||||
);
|
||||
|
||||
const invalidVideoPreview = await request('/api/tenant-content/imports/preview/videos', {
|
||||
userId: TENANT_ADMIN_USER_ID,
|
||||
method: 'POST',
|
||||
|
||||
299
scripts/import-worker-integration-test.js
Normal file
299
scripts/import-worker-integration-test.js
Normal file
@@ -0,0 +1,299 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import pg from 'pg';
|
||||
import { spawn } from 'node:child_process';
|
||||
|
||||
const databaseUrl = process.env.DATABASE_URL || 'postgresql://postgres:postgres@127.0.0.1:54322/postgres';
|
||||
const tenantId = '00000000-0000-0000-0000-000000000001';
|
||||
const adminUserId = '00000000-0000-0000-0000-000000000102';
|
||||
const ids = {
|
||||
region: '00000000-0000-0000-0000-000000000301',
|
||||
subject: '00000000-0000-0000-0000-000000000501',
|
||||
category: '00000000-0000-0000-0000-000000000601',
|
||||
contentEntry: '00000000-0000-0000-0000-000000000611',
|
||||
contentNodeSchoolTarget: '00000000-0000-0000-0000-000000000614',
|
||||
questionCollection: '00000000-0000-0000-0000-000000000615',
|
||||
};
|
||||
|
||||
function runWorkerOnce() {
|
||||
const child = spawn(process.execPath, ['apps/worker/dist/apps/worker/src/index.js', '--once', '--job', 'imports'], {
|
||||
cwd: process.cwd(),
|
||||
env: {
|
||||
...process.env,
|
||||
DATABASE_URL: databaseUrl,
|
||||
WORKER_IMPORT_BATCH_SIZE: '5',
|
||||
WORKER_IMPORT_ID: 'imports-integration-test',
|
||||
},
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
windowsHide: true,
|
||||
});
|
||||
let output = '';
|
||||
child.stdout.on('data', chunk => {
|
||||
output += chunk.toString();
|
||||
});
|
||||
child.stderr.on('data', chunk => {
|
||||
output += chunk.toString();
|
||||
});
|
||||
return new Promise((resolve, reject) => {
|
||||
child.on('error', reject);
|
||||
child.on('exit', code => {
|
||||
try {
|
||||
assert.equal(code, 0, `worker should exit 0\n${output}`);
|
||||
assert.match(output, /imports batch processed=\d+/, 'worker output should include imports summary');
|
||||
resolve(output);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function cleanup(pool) {
|
||||
await pool.query(
|
||||
`
|
||||
delete from public.question_collection_items
|
||||
where tenant_id = $1
|
||||
and question_id in (
|
||||
select id from public.questions
|
||||
where tenant_id = $1
|
||||
and (
|
||||
legacy_id like 'worker-import-question-%'
|
||||
or legacy_id like 'integration-import-async-choice-%'
|
||||
)
|
||||
)
|
||||
`,
|
||||
[tenantId],
|
||||
);
|
||||
await pool.query(
|
||||
`
|
||||
delete from public.question_versions
|
||||
where tenant_id = $1
|
||||
and question_id in (
|
||||
select id from public.questions
|
||||
where tenant_id = $1
|
||||
and (
|
||||
legacy_id like 'worker-import-question-%'
|
||||
or legacy_id like 'integration-import-async-choice-%'
|
||||
)
|
||||
)
|
||||
`,
|
||||
[tenantId],
|
||||
);
|
||||
await pool.query(
|
||||
`
|
||||
delete from public.questions
|
||||
where tenant_id = $1
|
||||
and (
|
||||
legacy_id like 'worker-import-question-%'
|
||||
or legacy_id like 'integration-import-async-choice-%'
|
||||
)
|
||||
`,
|
||||
[tenantId],
|
||||
);
|
||||
await pool.query(
|
||||
`
|
||||
delete from public.audit_logs
|
||||
where tenant_id = $1
|
||||
and target_type = 'content_import_job'
|
||||
and details::text like '%worker-import%'
|
||||
`,
|
||||
[tenantId],
|
||||
);
|
||||
await pool.query(
|
||||
`
|
||||
delete from public.content_import_jobs
|
||||
where tenant_id = $1
|
||||
and (
|
||||
source_name like 'worker-import-%'
|
||||
or source_name = 'async-question-import.json'
|
||||
)
|
||||
`,
|
||||
[tenantId],
|
||||
);
|
||||
}
|
||||
|
||||
async function createQueuedQuestionImport(pool) {
|
||||
const preview = await pool.query(
|
||||
`
|
||||
insert into public.content_import_jobs (
|
||||
tenant_id, created_by, import_type, source_format, status,
|
||||
source_name, source_hash, target_region_id, target_subject_id,
|
||||
target_category_id, target_content_node_id, target_collection_id,
|
||||
dry_run, total_count, valid_count, error_count, warning_count,
|
||||
summary, raw_payload, normalized_payload, execution_mode, queued_at,
|
||||
next_attempt_at, parser_metadata
|
||||
)
|
||||
values (
|
||||
$1, $2, 'questions', 'json', 'pending',
|
||||
'worker-import-questions.json', 'worker-import-source-hash',
|
||||
$3::uuid, $4::uuid, $5::uuid, $6::uuid, $7::uuid,
|
||||
false, 1, 1, 0, 0,
|
||||
$8::jsonb, $9::jsonb, $10::jsonb, 'async', now(), now(), '{}'::jsonb
|
||||
)
|
||||
returning id
|
||||
`,
|
||||
[
|
||||
tenantId,
|
||||
adminUserId,
|
||||
ids.region,
|
||||
ids.subject,
|
||||
ids.category,
|
||||
ids.contentNodeSchoolTarget,
|
||||
ids.questionCollection,
|
||||
JSON.stringify({
|
||||
target: {
|
||||
regionId: ids.region,
|
||||
subjectId: ids.subject,
|
||||
categoryId: ids.category,
|
||||
contentNodeId: ids.contentNodeSchoolTarget,
|
||||
collectionId: ids.questionCollection,
|
||||
},
|
||||
importOptions: { allowPartial: false },
|
||||
source: 'worker-import-integration',
|
||||
}),
|
||||
JSON.stringify([
|
||||
{
|
||||
legacyId: 'worker-import-question-001',
|
||||
type: 'choice',
|
||||
content: '异步导入题:worker 应该复用哪套导入规则?',
|
||||
options: ['自己重写', '复用后端导入 executor', '前端直写数据库', '跳过校验'],
|
||||
correctOptionIndices: [1],
|
||||
explanation: 'worker 和 API 必须复用同一套后端导入规则。',
|
||||
difficulty: 2,
|
||||
tags: ['worker-import'],
|
||||
},
|
||||
]),
|
||||
JSON.stringify([
|
||||
{
|
||||
legacyId: 'worker-import-question-001',
|
||||
type: 'choice',
|
||||
typeLabel: null,
|
||||
content: '异步导入题:worker 应该复用哪套导入规则?',
|
||||
options: ['自己重写', '复用后端导入 executor', '前端直写数据库', '跳过校验'],
|
||||
correctOptionIndex: 1,
|
||||
correctOptionIndices: [1],
|
||||
answerText: null,
|
||||
explanation: 'worker 和 API 必须复用同一套后端导入规则。',
|
||||
difficulty: 2,
|
||||
tags: ['worker-import'],
|
||||
mediaUrl: null,
|
||||
subQuestions: [],
|
||||
codeLang: null,
|
||||
codeTemplate: null,
|
||||
examMarkers: {},
|
||||
sourceHash: 'worker-import-question-hash-001',
|
||||
},
|
||||
]),
|
||||
],
|
||||
);
|
||||
const jobId = preview.rows[0].id;
|
||||
await pool.query(
|
||||
`
|
||||
insert into public.content_import_items (
|
||||
tenant_id, job_id, row_no, external_id, status, target_type,
|
||||
source_payload, normalized_payload, content_hash, issues_count
|
||||
)
|
||||
values ($1, $2, 1, 'worker-import-question-001', 'valid', 'question', $3::jsonb, $4::jsonb, 'worker-import-question-hash-001', 0)
|
||||
`,
|
||||
[
|
||||
tenantId,
|
||||
jobId,
|
||||
JSON.stringify({
|
||||
legacyId: 'worker-import-question-001',
|
||||
content: '异步导入题:worker 应该复用哪套导入规则?',
|
||||
}),
|
||||
JSON.stringify({
|
||||
legacyId: 'worker-import-question-001',
|
||||
type: 'choice',
|
||||
typeLabel: null,
|
||||
content: '异步导入题:worker 应该复用哪套导入规则?',
|
||||
options: ['自己重写', '复用后端导入 executor', '前端直写数据库', '跳过校验'],
|
||||
correctOptionIndex: 1,
|
||||
correctOptionIndices: [1],
|
||||
answerText: null,
|
||||
explanation: 'worker 和 API 必须复用同一套后端导入规则。',
|
||||
difficulty: 2,
|
||||
tags: ['worker-import'],
|
||||
mediaUrl: null,
|
||||
subQuestions: [],
|
||||
codeLang: null,
|
||||
codeTemplate: null,
|
||||
examMarkers: {},
|
||||
sourceHash: 'worker-import-question-hash-001',
|
||||
}),
|
||||
],
|
||||
);
|
||||
return jobId;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const pool = new pg.Pool({ connectionString: databaseUrl });
|
||||
try {
|
||||
await cleanup(pool);
|
||||
const jobId = await createQueuedQuestionImport(pool);
|
||||
|
||||
const output = await runWorkerOnce();
|
||||
assert.match(output, /completed=1/, 'worker should complete exactly the queued import job after cleanup');
|
||||
|
||||
const job = await pool.query(
|
||||
`
|
||||
select status, execution_mode, inserted_count, updated_count, skipped_count,
|
||||
locked_at, locked_by, attempt_count, error_message
|
||||
from public.content_import_jobs
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[tenantId, jobId],
|
||||
);
|
||||
assert.equal(job.rows[0]?.status, 'completed', 'queued import job should be completed');
|
||||
assert.equal(job.rows[0]?.execution_mode, 'async', 'job should keep async execution mode');
|
||||
assert.equal(Number(job.rows[0]?.inserted_count), 1, 'worker should insert one question');
|
||||
assert.equal(job.rows[0]?.locked_at, null, 'completed job should release lock');
|
||||
assert.equal(job.rows[0]?.locked_by, null, 'completed job should clear lock owner');
|
||||
assert.equal(Number(job.rows[0]?.attempt_count), 1, 'worker should record one attempt');
|
||||
assert.equal(job.rows[0]?.error_message, null, 'completed job should not retain error message');
|
||||
|
||||
const question = await pool.query(
|
||||
`
|
||||
select q.id, v.content
|
||||
from public.questions q
|
||||
join public.question_versions v on v.id = q.current_version_id
|
||||
where q.tenant_id = $1 and q.legacy_id = 'worker-import-question-001'
|
||||
limit 1
|
||||
`,
|
||||
[tenantId],
|
||||
);
|
||||
assert.equal(question.rows[0]?.content, '异步导入题:worker 应该复用哪套导入规则?', 'worker should import question content');
|
||||
|
||||
const collectionItem = await pool.query(
|
||||
`
|
||||
select 1
|
||||
from public.question_collection_items
|
||||
where tenant_id = $1 and collection_id = $2 and question_id = $3
|
||||
limit 1
|
||||
`,
|
||||
[tenantId, ids.questionCollection, question.rows[0]?.id],
|
||||
);
|
||||
assert.equal(collectionItem.rowCount, 1, 'worker should bind imported question to collection');
|
||||
|
||||
const audit = await pool.query(
|
||||
`
|
||||
select action
|
||||
from public.audit_logs
|
||||
where tenant_id = $1 and target_type = 'content_import_job' and target_id = $2
|
||||
order by created_at desc
|
||||
limit 1
|
||||
`,
|
||||
[tenantId, jobId],
|
||||
);
|
||||
assert.equal(audit.rows[0]?.action, 'content.import.questions.completed', 'worker import should write completion audit');
|
||||
|
||||
console.log('Import worker integration test complete.');
|
||||
} finally {
|
||||
await cleanup(pool).catch(() => {});
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
alter table public.content_import_jobs
|
||||
add column if not exists execution_mode text not null default 'sync',
|
||||
add column if not exists queued_at timestamptz,
|
||||
add column if not exists locked_at timestamptz,
|
||||
add column if not exists locked_by text,
|
||||
add column if not exists attempt_count integer not null default 0,
|
||||
add column if not exists max_attempts integer not null default 3,
|
||||
add column if not exists next_attempt_at timestamptz,
|
||||
add column if not exists parser_metadata jsonb not null default '{}'::jsonb;
|
||||
|
||||
do $$
|
||||
begin
|
||||
if not exists (select 1 from pg_constraint where conname = 'content_import_jobs_execution_mode_check') then
|
||||
alter table public.content_import_jobs
|
||||
add constraint content_import_jobs_execution_mode_check
|
||||
check (execution_mode in ('sync', 'async'));
|
||||
end if;
|
||||
|
||||
if not exists (select 1 from pg_constraint where conname = 'content_import_jobs_attempts_check') then
|
||||
alter table public.content_import_jobs
|
||||
add constraint content_import_jobs_attempts_check
|
||||
check (attempt_count >= 0 and max_attempts > 0 and attempt_count <= max_attempts);
|
||||
end if;
|
||||
end $$;
|
||||
|
||||
create index if not exists idx_content_import_jobs_async_queue
|
||||
on public.content_import_jobs(tenant_id, status, next_attempt_at, created_at)
|
||||
where execution_mode = 'async' and status = 'pending';
|
||||
Reference in New Issue
Block a user