From eee21f6eeddc8a012403bc1b2a636023765a3bc9 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 29 Jun 2026 08:00:17 +0800 Subject: [PATCH] feat: add spreadsheet async imports --- README.md | 15 +- apps/api/package.json | 1 + apps/api/src/core/db.ts | 4 +- apps/api/src/core/errors.ts | 9 + apps/api/src/core/http.ts | 13 +- .../tenant-content/import-spreadsheet.ts | 739 ++++++++++++++++ .../src/features/tenant-content/imports.ts | 693 ++++++++++----- apps/worker/package.json | 3 +- apps/worker/src/config.ts | 8 + apps/worker/src/index.ts | 16 + apps/worker/src/jobs/imports.ts | 221 +++++ docs/refactor/api-structure.md | 2 +- docs/refactor/backend-capability-status.md | 5 +- docs/refactor/backend-handoff-roadmap.md | 16 +- docs/refactor/backend-progress.md | 7 +- docs/refactor/blueprint-coverage.md | 6 +- docs/refactor/content-import-contract.md | 57 +- docs/refactor/frontend-handoff-index.md | 4 +- docs/refactor/implementation-status.md | 10 +- docs/refactor/legacy-feature-gap-matrix.md | 6 +- docs/refactor/next-development-todo.md | 10 +- docs/refactor/taro-frontend-integration.md | 61 +- package-lock.json | 826 ++++++++++++++++++ package.json | 1 + scripts/api-integration-test.js | 179 ++++ scripts/import-worker-integration-test.js | 299 +++++++ ...0014_spreadsheet_import_async_metadata.sql | 28 + 27 files changed, 2972 insertions(+), 267 deletions(-) create mode 100644 apps/api/src/core/errors.ts create mode 100644 apps/api/src/features/tenant-content/import-spreadsheet.ts create mode 100644 apps/worker/src/jobs/imports.ts create mode 100644 scripts/import-worker-integration-test.js create mode 100644 supabase/migrations/202606290014_spreadsheet_import_async_metadata.sql diff --git a/README.md b/README.md index 6c4d0c20..869addfa 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ - Supabase/PostgreSQL 多租户数据库 schema、RLS、索引、触发器。 - `apps/api` 独立业务 API，后续供 H5、Taro 小程序、管理后台统一调用；已支持 Supabase Auth JWT 和迁移期 `tk_` session 双入口。 - 租户后台能力：品牌、域名、公开设置、支付账户、登录配置、私密密钥掩码、活动内容、考试日期、题目反馈处理、激活码、优惠券、勋章管理/发放、成员权限、自定义角色模板、班级/教师/学生范围权限、学生批量导入、批量分班、学生备注、跟进任务、审计日志。 -- 租户内容能力：可配置题库入口、任意深度分类树、考试意向标记、题目集合、顺序/随机/全真模拟蓝图、题目录入/更新、视频绑定、分数线、单词、知识手册、资料资源台账、题目/单词/知识手册/分数线/视频 JSON 批量导入。 +- 租户内容能力：可配置题库入口、任意深度分类树、考试意向标记、题目集合、顺序/随机/全真模拟蓝图、题目录入/更新、视频绑定、分数线、单词、知识手册、资料资源台账、题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 批量导入。 - 学生端能力：题库入口、分类树、题目集合、顺序/随机/模考 session 组卷快照、答题、错题本、收藏夹、背单词进度、个人中心、勋章、考试倒计时、签到积分、题目反馈、排行榜、分数线、题目视频、订单详情/状态轮询、优惠券领取/抵扣、权益、激活码预检查/兑换、资料下载。 - 平台后台能力：租户管理、SaaS 套餐、订阅、账单、服务费收款、用量记录。 - 销售/代理/CRM 增长链路：邀请码、扫码/分享事件、首绑客资保护、销售统计、团队关系、CRM 配置和队列。 @@ -28,7 +28,7 @@ - Supabase Auth/JWT、租户角色模板、班级/教师/学生范围权限已可联调；生产前还要做真实云端 Auth/JWKS 回归和 RLS 深测。 - 阿里云/腾讯云短信、微信小程序登录、微信支付、支付宝主链路、微信/支付宝发起退款/查询确认/退款通知、支付/退款补偿 worker 已完成本地适配；微信网页登录、QQ 登录、手机号换绑、完整资金流水对账和真实生产账号联调还没接完。 - OSS/COS/Supabase Storage 上传下载签名 provider 已接入；上传后校验、PDF/图片预览和资源复检 worker 已完成，CDN 防盗链、杀毒扫描和视频动态水印还没完成。 -- Excel/CSV 导入和大批量异步导入 worker 还没完成；分数线/视频 JSON 批量导入已可联调。 +- Excel/CSV 导入解析已完成并复用 `content_import_jobs/items/issues` 管线；大批量异步导入 worker 基础已接入，支持 queued job 消费、重试和审计；导入后复检和模板下载/字段映射 UI 还没完成。 - 勋章管理/手动发放已可联调；自动发放规则、积分活动联动、分佣真实打款、结算导出、发票/凭证、CRM 轮询/定向分配、富卡片模板、失败告警和销售转化看板还没完成。 - Taro 跨端前端还没开始 scaffold。 - 根目录已清理为新 Supabase SaaS monorepo 编排层；旧 PocketBase/React 项目和旧构建产物仅保留在 `参考/` 目录作为迁移参考，不进入 Git 提交。 @@ -107,6 +107,12 @@ npm --workspace @tiku-saas/worker run commerce:once npm --workspace @tiku-saas/worker run assets:once ``` +单次运行内容导入 worker： + +```bash +npm --workspace @tiku-saas/worker run imports:once +``` + 默认本地数据库： ```text @@ -147,6 +153,7 @@ npm run test:api npm run test:worker:crm npm run test:worker:commerce npm run test:worker:assets +npm run test:worker:imports ``` ## API 模块 @@ -186,7 +193,7 @@ API 身份上下文： - 商户密钥、短信密钥、OAuth app secret 等必须进入 `app_private.tenant_secrets`，或后续生产 KMS/Vault。 - 资料、PDF、视频等资源必须先进入 `content_assets` 台账，再由 API 校验权限并下发签名 URL；生产环境应定时运行 assets worker 复检对象元数据，异常资源会被标记 failed 并退回 draft。 - 题库入口和分类使用 `content_entries/content_nodes`；题目列表和练习规则使用 `question_collections/practice_blueprints`，前端不要再把旧树字段当成唯一业务结构。 -- 批量导入必须先写 `content_import_jobs/items/issues`，保留原始 payload、规范化 payload、逐行问题和审计记录。题目、单词、知识手册、分数线和视频 JSON 导入已走这套后台校验管线，前端只做预检查和预览展示。 +- 批量导入必须先写 `content_import_jobs/items/issues`，保留原始 payload、规范化 payload、逐行问题和审计记录。题目、单词、知识手册、分数线和视频 JSON/CSV/Excel 导入已走这套后台校验管线；大批量任务可提交 `executionMode=async`，由 imports worker 消费，前端只轮询 job 状态和展示 issues。 - 支付 webhook 必须先设计幂等键和验签流程，再进入生产使用；生产环境还应定时运行 commerce worker 兜底供应商漏通知和处理中退款。 ## 最近一次验证 @@ -206,5 +213,5 @@ npm run check:refactor 1. 真实云端 Auth/JWKS 回归、RLS 深测和生产环境配置验收。 2. Taro 前端 scaffold，让 H5 和小程序共用同一套 API。 3. 对象存储 CDN 防盗链、杀毒扫描、视频动态水印和生命周期策略。 -4. Excel/CSV 导入；把现有 JSON 导入升级为可排队异步执行。 +4. 导入后复检、题库导出 PDF/Word/JSON、模板下载和字段映射 UI。 5. 微信网页/QQ 登录、完整资金流水对账、公共题库版本同步 worker、积分活动深化，以及排行榜防刷/预聚合。 diff --git a/apps/api/package.json b/apps/api/package.json index d90f76b9..85c76609 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -12,6 +12,7 @@ "dependencies": { "@supabase/storage-js": "^2.108.2", "ali-oss": "^6.23.0", + "exceljs": "^4.4.0", "jose": "^6.2.3", "pg": "^8.16.3" }, diff --git a/apps/api/src/core/db.ts b/apps/api/src/core/db.ts index cb9e0d2d..4b4e33eb 100644 --- a/apps/api/src/core/db.ts +++ b/apps/api/src/core/db.ts @@ -1,9 +1,9 @@ import type pg from 'pg'; import { createPool, query as runQuery, queryOne as runQueryOne } from '../../../../packages/db/src/index.js'; -import { config } from './config.js'; +import { DEFAULT_DATABASE_URL } from '../../../../packages/config/src/index.js'; export const pool = createPool({ - connectionString: config.databaseUrl, + connectionString: process.env.DATABASE_URL || DEFAULT_DATABASE_URL, max: 10, }); diff --git a/apps/api/src/core/errors.ts b/apps/api/src/core/errors.ts new file mode 100644 index 00000000..a38c9c58 --- /dev/null +++ b/apps/api/src/core/errors.ts @@ -0,0 +1,9 @@ +export class HttpError extends Error { + constructor( + public readonly statusCode: number, + message: string, + public readonly code = 'HTTP_ERROR', + ) { + super(message); + } +} diff --git a/apps/api/src/core/http.ts b/apps/api/src/core/http.ts index 8fe3aefe..7b0977dc 100644 --- a/apps/api/src/core/http.ts +++ b/apps/api/src/core/http.ts @@ -1,6 +1,9 @@ import type { IncomingMessage, ServerResponse } from 'node:http'; import { URL } from 'node:url'; import { config } from './config.js'; +import { HttpError } from './errors.js'; + +export { HttpError } from './errors.js'; export interface RequestContext { req: IncomingMessage; @@ -10,16 +13,6 @@ export interface RequestContext { export type Handler = (ctx: RequestContext) => Promise; -export class HttpError extends Error { - constructor( - public readonly statusCode: number, - message: string, - public readonly code = 'HTTP_ERROR', - ) { - super(message); - } -} - export function sendJson(res: ServerResponse, statusCode: number, body: unknown) { res.statusCode = statusCode; res.setHeader('content-type', 'application/json; charset=utf-8'); diff --git a/apps/api/src/features/tenant-content/import-spreadsheet.ts b/apps/api/src/features/tenant-content/import-spreadsheet.ts new file mode 100644 index 00000000..bf262c6d --- /dev/null +++ b/apps/api/src/features/tenant-content/import-spreadsheet.ts @@ -0,0 +1,739 @@ +import { Buffer } from 'node:buffer'; +import ExcelJS from 'exceljs'; +import { HttpError } from '../../core/http.js'; + +type JsonObject = Record; +type SpreadsheetImportType = 'questions' | 'vocabulary' | 'handbook' | 'scoreline' | 'videos'; +type SupportedSpreadsheetFormat = 'json' | 'csv' | 'excel'; + +interface ParsedSpreadsheet { + rows: JsonObject[]; + sheets: Record; + metadata: JsonObject; +} + +const MAX_SPREADSHEET_ROWS = 5000; +const MAX_SPREADSHEET_COLUMNS = 160; +const MAX_SPREADSHEET_CELL_CHARS = 100_000; +const MAX_EXCEL_FILE_BYTES = 8 * 1024 * 1024; + +const QUESTION_OPTION_KEYS = ['optionA', 'optionB', 'optionC', 'optionD', 'optionE', 'optionF', 'optionG', 'optionH']; + +function objectValue(value: unknown): JsonObject { + return value && typeof value === 'object' && !Array.isArray(value) ? value as JsonObject : {}; +} + +function stringValue(value: unknown) { + return typeof value === 'string' && value.trim() ? value.trim() : ''; +} + +function normalizedKey(value: string) { + return value + .trim() + .toLowerCase() + .replace(/[\s_\-./\\:：()（）[\]【】]/g, ''); +} + +function sourceFormatValue(value: unknown): SupportedSpreadsheetFormat { + const raw = stringValue(value).toLowerCase(); + if (!raw || raw === 'json') return 'json'; + if (raw === 'csv') return 'csv'; + if (['excel', 'xlsx'].includes(raw)) return 'excel'; + throw new HttpError(400, 'sourceFormat must be json, csv, or excel', 'IMPORT_FORMAT_NOT_SUPPORTED'); +} + +function assertRowLimit(count: number) { + if (count > MAX_SPREADSHEET_ROWS) { + throw new HttpError(400, `Spreadsheet import can contain at most ${MAX_SPREADSHEET_ROWS} rows`, 'IMPORT_TOO_LARGE'); + } +} + +function decodeBase64File(body: JsonObject) { + const raw = stringValue(body.fileBase64 ?? body.excelBase64 ?? body.csvBase64); + if (!raw) return null; + const base64 = raw.includes(',') ? raw.slice(raw.indexOf(',') + 1) : raw; + let buffer: Buffer; + try { + buffer = Buffer.from(base64, 'base64'); + } catch { + throw new HttpError(400, 'fileBase64 must be a valid base64 string', 'IMPORT_FILE_BASE64_INVALID'); + } + if (!buffer.length) { + throw new HttpError(400, 'fileBase64 is empty', 'IMPORT_FILE_EMPTY'); + } + if (buffer.length > MAX_EXCEL_FILE_BYTES) { + throw new HttpError(400, `Import file can be at most ${MAX_EXCEL_FILE_BYTES} bytes`, 'IMPORT_FILE_TOO_LARGE'); + } + return buffer; +} + +function primitiveCellValue(value: unknown): unknown { + if (value === undefined || value === null) return undefined; + if (value instanceof Date) return value.toISOString(); + if (typeof value === 'number' || typeof value === 'boolean') return value; + if (typeof value === 'string') { + const trimmed = value.trim(); + return trimmed.length > MAX_SPREADSHEET_CELL_CHARS ? trimmed.slice(0, MAX_SPREADSHEET_CELL_CHARS) : trimmed; + } + if (typeof value === 'object') { + const cell = value as { + text?: string; + result?: unknown; + formula?: string; + richText?: Array<{ text?: string }>; + hyperlink?: string; + }; + if (cell.result !== undefined) return primitiveCellValue(cell.result); + if (Array.isArray(cell.richText)) return cell.richText.map(part => part.text || '').join(''); + if (cell.text !== undefined) return primitiveCellValue(cell.text); + if (cell.hyperlink !== undefined) return primitiveCellValue(cell.hyperlink); + } + return String(value); +} + +function parseJsonLike(value: unknown): unknown { + if (typeof value !== 'string') return value; + const trimmed = value.trim(); + if (!trimmed) return undefined; + if ( + (trimmed.startsWith('[') && trimmed.endsWith(']')) || + (trimmed.startsWith('{') && trimmed.endsWith('}')) + ) { + try { + return JSON.parse(trimmed); + } catch { + return trimmed; + } + } + return trimmed; +} + +function splitList(value: unknown) { + const parsed = parseJsonLike(value); + if (Array.isArray(parsed)) return parsed.map(item => String(item).trim()).filter(Boolean); + if (parsed === undefined || parsed === null) return []; + return String(parsed) + .split(/[|,，;；\n]/) + .map(item => item.trim()) + .filter(Boolean); +} + +function parseBooleanLike(value: unknown): unknown { + if (typeof value === 'boolean') return value; + if (typeof value !== 'string') return value; + const normalized = value.trim().toLowerCase(); + if (['true', '1', 'yes', 'y', '是', '启用', '有效'].includes(normalized)) return true; + if (['false', '0', 'no', 'n', '否', '禁用', '无效'].includes(normalized)) return false; + return value; +} + +function parseNumericLike(value: unknown): unknown { + if (typeof value !== 'string') return value; + const trimmed = value.trim(); + if (!trimmed) return undefined; + if (!/^-?\d+(\.\d+)?$/.test(trimmed)) return value; + const parsed = Number(trimmed); + return Number.isFinite(parsed) ? parsed : value; +} + +function coerceValue(key: string, value: unknown) { + if (value === undefined || value === null || value === '') return undefined; + if ([ + 'options', + 'tags', + 'correctOptionIndices', + 'knowledgeTags', + ].includes(key)) { + return splitList(value); + } + if (['subQuestions', 'examMarkers', 'metadata', 'fieldValues', 'bindings'].includes(key)) { + return parseJsonLike(value); + } + if (['isActive', 'isHot', 'isFilter', 'isRequired', 'isVisible', 'isTrend', 'hasRestriction', 'isGeneral'].includes(key)) { + return parseBooleanLike(value); + } + if ([ + 'difficulty', + 'order', + 'sortOrder', + 'duration', + 'durationSeconds', + 'freePreviewSeconds', + 'year', + 'wordCount', + ].includes(key)) { + return parseNumericLike(value); + } + return parseJsonLike(value); +} + +function commonHeaderAlias(header: string) { + const key = normalizedKey(header); + const aliases: Record = { + id: 'legacyId', + legacyid: 'legacyId', + legacy_id: 'legacyId', + externalid: 'legacyId', + external_id: 'legacyId', + 旧id: 'legacyId', + 原id: 'legacyId', + 外部id: 'legacyId', + 排序: 'order', + sort: 'order', + sortorder: 'order', + sort_order: 'order', + 是否启用: 'isActive', + 状态: 'isActive', + 标签: 'tags', + tags: 'tags', + metadata: 'metadata', + 元数据: 'metadata', + }; + return aliases[key] || null; +} + +function importHeaderAlias(header: string, importType: SpreadsheetImportType) { + const key = normalizedKey(header); + const common = commonHeaderAlias(header); + const aliases: Record> = { + questions: { + type: 'type', + questiontype: 'type', + 题型: 'type', + typelabel: 'typeLabel', + 题型名称: 'typeLabel', + content: 'content', + title: 'content', + 题目: 'content', + 题干: 'content', + 问题: 'content', + 选项: 'options', + options: 'options', + a: 'optionA', + 选项a: 'optionA', + optiona: 'optionA', + b: 'optionB', + 选项b: 'optionB', + optionb: 'optionB', + c: 'optionC', + 选项c: 'optionC', + optionc: 'optionC', + d: 'optionD', + 选项d: 'optionD', + optiond: 'optionD', + e: 'optionE', + 选项e: 'optionE', + optione: 'optionE', + f: 'optionF', + 选项f: 'optionF', + optionf: 'optionF', + answer: 'answer', + correctanswer: 'answer', + correct: 'answer', + 答案: 'answer', + 正确答案: 'answer', + 正确选项: 'answer', + correctoptionindices: 'correctOptionIndices', + correct_option_indices: 'correctOptionIndices', + explanation: 'explanation', + 解析: 'explanation', + 答案解析: 'explanation', + difficulty: 'difficulty', + 难度: 'difficulty', + mediaurl: 'mediaUrl', + 图片: 'mediaUrl', + subquestions: 'subQuestions', + 子题: 'subQuestions', + exammarkers: 'examMarkers', + 考试标记: 'examMarkers', + }, + vocabulary: { + unitlegacyid: 'unitLegacyId', + unitid: 'unitLegacyId', + 单元id: 'unitLegacyId', + unitname: 'unitName', + unit: 'unitName', + 单元: 'unitName', + 单元名称: 'unitName', + unitdescription: 'unitDescription', + 单元描述: 'unitDescription', + unitorder: 'unitOrder', + 单元排序: 'unitOrder', + wordlegacyid: 'wordLegacyId', + wordid: 'wordLegacyId', + 单词id: 'wordLegacyId', + word: 'word', + 单词: 'word', + phonetic: 'phonetic', + pronunciation: 'phonetic', + 音标: 'phonetic', + meaning: 'meaning', + translation: 'meaning', + definition: 'meaning', + 释义: 'meaning', + 中文: 'meaning', + 意思: 'meaning', + example: 'example', + 例句: 'example', + exampletranslation: 'exampleTranslation', + 例句翻译: 'exampleTranslation', + difficulty: 'difficulty', + 难度: 'difficulty', + }, + handbook: { + subjectlegacyid: 'subjectLegacyId', + subjectid: 'subjectLegacyId', + 科目id: 'subjectLegacyId', + subjectname: 'subjectName', + subject: 'subjectName', + 科目: 'subjectName', + 手册: 'subjectName', + chapterlegacyid: 'chapterLegacyId', + chapterid: 'chapterLegacyId', + 章节id: 'chapterLegacyId', + chaptername: 'chapterName', + chapter: 'chapterName', + 章节: 'chapterName', + sectionlegacyid: 'sectionLegacyId', + sectionid: 'sectionLegacyId', + 小节id: 'sectionLegacyId', + sectionname: 'sectionName', + section: 'sectionName', + 小节: 'sectionName', + entrylegacyid: 'entryLegacyId', + entryid: 'entryLegacyId', + 知识点id: 'entryLegacyId', + title: 'title', + 标题: 'title', + 知识点: 'title', + content: 'content', + 正文: 'content', + 内容: 'content', + summary: 'summary', + 摘要: 'summary', + }, + scoreline: { + kind: 'kind', + itemtype: 'kind', + 类型: 'kind', + fieldkey: 'fieldKey', + 字段key: 'fieldKey', + fieldname: 'fieldName', + 字段名: 'fieldName', + fieldtype: 'fieldType', + 字段类型: 'fieldType', + unit: 'unit', + 单位: 'unit', + schoolid: 'schoolId', + 院校id: 'schoolId', + schoollegacyid: 'schoolLegacyId', + schoolname: 'schoolName', + school: 'schoolName', + 学校: 'schoolName', + 院校: 'schoolName', + majorid: 'majorId', + 专业id: 'majorId', + majorlegacyid: 'majorLegacyId', + majorname: 'majorName', + major: 'majorName', + 专业: 'majorName', + year: 'year', + 年份: 'year', + fieldvalues: 'fieldValues', + 动态字段: 'fieldValues', + 最低分: 'minScore', + 录取分: 'minScore', + 计划数: 'planCount', + 招生计划: 'planCount', + }, + videos: { + title: 'title', + name: 'title', + 视频标题: 'title', + 标题: 'title', + description: 'description', + 描述: 'description', + videourl: 'videoUrl', + url: 'videoUrl', + 视频地址: 'videoUrl', + thumbnailurl: 'thumbnailUrl', + 封面: 'thumbnailUrl', + duration: 'durationSeconds', + durationseconds: 'durationSeconds', + 时长: 'durationSeconds', + subjectid: 'subjectId', + 科目id: 'subjectId', + questionid: 'questionId', + 题目id: 'questionId', + legacyquestionid: 'legacyQuestionId', + 旧题目id: 'legacyQuestionId', + assetid: 'assetId', + 资源id: 'assetId', + accessmode: 'accessMode', + 访问模式: 'accessMode', + knowledgetags: 'knowledgeTags', + 知识点标签: 'knowledgeTags', + bindings: 'bindings', + 绑定题目: 'bindings', + }, + }; + return aliases[importType][key] || common || header.trim(); +} + +function normalizeSpreadsheetRow(raw: JsonObject, importType: SpreadsheetImportType) { + const row: JsonObject = {}; + for (const [header, value] of Object.entries(raw)) { + const key = importHeaderAlias(header, importType); + let coerced = coerceValue(key, value); + if (importType === 'scoreline' && key === header.trim()) { + coerced = parseNumericLike(coerced); + } + if (coerced !== undefined && coerced !== '') row[key] = coerced; + } + + if (importType === 'questions') { + const options = QUESTION_OPTION_KEYS.map(key => row[key]).filter(value => value !== undefined && value !== ''); + if (!row.options && options.length) row.options = options; + if (!row.correctOptionIndices && row.answer) { + row.correctOptionIndices = parseAnswerIndices(row.answer, Array.isArray(row.options) ? row.options.length : 0); + } + } + + if (importType === 'vocabulary') { + if (row.wordLegacyId && !row.legacyId) row.legacyId = row.wordLegacyId; + if (row.unitOrder && !row.order && !row.word) row.order = row.unitOrder; + } + + if (importType === 'handbook') { + if (row.entryLegacyId && !row.legacyId) row.legacyId = row.entryLegacyId; + } + + return row; +} + +function parseAnswerIndices(value: unknown, optionCount: number) { + const parts = splitList(value); + const indices: number[] = []; + for (const part of parts) { + const upper = String(part).trim().toUpperCase(); + if (/^[A-H]$/.test(upper)) { + indices.push(upper.charCodeAt(0) - 65); + continue; + } + const parsed = Number(upper); + if (Number.isFinite(parsed)) { + const integer = Math.trunc(parsed); + indices.push(integer === 0 ? 0 : optionCount > 0 && integer <= optionCount ? integer - 1 : integer); + } + } + return [...new Set(indices)]; +} + +function parseCsv(text: string, delimiter: string) { + const rows: string[][] = []; + let row: string[] = []; + let current = ''; + let quoted = false; + for (let index = 0; index < text.length; index += 1) { + const char = text[index]; + const next = text[index + 1]; + if (quoted) { + if (char === '"' && next === '"') { + current += '"'; + index += 1; + } else if (char === '"') { + quoted = false; + } else { + current += char; + } + continue; + } + if (char === '"') { + quoted = true; + } else if (char === delimiter) { + row.push(current); + current = ''; + } else if (char === '\n') { + row.push(current); + rows.push(row); + row = []; + current = ''; + } else if (char !== '\r') { + current += char; + } + } + row.push(current); + rows.push(row); + return rows.filter(item => item.some(cell => cell.trim())); +} + +function autoDelimiter(text: string) { + const sample = text.slice(0, 4096); + const candidates = [',', '\t', ';']; + return candidates + .map(delimiter => ({ delimiter, count: (sample.match(new RegExp(delimiter === '\t' ? '\\t' : delimiter, 'g')) || []).length })) + .sort((left, right) => right.count - left.count)[0]?.delimiter || ','; +} + +function rowsFromMatrix(matrix: string[][], importType: SpreadsheetImportType) { + if (matrix.length < 2) return []; + const headers = matrix[0].map((header, index) => stringValue(header) || `column${index + 1}`); + if (headers.length > MAX_SPREADSHEET_COLUMNS) { + throw new HttpError(400, `Spreadsheet can contain at most ${MAX_SPREADSHEET_COLUMNS} columns`, 'IMPORT_TOO_MANY_COLUMNS'); + } + const rows = matrix.slice(1).map(line => { + const row: JsonObject = {}; + headers.forEach((header, index) => { + const value = stringValue(line[index]); + if (value) row[header] = value; + }); + return normalizeSpreadsheetRow(row, importType); + }).filter(row => Object.keys(row).length > 0); + assertRowLimit(rows.length); + return rows; +} + +function parseCsvBody(body: JsonObject, importType: SpreadsheetImportType): ParsedSpreadsheet { + const buffer = decodeBase64File(body); + const text = stringValue(body.csvText ?? body.text ?? body.fileContent ?? body.payload) || buffer?.toString('utf8') || ''; + if (!text.trim()) { + throw new HttpError(400, 'CSV import requires csvText, fileContent, payload, or fileBase64', 'CSV_CONTENT_REQUIRED'); + } + const delimiter = stringValue(body.delimiter) || autoDelimiter(text); + const rows = rowsFromMatrix(parseCsv(text, delimiter), importType); + return { + rows, + sheets: { csv: rows }, + metadata: { + parser: 'csv', + delimiter: delimiter === '\t' ? 'tab' : delimiter, + rowCount: rows.length, + }, + }; +} + +function worksheetRows(worksheet: ExcelJS.Worksheet, importType: SpreadsheetImportType) { + const matrix: string[][] = []; + worksheet.eachRow({ includeEmpty: false }, row => { + const values: string[] = []; + const max = Math.min(row.cellCount, MAX_SPREADSHEET_COLUMNS); + for (let col = 1; col <= max; col += 1) { + const value = primitiveCellValue(row.getCell(col).value); + values.push(value === undefined || value === null ? '' : String(value)); + } + if (values.some(value => value.trim())) matrix.push(values); + }); + return rowsFromMatrix(matrix, importType); +} + +async function parseExcelBody(body: JsonObject, importType: SpreadsheetImportType): Promise { + const buffer = decodeBase64File(body); + if (!buffer) { + throw new HttpError(400, 'Excel import requires fileBase64', 'EXCEL_FILE_REQUIRED'); + } + const workbook = new ExcelJS.Workbook(); + try { + await workbook.xlsx.load(buffer as unknown as Parameters[0]); + } catch { + throw new HttpError(400, 'Excel file could not be parsed as .xlsx', 'EXCEL_PARSE_FAILED'); + } + + const sheetName = stringValue(body.sheetName); + const sheetIndex = Number(body.sheetIndex ?? 1); + let selected: ExcelJS.Worksheet | undefined; + if (sheetName) selected = workbook.getWorksheet(sheetName); + else if (Number.isFinite(sheetIndex) && sheetIndex > 0) selected = workbook.getWorksheet(Math.trunc(sheetIndex)); + selected = selected || workbook.worksheets.find(sheet => sheet.actualRowCount > 0); + if (!selected) { + throw new HttpError(400, 'Excel file has no non-empty worksheet', 'EXCEL_SHEET_EMPTY'); + } + + const sheets: Record = {}; + let totalRows = 0; + for (const sheet of workbook.worksheets) { + if (sheet.actualRowCount === 0) continue; + const rows = worksheetRows(sheet, importType); + sheets[normalizedKey(sheet.name)] = rows; + totalRows += rows.length; + } + assertRowLimit(totalRows); + + const selectedRows = sheets[normalizedKey(selected.name)] || []; + return { + rows: selectedRows, + sheets, + metadata: { + parser: 'exceljs', + workbookSheetCount: workbook.worksheets.length, + selectedSheet: selected.name, + rowCount: selectedRows.length, + totalParsedRows: totalRows, + }, + }; +} + +function sheetRows(parsed: ParsedSpreadsheet, names: string[]) { + for (const name of names) { + const rows = parsed.sheets[normalizedKey(name)]; + if (rows?.length) return rows; + } + return []; +} + +function stableLegacyId(value: unknown, fallback: string) { + return stringValue(value) || fallback; +} + +function buildVocabularyPayload(parsed: ParsedSpreadsheet) { + const explicitUnitRows = sheetRows(parsed, ['units', 'vocabulary_units', '单元', '单词单元']); + const explicitWordRows = sheetRows(parsed, ['words', 'vocabulary', 'vocabulary_words', '单词', '词汇']); + const rows = explicitWordRows.length || explicitUnitRows.length ? [...explicitUnitRows, ...explicitWordRows] : parsed.rows; + const units = new Map(); + + for (const row of rows) { + const rowType = String(row.rowType ?? row.kind ?? row.type ?? '').toLowerCase(); + const hasWord = Boolean(row.word); + const unitName = stringValue(row.unitName ?? row.unit) || '默认单词单元'; + const unitKey = stableLegacyId(row.unitLegacyId ?? row.unitId, unitName); + const unit = units.get(unitKey) || { + legacyId: unitKey, + name: unitName, + description: row.unitDescription ?? row.description ?? null, + order: row.unitOrder ?? row.order ?? units.size + 1, + isActive: row.isActive ?? true, + words: [], + }; + if (rowType.includes('unit') && !hasWord) { + unit.name = unitName; + unit.description = row.unitDescription ?? row.description ?? unit.description; + unit.order = row.unitOrder ?? row.order ?? unit.order; + unit.isActive = row.isActive ?? unit.isActive; + } + if (hasWord) { + const word = { ...row }; + word.legacyId = row.wordLegacyId ?? row.legacyId ?? null; + delete word.unitLegacyId; + delete word.unitId; + delete word.unitName; + delete word.unitDescription; + delete word.unitOrder; + delete word.wordLegacyId; + unit.words.push(word); + } + units.set(unitKey, unit); + } + + return { units: Array.from(units.values()) }; +} + +function getOrCreateNested( + map: Map, + key: string, + create: () => T, +) { + const existing = map.get(key); + if (existing) return existing; + const created = create(); + map.set(key, created); + return created; +} + +function buildHandbookPayload(parsed: ParsedSpreadsheet) { + const rows = sheetRows(parsed, ['entries', 'handbook', '知识点', '手册']) || parsed.rows; + const subjectMap = new Map }> }>(); + + for (const row of rows.length ? rows : parsed.rows) { + const subjectName = stringValue(row.subjectName ?? row.subject) || '默认知识手册'; + const subjectKey = stableLegacyId(row.subjectLegacyId ?? row.subjectId, subjectName); + const subject = getOrCreateNested(subjectMap, subjectKey, () => ({ + legacyId: subjectKey, + name: subjectName, + type: row.subjectType ?? row.type ?? null, + icon: row.icon ?? null, + color: row.color ?? null, + order: subjectMap.size + 1, + chapters: [], + })); + + const chapterName = stringValue(row.chapterName ?? row.chapter) || '默认章节'; + const chapterKey = stableLegacyId(row.chapterLegacyId ?? row.chapterId, `${subjectKey}:${chapterName}`); + let chapter = subject.chapters.find(item => item.legacyId === chapterKey); + if (!chapter) { + chapter = { legacyId: chapterKey, name: chapterName, order: subject.chapters.length + 1, sections: [] }; + subject.chapters.push(chapter); + } + + const sectionName = stringValue(row.sectionName ?? row.section) || ''; + const sectionKey = stableLegacyId(row.sectionLegacyId ?? row.sectionId, `${chapterKey}:${sectionName || 'default'}`); + let section = chapter.sections.find(item => item.legacyId === sectionKey); + if (!section) { + section = { legacyId: sectionKey, name: sectionName || null, order: chapter.sections.length + 1, entries: [] }; + chapter.sections.push(section); + } + + if (row.title || row.content) { + section.entries.push({ + legacyId: row.entryLegacyId ?? row.legacyId ?? null, + title: row.title ?? '未命名知识点', + summary: row.summary ?? null, + content: row.content ?? '', + tags: row.tags ?? [], + order: row.order ?? section.entries.length + 1, + isActive: row.isActive ?? true, + metadata: row.metadata ?? {}, + }); + } + } + + return { subjects: Array.from(subjectMap.values()) }; +} + +function buildScorelinePayload(parsed: ParsedSpreadsheet) { + const buckets = { + fields: sheetRows(parsed, ['fields', 'scoreline_fields', '字段']), + schools: sheetRows(parsed, ['schools', 'scoreline_schools', '院校', '学校']), + majors: sheetRows(parsed, ['majors', 'scoreline_majors', '专业']), + records: sheetRows(parsed, ['records', 'scoreline_records', '分数线', '记录']), + }; + if (Object.values(buckets).some(rows => rows.length > 0)) return buckets; + return { items: parsed.rows }; +} + +function buildVideosPayload(parsed: ParsedSpreadsheet) { + const rows = sheetRows(parsed, ['videos', 'video_explanations', '视频']) || parsed.rows; + return { videos: rows.length ? rows : parsed.rows }; +} + +export async function expandSpreadsheetImportBody(body: JsonObject, importType: SpreadsheetImportType): Promise { + const sourceFormat = sourceFormatValue(body.sourceFormat); + if (sourceFormat === 'json') return body; + + const parsed = sourceFormat === 'csv' + ? parseCsvBody(body, importType) + : await parseExcelBody(body, importType); + + if (!parsed.rows.length && !Object.values(parsed.sheets).some(rows => rows.length > 0)) { + throw new HttpError(400, 'Spreadsheet import payload must contain at least one data row', 'EMPTY_IMPORT_PAYLOAD'); + } + + const next: JsonObject = { + ...body, + sourceFormat, + spreadsheet: parsed.metadata, + }; + delete next.csvText; + delete next.text; + delete next.fileContent; + delete next.fileBase64; + delete next.excelBase64; + delete next.csvBase64; + delete next.payload; + + if (importType === 'questions') next.items = parsed.rows; + if (importType === 'vocabulary') Object.assign(next, buildVocabularyPayload(parsed)); + if (importType === 'handbook') Object.assign(next, buildHandbookPayload(parsed)); + if (importType === 'scoreline') Object.assign(next, buildScorelinePayload(parsed)); + if (importType === 'videos') Object.assign(next, buildVideosPayload(parsed)); + + return next; +} diff --git a/apps/api/src/features/tenant-content/imports.ts b/apps/api/src/features/tenant-content/imports.ts index 988bc833..804b1dbc 100644 --- a/apps/api/src/features/tenant-content/imports.ts +++ b/apps/api/src/features/tenant-content/imports.ts @@ -1,11 +1,9 @@ import { createHash, randomUUID } from 'node:crypto'; import type pg from 'pg'; -import { config } from '../../core/config.js'; -import { HttpError, type RequestContext } from '../../core/http.js'; -import { intParam, readJsonBody, requiredString, stringParam } from '../../core/request.js'; +import { HttpError } from '../../core/errors.js'; +import type { RequestContext } from '../../core/http.js'; import { query, queryOne, transaction } from '../../core/db.js'; -import { requireTenantContentEditor, type TenantContentAuth } from './auth.js'; -import { boolValue, nullableString } from './utils.js'; +import type { TenantContentAuth } from './auth.js'; type JsonObject = Record; @@ -114,8 +112,10 @@ interface NormalizedHandbookSubject { } type ContentImportType = 'vocabulary' | 'handbook' | 'scoreline' | 'videos'; +export type ExecutableContentImportType = 'questions' | ContentImportType; type EntryBoundImportType = 'vocabulary' | 'handbook'; type GenericTargetEntryType = EntryBoundImportType | null; +type SourceImportFormat = 'json' | 'csv' | 'excel'; type ScorelineImportKind = 'field' | 'school' | 'major' | 'record'; @@ -227,6 +227,19 @@ function stringValue(value: unknown) { return typeof value === 'string' && value.trim() ? value.trim() : ''; } +function sourceImportFormat(value: unknown): SourceImportFormat { + const normalized = stringValue(value).toLowerCase(); + if (!normalized || normalized === 'json') return 'json'; + if (normalized === 'csv') return 'csv'; + if (['excel', 'xlsx'].includes(normalized)) return 'excel'; + throw new HttpError(400, 'sourceFormat must be json, csv, or excel', 'IMPORT_FORMAT_NOT_SUPPORTED'); +} + +function isAsyncExecution(value: unknown) { + const normalized = stringValue(value).toLowerCase(); + return ['async', 'queued', 'queue'].includes(normalized); +} + function stringArrayValue(value: unknown) { if (!Array.isArray(value)) return []; return value.map(item => String(item).trim()).filter(Boolean); @@ -291,6 +304,40 @@ function boolishValue(value: unknown, fallback: boolean) { return fallback; } +function boolValue(value: unknown, fallback: boolean) { + return typeof value === 'boolean' ? value : fallback; +} + +function nullableString(value: unknown) { + return typeof value === 'string' && value.trim() ? value.trim() : null; +} + +function requiredString(body: JsonObject, key: string) { + const value = body[key]; + if (typeof value !== 'string' || !value.trim()) { + throw new HttpError(400, `${key} is required`, 'REQUIRED_FIELD'); + } + return value.trim(); +} + +async function routeDeps() { + const [{ config }, request, auth, spreadsheet] = await Promise.all([ + import('../../core/config.js'), + import('../../core/request.js'), + import('./auth.js'), + import('./import-spreadsheet.js'), + ]); + return { + config, + intParam: request.intParam, + readJsonBody: request.readJsonBody, + requiredString: request.requiredString, + stringParam: request.stringParam, + requireTenantContentEditor: auth.requireTenantContentEditor, + expandSpreadsheetImportBody: spreadsheet.expandSpreadsheetImportBody, + }; +} + function integerValue(value: unknown, fallback: number) { const numberValue = Number(value ?? fallback); return Number.isFinite(numberValue) ? Math.trunc(numberValue) : fallback; @@ -647,13 +694,9 @@ async function assertTargetReferences(client: pg.PoolClient, auth: TenantContent async function createQuestionPreviewJob(auth: TenantContentAuth, body: JsonObject): Promise { const rawItems = parseQuestionItems(body); - const sourceFormat = stringValue(body.sourceFormat) || 'json'; + const sourceFormat = sourceImportFormat(body.sourceFormat); const sourceName = stringValue(body.sourceName) || null; - if (sourceFormat !== 'json') { - throw new HttpError(400, 'Only json sourceFormat is supported by the synchronous API for now', 'IMPORT_FORMAT_NOT_SUPPORTED'); - } - return transaction(async client => { const target = await assertTargetReferences(client, auth, body); const normalizedItems = rawItems.map((raw, index) => { @@ -684,7 +727,7 @@ async function createQuestionPreviewJob(auth: TenantContentAuth, body: JsonObjec target_category_id, target_node_id, target_question_bank_id, target_entry_id, target_content_node_id, target_collection_id, dry_run, total_count, valid_count, error_count, warning_count, - summary, raw_payload, normalized_payload + summary, raw_payload, normalized_payload, parser_metadata ) values ( $1, $2, 'questions', $3, 'preview', @@ -692,7 +735,7 @@ async function createQuestionPreviewJob(auth: TenantContentAuth, body: JsonObjec $8::uuid, $9::uuid, $10::uuid, $11::uuid, $12::uuid, $13::uuid, true, $14, $15, $16, $17, - $18::jsonb, $19::jsonb, $20::jsonb + $18::jsonb, $19::jsonb, $20::jsonb, $21::jsonb ) returning id, status, total_count as "totalCount", valid_count as "validCount", error_count as "errorCount", warning_count as "warningCount" @@ -718,6 +761,7 @@ async function createQuestionPreviewJob(auth: TenantContentAuth, body: JsonObjec JSON.stringify({ target, generatedAt: new Date().toISOString() }), rawPayload, normalizedPayload, + JSON.stringify(objectValue(body.spreadsheet)), ], ); @@ -1659,11 +1703,8 @@ async function createGenericPreviewJob( normalizedItems: NormalizedImportItem[], expectedEntryType: GenericTargetEntryType = importType === 'vocabulary' || importType === 'handbook' ? importType : null, ): Promise> { - const sourceFormat = stringValue(body.sourceFormat) || 'json'; + const sourceFormat = sourceImportFormat(body.sourceFormat); const sourceName = stringValue(body.sourceName) || null; - if (sourceFormat !== 'json') { - throw new HttpError(400, 'Only json sourceFormat is supported by the synchronous API for now', 'IMPORT_FORMAT_NOT_SUPPORTED'); - } return transaction(async client => { const target = await assertGenericTargetReferences(client, auth, body, expectedEntryType); @@ -1678,13 +1719,13 @@ async function createGenericPreviewJob( tenant_id, created_by, import_type, source_format, status, source_name, source_hash, target_region_id, target_entry_id, target_content_node_id, dry_run, total_count, valid_count, - error_count, warning_count, summary, raw_payload, normalized_payload + error_count, warning_count, summary, raw_payload, normalized_payload, parser_metadata ) values ( $1, $2, $3, $4, 'preview', $5, $6, $7::uuid, $8::uuid, $9::uuid, true, $10, $11, - $12, $13, $14::jsonb, $15::jsonb, $16::jsonb + $12, $13, $14::jsonb, $15::jsonb, $16::jsonb, $17::jsonb ) returning id, status, total_count as "totalCount", valid_count as "validCount", error_count as "errorCount", warning_count as "warningCount" @@ -1706,6 +1747,7 @@ async function createGenericPreviewJob( JSON.stringify({ target, generatedAt: new Date().toISOString() }), JSON.stringify(normalizedItems.map(item => item.source)), JSON.stringify(normalizedItems.map(item => item.normalized).filter(Boolean)), + JSON.stringify(objectValue(body.spreadsheet)), ], ); @@ -3052,196 +3094,31 @@ async function importOneVideo( return { insertedCount, updatedCount, skippedCount }; } -async function runGenericImport( +interface ContentImportExecutionOptions { + jobId: string; + importType: ExecutableContentImportType; + allowPartial?: boolean; + allowQueuedJob?: boolean; +} + +interface ContentImportExecutionResult { + jobId: string; + status: string; + idempotent?: boolean; + insertedCount: number; + updatedCount: number; + skippedCount: number; + errorCount?: number; + warningCount?: number; +} + +async function executeQuestionsImportJob( auth: TenantContentAuth, - body: JsonObject, - importType: ContentImportType, - createPreview: () => Promise>, - importOne: ( - client: pg.PoolClient, - auth: TenantContentAuth, - job: { - id: string; - target_region_id: string | null; - target_entry_id: string | null; - target_content_node_id: string | null; - }, - item: { - id: string; - row_no: number; - normalized_payload: T; - }, - ) => Promise<{ insertedCount: number; updatedCount: number; skippedCount: number }>, + input: Omit, ) { - const allowPartial = boolValue(body.allowPartial, false); - const jobId = nullableString(body.previewJobId) || nullableString(body.jobId); - const createdPreview = jobId ? null : await createPreview(); - const finalJobId = jobId || createdPreview?.job.id || ''; - - const result = await transaction(async client => { - const job = await loadGenericPreviewJob(client, auth, finalJobId, importType); - if (job.status === 'completed' || job.status === 'completed_with_errors') { - return { - jobId: job.id, - status: job.status, - idempotent: true, - insertedCount: 0, - updatedCount: 0, - skippedCount: 0, - }; - } - - if (job.error_count > 0 && !allowPartial) { - await client.query( - ` - update public.content_import_jobs - set status = 'rejected', error_message = 'Preview contains validation errors', updated_at = now() - where tenant_id = $1 and id = $2 - `, - [auth.tenantId, job.id], - ); - throw new HttpError(409, 'Preview contains validation errors. Fix issues or set allowPartial=true.', 'IMPORT_HAS_ERRORS'); - } - - await client.query( - ` - update public.content_import_jobs - set status = 'importing', dry_run = false, started_at = coalesce(started_at, now()), updated_at = now() - where tenant_id = $1 and id = $2 - `, - [auth.tenantId, job.id], - ); - - const itemResult = await client.query<{ - id: string; - row_no: number; - normalized_payload: T; - }>( - ` - select id, row_no, normalized_payload - from public.content_import_items - where tenant_id = $1 and job_id = $2 and status = 'valid' - order by - case - when $3 = 'scoreline' then - case normalized_payload ->> 'kind' - when 'field' then 1 - when 'school' then 2 - when 'major' then 3 - when 'record' then 4 - else 9 - end - else 1 - end, - row_no asc - for update - `, - [auth.tenantId, job.id, importType], - ); - - let insertedCount = 0; - let updatedCount = 0; - let skippedCount = 0; - for (const item of itemResult.rows) { - const status = await importOne(client, auth, job, item); - insertedCount += status.insertedCount; - updatedCount += status.updatedCount; - skippedCount += status.skippedCount; - } - - const finalStatus = job.error_count > 0 ? 'completed_with_errors' : 'completed'; - await client.query( - ` - update public.content_import_jobs - set status = $3, - inserted_count = $4, - updated_count = $5, - skipped_count = $6, - summary = coalesce(summary, '{}'::jsonb) || $7::jsonb, - finished_at = now(), - updated_at = now() - where tenant_id = $1 and id = $2 - `, - [ - auth.tenantId, - job.id, - finalStatus, - insertedCount, - updatedCount, - skippedCount, - JSON.stringify({ insertedCount, updatedCount, skippedCount, importedAt: new Date().toISOString() }), - ], - ); - - await client.query( - ` - insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details) - values ($1, $2, $3, 'content_import_job', $4, $5::jsonb) - `, - [ - auth.tenantId, - auth.userId, - `content.import.${importType}.completed`, - job.id, - JSON.stringify({ insertedCount, updatedCount, skippedCount, allowPartial }), - ], - ); - - return { - jobId: job.id, - status: finalStatus, - insertedCount, - updatedCount, - skippedCount, - errorCount: job.error_count, - warningCount: job.warning_count, - }; - }); - - return { item: result, preview: createdPreview }; -} - -export async function previewQuestionsImportRoute(ctx: RequestContext) { - const auth = await requireTenantContentEditor(ctx); - const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); - return createQuestionPreviewJob(auth, body); -} - -export async function previewVocabularyImportRoute(ctx: RequestContext) { - const auth = await requireTenantContentEditor(ctx); - const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); - return createGenericPreviewJob(auth, body, 'vocabulary', 'vocabulary_unit', createVocabularyNormalizedItems(body)); -} - -export async function previewHandbookImportRoute(ctx: RequestContext) { - const auth = await requireTenantContentEditor(ctx); - const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); - return createGenericPreviewJob(auth, body, 'handbook', 'handbook_subject', createHandbookNormalizedItems(body)); -} - -export async function previewScorelineImportRoute(ctx: RequestContext) { - const auth = await requireTenantContentEditor(ctx); - const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); - return createGenericPreviewJob(auth, body, 'scoreline', 'scoreline_item', createScorelineNormalizedItems(body), null); -} - -export async function previewVideosImportRoute(ctx: RequestContext) { - const auth = await requireTenantContentEditor(ctx); - const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); - return createGenericPreviewJob(auth, body, 'videos', 'video_explanation', createVideoNormalizedItems(body), null); -} - -export async function importQuestionsRoute(ctx: RequestContext) { - const auth = await requireTenantContentEditor(ctx); - const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); - const allowPartial = boolValue(body.allowPartial, false); - const jobId = nullableString(body.previewJobId) || nullableString(body.jobId); - - const createdPreview = jobId ? null : await createQuestionPreviewJob(auth, body); - const finalJobId = jobId || createdPreview?.job.id || ''; - - const result = await transaction(async client => { - const job = await loadPreviewJob(client, auth, finalJobId); + const allowPartial = input.allowPartial === true; + return transaction(async client => { + const job = await loadPreviewJob(client, auth, input.jobId); if (job.status === 'completed' || job.status === 'completed_with_errors') { return { @@ -3253,12 +3130,19 @@ export async function importQuestionsRoute(ctx: RequestContext) { skippedCount: 0, }; } + if (job.status === 'pending' && input.allowQueuedJob !== true) { + throw new HttpError(409, 'Import job is queued for async processing', 'IMPORT_JOB_QUEUED'); + } if (job.error_count > 0 && !allowPartial) { await client.query( ` update public.content_import_jobs - set status = 'rejected', error_message = 'Preview contains validation errors', updated_at = now() + set status = 'rejected', + error_message = 'Preview contains validation errors', + locked_at = null, + locked_by = null, + updated_at = now() where tenant_id = $1 and id = $2 `, [auth.tenantId, job.id], @@ -3310,6 +3194,8 @@ export async function importQuestionsRoute(ctx: RequestContext) { skipped_count = $6, summary = coalesce(summary, '{}'::jsonb) || $7::jsonb, finished_at = now(), + locked_at = null, + locked_by = null, updated_at = now() where tenant_id = $1 and id = $2 `, @@ -3342,13 +3228,389 @@ export async function importQuestionsRoute(ctx: RequestContext) { warningCount: job.warning_count, }; }); +} + +async function executeGenericImportJob( + auth: TenantContentAuth, + input: Omit & { importType: ContentImportType }, + importOne: ( + client: pg.PoolClient, + auth: TenantContentAuth, + job: { + id: string; + target_region_id: string | null; + target_entry_id: string | null; + target_content_node_id: string | null; + }, + item: { + id: string; + row_no: number; + normalized_payload: T; + }, + ) => Promise<{ insertedCount: number; updatedCount: number; skippedCount: number }>, +) { + const allowPartial = input.allowPartial === true; + return transaction(async client => { + const job = await loadGenericPreviewJob(client, auth, input.jobId, input.importType); + if (job.status === 'completed' || job.status === 'completed_with_errors') { + return { + jobId: job.id, + status: job.status, + idempotent: true, + insertedCount: 0, + updatedCount: 0, + skippedCount: 0, + }; + } + if (job.status === 'pending' && input.allowQueuedJob !== true) { + throw new HttpError(409, 'Import job is queued for async processing', 'IMPORT_JOB_QUEUED'); + } + + if (job.error_count > 0 && !allowPartial) { + await client.query( + ` + update public.content_import_jobs + set status = 'rejected', + error_message = 'Preview contains validation errors', + locked_at = null, + locked_by = null, + updated_at = now() + where tenant_id = $1 and id = $2 + `, + [auth.tenantId, job.id], + ); + throw new HttpError(409, 'Preview contains validation errors. Fix issues or set allowPartial=true.', 'IMPORT_HAS_ERRORS'); + } + + await client.query( + ` + update public.content_import_jobs + set status = 'importing', dry_run = false, started_at = coalesce(started_at, now()), updated_at = now() + where tenant_id = $1 and id = $2 + `, + [auth.tenantId, job.id], + ); + + const itemResult = await client.query<{ + id: string; + row_no: number; + normalized_payload: T; + }>( + ` + select id, row_no, normalized_payload + from public.content_import_items + where tenant_id = $1 and job_id = $2 and status = 'valid' + order by + case + when $3 = 'scoreline' then + case normalized_payload ->> 'kind' + when 'field' then 1 + when 'school' then 2 + when 'major' then 3 + when 'record' then 4 + else 9 + end + else 1 + end, + row_no asc + for update + `, + [auth.tenantId, job.id, input.importType], + ); + + let insertedCount = 0; + let updatedCount = 0; + let skippedCount = 0; + for (const item of itemResult.rows) { + const status = await importOne(client, auth, job, item); + insertedCount += status.insertedCount; + updatedCount += status.updatedCount; + skippedCount += status.skippedCount; + } + + const finalStatus = job.error_count > 0 ? 'completed_with_errors' : 'completed'; + await client.query( + ` + update public.content_import_jobs + set status = $3, + inserted_count = $4, + updated_count = $5, + skipped_count = $6, + summary = coalesce(summary, '{}'::jsonb) || $7::jsonb, + finished_at = now(), + locked_at = null, + locked_by = null, + updated_at = now() + where tenant_id = $1 and id = $2 + `, + [ + auth.tenantId, + job.id, + finalStatus, + insertedCount, + updatedCount, + skippedCount, + JSON.stringify({ insertedCount, updatedCount, skippedCount, importedAt: new Date().toISOString() }), + ], + ); + + await client.query( + ` + insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details) + values ($1, $2, $3, 'content_import_job', $4, $5::jsonb) + `, + [ + auth.tenantId, + auth.userId, + `content.import.${input.importType}.completed`, + job.id, + JSON.stringify({ insertedCount, updatedCount, skippedCount, allowPartial }), + ], + ); + + return { + jobId: job.id, + status: finalStatus, + insertedCount, + updatedCount, + skippedCount, + errorCount: job.error_count, + warningCount: job.warning_count, + }; + }); +} + +export async function executeContentImportJob( + auth: TenantContentAuth, + input: ContentImportExecutionOptions, +): Promise { + if (input.importType === 'questions') { + return executeQuestionsImportJob(auth, input); + } + if (input.importType === 'vocabulary') { + return executeGenericImportJob(auth, { ...input, importType: 'vocabulary' }, importOneVocabularyUnit); + } + if (input.importType === 'handbook') { + return executeGenericImportJob(auth, { ...input, importType: 'handbook' }, importOneHandbookSubject); + } + if (input.importType === 'scoreline') { + return executeGenericImportJob(auth, { ...input, importType: 'scoreline' }, importOneScorelineItem); + } + if (input.importType === 'videos') { + return executeGenericImportJob(auth, { ...input, importType: 'videos' }, importOneVideo); + } + throw new HttpError(400, 'Unsupported import type', 'IMPORT_TYPE_NOT_SUPPORTED'); +} + +async function queueContentImportJob( + auth: TenantContentAuth, + input: ContentImportExecutionOptions, +) { + const allowPartial = input.allowPartial === true; + return transaction(async client => { + const result = await client.query<{ + id: string; + status: string; + error_count: number; + warning_count: number; + attempt_count: number; + max_attempts: number; + }>( + ` + select id, status, error_count, warning_count, attempt_count, max_attempts + from public.content_import_jobs + where tenant_id = $1 and id = $2 and import_type = $3 + limit 1 + for update + `, + [auth.tenantId, input.jobId, input.importType], + ); + const job = result.rows[0]; + if (!job) throw new HttpError(404, 'Import job not found', 'IMPORT_JOB_NOT_FOUND'); + if (job.status === 'completed' || job.status === 'completed_with_errors') { + return { + jobId: job.id, + status: job.status, + idempotent: true, + executionMode: 'async', + errorCount: job.error_count, + warningCount: job.warning_count, + }; + } + if (job.status === 'pending') { + return { + jobId: job.id, + status: job.status, + idempotent: true, + executionMode: 'async', + errorCount: job.error_count, + warningCount: job.warning_count, + }; + } + if (job.status === 'importing') { + throw new HttpError(409, 'Import job is already importing', 'IMPORT_JOB_NOT_READY'); + } + if (['failed', 'rejected'].includes(job.status)) { + throw new HttpError(409, `Import job is ${job.status}`, 'IMPORT_JOB_NOT_READY'); + } + if (job.error_count > 0 && !allowPartial) { + await client.query( + ` + update public.content_import_jobs + set status = 'rejected', + error_message = 'Preview contains validation errors', + updated_at = now() + where tenant_id = $1 and id = $2 + `, + [auth.tenantId, job.id], + ); + throw new HttpError(409, 'Preview contains validation errors. Fix issues or set allowPartial=true.', 'IMPORT_HAS_ERRORS'); + } + + const queuedAt = new Date().toISOString(); + await client.query( + ` + update public.content_import_jobs + set execution_mode = 'async', + status = 'pending', + dry_run = false, + queued_at = coalesce(queued_at, now()), + locked_at = null, + locked_by = null, + next_attempt_at = now(), + summary = coalesce(summary, '{}'::jsonb) || $3::jsonb, + updated_at = now() + where tenant_id = $1 and id = $2 + `, + [ + auth.tenantId, + job.id, + JSON.stringify({ + importOptions: { allowPartial }, + queuedAt, + }), + ], + ); + + await client.query( + ` + insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details) + values ($1, $2, $3, 'content_import_job', $4, $5::jsonb) + `, + [ + auth.tenantId, + auth.userId, + `content.import.${input.importType}.queued`, + job.id, + JSON.stringify({ allowPartial, executionMode: 'async', queuedAt }), + ], + ); + + return { + jobId: job.id, + status: 'pending', + executionMode: 'async', + errorCount: job.error_count, + warningCount: job.warning_count, + }; + }); +} + +async function runGenericImport( + auth: TenantContentAuth, + body: JsonObject, + importType: ContentImportType, + createPreview: () => Promise>, + importOne: ( + client: pg.PoolClient, + auth: TenantContentAuth, + job: { + id: string; + target_region_id: string | null; + target_entry_id: string | null; + target_content_node_id: string | null; + }, + item: { + id: string; + row_no: number; + normalized_payload: T; + }, + ) => Promise<{ insertedCount: number; updatedCount: number; skippedCount: number }>, +) { + const allowPartial = boolValue(body.allowPartial, false); + const jobId = nullableString(body.previewJobId) || nullableString(body.jobId); + const createdPreview = jobId ? null : await createPreview(); + const finalJobId = jobId || createdPreview?.job.id || ''; + const result = isAsyncExecution(body.executionMode ?? body.execution_mode) + ? await queueContentImportJob(auth, { jobId: finalJobId, importType, allowPartial }) + : await executeGenericImportJob(auth, { jobId: finalJobId, importType, allowPartial }, importOne); + + return { item: result, preview: createdPreview }; +} + +export async function previewQuestionsImportRoute(ctx: RequestContext) { + const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps(); + const auth = await requireTenantContentEditor(ctx); + const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const body = await expandSpreadsheetImportBody(rawBody, 'questions'); + return createQuestionPreviewJob(auth, body); +} + +export async function previewVocabularyImportRoute(ctx: RequestContext) { + const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps(); + const auth = await requireTenantContentEditor(ctx); + const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const body = await expandSpreadsheetImportBody(rawBody, 'vocabulary'); + return createGenericPreviewJob(auth, body, 'vocabulary', 'vocabulary_unit', createVocabularyNormalizedItems(body)); +} + +export async function previewHandbookImportRoute(ctx: RequestContext) { + const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps(); + const auth = await requireTenantContentEditor(ctx); + const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const body = await expandSpreadsheetImportBody(rawBody, 'handbook'); + return createGenericPreviewJob(auth, body, 'handbook', 'handbook_subject', createHandbookNormalizedItems(body)); +} + +export async function previewScorelineImportRoute(ctx: RequestContext) { + const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps(); + const auth = await requireTenantContentEditor(ctx); + const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const body = await expandSpreadsheetImportBody(rawBody, 'scoreline'); + return createGenericPreviewJob(auth, body, 'scoreline', 'scoreline_item', createScorelineNormalizedItems(body), null); +} + +export async function previewVideosImportRoute(ctx: RequestContext) { + const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps(); + const auth = await requireTenantContentEditor(ctx); + const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const body = await expandSpreadsheetImportBody(rawBody, 'videos'); + return createGenericPreviewJob(auth, body, 'videos', 'video_explanation', createVideoNormalizedItems(body), null); +} + +export async function importQuestionsRoute(ctx: RequestContext) { + const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps(); + const auth = await requireTenantContentEditor(ctx); + const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const body = await expandSpreadsheetImportBody(rawBody, 'questions'); + const allowPartial = boolValue(body.allowPartial, false); + const jobId = nullableString(body.previewJobId) || nullableString(body.jobId); + + const createdPreview = jobId ? null : await createQuestionPreviewJob(auth, body); + const finalJobId = jobId || createdPreview?.job.id || ''; + + const result = isAsyncExecution(body.executionMode ?? body.execution_mode) + ? await queueContentImportJob(auth, { jobId: finalJobId, importType: 'questions', allowPartial }) + : await executeContentImportJob(auth, { jobId: finalJobId, importType: 'questions', allowPartial }); return { item: result, preview: createdPreview }; } export async function importVocabularyRoute(ctx: RequestContext) { + const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps(); const auth = await requireTenantContentEditor(ctx); - const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const body = await expandSpreadsheetImportBody(rawBody, 'vocabulary'); return runGenericImport( auth, body, @@ -3359,8 +3621,10 @@ export async function importVocabularyRoute(ctx: RequestContext) { } export async function importHandbookRoute(ctx: RequestContext) { + const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps(); const auth = await requireTenantContentEditor(ctx); - const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const body = await expandSpreadsheetImportBody(rawBody, 'handbook'); return runGenericImport( auth, body, @@ -3371,8 +3635,10 @@ export async function importHandbookRoute(ctx: RequestContext) { } export async function importScorelineRoute(ctx: RequestContext) { + const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps(); const auth = await requireTenantContentEditor(ctx); - const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const body = await expandSpreadsheetImportBody(rawBody, 'scoreline'); return runGenericImport( auth, body, @@ -3383,8 +3649,10 @@ export async function importScorelineRoute(ctx: RequestContext) { } export async function importVideosRoute(ctx: RequestContext) { + const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps(); const auth = await requireTenantContentEditor(ctx); - const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes }); + const body = await expandSpreadsheetImportBody(rawBody, 'videos'); return runGenericImport( auth, body, @@ -3395,6 +3663,7 @@ export async function importVideosRoute(ctx: RequestContext) { } export async function importJobsRoute(ctx: RequestContext) { + const { intParam, requireTenantContentEditor, stringParam } = await routeDeps(); const auth = await requireTenantContentEditor(ctx); const limit = intParam(ctx, 'limit', 50, 200); const importType = stringParam(ctx, 'importType'); @@ -3423,6 +3692,7 @@ export async function importJobsRoute(ctx: RequestContext) { valid_count as "validCount", error_count as "errorCount", warning_count as "warningCount", inserted_count as "insertedCount", updated_count as "updatedCount", skipped_count as "skippedCount", + execution_mode as "executionMode", parser_metadata as "parserMetadata", summary, error_message as "errorMessage", started_at as "startedAt", finished_at as "finishedAt", created_by as "createdBy", created_at as "createdAt", updated_at as "updatedAt" @@ -3438,6 +3708,7 @@ export async function importJobsRoute(ctx: RequestContext) { } export async function importIssuesRoute(ctx: RequestContext) { + const { intParam, requireTenantContentEditor, requiredString, stringParam } = await routeDeps(); const auth = await requireTenantContentEditor(ctx); const jobId = requiredString({ jobId: stringParam(ctx, 'jobId') }, 'jobId'); const limit = intParam(ctx, 'limit', 500, 2000); diff --git a/apps/worker/package.json b/apps/worker/package.json index f025b2f3..38f5fc83 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -10,7 +10,8 @@ "check": "tsc -p tsconfig.json --noEmit", "crm:once": "tsx src/index.ts --once --job crm", "commerce:once": "tsx src/index.ts --once --job commerce", - "assets:once": "tsx src/index.ts --once --job assets" + "assets:once": "tsx src/index.ts --once --job assets", + "imports:once": "tsx src/index.ts --once --job imports" }, "dependencies": { "@supabase/storage-js": "^2.108.2", diff --git a/apps/worker/src/config.ts b/apps/worker/src/config.ts index c64b5efe..ae5d883a 100644 --- a/apps/worker/src/config.ts +++ b/apps/worker/src/config.ts @@ -17,6 +17,9 @@ export interface WorkerConfig { assetMinAgeSeconds: number; assetRecheckIntervalSeconds: number; assetRequestTimeoutMs: number; + importBatchSize: number; + importWorkerId: string; + importBackoffSeconds: number[]; storageMaxUploadBytes: number; storageAllowedMimePrefixes: string[]; storageAllowedMimeTypes: string[]; @@ -53,6 +56,11 @@ export const config: WorkerConfig = { assetMinAgeSeconds: envNumber('WORKER_ASSET_MIN_AGE_SECONDS', 300), assetRecheckIntervalSeconds: envNumber('WORKER_ASSET_RECHECK_INTERVAL_SECONDS', 60 * 60 * 24), assetRequestTimeoutMs: envNumber('WORKER_ASSET_REQUEST_TIMEOUT_MS', 10_000), + importBatchSize: envNumber('WORKER_IMPORT_BATCH_SIZE', 5), + importWorkerId: envString('WORKER_IMPORT_ID', `imports-${process.pid}`), + importBackoffSeconds: envList('WORKER_IMPORT_BACKOFF_SECONDS', '30,120,600,1800') + .map((value: string) => Number(value)) + .filter((value: number) => Number.isFinite(value) && value > 0), storageMaxUploadBytes: envNumber('STORAGE_MAX_UPLOAD_BYTES', 1024 * 1024 * 500), storageAllowedMimePrefixes: envList('STORAGE_ALLOWED_MIME_PREFIXES', 'image/,video/,audio/'), storageAllowedMimeTypes: envList( diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 3a748f8f..bf6ced53 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -4,6 +4,8 @@ import { processCrmBatch } from './jobs/crm.js'; import { processCommerceBatch } from './jobs/commerce.js'; import { processAssetBatch } from './jobs/assets.js'; +const extraClosers = new Set<() => Promise>(); + function hasArg(name: string) { return process.argv.includes(name); } @@ -37,6 +39,17 @@ async function runOnce() { ); return; } + if (job === 'imports') { + const { closeImportExecutorPool, processImportBatch } = await import('./jobs/imports.js'); + extraClosers.add(closeImportExecutorPool); + const result = await processImportBatch(); + console.log( + `[worker] imports batch processed=${result.processed}` + + ` completed=${result.completed} completedWithErrors=${result.completedWithErrors}` + + ` failed=${result.failed} retrying=${result.retrying} skipped=${result.skipped}`, + ); + return; + } throw new Error(`Unsupported worker job: ${job}`); } @@ -66,5 +79,8 @@ try { await runOnce(); } } finally { + for (const closeExtra of extraClosers) { + await closeExtra(); + } await closePool(); } diff --git a/apps/worker/src/jobs/imports.ts b/apps/worker/src/jobs/imports.ts new file mode 100644 index 00000000..8001da15 --- /dev/null +++ b/apps/worker/src/jobs/imports.ts @@ -0,0 +1,221 @@ +import { pool } from '../db.js'; +import { config } from '../config.js'; +import { executeContentImportJob, type ExecutableContentImportType } from '../../../api/src/features/tenant-content/imports.js'; +import { closePool as closeApiImportPool } from '../../../api/src/core/db.js'; + +interface ImportJobRow { + id: string; + tenantId: string; + createdBy: string | null; + importType: ExecutableContentImportType; + status: string; + attemptCount: number; + maxAttempts: number; + summary: Record; +} + +interface ImportWorkerResult { + processed: number; + completed: number; + completedWithErrors: number; + failed: number; + retrying: number; + skipped: number; +} + +function objectValue(value: unknown): Record { + return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : {}; +} + +function boolValue(value: unknown, fallback: boolean) { + return typeof value === 'boolean' ? value : fallback; +} + +function numberValue(value: unknown, fallback: number) { + const parsed = Number(value ?? fallback); + return Number.isFinite(parsed) ? parsed : fallback; +} + +function errorMessage(error: unknown) { + return error instanceof Error ? error.message : String(error); +} + +function errorCode(error: unknown) { + return typeof error === 'object' && error !== null && 'code' in error + ? String((error as { code?: unknown }).code || 'IMPORT_WORKER_ERROR') + : 'IMPORT_WORKER_ERROR'; +} + +function truncate(value: unknown, max = 1900) { + return String(value ?? '').slice(0, max); +} + +function backoffSeconds(attemptCount: number) { + const backoffs = config.importBackoffSeconds.length ? config.importBackoffSeconds : [30, 120, 600, 1800]; + return backoffs[Math.min(Math.max(0, attemptCount - 1), backoffs.length - 1)]; +} + +function importOptions(summary: Record) { + const options = objectValue(summary.importOptions); + return { + allowPartial: boolValue(options.allowPartial, false), + }; +} + +async function claimImportJobs() { + const client = await pool.connect(); + try { + await client.query('begin'); + const result = await client.query( + ` + select id, + tenant_id as "tenantId", + created_by as "createdBy", + import_type as "importType", + status, + attempt_count as "attemptCount", + max_attempts as "maxAttempts", + summary + from public.content_import_jobs + where execution_mode = 'async' + and status = 'pending' + and attempt_count < max_attempts + and (next_attempt_at is null or next_attempt_at <= now()) + order by created_at asc + limit $1 + for update skip locked + `, + [config.importBatchSize], + ); + + const ids = result.rows.map(row => row.id); + if (ids.length > 0) { + await client.query( + ` + update public.content_import_jobs + set locked_at = now(), + locked_by = $2, + attempt_count = attempt_count + 1, + updated_at = now() + where id = any($1::uuid[]) + `, + [ids, config.importWorkerId], + ); + } + await client.query('commit'); + return result.rows; + } catch (error) { + await client.query('rollback'); + throw error; + } finally { + client.release(); + } +} + +async function markImportFailed(job: ImportJobRow, error: unknown) { + const nextAttempt = job.attemptCount + 1; + const willRetry = nextAttempt < job.maxAttempts; + const status = willRetry ? 'pending' : 'failed'; + await pool.query( + ` + update public.content_import_jobs + set status = $3, + error_message = $4, + summary = coalesce(summary, '{}'::jsonb) || $5::jsonb, + next_attempt_at = case when $6::boolean then now() + make_interval(secs => $7::integer) else null end, + locked_at = null, + locked_by = null, + finished_at = case when $3 = 'failed' then now() else finished_at end, + updated_at = now() + where tenant_id = $1 and id = $2 + `, + [ + job.tenantId, + job.id, + status, + truncate(errorMessage(error)), + JSON.stringify({ + lastWorkerError: { + code: errorCode(error), + message: truncate(errorMessage(error)), + workerId: config.importWorkerId, + failedAt: new Date().toISOString(), + nextAttempt, + maxAttempts: job.maxAttempts, + willRetry, + }, + }), + willRetry, + backoffSeconds(nextAttempt), + ], + ); + + await pool.query( + ` + insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details) + values ($1, $2, $3, 'content_import_job', $4, $5::jsonb) + `, + [ + job.tenantId, + job.createdBy, + willRetry ? `content.import.${job.importType}.retry_scheduled` : `content.import.${job.importType}.failed`, + job.id, + JSON.stringify({ + code: errorCode(error), + message: truncate(errorMessage(error)), + workerId: config.importWorkerId, + nextAttempt, + maxAttempts: job.maxAttempts, + }), + ], + ); + + return willRetry ? 'retrying' : 'failed'; +} + +export async function processImportBatch(): Promise { + const jobs = await claimImportJobs(); + const result: ImportWorkerResult = { + processed: jobs.length, + completed: 0, + completedWithErrors: 0, + failed: 0, + retrying: 0, + skipped: 0, + }; + + for (const job of jobs) { + try { + const execution = await executeContentImportJob( + { + tenantId: job.tenantId, + userId: job.createdBy || job.tenantId, + role: 'system_worker', + permissions: { 'content:*': true }, + templatePermissions: {}, + }, + { + jobId: job.id, + importType: job.importType, + allowPartial: importOptions(job.summary).allowPartial, + allowQueuedJob: true, + }, + ); + + if (execution.idempotent) result.skipped += 1; + else if (execution.status === 'completed_with_errors') result.completedWithErrors += 1; + else if (execution.status === 'completed') result.completed += 1; + else result.skipped += 1; + } catch (error) { + const state = await markImportFailed(job, error); + if (state === 'retrying') result.retrying += 1; + else result.failed += 1; + } + } + + return result; +} + +export async function closeImportExecutorPool() { + await closeApiImportPool(); +} diff --git a/docs/refactor/api-structure.md b/docs/refactor/api-structure.md index b17b3a9b..72d353ef 100644 --- a/docs/refactor/api-structure.md +++ b/docs/refactor/api-structure.md @@ -74,4 +74,4 @@ types.ts 仅本领域使用的类型 - `learning` 创建练习 session 时必须保存 `question_ids` 快照，避免随机刷题和模考过程中题目集合变化导致答题记录无法复盘。 - 排行榜必须由后端按租户、地区、班级和可信用户上下文聚合，前端不能自行扫描答题记录、积分流水或单词进度后排名；后续高流量场景再通过 worker/materialized view 做日榜、周榜和防刷。 - 资料、PDF、视频等对象存储资源必须先进入 `content_assets` 台账，再通过 API/Edge Function 做权限校验和签名 URL 下发；前端不能直接拼 OSS/COS/Supabase Storage 私有地址。 -- 批量导入必须先写 `content_import_jobs/items/issues`，保留原始 payload、规范化 payload、逐行问题和审计记录；同步 API 当前支持题目 JSON，Excel/CSV 和其它内容类型应接入同一管线。 +- 批量导入必须先写 `content_import_jobs/items/issues`，保留原始 payload、规范化 payload、逐行问题和审计记录；题目、单词、知识手册、分数线和视频 JSON/CSV/Excel 都应进入同一管线，复杂大批量导入通过 `executionMode=async` 交给 imports worker 消费。 diff --git a/docs/refactor/backend-capability-status.md b/docs/refactor/backend-capability-status.md index 6838bd45..784cfcce 100644 --- a/docs/refactor/backend-capability-status.md +++ b/docs/refactor/backend-capability-status.md @@ -163,8 +163,8 @@ | 知识手册 JSON preview/import | 可联调 | 支持书籍/章节/小节/知识点归一化 | | 分数线 JSON preview/import | 可联调 | 支持 `fields/schools/majors/records` 分桶或 `items` 列表，后端校验租户地区和院校/专业引用 | | 视频 JSON preview/import | 可联调 | 支持 `videos/items`，后端校验题目、科目、资源引用，导入后写入 `question_videos` | -| Excel/CSV 导入 | 待补齐 | 应复用 `content_import_jobs` 管线 | -| 大批量异步导入 | 待补齐 | 需要 `apps/worker` | +| Excel/CSV 导入 | 可联调 | 题目、单词、知识手册、分数线、视频已支持 CSV 和 `.xlsx` 解析，解析后复用 `content_import_jobs/items/issues` 管线并保留 `parser_metadata` | +| 大批量异步导入 | 可联调 | `executionMode=async` 会将 preview job 置为 `pending`；`apps/worker --job imports` 抢占 queued job，复用 API 导入 executor，支持重试、清锁和审计；导入后复检待补 | | 公共题库版本同步 | 待补齐 | 当前采纳为快照复制；后续需 worker 做增量同步、冲突处理、版本升级通知和租户自改保护 | ## 当前验证 @@ -174,6 +174,7 @@ ```bash npm audit npm run check:refactor +npm run test:worker:imports ``` `check:refactor` 包含： diff --git a/docs/refactor/backend-handoff-roadmap.md b/docs/refactor/backend-handoff-roadmap.md index 95635d13..28c459fb 100644 --- a/docs/refactor/backend-handoff-roadmap.md +++ b/docs/refactor/backend-handoff-roadmap.md @@ -14,7 +14,7 @@ - 销售/代理/CRM 已经有邀请码、扫码/分享事件、首绑客资保护、团队关系、统计、CRM 配置和入队能力。 - 旧题库 JSON、单词模板、知识手册嵌套模板、分数线 JSON 和视频绑定 JSON 已经进入后端 preview/import 管线，由后端负责规范化、校验、幂等、审计和租户隔离。 -因此，后端现在已经具备进入 Taro 前端第一阶段联调的基础。需要注意的是，它还不是完整生产交付状态，真实云端鉴权、对象存储生产安全、支付/短信生产账号、微信网页登录/QQ 登录、真实数据 dry-run 迁移和大批量异步导入仍需要继续补齐或联调。 +因此，后端现在已经具备进入 Taro 前端第一阶段联调的基础。需要注意的是，它还不是完整生产交付状态，真实云端鉴权、对象存储生产安全、支付/短信生产账号、微信网页登录/QQ 登录、真实数据 dry-run 迁移、导入后复检和模板字段映射仍需要继续补齐或联调。 ## 后端模块进度 @@ -23,16 +23,16 @@ | 多租户底座 | 可联调 | 租户、域名、品牌、设置、RLS 基础、审计、Supabase JWT/API 身份映射 | 真实云端 Auth/JWKS 回归、生产 RLS 深测 | | 平台后台 | 基础完成 | 租户、套餐、订阅、账单、服务费、用量、公共题库授权 | 自动计费、平台审计、公共题库版本同步 | | 租户后台 | 可联调 | 品牌、域名、支付账户、登录配置、密钥掩码、活动、兑换码、优惠券、勋章管理/发放、成员权限、角色模板、菜单/模块/字段权限配置 API、班级/教师/学生范围权限 | 前端权限 UI、更细的数据范围组合 | -| 题库与练习 | 可联调 | 内容入口、任意深度分类、题目集合、顺序/随机/全真模拟蓝图、组卷快照、答题、错题、收藏、模考报告、排行榜、公共题库采纳快照 | 专项策略、公共题库版本同步、Excel 导入、排行榜防刷/预聚合 | -| 背单词 | 可联调 | 单元、单词、进度、收藏、统计、每日计划、JSON 导入、排行榜 | Excel 导入、更细复习参数 | -| 知识手册 | 可联调 | 科目、章节、条目、Markdown 内容、嵌套 JSON 导入 | 富文本资源、版本管理、附件/PDF 关联 | -| 分数线 | 可联调 | 院校、专业、动态字段、记录、年份、趋势、后台维护、JSON 导入 | 复杂筛选、AI 择校上下文 | -| 视频解析 | 可联调 | 单题视频、批量查询、后台视频绑定、JSON 导入、会员播放权限、播放次数扣减、签名 URL 和播放日志 | 深度防盗链、动态水印、播放统计 | +| 题库与练习 | 可联调 | 内容入口、任意深度分类、题目集合、顺序/随机/全真模拟蓝图、组卷快照、答题、错题、收藏、模考报告、排行榜、公共题库采纳快照 | 专项策略、公共题库版本同步、排行榜防刷/预聚合 | +| 背单词 | 可联调 | 单元、单词、进度、收藏、统计、每日计划、JSON/CSV/Excel 导入、排行榜 | 更细复习参数 | +| 知识手册 | 可联调 | 科目、章节、条目、Markdown 内容、嵌套 JSON/CSV/Excel 导入 | 富文本资源、版本管理、附件/PDF 关联 | +| 分数线 | 可联调 | 院校、专业、动态字段、记录、年份、趋势、后台维护、JSON/CSV/Excel 导入 | 复杂筛选、AI 择校上下文 | +| 视频解析 | 可联调 | 单题视频、批量查询、后台视频绑定、JSON/CSV/Excel 导入、会员播放权限、播放次数扣减、签名 URL 和播放日志 | 深度防盗链、动态水印、播放统计 | | 资料下载 | 部分完成 | 资源台账、SVIP 权限校验、`local_dev`/阿里云 OSS/腾讯 COS/Supabase Storage 上传下载签名、上传确认、PDF/图片预览签名、assets worker 复检异常下架 | PDF 渲染、CDN 防盗链、杀毒扫描、视频水印 | | 会员与订单 | 可联调 | 下单、订单详情/状态轮询、优惠券领取/抵扣、零元订单自动开通、手工确认权限保护、激活码预检查/兑换、微信支付、支付宝、微信/支付宝发起退款、微信/支付宝退款查询确认、微信/支付宝退款通知 webhook、支付/退款补偿 worker、权益发放 | 完整资金流水对账、异常订单运营台 | | 登录认证 | 可联调 | 短信 mock、阿里云/腾讯云短信 adapter、迁移期 session、Supabase Auth JWT、微信小程序登录、OAuth 配置表 | 微信网页登录、QQ 登录、手机号换绑、真实生产账号联调 | | 销售/代理/CRM | 基础完成 | 邀请码、首绑保护、团队关系、销售统计、CRM 入队 | 小程序码真实生成、分佣结算、钉钉/飞书/企微 worker | -| 内容导入 | 可联调 | 题目、单词、知识手册、分数线、视频 JSON preview/import、issue、job、审计、幂等 | Excel/CSV 解析、大批量异步 worker、导入后复检 | +| 内容导入 | 可联调 | 题目、单词、知识手册、分数线、视频 JSON/CSV/Excel preview/import、issue、job、审计、幂等、`executionMode=async` 和 imports worker | 导入后复检、模板下载/字段映射 | | 数据看板 | 可联调 | 租户 dashboard 聚合接口，收益、注册、学习、内容、激活码、反馈、趋势、24h 活跃、套餐销量和运营动态 | 预聚合 worker、缓存、慢 SQL 监控和销售转化看板 | | AI 择校推荐 | 未开始 | 暂无 | 数据上下文、AI JSON schema、报告渲染、PDF 生成 | | Taro 前端 | 未开始 | 旧 Web 已有新 API 适配雏形 | `apps/taro`、跨端 API client、H5/小程序页面和端到端测试 | @@ -87,7 +87,7 @@ - XPay 或其它实际支付网关 adapter。 - 阿里云/腾讯云短信、微信小程序登录、微信网页登录、QQ 登录。 - 公共题库/地区题库版本同步，租户按 SaaS 套餐购买地区、科目和题库范围的更细计费策略。 -- Excel/CSV 解析、大批量异步导入和导入后复检。 +- 导入后复检和导入模板/字段映射 UI；大批量导入已支持 `executionMode=async`，前端按 job 状态轮询。 - 视频深度防盗链、动态水印和播放统计。 - 数据看板 API：收益、注册趋势、答题次数、收入趋势、题型分布、题目总量、套餐销量、24h 活跃。 diff --git a/docs/refactor/backend-progress.md b/docs/refactor/backend-progress.md index 9eb9a4de..4c678831 100644 --- a/docs/refactor/backend-progress.md +++ b/docs/refactor/backend-progress.md @@ -32,6 +32,8 @@ - 已新增 commerce worker 和 `npm run test:worker:commerce`，用于补偿查询微信/支付宝支付、处理中退款和漏通知场景；支付成功会幂等更新订单/支付并开通权益，退款成功会幂等更新退款/订单/支付并在全额退款时撤销订单权益，测试覆盖密钥不泄露和重复执行不重复开通。 - 已新增 assets worker 和 `npm run test:worker:assets`，用于复检 `content_assets` 托管对象元数据；正常资源会写入复检证据，异常资源会自动下架为 `draft`、标记 `upload_status=failed`，并记录审计与安全标记。 - 已新增分数线/视频 JSON 导入接口和集成测试：分数线支持字段、院校、专业、记录、动态字段值；视频支持视频元数据、资源引用、访问模式和题目绑定，全部复用 `content_import_jobs/items/issues` 管线。 +- 已新增 CSV/Excel 导入解析层：题目、单词、知识手册、分数线和视频都可通过 `sourceFormat=csv/excel` 先解析为规范 payload，再进入同一套 preview/import、逐行 issue、幂等和审计管线；分数线支持多 Sheet Excel。 +- 已新增 imports worker 和 `npm run test:worker:imports`：确认导入时可传 `executionMode=async`，API 将 preview job 排队为 `pending`，worker 复用同一导入 executor 消费任务，支持抢占锁、attempt/backoff、失败重试、清锁和审计。 ## 已验证接口 @@ -251,11 +253,11 @@ GET /api/tenant-admin/audit-logs - 内容资源当前完成台账、租户后台维护、学生端 SVIP 下载权限，以及 `local_dev`、阿里云 OSS、腾讯 COS、Supabase Storage 的上传/下载签名 provider；上传确认和 assets worker 已支持对象元数据校验/复检。PDF 预览渲染、防盗链、水印和安全扫描仍需继续补。 - 题库内容导航当前以 `content_entries/content_nodes` 为主模型，可表达“入口 -> 多级分类 -> 院校/专业/学科/销售意向标记”；题目集合和练习方式由 `question_collections/practice_blueprints` 管理，练习 session 会保存当次题目 ID 快照。 - 练习访问控制由 `content_entries/content_nodes/question_collections/practice_blueprints` 的 `accessRules` 合并决定；普通用户消耗 `practice_daily_usage`，事件写入 `practice_access_events`，SVIP/staff 不消耗免费额度。 -- 批量导入当前支持题目、单词、知识手册、分数线、视频 JSON 预览、逐行 issue、job/item 台账、执行导入、幂等跳过，并可落到新内容入口、分类节点、分数线表或题目视频绑定。旧单词模板的 `vocabulary_units_示例数据` / `vocabulary_示例数据`、知识手册的书籍/章节/小节/知识点嵌套结构都由后端规范化。Excel/CSV 和大批量异步导入会继续复用同一套 `content_import_jobs` 管线。 +- 批量导入当前支持题目、单词、知识手册、分数线、视频 JSON/CSV/Excel 预览、逐行 issue、job/item 台账、同步执行或 `executionMode=async` 异步执行、幂等跳过，并可落到新内容入口、分类节点、分数线表或题目视频绑定。旧单词模板的 `vocabulary_units_示例数据` / `vocabulary_示例数据`、知识手册的书籍/章节/小节/知识点嵌套结构都由后端规范化。导入后复检、模板下载和字段映射 UI 后续补齐。 ## 下一步 -1. 完善内容导入和文件上传：Excel/CSV 解析、大批量异步导入、导入后复检，PDF 预览渲染、防盗链、杀毒扫描和视频水印。 +1. 完善内容导入和文件上传：导入后复检、模板下载/字段映射 UI，PDF 预览渲染、防盗链、杀毒扫描和视频水印。 2. 接入真实短信 provider：阿里云/腾讯云，密钥放 `app_private.tenant_secrets` 或生产 Vault。 3. 接入真实 OAuth provider：微信网页、微信小程序、QQ，并处理旧 PocketBase 身份映射。 4. 补完整资金流水对账、异常订单运营台和优惠券核销报表；支付/退款补偿、退款查询确认和退款通知主链路已完成。 @@ -269,5 +271,6 @@ npm run test:api npm run test:worker:crm npm run test:worker:commerce npm run test:worker:assets +npm run test:worker:imports npm run check:refactor ``` diff --git a/docs/refactor/blueprint-coverage.md b/docs/refactor/blueprint-coverage.md index ded7faf9..b3aa489e 100644 --- a/docs/refactor/blueprint-coverage.md +++ b/docs/refactor/blueprint-coverage.md @@ -18,9 +18,9 @@ | 平台超级管理员 | 部分完成 | 租户管理、SaaS 套餐、订阅、账单、服务费收款、用量记录 | 公共题库披露策略、地区/全国套餐权限、平台侧主题模板库、平台审计 | | 租户品牌和域名 | 基础完成 | 品牌、Logo、主题 JSON、公开资源、域名、租户公开配置 | 三套默认主题、主题可视化编辑、图标/图片上传 | | 租户成员权限 | 可联调 | owner/admin/operator/teacher/sales/agent/student，权限矩阵，成员启停，角色模板、菜单/模块/字段权限、班级/学生范围权限和审计查询 | 前端权限 UI、更细的数据范围组合 | -| 题库内容维护 | 可联调 | 内容入口、任意深度分类树、院校/专业/学科/销售意向标记、题目集合、顺序/随机/全真模拟练习蓝图、题目录入/更新、题目/单词/知识手册/分数线/视频 JSON 预览导入、视频绑定、分数线、单词、知识手册后台 API、公共题库授权和采纳快照 | Excel/CSV 批量导入、公共题库版本同步、可视化拖拽排序前端 | +| 题库内容维护 | 可联调 | 内容入口、任意深度分类树、院校/专业/学科/销售意向标记、题目集合、顺序/随机/全真模拟练习蓝图、题目录入/更新、题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 预览导入、`executionMode=async` 导入 worker、视频绑定、分数线、单词、知识手册后台 API、公共题库授权和采纳快照 | 导入后复检、模板/字段映射、公共题库版本同步、可视化拖拽排序前端 | | 学生刷题 | 基础完成 | 内容入口、分类树、题目集合、顺序刷题、随机刷题、全真模拟 session 题目快照、答题、错题本、收藏夹、模考交卷评分报告、错题复习计划、排行榜 | 专项练习策略、题型统计深度分析、排行榜防刷/预聚合 | -| 背单词 | 基础完成 | 单词单元、单词、进度、收藏、统计、每日复习计划、旧模板/新模板 JSON 预览导入、内容导航绑定、排行榜 | Excel 导入、更细复习参数 | +| 背单词 | 基础完成 | 单词单元、单词、进度、收藏、统计、每日复习计划、旧模板/新模板 JSON/CSV/Excel 预览导入、内容导航绑定、排行榜 | 更细复习参数 | | 知识手册 | 基础完成 | 科目、章节、条目只读与后台维护、书籍/章节/小节/知识点嵌套 JSON 预览导入、内容导航绑定 | 富文本资源、版本管理、附件/PDF 关联、Excel/Markdown 批量解析 | | 分数线 | 可联调 | 字段、院校、专业、记录、趋势、年份、JSON 批量导入 | 复杂动态筛选、AI 择校数据上下文 | | 视频解析会员 | 可联调 | 题目视频、批量查询、后台绑定、视频 JSON 导入、SVIP 权限、播放次数扣减、签名 URL、播放日志 | 深度防盗链、动态水印、播放统计 | @@ -36,7 +36,7 @@ ## 接下来优先级 -1. 完善内容导入和对象存储：Excel/CSV 解析、大批量异步导入、导入后复检、CDN 防盗链、杀毒扫描和视频水印。 +1. 完善内容导入和对象存储：导入后复检、模板下载/字段映射 UI、CDN 防盗链、杀毒扫描和视频水印。 2. 公共题库/地区题库授权：已完成披露和采纳快照；继续补版本同步、租户自改冲突处理和按 SaaS 套餐限制地区。 3. 学习统计增强：排行榜防刷/预聚合、断点续练、专项练习策略和更细题型分析。 4. 视频会员控制：深度防盗链、水印和播放统计。 diff --git a/docs/refactor/content-import-contract.md b/docs/refactor/content-import-contract.md index f6bb111a..5f0a4f63 100644 --- a/docs/refactor/content-import-contract.md +++ b/docs/refactor/content-import-contract.md @@ -50,6 +50,59 @@ GET /api/tenant-content/imports/issues - `content_import_issues` - `audit_logs` +## 格式支持 + +当前题目、单词、知识手册、分数线和视频导入均支持： + +- `sourceFormat=json`：直接提交规范 JSON 或兼容旧模板。 +- `sourceFormat=csv`：提交 `csvText`、`fileContent`、`payload` 或 `fileBase64`，后端按表头归一化。 +- `sourceFormat=excel`：提交 `.xlsx` 的 `fileBase64`，可选 `sheetName` 或 `sheetIndex`；多 Sheet 会被解析进同一个 `content_import_jobs`。 +- 同步导入默认在 API 请求内完成；大批量导入可在确认导入时传 `executionMode=async`，API 会把 job 置为 `pending`，由 imports worker 后台消费。 + +表格导入安全边界： + +- 单文件最大 8MB。 +- 单次最多解析 5000 行、160 列。 +- 单个单元格最多保留 100000 字符。 +- 后端只保存解析后的原始行和规范化 payload，不把 `fileBase64` 长期写入数据库。 +- 任务列表会返回 `sourceFormat`、`executionMode` 和 `parserMetadata`，前端可展示解析器、Sheet、行数等信息。 + +异步导入调用方式： + +```json +{ + "previewJobId": "uuid", + "executionMode": "async", + "allowPartial": false +} +``` + +异步导入状态流： + +```text +preview -> pending -> importing -> completed +preview -> pending -> importing -> completed_with_errors +preview -> pending -> failed +``` + +worker 命令： + +```bash +npm --workspace @tiku-saas/worker run imports:once +``` + +前端提交异步导入后不要重复同步执行同一 job；只需要轮询 `GET /api/tenant-content/imports` 并用 `GET /api/tenant-content/imports/issues` 展示问题行。worker 会按 `attempt_count/max_attempts` 记录重试，失败时写入 `errorMessage` 和审计日志。 + +推荐 CSV/Excel 表头： + +| 类型 | 常用表头 | +| --- | --- | +| 题目 | `legacyId`、`题型/type`、`题干/content`、`选项A`-`选项H`、`答案`、`解析`、`难度`、`标签` | +| 单词 | `unitLegacyId`、`unitName`、`wordLegacyId`、`word`、`phonetic`、`meaning`、`example`、`difficulty`、`tags` | +| 知识手册 | `subjectLegacyId`、`subjectName`、`chapterLegacyId`、`chapterName`、`sectionName`、`entryLegacyId`、`title`、`content`、`summary`、`tags` | +| 分数线 | 多 Sheet 推荐 `fields`、`schools`、`majors`、`records`；记录 Sheet 可包含任意动态字段列 | +| 视频 | `legacyId`、`title`、`videoUrl`、`thumbnailUrl`、`durationSeconds`、`subjectId`、`questionId` 或 `legacyQuestionId`、`accessMode` | + ## 单词导入 推荐新格式： @@ -262,5 +315,5 @@ GET /api/tenant-content/imports/issues ## 下一步 -- 增加 Excel/CSV 解析入口，但解析后仍进入同一套 preview/import 管线。 -- 增加异步 worker，处理大批量导入、重试和导入后校验。 +- 增加导入后校验/复检，确认导入行数、目标表记录、题目集合绑定和视频绑定一致。 +- 增加导入模板下载接口和后台可视化字段映射器。 diff --git a/docs/refactor/frontend-handoff-index.md b/docs/refactor/frontend-handoff-index.md index 017f141e..f7fd65fe 100644 --- a/docs/refactor/frontend-handoff-index.md +++ b/docs/refactor/frontend-handoff-index.md @@ -1,6 +1,6 @@ # 前端交接索引 -更新时间：2026-06-28 +更新时间：2026-06-29 这份文件是给 Taro/H5/小程序前端同事的入口。当前仓库的前端重构建议从 `apps/taro` 新建工程开始，不再把旧 React/Vite 前端搬回根目录继续开发。 @@ -45,7 +45,7 @@ - 不要把“Supabase 支持前端 Data API”误解为“本项目所有业务表都由 Taro 直写”；订单、支付、权益、租户后台、导入、CRM、私有资源必须走 RPC、`apps/api`、Edge Function 或 worker 这类后端命令层。 - 真实短信、微信登录、QQ 登录、微信支付、支付宝支付 provider 还未正式接完。 - 对象存储已完成签名 provider、上传后校验、PDF/图片预览和资源复检 worker，但 CDN 防盗链、视频水印和杀毒扫描还要补。 -- 分数线/视频 JSON 导入已可联调；大批量 Excel/CSV 解析和异步导入 worker 还未完成。 +- 题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 导入已可联调；大批量导入可传 `executionMode=async` 交给 imports worker，前端还要补上传预览、排队轮询、问题行展示、模板下载和字段映射 UI。 - 数据看板、分佣结算和勋章手动发放基础 API 已可联调；勋章自动发放、分佣真实打款/导出/凭证、AI 择校、主题模板市场等仍是后续商用增强项。 ## 前后端协作建议 diff --git a/docs/refactor/implementation-status.md b/docs/refactor/implementation-status.md index a332cfe4..98335629 100644 --- a/docs/refactor/implementation-status.md +++ b/docs/refactor/implementation-status.md @@ -4,7 +4,7 @@ ## 当前结论 -当前重构已经完成了 Supabase/PostgreSQL 多租户底座、核心业务表、PocketBase 数据导入器雏形、学生端核心 API、租户后台 API、平台后台 SaaS 账务 API、内容资产/题目/单词/知识手册/分数线/视频 JSON 批量导入基础闭环、题库入口/任意深度分类/题目集合/练习蓝图/组卷快照基础闭环，以及本地 Docker/API 构建验证。 +当前重构已经完成了 Supabase/PostgreSQL 多租户底座、核心业务表、PocketBase 数据导入器雏形、学生端核心 API、租户后台 API、平台后台 SaaS 账务 API、内容资产/题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 批量导入基础闭环、异步导入 worker、题库入口/任意深度分类/题目集合/练习蓝图/组卷快照基础闭环，以及本地 Docker/API 构建验证。 但这还不是完整商用交付状态，也不能说旧项目核心功能已经全部重构完成。现在更准确的状态是：后端商用架构骨架已经立住，核心业务正在按模块补齐。部分功能已经有可调用 API，部分功能只有数据模型和导入映射，部分功能还没有前端/自动化测试闭环。 @@ -24,7 +24,7 @@ | 模块 | 数据模型 | PocketBase 导入 | API | 自动化测试 | 当前状态 | | --- | --- | --- | --- | --- | --- | | 多租户隔离 | 已建 `tenants`、`tenant_domains`、`tenant_branding`、`tenant_settings`、RLS 基础 | 部分支持 | 租户解析、品牌、域名、支付账户、登录 provider、平台建租户已实现 | 核心 API 集成测试含租户隔离断言 | 基础可用，正式 JWT/RLS 权限闭环未完成 | -| 刷题题库 | 已建题库、题目、题目版本、内容入口、任意深度分类树、考试意向标记、题目集合、练习蓝图、导入任务台账、公共题库授权/采纳表 | 已支持核心映射，JSON 导入可落到新入口/节点/集合 | 题目列表、内容入口、分类树、集合题目、顺序/随机/全真模拟 session、答题提交、租户后台题目录入/更新、JSON 预览/导入、平台公共题库授权、租户采纳快照已实现 | 核心 API 集成测试含导航、组卷、导入、公共题库授权和采纳后组卷断言 | 新题库导航和组卷基础闭环可跑，公共题库采纳快照可联调；Excel 导入、公共题库全量/增量版本同步仍需补齐 | +| 刷题题库 | 已建题库、题目、题目版本、内容入口、任意深度分类树、考试意向标记、题目集合、练习蓝图、导入任务台账、公共题库授权/采纳表 | 已支持核心映射，JSON/CSV/Excel 导入可落到新入口/节点/集合 | 题目列表、内容入口、分类树、集合题目、顺序/随机/全真模拟 session、答题提交、租户后台题目录入/更新、JSON/CSV/Excel 预览/导入、异步导入 worker、平台公共题库授权、租户采纳快照已实现 | 核心 API 集成测试含导航、组卷、导入、公共题库授权和采纳后组卷断言 | 新题库导航和组卷基础闭环可跑，公共题库采纳快照可联调；公共题库全量/增量版本同步、导入后复检仍需补齐 | | 错题本 | 已建 `wrong_questions` | 已支持旧错题归一化 | 错题列表、答题自动入错题、移出错题已实现 | 仅烟测 | 基础功能已实现，复习计划和统计未完成 | | 收藏夹 | 已建 `favorite_questions` | 已支持旧收藏归一化 | 收藏/取消收藏、收藏列表已实现 | 仅烟测 | 基础功能已实现 | | 用户订阅/题库会员/SVIP | 已建 `orders`、`payments`、`entitlements`、`svip_plans`、激活码 | 已映射旧 SVIP/会员权益 | 下单、订单详情/状态轮询、手工支付确认权限保护、微信/支付宝支付、微信/支付宝发起退款、微信/支付宝退款查询确认、微信/支付宝退款通知 webhook、激活码预检查/兑换、优惠券抵扣、零元订单自动开通、权益查询已实现 | API 集成测试 | 商城主链路可联调，对账、支付补偿和异常订单自动处理待补 | @@ -36,7 +36,7 @@ | 个人中心 | 已建 `student_profiles`、会员权益、订单、练习记录、`badges/user_badges` | 已支持部分用户资料和勋章导入 | 个人资料、目标院校/专业、会员状态、最近练习、统计聚合、签到积分、题目反馈、考试倒计时、勋章 API 已实现 | API 集成测试 | 学生端基础个人中心已实现，账号绑定/换绑、学习报告可视化和更细任务系统待补 | | 活动/优惠 | 已建优惠券、激活码、激活码批次、banner、FAQ、公告、勋章等基础表 | 部分支持 | banner/FAQ/公告只读与租户后台维护、激活码预检查/兑换、激活码批次、批量生成激活码、优惠券维护、前台领取/下单抵扣、勋章维护和手动发放已实现 | 核心 API 集成测试 | 基础运营后台可用，勋章自动发放、复杂活动规则、营销自动化、核销报表待补 | | 销售/代理客资追踪 | 已建推荐码、首绑客资、团队关系、小程序码缓存、CRM 队列 | 旧 `referral_tracks` 已有映射基础 | 邀请码、扫码/分享事件、首绑保护、销售统计、客资明细、手动补绑、团队关系、CRM 配置/队列、CRM worker 推送已实现 | 核心 API 集成测试、CRM worker 集成测试 | 增长链路基础可用，真实微信小程序码、CRM 分配策略、富卡片和销售转化看板待补 | -| 租户后台 | 已建品牌、域名、设置、支付账户、登录 provider、私密密钥表、成员、审计日志、资源台账、导入台账、内容导航台账 | 不适用 | 概览、品牌、设置、域名、支付账户、登录配置、密钥掩码、活动内容、兑换码/优惠券、成员管理、权限矩阵、审计查询、内容入口/分类树/题目集合/练习蓝图维护、资源管理、题目/单词/知识手册/分数线/视频 JSON 导入已实现 | 核心 API 集成测试含角色/权限/租户隔离/密钥不泄露/导航/组卷/资源与导入断言 | 租户配置与运营闭环可用，前端权限 UI、Excel/CSV 导入和大批量异步导入待补 | +| 租户后台 | 已建品牌、域名、设置、支付账户、登录 provider、私密密钥表、成员、审计日志、资源台账、导入台账、内容导航台账 | 不适用 | 概览、品牌、设置、域名、支付账户、登录配置、密钥掩码、活动内容、兑换码/优惠券、成员管理、权限矩阵、审计查询、内容入口/分类树/题目集合/练习蓝图维护、资源管理、题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 同步/异步导入已实现 | 核心 API 集成测试含角色/权限/租户隔离/密钥不泄露/导航/组卷/资源与导入断言 | 租户配置与运营闭环可用，前端权限 UI、导入后复检和字段映射操作台待补 | | 平台后台 | 已建 SaaS 套餐、订阅、账单、服务费、用量 | 不适用 | 租户管理、账单、收款确认、用量记录、平台管理员 Supabase JWT 鉴权已实现 | API 集成测试 | 平台收费链路骨架可用，平台审计报表/自动计费待补 | | 登录认证 | 已建短信验证码、会话、OAuth provider 配置表，并支持 `auth_user_id` 映射 | 旧用户映射已预留 | 短信 mock 登录、迁移期 session、Supabase JWT 验签映射、微信小程序登录主链路已实现 | API 集成测试 | H5 Supabase Auth 可联调；真实短信/微信网页/QQ 登录生产联调待补 | | 数据导入 | 已建立 importer、risk report、validate | 已覆盖多类旧集合 | 命令行导入/校验 | `pb:import:validate` | 基础工具可用，需用真实完整数据做多轮 dry-run | @@ -268,7 +268,7 @@ platform-admin: 1. 正式鉴权：API 已支持 Supabase Auth JWT；生产前继续做真实云端 Auth/JWKS 回归、RLS 深测，并关闭 `x-user-id`、`x-platform-admin-key` 兼容入口。 2. 国内能力接入：短信、微信小程序登录、微信支付、支付宝支付、发起退款、退款查询确认和退款通知 webhook 的租户级配置入口与本地 provider 验证已具备；微信网页登录、QQ 登录、真实生产账号联调、对账和支付补偿仍需实现。 3. 核心缺口 API：学生端个人中心、分数线、题目视频详情、背单词进度/收藏、签到积分、题目反馈和勋章已补基础 API；下一步重点是账号绑定、学习报告可视化、后台统计和真实业务验收。 -4. 后台能力：题库录入、题目/单词/知识手册/分数线/视频 JSON 批量导入、资源台账、视频绑定、知识手册维护、分数线维护、品牌/商户/登录/活动/兑换码配置、销售客资、CRM 队列、成员权限、审计查询已补 API；Excel/CSV 导入、大批量异步导入和前端操作台待补。 +4. 后台能力：题库录入、题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 同步/异步批量导入、资源台账、视频绑定、知识手册维护、分数线维护、品牌/商户/登录/活动/兑换码配置、销售客资、CRM 队列、成员权限、审计查询已补 API；导入后复检、模板下载/字段映射和前端操作台待补。 5. 自动化测试：已建立核心 API、租户隔离、权限矩阵、后台维护、资源/导入、微信/支付宝支付 webhook、优惠券/激活码/订单状态集成测试；仍需真实数据导入回归、退款对账和前端端到端测试。 6. Taro 前端：建立 `apps/taro` 或等价跨端应用，把 H5 和小程序统一走同一套 API client。 7. 运维交付：生产环境变量、备份恢复、日志监控、异常告警、数据库迁移流程、灰度发布、回滚预案。 @@ -277,7 +277,7 @@ platform-admin: 为了先把旧项目核心业务补齐，再进入支付/短信等商用关键模块，建议按下面顺序继续： -1. 完善内容导入和文件上传：Excel/CSV 解析、大批量异步导入、导入后复检、CDN 防盗链、杀毒扫描，并把 JSON 导入扩展为异步 worker。 +1. 完善内容导入和文件上传：导入后复检、模板下载/字段映射 UI、CDN 防盗链、杀毒扫描。 2. 补公共题库版本同步 worker、租户套餐地区/科目/题库范围限制、主题模板系统。 3. 补学习统计增强：排行榜防刷/预聚合、断点续练、专项练习策略和更细题型分析。 4. 补视频商用控制：深度防盗链、动态水印和播放统计。 diff --git a/docs/refactor/legacy-feature-gap-matrix.md b/docs/refactor/legacy-feature-gap-matrix.md index a060d218..50372212 100644 --- a/docs/refactor/legacy-feature-gap-matrix.md +++ b/docs/refactor/legacy-feature-gap-matrix.md @@ -58,7 +58,7 @@ | 优惠券 | 已覆盖 | 后台配置、前台领取、同用户同券幂等、下单抵扣、全额优惠自动开通权益已有；复杂活动规则和核销报表待补 | | 激活码 | 已覆盖 | 批次、生成、预检查、兑换、自用码拒绝、地区校验主链路已有 | | 勋章管理 | 部分覆盖 | 后台勋章维护、手动发放、重复发放幂等、学生端勋章展示和权限隔离已覆盖；自动发放规则、积分活动联动和前端运营 UI 待补 | -| 题库录入 | 已覆盖 | 单题创建/更新、题目/单词/知识手册/分数线/视频 JSON 导入、集合/蓝图已有；Excel/CSV 和异步大批量导入待补 | +| 题库录入 | 已覆盖 | 单题创建/更新、题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 同步/异步导入、集合/蓝图已有；导入后复检、模板下载和字段映射 UI 待补 | | 题库导出 PDF/Word/JSON | 未覆盖 | 旧前端有导出组件；新后端需决定服务端导出、导出水印和权限审计 | | 题型分组/模拟卷配置 | 部分覆盖 | question_type_groups 表和 blueprint 有基础；后台配置体验待补 | | 背单词维护 | 已覆盖 | 单元/单词 CRUD 和导入已有 | @@ -98,7 +98,7 @@ 1. 排行榜增强：刷题、模考、背单词、积分排行榜主接口已有；还需防刷、日/周榜预聚合、运营后台排名看板。 2. 账号设置完整流：头像上传、绑定/更换手机号、微信/QQ 账号合并、密码/邮箱能力。 3. 题库导出：PDF/Word/JSON 导出、水印、导出审计和权限控制。 -4. 导入扩展：分数线/视频 JSON 导入已补齐；仍缺 Excel/CSV 解析、大批量异步 worker 和导入后复检。 +4. 导入扩展：题目/单词/知识手册/分数线/视频已支持 JSON、CSV 和 Excel 预览导入，并可用 `executionMode=async` 进入 imports worker；仍缺导入后复检和模板下载/字段映射 UI。 5. 公共题库商业化：平台公共/地区题库授权和租户快照采纳已完成基础闭环；还需版本同步、租户自改冲突处理和运营后台 UI。 6. CRM/销售结算：CRM worker、分佣规则、结算单、审核和打款状态基础闭环已完成；仍缺轮询/定向分配、打款导出、凭证和销售结算看板。 7. 题目反馈增强：处理通知、消息提醒、问题聚合统计和内容修复闭环。 @@ -117,7 +117,7 @@ 1. 微信/支付宝支付和 webhook 幂等。 2. 对象存储 PDF 预览、视频深度防盗链、动态水印。 -3. Excel/CSV 解析、大批量异步导入和导入后复检。 +3. 导入后复检、模板下载和字段映射 UI。 4. 数据看板预聚合 worker、销售/代理转化看板和分佣结算。 5. 公共题库版本同步、租户采纳后的更新策略和同步 worker。 diff --git a/docs/refactor/next-development-todo.md b/docs/refactor/next-development-todo.md index e26932af..60706ccf 100644 --- a/docs/refactor/next-development-todo.md +++ b/docs/refactor/next-development-todo.md @@ -16,7 +16,7 @@ - 练习组卷：`question_collections/practice_blueprints` 支持顺序、随机、全真模拟快照。 - 模考报告与学习统计：`practice_session_reports/practice_session_report_sections` 支持交卷、评分、题型/小节统计、错题解析汇总和历史查询；`/api/learning/stats`、`trend`、`practice-sessions/history`、`wrong-questions/review-plan` 可支撑个人中心和学习报告基础页。 - 练习访问控制：`practice_daily_usage/practice_access_events` 支持免费每日额度、SVIP 范围校验、SVIP-only 内容拦截和答题 session 快照保护。 -- 内容导入：题目、单词、知识手册、分数线、视频 JSON 预览、校验、导入、幂等、审计。 +- 内容导入：题目、单词、知识手册、分数线、视频 JSON/CSV/Excel 预览、校验、导入、幂等、审计。 - 租户组织范围：班级、班级成员、教师/班主任/助教/学生分组，教师按负责班级查看学生，字段权限可脱敏学生手机号。 - 学生运营管理：学生批量 upsert、禁用/恢复、批量分班、备注、跟进任务已完成接口和集成测试；后续补批量 CRM 推送和自动学习督导。 - 旧题库运营缺口已补一批：考试日期/倒计时、题目反馈/纠错处理、每日签到积分和积分流水、学习排行榜已完成接口和集成测试。 @@ -75,9 +75,9 @@ - 旧 PocketBase 用户账号和新身份体系的映射/补绑。 3. 导入体系扩展 - - Excel/CSV 导入。 - - 分数线/视频导入已完成 JSON preview/import；继续补 Excel/CSV 到规范 JSON 的解析适配。 - - 大批量导入异步 worker、重试、导入后校验。 + - 已完成题目、单词、知识手册、分数线、视频的 CSV/Excel 到规范 JSON 解析适配。 + - 继续补大批量导入异步 worker、重试、导入后校验。 + - 继续补模板下载、字段映射 UI 和导入前抽样校验。 4. 公共题库和租户授权 - 已完成平台公共题库/地区题库的基础授权、租户采纳和题目快照复制。 @@ -206,5 +206,5 @@ 2. 云服务器部署 Supabase/PostgreSQL 和 API，配置对象存储生产环境变量，跑 `check:refactor` 的远程等价测试。 3. 导出现有 PocketBase 数据，做完整 dry-run 迁移。 4. 开始 `apps/taro`，先接租户解析、首页、题库、背单词、知识手册。 -5. 并行补对象存储、真实登录、完整资金流水对账和公共题库版本同步 worker。 +5. 并行补对象存储、真实登录、完整资金流水对账、题库导出和公共题库版本同步 worker。 6. 前后端联调通过后，再做支付、权限、数据导入、资料下载、视频播放的商用验收。 diff --git a/docs/refactor/taro-frontend-integration.md b/docs/refactor/taro-frontend-integration.md index a9e08135..a51c97f4 100644 --- a/docs/refactor/taro-frontend-integration.md +++ b/docs/refactor/taro-frontend-integration.md @@ -713,7 +713,7 @@ content_entries ## 租户内容导入对接 -租户后台导入统一使用 preview -> issues -> import 流程，前端不要直接写 Supabase 表或绕过 `apps/api`。 +租户后台导入统一使用 preview -> issues -> import 流程，前端不要直接写 Supabase 表或绕过 `apps/api`。当前 JSON、CSV 和 Excel 都进入同一套后端规范化、逐行 issue、幂等和审计管线。 当前可联调： @@ -734,14 +734,63 @@ GET /api/tenant-content/imports/issues 前端流程： -1. 上传或粘贴 JSON，先调用对应 preview。 +1. 上传或粘贴 JSON/CSV/Excel，先调用对应 preview。 2. 展示 `job.totalCount/validCount/errorCount/warningCount`。 -3. 展示逐行 `issues`，错误行必须让运营修正；如果后端允许 `allowPartial`，也要二次确认。 -4. 用户确认后调用 import；导入完成后刷新内容列表、分数线列表或题目视频列表。 +3. 展示 `job.sourceFormat`、`job.parserMetadata`、逐行 `issues`，错误行必须让运营修正；如果后端允许 `allowPartial`，也要二次确认。 +4. 小批量确认后直接调用 import；大批量确认时传 `executionMode=async` 排队，前端轮询 job 状态。 +5. 导入完成后刷新内容列表、分数线列表或题目视频列表。 + +CSV 请求示例： + +```json +{ + "sourceFormat": "csv", + "sourceName": "questions.csv", + "csvText": "legacyId,题型,题干,选项A,选项B,答案\nq1,choice,题干,A,B,B", + "subjectId": "...", + "categoryId": "...", + "entryId": "...", + "contentNodeId": "...", + "collectionId": "..." +} +``` + +Excel 请求示例： + +```json +{ + "sourceFormat": "excel", + "sourceName": "scoreline.xlsx", + "fileBase64": "", + "sheetName": "records", + "regionId": "..." +} +``` + +异步确认导入示例： + +```json +{ + "previewJobId": "uuid", + "executionMode": "async", + "allowPartial": false +} +``` + +异步导入状态： + +```text +pending/importing：展示处理中，不允许重复同步执行同一 job。 +completed：刷新目标内容列表。 +completed_with_errors：刷新成功内容，并提示查看 issues。 +failed/rejected：展示 errorMessage 和 issues，允许运营修正后重新 preview。 +``` + +前端文件限制应与后端一致：单文件最大 8MB，最多 5000 行、160 列。后端不会保存原始 `fileBase64`，但前端仍不要把含隐私的导入文件写入长期缓存。 分数线导入前端注意： -- 后端支持 `fields/schools/majors/records` 分桶，也支持 `items` 列表。 +- 后端支持 `fields/schools/majors/records` 分桶，也支持 `items` 列表；Excel 可用 `fields`、`schools`、`majors`、`records` 多 Sheet。 - 页面筛选字段仍以 `/api/scoreline/fields` 为准，不要从导入 JSON 临时生成筛选 UI。 - `record` 至少需要 `schoolId`、`schoolLegacyId` 或 `schoolName`，否则 preview 会返回 issue。 @@ -1081,7 +1130,7 @@ ACTIVATION_CODE_REGION_MISMATCH - 角色模板：`GET/PUT /api/tenant-admin/role-templates`、`POST /api/tenant-admin/role-templates/disable` - 内容入口/分类树/题目集合/练习蓝图 - 题目/单词/知识手册/分数线/视频维护 -- 题目/单词/知识手册/分数线/视频 JSON 导入 preview/import/issues +- 题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 导入 preview/import/issues - Banner/FAQ/公告/激活码/优惠券 - 勋章：`GET/PUT /api/tenant-admin/badges`、`GET/POST /api/tenant-admin/badge-grants` - 考试日期：`GET/PUT /api/tenant-admin/exam-dates` diff --git a/package-lock.json b/package-lock.json index 9ec8df85..829b80a7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "dependencies": { "@supabase/storage-js": "^2.108.2", "ali-oss": "^6.23.0", + "exceljs": "^4.4.0", "jose": "^6.2.3", "pg": "^8.16.3" }, @@ -133,6 +134,47 @@ "node": ">=18" } }, + "node_modules/@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/format/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "license": "MIT" + }, + "node_modules/@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/@fast-csv/parse/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "license": "MIT" + }, "node_modules/@supabase/cli-darwin-arm64": { "version": "2.107.0", "resolved": "https://registry.npmjs.org/@supabase/cli-darwin-arm64/-/cli-darwin-arm64-2.107.0.tgz", @@ -373,12 +415,210 @@ "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", "license": "MIT" }, + "node_modules/archiver": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.2.tgz", + "integrity": "sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^2.1.0", + "async": "^3.2.4", + "buffer-crc32": "^0.2.1", + "readable-stream": "^3.6.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^2.2.0", + "zip-stream": "^4.1.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", + "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", + "license": "MIT", + "dependencies": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/archiver/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/big-integer": { + "version": "1.6.52", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", + "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", + "license": "Unlicense", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", + "license": "MIT", + "dependencies": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==", + "license": "MIT" + }, "node_modules/bowser": { "version": "1.9.4", "resolved": "https://registry.npmjs.org/bowser/-/bowser-1.9.4.tgz", "integrity": "sha512-9IdMmj2KjigRq6oWhmwv1W36pDuA4STQZ8q6YO9um+x07xgYNCD3Oou+WP/3L1HNz7iqythGet3/p4wvc8AAwQ==", "license": "MIT" }, + "node_modules/brace-expansion": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.15.tgz", + "integrity": "sha512-EwOCDEex4quD37XhqM3omwtMoJjr//isUZz1JopUNWms+4Z2ViyM/k1YIRePpoVNnQhENnxtFjLaxNHrT7xIUg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/buffer-indexof-polyfill": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", + "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", + "engines": { + "node": ">=0.2.0" + } + }, "node_modules/builtin-status-codes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", @@ -414,6 +654,53 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", + "license": "MIT/X11", + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, + "node_modules/compress-commons": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.2.tgz", + "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^4.0.2", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/compress-commons/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT" + }, "node_modules/content-type": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", @@ -435,6 +722,45 @@ "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", "license": "MIT" }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.3.tgz", + "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^3.4.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/crc32-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/dateformat": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-2.2.0.tgz", @@ -444,6 +770,12 @@ "node": "*" } }, + "node_modules/dayjs": { + "version": "1.11.21", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.21.tgz", + "integrity": "sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==", + "license": "MIT" + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -506,6 +838,15 @@ "node": ">= 0.4" } }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "license": "BSD-3-Clause", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -566,6 +907,40 @@ "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", "license": "MIT" }, + "node_modules/exceljs": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/exceljs/-/exceljs-4.4.0.tgz", + "integrity": "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg==", + "license": "MIT", + "dependencies": { + "archiver": "^5.0.0", + "dayjs": "^1.8.34", + "fast-csv": "^4.3.1", + "jszip": "^3.10.1", + "readable-stream": "^3.6.0", + "saxes": "^5.0.1", + "tmp": "^0.2.0", + "unzipper": "^0.10.11", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=8.3.0" + } + }, + "node_modules/exceljs/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/extend-shallow": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", @@ -578,6 +953,19 @@ "node": ">=0.10.0" } }, + "node_modules/fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "license": "MIT", + "dependencies": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/formstream": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/formstream/-/formstream-1.5.2.tgz", @@ -590,6 +978,18 @@ "pause-stream": "~0.0.11" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC" + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -605,6 +1005,22 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "dependencies": { + "graceful-fs": "^4.1.2", + "inherits": "~2.0.0", + "mkdirp": ">=0.5 0", + "rimraf": "2" + }, + "engines": { + "node": ">=0.6" + } + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -657,6 +1073,27 @@ "integrity": "sha512-mFXCZPJIlcYcth+N8267+mghfYN9h3EhsDa6JSnbA3Wrhh/XFpuowviFcsDeYZtKspQyWyJqfs4O6P8CHeTwzw==", "license": "MIT" }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -669,6 +1106,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -723,6 +1166,43 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "license": "MIT" + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -788,12 +1268,130 @@ "integrity": "sha512-OYWlK0j+roh+eyaMROlNbS5cd5R25Y+IUpdl7cNdB8HNrkgwQzIS7L9MegxOiWNBj9dQhA/yAxiMwCC5mwNoBw==", "license": "MIT" }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "license": "(MIT OR GPL-3.0-or-later)", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/listenercount": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", + "integrity": "sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ==", + "license": "ISC" + }, "node_modules/lodash": { "version": "4.18.1", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "license": "MIT" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.difference": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", + "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==", + "license": "MIT" + }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", + "license": "MIT" + }, + "node_modules/lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==", + "license": "MIT" + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "license": "MIT" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "license": "MIT" + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", + "license": "MIT" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==", + "license": "MIT" + }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==", + "license": "MIT" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "license": "MIT" + }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==", + "license": "MIT" + }, + "node_modules/lodash.union": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", + "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==", + "license": "MIT" + }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", + "license": "MIT" + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -824,6 +1422,18 @@ "node": ">=4.0.0" } }, + "node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", @@ -871,6 +1481,15 @@ "node": ">=8.0.0" } }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -932,6 +1551,21 @@ "node": ">=0.10.0" } }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/pause-stream": { "version": "0.0.11", "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", @@ -1131,6 +1765,49 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "license": "MIT" }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/brace-expansion": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.1.tgz", + "integrity": "sha512-WR1cURNjuvBLMZBMbqM0UoE+WAfdUcEV1ccD8PVBVOI+Z3ND4+SZbN8RsfT2bMuG1qwz5RFvPukSZm5fF2D5eA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -1146,6 +1823,18 @@ "node": ">=11.0.0" } }, + "node_modules/saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/sdk-base": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/sdk-base/-/sdk-base-2.0.1.tgz", @@ -1155,6 +1844,12 @@ "get-ready": "~1.0.0" } }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "license": "MIT" + }, "node_modules/side-channel": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", @@ -1302,6 +1997,36 @@ "@supabase/cli-windows-x64": "2.107.0" } }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tar-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/thenify": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", @@ -1329,12 +2054,30 @@ "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", "license": "MIT" }, + "node_modules/tmp": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", + "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, "node_modules/to-arraybuffer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz", "integrity": "sha512-okFlQcoGTi4LQBG/PgSYblw9VOyptsz2KJZqc6qtgGdes8VktzUQkj4BI2blit072iS8VODNcMA+tvnS9dnuMA==", "license": "MIT" }, + "node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "license": "MIT/X11", + "engines": { + "node": "*" + } + }, "node_modules/tsx": { "version": "4.22.4", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.4.tgz", @@ -1820,6 +2563,24 @@ "node": ">=0.10.0" } }, + "node_modules/unzipper": { + "version": "0.10.14", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.14.tgz", + "integrity": "sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g==", + "license": "MIT", + "dependencies": { + "big-integer": "^1.6.17", + "binary": "~0.3.0", + "bluebird": "~3.4.1", + "buffer-indexof-polyfill": "~1.0.0", + "duplexer2": "~0.1.4", + "fstream": "^1.0.12", + "graceful-fs": "^4.2.2", + "listenercount": "~1.0.1", + "readable-stream": "~2.3.6", + "setimmediate": "~1.0.4" + } + }, "node_modules/urllib": { "version": "2.44.1", "resolved": "https://registry.npmjs.org/urllib/-/urllib-2.44.1.tgz", @@ -1873,6 +2634,16 @@ "node": ">= 0.12.0" } }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/win-release": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/win-release/-/win-release-1.1.1.tgz", @@ -1922,6 +2693,12 @@ "node": ">=4.0" } }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "license": "MIT" + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -1931,6 +2708,55 @@ "node": ">=0.4" } }, + "node_modules/zip-stream": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", + "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^3.0.4", + "compress-commons": "^4.1.2", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/zip-stream/node_modules/archiver-utils": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-3.0.4.tgz", + "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", + "license": "MIT", + "dependencies": { + "glob": "^7.2.3", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/zip-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "packages/config": { "name": "@tiku-saas/config", "version": "0.1.0" diff --git a/package.json b/package.json index 08425839..459cac76 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "test:worker:crm": "npm run db:smoke-seed && npm run build:worker && node scripts/crm-worker-integration-test.js", "test:worker:commerce": "npm run db:smoke-seed && npm run build:worker && node scripts/commerce-worker-integration-test.js", "test:worker:assets": "npm run db:smoke-seed && npm run build:worker && node scripts/asset-worker-integration-test.js", + "test:worker:imports": "npm run db:smoke-seed && npm run build:worker && node scripts/import-worker-integration-test.js", "test:api:remote": "node scripts/api-integration-test.js", "pb:schema:summary": "npm --workspace @tiku-saas/import-pocketbase run schema:summary", "pb:schema:risk": "npm --workspace @tiku-saas/import-pocketbase run schema:risk", diff --git a/scripts/api-integration-test.js b/scripts/api-integration-test.js index d15d6ada..fbb361d1 100644 --- a/scripts/api-integration-test.js +++ b/scripts/api-integration-test.js @@ -4,6 +4,7 @@ import { spawn } from 'node:child_process'; import http from 'node:http'; import net from 'node:net'; import { SignJWT } from 'jose'; +import ExcelJS from 'exceljs'; const DEFAULT_DATABASE_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; const MAIN_TENANT_ID = process.env.TENANT_ID || '00000000-0000-0000-0000-000000000001'; @@ -2794,6 +2795,102 @@ async function testTenantContentAssetsAndImports() { 'catalog should expose imported question through the new collection binding', ); + const questionCsv = [ + ['legacyId', '题型', '题干', '选项A', '选项B', '选项C', '选项D', '答案', '解析', '难度', '标签'].join(','), + ['integration-import-csv-choice-001', 'choice', 'CSV导入题：多租户数据隔离应主要依赖什么？', '前端隐藏菜单', '后端权限和RLS', '浏览器缓存', '静态页面', 'B', '最终权限以后端和RLS为准。', '2', 'csv|import'].join(','), + ].join('\n'); + const csvQuestionPreview = await request('/api/tenant-content/imports/preview/questions', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { + sourceFormat: 'csv', + sourceName: 'questions.csv', + csvText: questionCsv, + subjectId: ids.subject, + categoryId: ids.category, + regionId: ids.region, + entryId: ids.contentEntry, + contentNodeId: ids.contentNodeSchoolTarget, + collectionId: ids.questionCollection, + }, + }); + assert.equal(csvQuestionPreview.job?.errorCount, 0, 'CSV question preview should have no errors'); + assert.equal(csvQuestionPreview.job?.validCount, 1, 'CSV question preview should normalize one row'); + assert.equal(csvQuestionPreview.items?.[0]?.normalized?.correctOptionIndices?.[0], 1, 'CSV answer B should map to option index 1'); + + const csvQuestionImport = await request('/api/tenant-content/imports/questions', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { previewJobId: csvQuestionPreview.job.id }, + }); + assert.equal(csvQuestionImport.item?.status, 'completed', 'CSV question import should complete'); + + const csvQuestionJobs = await request('/api/tenant-content/imports', { + userId: TENANT_ADMIN_USER_ID, + query: { importType: 'questions', limit: 20 }, + }); + assert.ok( + csvQuestionJobs.items?.some(item => item.id === csvQuestionPreview.job.id && item.sourceFormat === 'csv'), + 'CSV question import job should retain sourceFormat', + ); + + const importedCsvQuestions = await request('/api/catalog/questions', { + query: { collectionId: ids.questionCollection, limit: 150 }, + }); + assert.ok( + importedCsvQuestions.items?.some(item => item.content === 'CSV导入题：多租户数据隔离应主要依赖什么？'), + 'catalog should expose imported CSV question', + ); + + const asyncQuestionPreview = await request('/api/tenant-content/imports/preview/questions', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { + sourceName: 'async-question-import.json', + subjectId: ids.subject, + categoryId: ids.category, + regionId: ids.region, + entryId: ids.contentEntry, + contentNodeId: ids.contentNodeSchoolTarget, + collectionId: ids.questionCollection, + items: [ + { + legacyId: 'integration-import-async-choice-001', + type: 'choice', + content: '异步导入排队题：大批量导入应该交给谁执行？', + options: ['前端循环写表', '导入 worker', '用户刷新页面', '浏览器缓存'], + correctOptionIndices: [1], + explanation: '大批量导入由后端 worker 消费，避免请求超时和规则漂移。', + difficulty: 2, + tags: ['async-import'], + }, + ], + }, + }); + assert.equal(asyncQuestionPreview.job?.errorCount, 0, 'async preview should have no errors'); + const asyncQuestionQueued = await request('/api/tenant-content/imports/questions', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { previewJobId: asyncQuestionPreview.job.id, executionMode: 'async' }, + }); + assert.equal(asyncQuestionQueued.item?.status, 'pending', 'async import should queue the job'); + assert.equal(asyncQuestionQueued.item?.executionMode, 'async', 'async import response should expose execution mode'); + + const queuedAgain = await request('/api/tenant-content/imports/questions', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { previewJobId: asyncQuestionPreview.job.id, executionMode: 'async' }, + }); + assert.equal(queuedAgain.item?.idempotent, true, 'queueing the same async job should be idempotent'); + + const queuedSyncExecution = await request('/api/tenant-content/imports/questions', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { previewJobId: asyncQuestionPreview.job.id }, + expectStatus: 409, + }); + assert.equal(queuedSyncExecution.code, 'IMPORT_JOB_QUEUED', 'queued import job should not be executed synchronously'); + const vocabEntry = await request('/api/tenant-content/content-entries', { userId: TENANT_ADMIN_USER_ID, method: 'PUT', @@ -2913,6 +3010,42 @@ async function testTenantContentAssetsAndImports() { }); assert.ok(vocabularyWords.items?.some(item => item.word === 'abandon' && item.contentNodeId), 'catalog should expose imported vocabulary word with node binding'); + const vocabularyCsv = [ + ['unitLegacyId', 'unitName', 'wordLegacyId', 'word', 'phonetic', 'meaning', 'example', 'difficulty', 'tags'].join(','), + ['integration-vocab-csv-unit-001', 'CSV 高频词单元', 'integration-vocab-csv-word-scale', 'scale', '/skeɪl/', 'n. 规模；等级', 'SaaS platforms must scale safely.', '2', 'csv|高频'].join(','), + ].join('\n'); + const vocabularyCsvPreview = await request('/api/tenant-content/imports/preview/vocabulary', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { + sourceFormat: 'csv', + sourceName: 'vocabulary.csv', + csvText: vocabularyCsv, + regionId: ids.region, + entryId: vocabEntry.item.id, + contentNodeId: vocabRoot.item.id, + }, + }); + assert.equal(vocabularyCsvPreview.job?.errorCount, 0, 'CSV vocabulary preview should have no errors'); + assert.equal(vocabularyCsvPreview.items?.[0]?.normalized?.words?.[0]?.word, 'scale', 'CSV vocabulary should group words under units'); + + const vocabularyCsvImport = await request('/api/tenant-content/imports/vocabulary', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { previewJobId: vocabularyCsvPreview.job.id }, + }); + assert.equal(vocabularyCsvImport.item?.status, 'completed', 'CSV vocabulary import should complete'); + + const vocabularyUnitsAfterCsv = await request('/api/catalog/vocabulary-units', { + query: { regionId: ids.region }, + }); + const csvVocabularyUnit = vocabularyUnitsAfterCsv.items?.find(item => item.legacyId === 'integration-vocab-csv-unit-001'); + assert.ok(csvVocabularyUnit, 'catalog should expose CSV imported vocabulary unit'); + const csvVocabularyWords = await request('/api/catalog/vocabulary-words', { + query: { unitId: csvVocabularyUnit.id }, + }); + assert.ok(csvVocabularyWords.items?.some(item => item.word === 'scale'), 'catalog should expose CSV imported vocabulary word'); + const handbookEntry = await request('/api/tenant-content/content-entries', { userId: TENANT_ADMIN_USER_ID, method: 'PUT', @@ -3142,6 +3275,52 @@ async function testTenantContentAssetsAndImports() { 'mixed scoreline import should create record after school and major', ); + const scorelineWorkbook = new ExcelJS.Workbook(); + scorelineWorkbook.addWorksheet('fields').addRows([ + ['legacyId', 'fieldKey', 'fieldName', 'fieldType', 'isTrend'], + ['integration-scoreline-xlsx-field-min', 'xlsxMinScore', 'Excel最低分', 'number', '是'], + ]); + scorelineWorkbook.addWorksheet('schools').addRows([ + ['legacyId', 'schoolName', 'isHot'], + ['integration-scoreline-xlsx-school', 'Excel导入学院', 'true'], + ]); + scorelineWorkbook.addWorksheet('majors').addRows([ + ['legacyId', 'schoolLegacyId', 'majorName'], + ['integration-scoreline-xlsx-major', 'integration-scoreline-xlsx-school', 'Excel专业'], + ]); + scorelineWorkbook.addWorksheet('records').addRows([ + ['legacyId', 'schoolLegacyId', 'majorLegacyId', 'year', 'xlsxMinScore'], + ['integration-scoreline-xlsx-record-2027', 'integration-scoreline-xlsx-school', 'integration-scoreline-xlsx-major', 2027, 233], + ]); + const scorelineExcelBuffer = await scorelineWorkbook.xlsx.writeBuffer(); + const scorelineExcelPreview = await request('/api/tenant-content/imports/preview/scoreline', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { + sourceFormat: 'excel', + sourceName: 'scoreline.xlsx', + fileBase64: Buffer.from(scorelineExcelBuffer).toString('base64'), + regionId: ids.region, + }, + }); + assert.equal(scorelineExcelPreview.job?.errorCount, 0, 'Excel scoreline preview should have no errors'); + assert.equal(scorelineExcelPreview.job?.validCount, 4, 'Excel scoreline preview should parse multi-sheet workbook'); + + const scorelineExcelImport = await request('/api/tenant-content/imports/scoreline', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { previewJobId: scorelineExcelPreview.job.id }, + }); + assert.equal(scorelineExcelImport.item?.status, 'completed', 'Excel scoreline import should complete'); + + const excelScorelineRecords = await request('/api/scoreline/records', { + query: { regionId: ids.region, year: 2027, pageSize: 50 }, + }); + assert.ok( + excelScorelineRecords.items?.some(item => item.schoolName === 'Excel导入学院' && item.fieldValues?.xlsxMinScore === 233), + 'public scoreline records should expose imported Excel record', + ); + const invalidVideoPreview = await request('/api/tenant-content/imports/preview/videos', { userId: TENANT_ADMIN_USER_ID, method: 'POST', diff --git a/scripts/import-worker-integration-test.js b/scripts/import-worker-integration-test.js new file mode 100644 index 00000000..7eff4985 --- /dev/null +++ b/scripts/import-worker-integration-test.js @@ -0,0 +1,299 @@ +import assert from 'node:assert/strict'; +import pg from 'pg'; +import { spawn } from 'node:child_process'; + +const databaseUrl = process.env.DATABASE_URL || 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; +const tenantId = '00000000-0000-0000-0000-000000000001'; +const adminUserId = '00000000-0000-0000-0000-000000000102'; +const ids = { + region: '00000000-0000-0000-0000-000000000301', + subject: '00000000-0000-0000-0000-000000000501', + category: '00000000-0000-0000-0000-000000000601', + contentEntry: '00000000-0000-0000-0000-000000000611', + contentNodeSchoolTarget: '00000000-0000-0000-0000-000000000614', + questionCollection: '00000000-0000-0000-0000-000000000615', +}; + +function runWorkerOnce() { + const child = spawn(process.execPath, ['apps/worker/dist/apps/worker/src/index.js', '--once', '--job', 'imports'], { + cwd: process.cwd(), + env: { + ...process.env, + DATABASE_URL: databaseUrl, + WORKER_IMPORT_BATCH_SIZE: '5', + WORKER_IMPORT_ID: 'imports-integration-test', + }, + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + let output = ''; + child.stdout.on('data', chunk => { + output += chunk.toString(); + }); + child.stderr.on('data', chunk => { + output += chunk.toString(); + }); + return new Promise((resolve, reject) => { + child.on('error', reject); + child.on('exit', code => { + try { + assert.equal(code, 0, `worker should exit 0\n${output}`); + assert.match(output, /imports batch processed=\d+/, 'worker output should include imports summary'); + resolve(output); + } catch (error) { + reject(error); + } + }); + }); +} + +async function cleanup(pool) { + await pool.query( + ` + delete from public.question_collection_items + where tenant_id = $1 + and question_id in ( + select id from public.questions + where tenant_id = $1 + and ( + legacy_id like 'worker-import-question-%' + or legacy_id like 'integration-import-async-choice-%' + ) + ) + `, + [tenantId], + ); + await pool.query( + ` + delete from public.question_versions + where tenant_id = $1 + and question_id in ( + select id from public.questions + where tenant_id = $1 + and ( + legacy_id like 'worker-import-question-%' + or legacy_id like 'integration-import-async-choice-%' + ) + ) + `, + [tenantId], + ); + await pool.query( + ` + delete from public.questions + where tenant_id = $1 + and ( + legacy_id like 'worker-import-question-%' + or legacy_id like 'integration-import-async-choice-%' + ) + `, + [tenantId], + ); + await pool.query( + ` + delete from public.audit_logs + where tenant_id = $1 + and target_type = 'content_import_job' + and details::text like '%worker-import%' + `, + [tenantId], + ); + await pool.query( + ` + delete from public.content_import_jobs + where tenant_id = $1 + and ( + source_name like 'worker-import-%' + or source_name = 'async-question-import.json' + ) + `, + [tenantId], + ); +} + +async function createQueuedQuestionImport(pool) { + const preview = await pool.query( + ` + insert into public.content_import_jobs ( + tenant_id, created_by, import_type, source_format, status, + source_name, source_hash, target_region_id, target_subject_id, + target_category_id, target_content_node_id, target_collection_id, + dry_run, total_count, valid_count, error_count, warning_count, + summary, raw_payload, normalized_payload, execution_mode, queued_at, + next_attempt_at, parser_metadata + ) + values ( + $1, $2, 'questions', 'json', 'pending', + 'worker-import-questions.json', 'worker-import-source-hash', + $3::uuid, $4::uuid, $5::uuid, $6::uuid, $7::uuid, + false, 1, 1, 0, 0, + $8::jsonb, $9::jsonb, $10::jsonb, 'async', now(), now(), '{}'::jsonb + ) + returning id + `, + [ + tenantId, + adminUserId, + ids.region, + ids.subject, + ids.category, + ids.contentNodeSchoolTarget, + ids.questionCollection, + JSON.stringify({ + target: { + regionId: ids.region, + subjectId: ids.subject, + categoryId: ids.category, + contentNodeId: ids.contentNodeSchoolTarget, + collectionId: ids.questionCollection, + }, + importOptions: { allowPartial: false }, + source: 'worker-import-integration', + }), + JSON.stringify([ + { + legacyId: 'worker-import-question-001', + type: 'choice', + content: '异步导入题：worker 应该复用哪套导入规则？', + options: ['自己重写', '复用后端导入 executor', '前端直写数据库', '跳过校验'], + correctOptionIndices: [1], + explanation: 'worker 和 API 必须复用同一套后端导入规则。', + difficulty: 2, + tags: ['worker-import'], + }, + ]), + JSON.stringify([ + { + legacyId: 'worker-import-question-001', + type: 'choice', + typeLabel: null, + content: '异步导入题：worker 应该复用哪套导入规则？', + options: ['自己重写', '复用后端导入 executor', '前端直写数据库', '跳过校验'], + correctOptionIndex: 1, + correctOptionIndices: [1], + answerText: null, + explanation: 'worker 和 API 必须复用同一套后端导入规则。', + difficulty: 2, + tags: ['worker-import'], + mediaUrl: null, + subQuestions: [], + codeLang: null, + codeTemplate: null, + examMarkers: {}, + sourceHash: 'worker-import-question-hash-001', + }, + ]), + ], + ); + const jobId = preview.rows[0].id; + await pool.query( + ` + insert into public.content_import_items ( + tenant_id, job_id, row_no, external_id, status, target_type, + source_payload, normalized_payload, content_hash, issues_count + ) + values ($1, $2, 1, 'worker-import-question-001', 'valid', 'question', $3::jsonb, $4::jsonb, 'worker-import-question-hash-001', 0) + `, + [ + tenantId, + jobId, + JSON.stringify({ + legacyId: 'worker-import-question-001', + content: '异步导入题：worker 应该复用哪套导入规则？', + }), + JSON.stringify({ + legacyId: 'worker-import-question-001', + type: 'choice', + typeLabel: null, + content: '异步导入题：worker 应该复用哪套导入规则？', + options: ['自己重写', '复用后端导入 executor', '前端直写数据库', '跳过校验'], + correctOptionIndex: 1, + correctOptionIndices: [1], + answerText: null, + explanation: 'worker 和 API 必须复用同一套后端导入规则。', + difficulty: 2, + tags: ['worker-import'], + mediaUrl: null, + subQuestions: [], + codeLang: null, + codeTemplate: null, + examMarkers: {}, + sourceHash: 'worker-import-question-hash-001', + }), + ], + ); + return jobId; +} + +async function main() { + const pool = new pg.Pool({ connectionString: databaseUrl }); + try { + await cleanup(pool); + const jobId = await createQueuedQuestionImport(pool); + + const output = await runWorkerOnce(); + assert.match(output, /completed=1/, 'worker should complete exactly the queued import job after cleanup'); + + const job = await pool.query( + ` + select status, execution_mode, inserted_count, updated_count, skipped_count, + locked_at, locked_by, attempt_count, error_message + from public.content_import_jobs + where tenant_id = $1 and id = $2 + `, + [tenantId, jobId], + ); + assert.equal(job.rows[0]?.status, 'completed', 'queued import job should be completed'); + assert.equal(job.rows[0]?.execution_mode, 'async', 'job should keep async execution mode'); + assert.equal(Number(job.rows[0]?.inserted_count), 1, 'worker should insert one question'); + assert.equal(job.rows[0]?.locked_at, null, 'completed job should release lock'); + assert.equal(job.rows[0]?.locked_by, null, 'completed job should clear lock owner'); + assert.equal(Number(job.rows[0]?.attempt_count), 1, 'worker should record one attempt'); + assert.equal(job.rows[0]?.error_message, null, 'completed job should not retain error message'); + + const question = await pool.query( + ` + select q.id, v.content + from public.questions q + join public.question_versions v on v.id = q.current_version_id + where q.tenant_id = $1 and q.legacy_id = 'worker-import-question-001' + limit 1 + `, + [tenantId], + ); + assert.equal(question.rows[0]?.content, '异步导入题：worker 应该复用哪套导入规则？', 'worker should import question content'); + + const collectionItem = await pool.query( + ` + select 1 + from public.question_collection_items + where tenant_id = $1 and collection_id = $2 and question_id = $3 + limit 1 + `, + [tenantId, ids.questionCollection, question.rows[0]?.id], + ); + assert.equal(collectionItem.rowCount, 1, 'worker should bind imported question to collection'); + + const audit = await pool.query( + ` + select action + from public.audit_logs + where tenant_id = $1 and target_type = 'content_import_job' and target_id = $2 + order by created_at desc + limit 1 + `, + [tenantId, jobId], + ); + assert.equal(audit.rows[0]?.action, 'content.import.questions.completed', 'worker import should write completion audit'); + + console.log('Import worker integration test complete.'); + } finally { + await cleanup(pool).catch(() => {}); + await pool.end(); + } +} + +main().catch(error => { + console.error(error); + process.exit(1); +}); diff --git a/supabase/migrations/202606290014_spreadsheet_import_async_metadata.sql b/supabase/migrations/202606290014_spreadsheet_import_async_metadata.sql new file mode 100644 index 00000000..46d7a48e --- /dev/null +++ b/supabase/migrations/202606290014_spreadsheet_import_async_metadata.sql @@ -0,0 +1,28 @@ +alter table public.content_import_jobs + add column if not exists execution_mode text not null default 'sync', + add column if not exists queued_at timestamptz, + add column if not exists locked_at timestamptz, + add column if not exists locked_by text, + add column if not exists attempt_count integer not null default 0, + add column if not exists max_attempts integer not null default 3, + add column if not exists next_attempt_at timestamptz, + add column if not exists parser_metadata jsonb not null default '{}'::jsonb; + +do $$ +begin + if not exists (select 1 from pg_constraint where conname = 'content_import_jobs_execution_mode_check') then + alter table public.content_import_jobs + add constraint content_import_jobs_execution_mode_check + check (execution_mode in ('sync', 'async')); + end if; + + if not exists (select 1 from pg_constraint where conname = 'content_import_jobs_attempts_check') then + alter table public.content_import_jobs + add constraint content_import_jobs_attempts_check + check (attempt_count >= 0 and max_attempts > 0 and attempt_count <= max_attempts); + end if; +end $$; + +create index if not exists idx_content_import_jobs_async_queue + on public.content_import_jobs(tenant_id, status, next_attempt_at, created_at) + where execution_mode = 'async' and status = 'pending';