diff --git a/.env.example b/.env.example index ce10296c..087fd4c0 100644 --- a/.env.example +++ b/.env.example @@ -68,6 +68,14 @@ WORKER_PUBLIC_BANK_SYNC_COPY_LIMIT=1000 WORKER_PUBLIC_BANK_SYNC_ID=public-banks-1 WORKER_PUBLIC_BANK_SYNC_CLAIM_STALE_SECONDS=900 +# Worker 配置:题库 PDF/Word 导出。生产建议使用真实对象存储 provider;local_dev 只用于本地开发。 +WORKER_EXPORT_BATCH_SIZE=5 +WORKER_EXPORT_ID=exports-1 +WORKER_EXPORT_BACKOFF_SECONDS=30,120,600,1800 +EXPORT_LOCAL_STORAGE_ROOT=.local-storage +# Windows 本地可填 C:\Windows\Fonts\NotoSansSC-VF.ttf;Linux 生产建议安装 Noto Sans CJK 并配置路径。 +EXPORT_PDF_FONT_PATH= + # 对象存储配置:API 和 assets worker 共用 STORAGE_DEFAULT_PROVIDER=local_dev STORAGE_DEFAULT_BUCKET=tenant-assets diff --git a/.gitignore b/.gitignore index e5affc5e..cd418f08 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ scripts/satellite/sync-config.json # ✅ 临时/对比文件 stash_comparison.txt stash_diff.txt +.local-storage/ # ✅ 备份文件 *.backup diff --git a/README.md b/README.md index 6c19fffb..8ce62a84 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,9 @@ - 学生端能力:题库入口、分类树、题目集合、顺序/随机/模考 session 组卷快照、答题、错题本、收藏夹、背单词进度、个人中心、勋章、考试倒计时、签到积分、题目反馈、排行榜、分数线、题目视频、订单详情/状态轮询、优惠券领取/抵扣、权益、激活码预检查/兑换、资料下载。 - 平台后台能力:租户管理、SaaS 套餐、订阅、账单、服务费收款、用量记录、公共题库授权。 - 公共题库商业化能力:租户可采纳平台授权题库为本租户副本,并可手动或由 worker 自动同步平台新增/更新题目;同步会保护租户自改题目,返回冲突而不覆盖,后台可查询冲突明细。 -- 题库导出基础能力:租户内容编辑可按题目集合、内容入口或分类节点导出 JSON、`paper_json` 和打印 payload,后端强制租户隔离、答案/解析开关、复合题子题脱敏、导出 job 和审计。 +- 题库导出能力:租户内容编辑可按题目集合、内容入口或分类节点导出 JSON、`paper_json`、打印 payload、PDF 和 Word,后端强制租户隔离、答案/解析开关、复合题子题脱敏、导出 job 和审计;PDF/Word 由 exports worker 生成水印文件并发布到 `content_assets`。 - 销售/代理/CRM 增长链路:邀请码、扫码/分享事件、首绑客资保护、销售统计、团队关系、CRM 配置和队列。 -- `apps/worker` 后台任务进程:CRM webhook 队列消费、generic/钉钉/飞书/企微机器人发送、签名、失败重试和日志;commerce worker 可补偿查询微信/支付宝支付和退款状态;assets worker 可复检托管资源元数据并自动下架异常资源;public-banks worker 可自动同步公共题库采纳副本。 +- `apps/worker` 后台任务进程:CRM webhook 队列消费、generic/钉钉/飞书/企微机器人发送、签名、失败重试和日志;commerce worker 可补偿查询微信/支付宝支付和退款状态;assets worker 可复检托管资源元数据并自动下架异常资源;imports worker 可执行大批量导入;public-banks worker 可自动同步公共题库采纳副本;exports worker 可渲染 PDF/Word 导出文件。 - 销售/代理分佣结算基础闭环:租户默认比例、成员比例、激活码批次比例、订单/激活码归因、结算单生成、审核、线下打款状态和权限隔离。 - 订单售后基础闭环:退款请求、审核、处理状态流、微信/支付宝发起退款、微信/支付宝退款查询确认、微信/支付宝退款通知 webhook、退款金额累计、部分/全额退款订单状态、全额退款权益撤销、退款事件和审计日志。 - PocketBase schema/数据导入器雏形和导入后校验脚本。 @@ -31,7 +31,7 @@ - 阿里云/腾讯云短信、微信小程序登录、微信网页登录、QQ 登录、手机号绑定/换绑、微信支付、支付宝主链路、微信/支付宝发起退款/查询确认/退款通知、支付/退款补偿 worker 已完成本地适配;完整资金流水对账和真实生产账号联调还没接完。 - OSS/COS/Supabase Storage 上传下载签名 provider 已接入;上传后校验、PDF/图片预览和资源复检 worker 已完成,CDN 防盗链、杀毒扫描和视频动态水印还没完成。 - Excel/CSV 导入解析已完成并复用 `content_import_jobs/items/issues` 管线;大批量异步导入 worker 基础已接入,支持 queued job 消费、重试和审计;导入后复检、模板下载和字段映射 API 已完成,前端 UI 待接。 -- 题库导出目前完成服务端结构化 payload;PDF/Word 二进制生成、导出水印、发布到资料下载和导出 worker 还没完成。 +- 题库导出已完成服务端结构化 payload 和 PDF/Word 二进制 worker;后续还要补更精细试卷模板、每日一练素材导出和导出操作台体验。 - 勋章管理/手动发放已可联调;自动发放规则、积分活动联动、分佣真实打款、结算导出、发票/凭证、CRM 轮询/定向分配、富卡片模板、失败告警、销售转化看板、公共题库版本通知和冲突处理操作台还没完成。 - `apps/taro` 已建立 Taro 4 React 跨端前端地基,包含 H5 学生端、租户后台、平台后台三套构建入口、租户解析、统一 API client 和 Supabase Auth client 初始化;学生端第一批页面已接入登录、首页、题库、练习、背单词、知识手册、分数线、资料和个人中心;租户后台第一批页面已接入工作台、数据看板、学生/班级、题库内容、营销中心和租户设置;平台后台已接入工作台、租户管理、账务中心、公共题库授权,以及创建租户、状态变更、订阅、账单、收款、用量和题库授权第一版写操作。 - 根目录已清理为新 Supabase SaaS monorepo 编排层;旧 PocketBase/React 项目和旧构建产物仅保留在 `参考/` 目录作为迁移参考,不进入 Git 提交。 @@ -60,7 +60,7 @@ ```text apps/api/ Node.js 业务 API apps/taro/ Taro 4 React 跨端前端,H5 三入口,后续扩展小程序 -apps/worker/ 后台异步任务:CRM webhook、支付/退款补偿、资源复检、导入执行、公共题库同步等 +apps/worker/ 后台异步任务:CRM webhook、支付/退款补偿、资源复检、导入执行、公共题库同步、题库导出渲染等 packages/config/ 共享配置 packages/db/ PostgreSQL 连接池和查询封装 packages/domain/ 领域常量和共享类型 @@ -181,6 +181,12 @@ npm --workspace @tiku-saas/worker run imports:once npm --workspace @tiku-saas/worker run public-banks:once ``` +单次运行题库 PDF/Word 导出 worker: + +```bash +npm --workspace @tiku-saas/worker run exports:once +``` + 默认本地数据库: ```text @@ -226,6 +232,7 @@ npm run test:api npm run test:worker:crm npm run test:worker:commerce npm run test:worker:assets +npm run test:worker:exports npm run test:worker:imports npm run test:worker:public-banks ``` @@ -300,7 +307,7 @@ API 身份上下文: - 资料、PDF、视频等资源必须先进入 `content_assets` 台账,再由 API 校验权限并下发签名 URL;生产环境应定时运行 assets worker 复检对象元数据,异常资源会被标记 failed 并退回 draft。 - 题库入口和分类使用 `content_entries/content_nodes`;题目列表和练习规则使用 `question_collections/practice_blueprints`,前端不要再把旧树字段当成唯一业务结构。 - 批量导入必须先写 `content_import_jobs/items/issues`,保留原始 payload、规范化 payload、逐行问题和审计记录。题目、单词、知识手册、分数线和视频 JSON/CSV/Excel 导入已走这套后台校验管线;大批量任务可提交 `executionMode=async`,由 imports worker 消费,前端只轮询 job 状态和展示 issues。 -- 题库导出必须由后端按权限生成,不允许前端直接读取数据库拼导出文件;不开启答案/解析时,顶层题目和复合题子题都必须脱敏。 +- 题库导出必须由后端按权限生成,不允许前端直接读取数据库拼导出文件;不开启答案/解析时,顶层题目和复合题子题都必须脱敏;PDF/Word 只通过 exports worker 写入 `content_assets` 后再签名下载/预览。 - 支付 webhook 必须先设计幂等键和验签流程,再进入生产使用;生产环境还应定时运行 commerce worker 兜底供应商漏通知和处理中退款。 ## 最近一次验证 @@ -316,6 +323,7 @@ npm run test:worker:imports npm run test:worker:crm npm run test:worker:commerce npm run test:worker:assets +npm run test:worker:exports npm run test:worker:public-banks npm run check:taro npm run build:taro:h5:student @@ -335,5 +343,5 @@ git diff --check 1. 真实云端 Auth/JWKS 回归、RLS 深测和生产环境配置验收。 2. 继续补 Taro 前端:学生端视频/反馈/模考报告/订单收银台,租户后台写入表单/导入操作台/公共题库同步/角色模板 UI,平台后台租户详情/审计/自动计费增强,小程序兼容验证。 3. 对象存储 CDN 防盗链、杀毒扫描、视频动态水印和生命周期策略。 -4. 题库导出 PDF/Word worker、真实数据 dry-run、导入字段映射 UI 和复检结果操作台。 +4. 题库导出模板精排、每日一练运营工具、真实数据 dry-run、导入字段映射 UI 和复检结果操作台。 5. 真实 OAuth/短信/支付生产账号联调、完整资金流水对账、公共题库版本通知/冲突处理操作台、积分活动深化,以及排行榜防刷/预聚合。 diff --git a/apps/api/src/features/tenant-content/exports.ts b/apps/api/src/features/tenant-content/exports.ts index e1bd8893..afa4c3ed 100644 --- a/apps/api/src/features/tenant-content/exports.ts +++ b/apps/api/src/features/tenant-content/exports.ts @@ -6,7 +6,9 @@ import { query, transaction } from '../../core/db.js'; import { requireTenantContentEditor, type TenantContentAuth } from './auth.js'; import { boolValue, jsonObjectValue, nullableString } from './utils.js'; -const EXPORT_FORMATS = ['json', 'paper_json', 'print_payload']; +const INLINE_EXPORT_FORMATS = ['json', 'paper_json', 'print_payload']; +const BINARY_EXPORT_FORMATS = ['pdf', 'docx']; +const EXPORT_FORMATS = [...INLINE_EXPORT_FORMATS, ...BINARY_EXPORT_FORMATS]; const EXPORT_TYPES = ['questions', 'paper', 'daily_practice']; const SCOPE_TYPES = ['collection', 'entry', 'content_node']; const SUB_QUESTION_ANSWER_KEYS = new Set([ @@ -77,6 +79,44 @@ interface ExportQuestionRow { sourceHash: string | null; } +export interface BuiltExportPayload { + _tikuExport: string; + jobId: string; + tenantId: string; + exportedAt: string; + exportType: string; + format: string; + title: string; + scope: ExportScope; + options: Record; + summary: { + questionCount: number; + sectionCount: number; + totalScore: number; + durationMinutes: number | null; + }; + sections: Array<{ + key: string; + title: string; + questions: ReturnType[]; + totalScore: number; + questionCount: number; + }>; + questions: ReturnType[]; + files: Array<{ + filename: string; + mimeType: string; + encoding: 'base64'; + contentBase64: string; + }>; + renderHints: { + pdfLayout: string; + pageSize: string; + answerPlacement: string; + frontendRenderer: string; + }; +} + function choose(value: unknown, allowed: string[], fallback: string, code: string) { const candidate = nullableString(value) || fallback; if (!allowed.includes(candidate)) { @@ -99,6 +139,10 @@ function exportLimit(value: unknown) { return Math.max(1, Math.min(Math.trunc(parsed), 5000)); } +function isBinaryExportFormat(format: string) { + return BINARY_EXPORT_FORMATS.includes(format); +} + function jsonArray(value: unknown) { return Array.isArray(value) ? value : []; } @@ -199,7 +243,7 @@ function buildExportPayload(input: { questions: ReturnType[]; options: Record; createdAt: string; -}) { +}): BuiltExportPayload { const sections = groupBySection(input.questions); const title = typeof input.options.title === 'string' && input.options.title.trim() ? input.options.title.trim() @@ -424,6 +468,138 @@ async function loadQuestions(client: pg.PoolClient, tenantId: string, scopeType: return result.rows; } +async function createExportJob(input: { + client: pg.PoolClient; + auth: TenantContentAuth; + scope: ExportScope; + scopeType: ExportScope['scopeType']; + scopeId: string; + exportType: string; + format: string; + includeAnswers: boolean; + includeExplanations: boolean; + includeVideoRefs: boolean; + questionCount: number; + options: Record; + outputMetadata: Record; + status: 'pending' | 'completed'; + finishedAt?: boolean; +}) { + const job = await input.client.query<{ + id: string; + createdAt: string; + }>( + ` + insert into public.content_export_jobs ( + tenant_id, created_by, export_type, format, scope_type, scope_id, + status, include_answers, include_explanations, include_video_refs, + question_count, summary, options, output_metadata, finished_at + ) + values ( + $1, $2, $3, $4, $5, $6, + $7, $8, $9, $10, + $11, $12::jsonb, $13::jsonb, $14::jsonb, + case when $15::boolean then now() else null end + ) + returning id, created_at as "createdAt" + `, + [ + input.auth.tenantId, + input.auth.userId, + input.exportType, + input.format, + input.scopeType, + input.scopeId, + input.status, + input.includeAnswers, + input.includeExplanations, + input.includeVideoRefs, + input.questionCount, + JSON.stringify({ + questionCount: input.questionCount, + scopeName: input.scope.collectionName || input.scope.contentNodeName || input.scope.entryName, + format: input.format, + exportType: input.exportType, + }), + jsonObjectValue(input.options), + JSON.stringify(input.outputMetadata), + input.finishedAt === true, + ], + ); + return job.rows[0]; +} + +export async function buildExportPayloadForJob(input: { + client: pg.PoolClient; + tenantId: string; + jobId: string; + createdAt?: string; +}): Promise { + const job = await input.client.query<{ + id: string; + tenantId: string; + createdBy: string | null; + exportType: string; + format: string; + scopeType: ExportScope['scopeType']; + scopeId: string; + includeAnswers: boolean; + includeExplanations: boolean; + includeVideoRefs: boolean; + options: Record; + outputMetadata: Record; + createdAt: string; + }>( + ` + select id, tenant_id as "tenantId", created_by as "createdBy", + export_type as "exportType", format, + scope_type as "scopeType", scope_id as "scopeId", + include_answers as "includeAnswers", + include_explanations as "includeExplanations", + include_video_refs as "includeVideoRefs", + options, output_metadata as "outputMetadata", + created_at as "createdAt" + from public.content_export_jobs + where tenant_id = $1 and id = $2 + limit 1 + `, + [input.tenantId, input.jobId], + ); + const row = job.rows[0]; + if (!row) throw new HttpError(404, 'Export job not found', 'EXPORT_JOB_NOT_FOUND'); + + const scope = await loadScope(input.client, row.tenantId, row.scopeType, row.scopeId); + const maxQuestionLimit = Number(row.outputMetadata?.maxQuestionLimit ?? row.options?.limit ?? 1000); + const limit = exportLimit(maxQuestionLimit); + const rows = await loadQuestions(input.client, row.tenantId, row.scopeType, row.scopeId, limit); + const questions = rows.map(question => formatQuestion( + question, + row.includeAnswers, + row.includeExplanations, + row.includeVideoRefs, + )); + + return buildExportPayload({ + auth: { + tenantId: row.tenantId, + userId: row.createdBy || row.tenantId, + role: 'system_worker', + permissions: { 'content:*': true }, + templatePermissions: {}, + }, + jobId: row.id, + scope, + exportType: row.exportType, + format: row.format, + includeAnswers: row.includeAnswers, + includeExplanations: row.includeExplanations, + includeVideoRefs: row.includeVideoRefs, + questions, + options: row.options || {}, + createdAt: input.createdAt || row.createdAt || new Date().toISOString(), + }); +} + export async function createQuestionExportRoute(ctx: RequestContext) { const auth = await requireTenantContentEditor(ctx); const body = await readJsonBody(ctx); @@ -457,64 +633,47 @@ export async function createQuestionExportRoute(ctx: RequestContext) { createdAt, }; const outputHash = contentHash(prePayload); + const binaryExport = isBinaryExportFormat(format); - const job = await client.query<{ - id: string; - createdAt: string; - }>( - ` - insert into public.content_export_jobs ( - tenant_id, created_by, export_type, format, scope_type, scope_id, - status, include_answers, include_explanations, include_video_refs, - question_count, summary, options, output_metadata, finished_at - ) - values ( - $1, $2, $3, $4, $5, $6, - 'completed', $7, $8, $9, - $10, $11::jsonb, $12::jsonb, $13::jsonb, now() - ) - returning id, created_at as "createdAt" - `, - [ - auth.tenantId, - auth.userId, - exportType, - format, - scopeType, - scopeId, - includeAnswers, - includeExplanations, - includeVideoRefs, - questions.length, - JSON.stringify({ - questionCount: questions.length, - scopeName: scope.collectionName || scope.contentNodeName || scope.entryName, - format, - exportType, - }), - jsonObjectValue(options), - JSON.stringify({ - outputHash, - delivery: 'inline_payload', - maxQuestionLimit: limit, - }), - ], - ); - - const payload = buildExportPayload({ + const job = await createExportJob({ + client, auth, - jobId: job.rows[0].id, scope, + scopeType, + scopeId, exportType, format, includeAnswers, includeExplanations, includeVideoRefs, - questions, - options, - createdAt, + questionCount: questions.length, + options: { ...options, limit }, + outputMetadata: { + outputHash, + delivery: binaryExport ? 'content_asset_pending' : 'inline_payload', + maxQuestionLimit: limit, + renderer: binaryExport ? 'worker:exports' : 'api:inline-json', + }, + status: binaryExport ? 'pending' : 'completed', + finishedAt: !binaryExport, }); + const payload = binaryExport + ? null + : buildExportPayload({ + auth, + jobId: job.id, + scope, + exportType, + format, + includeAnswers, + includeExplanations, + includeVideoRefs, + questions, + options, + createdAt: job.createdAt || createdAt, + }); + await client.query( ` insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details) @@ -523,7 +682,7 @@ export async function createQuestionExportRoute(ctx: RequestContext) { [ auth.tenantId, auth.userId, - job.rows[0].id, + job.id, JSON.stringify({ scopeType, scopeId, @@ -534,15 +693,16 @@ export async function createQuestionExportRoute(ctx: RequestContext) { includeExplanations, includeVideoRefs, outputHash, + delivery: binaryExport ? 'content_asset_pending' : 'inline_payload', }), ], ); return { job: { - id: job.rows[0].id, - status: 'completed', - createdAt: job.rows[0].createdAt, + id: job.id, + status: binaryExport ? 'pending' : 'completed', + createdAt: job.createdAt, questionCount: questions.length, outputHash, }, @@ -580,7 +740,9 @@ export async function questionExportJobsRoute(ctx: RequestContext) { include_explanations as "includeExplanations", include_video_refs as "includeVideoRefs", question_count as "questionCount", summary, options, output_metadata as "outputMetadata", - error_message as "errorMessage", finished_at as "finishedAt", + asset_id as "assetId", error_message as "errorMessage", + started_at as "startedAt", finished_at as "finishedAt", + attempt_count as "attemptCount", max_attempts as "maxAttempts", created_by as "createdBy", created_at as "createdAt", updated_at as "updatedAt" from public.content_export_jobs where ${filters.join(' and ')} diff --git a/apps/worker/package.json b/apps/worker/package.json index 0e6263af..0c677910 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -12,15 +12,19 @@ "commerce:once": "tsx src/index.ts --once --job commerce", "assets:once": "tsx src/index.ts --once --job assets", "imports:once": "tsx src/index.ts --once --job imports", - "public-banks:once": "tsx src/index.ts --once --job public-banks" + "public-banks:once": "tsx src/index.ts --once --job public-banks", + "exports:once": "tsx src/index.ts --once --job exports" }, "dependencies": { "@supabase/storage-js": "^2.108.2", "ali-oss": "^6.23.0", + "docx": "^9.7.1", + "pdfkit": "^0.19.1", "pg": "^8.16.3" }, "devDependencies": { "@types/node": "^24.0.4", + "@types/pdfkit": "^0.17.6", "@types/pg": "^8.15.4", "tsx": "^4.20.3", "typescript": "^5.8.3" diff --git a/apps/worker/src/config.ts b/apps/worker/src/config.ts index 6f801a85..b0f48b58 100644 --- a/apps/worker/src/config.ts +++ b/apps/worker/src/config.ts @@ -24,6 +24,14 @@ export interface WorkerConfig { publicBankSyncCopyLimit: number; publicBankSyncWorkerId: string; publicBankSyncClaimStaleSeconds: number; + exportBatchSize: number; + exportWorkerId: string; + exportBackoffSeconds: number[]; + exportLocalStorageRoot: string; + exportPdfFontPath: string; + storageDefaultProvider: string; + storageDefaultBucket: string; + storagePublicBaseUrl: string; storageMaxUploadBytes: number; storageAllowedMimePrefixes: string[]; storageAllowedMimeTypes: string[]; @@ -69,6 +77,16 @@ export const config: WorkerConfig = { publicBankSyncCopyLimit: envNumber('WORKER_PUBLIC_BANK_SYNC_COPY_LIMIT', 1000), publicBankSyncWorkerId: envString('WORKER_PUBLIC_BANK_SYNC_ID', `public-banks-${process.pid}`), publicBankSyncClaimStaleSeconds: envNumber('WORKER_PUBLIC_BANK_SYNC_CLAIM_STALE_SECONDS', 15 * 60), + exportBatchSize: envNumber('WORKER_EXPORT_BATCH_SIZE', 5), + exportWorkerId: envString('WORKER_EXPORT_ID', `exports-${process.pid}`), + exportBackoffSeconds: envList('WORKER_EXPORT_BACKOFF_SECONDS', '30,120,600,1800') + .map((value: string) => Number(value)) + .filter((value: number) => Number.isFinite(value) && value > 0), + exportLocalStorageRoot: envString('EXPORT_LOCAL_STORAGE_ROOT', '.local-storage'), + exportPdfFontPath: envString('EXPORT_PDF_FONT_PATH', ''), + storageDefaultProvider: envString('STORAGE_DEFAULT_PROVIDER', 'local_dev'), + storageDefaultBucket: envString('STORAGE_DEFAULT_BUCKET', 'tenant-assets'), + storagePublicBaseUrl: envString('STORAGE_PUBLIC_BASE_URL', ''), storageMaxUploadBytes: envNumber('STORAGE_MAX_UPLOAD_BYTES', 1024 * 1024 * 500), storageAllowedMimePrefixes: envList('STORAGE_ALLOWED_MIME_PREFIXES', 'image/,video/,audio/'), storageAllowedMimeTypes: envList( diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 358ac566..3510c91b 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -61,6 +61,17 @@ async function runOnce() { ); return; } + if (job === 'exports') { + const { closeExportExecutorPool, processExportBatch } = await import('./jobs/exports.js'); + extraClosers.add(closeExportExecutorPool); + const result = await processExportBatch(); + console.log( + `[worker] exports batch processed=${result.processed}` + + ` completed=${result.completed} failed=${result.failed}` + + ` retrying=${result.retrying} skipped=${result.skipped}`, + ); + return; + } throw new Error(`Unsupported worker job: ${job}`); } diff --git a/apps/worker/src/jobs/export-renderer.ts b/apps/worker/src/jobs/export-renderer.ts new file mode 100644 index 00000000..3efb4820 --- /dev/null +++ b/apps/worker/src/jobs/export-renderer.ts @@ -0,0 +1,340 @@ +import fs from 'node:fs'; +import PDFDocument from 'pdfkit'; +import { + AlignmentType, + Document, + Footer, + Header, + HeadingLevel, + Packer, + PageNumber, + Paragraph, + TextRun, +} from 'docx'; +import type { BuiltExportPayload } from '../../../api/src/features/tenant-content/exports.js'; +import { config } from '../config.js'; + +interface RenderResult { + body: Buffer; + mimeType: string; + extension: 'pdf' | 'docx'; +} + +type ExportQuestion = BuiltExportPayload['questions'][number]; + +const PDF_FONT_CANDIDATES = [ + config.exportPdfFontPath, + 'C:\\Windows\\Fonts\\NotoSansSC-VF.ttf', + 'C:\\Windows\\Fonts\\msyh.ttc', + 'C:\\Windows\\Fonts\\simhei.ttf', + '/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc', + '/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc', + '/usr/share/fonts/truetype/wqy/wqy-microhei.ttc', +].filter(Boolean); + +function plainText(value: unknown) { + return String(value ?? '') + .replace(/!\[([^\]]*)\]\(([^)]+)\)/g, ' [图片: $1 $2] ') + .replace(/\[([^\]]+)\]\(([^)]+)\)/g, '$1') + .replace(//gi, '\n') + .replace(/<\/(p|div|li|tr|h[1-6])>/gi, '\n') + .replace(/<[^>]+>/g, '') + .replace(/ /g, ' ') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/&/g, '&') + .replace(/\r\n/g, '\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); +} + +function optionText(option: unknown, index: number) { + const prefix = `${String.fromCharCode(65 + index)}. `; + if (option && typeof option === 'object' && !Array.isArray(option)) { + const object = option as Record; + return prefix + plainText(object.text ?? object.label ?? object.content ?? JSON.stringify(object)); + } + return prefix + plainText(option); +} + +function answerText(question: ExportQuestion) { + const answers: string[] = []; + if (typeof question.correctOptionIndex === 'number') { + answers.push(String.fromCharCode(65 + question.correctOptionIndex)); + } + if (Array.isArray(question.correctOptionIndices) && question.correctOptionIndices.length) { + answers.push(question.correctOptionIndices.map(item => String.fromCharCode(65 + Number(item))).join(', ')); + } + if (typeof question.answerText === 'string' && question.answerText.trim()) { + answers.push(plainText(question.answerText)); + } + return Array.from(new Set(answers.filter(Boolean))).join(';'); +} + +function subQuestionLines(question: ExportQuestion, includeAnswers: boolean, includeExplanations: boolean) { + const subQuestions = Array.isArray(question.subQuestions) ? question.subQuestions : []; + const lines: string[] = []; + subQuestions.forEach((item, index) => { + if (!item || typeof item !== 'object' || Array.isArray(item)) return; + const object = item as Record; + lines.push(`(${index + 1}) ${plainText(object.content ?? object.stem ?? object.title ?? '')}`); + const options = Array.isArray(object.options) ? object.options : []; + options.forEach((option, optionIndex) => { + lines.push(` ${optionText(option, optionIndex)}`); + }); + if (includeAnswers) { + const answer = plainText(object.answerText ?? object.answer_text ?? object.answer ?? object.correctAnswer ?? object.referenceAnswer ?? ''); + const optionAnswer = Array.isArray(object.correctOptionIndices) + ? object.correctOptionIndices.map(value => String.fromCharCode(65 + Number(value))).join(', ') + : typeof object.correctOptionIndex === 'number' + ? String.fromCharCode(65 + object.correctOptionIndex) + : ''; + if (answer || optionAnswer) lines.push(` 答案:${answer || optionAnswer}`); + } + if (includeExplanations) { + const explanation = plainText(object.explanation ?? object.analysis ?? ''); + if (explanation) lines.push(` 解析:${explanation}`); + } + }); + return lines; +} + +function boolOption(payload: BuiltExportPayload, key: string) { + return payload.options && typeof payload.options[key] === 'boolean' ? payload.options[key] === true : false; +} + +function watermarkText(payload: BuiltExportPayload) { + const raw = payload.options?.watermarkText; + return typeof raw === 'string' && raw.trim() ? raw.trim().slice(0, 80) : ''; +} + +function findPdfFont() { + return PDF_FONT_CANDIDATES.find(candidate => fs.existsSync(candidate)); +} + +function collectPdf(doc: PDFKit.PDFDocument) { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + doc.on('data', chunk => chunks.push(Buffer.from(chunk))); + doc.on('end', () => resolve(Buffer.concat(chunks))); + doc.on('error', reject); + }); +} + +function renderPdfWatermark(doc: PDFKit.PDFDocument, text: string) { + if (!text) return; + const width = doc.page.width; + const height = doc.page.height; + doc.save(); + doc.rotate(-35, { origin: [width / 2, height / 2] }); + doc.opacity(0.08); + doc.fillColor('#64748b'); + doc.fontSize(36); + doc.text(text, 40, height / 2 - 40, { align: 'center', width: width - 80 }); + doc.opacity(1); + doc.restore(); + doc.fillColor('#111827'); +} + +function writePdfQuestion(doc: PDFKit.PDFDocument, question: ExportQuestion, index: number, payload: BuiltExportPayload) { + const includeAnswers = boolOption(payload, 'includeAnswers'); + const includeExplanations = boolOption(payload, 'includeExplanations'); + const typeLabel = typeof question.typeLabel === 'string' && question.typeLabel ? ` [${question.typeLabel}]` : ''; + const score = typeof question.score === 'number' ? ` (${question.score}分)` : ''; + doc.moveDown(0.4); + doc.fontSize(10).fillColor('#111827').text(`${index + 1}. ${plainText(question.content)}${typeLabel}${score}`, { + lineGap: 3, + }); + const options = Array.isArray(question.options) ? question.options : []; + options.forEach((option, optionIndex) => { + doc.fontSize(9).fillColor('#374151').text(optionText(option, optionIndex), { indent: 16, lineGap: 2 }); + }); + for (const line of subQuestionLines(question, includeAnswers, includeExplanations)) { + doc.fontSize(9).fillColor('#374151').text(line, { indent: line.startsWith('(') ? 16 : 26, lineGap: 2 }); + } + const answer = includeAnswers ? answerText(question) : ''; + if (answer) { + doc.fontSize(9).fillColor('#0f766e').text(`答案:${answer}`, { indent: 16, lineGap: 2 }); + } + if (includeExplanations && typeof question.explanation === 'string' && question.explanation.trim()) { + doc.fontSize(9).fillColor('#475569').text(`解析:${plainText(question.explanation)}`, { indent: 16, lineGap: 2 }); + } +} + +export async function renderExportPdf(payload: BuiltExportPayload): Promise { + const doc = new PDFDocument({ size: 'A4', margin: 48, bufferPages: true, autoFirstPage: true }); + const done = collectPdf(doc); + const fontPath = findPdfFont(); + if (fontPath) { + doc.registerFont('TikuCjk', fontPath); + doc.font('TikuCjk'); + } + + const watermark = watermarkText(payload); + renderPdfWatermark(doc, watermark); + doc.on('pageAdded', () => { + if (fontPath) doc.font('TikuCjk'); + renderPdfWatermark(doc, watermark); + }); + + doc.fontSize(18).fillColor('#0f172a').text(payload.title || '题库导出', { align: 'center' }); + doc.moveDown(0.4); + doc.fontSize(9).fillColor('#64748b').text( + `导出时间:${payload.exportedAt} 题量:${payload.summary.questionCount} 总分:${payload.summary.totalScore}`, + { align: 'center' }, + ); + if (watermark) { + doc.moveDown(0.3); + doc.fontSize(8).fillColor('#94a3b8').text(`水印:${watermark}`, { align: 'center' }); + } + + let questionIndex = 0; + for (const section of payload.sections) { + doc.moveDown(1); + doc.fontSize(13).fillColor('#1e3a8a').text(section.title || section.key, { underline: true }); + for (const question of section.questions) { + writePdfQuestion(doc, question, questionIndex, payload); + questionIndex += 1; + } + } + + const pages = doc.bufferedPageRange(); + for (let i = 0; i < pages.count; i += 1) { + doc.switchToPage(i); + if (fontPath) doc.font('TikuCjk'); + doc.fontSize(8).fillColor('#94a3b8').text( + `${payload.title || '题库导出'} · 第 ${i + 1} / ${pages.count} 页`, + 48, + doc.page.height - 34, + { align: 'center', width: doc.page.width - 96 }, + ); + } + doc.end(); + return { + body: await done, + mimeType: 'application/pdf', + extension: 'pdf', + }; +} + +function docxParagraph(text: string, options: { heading?: (typeof HeadingLevel)[keyof typeof HeadingLevel]; bold?: boolean } = {}) { + return new Paragraph({ + heading: options.heading, + spacing: { after: 140 }, + children: [ + new TextRun({ + text, + bold: options.bold, + font: 'Microsoft YaHei', + size: options.heading ? 28 : 21, + }), + ], + }); +} + +function docxQuestionParagraphs(question: ExportQuestion, index: number, payload: BuiltExportPayload) { + const includeAnswers = boolOption(payload, 'includeAnswers'); + const includeExplanations = boolOption(payload, 'includeExplanations'); + const paragraphs: Paragraph[] = []; + const typeLabel = typeof question.typeLabel === 'string' && question.typeLabel ? ` [${question.typeLabel}]` : ''; + const score = typeof question.score === 'number' ? ` (${question.score}分)` : ''; + paragraphs.push(docxParagraph(`${index + 1}. ${plainText(question.content)}${typeLabel}${score}`, { bold: true })); + const options = Array.isArray(question.options) ? question.options : []; + options.forEach((option, optionIndex) => { + paragraphs.push(docxParagraph(optionText(option, optionIndex))); + }); + for (const line of subQuestionLines(question, includeAnswers, includeExplanations)) { + paragraphs.push(docxParagraph(line)); + } + const answer = includeAnswers ? answerText(question) : ''; + if (answer) paragraphs.push(docxParagraph(`答案:${answer}`)); + if (includeExplanations && typeof question.explanation === 'string' && question.explanation.trim()) { + paragraphs.push(docxParagraph(`解析:${plainText(question.explanation)}`)); + } + return paragraphs; +} + +export async function renderExportDocx(payload: BuiltExportPayload): Promise { + const watermark = watermarkText(payload); + const children: Paragraph[] = [ + new Paragraph({ + heading: HeadingLevel.TITLE, + alignment: AlignmentType.CENTER, + spacing: { after: 180 }, + children: [new TextRun({ text: payload.title || '题库导出', bold: true, font: 'Microsoft YaHei', size: 36 })], + }), + new Paragraph({ + alignment: AlignmentType.CENTER, + spacing: { after: 180 }, + children: [ + new TextRun({ + text: `导出时间:${payload.exportedAt} 题量:${payload.summary.questionCount} 总分:${payload.summary.totalScore}`, + color: '64748B', + font: 'Microsoft YaHei', + size: 19, + }), + ], + }), + ]; + if (watermark) { + children.push(new Paragraph({ + alignment: AlignmentType.CENTER, + spacing: { after: 180 }, + children: [new TextRun({ text: `水印:${watermark}`, color: '94A3B8', font: 'Microsoft YaHei', size: 18 })], + })); + } + let questionIndex = 0; + for (const section of payload.sections) { + children.push(docxParagraph(section.title || section.key, { heading: HeadingLevel.HEADING_2, bold: true })); + for (const question of section.questions) { + children.push(...docxQuestionParagraphs(question, questionIndex, payload)); + questionIndex += 1; + } + } + + const doc = new Document({ + creator: 'Tiku SaaS Export Worker', + title: payload.title || '题库导出', + description: 'Generated by Tiku SaaS export worker', + sections: [ + { + headers: { + default: new Header({ + children: [ + new Paragraph({ + alignment: AlignmentType.CENTER, + children: [ + new TextRun({ + text: watermark || payload.title || '题库导出', + color: 'CBD5E1', + font: 'Microsoft YaHei', + size: 18, + }), + ], + }), + ], + }), + }, + footers: { + default: new Footer({ + children: [ + new Paragraph({ + alignment: AlignmentType.CENTER, + children: [ + new TextRun({ children: ['第 ', PageNumber.CURRENT, ' / ', PageNumber.TOTAL_PAGES, ' 页'], font: 'Microsoft YaHei', size: 18 }), + ], + }), + ], + }), + }, + children, + }, + ], + }); + + return { + body: await Packer.toBuffer(doc), + mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + extension: 'docx', + }; +} diff --git a/apps/worker/src/jobs/exports.ts b/apps/worker/src/jobs/exports.ts new file mode 100644 index 00000000..a3cb56ae --- /dev/null +++ b/apps/worker/src/jobs/exports.ts @@ -0,0 +1,496 @@ +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import type pg from 'pg'; +import { pool } from '../db.js'; +import { config } from '../config.js'; +import { closePool as closeApiExportPool } from '../../../api/src/core/db.js'; +import { buildExportPayloadForJob } from '../../../api/src/features/tenant-content/exports.js'; +import { normalizeStorageProvider, putStorageObject, storageBucket, validateObjectKey } from '../storage.js'; +import { renderExportDocx, renderExportPdf } from './export-renderer.js'; + +interface ExportJobRow { + id: string; + tenantId: string; + createdBy: string | null; + exportType: string; + format: 'pdf' | 'docx'; + scopeType: string; + scopeId: string; + status: string; + attemptCount: number; + maxAttempts: number; + options: Record; + outputMetadata: Record; + createdAt: string; +} + +interface ExportWorkerResult { + processed: number; + completed: number; + failed: number; + retrying: number; + skipped: number; +} + +interface RenderedExport { + body: Buffer; + mimeType: string; + extension: 'pdf' | 'docx'; +} + +const ASSET_VISIBILITIES = new Set(['public', 'tenant', 'members', 'svip', 'private']); + +function objectValue(value: unknown): Record { + return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : {}; +} + +function boolValue(value: unknown, fallback: boolean) { + return typeof value === 'boolean' ? value : fallback; +} + +function stringValue(value: unknown, fallback = '') { + return typeof value === 'string' && value.trim() ? value.trim() : fallback; +} + +function truncate(value: unknown, max = 1900) { + return String(value ?? '').slice(0, max); +} + +function errorMessage(error: unknown) { + return error instanceof Error ? error.message : String(error); +} + +function errorCode(error: unknown) { + return typeof error === 'object' && error !== null && 'code' in error + ? String((error as { code?: unknown }).code || 'EXPORT_WORKER_ERROR') + : 'EXPORT_WORKER_ERROR'; +} + +function backoffSeconds(attemptCount: number) { + const backoffs = config.exportBackoffSeconds.length ? config.exportBackoffSeconds : [30, 120, 600, 1800]; + return backoffs[Math.min(Math.max(0, attemptCount - 1), backoffs.length - 1)]; +} + +function safeFileBase(value: string) { + const ascii = value + .normalize('NFKD') + .replace(/[^\w.-]+/g, '-') + .replace(/-+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 80); + return ascii || 'question-export'; +} + +function safeDisplayFileName(title: string, extension: string) { + return `${title.trim().replace(/[\\/:*?"<>|]+/g, '-').replace(/\s+/g, '-').slice(0, 120) || '题库导出'}.${extension}`; +} + +function objectKeyFor(job: ExportJobRow, title: string, extension: string) { + const created = new Date(job.createdAt || Date.now()); + const yyyy = Number.isFinite(created.getTime()) ? created.getUTCFullYear() : new Date().getUTCFullYear(); + const mm = String((Number.isFinite(created.getTime()) ? created.getUTCMonth() : new Date().getUTCMonth()) + 1).padStart(2, '0'); + return validateObjectKey( + job.tenantId, + `${job.tenantId}/exports/${yyyy}/${mm}/${job.id}-${safeFileBase(title)}.${extension}`, + ); +} + +function assetVisibility(options: Record) { + const requested = stringValue(options.assetVisibility, boolValue(options.publishToAssets, false) ? 'tenant' : 'private'); + return ASSET_VISIBILITIES.has(requested) ? requested : 'private'; +} + +function assetStatus(options: Record) { + return boolValue(options.publishToAssets, false) ? 'active' : 'active'; +} + +function accessRules(options: Record) { + return objectValue(options.assetAccessRules); +} + +async function claimExportJobs() { + const client = await pool.connect(); + try { + await client.query('begin'); + const result = await client.query( + ` + select id, tenant_id as "tenantId", created_by as "createdBy", + export_type as "exportType", format, scope_type as "scopeType", + scope_id as "scopeId", status, attempt_count as "attemptCount", + max_attempts as "maxAttempts", options, + output_metadata as "outputMetadata", created_at as "createdAt" + from public.content_export_jobs + where status = 'pending' + and format in ('pdf', 'docx') + and attempt_count < max_attempts + and (next_attempt_at is null or next_attempt_at <= now()) + order by created_at asc + limit $1 + for update skip locked + `, + [config.exportBatchSize], + ); + + const ids = result.rows.map(row => row.id); + if (ids.length) { + await client.query( + ` + update public.content_export_jobs + set status = 'rendering', + locked_at = now(), + locked_by = $2, + attempt_count = attempt_count + 1, + started_at = coalesce(started_at, now()), + error_message = null, + updated_at = now() + where id = any($1::uuid[]) + `, + [ids, config.exportWorkerId], + ); + } + await client.query('commit'); + return result.rows; + } catch (error) { + await client.query('rollback'); + throw error; + } finally { + client.release(); + } +} + +async function renderJob(job: ExportJobRow): Promise<{ rendered: RenderedExport; title: string; questionCount: number }> { + const client = await pool.connect(); + try { + const payload = await buildExportPayloadForJob({ + client, + tenantId: job.tenantId, + jobId: job.id, + createdAt: new Date().toISOString(), + }); + const rendered = job.format === 'pdf' + ? await renderExportPdf(payload) + : await renderExportDocx(payload); + return { + rendered, + title: payload.title || '题库导出', + questionCount: payload.summary.questionCount, + }; + } finally { + client.release(); + } +} + +async function insertExportAsset( + client: pg.PoolClient, + job: ExportJobRow, + input: { + title: string; + fileName: string; + extension: 'pdf' | 'docx'; + mimeType: string; + bucket: string; + objectKey: string; + sizeBytes: number; + checksumSha256: string; + uploadDetails: Record; + }, +) { + const visibility = assetVisibility(job.options || {}); + const status = assetStatus(job.options || {}); + const assetType = input.extension === 'pdf' ? 'pdf' : 'document'; + const asset = await client.query<{ id: string }>( + ` + insert into public.content_assets ( + tenant_id, asset_key, title, category, description, file_name, + asset_type, storage_provider, bucket, object_key, mime_type, + file_size_bytes, checksum_sha256, upload_status, verified_at, + verified_size_bytes, verified_checksum_sha256, verification_details, + preview_status, visibility, is_public, status, access_rules, + metadata, created_by, updated_by, source + ) + values ( + $1, $2, $3, $4, $5, $6, + $7, $8, $9, $10, $11, + $12, $13, 'verified', now(), + $12, $13, $14::jsonb, + $15, $16, $17, $18, $19::jsonb, + $20::jsonb, $21, $21, 'content_export_worker' + ) + returning id + `, + [ + job.tenantId, + `content-export:${job.id}`, + input.title, + '题库导出', + `由题库导出任务 ${job.id} 自动生成`, + input.fileName, + assetType, + normalizeStorageProvider(String(job.outputMetadata?.storageProvider || config.storageDefaultProvider)), + input.bucket, + input.objectKey, + input.mimeType, + input.sizeBytes, + input.checksumSha256, + JSON.stringify({ + exportWorker: { + ...input.uploadDetails, + workerId: config.exportWorkerId, + checkedAt: new Date().toISOString(), + }, + }), + input.extension === 'pdf' ? 'ready' : 'none', + visibility, + visibility === 'public', + status, + JSON.stringify(accessRules(job.options || {})), + JSON.stringify({ + exportJobId: job.id, + exportType: job.exportType, + format: job.format, + scopeType: job.scopeType, + scopeId: job.scopeId, + publishToAssets: boolValue(job.options?.publishToAssets, false), + watermarkText: stringValue(job.options?.watermarkText, ''), + }), + job.createdBy, + ], + ); + return asset.rows[0].id; +} + +async function markExportCompleted(job: ExportJobRow, input: { + assetId: string; + title: string; + fileName: string; + mimeType: string; + sizeBytes: number; + checksumSha256: string; + bucket: string; + objectKey: string; + questionCount: number; + uploadDetails: Record; +}) { + const client = await pool.connect(); + try { + await client.query('begin'); + await client.query( + ` + update public.content_export_jobs + set status = 'completed', + asset_id = $3, + question_count = $4, + summary = coalesce(summary, '{}'::jsonb) || $5::jsonb, + output_metadata = coalesce(output_metadata, '{}'::jsonb) || $6::jsonb, + error_message = null, + locked_at = null, + locked_by = null, + next_attempt_at = null, + finished_at = now(), + updated_at = now() + where tenant_id = $1 and id = $2 + `, + [ + job.tenantId, + job.id, + input.assetId, + input.questionCount, + JSON.stringify({ + questionCount: input.questionCount, + renderedTitle: input.title, + }), + JSON.stringify({ + delivery: 'content_asset', + assetId: input.assetId, + fileName: input.fileName, + mimeType: input.mimeType, + sizeBytes: input.sizeBytes, + checksumSha256: input.checksumSha256, + bucket: input.bucket, + objectKey: input.objectKey, + renderer: 'worker:exports', + renderedAt: new Date().toISOString(), + upload: input.uploadDetails, + }), + ], + ); + await client.query( + ` + insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details) + values ($1, $2, 'content.questions.export_rendered', 'content_export_job', $3, $4::jsonb) + `, + [ + job.tenantId, + job.createdBy, + job.id, + JSON.stringify({ + assetId: input.assetId, + format: job.format, + mimeType: input.mimeType, + sizeBytes: input.sizeBytes, + checksumSha256: input.checksumSha256, + workerId: config.exportWorkerId, + }), + ], + ); + await client.query('commit'); + } catch (error) { + await client.query('rollback'); + throw error; + } finally { + client.release(); + } +} + +async function markExportFailed(job: ExportJobRow, error: unknown) { + const nextAttempt = job.attemptCount + 1; + const willRetry = nextAttempt < job.maxAttempts; + const status = willRetry ? 'pending' : 'failed'; + await pool.query( + ` + update public.content_export_jobs + set status = $3, + error_message = $4, + output_metadata = coalesce(output_metadata, '{}'::jsonb) || $5::jsonb, + next_attempt_at = case when $6::boolean then now() + make_interval(secs => $7::integer) else null end, + locked_at = null, + locked_by = null, + finished_at = case when $3 = 'failed' then now() else finished_at end, + updated_at = now() + where tenant_id = $1 and id = $2 + `, + [ + job.tenantId, + job.id, + status, + truncate(errorMessage(error)), + JSON.stringify({ + lastWorkerError: { + code: errorCode(error), + message: truncate(errorMessage(error)), + workerId: config.exportWorkerId, + failedAt: new Date().toISOString(), + nextAttempt, + maxAttempts: job.maxAttempts, + willRetry, + }, + }), + willRetry, + backoffSeconds(nextAttempt), + ], + ); + + await pool.query( + ` + insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details) + values ($1, $2, $3, 'content_export_job', $4, $5::jsonb) + `, + [ + job.tenantId, + job.createdBy, + willRetry ? 'content.questions.export_retry_scheduled' : 'content.questions.export_failed', + job.id, + JSON.stringify({ + code: errorCode(error), + message: truncate(errorMessage(error)), + workerId: config.exportWorkerId, + nextAttempt, + maxAttempts: job.maxAttempts, + }), + ], + ); + + return willRetry ? 'retrying' : 'failed'; +} + +async function processExportJob(job: ExportJobRow) { + const { rendered, title, questionCount } = await renderJob(job); + const extension = rendered.extension; + const bucket = storageBucket(String(job.outputMetadata?.storageBucket || '')); + const objectKey = objectKeyFor(job, title, extension); + const fileName = safeDisplayFileName(title, extension); + const upload = await putStorageObject({ + tenantId: job.tenantId, + provider: normalizeStorageProvider(String(job.outputMetadata?.storageProvider || config.storageDefaultProvider)), + bucket, + objectKey, + body: rendered.body, + mimeType: rendered.mimeType, + }); + + const client = await pool.connect(); + let assetId = ''; + try { + await client.query('begin'); + assetId = await insertExportAsset(client, job, { + title, + fileName, + extension, + mimeType: rendered.mimeType, + bucket, + objectKey, + sizeBytes: upload.sizeBytes, + checksumSha256: upload.checksumSha256, + uploadDetails: { + provider: upload.provider, + localPath: upload.localPath ? path.relative(process.cwd(), upload.localPath) : undefined, + etag: upload.etag, + }, + }); + await client.query('commit'); + } catch (error) { + await client.query('rollback'); + throw error; + } finally { + client.release(); + } + + await markExportCompleted(job, { + assetId, + title, + fileName, + mimeType: rendered.mimeType, + sizeBytes: upload.sizeBytes, + checksumSha256: upload.checksumSha256, + bucket, + objectKey, + questionCount, + uploadDetails: { + provider: upload.provider, + localPath: upload.localPath ? path.relative(process.cwd(), upload.localPath) : undefined, + etag: upload.etag, + }, + }); +} + +export async function processExportBatch(): Promise { + const jobs = await claimExportJobs(); + const result: ExportWorkerResult = { + processed: jobs.length, + completed: 0, + failed: 0, + retrying: 0, + skipped: 0, + }; + + for (const job of jobs) { + try { + if (job.format !== 'pdf' && job.format !== 'docx') { + result.skipped += 1; + continue; + } + await processExportJob(job); + result.completed += 1; + } catch (error) { + const state = await markExportFailed(job, error); + if (state === 'retrying') result.retrying += 1; + else result.failed += 1; + } + } + + return result; +} + +export async function closeExportExecutorPool() { + await closeApiExportPool(); +} diff --git a/apps/worker/src/storage.ts b/apps/worker/src/storage.ts new file mode 100644 index 00000000..b457b7e4 --- /dev/null +++ b/apps/worker/src/storage.ts @@ -0,0 +1,246 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { StorageClient } from '@supabase/storage-js'; +import { config } from './config.js'; + +export type StorageProviderName = 'external_url' | 'supabase_storage' | 'aliyun_oss' | 'tencent_cos' | 'qiniu_kodo' | 'local_dev'; + +interface PutObjectInput { + tenantId: string; + provider: StorageProviderName; + bucket: string; + objectKey: string; + body: Buffer; + mimeType: string; +} + +interface PutObjectResult { + provider: StorageProviderName; + bucket: string; + objectKey: string; + checksumSha256: string; + sizeBytes: number; + mimeType: string; + localPath?: string; + etag?: string | null; +} + +const SAFE_OBJECT_KEY_RE = /^[A-Za-z0-9][A-Za-z0-9._~!$&'()+,;=@/-]{0,1023}$/; +const SUPPORTED_UPLOAD_PROVIDERS = new Set(['local_dev', 'supabase_storage', 'aliyun_oss', 'tencent_cos']); + +export function normalizeStorageProvider(value: string | null | undefined): StorageProviderName { + const provider = (value?.trim() || config.storageDefaultProvider || 'local_dev') as StorageProviderName; + if (!SUPPORTED_UPLOAD_PROVIDERS.has(provider)) { + throw new Error(`Unsupported export storage provider: ${provider}`); + } + return provider; +} + +export function storageBucket(value: string | null | undefined) { + return value?.trim() || config.storageDefaultBucket || 'tenant-assets'; +} + +export function canonicalObjectKey(objectKey: string) { + return objectKey.replace(/^\/+/, '').replace(/\/{2,}/g, '/'); +} + +export function validateObjectKey(tenantId: string, objectKey: string) { + const clean = canonicalObjectKey(objectKey); + if (!clean || clean.includes('..') || clean.includes('\\') || clean.includes('%2f') || clean.includes('%2F')) { + throw new Error('Invalid objectKey'); + } + if (!SAFE_OBJECT_KEY_RE.test(clean)) { + throw new Error('objectKey contains unsafe characters'); + } + if (config.storageRequireTenantPrefix && !clean.startsWith(`${tenantId}/`)) { + throw new Error('objectKey must be scoped by tenantId prefix'); + } + return clean; +} + +function sha256(body: Buffer) { + return crypto.createHash('sha256').update(body).digest('hex'); +} + +function requireConfigured(condition: unknown, provider: StorageProviderName, missing: string) { + if (!condition) throw new Error(`${provider} is not configured: ${missing}`); +} + +async function putLocalDev(input: PutObjectInput, checksumSha256: string): Promise { + const root = path.resolve(process.cwd(), config.exportLocalStorageRoot || '.local-storage'); + const bucketRoot = path.resolve(root, input.bucket); + const target = path.resolve(bucketRoot, input.objectKey); + if (!target.startsWith(bucketRoot + path.sep)) { + throw new Error('Resolved local object path escapes storage root'); + } + await fs.mkdir(path.dirname(target), { recursive: true }); + await fs.writeFile(target, input.body); + return { + provider: input.provider, + bucket: input.bucket, + objectKey: input.objectKey, + checksumSha256, + sizeBytes: input.body.byteLength, + mimeType: input.mimeType, + localPath: target, + }; +} + +async function putAliyunOss(input: PutObjectInput, checksumSha256: string): Promise { + requireConfigured(config.aliyunOssAccessKeyId, 'aliyun_oss', 'ALIYUN_OSS_ACCESS_KEY_ID'); + requireConfigured(config.aliyunOssAccessKeySecret, 'aliyun_oss', 'ALIYUN_OSS_ACCESS_KEY_SECRET'); + requireConfigured(config.aliyunOssRegion || config.aliyunOssEndpoint, 'aliyun_oss', 'ALIYUN_OSS_REGION or ALIYUN_OSS_ENDPOINT'); + + const { default: OSS } = await import('ali-oss'); + const client = new OSS({ + region: config.aliyunOssRegion || undefined, + endpoint: config.aliyunOssEndpoint || undefined, + accessKeyId: config.aliyunOssAccessKeyId, + accessKeySecret: config.aliyunOssAccessKeySecret, + stsToken: config.aliyunOssStsToken || undefined, + bucket: input.bucket, + internal: config.aliyunOssInternal, + secure: true, + }); + const response = await (client as unknown as { + put: (objectKey: string, body: Buffer, options: Record) => Promise<{ res?: { headers?: Record } }>; + }).put(input.objectKey, input.body, { + headers: { + 'Content-Type': input.mimeType, + 'x-oss-meta-sha256': checksumSha256, + }, + }); + return { + provider: input.provider, + bucket: input.bucket, + objectKey: input.objectKey, + checksumSha256, + sizeBytes: input.body.byteLength, + mimeType: input.mimeType, + etag: response.res?.headers?.etag || null, + }; +} + +async function putSupabaseStorage(input: PutObjectInput, checksumSha256: string): Promise { + requireConfigured(config.supabaseStorageUrl, 'supabase_storage', 'SUPABASE_STORAGE_URL'); + requireConfigured(config.supabaseStorageServiceKey, 'supabase_storage', 'SUPABASE_STORAGE_SERVICE_KEY'); + const client = new StorageClient(config.supabaseStorageUrl.replace(/\/+$/, ''), { + apikey: config.supabaseStorageServiceKey, + authorization: `Bearer ${config.supabaseStorageServiceKey}`, + }); + const response = await client.from(input.bucket).upload(input.objectKey, input.body, { + contentType: input.mimeType, + upsert: true, + duplex: 'half', + metadata: { sha256: checksumSha256 }, + } as never); + if (response.error) { + throw new Error(response.error.message || 'Supabase Storage upload failed'); + } + return { + provider: input.provider, + bucket: input.bucket, + objectKey: input.objectKey, + checksumSha256, + sizeBytes: input.body.byteLength, + mimeType: input.mimeType, + }; +} + +function hmacSha1Hex(key: string | Buffer, value: string) { + return crypto.createHmac('sha1', key).update(value).digest('hex'); +} + +function sha1Hex(value: string) { + return crypto.createHash('sha1').update(value).digest('hex'); +} + +function cosEncodePath(objectKey: string) { + return objectKey + .split('/') + .map(part => encodeURIComponent(part).replace(/[!'()*]/g, char => `%${char.charCodeAt(0).toString(16).toUpperCase()}`)) + .join('/'); +} + +function cosHost(bucket: string) { + requireConfigured(config.tencentCosRegion, 'tencent_cos', 'TENCENT_COS_REGION'); + const bucketWithAppId = config.tencentCosAppId && !bucket.endsWith(`-${config.tencentCosAppId}`) + ? `${bucket}-${config.tencentCosAppId}` + : bucket; + return `${bucketWithAppId}.cos.${config.tencentCosRegion}.myqcloud.com`; +} + +function nowSeconds() { + return Math.floor(Date.now() / 1000); +} + +async function putTencentCos(input: PutObjectInput, checksumSha256: string): Promise { + requireConfigured(config.tencentCosSecretId, 'tencent_cos', 'TENCENT_COS_SECRET_ID'); + requireConfigured(config.tencentCosSecretKey, 'tencent_cos', 'TENCENT_COS_SECRET_KEY'); + const host = cosHost(input.bucket); + const start = nowSeconds(); + const end = start + 300; + const keyTime = `${start};${end}`; + const pathname = `/${cosEncodePath(input.objectKey)}`; + const signedHeaders: Record = { + host, + 'content-type': input.mimeType, + 'x-cos-meta-sha256': checksumSha256, + }; + const headerKeys = Object.keys(signedHeaders).sort(); + const headerList = headerKeys.join(';'); + const httpHeaders = headerKeys + .map(key => `${encodeURIComponent(key)}=${encodeURIComponent(signedHeaders[key]).toLowerCase()}`) + .join('&'); + const signedQuery: Record = {}; + if (config.tencentCosSecurityToken) signedQuery['x-cos-security-token'] = config.tencentCosSecurityToken; + const queryKeys = Object.keys(signedQuery).sort(); + const urlParamList = queryKeys.join(';'); + const httpParameters = queryKeys + .map(key => `${encodeURIComponent(key)}=${encodeURIComponent(signedQuery[key])}`) + .join('&'); + const httpString = `put\n${pathname}\n${httpParameters}\n${httpHeaders}\n`; + const stringToSign = `sha1\n${keyTime}\n${sha1Hex(httpString)}\n`; + const signKey = hmacSha1Hex(config.tencentCosSecretKey, keyTime); + const signature = hmacSha1Hex(signKey, stringToSign); + const query = new URLSearchParams(); + query.set('q-sign-algorithm', 'sha1'); + query.set('q-ak', config.tencentCosSecretId); + query.set('q-sign-time', keyTime); + query.set('q-key-time', keyTime); + query.set('q-header-list', headerList); + query.set('q-url-param-list', urlParamList); + query.set('q-signature', signature); + for (const key of queryKeys) query.set(key, signedQuery[key]); + + const response = await fetch(`https://${host}${pathname}?${query.toString()}`, { + method: 'PUT', + headers: signedHeaders, + body: input.body as unknown as BodyInit, + }); + if (!response.ok) { + throw new Error(`Tencent COS upload failed: ${response.status}`); + } + return { + provider: input.provider, + bucket: input.bucket, + objectKey: input.objectKey, + checksumSha256, + sizeBytes: input.body.byteLength, + mimeType: input.mimeType, + etag: response.headers.get('etag'), + }; +} + +export async function putStorageObject(input: PutObjectInput): Promise { + const provider = normalizeStorageProvider(input.provider); + const objectKey = validateObjectKey(input.tenantId, input.objectKey); + const normalized = { ...input, provider, objectKey }; + const checksumSha256 = sha256(input.body); + if (provider === 'local_dev') return putLocalDev(normalized, checksumSha256); + if (provider === 'aliyun_oss') return putAliyunOss(normalized, checksumSha256); + if (provider === 'tencent_cos') return putTencentCos(normalized, checksumSha256); + if (provider === 'supabase_storage') return putSupabaseStorage(normalized, checksumSha256); + throw new Error(`${provider} does not support worker object upload`); +} diff --git a/docs/refactor/backend-capability-status.md b/docs/refactor/backend-capability-status.md index 4227f7b7..5c78eee5 100644 --- a/docs/refactor/backend-capability-status.md +++ b/docs/refactor/backend-capability-status.md @@ -137,7 +137,7 @@ | 数据看板聚合接口 | 可联调 | `GET /api/tenant-admin/dashboard`;支持 `7d/30d/90d`、地区筛选、学生/学习/内容/订单/激活码/反馈卡片、趋势、24h 活跃、题型分布、科目排行、地区统计、套餐销量和运营动态 | | 平台公共题库授权 | 可联调 | `/api/platform-admin/question-banks`、`question-bank-grants`;支持按 SaaS 套餐、指定租户或全部活跃租户披露平台公共题库 | | 租户采纳/同步公共题库 | 可联调 | `/api/tenant-content/public-question-banks`、`public-question-banks/adopt`、`public-question-banks/sync`、`public-question-banks/conflicts`、`public-question-banks/conflicts/resolve`、`public-question-banks/conflicts/resolve-batch`、`tenant-content/notifications`;租户只能看到自己订阅/授权范围内题库,采纳后生成租户自己的题库、入口、集合和题目快照,可直接进入练习;平台更新后可手动或由 worker 自动同步,新增/更新和冲突会生成租户内容通知;租户自改题目会标记冲突并跳过;后台可查询最近一次冲突明细,并可单条或批量选择“采纳平台版本”/“保留本地版本”,操作会重新校验授权并写入逐条审计,冲突全部处理后相关通知自动 resolved | -| 题库导出基础 | 可联调 | `/api/tenant-content/exports/questions`、`/api/tenant-content/exports/jobs`;支持按题目集合、内容入口或分类节点导出 JSON/试卷 payload,后端校验租户内容编辑权限、跨租户隔离、答案/解析开关、复合题子题脱敏、导出 job 和审计;PDF/Word 二进制与水印 worker 后续补 | +| 题库导出 | 可联调 | `/api/tenant-content/exports/questions`、`/api/tenant-content/exports/jobs`;支持按题目集合、内容入口或分类节点导出 JSON/试卷 payload,也支持 `pdf/docx` 异步导出;后端校验租户内容编辑权限、跨租户隔离、答案/解析开关、复合题子题脱敏、导出 job 和审计;`apps/worker --job exports` 负责 PDF/Word 渲染、水印、生成 `content_assets`、记录 hash/size/assetId,前端通过资源签名接口下载/预览 | ## 销售、代理、CRM @@ -168,7 +168,7 @@ | 视频 JSON preview/import | 可联调 | 支持 `videos/items`,后端校验题目、科目、资源引用,导入后写入 `question_videos` | | Excel/CSV 导入 | 可联调 | 题目、单词、知识手册、分数线、视频已支持 CSV 和 `.xlsx` 解析,解析后复用 `content_import_jobs/items/issues` 管线并保留 `parser_metadata`;模板下载、字段映射 API、字段映射覆盖白名单、导入任务详情、异步 worker 状态、导入后复检已接入;Taro 租户内容页已接上传/粘贴 preview/import、字段别名编辑、异步任务轮询和复检详情第一版 | | 大批量异步导入 | 可联调 | `executionMode=async` 会将 preview job 置为 `pending`;`apps/worker --job imports` 抢占 queued job,复用 API 导入 executor,支持重试、清锁和审计 | -| 题库导出任务 | 可联调 | `content_export_jobs` 记录导出范围、格式、题量、输出 hash、选项和执行人;当前返回 inline base64 JSON 文件,前端可先下载 `.json` 或交给后续 PDF/Word worker 渲染 | +| 题库导出任务 | 可联调 | `content_export_jobs` 记录导出范围、格式、题量、输出 hash、选项、执行人、异步状态、重试次数和 `asset_id`;JSON 类导出返回 inline base64,`pdf/docx` 由 exports worker 渲染为资源台账文件 | | 公共题库自动同步增强 | 部分覆盖 | `apps/worker --job public-banks` 已可抢占待同步采纳记录、自动同步平台新增/更新题目、记录失败和审计;同步新增/更新和冲突会写入 `tenant_content_notifications`,租户后台已有通知列表、已读/忽略、单条和批量冲突处理第一版;后续需接入生产定时调度、失败告警和更完整运营消息 | ## 当前验证 @@ -189,6 +189,7 @@ npm run test:worker:imports npm run test:worker:crm npm run test:worker:commerce npm run test:worker:assets +npm run test:worker:exports npm run test:worker:public-banks git diff --check ``` diff --git a/docs/refactor/backend-handoff-roadmap.md b/docs/refactor/backend-handoff-roadmap.md index 528238ca..34f78b7c 100644 --- a/docs/refactor/backend-handoff-roadmap.md +++ b/docs/refactor/backend-handoff-roadmap.md @@ -23,12 +23,12 @@ | 多租户底座 | 可联调 | 租户、域名、品牌、设置、RLS 基础、审计、Supabase JWT/API 身份映射 | 真实云端 Auth/JWKS 回归、生产 RLS 深测 | | 平台后台 | 基础完成 | 租户、套餐、订阅、账单、服务费、用量、公共题库授权、公共题库自动同步 worker、公共题库冲突单条/批量处理 API、公共题库同步通知第一版 | 自动计费、平台审计、更完整运营消息 | | 租户后台 | 可联调 | 品牌、域名、支付账户、登录配置、密钥掩码、活动、兑换码、优惠券、勋章管理/发放、成员权限、角色模板、菜单/模块/字段权限配置 API、班级/教师/学生范围权限;Taro 工作台已接权限驱动模块入口,学生运营页已接学生创建/更新、禁用/恢复、批量导入、批量分班、备注和跟进任务第一版,租户设置页已接角色模板和成员绑定操作台第一版,营销中心已接 CRM 配置/队列和分佣结算操作台第一版 | 更细的数据范围组合、成员批量运营、真实打款/导出/凭证和完整权限菜单 | -| 题库与练习 | 可联调 | 内容入口、任意深度分类、题目集合、顺序/随机/全真模拟蓝图、组卷快照、客观题后端判分、主观题 `selfJudgedCorrect` 自评、阅读理解/案例分析 `subAnswers` 多小题判分、答题、错题、收藏、模考报告、排行榜、公共题库采纳快照、手动同步、自动同步 worker、冲突查询/单条和批量处理 API、公共题库同步通知、JSON/试卷 payload 导出 | 长题干/公式图片混排体验、PDF/Word 导出 worker、排行榜防刷/预聚合 | +| 题库与练习 | 可联调 | 内容入口、任意深度分类、题目集合、顺序/随机/全真模拟蓝图、组卷快照、客观题后端判分、主观题 `selfJudgedCorrect` 自评、阅读理解/案例分析 `subAnswers` 多小题判分、答题、错题、收藏、模考报告、排行榜、公共题库采纳快照、手动同步、自动同步 worker、冲突查询/单条和批量处理 API、公共题库同步通知、JSON/试卷 payload 导出、PDF/Word 异步导出 worker、水印和资料发布路径 | 长题干/公式图片混排体验、导出模板精排、排行榜防刷/预聚合 | | 背单词 | 可联调 | 单元、单词、进度、收藏、统计、每日计划、JSON/CSV/Excel 导入、排行榜 | 更细复习参数 | | 知识手册 | 可联调 | 科目、章节、条目、Markdown 内容、嵌套 JSON/CSV/Excel 导入 | 富文本资源、版本管理、附件/PDF 关联 | | 分数线 | 可联调 | 院校、专业、动态字段、记录、年份、趋势、后台维护、JSON/CSV/Excel 导入 | 复杂筛选、AI 择校上下文 | | 视频解析 | 可联调 | 单题视频、批量查询、后台视频绑定、JSON/CSV/Excel 导入、会员播放权限、播放次数扣减、签名 URL 和播放日志 | 深度防盗链、动态水印、播放统计 | -| 资料下载 | 部分完成 | 资源台账、SVIP 权限校验、`local_dev`/阿里云 OSS/腾讯 COS/Supabase Storage 上传下载签名、上传确认、PDF/图片预览签名、assets worker 复检异常下架 | PDF 渲染、CDN 防盗链、杀毒扫描、视频水印 | +| 资料下载 | 部分完成 | 资源台账、SVIP 权限校验、`local_dev`/阿里云 OSS/腾讯 COS/Supabase Storage 上传下载签名、上传确认、PDF/图片预览签名、assets worker 复检异常下架、题库导出 PDF/Word 可生成 `content_assets` 并走签名下载/预览 | CDN 防盗链、杀毒扫描、视频水印、资料前端操作体验 | | 会员与订单 | 可联调 | 下单、订单详情/状态轮询、优惠券领取/抵扣、零元订单自动开通、手工确认权限保护、激活码预检查/兑换、微信支付、支付宝、微信/支付宝发起退款、微信/支付宝退款查询确认、微信/支付宝退款通知 webhook、支付/退款补偿 worker、权益发放 | 完整资金流水对账、异常订单运营台 | | 登录认证 | 可联调 | 短信 mock、阿里云/腾讯云短信 adapter、迁移期 session、Supabase Auth JWT、微信小程序登录、微信网页登录、QQ 登录、手机号绑定/换绑、OAuth 配置表 | 真实生产账号和回调域名联调 | | 销售/代理/CRM | 可联调 | 邀请码、首绑保护、团队关系、销售统计、CRM 配置/队列、钉钉/飞书/企微 worker、分佣规则、成员分佣比例、订单/激活码归因、结算生成、审核和打款状态;Taro 营销中心已接第一版操作台 | 小程序码真实生成、CRM 分配策略、结算导出、真实打款、凭证、财务复核和销售转化看板 | @@ -76,7 +76,7 @@ ### P0:上云测试和前端主链路前必须处理 - 生产鉴权:API 已支持 Supabase Auth JWT;继续做真实云端 Auth/JWKS 回归、RLS 深测,并在生产关闭 `x-user-id` 与 `x-platform-admin-key` 兼容入口。 -- 对象存储:上传/下载签名已接入阿里云 OSS、腾讯云 COS、Supabase Storage;上传确认、PDF/图片预览签名和 assets worker 复检已完成,继续补 PDF 渲染、视频播放防盗链、杀毒扫描和水印。 +- 对象存储:上传/下载签名已接入阿里云 OSS、腾讯云 COS、Supabase Storage;上传确认、PDF/图片预览签名、assets worker 复检和题库导出 PDF/Word worker 已完成,继续补视频播放防盗链、杀毒扫描和动态水印。 - 真实数据 dry-run:导出 PocketBase 用户、题库、单词、知识手册、分数线、订单、权益,先跑 `npm run pb:import:dry-run`,再跑迁移和校验报告。 - 生产环境配置:`.env.example` 和 `npm run readiness:production` / `npm run readiness:production:db` 已补;继续补数据库迁移流程、备份恢复、日志、告警和 API 容器部署说明。 - Taro scaffold:`apps/taro` 地基已建立;学生端、租户后台、平台后台第一批 H5 页面已接真实 API,学生端已接地区选择、错题/收藏复习、阅读理解/案例分析多小题作答、题目反馈、视频解析、练习/模考报告、收银台、订单详情和售后入口第一版,平台后台关键写操作第一版已接入,租户工作台已接权限驱动模块入口,租户学生运营页已接学生创建/更新、禁用/恢复、批量导入、批量分班、备注和跟进任务第一版,租户内容页已接公共题库采纳/同步、冲突查看、单条/批量采纳平台或保留本地、导入问题、字段模板预览/下载、上传/粘贴预览、字段别名覆盖、同步/异步导入、异步轮询和复检详情第一版,租户设置页已接角色模板创建/编辑/停用、成员绑定模板和权限可见性配置第一版,租户营销中心已接 CRM 配置/队列和分佣结算操作台第一版;下一步补公式图片混排、更细数据范围 UI、平台后台审计增强和小程序兼容验证。 diff --git a/docs/refactor/frontend-handoff-index.md b/docs/refactor/frontend-handoff-index.md index 174ad99b..d5381a74 100644 --- a/docs/refactor/frontend-handoff-index.md +++ b/docs/refactor/frontend-handoff-index.md @@ -87,11 +87,11 @@ | 工作台 | `apps/taro/src/pages/tenant-admin/workbench/index.tsx` | `tenant-admin/overview`、`tenant-admin/dashboard`、`tenant-admin/permissions` | | 数据看板 | `apps/taro/src/pages/tenant-admin/dashboard/index.tsx` | `tenant-admin/dashboard` | | 学生运营 | `apps/taro/src/pages/tenant-admin/students/index.tsx` | `tenant-admin/classes`、`tenant-admin/teachers`、`tenant-admin/students`、`students/bulk-upsert`、`students/status`、`classes/members/bulk-assign`、`students/notes`、`students/followups` | -| 题库内容 | `apps/taro/src/pages/tenant-admin/content/index.tsx` | `tenant-content/content-entries`、`tenant-content/imports`、`imports/detail`、`imports/issues`、`imports/field-mapping`、`imports/templates`、`imports/post-check`、`tenant-content/public-question-banks`、`public-question-banks/adopt`、`public-question-banks/sync`、`public-question-banks/conflicts`、`public-question-banks/conflicts/resolve`、`public-question-banks/conflicts/resolve-batch`、`tenant-content/notifications`、`tenant-content/notifications/status` | +| 题库内容 | `apps/taro/src/pages/tenant-admin/content/index.tsx` | `tenant-content/content-entries`、`tenant-content/imports`、`imports/detail`、`imports/issues`、`imports/field-mapping`、`imports/templates`、`imports/post-check`、`tenant-content/exports/questions`、`tenant-content/exports/jobs`、`tenant-content/assets/sign-download`、`tenant-content/assets/sign-preview`、`tenant-content/public-question-banks`、`public-question-banks/adopt`、`public-question-banks/sync`、`public-question-banks/conflicts`、`public-question-banks/conflicts/resolve`、`public-question-banks/conflicts/resolve-batch`、`tenant-content/notifications`、`tenant-content/notifications/status` | | 营销中心 | `apps/taro/src/pages/tenant-admin/marketing/index.tsx` | `tenant-admin/coupons`、`code-batches`、`activation-codes`、`crm/config`、`crm/queue`、`commission/settings`、`member-rate`、`summary`、`orders`、`settlements`、`settlements/generate`、`settlements/status` | | 租户设置 | `apps/taro/src/pages/tenant-admin/settings/index.tsx` | `tenant-admin/overview`、`domains`、`payment-accounts`、`auth-providers`、`permissions`、`GET/PUT role-templates`、`POST role-templates/disable`、`GET/PUT members`、`POST members/disable` | -当前租户后台已有第一批运营操作:工作台按权限矩阵隐藏不可见模块;学生运营页支持学生创建/更新、状态禁用/恢复、批量导入、批量分班、学生备注、跟进任务和完成跟进;题库内容页支持公共题库采纳/同步、同步通知查看与已读/忽略、同步冲突查看、单条/批量采纳平台版本或保留本地版本、导入任务详情、异步 job 轮询、导入问题查看、字段映射/模板预览/下载、JSON/CSV/Excel 导入预览和执行、字段别名覆盖和导入后复检详情;营销中心支持 CRM 配置保存、队列按状态查看、分佣规则、成员分佣比例、分佣订单明细、结算单生成、审核通过/驳回和标记线下打款;租户设置页支持角色模板创建、编辑、停用、权限点、菜单、模块、字段、基础数据范围配置、成员搜索/新建、成员绑定模板、成员状态和额外权限覆盖。下一批需要继续补更精细的学生导入模板体验、更细数据范围 UI、真实打款 provider、结算导出和凭证。 +当前租户后台已有第一批运营操作:工作台按权限矩阵隐藏不可见模块;学生运营页支持学生创建/更新、状态禁用/恢复、批量导入、批量分班、学生备注、跟进任务和完成跟进;题库内容页支持公共题库采纳/同步、同步通知查看与已读/忽略、同步冲突查看、单条/批量采纳平台版本或保留本地版本、导入任务详情、异步 job 轮询、导入问题查看、字段映射/模板预览/下载、JSON/CSV/Excel 导入预览和执行、字段别名覆盖和导入后复检详情,也可接 JSON/试卷 payload 同步导出与 PDF/Word 异步导出 job 轮询,完成后用 `assetId` 走后台资源签名下载/预览;营销中心支持 CRM 配置保存、队列按状态查看、分佣规则、成员分佣比例、分佣订单明细、结算单生成、审核通过/驳回和标记线下打款;租户设置页支持角色模板创建、编辑、停用、权限点、菜单、模块、字段、基础数据范围配置、成员搜索/新建、成员绑定模板、成员状态和额外权限覆盖。下一批需要继续补更精细的学生导入模板体验、更细数据范围 UI、真实打款 provider、结算导出和凭证。 ## 已落地的 Taro 平台后台页面 diff --git a/docs/refactor/implementation-status.md b/docs/refactor/implementation-status.md index edabbe07..d02dddb0 100644 --- a/docs/refactor/implementation-status.md +++ b/docs/refactor/implementation-status.md @@ -4,7 +4,7 @@ ## 当前结论 -当前重构已经完成了 Supabase/PostgreSQL 多租户底座、核心业务表、PocketBase 数据导入器雏形、学生端核心 API、租户后台 API、平台后台 SaaS 账务 API、内容资产/题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 批量导入基础闭环、异步导入 worker、题库导出基础、题库入口/任意深度分类/题目集合/练习蓝图/组卷快照基础闭环,以及本地 Docker/API 构建验证。 +当前重构已经完成了 Supabase/PostgreSQL 多租户底座、核心业务表、PocketBase 数据导入器雏形、学生端核心 API、租户后台 API、平台后台 SaaS 账务 API、内容资产/题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 批量导入基础闭环、异步导入 worker、题库导出基础、PDF/Word 导出 worker、题库入口/任意深度分类/题目集合/练习蓝图/组卷快照基础闭环,以及本地 Docker/API 构建验证。 但这还不是完整商用交付状态,也不能说旧项目核心功能已经全部重构完成。现在更准确的状态是:后端商用架构骨架已经立住,核心业务正在按模块补齐。部分功能已经有可调用 API,部分功能只有数据模型和导入映射,部分功能还没有前端/自动化测试闭环。 @@ -24,7 +24,7 @@ | 模块 | 数据模型 | PocketBase 导入 | API | 自动化测试 | 当前状态 | | --- | --- | --- | --- | --- | --- | | 多租户隔离 | 已建 `tenants`、`tenant_domains`、`tenant_branding`、`tenant_settings`、RLS 基础 | 部分支持 | 租户解析、品牌、域名、支付账户、登录 provider、平台建租户已实现 | 核心 API 集成测试含租户隔离断言 | 基础可用,正式 JWT/RLS 权限闭环未完成 | -| 刷题题库 | 已建题库、题目、题目版本、内容入口、任意深度分类树、考试意向标记、题目集合、练习蓝图、导入任务台账、导出任务台账、公共题库授权/采纳表、租户内容通知表 | 已支持核心映射,JSON/CSV/Excel 导入可落到新入口/节点/集合,阅读理解/案例分析子题沿用 `subQuestions/sub_questions` | 题目列表、内容入口、分类树、集合题目、顺序/随机/全真模拟 session、答题提交、复合题 `subAnswers` 判分和报告明细、租户后台题目录入/更新、JSON/CSV/Excel 预览/导入、JSON/试卷 payload 导出、异步导入 worker、平台公共题库授权、租户采纳快照、手动同步、自动同步 worker、同步通知、冲突查询和单条/批量冲突处理已实现 | 核心 API 集成测试含导航、组卷、复合题后台录入/练习/判分/报告、导入、导出权限/脱敏、公共题库授权、采纳后组卷、同步新增题、通知隔离/已读/自动 resolved、租户自改冲突保护、单条/批量冲突处理和 worker 自动同步断言 | 新题库导航和组卷基础闭环可跑,阅读理解/案例分析多小题第一版可联调,公共题库采纳/手动/自动同步、同步通知、冲突查询/处理、导入后复检、模板下载、字段映射 API 和导出基础可联调;PDF/Word 导出 worker、公共题库生产调度/失败告警和更完整运营消息仍需补齐 | +| 刷题题库 | 已建题库、题目、题目版本、内容入口、任意深度分类树、考试意向标记、题目集合、练习蓝图、导入任务台账、导出任务台账、公共题库授权/采纳表、租户内容通知表 | 已支持核心映射,JSON/CSV/Excel 导入可落到新入口/节点/集合,阅读理解/案例分析子题沿用 `subQuestions/sub_questions` | 题目列表、内容入口、分类树、集合题目、顺序/随机/全真模拟 session、答题提交、复合题 `subAnswers` 判分和报告明细、租户后台题目录入/更新、JSON/CSV/Excel 预览/导入、JSON/试卷 payload 导出、PDF/Word 异步导出 worker、异步导入 worker、平台公共题库授权、租户采纳快照、手动同步、自动同步 worker、同步通知、冲突查询和单条/批量冲突处理已实现 | 核心 API 集成测试含导航、组卷、复合题后台录入/练习/判分/报告、导入、导出权限/脱敏、异步 PDF/Word job 创建、exports worker、公共题库授权、采纳后组卷、同步新增题、通知隔离/已读/自动 resolved、租户自改冲突保护、单条/批量冲突处理和 worker 自动同步断言 | 新题库导航和组卷基础闭环可跑,阅读理解/案例分析多小题第一版可联调,公共题库采纳/手动/自动同步、同步通知、冲突查询/处理、导入后复检、模板下载、字段映射 API、JSON/PDF/Word 导出可联调;公共题库生产调度/失败告警、更精细导出模板和更完整运营消息仍需补齐 | | 错题本 | 已建 `wrong_questions` | 已支持旧错题归一化 | 错题列表、答题自动入错题、移出错题已实现 | 仅烟测 | 基础功能已实现,复习计划和统计未完成 | | 收藏夹 | 已建 `favorite_questions` | 已支持旧收藏归一化 | 收藏/取消收藏、收藏列表已实现 | 仅烟测 | 基础功能已实现 | | 用户订阅/题库会员/SVIP | 已建 `orders`、`payments`、`entitlements`、`svip_plans`、激活码 | 已映射旧 SVIP/会员权益 | 下单、订单详情/状态轮询、手工支付确认权限保护、微信/支付宝支付、微信/支付宝发起退款、微信/支付宝退款查询确认、微信/支付宝退款通知 webhook、激活码预检查/兑换、优惠券抵扣、零元订单自动开通、权益查询已实现 | API 集成测试 | 商城主链路可联调,对账、支付补偿和异常订单自动处理待补 | @@ -32,7 +32,7 @@ | 知识手册 | 已建手册科目、章节、条目,并可绑定 `content_entries/content_nodes` | 已支持内容导入 | 只读 API、租户后台手册科目/章节/条目维护 API、嵌套 JSON 预览导入已实现 | 核心 API 集成测试含导入断言 | 学生端阅读、后台维护和批量 JSON 导入基础可用,富文本资源/版本管理待补 | | 分数线 | 已建院校、专业、字段、记录表 | 已支持导入映射 | 字段、院校、专业、记录、趋势、年份、租户后台维护 API、JSON 预览导入已实现 | 核心 API 集成测试含导入断言 | 查询、后台维护和批量 JSON 导入基础闭环已实现,复杂动态筛选和 AI 择校上下文待补 | | 题目视频讲解 | 已建 `video_explanations`、`question_videos` | 已支持导入映射 | 单题视频、批量预加载、通用视频搜索、播放签名、视频次数扣减、租户后台视频创建绑定 API、JSON 预览导入已实现 | 核心 API 集成测试含播放和导入断言 | 播放、权益、后台绑定和批量 JSON 导入链路已实现,深度防盗链、动态水印和播放统计待补 | -| 资料下载/PDF | 已扩展 `content_assets`,新增资源台账和导入任务表 | 旧 `app_assets/images` 兼容导入 | 租户后台资源管理、OSS/COS/Supabase Storage 上传/下载签名、上传确认、PDF/图片预览签名、学生端资料列表/下载权限已实现 | 核心 API 集成测试含 SVIP 资料下载,assets worker 测试 | 资料资源基础闭环可跑,CDN 防盗链、杀毒扫描和资料下载前端待补 | +| 资料下载/PDF | 已扩展 `content_assets`,新增资源台账和导入任务表 | 旧 `app_assets/images` 兼容导入 | 租户后台资源管理、OSS/COS/Supabase Storage 上传/下载签名、上传确认、PDF/图片预览签名、题库导出 PDF/Word 自动发布资源、学生端资料列表/下载权限已实现 | 核心 API 集成测试含 SVIP 资料下载,assets worker 和 exports worker 测试 | 资料资源基础闭环可跑,CDN 防盗链、杀毒扫描和资料下载前端待补 | | 个人中心 | 已建 `student_profiles`、会员权益、订单、练习记录、`badges/user_badges` | 已支持部分用户资料和勋章导入 | 个人资料、目标院校/专业、手机号绑定/换绑、会员状态、最近练习、统计聚合、签到积分、题目反馈、考试倒计时、勋章 API 已实现 | API 集成测试 | 学生端基础个人中心已实现,学习报告可视化和更细任务系统待补 | | 活动/优惠 | 已建优惠券、激活码、激活码批次、banner、FAQ、公告、勋章等基础表 | 部分支持 | banner/FAQ/公告只读与租户后台维护、激活码预检查/兑换、激活码批次、批量生成激活码、优惠券维护、前台领取/下单抵扣、勋章维护和手动发放已实现 | 核心 API 集成测试 | 基础运营后台可用,勋章自动发放、复杂活动规则、营销自动化、核销报表待补 | | 销售/代理客资追踪 | 已建推荐码、首绑客资、团队关系、小程序码缓存、CRM 队列 | 旧 `referral_tracks` 已有映射基础 | 邀请码、扫码/分享事件、首绑保护、销售统计、客资明细、手动补绑、团队关系、CRM 配置/队列、CRM worker 推送已实现 | 核心 API 集成测试、CRM worker 集成测试 | 增长链路基础可用,真实微信小程序码、CRM 分配策略、富卡片和销售转化看板待补 | diff --git a/docs/refactor/legacy-feature-gap-matrix.md b/docs/refactor/legacy-feature-gap-matrix.md index 3bc8ff12..3380e022 100644 --- a/docs/refactor/legacy-feature-gap-matrix.md +++ b/docs/refactor/legacy-feature-gap-matrix.md @@ -59,7 +59,7 @@ | 激活码 | 已覆盖 | 批次、生成、预检查、兑换、自用码拒绝、地区校验主链路已有 | | 勋章管理 | 部分覆盖 | 后台勋章维护、手动发放、重复发放幂等、学生端勋章展示和权限隔离已覆盖;自动发放规则、积分活动联动和前端运营 UI 待补 | | 题库录入 | 已覆盖 | 单题创建/更新、题目/单词/知识手册/分数线/视频 JSON/CSV/Excel 同步/异步导入、集合/蓝图、导入后复检、模板下载、字段映射 API 和导入任务详情已有;Taro 租户内容页已接上传/粘贴预览、模板下载、字段别名编辑、同步/异步执行、异步轮询和复检结果详情第一版;真实数据验收待补 | -| 题库导出 PDF/Word/JSON | 部分覆盖 | 服务端 JSON、`paper_json` 和打印 payload 导出基础已补,含租户内容编辑权限、跨租户拒绝、答案/解析开关、子题脱敏、导出 job 和审计;PDF/Word 二进制、水印、发布到资料下载和导出 worker 待补 | +| 题库导出 PDF/Word/JSON | 部分覆盖 | 服务端 JSON、`paper_json`、打印 payload、PDF 和 Word 异步导出已补,含租户内容编辑权限、跨租户拒绝、答案/解析开关、子题脱敏、导出 job、审计、水印、`content_assets` 发布/下载路径和 exports worker;后续补更精细试卷模板、每日一练素材导出和后台操作台体验 | | 题型分组/模拟卷配置 | 部分覆盖 | question_type_groups 表和 blueprint 有基础;后台配置体验待补 | | 背单词维护 | 已覆盖 | 单元/单词 CRUD 和导入已有 | | 知识手册维护 | 已覆盖 | subject/chapter/entry CRUD 和导入已有 | @@ -98,7 +98,7 @@ 1. 阅读理解/案例分析多小题:后端 `subAnswers` 判分、断点恢复、报告 `subResults` 和 Taro 作答 UI 第一版已完成;仍需优化长题干排版、公式图片混排和复盘体验。 2. 排行榜增强:刷题、模考、背单词、积分排行榜主接口已有;还需防刷、日/周榜预聚合、运营后台排名看板。 3. 账号设置完整流:绑定/更换手机号基础 API 已完成;仍缺头像上传、微信/QQ 账号合并、密码/邮箱能力。 -4. 题库导出:服务端 JSON/试卷 payload 导出、权限审计和答案脱敏已补;仍缺 PDF/Word 二进制生成、水印、资料发布和后台导出操作台。 +4. 题库导出:服务端 JSON/试卷 payload、PDF/Word 二进制生成、水印、资料发布路径、权限审计和答案脱敏已补;仍缺更精细试卷模板、每日一练素材导出和后台导出操作台体验。 5. 导入扩展:题目/单词/知识手册/分数线/视频已支持 JSON、CSV 和 Excel 预览导入,并可用 `executionMode=async` 进入 imports worker;导入后复检、导入任务详情、模板下载按钮、字段映射 API、Taro 字段别名编辑、异步轮询和 PocketBase JSON dry-run 报告已补,仍缺真实数据执行验收。 6. 公共题库商业化:平台公共/地区题库授权、租户快照采纳、手动同步、自动同步 worker、同步通知、冲突查询、租户自改冲突保护和单条/批量冲突处理已完成基础闭环;还需生产定时调度、失败告警和更完整运营后台消息。 7. CRM/销售结算:CRM worker、分佣规则、结算单、审核和打款状态基础闭环已完成;仍缺轮询/定向分配、打款导出、凭证和销售结算看板。 @@ -127,4 +127,4 @@ 1. 主题模板、素材库、主题预览/发布。 2. 班级、教师、学生分组和学习督导。 3. AI 择校推荐和 PDF 报告。 -4. 题库导出二进制生成、试卷排版 worker、每日一练运营工具。 +4. 题库导出模板精排、每日一练运营工具。 diff --git a/docs/refactor/next-development-todo.md b/docs/refactor/next-development-todo.md index 9e4771b5..449bb94e 100644 --- a/docs/refactor/next-development-todo.md +++ b/docs/refactor/next-development-todo.md @@ -27,6 +27,7 @@ - 租户后台数据看板已完成首版聚合 API:`GET /api/tenant-admin/dashboard`,支持租户/地区维度的收益、注册、学习、内容、激活码、反馈、趋势、24h 活跃、套餐销量和运营动态,前端可直接联调。 - 支付/退款补偿 worker 已完成:`apps/worker --job commerce` 可查询微信/支付宝支付和处理中退款,补偿漏通知订单,支付成功幂等开通权益,退款成功幂等更新退款/订单/支付并在全额退款时撤销订单权益。 - 内容资源复检 worker 已完成:`apps/worker --job assets` 可复检 `content_assets` 中的托管对象元数据,正常资源写回复检证据,异常资源自动置为 `failed + draft` 并写入审计和安全标记。 +- 题库导出 worker 已完成:`apps/worker --job exports` 可抢占 `pdf/docx` 导出任务,渲染 PDF/Word、水印、写入对象存储或本地开发存储,创建 `content_assets` 并回填 `assetId/hash/size`。 - 本地验证:`npm run check:refactor` 已通过。 当前更适合进入前端联调前阅读的总览文档: @@ -83,8 +84,8 @@ 4. 题库导出增强 - 已完成服务端 JSON、`paper_json`、`print_payload` 基础导出和导出 job 台账。 - - 当前前端可先下载 inline base64 JSON,或把 `paper_json` 交给后台打印预览组件渲染。 - - 继续补 PDF/Word 二进制生成 worker、页面水印、导出发布到资料下载、导出模板和操作台。 + - 已完成 PDF/Word 二进制生成 worker、页面水印、导出发布到 `content_assets`、后台资源签名下载/预览路径。 + - 继续补更精细的试卷模板、每日一练素材导出、导出操作台和多模板排版。 5. 公共题库和租户授权 - 已完成平台公共题库/地区题库的基础授权、租户采纳、题目快照复制和手动同步。 @@ -219,5 +220,5 @@ 3. 补平台后台增强:租户详情/编辑、平台审计报表、自动计费、账单批量操作和更细平台权限点。 4. 云服务器部署 Supabase/PostgreSQL 和 API,配置对象存储生产环境变量,跑 `check:refactor` 的远程等价测试。 5. 导出现有 PocketBase 数据,做完整 dry-run 迁移。 -6. 并行补对象存储、真实登录、完整资金流水对账、题库导出 PDF/Word worker、公共题库生产定时调度和失败告警。 +6. 并行补对象存储、真实登录、完整资金流水对账、题库导出模板精排/每日一练、公共题库生产定时调度和失败告警。 7. 前后端联调通过后,再做支付、权限、数据导入、资料下载、视频播放的商用验收。 diff --git a/docs/refactor/object-storage.md b/docs/refactor/object-storage.md index d0dfd089..9fb6d49c 100644 --- a/docs/refactor/object-storage.md +++ b/docs/refactor/object-storage.md @@ -70,6 +70,12 @@ POST /api/tenant-content/assets/sign-preview npm --workspace @tiku-saas/worker run assets:once ``` +题库 PDF/Word 导出 worker: + +```bash +npm --workspace @tiku-saas/worker run exports:once +``` + ## 标准上传流程 后台前端上传 PDF、图片、视频或资料包时必须走下面流程: @@ -83,6 +89,19 @@ npm --workspace @tiku-saas/worker run assets:once 7. 生产环境定时运行 assets worker,复检 `pending/verified` 托管对象的大小、MIME、SHA-256 等元数据。 8. 如果复检发现对象丢失、跨租户 objectKey、大小/MIME/checksum 不一致,worker 会把资源置为 `uploadStatus=failed`,并将 `active` 资源退回 `draft`,同时写入 `security_flags.assetRecheckFailed=true` 和 `audit_logs`。 +## 题库导出生成资源 + +题库导出 PDF/Word 不走前端上传,而是由后端和 worker 自动进入资源台账: + +1. 租户后台调用 `POST /api/tenant-content/exports/questions`,`format=pdf` 或 `format=docx`。 +2. API 校验租户内容权限、导出范围和答案/解析开关,创建 `content_export_jobs.status=pending`。 +3. `apps/worker --job exports` 抢占 pending job,复用后端导出 payload 规则渲染 PDF/Word,应用 `options.watermarkText` 水印。 +4. worker 将二进制文件写入配置的对象存储。`local_dev` 会写入 `EXPORT_LOCAL_STORAGE_ROOT`,生产建议使用 `supabase_storage`、`aliyun_oss` 或 `tencent_cos`。 +5. worker 创建 `content_assets`,设置 `uploadStatus=verified`,并把 `assetId`、文件名、大小和 SHA-256 回填到 `content_export_jobs.output_metadata`。 +6. 前端轮询 `GET /api/tenant-content/exports/jobs`,拿到 `assetId` 后通过 `POST /api/tenant-content/assets/sign-download` 或 `sign-preview` 下载/预览。 + +注意:PDF/Word 文件本身不直接存入数据库,数据库只保存 job、资源台账和校验 metadata。 + 确认上传示例: ```json diff --git a/docs/refactor/taro-frontend-integration.md b/docs/refactor/taro-frontend-integration.md index ab624671..5348ae60 100644 --- a/docs/refactor/taro-frontend-integration.md +++ b/docs/refactor/taro-frontend-integration.md @@ -1015,7 +1015,10 @@ failed:导入结果和目标表不一致,必须提示管理员排查,不 ## 题库导出对接 -租户后台题库导出统一走后端生成结构化 payload,前端不要直接查 Supabase 表拼导出文件。当前后端已支持 JSON、`paper_json` 和 `print_payload` 三类基础导出,适合先做后台“导出 JSON/试卷预览”功能;PDF/Word 二进制、水印和发布到资料下载后续由 worker 增强。 +租户后台题库导出统一走后端生成结构化 payload,前端不要直接查 Supabase 表拼导出文件。当前后端支持两种交付形态: + +- 同步结构化导出:`json`、`paper_json`、`print_payload`,接口直接返回 base64 JSON/payload。 +- 异步二进制导出:`pdf`、`docx`,接口先返回 `pending` job,由 `apps/worker --job exports` 渲染 PDF/Word、水印并写入 `content_assets`,前端轮询 job 后再走资源签名下载或预览。 可用接口: @@ -1066,6 +1069,27 @@ GET /api/tenant-content/exports/jobs } ``` +PDF/Word 异步导出: + +```json +{ + "scopeType": "collection", + "scopeId": "", + "format": "pdf", + "exportType": "paper", + "includeAnswers": false, + "includeExplanations": false, + "options": { + "title": "天津专升本模拟试卷", + "watermarkText": "仅供内部使用", + "publishToAssets": true, + "assetVisibility": "tenant" + } +} +``` + +`format` 可为 `pdf` 或 `docx`。`publishToAssets=true` 表示导出文件可作为资料资源展示给对应可见范围用户;不传时默认生成后台私有资源,仅后台可下载。`assetVisibility` 支持 `public`、`tenant`、`members`、`svip`、`private`,生产默认建议用 `tenant/private/svip`,不要轻易公开带题目的文件。 + 响应关键结构: ```json @@ -1102,14 +1126,54 @@ GET /api/tenant-content/exports/jobs } ``` +异步二进制导出初始响应: + +```json +{ + "job": { + "id": "...", + "status": "pending", + "questionCount": 20, + "outputHash": "..." + }, + "export": null +} +``` + +worker 完成后,`GET /api/tenant-content/exports/jobs` 返回: + +```json +{ + "items": [ + { + "id": "...", + "format": "pdf", + "status": "completed", + "assetId": "...", + "attemptCount": 1, + "outputMetadata": { + "delivery": "content_asset", + "fileName": "天津专升本模拟试卷.pdf", + "mimeType": "application/pdf", + "sizeBytes": 123456, + "checksumSha256": "...", + "assetId": "..." + } + } + ] +} +``` + 前端处理规则: - 导出按钮只给具备租户内容编辑权限的后台成员展示;接口仍以后端 `TENANT_CONTENT_EDITOR_REQUIRED` 为准。 - 下载 JSON 时使用 `files[0].contentBase64` 生成 Blob,文件名使用后端返回的 `filename`。 - `includeAnswers=false` 时,顶层答案字段和阅读理解/案例分析的子题答案都会被后端脱敏;前端不要在本地重新合并答案。 - `includeExplanations=false` 时,不展示解析,也不要从题目详情接口额外补解析。 -- `paper_json` 可先用于后台试卷预览和打印;正式 PDF/Word 导出等后端 worker 完成后再接二进制文件下载。 -- `GET /api/tenant-content/exports/jobs?scopeType=collection&scopeId=...` 用于后台导出历史;当前记录 metadata 和输出 hash,不长期保存完整导出内容。 +- `paper_json` 可用于后台试卷预览和打印;`pdf/docx` 用于正式文件下载、资料发布和带水印留档。 +- `GET /api/tenant-content/exports/jobs?scopeType=collection&scopeId=...` 用于后台导出历史;结构化导出只记录 metadata 和输出 hash,二进制导出记录 `assetId`、文件名、大小和 checksum。 +- `status=pending/rendering` 时展示生成中;`completed` 且存在 `assetId` 后,后台可调用 `POST /api/tenant-content/assets/sign-download` 下载,PDF 也可调用 `POST /api/tenant-content/assets/sign-preview` 预览。 +- 学生资料页只展示 `content_assets.status=active` 且非 `private` 的资源;后台私有导出不会出现在学生资料列表。 - 跨租户导出会返回 404 或 403,前端不要重试其它租户 ID。 ## 公共题库采纳对接 diff --git a/package-lock.json b/package-lock.json index a013a48b..d6e4e132 100644 --- a/package-lock.json +++ b/package-lock.json @@ -86,10 +86,13 @@ "dependencies": { "@supabase/storage-js": "^2.108.2", "ali-oss": "^6.23.0", + "docx": "^9.7.1", + "pdfkit": "^0.19.1", "pg": "^8.16.3" }, "devDependencies": { "@types/node": "^24.0.4", + "@types/pdfkit": "^0.17.6", "@types/pg": "^8.15.4", "tsx": "^4.20.3", "typescript": "^5.8.3" @@ -2895,6 +2898,30 @@ "dev": true, "license": "MIT" }, + "node_modules/@noble/ciphers": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-1.3.0.tgz", + "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -3777,6 +3804,16 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/@swc/helpers": { + "version": "0.5.23", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.23.tgz", + "integrity": "sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "tslib": "^2.8.0" + } + }, "node_modules/@swc/register": { "version": "0.1.10", "resolved": "https://registry.npmjs.org/@swc/register/-/register-0.1.10.tgz", @@ -4981,7 +5018,6 @@ "version": "25.6.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz", "integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==", - "dev": true, "license": "MIT", "peer": true, "dependencies": { @@ -4998,6 +5034,16 @@ "@types/node": "*" } }, + "node_modules/@types/pdfkit": { + "version": "0.17.6", + "resolved": "https://registry.npmjs.org/@types/pdfkit/-/pdfkit-0.17.6.tgz", + "integrity": "sha512-tIwzxk2uWKp0Cq9JIluQXJid77lYhF52EsIOwhsMF4iWLA6YneoBR1xVKYYdAysHuepUB0OX4tdwMiUDdGKmig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/pg": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.20.0.tgz", @@ -6312,6 +6358,24 @@ "node": ">=8" } }, + "node_modules/brotli": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz", + "integrity": "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.1.2" + } + }, + "node_modules/browserify-zlib": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz", + "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==", + "license": "MIT", + "dependencies": { + "pako": "~1.0.5" + } + }, "node_modules/browserslist": { "version": "4.28.4", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.4.tgz", @@ -8211,6 +8275,12 @@ "node": ">= 4.0.0" } }, + "node_modules/dfa": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/dfa/-/dfa-1.2.0.tgz", + "integrity": "sha512-ED3jP8saaweFTjeGX8HQPjeC1YYyZs98jGNZx6IiBvxW7JG5v492kamAQB3m2wop07CvU/RQmzcKr6bgcC5D/Q==", + "license": "MIT" + }, "node_modules/digest-header": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/digest-header/-/digest-header-1.1.0.tgz", @@ -8256,6 +8326,41 @@ "node": ">=6.0.0" } }, + "node_modules/docx": { + "version": "9.7.1", + "resolved": "https://registry.npmjs.org/docx/-/docx-9.7.1.tgz", + "integrity": "sha512-ilXFf9Moz47ABjFpDiA5s1w9lpb4EFSp7+5iiJSbfyYDM+bpZdAgLlSr7fW4aXhVe/E+F6QCv0EvRVFEd5CsWg==", + "license": "MIT", + "dependencies": { + "@types/node": "^25.2.3", + "hash.js": "^1.1.7", + "jszip": "^3.10.1", + "nanoid": "^5.1.3", + "xml": "^1.0.1", + "xml-js": "^1.6.8" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/docx/node_modules/nanoid": { + "version": "5.1.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.16.tgz", + "integrity": "sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.js" + }, + "engines": { + "node": "^18 || >=20" + } + }, "node_modules/dom-converter": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/dom-converter/-/dom-converter-0.2.0.tgz", @@ -9616,7 +9721,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, "license": "MIT" }, "node_modules/fast-glob": { @@ -9958,6 +10062,32 @@ } } }, + "node_modules/fontkit": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/fontkit/-/fontkit-2.0.4.tgz", + "integrity": "sha512-syetQadaUEDNdxdugga9CpEYVaQIxOwk7GlwZWWZ19//qW4zE5bknOKeMBDYAASwnpaSHKJITRLMF9m1fp3s6g==", + "license": "MIT", + "dependencies": { + "@swc/helpers": "^0.5.12", + "brotli": "^1.3.2", + "clone": "^2.1.2", + "dfa": "^1.2.0", + "fast-deep-equal": "^3.1.3", + "restructure": "^3.0.0", + "tiny-inflate": "^1.0.3", + "unicode-properties": "^1.4.0", + "unicode-trie": "^2.0.0" + } + }, + "node_modules/fontkit/node_modules/clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, "node_modules/for-each": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", @@ -10531,6 +10661,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/hash.js": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "minimalistic-assert": "^1.0.1" + } + }, "node_modules/hasown": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", @@ -11663,6 +11803,12 @@ "integrity": "sha512-pZe//GGmwJndub7ZghVHz7vjb2LgC1m8B07Au3eYqeqv9emhESByMXxaEgkUkEqJe87oBbSniGYoQNIBklc7IQ==", "license": "BSD-3-Clause" }, + "node_modules/js-md5": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/js-md5/-/js-md5-0.8.3.tgz", + "integrity": "sha512-qR0HB5uP6wCuRMrWPTrkMaev7MJZwJuuw4fnwAzRgP4J4/F8RwtodOKpGp4XpqsLBFzzgqIO42efFAyz2Et6KQ==", + "license": "MIT" + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -12336,6 +12482,25 @@ "url": "https://github.com/sponsors/antonk52" } }, + "node_modules/linebreak": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/linebreak/-/linebreak-1.1.0.tgz", + "integrity": "sha512-MHp03UImeVhB7XZtjd0E4n6+3xr5Dq/9xI/5FptGk5FrbDR3zagPa2DS6U8ks/3HjbKWG9Q1M2ufOzxV2qLYSQ==", + "license": "MIT", + "dependencies": { + "base64-js": "0.0.8", + "unicode-trie": "^2.0.0" + } + }, + "node_modules/linebreak/node_modules/base64-js": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-0.0.8.tgz", + "integrity": "sha512-3XSA2cR/h/73EzlXXdU6YNycmYI7+kicTxks4eJg2g39biHR84slg2+des+p7iHYhbRg/udIS4TD53WabcOUkw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", @@ -12876,7 +13041,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", - "dev": true, "license": "ISC" }, "node_modules/minimatch": { @@ -13997,6 +14161,20 @@ "through": "~2.3" } }, + "node_modules/pdfkit": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/pdfkit/-/pdfkit-0.19.1.tgz", + "integrity": "sha512-6Gzk+wDwTs4VSxsR5rCMTnIl5nlmkye1oWB0l2hDB1EX6ZNSIBroKQEv+2+fPPn+stVjyqzmsqRJVDfB9fo5DA==", + "license": "MIT", + "dependencies": { + "@noble/ciphers": "^1.0.0", + "@noble/hashes": "^1.6.0", + "fontkit": "^2.0.4", + "js-md5": "^0.8.3", + "linebreak": "^1.1.0", + "png-js": "^1.1.0" + } + }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -14242,6 +14420,14 @@ "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", "license": "MIT" }, + "node_modules/png-js": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/png-js/-/png-js-1.1.0.tgz", + "integrity": "sha512-PM/uYGzGdNSzqeOgly68+6wKQDL1SY0a/N+OEa/+br6LnHWOAJB0Npiamnodfq3jd2LS/i2fMeOKSAILjA+m5Q==", + "dependencies": { + "browserify-zlib": "^0.2.0" + } + }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", @@ -15772,6 +15958,12 @@ "node": ">=8" } }, + "node_modules/restructure": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/restructure/-/restructure-3.0.2.tgz", + "integrity": "sha512-gSfoiOEA0VPE6Tukkrr7I0RBdE0s7H1eFCDBk05l1KIQT1UIKNc5JZy6jdyW6eYH3aR3g5b3PuL77rq0hvwtAw==", + "license": "MIT" + }, "node_modules/retry": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", @@ -17637,6 +17829,12 @@ "dev": true, "license": "MIT" }, + "node_modules/tiny-inflate": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tiny-inflate/-/tiny-inflate-1.0.3.tgz", + "integrity": "sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw==", + "license": "MIT" + }, "node_modules/tmp": { "version": "0.2.7", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", @@ -18353,7 +18551,6 @@ "version": "7.19.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz", "integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==", - "dev": true, "license": "MIT" }, "node_modules/unescape": { @@ -18412,6 +18609,16 @@ "node": ">=4" } }, + "node_modules/unicode-properties": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/unicode-properties/-/unicode-properties-1.4.1.tgz", + "integrity": "sha512-CLjCCLQ6UuMxWnbIylkisbRj31qxHPAurvena/0iwSVbQ2G1VY5/HjV0IRabOEbDHlzZlRdCrD4NhB0JtU40Pg==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.0", + "unicode-trie": "^2.0.0" + } + }, "node_modules/unicode-property-aliases-ecmascript": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.2.0.tgz", @@ -18422,6 +18629,22 @@ "node": ">=4" } }, + "node_modules/unicode-trie": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-trie/-/unicode-trie-2.0.0.tgz", + "integrity": "sha512-x7bc76x0bm4prf1VLg79uhAzKw8DVboClSN5VxJuQ+LKDOVEW9CdH+VY7SP+vX7xCYQqzzgQpFqz15zeLvAtZQ==", + "license": "MIT", + "dependencies": { + "pako": "^0.2.5", + "tiny-inflate": "^1.0.0" + } + }, + "node_modules/unicode-trie/node_modules/pako": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", + "integrity": "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==", + "license": "MIT" + }, "node_modules/unicorn-magic": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.3.0.tgz", @@ -19363,6 +19586,24 @@ } } }, + "node_modules/xml": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", + "integrity": "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==", + "license": "MIT" + }, + "node_modules/xml-js": { + "version": "1.6.11", + "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz", + "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==", + "license": "MIT", + "dependencies": { + "sax": "^1.2.4" + }, + "bin": { + "xml-js": "bin/cli.js" + } + }, "node_modules/xml-name-validator": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", diff --git a/package.json b/package.json index 8a84222e..d7aed3e4 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "test:worker:crm": "npm run db:smoke-seed && npm run build:worker && node scripts/crm-worker-integration-test.js", "test:worker:commerce": "npm run db:smoke-seed && npm run build:worker && node scripts/commerce-worker-integration-test.js", "test:worker:assets": "npm run db:smoke-seed && npm run build:worker && node scripts/asset-worker-integration-test.js", + "test:worker:exports": "npm run db:smoke-seed && npm run build:worker && node scripts/export-worker-integration-test.js", "test:worker:imports": "npm run db:smoke-seed && npm run build:worker && node scripts/import-worker-integration-test.js", "test:worker:public-banks": "npm run db:smoke-seed && npm run build:worker && node scripts/public-bank-worker-integration-test.js", "test:readiness": "node scripts/production-readiness-check-test.js", diff --git a/scripts/api-integration-test.js b/scripts/api-integration-test.js index e39175be..c0dbf517 100644 --- a/scripts/api-integration-test.js +++ b/scripts/api-integration-test.js @@ -2814,6 +2814,45 @@ async function testTenantContentAdmin() { const exportedQuestionWithAnswer = paperExport.export?.questions?.find(item => item.id === question.item.id); assert.equal(exportedQuestionWithAnswer?.answerText, '4', 'paper export should include answers when requested'); + const pdfExportJob = await request('/api/tenant-content/exports/questions', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { + scopeType: 'collection', + scopeId: collection.item.id, + format: 'pdf', + exportType: 'paper', + includeAnswers: false, + includeExplanations: false, + options: { + title: '集成测试 PDF 异步导出', + watermarkText: '仅供内部测试', + publishToAssets: true, + assetVisibility: 'tenant', + }, + }, + }); + assert.equal(pdfExportJob.job?.status, 'pending', 'PDF export should create a pending async export job'); + assert.equal(pdfExportJob.export, null, 'PDF export should not return inline binary payload'); + + const docxExportJob = await request('/api/tenant-content/exports/questions', { + userId: TENANT_ADMIN_USER_ID, + method: 'POST', + body: { + scopeType: 'collection', + scopeId: collection.item.id, + format: 'docx', + exportType: 'paper', + includeAnswers: true, + includeExplanations: true, + options: { + title: '集成测试 Word 异步导出', + watermarkText: '仅供内部测试', + }, + }, + }); + assert.equal(docxExportJob.job?.status, 'pending', 'DOCX export should create a pending async export job'); + const partnerExportDenied = await request('/api/tenant-content/exports/questions', { tenantId: PARTNER_TENANT_ID, userId: PARTNER_TENANT_ADMIN_USER_ID, @@ -2831,6 +2870,10 @@ async function testTenantContentAdmin() { exportJobs.items?.some(item => item.id === exportNoAnswers.job.id && item.questionCount === 1), 'question export jobs should list completed export audit records', ); + const listedPdfJob = exportJobs.items?.find(item => item.id === pdfExportJob.job.id); + assert.equal(listedPdfJob?.status, 'pending', 'question export jobs should expose pending binary export jobs'); + assert.equal(listedPdfJob?.assetId, null, 'pending binary export job should not have an asset yet'); + assert.equal(listedPdfJob?.attemptCount, 0, 'pending binary export job should expose worker attempt count'); const invalidExportJobsScope = await request('/api/tenant-content/exports/jobs', { userId: TENANT_ADMIN_USER_ID, query: { scopeType: 'not_supported' }, diff --git a/scripts/export-worker-integration-test.js b/scripts/export-worker-integration-test.js new file mode 100644 index 00000000..0a668fdb --- /dev/null +++ b/scripts/export-worker-integration-test.js @@ -0,0 +1,249 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import pg from 'pg'; +import { spawn } from 'node:child_process'; +import JSZip from 'jszip'; + +const databaseUrl = process.env.DATABASE_URL || 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; +const tenantId = '00000000-0000-0000-0000-000000000001'; +const tenantAdminUserId = '00000000-0000-0000-0000-000000000102'; +const collectionId = '00000000-0000-0000-0000-000000000615'; +const localStorageRoot = '.local-storage-test'; + +const ids = { + pdfJob: '21000000-0000-0000-0000-000000000901', + docxJob: '21000000-0000-0000-0000-000000000902', + noAnswerDocxJob: '21000000-0000-0000-0000-000000000903', +}; + +const jobIds = [ids.pdfJob, ids.docxJob, ids.noAnswerDocxJob]; + +async function runWorkerOnce() { + const child = spawn(process.execPath, ['apps/worker/dist/apps/worker/src/index.js', '--once', '--job', 'exports'], { + cwd: process.cwd(), + env: { + ...process.env, + DATABASE_URL: databaseUrl, + WORKER_EXPORT_BATCH_SIZE: '20', + WORKER_EXPORT_ID: 'export-worker-test', + STORAGE_DEFAULT_PROVIDER: 'local_dev', + STORAGE_DEFAULT_BUCKET: 'tenant-assets', + EXPORT_LOCAL_STORAGE_ROOT: localStorageRoot, + STORAGE_REQUIRE_TENANT_PREFIX: 'true', + }, + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + let output = ''; + child.stdout.on('data', chunk => { + output += chunk.toString(); + }); + child.stderr.on('data', chunk => { + output += chunk.toString(); + }); + const code = await new Promise(resolve => child.on('exit', resolve)); + assert.equal(code, 0, `worker should exit 0\n${output}`); + assert.match(output, /exports batch processed=\d+/, 'worker output should include exports summary'); + return output; +} + +async function cleanup(pool) { + await pool.query( + ` + delete from public.audit_logs + where tenant_id = $1 + and ( + (target_type = 'content_export_job' and target_id = any($2::text[])) + or (target_type = 'content_asset' and details->>'exportJobId' = any($2::text[])) + ) + `, + [tenantId, jobIds], + ); + await pool.query( + ` + delete from public.content_assets + where tenant_id = $1 + and metadata->>'exportJobId' = any($2::text[]) + `, + [tenantId, jobIds], + ); + await pool.query( + ` + delete from public.content_export_jobs + where tenant_id = $1 and id = any($2::uuid[]) + `, + [tenantId, jobIds], + ); + await fs.rm(path.resolve(process.cwd(), localStorageRoot), { recursive: true, force: true }); +} + +async function seed(pool) { + await pool.query( + ` + insert into public.content_export_jobs ( + id, tenant_id, created_by, export_type, format, scope_type, scope_id, + status, include_answers, include_explanations, include_video_refs, + question_count, summary, options, output_metadata, created_at, updated_at + ) + values + ( + $1, $4, $5, 'paper', 'pdf', 'collection', $6, + 'pending', false, false, false, + 0, '{"source":"export-worker-test"}'::jsonb, + '{"title":"导出测试无答案试卷","watermarkText":"内部水印","publishToAssets":true,"assetVisibility":"tenant","limit":50}'::jsonb, + '{"delivery":"content_asset_pending","maxQuestionLimit":50}'::jsonb, + now() - interval '1 minute', now() - interval '1 minute' + ), + ( + $2, $4, $5, 'paper', 'docx', 'collection', $6, + 'pending', true, true, false, + 0, '{"source":"export-worker-test"}'::jsonb, + '{"title":"导出测试含答案文档","watermarkText":"内部水印","publishToAssets":false,"assetVisibility":"private","limit":50}'::jsonb, + '{"delivery":"content_asset_pending","maxQuestionLimit":50}'::jsonb, + now() - interval '1 minute', now() - interval '1 minute' + ), + ( + $3, $4, $5, 'paper', 'docx', 'collection', $6, + 'pending', false, false, false, + 0, '{"source":"export-worker-test"}'::jsonb, + '{"title":"导出测试无答案文档","watermarkText":"内部水印","publishToAssets":false,"assetVisibility":"private","limit":50}'::jsonb, + '{"delivery":"content_asset_pending","maxQuestionLimit":50}'::jsonb, + now() - interval '1 minute', now() - interval '1 minute' + ) + `, + [ids.pdfJob, ids.docxJob, ids.noAnswerDocxJob, tenantId, tenantAdminUserId, collectionId], + ); +} + +function countFromWorkerOutput(output, name) { + const match = output.match(new RegExp(`${name}=(\\d+)`)); + return match ? Number(match[1]) : 0; +} + +async function generatedFileBody(asset) { + const fullPath = path.resolve(process.cwd(), localStorageRoot, asset.bucket, asset.object_key); + return fs.readFile(fullPath); +} + +async function assertGeneratedFile(asset) { + const body = await generatedFileBody(asset); + assert.equal(body.length, Number(asset.file_size_bytes), 'generated file size should match content_assets'); + assert.ok(body.length > 500, 'generated binary export should not be empty'); + if (asset.mime_type === 'application/pdf') { + assert.equal(body.subarray(0, 4).toString('utf8'), '%PDF', 'PDF export should have PDF header'); + } else { + assert.equal(body.subarray(0, 2).toString('hex'), '504b', 'DOCX export should be a zipped Office file'); + } +} + +async function docxText(asset) { + const zip = await JSZip.loadAsync(await generatedFileBody(asset)); + const xml = await zip.file('word/document.xml')?.async('string'); + assert.ok(xml, 'DOCX should contain word/document.xml'); + return xml + .replace(/<[^>]+>/g, '') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/&/g, '&'); +} + +async function main() { + const pool = new pg.Pool({ connectionString: databaseUrl }); + let seeded = false; + try { + await pool.query('begin'); + await cleanup(pool); + await seed(pool); + await pool.query('commit'); + seeded = true; + + const output = await runWorkerOnce(); + assert.ok(countFromWorkerOutput(output, 'completed') >= 3, 'worker should complete all binary export jobs'); + + const jobs = await pool.query( + ` + select id, status, asset_id, question_count, output_metadata + from public.content_export_jobs + where tenant_id = $1 and id = any($2::uuid[]) + order by id + `, + [tenantId, jobIds], + ); + assert.equal(jobs.rowCount, 3, 'all export jobs should still exist'); + for (const job of jobs.rows) { + assert.equal(job.status, 'completed', 'export job should be completed'); + assert.ok(job.asset_id, 'completed export job should reference a content asset'); + assert.equal(job.output_metadata?.delivery, 'content_asset', 'export job should switch delivery to content_asset'); + assert.ok(job.output_metadata?.checksumSha256, 'export job should record output checksum'); + assert.ok(Number(job.question_count) >= 1, 'export job should record question count'); + } + + const assets = await pool.query( + ` + select id, asset_type, title, file_name, storage_provider, bucket, + object_key, mime_type, file_size_bytes, checksum_sha256, + upload_status, visibility, status, metadata + from public.content_assets + where tenant_id = $1 and metadata->>'exportJobId' = any($2::text[]) + order by file_name + `, + [tenantId, jobIds], + ); + assert.equal(assets.rowCount, 3, 'worker should create three export assets'); + const pdfAsset = assets.rows.find(asset => asset.metadata?.exportJobId === ids.pdfJob); + const docxAsset = assets.rows.find(asset => asset.metadata?.exportJobId === ids.docxJob); + const noAnswerDocxAsset = assets.rows.find(asset => asset.metadata?.exportJobId === ids.noAnswerDocxJob); + assert.ok(pdfAsset, 'PDF export should create a PDF content asset'); + assert.ok(docxAsset, 'DOCX export should create a Word content asset'); + assert.ok(noAnswerDocxAsset, 'no-answer DOCX export should create a Word content asset'); + assert.equal(pdfAsset.mime_type, 'application/pdf', 'PDF asset should use application/pdf'); + assert.ok(docxAsset.mime_type?.includes('wordprocessingml'), 'DOCX asset should use Word MIME type'); + assert.equal(pdfAsset.upload_status, 'verified', 'PDF asset should be verified'); + assert.equal(docxAsset.upload_status, 'verified', 'DOCX asset should be verified'); + assert.equal(noAnswerDocxAsset.upload_status, 'verified', 'no-answer DOCX asset should be verified'); + assert.equal(pdfAsset.visibility, 'tenant', 'published PDF export should use requested asset visibility'); + assert.equal(docxAsset.visibility, 'private', 'non-published DOCX export should remain private'); + assert.equal(noAnswerDocxAsset.visibility, 'private', 'no-answer DOCX export should remain private'); + await assertGeneratedFile(pdfAsset); + await assertGeneratedFile(docxAsset); + await assertGeneratedFile(noAnswerDocxAsset); + + const answeredDocxText = await docxText(docxAsset); + assert.match(answeredDocxText, /答案/, 'DOCX with answers should include answer labels'); + assert.match(answeredDocxText, /后端需要统一校验租户/, 'DOCX with answers should include subjective answer text'); + const noAnswerDocxText = await docxText(noAnswerDocxAsset); + assert.doesNotMatch(noAnswerDocxText, /后端需要统一校验租户/, 'DOCX without answers should not leak subjective answer text'); + assert.doesNotMatch(noAnswerDocxText, /基础加法。/, 'DOCX without explanations should not leak explanations'); + + const audits = await pool.query( + ` + select action, details + from public.audit_logs + where tenant_id = $1 + and target_type = 'content_export_job' + and target_id = any($2::text[]) + `, + [tenantId, jobIds], + ); + assert.ok( + audits.rows.some(row => row.action === 'content.questions.export_rendered' && row.details?.assetId), + 'worker should write rendered audit log', + ); + + console.log('Export worker integration test complete.'); + } catch (error) { + await pool.query('rollback').catch(() => {}); + throw error; + } finally { + if (seeded) { + await cleanup(pool).catch(() => {}); + } + await pool.end(); + } +} + +main().catch(error => { + console.error(error); + process.exit(1); +}); diff --git a/supabase/migrations/202606290018_content_export_binary_worker.sql b/supabase/migrations/202606290018_content_export_binary_worker.sql new file mode 100644 index 00000000..cd0f9447 --- /dev/null +++ b/supabase/migrations/202606290018_content_export_binary_worker.sql @@ -0,0 +1,40 @@ +alter table public.content_export_jobs + drop constraint if exists content_export_jobs_format_check, + drop constraint if exists content_export_jobs_status_check; + +alter table public.content_export_jobs + add constraint content_export_jobs_format_check + check (format in ('json', 'paper_json', 'print_payload', 'pdf', 'docx')); + +alter table public.content_export_jobs + add constraint content_export_jobs_status_check + check (status in ('pending', 'rendering', 'completed', 'failed')); + +alter table public.content_export_jobs + alter column status set default 'pending'; + +alter table public.content_export_jobs + add column if not exists asset_id uuid references public.content_assets(id) on delete set null, + add column if not exists locked_at timestamptz, + add column if not exists locked_by text, + add column if not exists attempt_count integer not null default 0, + add column if not exists max_attempts integer not null default 3, + add column if not exists next_attempt_at timestamptz, + add column if not exists started_at timestamptz; + +do $$ +begin + if not exists (select 1 from pg_constraint where conname = 'content_export_jobs_attempt_count_check') then + alter table public.content_export_jobs + add constraint content_export_jobs_attempt_count_check + check (attempt_count >= 0 and max_attempts > 0 and attempt_count <= max_attempts); + end if; +end $$; + +create index if not exists idx_content_export_jobs_worker_queue + on public.content_export_jobs(status, next_attempt_at, created_at) + where status = 'pending'; + +create index if not exists idx_content_export_jobs_asset + on public.content_export_jobs(tenant_id, asset_id) + where asset_id is not null;