forked from wangziqi/gongxue-base
feat: render question exports as assets
This commit is contained in:
Codex
@@ -12,15 +12,19 @@
|
||||
"commerce:once": "tsx src/index.ts --once --job commerce",
|
||||
"assets:once": "tsx src/index.ts --once --job assets",
|
||||
"imports:once": "tsx src/index.ts --once --job imports",
|
||||
"public-banks:once": "tsx src/index.ts --once --job public-banks"
|
||||
"public-banks:once": "tsx src/index.ts --once --job public-banks",
|
||||
"exports:once": "tsx src/index.ts --once --job exports"
|
||||
},
|
||||
"dependencies": {
|
||||
"@supabase/storage-js": "^2.108.2",
|
||||
"ali-oss": "^6.23.0",
|
||||
"docx": "^9.7.1",
|
||||
"pdfkit": "^0.19.1",
|
||||
"pg": "^8.16.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.0.4",
|
||||
"@types/pdfkit": "^0.17.6",
|
||||
"@types/pg": "^8.15.4",
|
||||
"tsx": "^4.20.3",
|
||||
"typescript": "^5.8.3"
|
||||
|
||||
@@ -24,6 +24,14 @@ export interface WorkerConfig {
|
||||
publicBankSyncCopyLimit: number;
|
||||
publicBankSyncWorkerId: string;
|
||||
publicBankSyncClaimStaleSeconds: number;
|
||||
exportBatchSize: number;
|
||||
exportWorkerId: string;
|
||||
exportBackoffSeconds: number[];
|
||||
exportLocalStorageRoot: string;
|
||||
exportPdfFontPath: string;
|
||||
storageDefaultProvider: string;
|
||||
storageDefaultBucket: string;
|
||||
storagePublicBaseUrl: string;
|
||||
storageMaxUploadBytes: number;
|
||||
storageAllowedMimePrefixes: string[];
|
||||
storageAllowedMimeTypes: string[];
|
||||
@@ -69,6 +77,16 @@ export const config: WorkerConfig = {
|
||||
publicBankSyncCopyLimit: envNumber('WORKER_PUBLIC_BANK_SYNC_COPY_LIMIT', 1000),
|
||||
publicBankSyncWorkerId: envString('WORKER_PUBLIC_BANK_SYNC_ID', `public-banks-${process.pid}`),
|
||||
publicBankSyncClaimStaleSeconds: envNumber('WORKER_PUBLIC_BANK_SYNC_CLAIM_STALE_SECONDS', 15 * 60),
|
||||
exportBatchSize: envNumber('WORKER_EXPORT_BATCH_SIZE', 5),
|
||||
exportWorkerId: envString('WORKER_EXPORT_ID', `exports-${process.pid}`),
|
||||
exportBackoffSeconds: envList('WORKER_EXPORT_BACKOFF_SECONDS', '30,120,600,1800')
|
||||
.map((value: string) => Number(value))
|
||||
.filter((value: number) => Number.isFinite(value) && value > 0),
|
||||
exportLocalStorageRoot: envString('EXPORT_LOCAL_STORAGE_ROOT', '.local-storage'),
|
||||
exportPdfFontPath: envString('EXPORT_PDF_FONT_PATH', ''),
|
||||
storageDefaultProvider: envString('STORAGE_DEFAULT_PROVIDER', 'local_dev'),
|
||||
storageDefaultBucket: envString('STORAGE_DEFAULT_BUCKET', 'tenant-assets'),
|
||||
storagePublicBaseUrl: envString('STORAGE_PUBLIC_BASE_URL', ''),
|
||||
storageMaxUploadBytes: envNumber('STORAGE_MAX_UPLOAD_BYTES', 1024 * 1024 * 500),
|
||||
storageAllowedMimePrefixes: envList('STORAGE_ALLOWED_MIME_PREFIXES', 'image/,video/,audio/'),
|
||||
storageAllowedMimeTypes: envList(
|
||||
|
||||
@@ -61,6 +61,17 @@ async function runOnce() {
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (job === 'exports') {
|
||||
const { closeExportExecutorPool, processExportBatch } = await import('./jobs/exports.js');
|
||||
extraClosers.add(closeExportExecutorPool);
|
||||
const result = await processExportBatch();
|
||||
console.log(
|
||||
`[worker] exports batch processed=${result.processed}`
|
||||
+ ` completed=${result.completed} failed=${result.failed}`
|
||||
+ ` retrying=${result.retrying} skipped=${result.skipped}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
throw new Error(`Unsupported worker job: ${job}`);
|
||||
}
|
||||
|
||||
|
||||
340
apps/worker/src/jobs/export-renderer.ts
Normal file
340
apps/worker/src/jobs/export-renderer.ts
Normal file
@@ -0,0 +1,340 @@
|
||||
import fs from 'node:fs';
|
||||
import PDFDocument from 'pdfkit';
|
||||
import {
|
||||
AlignmentType,
|
||||
Document,
|
||||
Footer,
|
||||
Header,
|
||||
HeadingLevel,
|
||||
Packer,
|
||||
PageNumber,
|
||||
Paragraph,
|
||||
TextRun,
|
||||
} from 'docx';
|
||||
import type { BuiltExportPayload } from '../../../api/src/features/tenant-content/exports.js';
|
||||
import { config } from '../config.js';
|
||||
|
||||
interface RenderResult {
|
||||
body: Buffer;
|
||||
mimeType: string;
|
||||
extension: 'pdf' | 'docx';
|
||||
}
|
||||
|
||||
type ExportQuestion = BuiltExportPayload['questions'][number];
|
||||
|
||||
const PDF_FONT_CANDIDATES = [
|
||||
config.exportPdfFontPath,
|
||||
'C:\\Windows\\Fonts\\NotoSansSC-VF.ttf',
|
||||
'C:\\Windows\\Fonts\\msyh.ttc',
|
||||
'C:\\Windows\\Fonts\\simhei.ttf',
|
||||
'/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc',
|
||||
'/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc',
|
||||
'/usr/share/fonts/truetype/wqy/wqy-microhei.ttc',
|
||||
].filter(Boolean);
|
||||
|
||||
function plainText(value: unknown) {
|
||||
return String(value ?? '')
|
||||
.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, ' [图片: $1 $2] ')
|
||||
.replace(/\[([^\]]+)\]\(([^)]+)\)/g, '$1')
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<\/(p|div|li|tr|h[1-6])>/gi, '\n')
|
||||
.replace(/<[^>]+>/g, '')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/\r\n/g, '\n')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function optionText(option: unknown, index: number) {
|
||||
const prefix = `${String.fromCharCode(65 + index)}. `;
|
||||
if (option && typeof option === 'object' && !Array.isArray(option)) {
|
||||
const object = option as Record<string, unknown>;
|
||||
return prefix + plainText(object.text ?? object.label ?? object.content ?? JSON.stringify(object));
|
||||
}
|
||||
return prefix + plainText(option);
|
||||
}
|
||||
|
||||
function answerText(question: ExportQuestion) {
|
||||
const answers: string[] = [];
|
||||
if (typeof question.correctOptionIndex === 'number') {
|
||||
answers.push(String.fromCharCode(65 + question.correctOptionIndex));
|
||||
}
|
||||
if (Array.isArray(question.correctOptionIndices) && question.correctOptionIndices.length) {
|
||||
answers.push(question.correctOptionIndices.map(item => String.fromCharCode(65 + Number(item))).join(', '));
|
||||
}
|
||||
if (typeof question.answerText === 'string' && question.answerText.trim()) {
|
||||
answers.push(plainText(question.answerText));
|
||||
}
|
||||
return Array.from(new Set(answers.filter(Boolean))).join('；');
|
||||
}
|
||||
|
||||
function subQuestionLines(question: ExportQuestion, includeAnswers: boolean, includeExplanations: boolean) {
|
||||
const subQuestions = Array.isArray(question.subQuestions) ? question.subQuestions : [];
|
||||
const lines: string[] = [];
|
||||
subQuestions.forEach((item, index) => {
|
||||
if (!item || typeof item !== 'object' || Array.isArray(item)) return;
|
||||
const object = item as Record<string, unknown>;
|
||||
lines.push(`(${index + 1}) ${plainText(object.content ?? object.stem ?? object.title ?? '')}`);
|
||||
const options = Array.isArray(object.options) ? object.options : [];
|
||||
options.forEach((option, optionIndex) => {
|
||||
lines.push(` ${optionText(option, optionIndex)}`);
|
||||
});
|
||||
if (includeAnswers) {
|
||||
const answer = plainText(object.answerText ?? object.answer_text ?? object.answer ?? object.correctAnswer ?? object.referenceAnswer ?? '');
|
||||
const optionAnswer = Array.isArray(object.correctOptionIndices)
|
||||
? object.correctOptionIndices.map(value => String.fromCharCode(65 + Number(value))).join(', ')
|
||||
: typeof object.correctOptionIndex === 'number'
|
||||
? String.fromCharCode(65 + object.correctOptionIndex)
|
||||
: '';
|
||||
if (answer || optionAnswer) lines.push(` 答案：${answer || optionAnswer}`);
|
||||
}
|
||||
if (includeExplanations) {
|
||||
const explanation = plainText(object.explanation ?? object.analysis ?? '');
|
||||
if (explanation) lines.push(` 解析：${explanation}`);
|
||||
}
|
||||
});
|
||||
return lines;
|
||||
}
|
||||
|
||||
function boolOption(payload: BuiltExportPayload, key: string) {
|
||||
return payload.options && typeof payload.options[key] === 'boolean' ? payload.options[key] === true : false;
|
||||
}
|
||||
|
||||
function watermarkText(payload: BuiltExportPayload) {
|
||||
const raw = payload.options?.watermarkText;
|
||||
return typeof raw === 'string' && raw.trim() ? raw.trim().slice(0, 80) : '';
|
||||
}
|
||||
|
||||
function findPdfFont() {
|
||||
return PDF_FONT_CANDIDATES.find(candidate => fs.existsSync(candidate));
|
||||
}
|
||||
|
||||
function collectPdf(doc: PDFKit.PDFDocument) {
|
||||
return new Promise<Buffer>((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
doc.on('data', chunk => chunks.push(Buffer.from(chunk)));
|
||||
doc.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
doc.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function renderPdfWatermark(doc: PDFKit.PDFDocument, text: string) {
|
||||
if (!text) return;
|
||||
const width = doc.page.width;
|
||||
const height = doc.page.height;
|
||||
doc.save();
|
||||
doc.rotate(-35, { origin: [width / 2, height / 2] });
|
||||
doc.opacity(0.08);
|
||||
doc.fillColor('#64748b');
|
||||
doc.fontSize(36);
|
||||
doc.text(text, 40, height / 2 - 40, { align: 'center', width: width - 80 });
|
||||
doc.opacity(1);
|
||||
doc.restore();
|
||||
doc.fillColor('#111827');
|
||||
}
|
||||
|
||||
function writePdfQuestion(doc: PDFKit.PDFDocument, question: ExportQuestion, index: number, payload: BuiltExportPayload) {
|
||||
const includeAnswers = boolOption(payload, 'includeAnswers');
|
||||
const includeExplanations = boolOption(payload, 'includeExplanations');
|
||||
const typeLabel = typeof question.typeLabel === 'string' && question.typeLabel ? ` [${question.typeLabel}]` : '';
|
||||
const score = typeof question.score === 'number' ? ` (${question.score}分)` : '';
|
||||
doc.moveDown(0.4);
|
||||
doc.fontSize(10).fillColor('#111827').text(`${index + 1}. ${plainText(question.content)}${typeLabel}${score}`, {
|
||||
lineGap: 3,
|
||||
});
|
||||
const options = Array.isArray(question.options) ? question.options : [];
|
||||
options.forEach((option, optionIndex) => {
|
||||
doc.fontSize(9).fillColor('#374151').text(optionText(option, optionIndex), { indent: 16, lineGap: 2 });
|
||||
});
|
||||
for (const line of subQuestionLines(question, includeAnswers, includeExplanations)) {
|
||||
doc.fontSize(9).fillColor('#374151').text(line, { indent: line.startsWith('(') ? 16 : 26, lineGap: 2 });
|
||||
}
|
||||
const answer = includeAnswers ? answerText(question) : '';
|
||||
if (answer) {
|
||||
doc.fontSize(9).fillColor('#0f766e').text(`答案：${answer}`, { indent: 16, lineGap: 2 });
|
||||
}
|
||||
if (includeExplanations && typeof question.explanation === 'string' && question.explanation.trim()) {
|
||||
doc.fontSize(9).fillColor('#475569').text(`解析：${plainText(question.explanation)}`, { indent: 16, lineGap: 2 });
|
||||
}
|
||||
}
|
||||
|
||||
export async function renderExportPdf(payload: BuiltExportPayload): Promise<RenderResult> {
|
||||
const doc = new PDFDocument({ size: 'A4', margin: 48, bufferPages: true, autoFirstPage: true });
|
||||
const done = collectPdf(doc);
|
||||
const fontPath = findPdfFont();
|
||||
if (fontPath) {
|
||||
doc.registerFont('TikuCjk', fontPath);
|
||||
doc.font('TikuCjk');
|
||||
}
|
||||
|
||||
const watermark = watermarkText(payload);
|
||||
renderPdfWatermark(doc, watermark);
|
||||
doc.on('pageAdded', () => {
|
||||
if (fontPath) doc.font('TikuCjk');
|
||||
renderPdfWatermark(doc, watermark);
|
||||
});
|
||||
|
||||
doc.fontSize(18).fillColor('#0f172a').text(payload.title || '题库导出', { align: 'center' });
|
||||
doc.moveDown(0.4);
|
||||
doc.fontSize(9).fillColor('#64748b').text(
|
||||
`导出时间：${payload.exportedAt} 题量：${payload.summary.questionCount} 总分：${payload.summary.totalScore}`,
|
||||
{ align: 'center' },
|
||||
);
|
||||
if (watermark) {
|
||||
doc.moveDown(0.3);
|
||||
doc.fontSize(8).fillColor('#94a3b8').text(`水印：${watermark}`, { align: 'center' });
|
||||
}
|
||||
|
||||
let questionIndex = 0;
|
||||
for (const section of payload.sections) {
|
||||
doc.moveDown(1);
|
||||
doc.fontSize(13).fillColor('#1e3a8a').text(section.title || section.key, { underline: true });
|
||||
for (const question of section.questions) {
|
||||
writePdfQuestion(doc, question, questionIndex, payload);
|
||||
questionIndex += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const pages = doc.bufferedPageRange();
|
||||
for (let i = 0; i < pages.count; i += 1) {
|
||||
doc.switchToPage(i);
|
||||
if (fontPath) doc.font('TikuCjk');
|
||||
doc.fontSize(8).fillColor('#94a3b8').text(
|
||||
`${payload.title || '题库导出'} · 第 ${i + 1} / ${pages.count} 页`,
|
||||
48,
|
||||
doc.page.height - 34,
|
||||
{ align: 'center', width: doc.page.width - 96 },
|
||||
);
|
||||
}
|
||||
doc.end();
|
||||
return {
|
||||
body: await done,
|
||||
mimeType: 'application/pdf',
|
||||
extension: 'pdf',
|
||||
};
|
||||
}
|
||||
|
||||
function docxParagraph(text: string, options: { heading?: (typeof HeadingLevel)[keyof typeof HeadingLevel]; bold?: boolean } = {}) {
|
||||
return new Paragraph({
|
||||
heading: options.heading,
|
||||
spacing: { after: 140 },
|
||||
children: [
|
||||
new TextRun({
|
||||
text,
|
||||
bold: options.bold,
|
||||
font: 'Microsoft YaHei',
|
||||
size: options.heading ? 28 : 21,
|
||||
}),
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
function docxQuestionParagraphs(question: ExportQuestion, index: number, payload: BuiltExportPayload) {
|
||||
const includeAnswers = boolOption(payload, 'includeAnswers');
|
||||
const includeExplanations = boolOption(payload, 'includeExplanations');
|
||||
const paragraphs: Paragraph[] = [];
|
||||
const typeLabel = typeof question.typeLabel === 'string' && question.typeLabel ? ` [${question.typeLabel}]` : '';
|
||||
const score = typeof question.score === 'number' ? ` (${question.score}分)` : '';
|
||||
paragraphs.push(docxParagraph(`${index + 1}. ${plainText(question.content)}${typeLabel}${score}`, { bold: true }));
|
||||
const options = Array.isArray(question.options) ? question.options : [];
|
||||
options.forEach((option, optionIndex) => {
|
||||
paragraphs.push(docxParagraph(optionText(option, optionIndex)));
|
||||
});
|
||||
for (const line of subQuestionLines(question, includeAnswers, includeExplanations)) {
|
||||
paragraphs.push(docxParagraph(line));
|
||||
}
|
||||
const answer = includeAnswers ? answerText(question) : '';
|
||||
if (answer) paragraphs.push(docxParagraph(`答案：${answer}`));
|
||||
if (includeExplanations && typeof question.explanation === 'string' && question.explanation.trim()) {
|
||||
paragraphs.push(docxParagraph(`解析：${plainText(question.explanation)}`));
|
||||
}
|
||||
return paragraphs;
|
||||
}
|
||||
|
||||
export async function renderExportDocx(payload: BuiltExportPayload): Promise<RenderResult> {
|
||||
const watermark = watermarkText(payload);
|
||||
const children: Paragraph[] = [
|
||||
new Paragraph({
|
||||
heading: HeadingLevel.TITLE,
|
||||
alignment: AlignmentType.CENTER,
|
||||
spacing: { after: 180 },
|
||||
children: [new TextRun({ text: payload.title || '题库导出', bold: true, font: 'Microsoft YaHei', size: 36 })],
|
||||
}),
|
||||
new Paragraph({
|
||||
alignment: AlignmentType.CENTER,
|
||||
spacing: { after: 180 },
|
||||
children: [
|
||||
new TextRun({
|
||||
text: `导出时间：${payload.exportedAt} 题量：${payload.summary.questionCount} 总分：${payload.summary.totalScore}`,
|
||||
color: '64748B',
|
||||
font: 'Microsoft YaHei',
|
||||
size: 19,
|
||||
}),
|
||||
],
|
||||
}),
|
||||
];
|
||||
if (watermark) {
|
||||
children.push(new Paragraph({
|
||||
alignment: AlignmentType.CENTER,
|
||||
spacing: { after: 180 },
|
||||
children: [new TextRun({ text: `水印：${watermark}`, color: '94A3B8', font: 'Microsoft YaHei', size: 18 })],
|
||||
}));
|
||||
}
|
||||
let questionIndex = 0;
|
||||
for (const section of payload.sections) {
|
||||
children.push(docxParagraph(section.title || section.key, { heading: HeadingLevel.HEADING_2, bold: true }));
|
||||
for (const question of section.questions) {
|
||||
children.push(...docxQuestionParagraphs(question, questionIndex, payload));
|
||||
questionIndex += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const doc = new Document({
|
||||
creator: 'Tiku SaaS Export Worker',
|
||||
title: payload.title || '题库导出',
|
||||
description: 'Generated by Tiku SaaS export worker',
|
||||
sections: [
|
||||
{
|
||||
headers: {
|
||||
default: new Header({
|
||||
children: [
|
||||
new Paragraph({
|
||||
alignment: AlignmentType.CENTER,
|
||||
children: [
|
||||
new TextRun({
|
||||
text: watermark || payload.title || '题库导出',
|
||||
color: 'CBD5E1',
|
||||
font: 'Microsoft YaHei',
|
||||
size: 18,
|
||||
}),
|
||||
],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
},
|
||||
footers: {
|
||||
default: new Footer({
|
||||
children: [
|
||||
new Paragraph({
|
||||
alignment: AlignmentType.CENTER,
|
||||
children: [
|
||||
new TextRun({ children: ['第 ', PageNumber.CURRENT, ' / ', PageNumber.TOTAL_PAGES, ' 页'], font: 'Microsoft YaHei', size: 18 }),
|
||||
],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
},
|
||||
children,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
return {
|
||||
body: await Packer.toBuffer(doc),
|
||||
mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
extension: 'docx',
|
||||
};
|
||||
}
|
||||
496
apps/worker/src/jobs/exports.ts
Normal file
496
apps/worker/src/jobs/exports.ts
Normal file
@@ -0,0 +1,496 @@
|
||||
import path from 'node:path';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type pg from 'pg';
|
||||
import { pool } from '../db.js';
|
||||
import { config } from '../config.js';
|
||||
import { closePool as closeApiExportPool } from '../../../api/src/core/db.js';
|
||||
import { buildExportPayloadForJob } from '../../../api/src/features/tenant-content/exports.js';
|
||||
import { normalizeStorageProvider, putStorageObject, storageBucket, validateObjectKey } from '../storage.js';
|
||||
import { renderExportDocx, renderExportPdf } from './export-renderer.js';
|
||||
|
||||
interface ExportJobRow {
|
||||
id: string;
|
||||
tenantId: string;
|
||||
createdBy: string | null;
|
||||
exportType: string;
|
||||
format: 'pdf' | 'docx';
|
||||
scopeType: string;
|
||||
scopeId: string;
|
||||
status: string;
|
||||
attemptCount: number;
|
||||
maxAttempts: number;
|
||||
options: Record<string, unknown>;
|
||||
outputMetadata: Record<string, unknown>;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
interface ExportWorkerResult {
|
||||
processed: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
retrying: number;
|
||||
skipped: number;
|
||||
}
|
||||
|
||||
interface RenderedExport {
|
||||
body: Buffer;
|
||||
mimeType: string;
|
||||
extension: 'pdf' | 'docx';
|
||||
}
|
||||
|
||||
const ASSET_VISIBILITIES = new Set(['public', 'tenant', 'members', 'svip', 'private']);
|
||||
|
||||
function objectValue(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function boolValue(value: unknown, fallback: boolean) {
|
||||
return typeof value === 'boolean' ? value : fallback;
|
||||
}
|
||||
|
||||
function stringValue(value: unknown, fallback = '') {
|
||||
return typeof value === 'string' && value.trim() ? value.trim() : fallback;
|
||||
}
|
||||
|
||||
function truncate(value: unknown, max = 1900) {
|
||||
return String(value ?? '').slice(0, max);
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function errorCode(error: unknown) {
|
||||
return typeof error === 'object' && error !== null && 'code' in error
|
||||
? String((error as { code?: unknown }).code || 'EXPORT_WORKER_ERROR')
|
||||
: 'EXPORT_WORKER_ERROR';
|
||||
}
|
||||
|
||||
function backoffSeconds(attemptCount: number) {
|
||||
const backoffs = config.exportBackoffSeconds.length ? config.exportBackoffSeconds : [30, 120, 600, 1800];
|
||||
return backoffs[Math.min(Math.max(0, attemptCount - 1), backoffs.length - 1)];
|
||||
}
|
||||
|
||||
function safeFileBase(value: string) {
|
||||
const ascii = value
|
||||
.normalize('NFKD')
|
||||
.replace(/[^\w.-]+/g, '-')
|
||||
.replace(/-+/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
.slice(0, 80);
|
||||
return ascii || 'question-export';
|
||||
}
|
||||
|
||||
function safeDisplayFileName(title: string, extension: string) {
|
||||
return `${title.trim().replace(/[\\/:*?"<>|]+/g, '-').replace(/\s+/g, '-').slice(0, 120) || '题库导出'}.${extension}`;
|
||||
}
|
||||
|
||||
function objectKeyFor(job: ExportJobRow, title: string, extension: string) {
|
||||
const created = new Date(job.createdAt || Date.now());
|
||||
const yyyy = Number.isFinite(created.getTime()) ? created.getUTCFullYear() : new Date().getUTCFullYear();
|
||||
const mm = String((Number.isFinite(created.getTime()) ? created.getUTCMonth() : new Date().getUTCMonth()) + 1).padStart(2, '0');
|
||||
return validateObjectKey(
|
||||
job.tenantId,
|
||||
`${job.tenantId}/exports/${yyyy}/${mm}/${job.id}-${safeFileBase(title)}.${extension}`,
|
||||
);
|
||||
}
|
||||
|
||||
function assetVisibility(options: Record<string, unknown>) {
|
||||
const requested = stringValue(options.assetVisibility, boolValue(options.publishToAssets, false) ? 'tenant' : 'private');
|
||||
return ASSET_VISIBILITIES.has(requested) ? requested : 'private';
|
||||
}
|
||||
|
||||
function assetStatus(options: Record<string, unknown>) {
|
||||
return boolValue(options.publishToAssets, false) ? 'active' : 'active';
|
||||
}
|
||||
|
||||
function accessRules(options: Record<string, unknown>) {
|
||||
return objectValue(options.assetAccessRules);
|
||||
}
|
||||
|
||||
async function claimExportJobs() {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('begin');
|
||||
const result = await client.query<ExportJobRow>(
|
||||
`
|
||||
select id, tenant_id as "tenantId", created_by as "createdBy",
|
||||
export_type as "exportType", format, scope_type as "scopeType",
|
||||
scope_id as "scopeId", status, attempt_count as "attemptCount",
|
||||
max_attempts as "maxAttempts", options,
|
||||
output_metadata as "outputMetadata", created_at as "createdAt"
|
||||
from public.content_export_jobs
|
||||
where status = 'pending'
|
||||
and format in ('pdf', 'docx')
|
||||
and attempt_count < max_attempts
|
||||
and (next_attempt_at is null or next_attempt_at <= now())
|
||||
order by created_at asc
|
||||
limit $1
|
||||
for update skip locked
|
||||
`,
|
||||
[config.exportBatchSize],
|
||||
);
|
||||
|
||||
const ids = result.rows.map(row => row.id);
|
||||
if (ids.length) {
|
||||
await client.query(
|
||||
`
|
||||
update public.content_export_jobs
|
||||
set status = 'rendering',
|
||||
locked_at = now(),
|
||||
locked_by = $2,
|
||||
attempt_count = attempt_count + 1,
|
||||
started_at = coalesce(started_at, now()),
|
||||
error_message = null,
|
||||
updated_at = now()
|
||||
where id = any($1::uuid[])
|
||||
`,
|
||||
[ids, config.exportWorkerId],
|
||||
);
|
||||
}
|
||||
await client.query('commit');
|
||||
return result.rows;
|
||||
} catch (error) {
|
||||
await client.query('rollback');
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function renderJob(job: ExportJobRow): Promise<{ rendered: RenderedExport; title: string; questionCount: number }> {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
const payload = await buildExportPayloadForJob({
|
||||
client,
|
||||
tenantId: job.tenantId,
|
||||
jobId: job.id,
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
const rendered = job.format === 'pdf'
|
||||
? await renderExportPdf(payload)
|
||||
: await renderExportDocx(payload);
|
||||
return {
|
||||
rendered,
|
||||
title: payload.title || '题库导出',
|
||||
questionCount: payload.summary.questionCount,
|
||||
};
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function insertExportAsset(
|
||||
client: pg.PoolClient,
|
||||
job: ExportJobRow,
|
||||
input: {
|
||||
title: string;
|
||||
fileName: string;
|
||||
extension: 'pdf' | 'docx';
|
||||
mimeType: string;
|
||||
bucket: string;
|
||||
objectKey: string;
|
||||
sizeBytes: number;
|
||||
checksumSha256: string;
|
||||
uploadDetails: Record<string, unknown>;
|
||||
},
|
||||
) {
|
||||
const visibility = assetVisibility(job.options || {});
|
||||
const status = assetStatus(job.options || {});
|
||||
const assetType = input.extension === 'pdf' ? 'pdf' : 'document';
|
||||
const asset = await client.query<{ id: string }>(
|
||||
`
|
||||
insert into public.content_assets (
|
||||
tenant_id, asset_key, title, category, description, file_name,
|
||||
asset_type, storage_provider, bucket, object_key, mime_type,
|
||||
file_size_bytes, checksum_sha256, upload_status, verified_at,
|
||||
verified_size_bytes, verified_checksum_sha256, verification_details,
|
||||
preview_status, visibility, is_public, status, access_rules,
|
||||
metadata, created_by, updated_by, source
|
||||
)
|
||||
values (
|
||||
$1, $2, $3, $4, $5, $6,
|
||||
$7, $8, $9, $10, $11,
|
||||
$12, $13, 'verified', now(),
|
||||
$12, $13, $14::jsonb,
|
||||
$15, $16, $17, $18, $19::jsonb,
|
||||
$20::jsonb, $21, $21, 'content_export_worker'
|
||||
)
|
||||
returning id
|
||||
`,
|
||||
[
|
||||
job.tenantId,
|
||||
`content-export:${job.id}`,
|
||||
input.title,
|
||||
'题库导出',
|
||||
`由题库导出任务 ${job.id} 自动生成`,
|
||||
input.fileName,
|
||||
assetType,
|
||||
normalizeStorageProvider(String(job.outputMetadata?.storageProvider || config.storageDefaultProvider)),
|
||||
input.bucket,
|
||||
input.objectKey,
|
||||
input.mimeType,
|
||||
input.sizeBytes,
|
||||
input.checksumSha256,
|
||||
JSON.stringify({
|
||||
exportWorker: {
|
||||
...input.uploadDetails,
|
||||
workerId: config.exportWorkerId,
|
||||
checkedAt: new Date().toISOString(),
|
||||
},
|
||||
}),
|
||||
input.extension === 'pdf' ? 'ready' : 'none',
|
||||
visibility,
|
||||
visibility === 'public',
|
||||
status,
|
||||
JSON.stringify(accessRules(job.options || {})),
|
||||
JSON.stringify({
|
||||
exportJobId: job.id,
|
||||
exportType: job.exportType,
|
||||
format: job.format,
|
||||
scopeType: job.scopeType,
|
||||
scopeId: job.scopeId,
|
||||
publishToAssets: boolValue(job.options?.publishToAssets, false),
|
||||
watermarkText: stringValue(job.options?.watermarkText, ''),
|
||||
}),
|
||||
job.createdBy,
|
||||
],
|
||||
);
|
||||
return asset.rows[0].id;
|
||||
}
|
||||
|
||||
async function markExportCompleted(job: ExportJobRow, input: {
|
||||
assetId: string;
|
||||
title: string;
|
||||
fileName: string;
|
||||
mimeType: string;
|
||||
sizeBytes: number;
|
||||
checksumSha256: string;
|
||||
bucket: string;
|
||||
objectKey: string;
|
||||
questionCount: number;
|
||||
uploadDetails: Record<string, unknown>;
|
||||
}) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('begin');
|
||||
await client.query(
|
||||
`
|
||||
update public.content_export_jobs
|
||||
set status = 'completed',
|
||||
asset_id = $3,
|
||||
question_count = $4,
|
||||
summary = coalesce(summary, '{}'::jsonb) || $5::jsonb,
|
||||
output_metadata = coalesce(output_metadata, '{}'::jsonb) || $6::jsonb,
|
||||
error_message = null,
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
next_attempt_at = null,
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[
|
||||
job.tenantId,
|
||||
job.id,
|
||||
input.assetId,
|
||||
input.questionCount,
|
||||
JSON.stringify({
|
||||
questionCount: input.questionCount,
|
||||
renderedTitle: input.title,
|
||||
}),
|
||||
JSON.stringify({
|
||||
delivery: 'content_asset',
|
||||
assetId: input.assetId,
|
||||
fileName: input.fileName,
|
||||
mimeType: input.mimeType,
|
||||
sizeBytes: input.sizeBytes,
|
||||
checksumSha256: input.checksumSha256,
|
||||
bucket: input.bucket,
|
||||
objectKey: input.objectKey,
|
||||
renderer: 'worker:exports',
|
||||
renderedAt: new Date().toISOString(),
|
||||
upload: input.uploadDetails,
|
||||
}),
|
||||
],
|
||||
);
|
||||
await client.query(
|
||||
`
|
||||
insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details)
|
||||
values ($1, $2, 'content.questions.export_rendered', 'content_export_job', $3, $4::jsonb)
|
||||
`,
|
||||
[
|
||||
job.tenantId,
|
||||
job.createdBy,
|
||||
job.id,
|
||||
JSON.stringify({
|
||||
assetId: input.assetId,
|
||||
format: job.format,
|
||||
mimeType: input.mimeType,
|
||||
sizeBytes: input.sizeBytes,
|
||||
checksumSha256: input.checksumSha256,
|
||||
workerId: config.exportWorkerId,
|
||||
}),
|
||||
],
|
||||
);
|
||||
await client.query('commit');
|
||||
} catch (error) {
|
||||
await client.query('rollback');
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function markExportFailed(job: ExportJobRow, error: unknown) {
|
||||
const nextAttempt = job.attemptCount + 1;
|
||||
const willRetry = nextAttempt < job.maxAttempts;
|
||||
const status = willRetry ? 'pending' : 'failed';
|
||||
await pool.query(
|
||||
`
|
||||
update public.content_export_jobs
|
||||
set status = $3,
|
||||
error_message = $4,
|
||||
output_metadata = coalesce(output_metadata, '{}'::jsonb) || $5::jsonb,
|
||||
next_attempt_at = case when $6::boolean then now() + make_interval(secs => $7::integer) else null end,
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
finished_at = case when $3 = 'failed' then now() else finished_at end,
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[
|
||||
job.tenantId,
|
||||
job.id,
|
||||
status,
|
||||
truncate(errorMessage(error)),
|
||||
JSON.stringify({
|
||||
lastWorkerError: {
|
||||
code: errorCode(error),
|
||||
message: truncate(errorMessage(error)),
|
||||
workerId: config.exportWorkerId,
|
||||
failedAt: new Date().toISOString(),
|
||||
nextAttempt,
|
||||
maxAttempts: job.maxAttempts,
|
||||
willRetry,
|
||||
},
|
||||
}),
|
||||
willRetry,
|
||||
backoffSeconds(nextAttempt),
|
||||
],
|
||||
);
|
||||
|
||||
await pool.query(
|
||||
`
|
||||
insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details)
|
||||
values ($1, $2, $3, 'content_export_job', $4, $5::jsonb)
|
||||
`,
|
||||
[
|
||||
job.tenantId,
|
||||
job.createdBy,
|
||||
willRetry ? 'content.questions.export_retry_scheduled' : 'content.questions.export_failed',
|
||||
job.id,
|
||||
JSON.stringify({
|
||||
code: errorCode(error),
|
||||
message: truncate(errorMessage(error)),
|
||||
workerId: config.exportWorkerId,
|
||||
nextAttempt,
|
||||
maxAttempts: job.maxAttempts,
|
||||
}),
|
||||
],
|
||||
);
|
||||
|
||||
return willRetry ? 'retrying' : 'failed';
|
||||
}
|
||||
|
||||
async function processExportJob(job: ExportJobRow) {
|
||||
const { rendered, title, questionCount } = await renderJob(job);
|
||||
const extension = rendered.extension;
|
||||
const bucket = storageBucket(String(job.outputMetadata?.storageBucket || ''));
|
||||
const objectKey = objectKeyFor(job, title, extension);
|
||||
const fileName = safeDisplayFileName(title, extension);
|
||||
const upload = await putStorageObject({
|
||||
tenantId: job.tenantId,
|
||||
provider: normalizeStorageProvider(String(job.outputMetadata?.storageProvider || config.storageDefaultProvider)),
|
||||
bucket,
|
||||
objectKey,
|
||||
body: rendered.body,
|
||||
mimeType: rendered.mimeType,
|
||||
});
|
||||
|
||||
const client = await pool.connect();
|
||||
let assetId = '';
|
||||
try {
|
||||
await client.query('begin');
|
||||
assetId = await insertExportAsset(client, job, {
|
||||
title,
|
||||
fileName,
|
||||
extension,
|
||||
mimeType: rendered.mimeType,
|
||||
bucket,
|
||||
objectKey,
|
||||
sizeBytes: upload.sizeBytes,
|
||||
checksumSha256: upload.checksumSha256,
|
||||
uploadDetails: {
|
||||
provider: upload.provider,
|
||||
localPath: upload.localPath ? path.relative(process.cwd(), upload.localPath) : undefined,
|
||||
etag: upload.etag,
|
||||
},
|
||||
});
|
||||
await client.query('commit');
|
||||
} catch (error) {
|
||||
await client.query('rollback');
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
|
||||
await markExportCompleted(job, {
|
||||
assetId,
|
||||
title,
|
||||
fileName,
|
||||
mimeType: rendered.mimeType,
|
||||
sizeBytes: upload.sizeBytes,
|
||||
checksumSha256: upload.checksumSha256,
|
||||
bucket,
|
||||
objectKey,
|
||||
questionCount,
|
||||
uploadDetails: {
|
||||
provider: upload.provider,
|
||||
localPath: upload.localPath ? path.relative(process.cwd(), upload.localPath) : undefined,
|
||||
etag: upload.etag,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function processExportBatch(): Promise<ExportWorkerResult> {
|
||||
const jobs = await claimExportJobs();
|
||||
const result: ExportWorkerResult = {
|
||||
processed: jobs.length,
|
||||
completed: 0,
|
||||
failed: 0,
|
||||
retrying: 0,
|
||||
skipped: 0,
|
||||
};
|
||||
|
||||
for (const job of jobs) {
|
||||
try {
|
||||
if (job.format !== 'pdf' && job.format !== 'docx') {
|
||||
result.skipped += 1;
|
||||
continue;
|
||||
}
|
||||
await processExportJob(job);
|
||||
result.completed += 1;
|
||||
} catch (error) {
|
||||
const state = await markExportFailed(job, error);
|
||||
if (state === 'retrying') result.retrying += 1;
|
||||
else result.failed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function closeExportExecutorPool() {
|
||||
await closeApiExportPool();
|
||||
}
|
||||
246
apps/worker/src/storage.ts
Normal file
246
apps/worker/src/storage.ts
Normal file
@@ -0,0 +1,246 @@
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import { StorageClient } from '@supabase/storage-js';
|
||||
import { config } from './config.js';
|
||||
|
||||
export type StorageProviderName = 'external_url' | 'supabase_storage' | 'aliyun_oss' | 'tencent_cos' | 'qiniu_kodo' | 'local_dev';
|
||||
|
||||
interface PutObjectInput {
|
||||
tenantId: string;
|
||||
provider: StorageProviderName;
|
||||
bucket: string;
|
||||
objectKey: string;
|
||||
body: Buffer;
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
interface PutObjectResult {
|
||||
provider: StorageProviderName;
|
||||
bucket: string;
|
||||
objectKey: string;
|
||||
checksumSha256: string;
|
||||
sizeBytes: number;
|
||||
mimeType: string;
|
||||
localPath?: string;
|
||||
etag?: string | null;
|
||||
}
|
||||
|
||||
const SAFE_OBJECT_KEY_RE = /^[A-Za-z0-9][A-Za-z0-9._~!$&'()+,;=@/-]{0,1023}$/;
|
||||
const SUPPORTED_UPLOAD_PROVIDERS = new Set<StorageProviderName>(['local_dev', 'supabase_storage', 'aliyun_oss', 'tencent_cos']);
|
||||
|
||||
export function normalizeStorageProvider(value: string | null | undefined): StorageProviderName {
|
||||
const provider = (value?.trim() || config.storageDefaultProvider || 'local_dev') as StorageProviderName;
|
||||
if (!SUPPORTED_UPLOAD_PROVIDERS.has(provider)) {
|
||||
throw new Error(`Unsupported export storage provider: ${provider}`);
|
||||
}
|
||||
return provider;
|
||||
}
|
||||
|
||||
export function storageBucket(value: string | null | undefined) {
|
||||
return value?.trim() || config.storageDefaultBucket || 'tenant-assets';
|
||||
}
|
||||
|
||||
export function canonicalObjectKey(objectKey: string) {
|
||||
return objectKey.replace(/^\/+/, '').replace(/\/{2,}/g, '/');
|
||||
}
|
||||
|
||||
export function validateObjectKey(tenantId: string, objectKey: string) {
|
||||
const clean = canonicalObjectKey(objectKey);
|
||||
if (!clean || clean.includes('..') || clean.includes('\\') || clean.includes('%2f') || clean.includes('%2F')) {
|
||||
throw new Error('Invalid objectKey');
|
||||
}
|
||||
if (!SAFE_OBJECT_KEY_RE.test(clean)) {
|
||||
throw new Error('objectKey contains unsafe characters');
|
||||
}
|
||||
if (config.storageRequireTenantPrefix && !clean.startsWith(`${tenantId}/`)) {
|
||||
throw new Error('objectKey must be scoped by tenantId prefix');
|
||||
}
|
||||
return clean;
|
||||
}
|
||||
|
||||
function sha256(body: Buffer) {
|
||||
return crypto.createHash('sha256').update(body).digest('hex');
|
||||
}
|
||||
|
||||
function requireConfigured(condition: unknown, provider: StorageProviderName, missing: string) {
|
||||
if (!condition) throw new Error(`${provider} is not configured: ${missing}`);
|
||||
}
|
||||
|
||||
async function putLocalDev(input: PutObjectInput, checksumSha256: string): Promise<PutObjectResult> {
|
||||
const root = path.resolve(process.cwd(), config.exportLocalStorageRoot || '.local-storage');
|
||||
const bucketRoot = path.resolve(root, input.bucket);
|
||||
const target = path.resolve(bucketRoot, input.objectKey);
|
||||
if (!target.startsWith(bucketRoot + path.sep)) {
|
||||
throw new Error('Resolved local object path escapes storage root');
|
||||
}
|
||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
||||
await fs.writeFile(target, input.body);
|
||||
return {
|
||||
provider: input.provider,
|
||||
bucket: input.bucket,
|
||||
objectKey: input.objectKey,
|
||||
checksumSha256,
|
||||
sizeBytes: input.body.byteLength,
|
||||
mimeType: input.mimeType,
|
||||
localPath: target,
|
||||
};
|
||||
}
|
||||
|
||||
async function putAliyunOss(input: PutObjectInput, checksumSha256: string): Promise<PutObjectResult> {
|
||||
requireConfigured(config.aliyunOssAccessKeyId, 'aliyun_oss', 'ALIYUN_OSS_ACCESS_KEY_ID');
|
||||
requireConfigured(config.aliyunOssAccessKeySecret, 'aliyun_oss', 'ALIYUN_OSS_ACCESS_KEY_SECRET');
|
||||
requireConfigured(config.aliyunOssRegion || config.aliyunOssEndpoint, 'aliyun_oss', 'ALIYUN_OSS_REGION or ALIYUN_OSS_ENDPOINT');
|
||||
|
||||
const { default: OSS } = await import('ali-oss');
|
||||
const client = new OSS({
|
||||
region: config.aliyunOssRegion || undefined,
|
||||
endpoint: config.aliyunOssEndpoint || undefined,
|
||||
accessKeyId: config.aliyunOssAccessKeyId,
|
||||
accessKeySecret: config.aliyunOssAccessKeySecret,
|
||||
stsToken: config.aliyunOssStsToken || undefined,
|
||||
bucket: input.bucket,
|
||||
internal: config.aliyunOssInternal,
|
||||
secure: true,
|
||||
});
|
||||
const response = await (client as unknown as {
|
||||
put: (objectKey: string, body: Buffer, options: Record<string, unknown>) => Promise<{ res?: { headers?: Record<string, string> } }>;
|
||||
}).put(input.objectKey, input.body, {
|
||||
headers: {
|
||||
'Content-Type': input.mimeType,
|
||||
'x-oss-meta-sha256': checksumSha256,
|
||||
},
|
||||
});
|
||||
return {
|
||||
provider: input.provider,
|
||||
bucket: input.bucket,
|
||||
objectKey: input.objectKey,
|
||||
checksumSha256,
|
||||
sizeBytes: input.body.byteLength,
|
||||
mimeType: input.mimeType,
|
||||
etag: response.res?.headers?.etag || null,
|
||||
};
|
||||
}
|
||||
|
||||
async function putSupabaseStorage(input: PutObjectInput, checksumSha256: string): Promise<PutObjectResult> {
|
||||
requireConfigured(config.supabaseStorageUrl, 'supabase_storage', 'SUPABASE_STORAGE_URL');
|
||||
requireConfigured(config.supabaseStorageServiceKey, 'supabase_storage', 'SUPABASE_STORAGE_SERVICE_KEY');
|
||||
const client = new StorageClient(config.supabaseStorageUrl.replace(/\/+$/, ''), {
|
||||
apikey: config.supabaseStorageServiceKey,
|
||||
authorization: `Bearer ${config.supabaseStorageServiceKey}`,
|
||||
});
|
||||
const response = await client.from(input.bucket).upload(input.objectKey, input.body, {
|
||||
contentType: input.mimeType,
|
||||
upsert: true,
|
||||
duplex: 'half',
|
||||
metadata: { sha256: checksumSha256 },
|
||||
} as never);
|
||||
if (response.error) {
|
||||
throw new Error(response.error.message || 'Supabase Storage upload failed');
|
||||
}
|
||||
return {
|
||||
provider: input.provider,
|
||||
bucket: input.bucket,
|
||||
objectKey: input.objectKey,
|
||||
checksumSha256,
|
||||
sizeBytes: input.body.byteLength,
|
||||
mimeType: input.mimeType,
|
||||
};
|
||||
}
|
||||
|
||||
function hmacSha1Hex(key: string | Buffer, value: string) {
|
||||
return crypto.createHmac('sha1', key).update(value).digest('hex');
|
||||
}
|
||||
|
||||
function sha1Hex(value: string) {
|
||||
return crypto.createHash('sha1').update(value).digest('hex');
|
||||
}
|
||||
|
||||
function cosEncodePath(objectKey: string) {
|
||||
return objectKey
|
||||
.split('/')
|
||||
.map(part => encodeURIComponent(part).replace(/[!'()*]/g, char => `%${char.charCodeAt(0).toString(16).toUpperCase()}`))
|
||||
.join('/');
|
||||
}
|
||||
|
||||
function cosHost(bucket: string) {
|
||||
requireConfigured(config.tencentCosRegion, 'tencent_cos', 'TENCENT_COS_REGION');
|
||||
const bucketWithAppId = config.tencentCosAppId && !bucket.endsWith(`-${config.tencentCosAppId}`)
|
||||
? `${bucket}-${config.tencentCosAppId}`
|
||||
: bucket;
|
||||
return `${bucketWithAppId}.cos.${config.tencentCosRegion}.myqcloud.com`;
|
||||
}
|
||||
|
||||
function nowSeconds() {
|
||||
return Math.floor(Date.now() / 1000);
|
||||
}
|
||||
|
||||
async function putTencentCos(input: PutObjectInput, checksumSha256: string): Promise<PutObjectResult> {
|
||||
requireConfigured(config.tencentCosSecretId, 'tencent_cos', 'TENCENT_COS_SECRET_ID');
|
||||
requireConfigured(config.tencentCosSecretKey, 'tencent_cos', 'TENCENT_COS_SECRET_KEY');
|
||||
const host = cosHost(input.bucket);
|
||||
const start = nowSeconds();
|
||||
const end = start + 300;
|
||||
const keyTime = `${start};${end}`;
|
||||
const pathname = `/${cosEncodePath(input.objectKey)}`;
|
||||
const signedHeaders: Record<string, string> = {
|
||||
host,
|
||||
'content-type': input.mimeType,
|
||||
'x-cos-meta-sha256': checksumSha256,
|
||||
};
|
||||
const headerKeys = Object.keys(signedHeaders).sort();
|
||||
const headerList = headerKeys.join(';');
|
||||
const httpHeaders = headerKeys
|
||||
.map(key => `${encodeURIComponent(key)}=${encodeURIComponent(signedHeaders[key]).toLowerCase()}`)
|
||||
.join('&');
|
||||
const signedQuery: Record<string, string> = {};
|
||||
if (config.tencentCosSecurityToken) signedQuery['x-cos-security-token'] = config.tencentCosSecurityToken;
|
||||
const queryKeys = Object.keys(signedQuery).sort();
|
||||
const urlParamList = queryKeys.join(';');
|
||||
const httpParameters = queryKeys
|
||||
.map(key => `${encodeURIComponent(key)}=${encodeURIComponent(signedQuery[key])}`)
|
||||
.join('&');
|
||||
const httpString = `put\n${pathname}\n${httpParameters}\n${httpHeaders}\n`;
|
||||
const stringToSign = `sha1\n${keyTime}\n${sha1Hex(httpString)}\n`;
|
||||
const signKey = hmacSha1Hex(config.tencentCosSecretKey, keyTime);
|
||||
const signature = hmacSha1Hex(signKey, stringToSign);
|
||||
const query = new URLSearchParams();
|
||||
query.set('q-sign-algorithm', 'sha1');
|
||||
query.set('q-ak', config.tencentCosSecretId);
|
||||
query.set('q-sign-time', keyTime);
|
||||
query.set('q-key-time', keyTime);
|
||||
query.set('q-header-list', headerList);
|
||||
query.set('q-url-param-list', urlParamList);
|
||||
query.set('q-signature', signature);
|
||||
for (const key of queryKeys) query.set(key, signedQuery[key]);
|
||||
|
||||
const response = await fetch(`https://${host}${pathname}?${query.toString()}`, {
|
||||
method: 'PUT',
|
||||
headers: signedHeaders,
|
||||
body: input.body as unknown as BodyInit,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Tencent COS upload failed: ${response.status}`);
|
||||
}
|
||||
return {
|
||||
provider: input.provider,
|
||||
bucket: input.bucket,
|
||||
objectKey: input.objectKey,
|
||||
checksumSha256,
|
||||
sizeBytes: input.body.byteLength,
|
||||
mimeType: input.mimeType,
|
||||
etag: response.headers.get('etag'),
|
||||
};
|
||||
}
|
||||
|
||||
export async function putStorageObject(input: PutObjectInput): Promise<PutObjectResult> {
|
||||
const provider = normalizeStorageProvider(input.provider);
|
||||
const objectKey = validateObjectKey(input.tenantId, input.objectKey);
|
||||
const normalized = { ...input, provider, objectKey };
|
||||
const checksumSha256 = sha256(input.body);
|
||||
if (provider === 'local_dev') return putLocalDev(normalized, checksumSha256);
|
||||
if (provider === 'aliyun_oss') return putAliyunOss(normalized, checksumSha256);
|
||||
if (provider === 'tencent_cos') return putTencentCos(normalized, checksumSha256);
|
||||
if (provider === 'supabase_storage') return putSupabaseStorage(normalized, checksumSha256);
|
||||
throw new Error(`${provider} does not support worker object upload`);
|
||||
}
|
||||
Reference in New Issue
Block a user