forked from wangziqi/gongxue-base
feat: add spreadsheet async imports
This commit is contained in:
Codex
@@ -12,6 +12,7 @@
|
||||
"dependencies": {
|
||||
"@supabase/storage-js": "^2.108.2",
|
||||
"ali-oss": "^6.23.0",
|
||||
"exceljs": "^4.4.0",
|
||||
"jose": "^6.2.3",
|
||||
"pg": "^8.16.3"
|
||||
},
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import type pg from 'pg';
|
||||
import { createPool, query as runQuery, queryOne as runQueryOne } from '../../../../packages/db/src/index.js';
|
||||
import { config } from './config.js';
|
||||
import { DEFAULT_DATABASE_URL } from '../../../../packages/config/src/index.js';
|
||||
|
||||
export const pool = createPool({
|
||||
connectionString: config.databaseUrl,
|
||||
connectionString: process.env.DATABASE_URL || DEFAULT_DATABASE_URL,
|
||||
max: 10,
|
||||
});
|
||||
|
||||
|
||||
9
apps/api/src/core/errors.ts
Normal file
9
apps/api/src/core/errors.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
export class HttpError extends Error {
|
||||
constructor(
|
||||
public readonly statusCode: number,
|
||||
message: string,
|
||||
public readonly code = 'HTTP_ERROR',
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
import type { IncomingMessage, ServerResponse } from 'node:http';
|
||||
import { URL } from 'node:url';
|
||||
import { config } from './config.js';
|
||||
import { HttpError } from './errors.js';
|
||||
|
||||
export { HttpError } from './errors.js';
|
||||
|
||||
export interface RequestContext {
|
||||
req: IncomingMessage;
|
||||
@@ -10,16 +13,6 @@ export interface RequestContext {
|
||||
|
||||
export type Handler = (ctx: RequestContext) => Promise<unknown>;
|
||||
|
||||
export class HttpError extends Error {
|
||||
constructor(
|
||||
public readonly statusCode: number,
|
||||
message: string,
|
||||
public readonly code = 'HTTP_ERROR',
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export function sendJson(res: ServerResponse, statusCode: number, body: unknown) {
|
||||
res.statusCode = statusCode;
|
||||
res.setHeader('content-type', 'application/json; charset=utf-8');
|
||||
|
||||
739
apps/api/src/features/tenant-content/import-spreadsheet.ts
Normal file
739
apps/api/src/features/tenant-content/import-spreadsheet.ts
Normal file
@@ -0,0 +1,739 @@
|
||||
import { Buffer } from 'node:buffer';
|
||||
import ExcelJS from 'exceljs';
|
||||
import { HttpError } from '../../core/http.js';
|
||||
|
||||
type JsonObject = Record<string, unknown>;
|
||||
type SpreadsheetImportType = 'questions' | 'vocabulary' | 'handbook' | 'scoreline' | 'videos';
|
||||
type SupportedSpreadsheetFormat = 'json' | 'csv' | 'excel';
|
||||
|
||||
interface ParsedSpreadsheet {
|
||||
rows: JsonObject[];
|
||||
sheets: Record<string, JsonObject[]>;
|
||||
metadata: JsonObject;
|
||||
}
|
||||
|
||||
const MAX_SPREADSHEET_ROWS = 5000;
|
||||
const MAX_SPREADSHEET_COLUMNS = 160;
|
||||
const MAX_SPREADSHEET_CELL_CHARS = 100_000;
|
||||
const MAX_EXCEL_FILE_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
const QUESTION_OPTION_KEYS = ['optionA', 'optionB', 'optionC', 'optionD', 'optionE', 'optionF', 'optionG', 'optionH'];
|
||||
|
||||
function objectValue(value: unknown): JsonObject {
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? value as JsonObject : {};
|
||||
}
|
||||
|
||||
function stringValue(value: unknown) {
|
||||
return typeof value === 'string' && value.trim() ? value.trim() : '';
|
||||
}
|
||||
|
||||
function normalizedKey(value: string) {
|
||||
return value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[\s_\-./\\:：()（）[\]【】]/g, '');
|
||||
}
|
||||
|
||||
function sourceFormatValue(value: unknown): SupportedSpreadsheetFormat {
|
||||
const raw = stringValue(value).toLowerCase();
|
||||
if (!raw || raw === 'json') return 'json';
|
||||
if (raw === 'csv') return 'csv';
|
||||
if (['excel', 'xlsx'].includes(raw)) return 'excel';
|
||||
throw new HttpError(400, 'sourceFormat must be json, csv, or excel', 'IMPORT_FORMAT_NOT_SUPPORTED');
|
||||
}
|
||||
|
||||
function assertRowLimit(count: number) {
|
||||
if (count > MAX_SPREADSHEET_ROWS) {
|
||||
throw new HttpError(400, `Spreadsheet import can contain at most ${MAX_SPREADSHEET_ROWS} rows`, 'IMPORT_TOO_LARGE');
|
||||
}
|
||||
}
|
||||
|
||||
function decodeBase64File(body: JsonObject) {
|
||||
const raw = stringValue(body.fileBase64 ?? body.excelBase64 ?? body.csvBase64);
|
||||
if (!raw) return null;
|
||||
const base64 = raw.includes(',') ? raw.slice(raw.indexOf(',') + 1) : raw;
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
buffer = Buffer.from(base64, 'base64');
|
||||
} catch {
|
||||
throw new HttpError(400, 'fileBase64 must be a valid base64 string', 'IMPORT_FILE_BASE64_INVALID');
|
||||
}
|
||||
if (!buffer.length) {
|
||||
throw new HttpError(400, 'fileBase64 is empty', 'IMPORT_FILE_EMPTY');
|
||||
}
|
||||
if (buffer.length > MAX_EXCEL_FILE_BYTES) {
|
||||
throw new HttpError(400, `Import file can be at most ${MAX_EXCEL_FILE_BYTES} bytes`, 'IMPORT_FILE_TOO_LARGE');
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
|
||||
function primitiveCellValue(value: unknown): unknown {
|
||||
if (value === undefined || value === null) return undefined;
|
||||
if (value instanceof Date) return value.toISOString();
|
||||
if (typeof value === 'number' || typeof value === 'boolean') return value;
|
||||
if (typeof value === 'string') {
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > MAX_SPREADSHEET_CELL_CHARS ? trimmed.slice(0, MAX_SPREADSHEET_CELL_CHARS) : trimmed;
|
||||
}
|
||||
if (typeof value === 'object') {
|
||||
const cell = value as {
|
||||
text?: string;
|
||||
result?: unknown;
|
||||
formula?: string;
|
||||
richText?: Array<{ text?: string }>;
|
||||
hyperlink?: string;
|
||||
};
|
||||
if (cell.result !== undefined) return primitiveCellValue(cell.result);
|
||||
if (Array.isArray(cell.richText)) return cell.richText.map(part => part.text || '').join('');
|
||||
if (cell.text !== undefined) return primitiveCellValue(cell.text);
|
||||
if (cell.hyperlink !== undefined) return primitiveCellValue(cell.hyperlink);
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function parseJsonLike(value: unknown): unknown {
|
||||
if (typeof value !== 'string') return value;
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) return undefined;
|
||||
if (
|
||||
(trimmed.startsWith('[') && trimmed.endsWith(']')) ||
|
||||
(trimmed.startsWith('{') && trimmed.endsWith('}'))
|
||||
) {
|
||||
try {
|
||||
return JSON.parse(trimmed);
|
||||
} catch {
|
||||
return trimmed;
|
||||
}
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function splitList(value: unknown) {
|
||||
const parsed = parseJsonLike(value);
|
||||
if (Array.isArray(parsed)) return parsed.map(item => String(item).trim()).filter(Boolean);
|
||||
if (parsed === undefined || parsed === null) return [];
|
||||
return String(parsed)
|
||||
.split(/[|,，;；\n]/)
|
||||
.map(item => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function parseBooleanLike(value: unknown): unknown {
|
||||
if (typeof value === 'boolean') return value;
|
||||
if (typeof value !== 'string') return value;
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (['true', '1', 'yes', 'y', '是', '启用', '有效'].includes(normalized)) return true;
|
||||
if (['false', '0', 'no', 'n', '否', '禁用', '无效'].includes(normalized)) return false;
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseNumericLike(value: unknown): unknown {
|
||||
if (typeof value !== 'string') return value;
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) return undefined;
|
||||
if (!/^-?\d+(\.\d+)?$/.test(trimmed)) return value;
|
||||
const parsed = Number(trimmed);
|
||||
return Number.isFinite(parsed) ? parsed : value;
|
||||
}
|
||||
|
||||
function coerceValue(key: string, value: unknown) {
|
||||
if (value === undefined || value === null || value === '') return undefined;
|
||||
if ([
|
||||
'options',
|
||||
'tags',
|
||||
'correctOptionIndices',
|
||||
'knowledgeTags',
|
||||
].includes(key)) {
|
||||
return splitList(value);
|
||||
}
|
||||
if (['subQuestions', 'examMarkers', 'metadata', 'fieldValues', 'bindings'].includes(key)) {
|
||||
return parseJsonLike(value);
|
||||
}
|
||||
if (['isActive', 'isHot', 'isFilter', 'isRequired', 'isVisible', 'isTrend', 'hasRestriction', 'isGeneral'].includes(key)) {
|
||||
return parseBooleanLike(value);
|
||||
}
|
||||
if ([
|
||||
'difficulty',
|
||||
'order',
|
||||
'sortOrder',
|
||||
'duration',
|
||||
'durationSeconds',
|
||||
'freePreviewSeconds',
|
||||
'year',
|
||||
'wordCount',
|
||||
].includes(key)) {
|
||||
return parseNumericLike(value);
|
||||
}
|
||||
return parseJsonLike(value);
|
||||
}
|
||||
|
||||
function commonHeaderAlias(header: string) {
|
||||
const key = normalizedKey(header);
|
||||
const aliases: Record<string, string> = {
|
||||
id: 'legacyId',
|
||||
legacyid: 'legacyId',
|
||||
legacy_id: 'legacyId',
|
||||
externalid: 'legacyId',
|
||||
external_id: 'legacyId',
|
||||
旧id: 'legacyId',
|
||||
原id: 'legacyId',
|
||||
外部id: 'legacyId',
|
||||
排序: 'order',
|
||||
sort: 'order',
|
||||
sortorder: 'order',
|
||||
sort_order: 'order',
|
||||
是否启用: 'isActive',
|
||||
状态: 'isActive',
|
||||
标签: 'tags',
|
||||
tags: 'tags',
|
||||
metadata: 'metadata',
|
||||
元数据: 'metadata',
|
||||
};
|
||||
return aliases[key] || null;
|
||||
}
|
||||
|
||||
function importHeaderAlias(header: string, importType: SpreadsheetImportType) {
|
||||
const key = normalizedKey(header);
|
||||
const common = commonHeaderAlias(header);
|
||||
const aliases: Record<SpreadsheetImportType, Record<string, string>> = {
|
||||
questions: {
|
||||
type: 'type',
|
||||
questiontype: 'type',
|
||||
题型: 'type',
|
||||
typelabel: 'typeLabel',
|
||||
题型名称: 'typeLabel',
|
||||
content: 'content',
|
||||
title: 'content',
|
||||
题目: 'content',
|
||||
题干: 'content',
|
||||
问题: 'content',
|
||||
选项: 'options',
|
||||
options: 'options',
|
||||
a: 'optionA',
|
||||
选项a: 'optionA',
|
||||
optiona: 'optionA',
|
||||
b: 'optionB',
|
||||
选项b: 'optionB',
|
||||
optionb: 'optionB',
|
||||
c: 'optionC',
|
||||
选项c: 'optionC',
|
||||
optionc: 'optionC',
|
||||
d: 'optionD',
|
||||
选项d: 'optionD',
|
||||
optiond: 'optionD',
|
||||
e: 'optionE',
|
||||
选项e: 'optionE',
|
||||
optione: 'optionE',
|
||||
f: 'optionF',
|
||||
选项f: 'optionF',
|
||||
optionf: 'optionF',
|
||||
answer: 'answer',
|
||||
correctanswer: 'answer',
|
||||
correct: 'answer',
|
||||
答案: 'answer',
|
||||
正确答案: 'answer',
|
||||
正确选项: 'answer',
|
||||
correctoptionindices: 'correctOptionIndices',
|
||||
correct_option_indices: 'correctOptionIndices',
|
||||
explanation: 'explanation',
|
||||
解析: 'explanation',
|
||||
答案解析: 'explanation',
|
||||
difficulty: 'difficulty',
|
||||
难度: 'difficulty',
|
||||
mediaurl: 'mediaUrl',
|
||||
图片: 'mediaUrl',
|
||||
subquestions: 'subQuestions',
|
||||
子题: 'subQuestions',
|
||||
exammarkers: 'examMarkers',
|
||||
考试标记: 'examMarkers',
|
||||
},
|
||||
vocabulary: {
|
||||
unitlegacyid: 'unitLegacyId',
|
||||
unitid: 'unitLegacyId',
|
||||
单元id: 'unitLegacyId',
|
||||
unitname: 'unitName',
|
||||
unit: 'unitName',
|
||||
单元: 'unitName',
|
||||
单元名称: 'unitName',
|
||||
unitdescription: 'unitDescription',
|
||||
单元描述: 'unitDescription',
|
||||
unitorder: 'unitOrder',
|
||||
单元排序: 'unitOrder',
|
||||
wordlegacyid: 'wordLegacyId',
|
||||
wordid: 'wordLegacyId',
|
||||
单词id: 'wordLegacyId',
|
||||
word: 'word',
|
||||
单词: 'word',
|
||||
phonetic: 'phonetic',
|
||||
pronunciation: 'phonetic',
|
||||
音标: 'phonetic',
|
||||
meaning: 'meaning',
|
||||
translation: 'meaning',
|
||||
definition: 'meaning',
|
||||
释义: 'meaning',
|
||||
中文: 'meaning',
|
||||
意思: 'meaning',
|
||||
example: 'example',
|
||||
例句: 'example',
|
||||
exampletranslation: 'exampleTranslation',
|
||||
例句翻译: 'exampleTranslation',
|
||||
difficulty: 'difficulty',
|
||||
难度: 'difficulty',
|
||||
},
|
||||
handbook: {
|
||||
subjectlegacyid: 'subjectLegacyId',
|
||||
subjectid: 'subjectLegacyId',
|
||||
科目id: 'subjectLegacyId',
|
||||
subjectname: 'subjectName',
|
||||
subject: 'subjectName',
|
||||
科目: 'subjectName',
|
||||
手册: 'subjectName',
|
||||
chapterlegacyid: 'chapterLegacyId',
|
||||
chapterid: 'chapterLegacyId',
|
||||
章节id: 'chapterLegacyId',
|
||||
chaptername: 'chapterName',
|
||||
chapter: 'chapterName',
|
||||
章节: 'chapterName',
|
||||
sectionlegacyid: 'sectionLegacyId',
|
||||
sectionid: 'sectionLegacyId',
|
||||
小节id: 'sectionLegacyId',
|
||||
sectionname: 'sectionName',
|
||||
section: 'sectionName',
|
||||
小节: 'sectionName',
|
||||
entrylegacyid: 'entryLegacyId',
|
||||
entryid: 'entryLegacyId',
|
||||
知识点id: 'entryLegacyId',
|
||||
title: 'title',
|
||||
标题: 'title',
|
||||
知识点: 'title',
|
||||
content: 'content',
|
||||
正文: 'content',
|
||||
内容: 'content',
|
||||
summary: 'summary',
|
||||
摘要: 'summary',
|
||||
},
|
||||
scoreline: {
|
||||
kind: 'kind',
|
||||
itemtype: 'kind',
|
||||
类型: 'kind',
|
||||
fieldkey: 'fieldKey',
|
||||
字段key: 'fieldKey',
|
||||
fieldname: 'fieldName',
|
||||
字段名: 'fieldName',
|
||||
fieldtype: 'fieldType',
|
||||
字段类型: 'fieldType',
|
||||
unit: 'unit',
|
||||
单位: 'unit',
|
||||
schoolid: 'schoolId',
|
||||
院校id: 'schoolId',
|
||||
schoollegacyid: 'schoolLegacyId',
|
||||
schoolname: 'schoolName',
|
||||
school: 'schoolName',
|
||||
学校: 'schoolName',
|
||||
院校: 'schoolName',
|
||||
majorid: 'majorId',
|
||||
专业id: 'majorId',
|
||||
majorlegacyid: 'majorLegacyId',
|
||||
majorname: 'majorName',
|
||||
major: 'majorName',
|
||||
专业: 'majorName',
|
||||
year: 'year',
|
||||
年份: 'year',
|
||||
fieldvalues: 'fieldValues',
|
||||
动态字段: 'fieldValues',
|
||||
最低分: 'minScore',
|
||||
录取分: 'minScore',
|
||||
计划数: 'planCount',
|
||||
招生计划: 'planCount',
|
||||
},
|
||||
videos: {
|
||||
title: 'title',
|
||||
name: 'title',
|
||||
视频标题: 'title',
|
||||
标题: 'title',
|
||||
description: 'description',
|
||||
描述: 'description',
|
||||
videourl: 'videoUrl',
|
||||
url: 'videoUrl',
|
||||
视频地址: 'videoUrl',
|
||||
thumbnailurl: 'thumbnailUrl',
|
||||
封面: 'thumbnailUrl',
|
||||
duration: 'durationSeconds',
|
||||
durationseconds: 'durationSeconds',
|
||||
时长: 'durationSeconds',
|
||||
subjectid: 'subjectId',
|
||||
科目id: 'subjectId',
|
||||
questionid: 'questionId',
|
||||
题目id: 'questionId',
|
||||
legacyquestionid: 'legacyQuestionId',
|
||||
旧题目id: 'legacyQuestionId',
|
||||
assetid: 'assetId',
|
||||
资源id: 'assetId',
|
||||
accessmode: 'accessMode',
|
||||
访问模式: 'accessMode',
|
||||
knowledgetags: 'knowledgeTags',
|
||||
知识点标签: 'knowledgeTags',
|
||||
bindings: 'bindings',
|
||||
绑定题目: 'bindings',
|
||||
},
|
||||
};
|
||||
return aliases[importType][key] || common || header.trim();
|
||||
}
|
||||
|
||||
function normalizeSpreadsheetRow(raw: JsonObject, importType: SpreadsheetImportType) {
|
||||
const row: JsonObject = {};
|
||||
for (const [header, value] of Object.entries(raw)) {
|
||||
const key = importHeaderAlias(header, importType);
|
||||
let coerced = coerceValue(key, value);
|
||||
if (importType === 'scoreline' && key === header.trim()) {
|
||||
coerced = parseNumericLike(coerced);
|
||||
}
|
||||
if (coerced !== undefined && coerced !== '') row[key] = coerced;
|
||||
}
|
||||
|
||||
if (importType === 'questions') {
|
||||
const options = QUESTION_OPTION_KEYS.map(key => row[key]).filter(value => value !== undefined && value !== '');
|
||||
if (!row.options && options.length) row.options = options;
|
||||
if (!row.correctOptionIndices && row.answer) {
|
||||
row.correctOptionIndices = parseAnswerIndices(row.answer, Array.isArray(row.options) ? row.options.length : 0);
|
||||
}
|
||||
}
|
||||
|
||||
if (importType === 'vocabulary') {
|
||||
if (row.wordLegacyId && !row.legacyId) row.legacyId = row.wordLegacyId;
|
||||
if (row.unitOrder && !row.order && !row.word) row.order = row.unitOrder;
|
||||
}
|
||||
|
||||
if (importType === 'handbook') {
|
||||
if (row.entryLegacyId && !row.legacyId) row.legacyId = row.entryLegacyId;
|
||||
}
|
||||
|
||||
return row;
|
||||
}
|
||||
|
||||
function parseAnswerIndices(value: unknown, optionCount: number) {
|
||||
const parts = splitList(value);
|
||||
const indices: number[] = [];
|
||||
for (const part of parts) {
|
||||
const upper = String(part).trim().toUpperCase();
|
||||
if (/^[A-H]$/.test(upper)) {
|
||||
indices.push(upper.charCodeAt(0) - 65);
|
||||
continue;
|
||||
}
|
||||
const parsed = Number(upper);
|
||||
if (Number.isFinite(parsed)) {
|
||||
const integer = Math.trunc(parsed);
|
||||
indices.push(integer === 0 ? 0 : optionCount > 0 && integer <= optionCount ? integer - 1 : integer);
|
||||
}
|
||||
}
|
||||
return [...new Set(indices)];
|
||||
}
|
||||
|
||||
function parseCsv(text: string, delimiter: string) {
|
||||
const rows: string[][] = [];
|
||||
let row: string[] = [];
|
||||
let current = '';
|
||||
let quoted = false;
|
||||
for (let index = 0; index < text.length; index += 1) {
|
||||
const char = text[index];
|
||||
const next = text[index + 1];
|
||||
if (quoted) {
|
||||
if (char === '"' && next === '"') {
|
||||
current += '"';
|
||||
index += 1;
|
||||
} else if (char === '"') {
|
||||
quoted = false;
|
||||
} else {
|
||||
current += char;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (char === '"') {
|
||||
quoted = true;
|
||||
} else if (char === delimiter) {
|
||||
row.push(current);
|
||||
current = '';
|
||||
} else if (char === '\n') {
|
||||
row.push(current);
|
||||
rows.push(row);
|
||||
row = [];
|
||||
current = '';
|
||||
} else if (char !== '\r') {
|
||||
current += char;
|
||||
}
|
||||
}
|
||||
row.push(current);
|
||||
rows.push(row);
|
||||
return rows.filter(item => item.some(cell => cell.trim()));
|
||||
}
|
||||
|
||||
function autoDelimiter(text: string) {
|
||||
const sample = text.slice(0, 4096);
|
||||
const candidates = [',', '\t', ';'];
|
||||
return candidates
|
||||
.map(delimiter => ({ delimiter, count: (sample.match(new RegExp(delimiter === '\t' ? '\\t' : delimiter, 'g')) || []).length }))
|
||||
.sort((left, right) => right.count - left.count)[0]?.delimiter || ',';
|
||||
}
|
||||
|
||||
function rowsFromMatrix(matrix: string[][], importType: SpreadsheetImportType) {
|
||||
if (matrix.length < 2) return [];
|
||||
const headers = matrix[0].map((header, index) => stringValue(header) || `column${index + 1}`);
|
||||
if (headers.length > MAX_SPREADSHEET_COLUMNS) {
|
||||
throw new HttpError(400, `Spreadsheet can contain at most ${MAX_SPREADSHEET_COLUMNS} columns`, 'IMPORT_TOO_MANY_COLUMNS');
|
||||
}
|
||||
const rows = matrix.slice(1).map(line => {
|
||||
const row: JsonObject = {};
|
||||
headers.forEach((header, index) => {
|
||||
const value = stringValue(line[index]);
|
||||
if (value) row[header] = value;
|
||||
});
|
||||
return normalizeSpreadsheetRow(row, importType);
|
||||
}).filter(row => Object.keys(row).length > 0);
|
||||
assertRowLimit(rows.length);
|
||||
return rows;
|
||||
}
|
||||
|
||||
function parseCsvBody(body: JsonObject, importType: SpreadsheetImportType): ParsedSpreadsheet {
|
||||
const buffer = decodeBase64File(body);
|
||||
const text = stringValue(body.csvText ?? body.text ?? body.fileContent ?? body.payload) || buffer?.toString('utf8') || '';
|
||||
if (!text.trim()) {
|
||||
throw new HttpError(400, 'CSV import requires csvText, fileContent, payload, or fileBase64', 'CSV_CONTENT_REQUIRED');
|
||||
}
|
||||
const delimiter = stringValue(body.delimiter) || autoDelimiter(text);
|
||||
const rows = rowsFromMatrix(parseCsv(text, delimiter), importType);
|
||||
return {
|
||||
rows,
|
||||
sheets: { csv: rows },
|
||||
metadata: {
|
||||
parser: 'csv',
|
||||
delimiter: delimiter === '\t' ? 'tab' : delimiter,
|
||||
rowCount: rows.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function worksheetRows(worksheet: ExcelJS.Worksheet, importType: SpreadsheetImportType) {
|
||||
const matrix: string[][] = [];
|
||||
worksheet.eachRow({ includeEmpty: false }, row => {
|
||||
const values: string[] = [];
|
||||
const max = Math.min(row.cellCount, MAX_SPREADSHEET_COLUMNS);
|
||||
for (let col = 1; col <= max; col += 1) {
|
||||
const value = primitiveCellValue(row.getCell(col).value);
|
||||
values.push(value === undefined || value === null ? '' : String(value));
|
||||
}
|
||||
if (values.some(value => value.trim())) matrix.push(values);
|
||||
});
|
||||
return rowsFromMatrix(matrix, importType);
|
||||
}
|
||||
|
||||
async function parseExcelBody(body: JsonObject, importType: SpreadsheetImportType): Promise<ParsedSpreadsheet> {
|
||||
const buffer = decodeBase64File(body);
|
||||
if (!buffer) {
|
||||
throw new HttpError(400, 'Excel import requires fileBase64', 'EXCEL_FILE_REQUIRED');
|
||||
}
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
try {
|
||||
await workbook.xlsx.load(buffer as unknown as Parameters<typeof workbook.xlsx.load>[0]);
|
||||
} catch {
|
||||
throw new HttpError(400, 'Excel file could not be parsed as .xlsx', 'EXCEL_PARSE_FAILED');
|
||||
}
|
||||
|
||||
const sheetName = stringValue(body.sheetName);
|
||||
const sheetIndex = Number(body.sheetIndex ?? 1);
|
||||
let selected: ExcelJS.Worksheet | undefined;
|
||||
if (sheetName) selected = workbook.getWorksheet(sheetName);
|
||||
else if (Number.isFinite(sheetIndex) && sheetIndex > 0) selected = workbook.getWorksheet(Math.trunc(sheetIndex));
|
||||
selected = selected || workbook.worksheets.find(sheet => sheet.actualRowCount > 0);
|
||||
if (!selected) {
|
||||
throw new HttpError(400, 'Excel file has no non-empty worksheet', 'EXCEL_SHEET_EMPTY');
|
||||
}
|
||||
|
||||
const sheets: Record<string, JsonObject[]> = {};
|
||||
let totalRows = 0;
|
||||
for (const sheet of workbook.worksheets) {
|
||||
if (sheet.actualRowCount === 0) continue;
|
||||
const rows = worksheetRows(sheet, importType);
|
||||
sheets[normalizedKey(sheet.name)] = rows;
|
||||
totalRows += rows.length;
|
||||
}
|
||||
assertRowLimit(totalRows);
|
||||
|
||||
const selectedRows = sheets[normalizedKey(selected.name)] || [];
|
||||
return {
|
||||
rows: selectedRows,
|
||||
sheets,
|
||||
metadata: {
|
||||
parser: 'exceljs',
|
||||
workbookSheetCount: workbook.worksheets.length,
|
||||
selectedSheet: selected.name,
|
||||
rowCount: selectedRows.length,
|
||||
totalParsedRows: totalRows,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function sheetRows(parsed: ParsedSpreadsheet, names: string[]) {
|
||||
for (const name of names) {
|
||||
const rows = parsed.sheets[normalizedKey(name)];
|
||||
if (rows?.length) return rows;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function stableLegacyId(value: unknown, fallback: string) {
|
||||
return stringValue(value) || fallback;
|
||||
}
|
||||
|
||||
function buildVocabularyPayload(parsed: ParsedSpreadsheet) {
|
||||
const explicitUnitRows = sheetRows(parsed, ['units', 'vocabulary_units', '单元', '单词单元']);
|
||||
const explicitWordRows = sheetRows(parsed, ['words', 'vocabulary', 'vocabulary_words', '单词', '词汇']);
|
||||
const rows = explicitWordRows.length || explicitUnitRows.length ? [...explicitUnitRows, ...explicitWordRows] : parsed.rows;
|
||||
const units = new Map<string, JsonObject & { words: JsonObject[] }>();
|
||||
|
||||
for (const row of rows) {
|
||||
const rowType = String(row.rowType ?? row.kind ?? row.type ?? '').toLowerCase();
|
||||
const hasWord = Boolean(row.word);
|
||||
const unitName = stringValue(row.unitName ?? row.unit) || '默认单词单元';
|
||||
const unitKey = stableLegacyId(row.unitLegacyId ?? row.unitId, unitName);
|
||||
const unit = units.get(unitKey) || {
|
||||
legacyId: unitKey,
|
||||
name: unitName,
|
||||
description: row.unitDescription ?? row.description ?? null,
|
||||
order: row.unitOrder ?? row.order ?? units.size + 1,
|
||||
isActive: row.isActive ?? true,
|
||||
words: [],
|
||||
};
|
||||
if (rowType.includes('unit') && !hasWord) {
|
||||
unit.name = unitName;
|
||||
unit.description = row.unitDescription ?? row.description ?? unit.description;
|
||||
unit.order = row.unitOrder ?? row.order ?? unit.order;
|
||||
unit.isActive = row.isActive ?? unit.isActive;
|
||||
}
|
||||
if (hasWord) {
|
||||
const word = { ...row };
|
||||
word.legacyId = row.wordLegacyId ?? row.legacyId ?? null;
|
||||
delete word.unitLegacyId;
|
||||
delete word.unitId;
|
||||
delete word.unitName;
|
||||
delete word.unitDescription;
|
||||
delete word.unitOrder;
|
||||
delete word.wordLegacyId;
|
||||
unit.words.push(word);
|
||||
}
|
||||
units.set(unitKey, unit);
|
||||
}
|
||||
|
||||
return { units: Array.from(units.values()) };
|
||||
}
|
||||
|
||||
function getOrCreateNested<T extends JsonObject>(
|
||||
map: Map<string, T>,
|
||||
key: string,
|
||||
create: () => T,
|
||||
) {
|
||||
const existing = map.get(key);
|
||||
if (existing) return existing;
|
||||
const created = create();
|
||||
map.set(key, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
function buildHandbookPayload(parsed: ParsedSpreadsheet) {
|
||||
const rows = sheetRows(parsed, ['entries', 'handbook', '知识点', '手册']) || parsed.rows;
|
||||
const subjectMap = new Map<string, JsonObject & { chapters: Array<JsonObject & { sections: Array<JsonObject & { entries: JsonObject[] }> }> }>();
|
||||
|
||||
for (const row of rows.length ? rows : parsed.rows) {
|
||||
const subjectName = stringValue(row.subjectName ?? row.subject) || '默认知识手册';
|
||||
const subjectKey = stableLegacyId(row.subjectLegacyId ?? row.subjectId, subjectName);
|
||||
const subject = getOrCreateNested(subjectMap, subjectKey, () => ({
|
||||
legacyId: subjectKey,
|
||||
name: subjectName,
|
||||
type: row.subjectType ?? row.type ?? null,
|
||||
icon: row.icon ?? null,
|
||||
color: row.color ?? null,
|
||||
order: subjectMap.size + 1,
|
||||
chapters: [],
|
||||
}));
|
||||
|
||||
const chapterName = stringValue(row.chapterName ?? row.chapter) || '默认章节';
|
||||
const chapterKey = stableLegacyId(row.chapterLegacyId ?? row.chapterId, `${subjectKey}:${chapterName}`);
|
||||
let chapter = subject.chapters.find(item => item.legacyId === chapterKey);
|
||||
if (!chapter) {
|
||||
chapter = { legacyId: chapterKey, name: chapterName, order: subject.chapters.length + 1, sections: [] };
|
||||
subject.chapters.push(chapter);
|
||||
}
|
||||
|
||||
const sectionName = stringValue(row.sectionName ?? row.section) || '';
|
||||
const sectionKey = stableLegacyId(row.sectionLegacyId ?? row.sectionId, `${chapterKey}:${sectionName || 'default'}`);
|
||||
let section = chapter.sections.find(item => item.legacyId === sectionKey);
|
||||
if (!section) {
|
||||
section = { legacyId: sectionKey, name: sectionName || null, order: chapter.sections.length + 1, entries: [] };
|
||||
chapter.sections.push(section);
|
||||
}
|
||||
|
||||
if (row.title || row.content) {
|
||||
section.entries.push({
|
||||
legacyId: row.entryLegacyId ?? row.legacyId ?? null,
|
||||
title: row.title ?? '未命名知识点',
|
||||
summary: row.summary ?? null,
|
||||
content: row.content ?? '',
|
||||
tags: row.tags ?? [],
|
||||
order: row.order ?? section.entries.length + 1,
|
||||
isActive: row.isActive ?? true,
|
||||
metadata: row.metadata ?? {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { subjects: Array.from(subjectMap.values()) };
|
||||
}
|
||||
|
||||
function buildScorelinePayload(parsed: ParsedSpreadsheet) {
|
||||
const buckets = {
|
||||
fields: sheetRows(parsed, ['fields', 'scoreline_fields', '字段']),
|
||||
schools: sheetRows(parsed, ['schools', 'scoreline_schools', '院校', '学校']),
|
||||
majors: sheetRows(parsed, ['majors', 'scoreline_majors', '专业']),
|
||||
records: sheetRows(parsed, ['records', 'scoreline_records', '分数线', '记录']),
|
||||
};
|
||||
if (Object.values(buckets).some(rows => rows.length > 0)) return buckets;
|
||||
return { items: parsed.rows };
|
||||
}
|
||||
|
||||
function buildVideosPayload(parsed: ParsedSpreadsheet) {
|
||||
const rows = sheetRows(parsed, ['videos', 'video_explanations', '视频']) || parsed.rows;
|
||||
return { videos: rows.length ? rows : parsed.rows };
|
||||
}
|
||||
|
||||
export async function expandSpreadsheetImportBody(body: JsonObject, importType: SpreadsheetImportType): Promise<JsonObject> {
|
||||
const sourceFormat = sourceFormatValue(body.sourceFormat);
|
||||
if (sourceFormat === 'json') return body;
|
||||
|
||||
const parsed = sourceFormat === 'csv'
|
||||
? parseCsvBody(body, importType)
|
||||
: await parseExcelBody(body, importType);
|
||||
|
||||
if (!parsed.rows.length && !Object.values(parsed.sheets).some(rows => rows.length > 0)) {
|
||||
throw new HttpError(400, 'Spreadsheet import payload must contain at least one data row', 'EMPTY_IMPORT_PAYLOAD');
|
||||
}
|
||||
|
||||
const next: JsonObject = {
|
||||
...body,
|
||||
sourceFormat,
|
||||
spreadsheet: parsed.metadata,
|
||||
};
|
||||
delete next.csvText;
|
||||
delete next.text;
|
||||
delete next.fileContent;
|
||||
delete next.fileBase64;
|
||||
delete next.excelBase64;
|
||||
delete next.csvBase64;
|
||||
delete next.payload;
|
||||
|
||||
if (importType === 'questions') next.items = parsed.rows;
|
||||
if (importType === 'vocabulary') Object.assign(next, buildVocabularyPayload(parsed));
|
||||
if (importType === 'handbook') Object.assign(next, buildHandbookPayload(parsed));
|
||||
if (importType === 'scoreline') Object.assign(next, buildScorelinePayload(parsed));
|
||||
if (importType === 'videos') Object.assign(next, buildVideosPayload(parsed));
|
||||
|
||||
return next;
|
||||
}
|
||||
@@ -1,11 +1,9 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import type pg from 'pg';
|
||||
import { config } from '../../core/config.js';
|
||||
import { HttpError, type RequestContext } from '../../core/http.js';
|
||||
import { intParam, readJsonBody, requiredString, stringParam } from '../../core/request.js';
|
||||
import { HttpError } from '../../core/errors.js';
|
||||
import type { RequestContext } from '../../core/http.js';
|
||||
import { query, queryOne, transaction } from '../../core/db.js';
|
||||
import { requireTenantContentEditor, type TenantContentAuth } from './auth.js';
|
||||
import { boolValue, nullableString } from './utils.js';
|
||||
import type { TenantContentAuth } from './auth.js';
|
||||
|
||||
type JsonObject = Record<string, unknown>;
|
||||
|
||||
@@ -114,8 +112,10 @@ interface NormalizedHandbookSubject {
|
||||
}
|
||||
|
||||
type ContentImportType = 'vocabulary' | 'handbook' | 'scoreline' | 'videos';
|
||||
export type ExecutableContentImportType = 'questions' | ContentImportType;
|
||||
type EntryBoundImportType = 'vocabulary' | 'handbook';
|
||||
type GenericTargetEntryType = EntryBoundImportType | null;
|
||||
type SourceImportFormat = 'json' | 'csv' | 'excel';
|
||||
|
||||
type ScorelineImportKind = 'field' | 'school' | 'major' | 'record';
|
||||
|
||||
@@ -227,6 +227,19 @@ function stringValue(value: unknown) {
|
||||
return typeof value === 'string' && value.trim() ? value.trim() : '';
|
||||
}
|
||||
|
||||
function sourceImportFormat(value: unknown): SourceImportFormat {
|
||||
const normalized = stringValue(value).toLowerCase();
|
||||
if (!normalized || normalized === 'json') return 'json';
|
||||
if (normalized === 'csv') return 'csv';
|
||||
if (['excel', 'xlsx'].includes(normalized)) return 'excel';
|
||||
throw new HttpError(400, 'sourceFormat must be json, csv, or excel', 'IMPORT_FORMAT_NOT_SUPPORTED');
|
||||
}
|
||||
|
||||
function isAsyncExecution(value: unknown) {
|
||||
const normalized = stringValue(value).toLowerCase();
|
||||
return ['async', 'queued', 'queue'].includes(normalized);
|
||||
}
|
||||
|
||||
function stringArrayValue(value: unknown) {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.map(item => String(item).trim()).filter(Boolean);
|
||||
@@ -291,6 +304,40 @@ function boolishValue(value: unknown, fallback: boolean) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function boolValue(value: unknown, fallback: boolean) {
|
||||
return typeof value === 'boolean' ? value : fallback;
|
||||
}
|
||||
|
||||
function nullableString(value: unknown) {
|
||||
return typeof value === 'string' && value.trim() ? value.trim() : null;
|
||||
}
|
||||
|
||||
function requiredString(body: JsonObject, key: string) {
|
||||
const value = body[key];
|
||||
if (typeof value !== 'string' || !value.trim()) {
|
||||
throw new HttpError(400, `${key} is required`, 'REQUIRED_FIELD');
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
async function routeDeps() {
|
||||
const [{ config }, request, auth, spreadsheet] = await Promise.all([
|
||||
import('../../core/config.js'),
|
||||
import('../../core/request.js'),
|
||||
import('./auth.js'),
|
||||
import('./import-spreadsheet.js'),
|
||||
]);
|
||||
return {
|
||||
config,
|
||||
intParam: request.intParam,
|
||||
readJsonBody: request.readJsonBody,
|
||||
requiredString: request.requiredString,
|
||||
stringParam: request.stringParam,
|
||||
requireTenantContentEditor: auth.requireTenantContentEditor,
|
||||
expandSpreadsheetImportBody: spreadsheet.expandSpreadsheetImportBody,
|
||||
};
|
||||
}
|
||||
|
||||
function integerValue(value: unknown, fallback: number) {
|
||||
const numberValue = Number(value ?? fallback);
|
||||
return Number.isFinite(numberValue) ? Math.trunc(numberValue) : fallback;
|
||||
@@ -647,13 +694,9 @@ async function assertTargetReferences(client: pg.PoolClient, auth: TenantContent
|
||||
|
||||
async function createQuestionPreviewJob(auth: TenantContentAuth, body: JsonObject): Promise<PreviewResult> {
|
||||
const rawItems = parseQuestionItems(body);
|
||||
const sourceFormat = stringValue(body.sourceFormat) || 'json';
|
||||
const sourceFormat = sourceImportFormat(body.sourceFormat);
|
||||
const sourceName = stringValue(body.sourceName) || null;
|
||||
|
||||
if (sourceFormat !== 'json') {
|
||||
throw new HttpError(400, 'Only json sourceFormat is supported by the synchronous API for now', 'IMPORT_FORMAT_NOT_SUPPORTED');
|
||||
}
|
||||
|
||||
return transaction(async client => {
|
||||
const target = await assertTargetReferences(client, auth, body);
|
||||
const normalizedItems = rawItems.map((raw, index) => {
|
||||
@@ -684,7 +727,7 @@ async function createQuestionPreviewJob(auth: TenantContentAuth, body: JsonObjec
|
||||
target_category_id, target_node_id, target_question_bank_id,
|
||||
target_entry_id, target_content_node_id, target_collection_id,
|
||||
dry_run, total_count, valid_count, error_count, warning_count,
|
||||
summary, raw_payload, normalized_payload
|
||||
summary, raw_payload, normalized_payload, parser_metadata
|
||||
)
|
||||
values (
|
||||
$1, $2, 'questions', $3, 'preview',
|
||||
@@ -692,7 +735,7 @@ async function createQuestionPreviewJob(auth: TenantContentAuth, body: JsonObjec
|
||||
$8::uuid, $9::uuid, $10::uuid,
|
||||
$11::uuid, $12::uuid, $13::uuid,
|
||||
true, $14, $15, $16, $17,
|
||||
$18::jsonb, $19::jsonb, $20::jsonb
|
||||
$18::jsonb, $19::jsonb, $20::jsonb, $21::jsonb
|
||||
)
|
||||
returning id, status, total_count as "totalCount", valid_count as "validCount",
|
||||
error_count as "errorCount", warning_count as "warningCount"
|
||||
@@ -718,6 +761,7 @@ async function createQuestionPreviewJob(auth: TenantContentAuth, body: JsonObjec
|
||||
JSON.stringify({ target, generatedAt: new Date().toISOString() }),
|
||||
rawPayload,
|
||||
normalizedPayload,
|
||||
JSON.stringify(objectValue(body.spreadsheet)),
|
||||
],
|
||||
);
|
||||
|
||||
@@ -1659,11 +1703,8 @@ async function createGenericPreviewJob<T>(
|
||||
normalizedItems: NormalizedImportItem<T>[],
|
||||
expectedEntryType: GenericTargetEntryType = importType === 'vocabulary' || importType === 'handbook' ? importType : null,
|
||||
): Promise<PreviewResult<T>> {
|
||||
const sourceFormat = stringValue(body.sourceFormat) || 'json';
|
||||
const sourceFormat = sourceImportFormat(body.sourceFormat);
|
||||
const sourceName = stringValue(body.sourceName) || null;
|
||||
if (sourceFormat !== 'json') {
|
||||
throw new HttpError(400, 'Only json sourceFormat is supported by the synchronous API for now', 'IMPORT_FORMAT_NOT_SUPPORTED');
|
||||
}
|
||||
|
||||
return transaction(async client => {
|
||||
const target = await assertGenericTargetReferences(client, auth, body, expectedEntryType);
|
||||
@@ -1678,13 +1719,13 @@ async function createGenericPreviewJob<T>(
|
||||
tenant_id, created_by, import_type, source_format, status,
|
||||
source_name, source_hash, target_region_id, target_entry_id,
|
||||
target_content_node_id, dry_run, total_count, valid_count,
|
||||
error_count, warning_count, summary, raw_payload, normalized_payload
|
||||
error_count, warning_count, summary, raw_payload, normalized_payload, parser_metadata
|
||||
)
|
||||
values (
|
||||
$1, $2, $3, $4, 'preview',
|
||||
$5, $6, $7::uuid, $8::uuid,
|
||||
$9::uuid, true, $10, $11,
|
||||
$12, $13, $14::jsonb, $15::jsonb, $16::jsonb
|
||||
$12, $13, $14::jsonb, $15::jsonb, $16::jsonb, $17::jsonb
|
||||
)
|
||||
returning id, status, total_count as "totalCount", valid_count as "validCount",
|
||||
error_count as "errorCount", warning_count as "warningCount"
|
||||
@@ -1706,6 +1747,7 @@ async function createGenericPreviewJob<T>(
|
||||
JSON.stringify({ target, generatedAt: new Date().toISOString() }),
|
||||
JSON.stringify(normalizedItems.map(item => item.source)),
|
||||
JSON.stringify(normalizedItems.map(item => item.normalized).filter(Boolean)),
|
||||
JSON.stringify(objectValue(body.spreadsheet)),
|
||||
],
|
||||
);
|
||||
|
||||
@@ -3052,196 +3094,31 @@ async function importOneVideo(
|
||||
return { insertedCount, updatedCount, skippedCount };
|
||||
}
|
||||
|
||||
async function runGenericImport<T>(
|
||||
interface ContentImportExecutionOptions {
|
||||
jobId: string;
|
||||
importType: ExecutableContentImportType;
|
||||
allowPartial?: boolean;
|
||||
allowQueuedJob?: boolean;
|
||||
}
|
||||
|
||||
interface ContentImportExecutionResult {
|
||||
jobId: string;
|
||||
status: string;
|
||||
idempotent?: boolean;
|
||||
insertedCount: number;
|
||||
updatedCount: number;
|
||||
skippedCount: number;
|
||||
errorCount?: number;
|
||||
warningCount?: number;
|
||||
}
|
||||
|
||||
async function executeQuestionsImportJob(
|
||||
auth: TenantContentAuth,
|
||||
body: JsonObject,
|
||||
importType: ContentImportType,
|
||||
createPreview: () => Promise<PreviewResult<T>>,
|
||||
importOne: (
|
||||
client: pg.PoolClient,
|
||||
auth: TenantContentAuth,
|
||||
job: {
|
||||
id: string;
|
||||
target_region_id: string | null;
|
||||
target_entry_id: string | null;
|
||||
target_content_node_id: string | null;
|
||||
},
|
||||
item: {
|
||||
id: string;
|
||||
row_no: number;
|
||||
normalized_payload: T;
|
||||
},
|
||||
) => Promise<{ insertedCount: number; updatedCount: number; skippedCount: number }>,
|
||||
input: Omit<ContentImportExecutionOptions, 'importType'>,
|
||||
) {
|
||||
const allowPartial = boolValue(body.allowPartial, false);
|
||||
const jobId = nullableString(body.previewJobId) || nullableString(body.jobId);
|
||||
const createdPreview = jobId ? null : await createPreview();
|
||||
const finalJobId = jobId || createdPreview?.job.id || '';
|
||||
|
||||
const result = await transaction(async client => {
|
||||
const job = await loadGenericPreviewJob(client, auth, finalJobId, importType);
|
||||
if (job.status === 'completed' || job.status === 'completed_with_errors') {
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: job.status,
|
||||
idempotent: true,
|
||||
insertedCount: 0,
|
||||
updatedCount: 0,
|
||||
skippedCount: 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (job.error_count > 0 && !allowPartial) {
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = 'rejected', error_message = 'Preview contains validation errors', updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[auth.tenantId, job.id],
|
||||
);
|
||||
throw new HttpError(409, 'Preview contains validation errors. Fix issues or set allowPartial=true.', 'IMPORT_HAS_ERRORS');
|
||||
}
|
||||
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = 'importing', dry_run = false, started_at = coalesce(started_at, now()), updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[auth.tenantId, job.id],
|
||||
);
|
||||
|
||||
const itemResult = await client.query<{
|
||||
id: string;
|
||||
row_no: number;
|
||||
normalized_payload: T;
|
||||
}>(
|
||||
`
|
||||
select id, row_no, normalized_payload
|
||||
from public.content_import_items
|
||||
where tenant_id = $1 and job_id = $2 and status = 'valid'
|
||||
order by
|
||||
case
|
||||
when $3 = 'scoreline' then
|
||||
case normalized_payload ->> 'kind'
|
||||
when 'field' then 1
|
||||
when 'school' then 2
|
||||
when 'major' then 3
|
||||
when 'record' then 4
|
||||
else 9
|
||||
end
|
||||
else 1
|
||||
end,
|
||||
row_no asc
|
||||
for update
|
||||
`,
|
||||
[auth.tenantId, job.id, importType],
|
||||
);
|
||||
|
||||
let insertedCount = 0;
|
||||
let updatedCount = 0;
|
||||
let skippedCount = 0;
|
||||
for (const item of itemResult.rows) {
|
||||
const status = await importOne(client, auth, job, item);
|
||||
insertedCount += status.insertedCount;
|
||||
updatedCount += status.updatedCount;
|
||||
skippedCount += status.skippedCount;
|
||||
}
|
||||
|
||||
const finalStatus = job.error_count > 0 ? 'completed_with_errors' : 'completed';
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = $3,
|
||||
inserted_count = $4,
|
||||
updated_count = $5,
|
||||
skipped_count = $6,
|
||||
summary = coalesce(summary, '{}'::jsonb) || $7::jsonb,
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[
|
||||
auth.tenantId,
|
||||
job.id,
|
||||
finalStatus,
|
||||
insertedCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
JSON.stringify({ insertedCount, updatedCount, skippedCount, importedAt: new Date().toISOString() }),
|
||||
],
|
||||
);
|
||||
|
||||
await client.query(
|
||||
`
|
||||
insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details)
|
||||
values ($1, $2, $3, 'content_import_job', $4, $5::jsonb)
|
||||
`,
|
||||
[
|
||||
auth.tenantId,
|
||||
auth.userId,
|
||||
`content.import.${importType}.completed`,
|
||||
job.id,
|
||||
JSON.stringify({ insertedCount, updatedCount, skippedCount, allowPartial }),
|
||||
],
|
||||
);
|
||||
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: finalStatus,
|
||||
insertedCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
errorCount: job.error_count,
|
||||
warningCount: job.warning_count,
|
||||
};
|
||||
});
|
||||
|
||||
return { item: result, preview: createdPreview };
|
||||
}
|
||||
|
||||
export async function previewQuestionsImportRoute(ctx: RequestContext) {
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
return createQuestionPreviewJob(auth, body);
|
||||
}
|
||||
|
||||
export async function previewVocabularyImportRoute(ctx: RequestContext) {
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
return createGenericPreviewJob(auth, body, 'vocabulary', 'vocabulary_unit', createVocabularyNormalizedItems(body));
|
||||
}
|
||||
|
||||
export async function previewHandbookImportRoute(ctx: RequestContext) {
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
return createGenericPreviewJob(auth, body, 'handbook', 'handbook_subject', createHandbookNormalizedItems(body));
|
||||
}
|
||||
|
||||
export async function previewScorelineImportRoute(ctx: RequestContext) {
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
return createGenericPreviewJob(auth, body, 'scoreline', 'scoreline_item', createScorelineNormalizedItems(body), null);
|
||||
}
|
||||
|
||||
export async function previewVideosImportRoute(ctx: RequestContext) {
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
return createGenericPreviewJob(auth, body, 'videos', 'video_explanation', createVideoNormalizedItems(body), null);
|
||||
}
|
||||
|
||||
export async function importQuestionsRoute(ctx: RequestContext) {
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const allowPartial = boolValue(body.allowPartial, false);
|
||||
const jobId = nullableString(body.previewJobId) || nullableString(body.jobId);
|
||||
|
||||
const createdPreview = jobId ? null : await createQuestionPreviewJob(auth, body);
|
||||
const finalJobId = jobId || createdPreview?.job.id || '';
|
||||
|
||||
const result = await transaction(async client => {
|
||||
const job = await loadPreviewJob(client, auth, finalJobId);
|
||||
const allowPartial = input.allowPartial === true;
|
||||
return transaction(async client => {
|
||||
const job = await loadPreviewJob(client, auth, input.jobId);
|
||||
|
||||
if (job.status === 'completed' || job.status === 'completed_with_errors') {
|
||||
return {
|
||||
@@ -3253,12 +3130,19 @@ export async function importQuestionsRoute(ctx: RequestContext) {
|
||||
skippedCount: 0,
|
||||
};
|
||||
}
|
||||
if (job.status === 'pending' && input.allowQueuedJob !== true) {
|
||||
throw new HttpError(409, 'Import job is queued for async processing', 'IMPORT_JOB_QUEUED');
|
||||
}
|
||||
|
||||
if (job.error_count > 0 && !allowPartial) {
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = 'rejected', error_message = 'Preview contains validation errors', updated_at = now()
|
||||
set status = 'rejected',
|
||||
error_message = 'Preview contains validation errors',
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[auth.tenantId, job.id],
|
||||
@@ -3310,6 +3194,8 @@ export async function importQuestionsRoute(ctx: RequestContext) {
|
||||
skipped_count = $6,
|
||||
summary = coalesce(summary, '{}'::jsonb) || $7::jsonb,
|
||||
finished_at = now(),
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
@@ -3342,13 +3228,389 @@ export async function importQuestionsRoute(ctx: RequestContext) {
|
||||
warningCount: job.warning_count,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function executeGenericImportJob<T>(
|
||||
auth: TenantContentAuth,
|
||||
input: Omit<ContentImportExecutionOptions, 'importType'> & { importType: ContentImportType },
|
||||
importOne: (
|
||||
client: pg.PoolClient,
|
||||
auth: TenantContentAuth,
|
||||
job: {
|
||||
id: string;
|
||||
target_region_id: string | null;
|
||||
target_entry_id: string | null;
|
||||
target_content_node_id: string | null;
|
||||
},
|
||||
item: {
|
||||
id: string;
|
||||
row_no: number;
|
||||
normalized_payload: T;
|
||||
},
|
||||
) => Promise<{ insertedCount: number; updatedCount: number; skippedCount: number }>,
|
||||
) {
|
||||
const allowPartial = input.allowPartial === true;
|
||||
return transaction(async client => {
|
||||
const job = await loadGenericPreviewJob(client, auth, input.jobId, input.importType);
|
||||
if (job.status === 'completed' || job.status === 'completed_with_errors') {
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: job.status,
|
||||
idempotent: true,
|
||||
insertedCount: 0,
|
||||
updatedCount: 0,
|
||||
skippedCount: 0,
|
||||
};
|
||||
}
|
||||
if (job.status === 'pending' && input.allowQueuedJob !== true) {
|
||||
throw new HttpError(409, 'Import job is queued for async processing', 'IMPORT_JOB_QUEUED');
|
||||
}
|
||||
|
||||
if (job.error_count > 0 && !allowPartial) {
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = 'rejected',
|
||||
error_message = 'Preview contains validation errors',
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[auth.tenantId, job.id],
|
||||
);
|
||||
throw new HttpError(409, 'Preview contains validation errors. Fix issues or set allowPartial=true.', 'IMPORT_HAS_ERRORS');
|
||||
}
|
||||
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = 'importing', dry_run = false, started_at = coalesce(started_at, now()), updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[auth.tenantId, job.id],
|
||||
);
|
||||
|
||||
const itemResult = await client.query<{
|
||||
id: string;
|
||||
row_no: number;
|
||||
normalized_payload: T;
|
||||
}>(
|
||||
`
|
||||
select id, row_no, normalized_payload
|
||||
from public.content_import_items
|
||||
where tenant_id = $1 and job_id = $2 and status = 'valid'
|
||||
order by
|
||||
case
|
||||
when $3 = 'scoreline' then
|
||||
case normalized_payload ->> 'kind'
|
||||
when 'field' then 1
|
||||
when 'school' then 2
|
||||
when 'major' then 3
|
||||
when 'record' then 4
|
||||
else 9
|
||||
end
|
||||
else 1
|
||||
end,
|
||||
row_no asc
|
||||
for update
|
||||
`,
|
||||
[auth.tenantId, job.id, input.importType],
|
||||
);
|
||||
|
||||
let insertedCount = 0;
|
||||
let updatedCount = 0;
|
||||
let skippedCount = 0;
|
||||
for (const item of itemResult.rows) {
|
||||
const status = await importOne(client, auth, job, item);
|
||||
insertedCount += status.insertedCount;
|
||||
updatedCount += status.updatedCount;
|
||||
skippedCount += status.skippedCount;
|
||||
}
|
||||
|
||||
const finalStatus = job.error_count > 0 ? 'completed_with_errors' : 'completed';
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = $3,
|
||||
inserted_count = $4,
|
||||
updated_count = $5,
|
||||
skipped_count = $6,
|
||||
summary = coalesce(summary, '{}'::jsonb) || $7::jsonb,
|
||||
finished_at = now(),
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[
|
||||
auth.tenantId,
|
||||
job.id,
|
||||
finalStatus,
|
||||
insertedCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
JSON.stringify({ insertedCount, updatedCount, skippedCount, importedAt: new Date().toISOString() }),
|
||||
],
|
||||
);
|
||||
|
||||
await client.query(
|
||||
`
|
||||
insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details)
|
||||
values ($1, $2, $3, 'content_import_job', $4, $5::jsonb)
|
||||
`,
|
||||
[
|
||||
auth.tenantId,
|
||||
auth.userId,
|
||||
`content.import.${input.importType}.completed`,
|
||||
job.id,
|
||||
JSON.stringify({ insertedCount, updatedCount, skippedCount, allowPartial }),
|
||||
],
|
||||
);
|
||||
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: finalStatus,
|
||||
insertedCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
errorCount: job.error_count,
|
||||
warningCount: job.warning_count,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function executeContentImportJob(
|
||||
auth: TenantContentAuth,
|
||||
input: ContentImportExecutionOptions,
|
||||
): Promise<ContentImportExecutionResult> {
|
||||
if (input.importType === 'questions') {
|
||||
return executeQuestionsImportJob(auth, input);
|
||||
}
|
||||
if (input.importType === 'vocabulary') {
|
||||
return executeGenericImportJob(auth, { ...input, importType: 'vocabulary' }, importOneVocabularyUnit);
|
||||
}
|
||||
if (input.importType === 'handbook') {
|
||||
return executeGenericImportJob(auth, { ...input, importType: 'handbook' }, importOneHandbookSubject);
|
||||
}
|
||||
if (input.importType === 'scoreline') {
|
||||
return executeGenericImportJob(auth, { ...input, importType: 'scoreline' }, importOneScorelineItem);
|
||||
}
|
||||
if (input.importType === 'videos') {
|
||||
return executeGenericImportJob(auth, { ...input, importType: 'videos' }, importOneVideo);
|
||||
}
|
||||
throw new HttpError(400, 'Unsupported import type', 'IMPORT_TYPE_NOT_SUPPORTED');
|
||||
}
|
||||
|
||||
async function queueContentImportJob(
|
||||
auth: TenantContentAuth,
|
||||
input: ContentImportExecutionOptions,
|
||||
) {
|
||||
const allowPartial = input.allowPartial === true;
|
||||
return transaction(async client => {
|
||||
const result = await client.query<{
|
||||
id: string;
|
||||
status: string;
|
||||
error_count: number;
|
||||
warning_count: number;
|
||||
attempt_count: number;
|
||||
max_attempts: number;
|
||||
}>(
|
||||
`
|
||||
select id, status, error_count, warning_count, attempt_count, max_attempts
|
||||
from public.content_import_jobs
|
||||
where tenant_id = $1 and id = $2 and import_type = $3
|
||||
limit 1
|
||||
for update
|
||||
`,
|
||||
[auth.tenantId, input.jobId, input.importType],
|
||||
);
|
||||
const job = result.rows[0];
|
||||
if (!job) throw new HttpError(404, 'Import job not found', 'IMPORT_JOB_NOT_FOUND');
|
||||
if (job.status === 'completed' || job.status === 'completed_with_errors') {
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: job.status,
|
||||
idempotent: true,
|
||||
executionMode: 'async',
|
||||
errorCount: job.error_count,
|
||||
warningCount: job.warning_count,
|
||||
};
|
||||
}
|
||||
if (job.status === 'pending') {
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: job.status,
|
||||
idempotent: true,
|
||||
executionMode: 'async',
|
||||
errorCount: job.error_count,
|
||||
warningCount: job.warning_count,
|
||||
};
|
||||
}
|
||||
if (job.status === 'importing') {
|
||||
throw new HttpError(409, 'Import job is already importing', 'IMPORT_JOB_NOT_READY');
|
||||
}
|
||||
if (['failed', 'rejected'].includes(job.status)) {
|
||||
throw new HttpError(409, `Import job is ${job.status}`, 'IMPORT_JOB_NOT_READY');
|
||||
}
|
||||
if (job.error_count > 0 && !allowPartial) {
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = 'rejected',
|
||||
error_message = 'Preview contains validation errors',
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[auth.tenantId, job.id],
|
||||
);
|
||||
throw new HttpError(409, 'Preview contains validation errors. Fix issues or set allowPartial=true.', 'IMPORT_HAS_ERRORS');
|
||||
}
|
||||
|
||||
const queuedAt = new Date().toISOString();
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set execution_mode = 'async',
|
||||
status = 'pending',
|
||||
dry_run = false,
|
||||
queued_at = coalesce(queued_at, now()),
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
next_attempt_at = now(),
|
||||
summary = coalesce(summary, '{}'::jsonb) || $3::jsonb,
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[
|
||||
auth.tenantId,
|
||||
job.id,
|
||||
JSON.stringify({
|
||||
importOptions: { allowPartial },
|
||||
queuedAt,
|
||||
}),
|
||||
],
|
||||
);
|
||||
|
||||
await client.query(
|
||||
`
|
||||
insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details)
|
||||
values ($1, $2, $3, 'content_import_job', $4, $5::jsonb)
|
||||
`,
|
||||
[
|
||||
auth.tenantId,
|
||||
auth.userId,
|
||||
`content.import.${input.importType}.queued`,
|
||||
job.id,
|
||||
JSON.stringify({ allowPartial, executionMode: 'async', queuedAt }),
|
||||
],
|
||||
);
|
||||
|
||||
return {
|
||||
jobId: job.id,
|
||||
status: 'pending',
|
||||
executionMode: 'async',
|
||||
errorCount: job.error_count,
|
||||
warningCount: job.warning_count,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function runGenericImport<T>(
|
||||
auth: TenantContentAuth,
|
||||
body: JsonObject,
|
||||
importType: ContentImportType,
|
||||
createPreview: () => Promise<PreviewResult<T>>,
|
||||
importOne: (
|
||||
client: pg.PoolClient,
|
||||
auth: TenantContentAuth,
|
||||
job: {
|
||||
id: string;
|
||||
target_region_id: string | null;
|
||||
target_entry_id: string | null;
|
||||
target_content_node_id: string | null;
|
||||
},
|
||||
item: {
|
||||
id: string;
|
||||
row_no: number;
|
||||
normalized_payload: T;
|
||||
},
|
||||
) => Promise<{ insertedCount: number; updatedCount: number; skippedCount: number }>,
|
||||
) {
|
||||
const allowPartial = boolValue(body.allowPartial, false);
|
||||
const jobId = nullableString(body.previewJobId) || nullableString(body.jobId);
|
||||
const createdPreview = jobId ? null : await createPreview();
|
||||
const finalJobId = jobId || createdPreview?.job.id || '';
|
||||
const result = isAsyncExecution(body.executionMode ?? body.execution_mode)
|
||||
? await queueContentImportJob(auth, { jobId: finalJobId, importType, allowPartial })
|
||||
: await executeGenericImportJob(auth, { jobId: finalJobId, importType, allowPartial }, importOne);
|
||||
|
||||
return { item: result, preview: createdPreview };
|
||||
}
|
||||
|
||||
export async function previewQuestionsImportRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'questions');
|
||||
return createQuestionPreviewJob(auth, body);
|
||||
}
|
||||
|
||||
export async function previewVocabularyImportRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'vocabulary');
|
||||
return createGenericPreviewJob(auth, body, 'vocabulary', 'vocabulary_unit', createVocabularyNormalizedItems(body));
|
||||
}
|
||||
|
||||
export async function previewHandbookImportRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'handbook');
|
||||
return createGenericPreviewJob(auth, body, 'handbook', 'handbook_subject', createHandbookNormalizedItems(body));
|
||||
}
|
||||
|
||||
export async function previewScorelineImportRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'scoreline');
|
||||
return createGenericPreviewJob(auth, body, 'scoreline', 'scoreline_item', createScorelineNormalizedItems(body), null);
|
||||
}
|
||||
|
||||
export async function previewVideosImportRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'videos');
|
||||
return createGenericPreviewJob(auth, body, 'videos', 'video_explanation', createVideoNormalizedItems(body), null);
|
||||
}
|
||||
|
||||
export async function importQuestionsRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'questions');
|
||||
const allowPartial = boolValue(body.allowPartial, false);
|
||||
const jobId = nullableString(body.previewJobId) || nullableString(body.jobId);
|
||||
|
||||
const createdPreview = jobId ? null : await createQuestionPreviewJob(auth, body);
|
||||
const finalJobId = jobId || createdPreview?.job.id || '';
|
||||
|
||||
const result = isAsyncExecution(body.executionMode ?? body.execution_mode)
|
||||
? await queueContentImportJob(auth, { jobId: finalJobId, importType: 'questions', allowPartial })
|
||||
: await executeContentImportJob(auth, { jobId: finalJobId, importType: 'questions', allowPartial });
|
||||
|
||||
return { item: result, preview: createdPreview };
|
||||
}
|
||||
|
||||
export async function importVocabularyRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'vocabulary');
|
||||
return runGenericImport(
|
||||
auth,
|
||||
body,
|
||||
@@ -3359,8 +3621,10 @@ export async function importVocabularyRoute(ctx: RequestContext) {
|
||||
}
|
||||
|
||||
export async function importHandbookRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'handbook');
|
||||
return runGenericImport(
|
||||
auth,
|
||||
body,
|
||||
@@ -3371,8 +3635,10 @@ export async function importHandbookRoute(ctx: RequestContext) {
|
||||
}
|
||||
|
||||
export async function importScorelineRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'scoreline');
|
||||
return runGenericImport(
|
||||
auth,
|
||||
body,
|
||||
@@ -3383,8 +3649,10 @@ export async function importScorelineRoute(ctx: RequestContext) {
|
||||
}
|
||||
|
||||
export async function importVideosRoute(ctx: RequestContext) {
|
||||
const { config, expandSpreadsheetImportBody, readJsonBody, requireTenantContentEditor } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const body = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const rawBody = await readJsonBody(ctx, { maxBytes: config.maxImportJsonBodyBytes });
|
||||
const body = await expandSpreadsheetImportBody(rawBody, 'videos');
|
||||
return runGenericImport(
|
||||
auth,
|
||||
body,
|
||||
@@ -3395,6 +3663,7 @@ export async function importVideosRoute(ctx: RequestContext) {
|
||||
}
|
||||
|
||||
export async function importJobsRoute(ctx: RequestContext) {
|
||||
const { intParam, requireTenantContentEditor, stringParam } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const limit = intParam(ctx, 'limit', 50, 200);
|
||||
const importType = stringParam(ctx, 'importType');
|
||||
@@ -3423,6 +3692,7 @@ export async function importJobsRoute(ctx: RequestContext) {
|
||||
valid_count as "validCount", error_count as "errorCount",
|
||||
warning_count as "warningCount", inserted_count as "insertedCount",
|
||||
updated_count as "updatedCount", skipped_count as "skippedCount",
|
||||
execution_mode as "executionMode", parser_metadata as "parserMetadata",
|
||||
summary, error_message as "errorMessage",
|
||||
started_at as "startedAt", finished_at as "finishedAt",
|
||||
created_by as "createdBy", created_at as "createdAt", updated_at as "updatedAt"
|
||||
@@ -3438,6 +3708,7 @@ export async function importJobsRoute(ctx: RequestContext) {
|
||||
}
|
||||
|
||||
export async function importIssuesRoute(ctx: RequestContext) {
|
||||
const { intParam, requireTenantContentEditor, requiredString, stringParam } = await routeDeps();
|
||||
const auth = await requireTenantContentEditor(ctx);
|
||||
const jobId = requiredString({ jobId: stringParam(ctx, 'jobId') }, 'jobId');
|
||||
const limit = intParam(ctx, 'limit', 500, 2000);
|
||||
|
||||
@@ -10,7 +10,8 @@
|
||||
"check": "tsc -p tsconfig.json --noEmit",
|
||||
"crm:once": "tsx src/index.ts --once --job crm",
|
||||
"commerce:once": "tsx src/index.ts --once --job commerce",
|
||||
"assets:once": "tsx src/index.ts --once --job assets"
|
||||
"assets:once": "tsx src/index.ts --once --job assets",
|
||||
"imports:once": "tsx src/index.ts --once --job imports"
|
||||
},
|
||||
"dependencies": {
|
||||
"@supabase/storage-js": "^2.108.2",
|
||||
|
||||
@@ -17,6 +17,9 @@ export interface WorkerConfig {
|
||||
assetMinAgeSeconds: number;
|
||||
assetRecheckIntervalSeconds: number;
|
||||
assetRequestTimeoutMs: number;
|
||||
importBatchSize: number;
|
||||
importWorkerId: string;
|
||||
importBackoffSeconds: number[];
|
||||
storageMaxUploadBytes: number;
|
||||
storageAllowedMimePrefixes: string[];
|
||||
storageAllowedMimeTypes: string[];
|
||||
@@ -53,6 +56,11 @@ export const config: WorkerConfig = {
|
||||
assetMinAgeSeconds: envNumber('WORKER_ASSET_MIN_AGE_SECONDS', 300),
|
||||
assetRecheckIntervalSeconds: envNumber('WORKER_ASSET_RECHECK_INTERVAL_SECONDS', 60 * 60 * 24),
|
||||
assetRequestTimeoutMs: envNumber('WORKER_ASSET_REQUEST_TIMEOUT_MS', 10_000),
|
||||
importBatchSize: envNumber('WORKER_IMPORT_BATCH_SIZE', 5),
|
||||
importWorkerId: envString('WORKER_IMPORT_ID', `imports-${process.pid}`),
|
||||
importBackoffSeconds: envList('WORKER_IMPORT_BACKOFF_SECONDS', '30,120,600,1800')
|
||||
.map((value: string) => Number(value))
|
||||
.filter((value: number) => Number.isFinite(value) && value > 0),
|
||||
storageMaxUploadBytes: envNumber('STORAGE_MAX_UPLOAD_BYTES', 1024 * 1024 * 500),
|
||||
storageAllowedMimePrefixes: envList('STORAGE_ALLOWED_MIME_PREFIXES', 'image/,video/,audio/'),
|
||||
storageAllowedMimeTypes: envList(
|
||||
|
||||
@@ -4,6 +4,8 @@ import { processCrmBatch } from './jobs/crm.js';
|
||||
import { processCommerceBatch } from './jobs/commerce.js';
|
||||
import { processAssetBatch } from './jobs/assets.js';
|
||||
|
||||
const extraClosers = new Set<() => Promise<void>>();
|
||||
|
||||
function hasArg(name: string) {
|
||||
return process.argv.includes(name);
|
||||
}
|
||||
@@ -37,6 +39,17 @@ async function runOnce() {
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (job === 'imports') {
|
||||
const { closeImportExecutorPool, processImportBatch } = await import('./jobs/imports.js');
|
||||
extraClosers.add(closeImportExecutorPool);
|
||||
const result = await processImportBatch();
|
||||
console.log(
|
||||
`[worker] imports batch processed=${result.processed}`
|
||||
+ ` completed=${result.completed} completedWithErrors=${result.completedWithErrors}`
|
||||
+ ` failed=${result.failed} retrying=${result.retrying} skipped=${result.skipped}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
throw new Error(`Unsupported worker job: ${job}`);
|
||||
}
|
||||
|
||||
@@ -66,5 +79,8 @@ try {
|
||||
await runOnce();
|
||||
}
|
||||
} finally {
|
||||
for (const closeExtra of extraClosers) {
|
||||
await closeExtra();
|
||||
}
|
||||
await closePool();
|
||||
}
|
||||
|
||||
221
apps/worker/src/jobs/imports.ts
Normal file
221
apps/worker/src/jobs/imports.ts
Normal file
@@ -0,0 +1,221 @@
|
||||
import { pool } from '../db.js';
|
||||
import { config } from '../config.js';
|
||||
import { executeContentImportJob, type ExecutableContentImportType } from '../../../api/src/features/tenant-content/imports.js';
|
||||
import { closePool as closeApiImportPool } from '../../../api/src/core/db.js';
|
||||
|
||||
interface ImportJobRow {
|
||||
id: string;
|
||||
tenantId: string;
|
||||
createdBy: string | null;
|
||||
importType: ExecutableContentImportType;
|
||||
status: string;
|
||||
attemptCount: number;
|
||||
maxAttempts: number;
|
||||
summary: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface ImportWorkerResult {
|
||||
processed: number;
|
||||
completed: number;
|
||||
completedWithErrors: number;
|
||||
failed: number;
|
||||
retrying: number;
|
||||
skipped: number;
|
||||
}
|
||||
|
||||
function objectValue(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function boolValue(value: unknown, fallback: boolean) {
|
||||
return typeof value === 'boolean' ? value : fallback;
|
||||
}
|
||||
|
||||
function numberValue(value: unknown, fallback: number) {
|
||||
const parsed = Number(value ?? fallback);
|
||||
return Number.isFinite(parsed) ? parsed : fallback;
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function errorCode(error: unknown) {
|
||||
return typeof error === 'object' && error !== null && 'code' in error
|
||||
? String((error as { code?: unknown }).code || 'IMPORT_WORKER_ERROR')
|
||||
: 'IMPORT_WORKER_ERROR';
|
||||
}
|
||||
|
||||
function truncate(value: unknown, max = 1900) {
|
||||
return String(value ?? '').slice(0, max);
|
||||
}
|
||||
|
||||
function backoffSeconds(attemptCount: number) {
|
||||
const backoffs = config.importBackoffSeconds.length ? config.importBackoffSeconds : [30, 120, 600, 1800];
|
||||
return backoffs[Math.min(Math.max(0, attemptCount - 1), backoffs.length - 1)];
|
||||
}
|
||||
|
||||
function importOptions(summary: Record<string, unknown>) {
|
||||
const options = objectValue(summary.importOptions);
|
||||
return {
|
||||
allowPartial: boolValue(options.allowPartial, false),
|
||||
};
|
||||
}
|
||||
|
||||
async function claimImportJobs() {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('begin');
|
||||
const result = await client.query<ImportJobRow>(
|
||||
`
|
||||
select id,
|
||||
tenant_id as "tenantId",
|
||||
created_by as "createdBy",
|
||||
import_type as "importType",
|
||||
status,
|
||||
attempt_count as "attemptCount",
|
||||
max_attempts as "maxAttempts",
|
||||
summary
|
||||
from public.content_import_jobs
|
||||
where execution_mode = 'async'
|
||||
and status = 'pending'
|
||||
and attempt_count < max_attempts
|
||||
and (next_attempt_at is null or next_attempt_at <= now())
|
||||
order by created_at asc
|
||||
limit $1
|
||||
for update skip locked
|
||||
`,
|
||||
[config.importBatchSize],
|
||||
);
|
||||
|
||||
const ids = result.rows.map(row => row.id);
|
||||
if (ids.length > 0) {
|
||||
await client.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set locked_at = now(),
|
||||
locked_by = $2,
|
||||
attempt_count = attempt_count + 1,
|
||||
updated_at = now()
|
||||
where id = any($1::uuid[])
|
||||
`,
|
||||
[ids, config.importWorkerId],
|
||||
);
|
||||
}
|
||||
await client.query('commit');
|
||||
return result.rows;
|
||||
} catch (error) {
|
||||
await client.query('rollback');
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function markImportFailed(job: ImportJobRow, error: unknown) {
|
||||
const nextAttempt = job.attemptCount + 1;
|
||||
const willRetry = nextAttempt < job.maxAttempts;
|
||||
const status = willRetry ? 'pending' : 'failed';
|
||||
await pool.query(
|
||||
`
|
||||
update public.content_import_jobs
|
||||
set status = $3,
|
||||
error_message = $4,
|
||||
summary = coalesce(summary, '{}'::jsonb) || $5::jsonb,
|
||||
next_attempt_at = case when $6::boolean then now() + make_interval(secs => $7::integer) else null end,
|
||||
locked_at = null,
|
||||
locked_by = null,
|
||||
finished_at = case when $3 = 'failed' then now() else finished_at end,
|
||||
updated_at = now()
|
||||
where tenant_id = $1 and id = $2
|
||||
`,
|
||||
[
|
||||
job.tenantId,
|
||||
job.id,
|
||||
status,
|
||||
truncate(errorMessage(error)),
|
||||
JSON.stringify({
|
||||
lastWorkerError: {
|
||||
code: errorCode(error),
|
||||
message: truncate(errorMessage(error)),
|
||||
workerId: config.importWorkerId,
|
||||
failedAt: new Date().toISOString(),
|
||||
nextAttempt,
|
||||
maxAttempts: job.maxAttempts,
|
||||
willRetry,
|
||||
},
|
||||
}),
|
||||
willRetry,
|
||||
backoffSeconds(nextAttempt),
|
||||
],
|
||||
);
|
||||
|
||||
await pool.query(
|
||||
`
|
||||
insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details)
|
||||
values ($1, $2, $3, 'content_import_job', $4, $5::jsonb)
|
||||
`,
|
||||
[
|
||||
job.tenantId,
|
||||
job.createdBy,
|
||||
willRetry ? `content.import.${job.importType}.retry_scheduled` : `content.import.${job.importType}.failed`,
|
||||
job.id,
|
||||
JSON.stringify({
|
||||
code: errorCode(error),
|
||||
message: truncate(errorMessage(error)),
|
||||
workerId: config.importWorkerId,
|
||||
nextAttempt,
|
||||
maxAttempts: job.maxAttempts,
|
||||
}),
|
||||
],
|
||||
);
|
||||
|
||||
return willRetry ? 'retrying' : 'failed';
|
||||
}
|
||||
|
||||
export async function processImportBatch(): Promise<ImportWorkerResult> {
|
||||
const jobs = await claimImportJobs();
|
||||
const result: ImportWorkerResult = {
|
||||
processed: jobs.length,
|
||||
completed: 0,
|
||||
completedWithErrors: 0,
|
||||
failed: 0,
|
||||
retrying: 0,
|
||||
skipped: 0,
|
||||
};
|
||||
|
||||
for (const job of jobs) {
|
||||
try {
|
||||
const execution = await executeContentImportJob(
|
||||
{
|
||||
tenantId: job.tenantId,
|
||||
userId: job.createdBy || job.tenantId,
|
||||
role: 'system_worker',
|
||||
permissions: { 'content:*': true },
|
||||
templatePermissions: {},
|
||||
},
|
||||
{
|
||||
jobId: job.id,
|
||||
importType: job.importType,
|
||||
allowPartial: importOptions(job.summary).allowPartial,
|
||||
allowQueuedJob: true,
|
||||
},
|
||||
);
|
||||
|
||||
if (execution.idempotent) result.skipped += 1;
|
||||
else if (execution.status === 'completed_with_errors') result.completedWithErrors += 1;
|
||||
else if (execution.status === 'completed') result.completed += 1;
|
||||
else result.skipped += 1;
|
||||
} catch (error) {
|
||||
const state = await markImportFailed(job, error);
|
||||
if (state === 'retrying') result.retrying += 1;
|
||||
else result.failed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function closeImportExecutorPool() {
|
||||
await closeApiImportPool();
|
||||
}
|
||||
Reference in New Issue
Block a user