feat: add spreadsheet async imports

2026-06-29 08:00:17 +08:00
parent 78a26d1df2
commit eee21f6eed
27 changed files with 2972 additions and 267 deletions
--- a/scripts/api-integration-test.js
+++ b/scripts/api-integration-test.js
@@ -4,6 +4,7 @@ import { spawn } from 'node:child_process';
 import http from 'node:http';
 import net from 'node:net';
 import { SignJWT } from 'jose';
+import ExcelJS from 'exceljs';

 const DEFAULT_DATABASE_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres';
 const MAIN_TENANT_ID = process.env.TENANT_ID || '00000000-0000-0000-0000-000000000001';
@@ -2794,6 +2795,102 @@ async function testTenantContentAssetsAndImports() {
    'catalog should expose imported question through the new collection binding',
  );

+  const questionCsv = [
+    ['legacyId', '题型', '题干', '选项A', '选项B', '选项C', '选项D', '答案', '解析', '难度', '标签'].join(','),
+    ['integration-import-csv-choice-001', 'choice', 'CSV导入题：多租户数据隔离应主要依赖什么？', '前端隐藏菜单', '后端权限和RLS', '浏览器缓存', '静态页面', 'B', '最终权限以后端和RLS为准。', '2', 'csv|import'].join(','),
+  ].join('\n');
+  const csvQuestionPreview = await request('/api/tenant-content/imports/preview/questions', {
+    userId: TENANT_ADMIN_USER_ID,
+    method: 'POST',
+    body: {
+      sourceFormat: 'csv',
+      sourceName: 'questions.csv',
+      csvText: questionCsv,
+      subjectId: ids.subject,
+      categoryId: ids.category,
+      regionId: ids.region,
+      entryId: ids.contentEntry,
+      contentNodeId: ids.contentNodeSchoolTarget,
+      collectionId: ids.questionCollection,
+    },
+  });
+  assert.equal(csvQuestionPreview.job?.errorCount, 0, 'CSV question preview should have no errors');
+  assert.equal(csvQuestionPreview.job?.validCount, 1, 'CSV question preview should normalize one row');
+  assert.equal(csvQuestionPreview.items?.[0]?.normalized?.correctOptionIndices?.[0], 1, 'CSV answer B should map to option index 1');
+
+  const csvQuestionImport = await request('/api/tenant-content/imports/questions', {
+    userId: TENANT_ADMIN_USER_ID,
+    method: 'POST',
+    body: { previewJobId: csvQuestionPreview.job.id },
+  });
+  assert.equal(csvQuestionImport.item?.status, 'completed', 'CSV question import should complete');
+
+  const csvQuestionJobs = await request('/api/tenant-content/imports', {
+    userId: TENANT_ADMIN_USER_ID,
+    query: { importType: 'questions', limit: 20 },
+  });
+  assert.ok(
+    csvQuestionJobs.items?.some(item => item.id === csvQuestionPreview.job.id && item.sourceFormat === 'csv'),
+    'CSV question import job should retain sourceFormat',
+  );
+
+  const importedCsvQuestions = await request('/api/catalog/questions', {
+    query: { collectionId: ids.questionCollection, limit: 150 },
+  });
+  assert.ok(
+    importedCsvQuestions.items?.some(item => item.content === 'CSV导入题：多租户数据隔离应主要依赖什么？'),
+    'catalog should expose imported CSV question',
+  );
+
+  const asyncQuestionPreview = await request('/api/tenant-content/imports/preview/questions', {
+    userId: TENANT_ADMIN_USER_ID,
+    method: 'POST',
+    body: {
+      sourceName: 'async-question-import.json',
+      subjectId: ids.subject,
+      categoryId: ids.category,
+      regionId: ids.region,
+      entryId: ids.contentEntry,
+      contentNodeId: ids.contentNodeSchoolTarget,
+      collectionId: ids.questionCollection,
+      items: [
+        {
+          legacyId: 'integration-import-async-choice-001',
+          type: 'choice',
+          content: '异步导入排队题：大批量导入应该交给谁执行？',
+          options: ['前端循环写表', '导入 worker', '用户刷新页面', '浏览器缓存'],
+          correctOptionIndices: [1],
+          explanation: '大批量导入由后端 worker 消费，避免请求超时和规则漂移。',
+          difficulty: 2,
+          tags: ['async-import'],
+        },
+      ],
+    },
+  });
+  assert.equal(asyncQuestionPreview.job?.errorCount, 0, 'async preview should have no errors');
+  const asyncQuestionQueued = await request('/api/tenant-content/imports/questions', {
+    userId: TENANT_ADMIN_USER_ID,
+    method: 'POST',
+    body: { previewJobId: asyncQuestionPreview.job.id, executionMode: 'async' },
+  });
+  assert.equal(asyncQuestionQueued.item?.status, 'pending', 'async import should queue the job');
+  assert.equal(asyncQuestionQueued.item?.executionMode, 'async', 'async import response should expose execution mode');
+
+  const queuedAgain = await request('/api/tenant-content/imports/questions', {
+    userId: TENANT_ADMIN_USER_ID,
+    method: 'POST',
+    body: { previewJobId: asyncQuestionPreview.job.id, executionMode: 'async' },
+  });
+  assert.equal(queuedAgain.item?.idempotent, true, 'queueing the same async job should be idempotent');
+
+  const queuedSyncExecution = await request('/api/tenant-content/imports/questions', {
+    userId: TENANT_ADMIN_USER_ID,
+    method: 'POST',
+    body: { previewJobId: asyncQuestionPreview.job.id },
+    expectStatus: 409,
+  });
+  assert.equal(queuedSyncExecution.code, 'IMPORT_JOB_QUEUED', 'queued import job should not be executed synchronously');
+
  const vocabEntry = await request('/api/tenant-content/content-entries', {
    userId: TENANT_ADMIN_USER_ID,
    method: 'PUT',
@@ -2913,6 +3010,42 @@ async function testTenantContentAssetsAndImports() {
  });
  assert.ok(vocabularyWords.items?.some(item => item.word === 'abandon' && item.contentNodeId), 'catalog should expose imported vocabulary word with node binding');

+  const vocabularyCsv = [
+    ['unitLegacyId', 'unitName', 'wordLegacyId', 'word', 'phonetic', 'meaning', 'example', 'difficulty', 'tags'].join(','),
+    ['integration-vocab-csv-unit-001', 'CSV 高频词单元', 'integration-vocab-csv-word-scale', 'scale', '/skeɪl/', 'n. 规模；等级', 'SaaS platforms must scale safely.', '2', 'csv|高频'].join(','),
+  ].join('\n');
+  const vocabularyCsvPreview = await request('/api/tenant-content/imports/preview/vocabulary', {
+    userId: TENANT_ADMIN_USER_ID,
+    method: 'POST',
+    body: {
+      sourceFormat: 'csv',
+      sourceName: 'vocabulary.csv',
+      csvText: vocabularyCsv,
+      regionId: ids.region,
+      entryId: vocabEntry.item.id,
+      contentNodeId: vocabRoot.item.id,
+    },
+  });
+  assert.equal(vocabularyCsvPreview.job?.errorCount, 0, 'CSV vocabulary preview should have no errors');
+  assert.equal(vocabularyCsvPreview.items?.[0]?.normalized?.words?.[0]?.word, 'scale', 'CSV vocabulary should group words under units');
+
+  const vocabularyCsvImport = await request('/api/tenant-content/imports/vocabulary', {
+    userId: TENANT_ADMIN_USER_ID,
+    method: 'POST',
+    body: { previewJobId: vocabularyCsvPreview.job.id },
+  });
+  assert.equal(vocabularyCsvImport.item?.status, 'completed', 'CSV vocabulary import should complete');
+
+  const vocabularyUnitsAfterCsv = await request('/api/catalog/vocabulary-units', {
+    query: { regionId: ids.region },
+  });
+  const csvVocabularyUnit = vocabularyUnitsAfterCsv.items?.find(item => item.legacyId === 'integration-vocab-csv-unit-001');
+  assert.ok(csvVocabularyUnit, 'catalog should expose CSV imported vocabulary unit');
+  const csvVocabularyWords = await request('/api/catalog/vocabulary-words', {
+    query: { unitId: csvVocabularyUnit.id },
+  });
+  assert.ok(csvVocabularyWords.items?.some(item => item.word === 'scale'), 'catalog should expose CSV imported vocabulary word');
+
  const handbookEntry = await request('/api/tenant-content/content-entries', {
    userId: TENANT_ADMIN_USER_ID,
    method: 'PUT',
@@ -3142,6 +3275,52 @@ async function testTenantContentAssetsAndImports() {
    'mixed scoreline import should create record after school and major',
  );

+  const scorelineWorkbook = new ExcelJS.Workbook();
+  scorelineWorkbook.addWorksheet('fields').addRows([
+    ['legacyId', 'fieldKey', 'fieldName', 'fieldType', 'isTrend'],
+    ['integration-scoreline-xlsx-field-min', 'xlsxMinScore', 'Excel最低分', 'number', '是'],
+  ]);
+  scorelineWorkbook.addWorksheet('schools').addRows([
+    ['legacyId', 'schoolName', 'isHot'],
+    ['integration-scoreline-xlsx-school', 'Excel导入学院', 'true'],
+  ]);
+  scorelineWorkbook.addWorksheet('majors').addRows([
+    ['legacyId', 'schoolLegacyId', 'majorName'],
+    ['integration-scoreline-xlsx-major', 'integration-scoreline-xlsx-school', 'Excel专业'],
+  ]);
+  scorelineWorkbook.addWorksheet('records').addRows([
+    ['legacyId', 'schoolLegacyId', 'majorLegacyId', 'year', 'xlsxMinScore'],
+    ['integration-scoreline-xlsx-record-2027', 'integration-scoreline-xlsx-school', 'integration-scoreline-xlsx-major', 2027, 233],
+  ]);
+  const scorelineExcelBuffer = await scorelineWorkbook.xlsx.writeBuffer();
+  const scorelineExcelPreview = await request('/api/tenant-content/imports/preview/scoreline', {
+    userId: TENANT_ADMIN_USER_ID,
+    method: 'POST',
+    body: {
+      sourceFormat: 'excel',
+      sourceName: 'scoreline.xlsx',
+      fileBase64: Buffer.from(scorelineExcelBuffer).toString('base64'),
+      regionId: ids.region,
+    },
+  });
+  assert.equal(scorelineExcelPreview.job?.errorCount, 0, 'Excel scoreline preview should have no errors');
+  assert.equal(scorelineExcelPreview.job?.validCount, 4, 'Excel scoreline preview should parse multi-sheet workbook');
+
+  const scorelineExcelImport = await request('/api/tenant-content/imports/scoreline', {
+    userId: TENANT_ADMIN_USER_ID,
+    method: 'POST',
+    body: { previewJobId: scorelineExcelPreview.job.id },
+  });
+  assert.equal(scorelineExcelImport.item?.status, 'completed', 'Excel scoreline import should complete');
+
+  const excelScorelineRecords = await request('/api/scoreline/records', {
+    query: { regionId: ids.region, year: 2027, pageSize: 50 },
+  });
+  assert.ok(
+    excelScorelineRecords.items?.some(item => item.schoolName === 'Excel导入学院' && item.fieldValues?.xlsxMinScore === 233),
+    'public scoreline records should expose imported Excel record',
+  );
+
  const invalidVideoPreview = await request('/api/tenant-content/imports/preview/videos', {
    userId: TENANT_ADMIN_USER_ID,
    method: 'POST',