xiongyuxing/gongxue-base
1
0
Fork 0
forked from wangziqi/gongxue-base
Code Pull Requests Activity

feat: add spreadsheet async imports

Browse Source
This commit is contained in:
Codex
2026-06-29 08:00:17 +08:00
parent 78a26d1df2
commit eee21f6eed
27 changed files with 2972 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

179
scripts/api-integration-test.js
View File

@@ -4,6 +4,7 @@ import { spawn } from 'node:child_process';
import http from 'node:http';
import net from 'node:net';
import { SignJWT } from 'jose';
import ExcelJS from 'exceljs';
const DEFAULT_DATABASE_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres';
const MAIN_TENANT_ID = process.env.TENANT_ID || '00000000-0000-0000-0000-000000000001';
@@ -2794,6 +2795,102 @@ async function testTenantContentAssetsAndImports() {
'catalog should expose imported question through the new collection binding',
);
const questionCsv = [
['legacyId', '题型', '题干', '选项A', '选项B', '选项C', '选项D', '答案', '解析', '难度', '标签'].join(','),
['integration-import-csv-choice-001', 'choice', 'CSV导入题：多租户数据隔离应主要依赖什么？', '前端隐藏菜单', '后端权限和RLS', '浏览器缓存', '静态页面', 'B', '最终权限以后端和RLS为准。', '2', 'csv|import'].join(','),
].join('\n');
const csvQuestionPreview = await request('/api/tenant-content/imports/preview/questions', {
userId: TENANT_ADMIN_USER_ID,
method: 'POST',
body: {
sourceFormat: 'csv',
sourceName: 'questions.csv',
csvText: questionCsv,
subjectId: ids.subject,
categoryId: ids.category,
regionId: ids.region,
entryId: ids.contentEntry,
contentNodeId: ids.contentNodeSchoolTarget,
collectionId: ids.questionCollection,
},
});
assert.equal(csvQuestionPreview.job?.errorCount, 0, 'CSV question preview should have no errors');
assert.equal(csvQuestionPreview.job?.validCount, 1, 'CSV question preview should normalize one row');
assert.equal(csvQuestionPreview.items?.[0]?.normalized?.correctOptionIndices?.[0], 1, 'CSV answer B should map to option index 1');
const csvQuestionImport = await request('/api/tenant-content/imports/questions', {
userId: TENANT_ADMIN_USER_ID,
method: 'POST',
body: { previewJobId: csvQuestionPreview.job.id },
});
assert.equal(csvQuestionImport.item?.status, 'completed', 'CSV question import should complete');
const csvQuestionJobs = await request('/api/tenant-content/imports', {
userId: TENANT_ADMIN_USER_ID,
query: { importType: 'questions', limit: 20 },
});
assert.ok(
csvQuestionJobs.items?.some(item => item.id === csvQuestionPreview.job.id && item.sourceFormat === 'csv'),
'CSV question import job should retain sourceFormat',
);
const importedCsvQuestions = await request('/api/catalog/questions', {
query: { collectionId: ids.questionCollection, limit: 150 },
});
assert.ok(
importedCsvQuestions.items?.some(item => item.content === 'CSV导入题：多租户数据隔离应主要依赖什么？'),
'catalog should expose imported CSV question',
);
const asyncQuestionPreview = await request('/api/tenant-content/imports/preview/questions', {
userId: TENANT_ADMIN_USER_ID,
method: 'POST',
body: {
sourceName: 'async-question-import.json',
subjectId: ids.subject,
categoryId: ids.category,
regionId: ids.region,
entryId: ids.contentEntry,
contentNodeId: ids.contentNodeSchoolTarget,
collectionId: ids.questionCollection,
items: [
{
legacyId: 'integration-import-async-choice-001',
type: 'choice',
content: '异步导入排队题：大批量导入应该交给谁执行？',
options: ['前端循环写表', '导入 worker', '用户刷新页面', '浏览器缓存'],
correctOptionIndices: [1],
explanation: '大批量导入由后端 worker 消费，避免请求超时和规则漂移。',
difficulty: 2,
tags: ['async-import'],
},
],
},
});
assert.equal(asyncQuestionPreview.job?.errorCount, 0, 'async preview should have no errors');
const asyncQuestionQueued = await request('/api/tenant-content/imports/questions', {
userId: TENANT_ADMIN_USER_ID,
method: 'POST',
body: { previewJobId: asyncQuestionPreview.job.id, executionMode: 'async' },
});
assert.equal(asyncQuestionQueued.item?.status, 'pending', 'async import should queue the job');
assert.equal(asyncQuestionQueued.item?.executionMode, 'async', 'async import response should expose execution mode');
const queuedAgain = await request('/api/tenant-content/imports/questions', {
userId: TENANT_ADMIN_USER_ID,
method: 'POST',
body: { previewJobId: asyncQuestionPreview.job.id, executionMode: 'async' },
});
assert.equal(queuedAgain.item?.idempotent, true, 'queueing the same async job should be idempotent');
const queuedSyncExecution = await request('/api/tenant-content/imports/questions', {
userId: TENANT_ADMIN_USER_ID,
method: 'POST',
body: { previewJobId: asyncQuestionPreview.job.id },
expectStatus: 409,
});
assert.equal(queuedSyncExecution.code, 'IMPORT_JOB_QUEUED', 'queued import job should not be executed synchronously');
const vocabEntry = await request('/api/tenant-content/content-entries', {
userId: TENANT_ADMIN_USER_ID,
method: 'PUT',
@@ -2913,6 +3010,42 @@ async function testTenantContentAssetsAndImports() {
});
assert.ok(vocabularyWords.items?.some(item => item.word === 'abandon' && item.contentNodeId), 'catalog should expose imported vocabulary word with node binding');
const vocabularyCsv = [
['unitLegacyId', 'unitName', 'wordLegacyId', 'word', 'phonetic', 'meaning', 'example', 'difficulty', 'tags'].join(','),
['integration-vocab-csv-unit-001', 'CSV 高频词单元', 'integration-vocab-csv-word-scale', 'scale', '/skeɪl/', 'n. 规模；等级', 'SaaS platforms must scale safely.', '2', 'csv|高频'].join(','),
].join('\n');
const vocabularyCsvPreview = await request('/api/tenant-content/imports/preview/vocabulary', {
userId: TENANT_ADMIN_USER_ID,
method: 'POST',
body: {
sourceFormat: 'csv',
sourceName: 'vocabulary.csv',
csvText: vocabularyCsv,
regionId: ids.region,
entryId: vocabEntry.item.id,
contentNodeId: vocabRoot.item.id,
},
});
assert.equal(vocabularyCsvPreview.job?.errorCount, 0, 'CSV vocabulary preview should have no errors');
assert.equal(vocabularyCsvPreview.items?.[0]?.normalized?.words?.[0]?.word, 'scale', 'CSV vocabulary should group words under units');
const vocabularyCsvImport = await request('/api/tenant-content/imports/vocabulary', {
userId: TENANT_ADMIN_USER_ID,
method: 'POST',
body: { previewJobId: vocabularyCsvPreview.job.id },
});
assert.equal(vocabularyCsvImport.item?.status, 'completed', 'CSV vocabulary import should complete');
const vocabularyUnitsAfterCsv = await request('/api/catalog/vocabulary-units', {
query: { regionId: ids.region },
});
const csvVocabularyUnit = vocabularyUnitsAfterCsv.items?.find(item => item.legacyId === 'integration-vocab-csv-unit-001');
assert.ok(csvVocabularyUnit, 'catalog should expose CSV imported vocabulary unit');
const csvVocabularyWords = await request('/api/catalog/vocabulary-words', {
query: { unitId: csvVocabularyUnit.id },
});
assert.ok(csvVocabularyWords.items?.some(item => item.word === 'scale'), 'catalog should expose CSV imported vocabulary word');
const handbookEntry = await request('/api/tenant-content/content-entries', {
userId: TENANT_ADMIN_USER_ID,
method: 'PUT',
@@ -3142,6 +3275,52 @@ async function testTenantContentAssetsAndImports() {
'mixed scoreline import should create record after school and major',
);
const scorelineWorkbook = new ExcelJS.Workbook();
scorelineWorkbook.addWorksheet('fields').addRows([
['legacyId', 'fieldKey', 'fieldName', 'fieldType', 'isTrend'],
['integration-scoreline-xlsx-field-min', 'xlsxMinScore', 'Excel最低分', 'number', '是'],
]);
scorelineWorkbook.addWorksheet('schools').addRows([
['legacyId', 'schoolName', 'isHot'],
['integration-scoreline-xlsx-school', 'Excel导入学院', 'true'],
]);
scorelineWorkbook.addWorksheet('majors').addRows([
['legacyId', 'schoolLegacyId', 'majorName'],
['integration-scoreline-xlsx-major', 'integration-scoreline-xlsx-school', 'Excel专业'],
]);
scorelineWorkbook.addWorksheet('records').addRows([
['legacyId', 'schoolLegacyId', 'majorLegacyId', 'year', 'xlsxMinScore'],
['integration-scoreline-xlsx-record-2027', 'integration-scoreline-xlsx-school', 'integration-scoreline-xlsx-major', 2027, 233],
]);
const scorelineExcelBuffer = await scorelineWorkbook.xlsx.writeBuffer();
const scorelineExcelPreview = await request('/api/tenant-content/imports/preview/scoreline', {
userId: TENANT_ADMIN_USER_ID,
method: 'POST',
body: {
sourceFormat: 'excel',
sourceName: 'scoreline.xlsx',
fileBase64: Buffer.from(scorelineExcelBuffer).toString('base64'),
regionId: ids.region,
},
});
assert.equal(scorelineExcelPreview.job?.errorCount, 0, 'Excel scoreline preview should have no errors');
assert.equal(scorelineExcelPreview.job?.validCount, 4, 'Excel scoreline preview should parse multi-sheet workbook');
const scorelineExcelImport = await request('/api/tenant-content/imports/scoreline', {
userId: TENANT_ADMIN_USER_ID,
method: 'POST',
body: { previewJobId: scorelineExcelPreview.job.id },
});
assert.equal(scorelineExcelImport.item?.status, 'completed', 'Excel scoreline import should complete');
const excelScorelineRecords = await request('/api/scoreline/records', {
query: { regionId: ids.region, year: 2027, pageSize: 50 },
});
assert.ok(
excelScorelineRecords.items?.some(item => item.schoolName === 'Excel导入学院' && item.fieldValues?.xlsxMinScore === 233),
'public scoreline records should expose imported Excel record',
);
const invalidVideoPreview = await request('/api/tenant-content/imports/preview/videos', {
userId: TENANT_ADMIN_USER_ID,
method: 'POST',

299
scripts/import-worker-integration-test.js Normal file
View File

@@ -0,0 +1,299 @@
import assert from 'node:assert/strict';
import pg from 'pg';
import { spawn } from 'node:child_process';
const databaseUrl = process.env.DATABASE_URL || 'postgresql://postgres:postgres@127.0.0.1:54322/postgres';
const tenantId = '00000000-0000-0000-0000-000000000001';
const adminUserId = '00000000-0000-0000-0000-000000000102';
const ids = {
region: '00000000-0000-0000-0000-000000000301',
subject: '00000000-0000-0000-0000-000000000501',
category: '00000000-0000-0000-0000-000000000601',
contentEntry: '00000000-0000-0000-0000-000000000611',
contentNodeSchoolTarget: '00000000-0000-0000-0000-000000000614',
questionCollection: '00000000-0000-0000-0000-000000000615',
};
function runWorkerOnce() {
const child = spawn(process.execPath, ['apps/worker/dist/apps/worker/src/index.js', '--once', '--job', 'imports'], {
cwd: process.cwd(),
env: {
...process.env,
DATABASE_URL: databaseUrl,
WORKER_IMPORT_BATCH_SIZE: '5',
WORKER_IMPORT_ID: 'imports-integration-test',
},
stdio: ['ignore', 'pipe', 'pipe'],
windowsHide: true,
});
let output = '';
child.stdout.on('data', chunk => {
output += chunk.toString();
});
child.stderr.on('data', chunk => {
output += chunk.toString();
});
return new Promise((resolve, reject) => {
child.on('error', reject);
child.on('exit', code => {
try {
assert.equal(code, 0, `worker should exit 0\n${output}`);
assert.match(output, /imports batch processed=\d+/, 'worker output should include imports summary');
resolve(output);
} catch (error) {
reject(error);
}
});
});
}
async function cleanup(pool) {
await pool.query(
`
delete from public.question_collection_items
where tenant_id = $1
and question_id in (
select id from public.questions
where tenant_id = $1
and (
legacy_id like 'worker-import-question-%'
or legacy_id like 'integration-import-async-choice-%'
)
)
`,
[tenantId],
);
await pool.query(
`
delete from public.question_versions
where tenant_id = $1
and question_id in (
select id from public.questions
where tenant_id = $1
and (
legacy_id like 'worker-import-question-%'
or legacy_id like 'integration-import-async-choice-%'
)
)
`,
[tenantId],
);
await pool.query(
`
delete from public.questions
where tenant_id = $1
and (
legacy_id like 'worker-import-question-%'
or legacy_id like 'integration-import-async-choice-%'
)
`,
[tenantId],
);
await pool.query(
`
delete from public.audit_logs
where tenant_id = $1
and target_type = 'content_import_job'
and details::text like '%worker-import%'
`,
[tenantId],
);
await pool.query(
`
delete from public.content_import_jobs
where tenant_id = $1
and (
source_name like 'worker-import-%'
or source_name = 'async-question-import.json'
)
`,
[tenantId],
);
}
async function createQueuedQuestionImport(pool) {
const preview = await pool.query(
`
insert into public.content_import_jobs (
tenant_id, created_by, import_type, source_format, status,
source_name, source_hash, target_region_id, target_subject_id,
target_category_id, target_content_node_id, target_collection_id,
dry_run, total_count, valid_count, error_count, warning_count,
summary, raw_payload, normalized_payload, execution_mode, queued_at,
next_attempt_at, parser_metadata
)
values (
$1, $2, 'questions', 'json', 'pending',
'worker-import-questions.json', 'worker-import-source-hash',
$3::uuid, $4::uuid, $5::uuid, $6::uuid, $7::uuid,
false, 1, 1, 0, 0,
$8::jsonb, $9::jsonb, $10::jsonb, 'async', now(), now(), '{}'::jsonb
)
returning id
`,
[
tenantId,
adminUserId,
ids.region,
ids.subject,
ids.category,
ids.contentNodeSchoolTarget,
ids.questionCollection,
JSON.stringify({
target: {
regionId: ids.region,
subjectId: ids.subject,
categoryId: ids.category,
contentNodeId: ids.contentNodeSchoolTarget,
collectionId: ids.questionCollection,
},
importOptions: { allowPartial: false },
source: 'worker-import-integration',
}),
JSON.stringify([
{
legacyId: 'worker-import-question-001',
type: 'choice',
content: '异步导入题：worker 应该复用哪套导入规则？',
options: ['自己重写', '复用后端导入 executor', '前端直写数据库', '跳过校验'],
correctOptionIndices: [1],
explanation: 'worker 和 API 必须复用同一套后端导入规则。',
difficulty: 2,
tags: ['worker-import'],
},
]),
JSON.stringify([
{
legacyId: 'worker-import-question-001',
type: 'choice',
typeLabel: null,
content: '异步导入题：worker 应该复用哪套导入规则？',
options: ['自己重写', '复用后端导入 executor', '前端直写数据库', '跳过校验'],
correctOptionIndex: 1,
correctOptionIndices: [1],
answerText: null,
explanation: 'worker 和 API 必须复用同一套后端导入规则。',
difficulty: 2,
tags: ['worker-import'],
mediaUrl: null,
subQuestions: [],
codeLang: null,
codeTemplate: null,
examMarkers: {},
sourceHash: 'worker-import-question-hash-001',
},
]),
],
);
const jobId = preview.rows[0].id;
await pool.query(
`
insert into public.content_import_items (
tenant_id, job_id, row_no, external_id, status, target_type,
source_payload, normalized_payload, content_hash, issues_count
)
values ($1, $2, 1, 'worker-import-question-001', 'valid', 'question', $3::jsonb, $4::jsonb, 'worker-import-question-hash-001', 0)
`,
[
tenantId,
jobId,
JSON.stringify({
legacyId: 'worker-import-question-001',
content: '异步导入题：worker 应该复用哪套导入规则？',
}),
JSON.stringify({
legacyId: 'worker-import-question-001',
type: 'choice',
typeLabel: null,
content: '异步导入题：worker 应该复用哪套导入规则？',
options: ['自己重写', '复用后端导入 executor', '前端直写数据库', '跳过校验'],
correctOptionIndex: 1,
correctOptionIndices: [1],
answerText: null,
explanation: 'worker 和 API 必须复用同一套后端导入规则。',
difficulty: 2,
tags: ['worker-import'],
mediaUrl: null,
subQuestions: [],
codeLang: null,
codeTemplate: null,
examMarkers: {},
sourceHash: 'worker-import-question-hash-001',
}),
],
);
return jobId;
}
async function main() {
const pool = new pg.Pool({ connectionString: databaseUrl });
try {
await cleanup(pool);
const jobId = await createQueuedQuestionImport(pool);
const output = await runWorkerOnce();
assert.match(output, /completed=1/, 'worker should complete exactly the queued import job after cleanup');
const job = await pool.query(
`
select status, execution_mode, inserted_count, updated_count, skipped_count,
locked_at, locked_by, attempt_count, error_message
from public.content_import_jobs
where tenant_id = $1 and id = $2
`,
[tenantId, jobId],
);
assert.equal(job.rows[0]?.status, 'completed', 'queued import job should be completed');
assert.equal(job.rows[0]?.execution_mode, 'async', 'job should keep async execution mode');
assert.equal(Number(job.rows[0]?.inserted_count), 1, 'worker should insert one question');
assert.equal(job.rows[0]?.locked_at, null, 'completed job should release lock');
assert.equal(job.rows[0]?.locked_by, null, 'completed job should clear lock owner');
assert.equal(Number(job.rows[0]?.attempt_count), 1, 'worker should record one attempt');
assert.equal(job.rows[0]?.error_message, null, 'completed job should not retain error message');
const question = await pool.query(
`
select q.id, v.content
from public.questions q
join public.question_versions v on v.id = q.current_version_id
where q.tenant_id = $1 and q.legacy_id = 'worker-import-question-001'
limit 1
`,
[tenantId],
);
assert.equal(question.rows[0]?.content, '异步导入题：worker 应该复用哪套导入规则？', 'worker should import question content');
const collectionItem = await pool.query(
`
select 1
from public.question_collection_items
where tenant_id = $1 and collection_id = $2 and question_id = $3
limit 1
`,
[tenantId, ids.questionCollection, question.rows[0]?.id],
);
assert.equal(collectionItem.rowCount, 1, 'worker should bind imported question to collection');
const audit = await pool.query(
`
select action
from public.audit_logs
where tenant_id = $1 and target_type = 'content_import_job' and target_id = $2
order by created_at desc
limit 1
`,
[tenantId, jobId],
);
assert.equal(audit.rows[0]?.action, 'content.import.questions.completed', 'worker import should write completion audit');
console.log('Import worker integration test complete.');
} finally {
await cleanup(pool).catch(() => {});
await pool.end();
}
}
main().catch(error => {
console.error(error);
process.exit(1);
});