feat(imports): 导入向导兼容旧版工作簿并完善执行/预览
- 新增 xls 工作簿回退解析与测试 - 执行/预览/权限校验逻辑完善
This commit is contained in:
37
apps/server/src/imports/imports.access.spec.ts
Normal file
37
apps/server/src/imports/imports.access.spec.ts
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
import { ForbiddenException } from '@nestjs/common';
|
||||||
|
import { assertStepPermission, permittedStepKeys } from './imports.access';
|
||||||
|
|
||||||
|
describe('permittedStepKeys', () => {
|
||||||
|
it('超级管理员拥有全部导入阶段', () => {
|
||||||
|
expect(
|
||||||
|
permittedStepKeys({ id: 1, permissions: [], isSuperAdmin: true }),
|
||||||
|
).toEqual(['students', 'rooms', 'checkins', 'transfers']);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('按细分权限计算可提交阶段', () => {
|
||||||
|
expect(
|
||||||
|
permittedStepKeys({
|
||||||
|
id: 1,
|
||||||
|
permissions: ['student:import', 'room:edit', 'occupancy:checkin'],
|
||||||
|
isSuperAdmin: false,
|
||||||
|
}),
|
||||||
|
).toEqual(['students', 'rooms', 'checkins']);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('无导入权限时仅返回空数组', () => {
|
||||||
|
expect(
|
||||||
|
permittedStepKeys({ id: 1, permissions: ['ai:chat:use'], isSuperAdmin: false }),
|
||||||
|
).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('assertStepPermission', () => {
|
||||||
|
it('无对应细分权限时拒绝提交', () => {
|
||||||
|
expect(() =>
|
||||||
|
assertStepPermission(
|
||||||
|
{ id: 1, permissions: ['ai:chat:use'], isSuperAdmin: false },
|
||||||
|
'transfers',
|
||||||
|
),
|
||||||
|
).toThrow(ForbiddenException);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -2,7 +2,7 @@ import { ForbiddenException, NotFoundException } from '@nestjs/common';
|
|||||||
import { Repository } from 'typeorm';
|
import { Repository } from 'typeorm';
|
||||||
import { ImportRun } from './entities/import-run.entity';
|
import { ImportRun } from './entities/import-run.entity';
|
||||||
import { ImportStep } from './entities/import-step.entity';
|
import { ImportStep } from './entities/import-step.entity';
|
||||||
import { IMPORT_STEP_LABELS } from './imports.types';
|
import { IMPORT_STEP_LABELS, IMPORT_STEP_ORDER } from './imports.types';
|
||||||
import type { ImportStepKey } from './imports.types';
|
import type { ImportStepKey } from './imports.types';
|
||||||
|
|
||||||
export interface ImportPrincipal {
|
export interface ImportPrincipal {
|
||||||
@@ -45,3 +45,11 @@ export function assertStepPermission(principal: ImportPrincipal, stepKey: Import
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Step keys the current principal may actually commit. */
|
||||||
|
export function permittedStepKeys(principal: ImportPrincipal): ImportStepKey[] {
|
||||||
|
if (principal.isSuperAdmin) return [...IMPORT_STEP_ORDER];
|
||||||
|
return IMPORT_STEP_ORDER.filter((stepKey) =>
|
||||||
|
STEP_PERMISSIONS[stepKey].some((code) => principal.permissions.includes(code)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,6 +7,20 @@ describe('ImportsController', () => {
|
|||||||
connection: { remoteAddress: '127.0.0.1' },
|
connection: { remoteAddress: '127.0.0.1' },
|
||||||
};
|
};
|
||||||
|
|
||||||
|
it('路由门槛允许 AI 用户打开本人 run,同时保留原有导入权限', () => {
|
||||||
|
const permissions = Reflect.getMetadata('permissions', ImportsController) as string[];
|
||||||
|
expect(permissions).toContain('ai:chat:use');
|
||||||
|
expect(permissions).toEqual(
|
||||||
|
expect.arrayContaining([
|
||||||
|
'student:import',
|
||||||
|
'room:create',
|
||||||
|
'room:edit',
|
||||||
|
'occupancy:checkin',
|
||||||
|
'occupancy:transfer',
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('提交导入阶段成功后写入操作日志', async () => {
|
it('提交导入阶段成功后写入操作日志', async () => {
|
||||||
const commitStep = jest.fn().mockResolvedValue({
|
const commitStep = jest.fn().mockResolvedValue({
|
||||||
runId: 'run-1',
|
runId: 'run-1',
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ const IMPORT_GATE_PERMISSIONS = [
|
|||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
@Controller('imports')
|
@Controller('imports')
|
||||||
@RequirePermission(...IMPORT_GATE_PERMISSIONS)
|
@RequirePermission('ai:chat:use', ...IMPORT_GATE_PERMISSIONS)
|
||||||
export class ImportsController {
|
export class ImportsController {
|
||||||
constructor(
|
constructor(
|
||||||
private readonly importsService: ImportsService,
|
private readonly importsService: ImportsService,
|
||||||
|
|||||||
@@ -50,7 +50,10 @@ export function cellValue(cell: ExcelJS.Cell | undefined): CellValue {
|
|||||||
|
|
||||||
export function parseDateValue(value: CellValue): string | null {
|
export function parseDateValue(value: CellValue): string | null {
|
||||||
if (value instanceof Date && !Number.isNaN(value.getTime())) {
|
if (value instanceof Date && !Number.isNaN(value.getTime())) {
|
||||||
return value.toISOString().slice(0, 10);
|
const year = value.getFullYear();
|
||||||
|
const month = String(value.getMonth() + 1).padStart(2, '0');
|
||||||
|
const day = String(value.getDate()).padStart(2, '0');
|
||||||
|
return `${year}-${month}-${day}`;
|
||||||
}
|
}
|
||||||
const raw = textValue(value);
|
const raw = textValue(value);
|
||||||
if (!raw) return null;
|
if (!raw) return null;
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ describe('buildPreflightReport', () => {
|
|||||||
update: 0,
|
update: 0,
|
||||||
error: 0,
|
error: 0,
|
||||||
skip: 0,
|
skip: 0,
|
||||||
|
headers: ['姓名', '学号', '手机号'],
|
||||||
mapping: { name: '姓名', studentNo: '学号', phone: '手机号' },
|
mapping: { name: '姓名', studentNo: '学号', phone: '手机号' },
|
||||||
});
|
});
|
||||||
expect(report.blocks).toEqual([]);
|
expect(report.blocks).toEqual([]);
|
||||||
|
|||||||
@@ -183,7 +183,7 @@ async function analyzeStage(
|
|||||||
code,
|
code,
|
||||||
stepKey,
|
stepKey,
|
||||||
sheet: sheet.name,
|
sheet: sheet.name,
|
||||||
rowNumber: i + 2,
|
rowNumber: sheet.rowNumbers?.[i] ?? (sheet.headerRow ?? 1) + i + 1,
|
||||||
errors: result.errors,
|
errors: result.errors,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -209,6 +209,7 @@ async function analyzeStage(
|
|||||||
stepKey,
|
stepKey,
|
||||||
label: IMPORT_STEP_LABELS[stepKey],
|
label: IMPORT_STEP_LABELS[stepKey],
|
||||||
sheetNames: sheets.map((sheet) => sheet.name),
|
sheetNames: sheets.map((sheet) => sheet.name),
|
||||||
|
headers: [...new Set(sheets.flatMap((sheet) => sheet.headers))],
|
||||||
total,
|
total,
|
||||||
create,
|
create,
|
||||||
update,
|
update,
|
||||||
|
|||||||
@@ -55,8 +55,15 @@ export class ImportPreviewService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const sheetsData =
|
const sheetsData =
|
||||||
parseJson<Array<{ name: string; headers: string[]; rows: CellValue[][] }>>(run.sheetsJson) ??
|
parseJson<
|
||||||
[];
|
Array<{
|
||||||
|
name: string;
|
||||||
|
headers: string[];
|
||||||
|
rows: CellValue[][];
|
||||||
|
headerRow?: number;
|
||||||
|
rowNumbers?: number[];
|
||||||
|
}>
|
||||||
|
>(run.sheetsJson) ?? [];
|
||||||
const settings = parseJson<ImportRunSettings>(run.settingsJson) ?? {};
|
const settings = parseJson<ImportRunSettings>(run.settingsJson) ?? {};
|
||||||
const sheetNames = body.sheets?.length
|
const sheetNames = body.sheets?.length
|
||||||
? body.sheets
|
? body.sheets
|
||||||
@@ -132,7 +139,7 @@ export class ImportPreviewService {
|
|||||||
runId,
|
runId,
|
||||||
stepId: step.id,
|
stepId: step.id,
|
||||||
sheetName,
|
sheetName,
|
||||||
rowNumber: i + 2,
|
rowNumber: sheet.rowNumbers?.[i] ?? (sheet.headerRow ?? 1) + i + 1,
|
||||||
rawJson: JSON.stringify(raw),
|
rawJson: JSON.stringify(raw),
|
||||||
normalizedJson: JSON.stringify(normalized),
|
normalizedJson: JSON.stringify(normalized),
|
||||||
matchKey: result.matchKey,
|
matchKey: result.matchKey,
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import type {
|
|||||||
} from './imports.types';
|
} from './imports.types';
|
||||||
import { parseJson } from './imports.helpers';
|
import { parseJson } from './imports.helpers';
|
||||||
import { parseSheets } from './imports.workbook';
|
import { parseSheets } from './imports.workbook';
|
||||||
|
import type { ImportSheetData } from './imports.workbook';
|
||||||
import { autoAssignedSheets, resolveAssignedSheets, suggestMapping, suggestStep } from './imports.mapping';
|
import { autoAssignedSheets, resolveAssignedSheets, suggestMapping, suggestStep } from './imports.mapping';
|
||||||
import { findOwnedRun } from './imports.access';
|
import { findOwnedRun } from './imports.access';
|
||||||
import type { ImportPrincipal } from './imports.access';
|
import type { ImportPrincipal } from './imports.access';
|
||||||
@@ -42,7 +43,35 @@ export class ImportRunService {
|
|||||||
if (!file.buffer || file.buffer.length === 0) {
|
if (!file.buffer || file.buffer.length === 0) {
|
||||||
throw new BadRequestException('上传文件为空');
|
throw new BadRequestException('上传文件为空');
|
||||||
}
|
}
|
||||||
const sheets = await parseSheets(file.buffer, file.originalName, file.mimeType);
|
const headerRows =
|
||||||
|
stages && stages.length > 0
|
||||||
|
? [...new Set(stages.map((stage) => stage.headerRow ?? 1))]
|
||||||
|
: [1];
|
||||||
|
const parsedByHeaderRow = new Map<number, ImportSheetData[]>();
|
||||||
|
for (const headerRow of headerRows) {
|
||||||
|
parsedByHeaderRow.set(
|
||||||
|
headerRow,
|
||||||
|
await parseSheets(file.buffer, file.originalName, file.mimeType, headerRow),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const views = new Map<string, ImportSheetData>();
|
||||||
|
if (stages && stages.length > 0) {
|
||||||
|
// v1 limitation: a sheet referenced by multiple stages with different
|
||||||
|
// header rows keeps the view of the last stage that named it.
|
||||||
|
for (const stage of stages) {
|
||||||
|
const sheetName = stage.sheet?.trim();
|
||||||
|
if (!sheetName) continue;
|
||||||
|
const view = parsedByHeaderRow
|
||||||
|
.get(stage.headerRow ?? 1)
|
||||||
|
?.find((sheet) => sheet.name === sheetName);
|
||||||
|
if (view) views.set(sheetName, view);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for (const sheet of parsedByHeaderRow.get(1) ?? []) {
|
||||||
|
views.set(sheet.name, sheet);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const sheets = [...views.values()];
|
||||||
|
|
||||||
const runId = randomUUID();
|
const runId = randomUUID();
|
||||||
const run = this.runs.create({
|
const run = this.runs.create({
|
||||||
@@ -85,6 +114,20 @@ export class ImportRunService {
|
|||||||
const firstSheet = sheets.find((s) => s.name === assigned[0]);
|
const firstSheet = sheets.find((s) => s.name === assigned[0]);
|
||||||
const mapping =
|
const mapping =
|
||||||
mappingByStep?.[stepKey] ?? suggestMapping(firstSheet?.headers ?? [], stepKey);
|
mappingByStep?.[stepKey] ?? suggestMapping(firstSheet?.headers ?? [], stepKey);
|
||||||
|
if (mapping) {
|
||||||
|
const allowedHeaders = new Set<string>();
|
||||||
|
for (const sheetName of assigned) {
|
||||||
|
const sheet = sheets.find((s) => s.name === sheetName);
|
||||||
|
for (const header of sheet?.headers ?? []) allowedHeaders.add(header);
|
||||||
|
}
|
||||||
|
for (const [field, header] of Object.entries(mapping)) {
|
||||||
|
if (header && allowedHeaders.size > 0 && !allowedHeaders.has(header)) {
|
||||||
|
throw new BadRequestException(
|
||||||
|
`「${IMPORT_STEP_LABELS[stepKey]}」列映射「${header}」(字段 ${field})不在工作表表头中`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
stepRecords.push(
|
stepRecords.push(
|
||||||
this.steps.create({
|
this.steps.create({
|
||||||
runId,
|
runId,
|
||||||
@@ -108,8 +151,15 @@ export class ImportRunService {
|
|||||||
const run = await findOwnedRun(this.runs, userId, runId);
|
const run = await findOwnedRun(this.runs, userId, runId);
|
||||||
const stepRecords = await this.steps.find({ where: { runId }, order: { id: 'ASC' } });
|
const stepRecords = await this.steps.find({ where: { runId }, order: { id: 'ASC' } });
|
||||||
const sheets =
|
const sheets =
|
||||||
parseJson<Array<{ name: string; headers: string[]; rows: CellValue[][] }>>(run.sheetsJson) ??
|
parseJson<
|
||||||
[];
|
Array<{
|
||||||
|
name: string;
|
||||||
|
headers: string[];
|
||||||
|
rows: CellValue[][];
|
||||||
|
headerRow?: number;
|
||||||
|
rowNumbers?: number[];
|
||||||
|
}>
|
||||||
|
>(run.sheetsJson) ?? [];
|
||||||
return {
|
return {
|
||||||
id: run.id,
|
id: run.id,
|
||||||
fileName: run.fileName,
|
fileName: run.fileName,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import { ImportRun } from './entities/import-run.entity';
|
|||||||
import { ImportStep } from './entities/import-step.entity';
|
import { ImportStep } from './entities/import-step.entity';
|
||||||
import { ImportRow } from './entities/import-row.entity';
|
import { ImportRow } from './entities/import-row.entity';
|
||||||
import { ImportsService } from './imports.service';
|
import { ImportsService } from './imports.service';
|
||||||
|
import * as workbookModule from './imports.workbook';
|
||||||
import type { ParsedImportFile } from './imports.types';
|
import type { ParsedImportFile } from './imports.types';
|
||||||
|
|
||||||
function makeRowsRepo() {
|
function makeRowsRepo() {
|
||||||
@@ -810,4 +811,101 @@ describe('ImportsService', () => {
|
|||||||
expect(result.rows[0]).toMatchObject({ action: 'skip', status: 'valid' });
|
expect(result.rows[0]).toMatchObject({ action: 'skip', status: 'valid' });
|
||||||
expect(result.rows[0].errors.join(';')).toContain('按策略跳过');
|
expect(result.rows[0].errors.join(';')).toContain('按策略跳过');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('preflightFile 透传 headerRow 到解析层', async () => {
|
||||||
|
const parseSpy = jest
|
||||||
|
.spyOn(workbookModule, 'parseSheets')
|
||||||
|
.mockResolvedValue([]);
|
||||||
|
try {
|
||||||
|
const service = new ImportsService(
|
||||||
|
makeRunsRepo({} as ImportRun) as never,
|
||||||
|
makeStepsRepo({} as ImportStep) as never,
|
||||||
|
makeRowsRepo() as never,
|
||||||
|
{} as never,
|
||||||
|
);
|
||||||
|
const report = await service.preflightFile(fileOf('students.xlsx', Buffer.from('x')), 3);
|
||||||
|
expect(parseSpy).toHaveBeenCalledWith(
|
||||||
|
expect.any(Buffer),
|
||||||
|
'students.xlsx',
|
||||||
|
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||||
|
3,
|
||||||
|
);
|
||||||
|
expect(report.verdict).toBe('blocked');
|
||||||
|
} finally {
|
||||||
|
parseSpy.mockRestore();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('createRun 按 stages 的 headerRow 生成对应工作表视图并写入 sheetsJson', async () => {
|
||||||
|
const workbook = new ExcelJS.Workbook();
|
||||||
|
const worksheet = workbook.addWorksheet('名单');
|
||||||
|
worksheet.addRow(['标题行', null]);
|
||||||
|
worksheet.addRow(['姓名', '学号']);
|
||||||
|
worksheet.addRow(['', '']);
|
||||||
|
worksheet.addRow(['张三', '2024001']);
|
||||||
|
const buffer = (await workbook.xlsx.writeBuffer()) as Buffer;
|
||||||
|
const run = {
|
||||||
|
id: 'run-h',
|
||||||
|
userId: 7,
|
||||||
|
conversationId: null,
|
||||||
|
source: 'manual',
|
||||||
|
fileName: 'students.xlsx',
|
||||||
|
sheetsJson: '[]',
|
||||||
|
status: 'ready',
|
||||||
|
currentStepKey: null,
|
||||||
|
error: null,
|
||||||
|
createdAt: new Date(),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
} as ImportRun;
|
||||||
|
const runsRepo = makeRunsRepo(run);
|
||||||
|
const stepsRepo = makeStepsRepo({} as ImportStep);
|
||||||
|
const service = new ImportsService(
|
||||||
|
runsRepo as never,
|
||||||
|
stepsRepo as never,
|
||||||
|
makeRowsRepo() as never,
|
||||||
|
{} as never,
|
||||||
|
);
|
||||||
|
|
||||||
|
await service.createRun(
|
||||||
|
principal,
|
||||||
|
'manual',
|
||||||
|
fileOf('students.xlsx', buffer),
|
||||||
|
null,
|
||||||
|
[{ stepKey: 'students', sheet: '名单', headerRow: 2 }],
|
||||||
|
);
|
||||||
|
|
||||||
|
const created = runsRepo.create.mock.calls[0][0] as { sheetsJson: string };
|
||||||
|
const sheets = JSON.parse(created.sheetsJson) as Array<{
|
||||||
|
headers: string[];
|
||||||
|
rows: unknown[][];
|
||||||
|
headerRow: number;
|
||||||
|
rowNumbers: number[];
|
||||||
|
}>;
|
||||||
|
expect(sheets).toHaveLength(1);
|
||||||
|
expect(sheets[0].headers).toEqual(['姓名', '学号']);
|
||||||
|
expect(sheets[0].rows).toEqual([['张三', '2024001']]);
|
||||||
|
expect(sheets[0].headerRow).toBe(2);
|
||||||
|
expect(sheets[0].rowNumbers).toEqual([4]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('createRun 拒绝映射到工作表表头之外的列名', async () => {
|
||||||
|
const buffer = await xlsxBuffer(studentSheet());
|
||||||
|
const service = new ImportsService(
|
||||||
|
makeRunsRepo({} as ImportRun) as never,
|
||||||
|
makeStepsRepo({} as ImportStep) as never,
|
||||||
|
makeRowsRepo() as never,
|
||||||
|
{} as never,
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.createRun(
|
||||||
|
principal,
|
||||||
|
'manual',
|
||||||
|
fileOf('students.xlsx', buffer),
|
||||||
|
null,
|
||||||
|
[{ stepKey: 'students', sheet: '学生' }],
|
||||||
|
{ students: { name: '姓名', studentNo: '不存在的列' } },
|
||||||
|
),
|
||||||
|
).rejects.toThrow('不在工作表表头中');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -72,11 +72,14 @@ export class ImportsService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** 上传后的只读预检:解析文件并生成可插入性报告,不写库。 */
|
/** 上传后的只读预检:解析文件并生成可插入性报告,不写库。 */
|
||||||
async preflightFile(file: ParsedImportFile): Promise<import('./imports.types').PreflightReport> {
|
async preflightFile(
|
||||||
|
file: ParsedImportFile,
|
||||||
|
headerRow = 1,
|
||||||
|
): Promise<import('./imports.types').PreflightReport> {
|
||||||
if (!file.buffer || file.buffer.length === 0) {
|
if (!file.buffer || file.buffer.length === 0) {
|
||||||
throw new BadRequestException('上传文件为空');
|
throw new BadRequestException('上传文件为空');
|
||||||
}
|
}
|
||||||
const sheets = await parseSheets(file.buffer, file.originalName, file.mimeType);
|
const sheets = await parseSheets(file.buffer, file.originalName, file.mimeType, headerRow);
|
||||||
return buildPreflightReport(this.dataSource, sheets);
|
return buildPreflightReport(this.dataSource, sheets);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -153,6 +153,8 @@ export interface PreflightStageStat {
|
|||||||
stepKey: ImportStepKey;
|
stepKey: ImportStepKey;
|
||||||
label: string;
|
label: string;
|
||||||
sheetNames: string[];
|
sheetNames: string[];
|
||||||
|
/** 该阶段所有工作表的表头并集,供前端预检卡渲染列映射选项。 */
|
||||||
|
headers: string[];
|
||||||
total: number;
|
total: number;
|
||||||
create: number;
|
create: number;
|
||||||
update: number;
|
update: number;
|
||||||
@@ -225,6 +227,12 @@ export interface PreflightReport {
|
|||||||
questions: PreflightQuestion[];
|
questions: PreflightQuestion[];
|
||||||
nextSteps: PreflightNextStep[];
|
nextSteps: PreflightNextStep[];
|
||||||
errorSamples: PreflightErrorSample[];
|
errorSamples: PreflightErrorSample[];
|
||||||
|
/** 以下字段由 AI 预检卡使用,普通预检报告生成时不设置。 */
|
||||||
|
attachmentId?: number;
|
||||||
|
headerRow?: number;
|
||||||
|
permittedSteps?: ImportStepKey[];
|
||||||
|
resolved?: boolean;
|
||||||
|
runId?: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface StepPreviewResult {
|
export interface StepPreviewResult {
|
||||||
|
|||||||
160
apps/server/src/imports/imports.workbook-fallback.ts
Normal file
160
apps/server/src/imports/imports.workbook-fallback.ts
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
import JSZip from 'jszip';
|
||||||
|
import type { ImportSheetData } from './imports.workbook';
|
||||||
|
|
||||||
|
const MAX_SHEETS = 30;
|
||||||
|
const MAX_ROWS_PER_SHEET = 3000;
|
||||||
|
const MAX_COLS_PER_SHEET = 60;
|
||||||
|
|
||||||
|
export interface ExcelFallbackSheetRows {
|
||||||
|
name: string;
|
||||||
|
rows: string[][];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read an .xlsx workbook without relying on ExcelJS. WPS-produced files
|
||||||
|
* sometimes prefix every OOXML element with a namespace that ExcelJS cannot
|
||||||
|
* load; this fallback strips the prefixes and parses shared strings directly.
|
||||||
|
*/
|
||||||
|
export async function readXlsxSheetsFallback(buffer: Buffer): Promise<ExcelFallbackSheetRows[]> {
|
||||||
|
const zip = await JSZip.loadAsync(buffer);
|
||||||
|
const readEntry = async (name: string): Promise<string | null> => {
|
||||||
|
const entry = zip.file(name);
|
||||||
|
return entry ? entry.async('string') : null;
|
||||||
|
};
|
||||||
|
const workbookXml = await readEntry('xl/workbook.xml');
|
||||||
|
if (!workbookXml) throw new Error('workbook.xml missing');
|
||||||
|
const stripPrefixes = (value: string): string =>
|
||||||
|
value.replace(/<(\/?)([a-zA-Z][\w.]*):/g, '<$1');
|
||||||
|
const relsXml = stripPrefixes((await readEntry('xl/_rels/workbook.xml.rels')) ?? '');
|
||||||
|
const relTargets = new Map<string, string>();
|
||||||
|
for (const match of relsXml.matchAll(
|
||||||
|
/<Relationship[^>]*\bId="([^"]+)"[^>]*\bTarget="([^"]+)"/g,
|
||||||
|
)) {
|
||||||
|
const target = match[2].replace(/^\/+/, '');
|
||||||
|
relTargets.set(match[1], target.startsWith('xl/') ? target : `xl/${target}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const sharedStrings = await parseSharedStringsFallback(readEntry);
|
||||||
|
const sheets: ExcelFallbackSheetRows[] = [];
|
||||||
|
const cleanWorkbook = stripPrefixes(workbookXml);
|
||||||
|
for (const match of cleanWorkbook.matchAll(/<sheet\b[^>]*\/?>/g)) {
|
||||||
|
const tag = match[0].replace(/<sheet\b/, '<sheet').replace(/\/?>$/, '>');
|
||||||
|
const name = tag.match(/\bname="([^"]+)"/)?.[1];
|
||||||
|
const rid = tag.match(/\br:id="([^"]+)"/)?.[1];
|
||||||
|
if (!name || !rid) continue;
|
||||||
|
const target = relTargets.get(rid);
|
||||||
|
const sheetXml = target ? await readEntry(target) : null;
|
||||||
|
if (!sheetXml) continue;
|
||||||
|
sheets.push({
|
||||||
|
name: unescapeXml(name),
|
||||||
|
rows: sheetRowsFromXmlFallback(sheetXml, sharedStrings),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return sheets;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Convert fallback rows (all physical rows as strings) into import sheet views. */
|
||||||
|
export function fallbackSheetsToImportSheets(
|
||||||
|
sheets: ExcelFallbackSheetRows[],
|
||||||
|
headerRow = 1,
|
||||||
|
): ImportSheetData[] {
|
||||||
|
const result: ImportSheetData[] = [];
|
||||||
|
for (const sheet of sheets.slice(0, MAX_SHEETS)) {
|
||||||
|
const headerIndex = headerRow - 1;
|
||||||
|
const headers =
|
||||||
|
sheet.rows[headerIndex]?.slice(0, MAX_COLS_PER_SHEET).map((header) => header.trim()) ?? [];
|
||||||
|
if (!headers.some((header) => header.trim() !== '')) continue;
|
||||||
|
const rows: ImportSheetData['rows'] = [];
|
||||||
|
const rowNumbers: number[] = [];
|
||||||
|
for (let i = headerIndex + 1; i < sheet.rows.length && rows.length < MAX_ROWS_PER_SHEET; i += 1) {
|
||||||
|
const values = sheet.rows[i].slice(0, headers.length);
|
||||||
|
if (values.every((value) => value === '')) continue;
|
||||||
|
rows.push(values);
|
||||||
|
rowNumbers.push(i + 1);
|
||||||
|
}
|
||||||
|
if (rows.length === 0) continue;
|
||||||
|
result.push({
|
||||||
|
name: sheet.name,
|
||||||
|
headers,
|
||||||
|
rows,
|
||||||
|
headerRow,
|
||||||
|
rowNumbers,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function parseSharedStringsFallback(
|
||||||
|
readEntry: (name: string) => Promise<string | null>,
|
||||||
|
): Promise<string[]> {
|
||||||
|
const xml = await readEntry('xl/sharedStrings.xml');
|
||||||
|
if (!xml) return [];
|
||||||
|
const clean = xml.replace(/<(\/?)([a-zA-Z][\w.]*):/g, '<$1');
|
||||||
|
const strings: string[] = [];
|
||||||
|
for (const match of clean.matchAll(/<si\b[^>]*>([\s\S]*?)<\/si>/gs)) {
|
||||||
|
const texts = [...match[1].matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/g)].map((part) =>
|
||||||
|
unescapeXml(part[1]),
|
||||||
|
);
|
||||||
|
strings.push(texts.join(''));
|
||||||
|
}
|
||||||
|
return strings;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sheetRowsFromXmlFallback(sheetXml: string, sharedStrings: string[]): string[][] {
|
||||||
|
const rows: string[][] = [];
|
||||||
|
const xml = sheetXml.replace(/<(\/?)([a-zA-Z][\w.]*):/g, '<$1');
|
||||||
|
for (const rowMatch of xml.matchAll(/<row\b[^>]*>([\s\S]*?)<\/row>/gs)) {
|
||||||
|
const cells = new Map<number, string>();
|
||||||
|
let maxColumn = -1;
|
||||||
|
for (const cellMatch of rowMatch[1].matchAll(/<c\b([^>]*)\/?>([\s\S]*?)<\/c>/gs)) {
|
||||||
|
const attrs = cellMatch[1];
|
||||||
|
const refMatch = attrs.match(/\br="([A-Z]+)\d+"/);
|
||||||
|
const column = refMatch ? columnIndex(refMatch[1]) : -1;
|
||||||
|
const type = attrs.match(/\bt="([^"]+)"/)?.[1] ?? 'n';
|
||||||
|
const body = cellMatch[2] ?? '';
|
||||||
|
let value = '';
|
||||||
|
if (type === 's') {
|
||||||
|
const index = Number(body.match(/<v>([^<]*)<\/v>/)?.[1] ?? '');
|
||||||
|
value = Number.isInteger(index) ? (sharedStrings[index] ?? '') : '';
|
||||||
|
} else if (type === 'inlineStr') {
|
||||||
|
const texts = [...body.matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/g)].map((part) =>
|
||||||
|
unescapeXml(part[1]),
|
||||||
|
);
|
||||||
|
value = texts.join('');
|
||||||
|
} else {
|
||||||
|
value = unescapeXml(body.match(/<v>([\s\S]*?)<\/v>/)?.[1] ?? '');
|
||||||
|
if (type === 'b') value = value === '1' ? 'true' : 'false';
|
||||||
|
}
|
||||||
|
if (column >= 0) {
|
||||||
|
cells.set(column, value);
|
||||||
|
maxColumn = Math.max(maxColumn, column);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (maxColumn < 0) continue;
|
||||||
|
const values = Array.from({ length: maxColumn + 1 }, (_, index) => cells.get(index) ?? '');
|
||||||
|
if (values.every((value) => value === '')) continue;
|
||||||
|
rows.push(values);
|
||||||
|
}
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
function columnIndex(letters: string): number {
|
||||||
|
let index = 0;
|
||||||
|
for (const char of letters.toUpperCase()) {
|
||||||
|
index = index * 26 + (char.charCodeAt(0) - 64);
|
||||||
|
}
|
||||||
|
return index - 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
function unescapeXml(value: string): string {
|
||||||
|
return value
|
||||||
|
.replace(/</g, '<')
|
||||||
|
.replace(/>/g, '>')
|
||||||
|
.replace(/"/g, '"')
|
||||||
|
.replace(/'/g, "'")
|
||||||
|
.replace(/&/g, '&')
|
||||||
|
.replace(/&#x([0-9a-fA-F]+);/g, (_all, hex: string) =>
|
||||||
|
String.fromCodePoint(Number.parseInt(hex, 16)),
|
||||||
|
)
|
||||||
|
.replace(/&#(\d+);/g, (_all, dec: string) => String.fromCodePoint(Number(dec)));
|
||||||
|
}
|
||||||
104
apps/server/src/imports/imports.workbook.spec.ts
Normal file
104
apps/server/src/imports/imports.workbook.spec.ts
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
import { BadRequestException } from '@nestjs/common';
|
||||||
|
import * as ExcelJS from 'exceljs';
|
||||||
|
import JSZip from 'jszip';
|
||||||
|
import { parseDateValue } from './imports.helpers';
|
||||||
|
import { parseSheets } from './imports.workbook';
|
||||||
|
|
||||||
|
async function xlsxBuffer(rows: Array<Array<unknown>>): Promise<Buffer> {
|
||||||
|
const workbook = new ExcelJS.Workbook();
|
||||||
|
const worksheet = workbook.addWorksheet('名单');
|
||||||
|
rows.forEach((row, index) => worksheet.addRow(row));
|
||||||
|
return (await workbook.xlsx.writeBuffer()) as Buffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function wpsNamespaceXlsxBuffer(): Promise<Buffer> {
|
||||||
|
const zip = new JSZip();
|
||||||
|
zip.file('[Content_Types].xml', '<?xml version="1.0" encoding="UTF-8"?><Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"/>');
|
||||||
|
zip.file(
|
||||||
|
'xl/workbook.xml',
|
||||||
|
`<x:workbook xmlns:x="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
|
||||||
|
<x:sheets><x:sheet name="名单" sheetId="1" state="visible" r:id="rId1"/></x:sheets>
|
||||||
|
</x:workbook>`,
|
||||||
|
);
|
||||||
|
zip.file(
|
||||||
|
'xl/_rels/workbook.xml.rels',
|
||||||
|
`<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||||
|
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/>
|
||||||
|
</Relationships>`,
|
||||||
|
);
|
||||||
|
zip.file(
|
||||||
|
'xl/sharedStrings.xml',
|
||||||
|
`<x:sst xmlns:x="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><x:si><x:t>张三</x:t></x:si></x:sst>`,
|
||||||
|
);
|
||||||
|
zip.file(
|
||||||
|
'xl/worksheets/sheet1.xml',
|
||||||
|
`<x:worksheet xmlns:x="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||||
|
<x:sheetData>
|
||||||
|
<x:row r="1"><x:c r="A1" t="inlineStr"><x:is><x:t>姓名</x:t></x:is></x:c><x:c r="B1" t="inlineStr"><x:is><x:t>手机号</x:t></x:is></x:c></x:row>
|
||||||
|
<x:row r="2"><x:c r="A2" t="s"><x:v>0</x:v></x:c><x:c r="B2"><x:v>13800138000</x:v></x:c></x:row>
|
||||||
|
</x:sheetData>
|
||||||
|
</x:worksheet>`,
|
||||||
|
);
|
||||||
|
return Buffer.from(await zip.generateAsync({ type: 'nodebuffer' }));
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('parseSheets', () => {
|
||||||
|
it('从 headerRow 取表头,并保留空行后的物理行号', async () => {
|
||||||
|
const buffer = await xlsxBuffer(
|
||||||
|
[
|
||||||
|
['标题行', null],
|
||||||
|
['姓名', '学号'],
|
||||||
|
['', ''],
|
||||||
|
['张三', '2024001'],
|
||||||
|
['李四', '2024002'],
|
||||||
|
],
|
||||||
|
);
|
||||||
|
const sheets = await parseSheets(
|
||||||
|
buffer,
|
||||||
|
'students.xlsx',
|
||||||
|
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||||
|
2,
|
||||||
|
);
|
||||||
|
expect(sheets).toHaveLength(1);
|
||||||
|
expect(sheets[0].headers).toEqual(['姓名', '学号']);
|
||||||
|
expect(sheets[0].rows).toEqual([
|
||||||
|
['张三', '2024001'],
|
||||||
|
['李四', '2024002'],
|
||||||
|
]);
|
||||||
|
expect(sheets[0].rowNumbers).toEqual([4, 5]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('解析 CSV 并默认使用第一行作为表头', async () => {
|
||||||
|
const buffer = Buffer.from('姓名,学号\n张三,2024001\n', 'utf8');
|
||||||
|
const sheets = await parseSheets(buffer, 'students.csv', 'text/csv');
|
||||||
|
expect(sheets).toHaveLength(1);
|
||||||
|
expect(sheets[0].headers).toEqual(['姓名', '学号']);
|
||||||
|
expect(sheets[0].rows[0]).toEqual(['张三', 2024001]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('ExcelJS 失败时回退到 WPS 命名空间解析', async () => {
|
||||||
|
const buffer = await wpsNamespaceXlsxBuffer();
|
||||||
|
const sheets = await parseSheets(
|
||||||
|
buffer,
|
||||||
|
'students.xlsx',
|
||||||
|
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||||
|
);
|
||||||
|
expect(sheets).toHaveLength(1);
|
||||||
|
expect(sheets[0].name).toBe('名单');
|
||||||
|
expect(sheets[0].headers).toEqual(['姓名', '手机号']);
|
||||||
|
expect(sheets[0].rows).toEqual([['张三', '13800138000']]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('拒绝 .xls 文件', async () => {
|
||||||
|
await expect(
|
||||||
|
parseSheets(Buffer.from('not excel'), 'a.xls', 'application/vnd.ms-excel'),
|
||||||
|
).rejects.toBeInstanceOf(BadRequestException);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('parseDateValue', () => {
|
||||||
|
it('按进程本地时区格式化为 YYYY-MM-DD(CST 午夜不偏移到前一天)', () => {
|
||||||
|
expect(parseDateValue(new Date(2026, 0, 1))).toBe('2026-01-01');
|
||||||
|
expect(parseDateValue(new Date(2026, 11, 31))).toBe('2026-12-31');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -2,6 +2,7 @@ import { BadRequestException } from '@nestjs/common';
|
|||||||
import * as ExcelJS from 'exceljs';
|
import * as ExcelJS from 'exceljs';
|
||||||
import { Readable } from 'node:stream';
|
import { Readable } from 'node:stream';
|
||||||
import { cellValue, textValue } from './imports.helpers';
|
import { cellValue, textValue } from './imports.helpers';
|
||||||
|
import { fallbackSheetsToImportSheets, readXlsxSheetsFallback } from './imports.workbook-fallback';
|
||||||
import type { CellValue } from './imports.types';
|
import type { CellValue } from './imports.types';
|
||||||
|
|
||||||
const MAX_SHEETS = 30;
|
const MAX_SHEETS = 30;
|
||||||
@@ -12,30 +13,38 @@ export interface ImportSheetData {
|
|||||||
name: string;
|
name: string;
|
||||||
headers: string[];
|
headers: string[];
|
||||||
rows: CellValue[][];
|
rows: CellValue[][];
|
||||||
|
/** 1-based header row used to build this view; defaults to 1. */
|
||||||
|
headerRow?: number;
|
||||||
|
/** Physical 1-based row numbers for each entry in `rows` (after headerRow). */
|
||||||
|
rowNumbers?: number[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export function extractSheets(workbook: ExcelJS.Workbook): ImportSheetData[] {
|
export function extractSheets(workbook: ExcelJS.Workbook, headerRow = 1): ImportSheetData[] {
|
||||||
const sheets: ImportSheetData[] = [];
|
const sheets: ImportSheetData[] = [];
|
||||||
for (const worksheet of workbook.worksheets) {
|
for (const worksheet of workbook.worksheets) {
|
||||||
if (sheets.length >= MAX_SHEETS) break;
|
if (sheets.length >= MAX_SHEETS) break;
|
||||||
const headers: string[] = [];
|
const headers: string[] = [];
|
||||||
const rows: CellValue[][] = [];
|
const rows: CellValue[][] = [];
|
||||||
const firstRow = worksheet.getRow(1);
|
const rowNumbers: number[] = [];
|
||||||
|
const firstRow = worksheet.getRow(headerRow);
|
||||||
for (let col = 1; col <= Math.min(firstRow.cellCount, MAX_COLS_PER_SHEET); col += 1) {
|
for (let col = 1; col <= Math.min(firstRow.cellCount, MAX_COLS_PER_SHEET); col += 1) {
|
||||||
const header = textValue(cellValue(firstRow.getCell(col)));
|
const header = textValue(cellValue(firstRow.getCell(col)));
|
||||||
headers.push(header);
|
headers.push(header);
|
||||||
}
|
}
|
||||||
if (!headers.some(Boolean)) continue;
|
if (!headers.some(Boolean)) continue;
|
||||||
worksheet.eachRow({ includeEmpty: false }, (row, rowNumber) => {
|
worksheet.eachRow({ includeEmpty: false }, (row, rowNumber) => {
|
||||||
if (rowNumber === 1 || rows.length >= MAX_ROWS_PER_SHEET) return;
|
if (rowNumber <= headerRow || rows.length >= MAX_ROWS_PER_SHEET) return;
|
||||||
const values: CellValue[] = [];
|
const values: CellValue[] = [];
|
||||||
for (let col = 1; col <= headers.length; col += 1) {
|
for (let col = 1; col <= headers.length; col += 1) {
|
||||||
values.push(cellValue(row.getCell(col)));
|
values.push(cellValue(row.getCell(col)));
|
||||||
}
|
}
|
||||||
if (values.every((v) => v === null || textValue(v) === '')) return;
|
if (values.every((v) => v === null || textValue(v) === '')) return;
|
||||||
rows.push(values);
|
rows.push(values);
|
||||||
|
rowNumbers.push(rowNumber);
|
||||||
});
|
});
|
||||||
if (rows.length > 0) sheets.push({ name: worksheet.name, headers, rows });
|
if (rows.length > 0) {
|
||||||
|
sheets.push({ name: worksheet.name, headers, rows, headerRow, rowNumbers });
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return sheets;
|
return sheets;
|
||||||
}
|
}
|
||||||
@@ -61,6 +70,7 @@ export async function parseSheets(
|
|||||||
buffer: Buffer,
|
buffer: Buffer,
|
||||||
originalName: string,
|
originalName: string,
|
||||||
mimeType: string,
|
mimeType: string,
|
||||||
|
headerRow = 1,
|
||||||
): Promise<ImportSheetData[]> {
|
): Promise<ImportSheetData[]> {
|
||||||
const kind = detectWorkbookKind(originalName, mimeType);
|
const kind = detectWorkbookKind(originalName, mimeType);
|
||||||
if (!kind) {
|
if (!kind) {
|
||||||
@@ -76,13 +86,24 @@ export async function parseSheets(
|
|||||||
} else {
|
} else {
|
||||||
await workbook.xlsx.load(buffer.buffer as ArrayBuffer);
|
await workbook.xlsx.load(buffer.buffer as ArrayBuffer);
|
||||||
}
|
}
|
||||||
const sheets = extractSheets(workbook);
|
const sheets = extractSheets(workbook, headerRow);
|
||||||
if (sheets.length === 0) {
|
if (sheets.length === 0) {
|
||||||
throw new BadRequestException('文件中没有可用的工作表数据');
|
throw new BadRequestException('文件中没有可用的工作表数据');
|
||||||
}
|
}
|
||||||
return sheets;
|
return sheets;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof BadRequestException) throw error;
|
if (error instanceof BadRequestException) throw error;
|
||||||
|
if (kind === 'xlsx') {
|
||||||
|
try {
|
||||||
|
const fallbackSheets = fallbackSheetsToImportSheets(
|
||||||
|
await readXlsxSheetsFallback(buffer),
|
||||||
|
headerRow,
|
||||||
|
);
|
||||||
|
if (fallbackSheets.length > 0) return fallbackSheets;
|
||||||
|
} catch {
|
||||||
|
// fall through to the readable error below
|
||||||
|
}
|
||||||
|
}
|
||||||
throw new BadRequestException('Excel 文件解析失败,请检查文件格式');
|
throw new BadRequestException('Excel 文件解析失败,请检查文件格式');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user