feat(imports): 导入向导兼容旧版工作簿并完善执行/预览
- 新增 xls 工作簿回退解析与测试 - 执行/预览/权限校验逻辑完善
This commit is contained in:
37
apps/server/src/imports/imports.access.spec.ts
Normal file
37
apps/server/src/imports/imports.access.spec.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { ForbiddenException } from '@nestjs/common';
|
||||
import { assertStepPermission, permittedStepKeys } from './imports.access';
|
||||
|
||||
describe('permittedStepKeys', () => {
|
||||
it('超级管理员拥有全部导入阶段', () => {
|
||||
expect(
|
||||
permittedStepKeys({ id: 1, permissions: [], isSuperAdmin: true }),
|
||||
).toEqual(['students', 'rooms', 'checkins', 'transfers']);
|
||||
});
|
||||
|
||||
it('按细分权限计算可提交阶段', () => {
|
||||
expect(
|
||||
permittedStepKeys({
|
||||
id: 1,
|
||||
permissions: ['student:import', 'room:edit', 'occupancy:checkin'],
|
||||
isSuperAdmin: false,
|
||||
}),
|
||||
).toEqual(['students', 'rooms', 'checkins']);
|
||||
});
|
||||
|
||||
it('无导入权限时仅返回空数组', () => {
|
||||
expect(
|
||||
permittedStepKeys({ id: 1, permissions: ['ai:chat:use'], isSuperAdmin: false }),
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('assertStepPermission', () => {
|
||||
it('无对应细分权限时拒绝提交', () => {
|
||||
expect(() =>
|
||||
assertStepPermission(
|
||||
{ id: 1, permissions: ['ai:chat:use'], isSuperAdmin: false },
|
||||
'transfers',
|
||||
),
|
||||
).toThrow(ForbiddenException);
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,7 @@ import { ForbiddenException, NotFoundException } from '@nestjs/common';
|
||||
import { Repository } from 'typeorm';
|
||||
import { ImportRun } from './entities/import-run.entity';
|
||||
import { ImportStep } from './entities/import-step.entity';
|
||||
import { IMPORT_STEP_LABELS } from './imports.types';
|
||||
import { IMPORT_STEP_LABELS, IMPORT_STEP_ORDER } from './imports.types';
|
||||
import type { ImportStepKey } from './imports.types';
|
||||
|
||||
export interface ImportPrincipal {
|
||||
@@ -45,3 +45,11 @@ export function assertStepPermission(principal: ImportPrincipal, stepKey: Import
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Step keys the current principal may actually commit. */
|
||||
export function permittedStepKeys(principal: ImportPrincipal): ImportStepKey[] {
|
||||
if (principal.isSuperAdmin) return [...IMPORT_STEP_ORDER];
|
||||
return IMPORT_STEP_ORDER.filter((stepKey) =>
|
||||
STEP_PERMISSIONS[stepKey].some((code) => principal.permissions.includes(code)),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -7,6 +7,20 @@ describe('ImportsController', () => {
|
||||
connection: { remoteAddress: '127.0.0.1' },
|
||||
};
|
||||
|
||||
it('路由门槛允许 AI 用户打开本人 run,同时保留原有导入权限', () => {
|
||||
const permissions = Reflect.getMetadata('permissions', ImportsController) as string[];
|
||||
expect(permissions).toContain('ai:chat:use');
|
||||
expect(permissions).toEqual(
|
||||
expect.arrayContaining([
|
||||
'student:import',
|
||||
'room:create',
|
||||
'room:edit',
|
||||
'occupancy:checkin',
|
||||
'occupancy:transfer',
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('提交导入阶段成功后写入操作日志', async () => {
|
||||
const commitStep = jest.fn().mockResolvedValue({
|
||||
runId: 'run-1',
|
||||
|
||||
@@ -38,7 +38,7 @@ const IMPORT_GATE_PERMISSIONS = [
|
||||
] as const;
|
||||
|
||||
@Controller('imports')
|
||||
@RequirePermission(...IMPORT_GATE_PERMISSIONS)
|
||||
@RequirePermission('ai:chat:use', ...IMPORT_GATE_PERMISSIONS)
|
||||
export class ImportsController {
|
||||
constructor(
|
||||
private readonly importsService: ImportsService,
|
||||
|
||||
@@ -50,7 +50,10 @@ export function cellValue(cell: ExcelJS.Cell | undefined): CellValue {
|
||||
|
||||
export function parseDateValue(value: CellValue): string | null {
|
||||
if (value instanceof Date && !Number.isNaN(value.getTime())) {
|
||||
return value.toISOString().slice(0, 10);
|
||||
const year = value.getFullYear();
|
||||
const month = String(value.getMonth() + 1).padStart(2, '0');
|
||||
const day = String(value.getDate()).padStart(2, '0');
|
||||
return `${year}-${month}-${day}`;
|
||||
}
|
||||
const raw = textValue(value);
|
||||
if (!raw) return null;
|
||||
|
||||
@@ -52,6 +52,7 @@ describe('buildPreflightReport', () => {
|
||||
update: 0,
|
||||
error: 0,
|
||||
skip: 0,
|
||||
headers: ['姓名', '学号', '手机号'],
|
||||
mapping: { name: '姓名', studentNo: '学号', phone: '手机号' },
|
||||
});
|
||||
expect(report.blocks).toEqual([]);
|
||||
|
||||
@@ -183,7 +183,7 @@ async function analyzeStage(
|
||||
code,
|
||||
stepKey,
|
||||
sheet: sheet.name,
|
||||
rowNumber: i + 2,
|
||||
rowNumber: sheet.rowNumbers?.[i] ?? (sheet.headerRow ?? 1) + i + 1,
|
||||
errors: result.errors,
|
||||
});
|
||||
}
|
||||
@@ -209,6 +209,7 @@ async function analyzeStage(
|
||||
stepKey,
|
||||
label: IMPORT_STEP_LABELS[stepKey],
|
||||
sheetNames: sheets.map((sheet) => sheet.name),
|
||||
headers: [...new Set(sheets.flatMap((sheet) => sheet.headers))],
|
||||
total,
|
||||
create,
|
||||
update,
|
||||
|
||||
@@ -55,8 +55,15 @@ export class ImportPreviewService {
|
||||
}
|
||||
|
||||
const sheetsData =
|
||||
parseJson<Array<{ name: string; headers: string[]; rows: CellValue[][] }>>(run.sheetsJson) ??
|
||||
[];
|
||||
parseJson<
|
||||
Array<{
|
||||
name: string;
|
||||
headers: string[];
|
||||
rows: CellValue[][];
|
||||
headerRow?: number;
|
||||
rowNumbers?: number[];
|
||||
}>
|
||||
>(run.sheetsJson) ?? [];
|
||||
const settings = parseJson<ImportRunSettings>(run.settingsJson) ?? {};
|
||||
const sheetNames = body.sheets?.length
|
||||
? body.sheets
|
||||
@@ -132,7 +139,7 @@ export class ImportPreviewService {
|
||||
runId,
|
||||
stepId: step.id,
|
||||
sheetName,
|
||||
rowNumber: i + 2,
|
||||
rowNumber: sheet.rowNumbers?.[i] ?? (sheet.headerRow ?? 1) + i + 1,
|
||||
rawJson: JSON.stringify(raw),
|
||||
normalizedJson: JSON.stringify(normalized),
|
||||
matchKey: result.matchKey,
|
||||
|
||||
@@ -17,6 +17,7 @@ import type {
|
||||
} from './imports.types';
|
||||
import { parseJson } from './imports.helpers';
|
||||
import { parseSheets } from './imports.workbook';
|
||||
import type { ImportSheetData } from './imports.workbook';
|
||||
import { autoAssignedSheets, resolveAssignedSheets, suggestMapping, suggestStep } from './imports.mapping';
|
||||
import { findOwnedRun } from './imports.access';
|
||||
import type { ImportPrincipal } from './imports.access';
|
||||
@@ -42,7 +43,35 @@ export class ImportRunService {
|
||||
if (!file.buffer || file.buffer.length === 0) {
|
||||
throw new BadRequestException('上传文件为空');
|
||||
}
|
||||
const sheets = await parseSheets(file.buffer, file.originalName, file.mimeType);
|
||||
const headerRows =
|
||||
stages && stages.length > 0
|
||||
? [...new Set(stages.map((stage) => stage.headerRow ?? 1))]
|
||||
: [1];
|
||||
const parsedByHeaderRow = new Map<number, ImportSheetData[]>();
|
||||
for (const headerRow of headerRows) {
|
||||
parsedByHeaderRow.set(
|
||||
headerRow,
|
||||
await parseSheets(file.buffer, file.originalName, file.mimeType, headerRow),
|
||||
);
|
||||
}
|
||||
const views = new Map<string, ImportSheetData>();
|
||||
if (stages && stages.length > 0) {
|
||||
// v1 limitation: a sheet referenced by multiple stages with different
|
||||
// header rows keeps the view of the last stage that named it.
|
||||
for (const stage of stages) {
|
||||
const sheetName = stage.sheet?.trim();
|
||||
if (!sheetName) continue;
|
||||
const view = parsedByHeaderRow
|
||||
.get(stage.headerRow ?? 1)
|
||||
?.find((sheet) => sheet.name === sheetName);
|
||||
if (view) views.set(sheetName, view);
|
||||
}
|
||||
} else {
|
||||
for (const sheet of parsedByHeaderRow.get(1) ?? []) {
|
||||
views.set(sheet.name, sheet);
|
||||
}
|
||||
}
|
||||
const sheets = [...views.values()];
|
||||
|
||||
const runId = randomUUID();
|
||||
const run = this.runs.create({
|
||||
@@ -85,6 +114,20 @@ export class ImportRunService {
|
||||
const firstSheet = sheets.find((s) => s.name === assigned[0]);
|
||||
const mapping =
|
||||
mappingByStep?.[stepKey] ?? suggestMapping(firstSheet?.headers ?? [], stepKey);
|
||||
if (mapping) {
|
||||
const allowedHeaders = new Set<string>();
|
||||
for (const sheetName of assigned) {
|
||||
const sheet = sheets.find((s) => s.name === sheetName);
|
||||
for (const header of sheet?.headers ?? []) allowedHeaders.add(header);
|
||||
}
|
||||
for (const [field, header] of Object.entries(mapping)) {
|
||||
if (header && allowedHeaders.size > 0 && !allowedHeaders.has(header)) {
|
||||
throw new BadRequestException(
|
||||
`「${IMPORT_STEP_LABELS[stepKey]}」列映射「${header}」(字段 ${field})不在工作表表头中`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
stepRecords.push(
|
||||
this.steps.create({
|
||||
runId,
|
||||
@@ -108,8 +151,15 @@ export class ImportRunService {
|
||||
const run = await findOwnedRun(this.runs, userId, runId);
|
||||
const stepRecords = await this.steps.find({ where: { runId }, order: { id: 'ASC' } });
|
||||
const sheets =
|
||||
parseJson<Array<{ name: string; headers: string[]; rows: CellValue[][] }>>(run.sheetsJson) ??
|
||||
[];
|
||||
parseJson<
|
||||
Array<{
|
||||
name: string;
|
||||
headers: string[];
|
||||
rows: CellValue[][];
|
||||
headerRow?: number;
|
||||
rowNumbers?: number[];
|
||||
}>
|
||||
>(run.sheetsJson) ?? [];
|
||||
return {
|
||||
id: run.id,
|
||||
fileName: run.fileName,
|
||||
|
||||
@@ -8,6 +8,7 @@ import { ImportRun } from './entities/import-run.entity';
|
||||
import { ImportStep } from './entities/import-step.entity';
|
||||
import { ImportRow } from './entities/import-row.entity';
|
||||
import { ImportsService } from './imports.service';
|
||||
import * as workbookModule from './imports.workbook';
|
||||
import type { ParsedImportFile } from './imports.types';
|
||||
|
||||
function makeRowsRepo() {
|
||||
@@ -810,4 +811,101 @@ describe('ImportsService', () => {
|
||||
expect(result.rows[0]).toMatchObject({ action: 'skip', status: 'valid' });
|
||||
expect(result.rows[0].errors.join(';')).toContain('按策略跳过');
|
||||
});
|
||||
|
||||
it('preflightFile 透传 headerRow 到解析层', async () => {
|
||||
const parseSpy = jest
|
||||
.spyOn(workbookModule, 'parseSheets')
|
||||
.mockResolvedValue([]);
|
||||
try {
|
||||
const service = new ImportsService(
|
||||
makeRunsRepo({} as ImportRun) as never,
|
||||
makeStepsRepo({} as ImportStep) as never,
|
||||
makeRowsRepo() as never,
|
||||
{} as never,
|
||||
);
|
||||
const report = await service.preflightFile(fileOf('students.xlsx', Buffer.from('x')), 3);
|
||||
expect(parseSpy).toHaveBeenCalledWith(
|
||||
expect.any(Buffer),
|
||||
'students.xlsx',
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
3,
|
||||
);
|
||||
expect(report.verdict).toBe('blocked');
|
||||
} finally {
|
||||
parseSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('createRun 按 stages 的 headerRow 生成对应工作表视图并写入 sheetsJson', async () => {
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
const worksheet = workbook.addWorksheet('名单');
|
||||
worksheet.addRow(['标题行', null]);
|
||||
worksheet.addRow(['姓名', '学号']);
|
||||
worksheet.addRow(['', '']);
|
||||
worksheet.addRow(['张三', '2024001']);
|
||||
const buffer = (await workbook.xlsx.writeBuffer()) as Buffer;
|
||||
const run = {
|
||||
id: 'run-h',
|
||||
userId: 7,
|
||||
conversationId: null,
|
||||
source: 'manual',
|
||||
fileName: 'students.xlsx',
|
||||
sheetsJson: '[]',
|
||||
status: 'ready',
|
||||
currentStepKey: null,
|
||||
error: null,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
} as ImportRun;
|
||||
const runsRepo = makeRunsRepo(run);
|
||||
const stepsRepo = makeStepsRepo({} as ImportStep);
|
||||
const service = new ImportsService(
|
||||
runsRepo as never,
|
||||
stepsRepo as never,
|
||||
makeRowsRepo() as never,
|
||||
{} as never,
|
||||
);
|
||||
|
||||
await service.createRun(
|
||||
principal,
|
||||
'manual',
|
||||
fileOf('students.xlsx', buffer),
|
||||
null,
|
||||
[{ stepKey: 'students', sheet: '名单', headerRow: 2 }],
|
||||
);
|
||||
|
||||
const created = runsRepo.create.mock.calls[0][0] as { sheetsJson: string };
|
||||
const sheets = JSON.parse(created.sheetsJson) as Array<{
|
||||
headers: string[];
|
||||
rows: unknown[][];
|
||||
headerRow: number;
|
||||
rowNumbers: number[];
|
||||
}>;
|
||||
expect(sheets).toHaveLength(1);
|
||||
expect(sheets[0].headers).toEqual(['姓名', '学号']);
|
||||
expect(sheets[0].rows).toEqual([['张三', '2024001']]);
|
||||
expect(sheets[0].headerRow).toBe(2);
|
||||
expect(sheets[0].rowNumbers).toEqual([4]);
|
||||
});
|
||||
|
||||
it('createRun 拒绝映射到工作表表头之外的列名', async () => {
|
||||
const buffer = await xlsxBuffer(studentSheet());
|
||||
const service = new ImportsService(
|
||||
makeRunsRepo({} as ImportRun) as never,
|
||||
makeStepsRepo({} as ImportStep) as never,
|
||||
makeRowsRepo() as never,
|
||||
{} as never,
|
||||
);
|
||||
|
||||
await expect(
|
||||
service.createRun(
|
||||
principal,
|
||||
'manual',
|
||||
fileOf('students.xlsx', buffer),
|
||||
null,
|
||||
[{ stepKey: 'students', sheet: '学生' }],
|
||||
{ students: { name: '姓名', studentNo: '不存在的列' } },
|
||||
),
|
||||
).rejects.toThrow('不在工作表表头中');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -72,11 +72,14 @@ export class ImportsService {
|
||||
}
|
||||
|
||||
/** 上传后的只读预检:解析文件并生成可插入性报告,不写库。 */
|
||||
async preflightFile(file: ParsedImportFile): Promise<import('./imports.types').PreflightReport> {
|
||||
async preflightFile(
|
||||
file: ParsedImportFile,
|
||||
headerRow = 1,
|
||||
): Promise<import('./imports.types').PreflightReport> {
|
||||
if (!file.buffer || file.buffer.length === 0) {
|
||||
throw new BadRequestException('上传文件为空');
|
||||
}
|
||||
const sheets = await parseSheets(file.buffer, file.originalName, file.mimeType);
|
||||
const sheets = await parseSheets(file.buffer, file.originalName, file.mimeType, headerRow);
|
||||
return buildPreflightReport(this.dataSource, sheets);
|
||||
}
|
||||
|
||||
|
||||
@@ -153,6 +153,8 @@ export interface PreflightStageStat {
|
||||
stepKey: ImportStepKey;
|
||||
label: string;
|
||||
sheetNames: string[];
|
||||
/** 该阶段所有工作表的表头并集,供前端预检卡渲染列映射选项。 */
|
||||
headers: string[];
|
||||
total: number;
|
||||
create: number;
|
||||
update: number;
|
||||
@@ -225,6 +227,12 @@ export interface PreflightReport {
|
||||
questions: PreflightQuestion[];
|
||||
nextSteps: PreflightNextStep[];
|
||||
errorSamples: PreflightErrorSample[];
|
||||
/** 以下字段由 AI 预检卡使用,普通预检报告生成时不设置。 */
|
||||
attachmentId?: number;
|
||||
headerRow?: number;
|
||||
permittedSteps?: ImportStepKey[];
|
||||
resolved?: boolean;
|
||||
runId?: string | null;
|
||||
}
|
||||
|
||||
export interface StepPreviewResult {
|
||||
|
||||
160
apps/server/src/imports/imports.workbook-fallback.ts
Normal file
160
apps/server/src/imports/imports.workbook-fallback.ts
Normal file
@@ -0,0 +1,160 @@
|
||||
import JSZip from 'jszip';
|
||||
import type { ImportSheetData } from './imports.workbook';
|
||||
|
||||
const MAX_SHEETS = 30;
|
||||
const MAX_ROWS_PER_SHEET = 3000;
|
||||
const MAX_COLS_PER_SHEET = 60;
|
||||
|
||||
export interface ExcelFallbackSheetRows {
|
||||
name: string;
|
||||
rows: string[][];
|
||||
}
|
||||
|
||||
/**
|
||||
* Read an .xlsx workbook without relying on ExcelJS. WPS-produced files
|
||||
* sometimes prefix every OOXML element with a namespace that ExcelJS cannot
|
||||
* load; this fallback strips the prefixes and parses shared strings directly.
|
||||
*/
|
||||
export async function readXlsxSheetsFallback(buffer: Buffer): Promise<ExcelFallbackSheetRows[]> {
|
||||
const zip = await JSZip.loadAsync(buffer);
|
||||
const readEntry = async (name: string): Promise<string | null> => {
|
||||
const entry = zip.file(name);
|
||||
return entry ? entry.async('string') : null;
|
||||
};
|
||||
const workbookXml = await readEntry('xl/workbook.xml');
|
||||
if (!workbookXml) throw new Error('workbook.xml missing');
|
||||
const stripPrefixes = (value: string): string =>
|
||||
value.replace(/<(\/?)([a-zA-Z][\w.]*):/g, '<$1');
|
||||
const relsXml = stripPrefixes((await readEntry('xl/_rels/workbook.xml.rels')) ?? '');
|
||||
const relTargets = new Map<string, string>();
|
||||
for (const match of relsXml.matchAll(
|
||||
/<Relationship[^>]*\bId="([^"]+)"[^>]*\bTarget="([^"]+)"/g,
|
||||
)) {
|
||||
const target = match[2].replace(/^\/+/, '');
|
||||
relTargets.set(match[1], target.startsWith('xl/') ? target : `xl/${target}`);
|
||||
}
|
||||
|
||||
const sharedStrings = await parseSharedStringsFallback(readEntry);
|
||||
const sheets: ExcelFallbackSheetRows[] = [];
|
||||
const cleanWorkbook = stripPrefixes(workbookXml);
|
||||
for (const match of cleanWorkbook.matchAll(/<sheet\b[^>]*\/?>/g)) {
|
||||
const tag = match[0].replace(/<sheet\b/, '<sheet').replace(/\/?>$/, '>');
|
||||
const name = tag.match(/\bname="([^"]+)"/)?.[1];
|
||||
const rid = tag.match(/\br:id="([^"]+)"/)?.[1];
|
||||
if (!name || !rid) continue;
|
||||
const target = relTargets.get(rid);
|
||||
const sheetXml = target ? await readEntry(target) : null;
|
||||
if (!sheetXml) continue;
|
||||
sheets.push({
|
||||
name: unescapeXml(name),
|
||||
rows: sheetRowsFromXmlFallback(sheetXml, sharedStrings),
|
||||
});
|
||||
}
|
||||
return sheets;
|
||||
}
|
||||
|
||||
/** Convert fallback rows (all physical rows as strings) into import sheet views. */
|
||||
export function fallbackSheetsToImportSheets(
|
||||
sheets: ExcelFallbackSheetRows[],
|
||||
headerRow = 1,
|
||||
): ImportSheetData[] {
|
||||
const result: ImportSheetData[] = [];
|
||||
for (const sheet of sheets.slice(0, MAX_SHEETS)) {
|
||||
const headerIndex = headerRow - 1;
|
||||
const headers =
|
||||
sheet.rows[headerIndex]?.slice(0, MAX_COLS_PER_SHEET).map((header) => header.trim()) ?? [];
|
||||
if (!headers.some((header) => header.trim() !== '')) continue;
|
||||
const rows: ImportSheetData['rows'] = [];
|
||||
const rowNumbers: number[] = [];
|
||||
for (let i = headerIndex + 1; i < sheet.rows.length && rows.length < MAX_ROWS_PER_SHEET; i += 1) {
|
||||
const values = sheet.rows[i].slice(0, headers.length);
|
||||
if (values.every((value) => value === '')) continue;
|
||||
rows.push(values);
|
||||
rowNumbers.push(i + 1);
|
||||
}
|
||||
if (rows.length === 0) continue;
|
||||
result.push({
|
||||
name: sheet.name,
|
||||
headers,
|
||||
rows,
|
||||
headerRow,
|
||||
rowNumbers,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function parseSharedStringsFallback(
|
||||
readEntry: (name: string) => Promise<string | null>,
|
||||
): Promise<string[]> {
|
||||
const xml = await readEntry('xl/sharedStrings.xml');
|
||||
if (!xml) return [];
|
||||
const clean = xml.replace(/<(\/?)([a-zA-Z][\w.]*):/g, '<$1');
|
||||
const strings: string[] = [];
|
||||
for (const match of clean.matchAll(/<si\b[^>]*>([\s\S]*?)<\/si>/gs)) {
|
||||
const texts = [...match[1].matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/g)].map((part) =>
|
||||
unescapeXml(part[1]),
|
||||
);
|
||||
strings.push(texts.join(''));
|
||||
}
|
||||
return strings;
|
||||
}
|
||||
|
||||
function sheetRowsFromXmlFallback(sheetXml: string, sharedStrings: string[]): string[][] {
|
||||
const rows: string[][] = [];
|
||||
const xml = sheetXml.replace(/<(\/?)([a-zA-Z][\w.]*):/g, '<$1');
|
||||
for (const rowMatch of xml.matchAll(/<row\b[^>]*>([\s\S]*?)<\/row>/gs)) {
|
||||
const cells = new Map<number, string>();
|
||||
let maxColumn = -1;
|
||||
for (const cellMatch of rowMatch[1].matchAll(/<c\b([^>]*)\/?>([\s\S]*?)<\/c>/gs)) {
|
||||
const attrs = cellMatch[1];
|
||||
const refMatch = attrs.match(/\br="([A-Z]+)\d+"/);
|
||||
const column = refMatch ? columnIndex(refMatch[1]) : -1;
|
||||
const type = attrs.match(/\bt="([^"]+)"/)?.[1] ?? 'n';
|
||||
const body = cellMatch[2] ?? '';
|
||||
let value = '';
|
||||
if (type === 's') {
|
||||
const index = Number(body.match(/<v>([^<]*)<\/v>/)?.[1] ?? '');
|
||||
value = Number.isInteger(index) ? (sharedStrings[index] ?? '') : '';
|
||||
} else if (type === 'inlineStr') {
|
||||
const texts = [...body.matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/g)].map((part) =>
|
||||
unescapeXml(part[1]),
|
||||
);
|
||||
value = texts.join('');
|
||||
} else {
|
||||
value = unescapeXml(body.match(/<v>([\s\S]*?)<\/v>/)?.[1] ?? '');
|
||||
if (type === 'b') value = value === '1' ? 'true' : 'false';
|
||||
}
|
||||
if (column >= 0) {
|
||||
cells.set(column, value);
|
||||
maxColumn = Math.max(maxColumn, column);
|
||||
}
|
||||
}
|
||||
if (maxColumn < 0) continue;
|
||||
const values = Array.from({ length: maxColumn + 1 }, (_, index) => cells.get(index) ?? '');
|
||||
if (values.every((value) => value === '')) continue;
|
||||
rows.push(values);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
function columnIndex(letters: string): number {
|
||||
let index = 0;
|
||||
for (const char of letters.toUpperCase()) {
|
||||
index = index * 26 + (char.charCodeAt(0) - 64);
|
||||
}
|
||||
return index - 1;
|
||||
}
|
||||
|
||||
function unescapeXml(value: string): string {
|
||||
return value
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, '&')
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_all, hex: string) =>
|
||||
String.fromCodePoint(Number.parseInt(hex, 16)),
|
||||
)
|
||||
.replace(/&#(\d+);/g, (_all, dec: string) => String.fromCodePoint(Number(dec)));
|
||||
}
|
||||
104
apps/server/src/imports/imports.workbook.spec.ts
Normal file
104
apps/server/src/imports/imports.workbook.spec.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import * as ExcelJS from 'exceljs';
|
||||
import JSZip from 'jszip';
|
||||
import { parseDateValue } from './imports.helpers';
|
||||
import { parseSheets } from './imports.workbook';
|
||||
|
||||
async function xlsxBuffer(rows: Array<Array<unknown>>): Promise<Buffer> {
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
const worksheet = workbook.addWorksheet('名单');
|
||||
rows.forEach((row, index) => worksheet.addRow(row));
|
||||
return (await workbook.xlsx.writeBuffer()) as Buffer;
|
||||
}
|
||||
|
||||
async function wpsNamespaceXlsxBuffer(): Promise<Buffer> {
|
||||
const zip = new JSZip();
|
||||
zip.file('[Content_Types].xml', '<?xml version="1.0" encoding="UTF-8"?><Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"/>');
|
||||
zip.file(
|
||||
'xl/workbook.xml',
|
||||
`<x:workbook xmlns:x="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
|
||||
<x:sheets><x:sheet name="名单" sheetId="1" state="visible" r:id="rId1"/></x:sheets>
|
||||
</x:workbook>`,
|
||||
);
|
||||
zip.file(
|
||||
'xl/_rels/workbook.xml.rels',
|
||||
`<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/>
|
||||
</Relationships>`,
|
||||
);
|
||||
zip.file(
|
||||
'xl/sharedStrings.xml',
|
||||
`<x:sst xmlns:x="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><x:si><x:t>张三</x:t></x:si></x:sst>`,
|
||||
);
|
||||
zip.file(
|
||||
'xl/worksheets/sheet1.xml',
|
||||
`<x:worksheet xmlns:x="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<x:sheetData>
|
||||
<x:row r="1"><x:c r="A1" t="inlineStr"><x:is><x:t>姓名</x:t></x:is></x:c><x:c r="B1" t="inlineStr"><x:is><x:t>手机号</x:t></x:is></x:c></x:row>
|
||||
<x:row r="2"><x:c r="A2" t="s"><x:v>0</x:v></x:c><x:c r="B2"><x:v>13800138000</x:v></x:c></x:row>
|
||||
</x:sheetData>
|
||||
</x:worksheet>`,
|
||||
);
|
||||
return Buffer.from(await zip.generateAsync({ type: 'nodebuffer' }));
|
||||
}
|
||||
|
||||
describe('parseSheets', () => {
|
||||
it('从 headerRow 取表头,并保留空行后的物理行号', async () => {
|
||||
const buffer = await xlsxBuffer(
|
||||
[
|
||||
['标题行', null],
|
||||
['姓名', '学号'],
|
||||
['', ''],
|
||||
['张三', '2024001'],
|
||||
['李四', '2024002'],
|
||||
],
|
||||
);
|
||||
const sheets = await parseSheets(
|
||||
buffer,
|
||||
'students.xlsx',
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
2,
|
||||
);
|
||||
expect(sheets).toHaveLength(1);
|
||||
expect(sheets[0].headers).toEqual(['姓名', '学号']);
|
||||
expect(sheets[0].rows).toEqual([
|
||||
['张三', '2024001'],
|
||||
['李四', '2024002'],
|
||||
]);
|
||||
expect(sheets[0].rowNumbers).toEqual([4, 5]);
|
||||
});
|
||||
|
||||
it('解析 CSV 并默认使用第一行作为表头', async () => {
|
||||
const buffer = Buffer.from('姓名,学号\n张三,2024001\n', 'utf8');
|
||||
const sheets = await parseSheets(buffer, 'students.csv', 'text/csv');
|
||||
expect(sheets).toHaveLength(1);
|
||||
expect(sheets[0].headers).toEqual(['姓名', '学号']);
|
||||
expect(sheets[0].rows[0]).toEqual(['张三', 2024001]);
|
||||
});
|
||||
|
||||
it('ExcelJS 失败时回退到 WPS 命名空间解析', async () => {
|
||||
const buffer = await wpsNamespaceXlsxBuffer();
|
||||
const sheets = await parseSheets(
|
||||
buffer,
|
||||
'students.xlsx',
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
);
|
||||
expect(sheets).toHaveLength(1);
|
||||
expect(sheets[0].name).toBe('名单');
|
||||
expect(sheets[0].headers).toEqual(['姓名', '手机号']);
|
||||
expect(sheets[0].rows).toEqual([['张三', '13800138000']]);
|
||||
});
|
||||
|
||||
it('拒绝 .xls 文件', async () => {
|
||||
await expect(
|
||||
parseSheets(Buffer.from('not excel'), 'a.xls', 'application/vnd.ms-excel'),
|
||||
).rejects.toBeInstanceOf(BadRequestException);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseDateValue', () => {
|
||||
it('按进程本地时区格式化为 YYYY-MM-DD(CST 午夜不偏移到前一天)', () => {
|
||||
expect(parseDateValue(new Date(2026, 0, 1))).toBe('2026-01-01');
|
||||
expect(parseDateValue(new Date(2026, 11, 31))).toBe('2026-12-31');
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { BadRequestException } from '@nestjs/common';
|
||||
import * as ExcelJS from 'exceljs';
|
||||
import { Readable } from 'node:stream';
|
||||
import { cellValue, textValue } from './imports.helpers';
|
||||
import { fallbackSheetsToImportSheets, readXlsxSheetsFallback } from './imports.workbook-fallback';
|
||||
import type { CellValue } from './imports.types';
|
||||
|
||||
const MAX_SHEETS = 30;
|
||||
@@ -12,30 +13,38 @@ export interface ImportSheetData {
|
||||
name: string;
|
||||
headers: string[];
|
||||
rows: CellValue[][];
|
||||
/** 1-based header row used to build this view; defaults to 1. */
|
||||
headerRow?: number;
|
||||
/** Physical 1-based row numbers for each entry in `rows` (after headerRow). */
|
||||
rowNumbers?: number[];
|
||||
}
|
||||
|
||||
export function extractSheets(workbook: ExcelJS.Workbook): ImportSheetData[] {
|
||||
export function extractSheets(workbook: ExcelJS.Workbook, headerRow = 1): ImportSheetData[] {
|
||||
const sheets: ImportSheetData[] = [];
|
||||
for (const worksheet of workbook.worksheets) {
|
||||
if (sheets.length >= MAX_SHEETS) break;
|
||||
const headers: string[] = [];
|
||||
const rows: CellValue[][] = [];
|
||||
const firstRow = worksheet.getRow(1);
|
||||
const rowNumbers: number[] = [];
|
||||
const firstRow = worksheet.getRow(headerRow);
|
||||
for (let col = 1; col <= Math.min(firstRow.cellCount, MAX_COLS_PER_SHEET); col += 1) {
|
||||
const header = textValue(cellValue(firstRow.getCell(col)));
|
||||
headers.push(header);
|
||||
}
|
||||
if (!headers.some(Boolean)) continue;
|
||||
worksheet.eachRow({ includeEmpty: false }, (row, rowNumber) => {
|
||||
if (rowNumber === 1 || rows.length >= MAX_ROWS_PER_SHEET) return;
|
||||
if (rowNumber <= headerRow || rows.length >= MAX_ROWS_PER_SHEET) return;
|
||||
const values: CellValue[] = [];
|
||||
for (let col = 1; col <= headers.length; col += 1) {
|
||||
values.push(cellValue(row.getCell(col)));
|
||||
}
|
||||
if (values.every((v) => v === null || textValue(v) === '')) return;
|
||||
rows.push(values);
|
||||
rowNumbers.push(rowNumber);
|
||||
});
|
||||
if (rows.length > 0) sheets.push({ name: worksheet.name, headers, rows });
|
||||
if (rows.length > 0) {
|
||||
sheets.push({ name: worksheet.name, headers, rows, headerRow, rowNumbers });
|
||||
}
|
||||
}
|
||||
return sheets;
|
||||
}
|
||||
@@ -61,6 +70,7 @@ export async function parseSheets(
|
||||
buffer: Buffer,
|
||||
originalName: string,
|
||||
mimeType: string,
|
||||
headerRow = 1,
|
||||
): Promise<ImportSheetData[]> {
|
||||
const kind = detectWorkbookKind(originalName, mimeType);
|
||||
if (!kind) {
|
||||
@@ -76,13 +86,24 @@ export async function parseSheets(
|
||||
} else {
|
||||
await workbook.xlsx.load(buffer.buffer as ArrayBuffer);
|
||||
}
|
||||
const sheets = extractSheets(workbook);
|
||||
const sheets = extractSheets(workbook, headerRow);
|
||||
if (sheets.length === 0) {
|
||||
throw new BadRequestException('文件中没有可用的工作表数据');
|
||||
}
|
||||
return sheets;
|
||||
} catch (error) {
|
||||
if (error instanceof BadRequestException) throw error;
|
||||
if (kind === 'xlsx') {
|
||||
try {
|
||||
const fallbackSheets = fallbackSheetsToImportSheets(
|
||||
await readXlsxSheetsFallback(buffer),
|
||||
headerRow,
|
||||
);
|
||||
if (fallbackSheets.length > 0) return fallbackSheets;
|
||||
} catch {
|
||||
// fall through to the readable error below
|
||||
}
|
||||
}
|
||||
throw new BadRequestException('Excel 文件解析失败,请检查文件格式');
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user