Files
gongxue-base/apps/server/src/students/students.import.service.ts
wangziqi f50301148d fix(correctness): 并发/事务/实体/时区/状态一致性修复
由 OCR(open-codereview.ai,deepseek-v4-flash)审查驱动修复:
- wallets 原子扣款防 double-spend;refund 条件更新幂等;findTransactions 分页
- financial/imports/occupancies/attendance 事务与 advisory lock;重复生成/提交幂等
- 矛盾校验器、日期区间、实体双映射/DECIMAL/nullable、时区统一(china-time)
- rbac-seed 防重激活、exam 权限恢复、状态一致性、路由顺序、N+1/IN 分块等性能项

Reviewed-by: OCR (open-codereview.ai)
2026-08-09 21:29:54 +08:00

609 lines
24 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { EntityManager, In, Repository } from 'typeorm';
import { Student } from '../entities/student.entity';
import { StudentProfile } from '../entities/student-profile.entity';
import { StudentEnrollment } from '../entities/student-enrollment.entity';
import { ExamScore } from '../entities/exam-score.entity';
import { LearningRecord } from '../entities/learning-record.entity';
import { ResultArchive } from '../entities/result-archive.entity';
import { Organization } from '../entities/organization.entity';
import type {
ExamScoreImportRow,
LearningRecordImportRow,
StudentEnrollmentImportRow,
StudentImportRow,
StudentWorkbookImport,
} from './student-import';
import { getHostOrganizationId } from './students.organization';
import { addDaysToDateOnly } from '../common/china-time';
/** 学生查重/匹配的内存索引phone 与 idNumber 各一张 Mapkey 为去空白后的值)。 */
type StudentLookupIndex = {
byPhone: Map<string, Student>;
byIdNumber: Map<string, Student>;
};
@Injectable()
export class StudentsImportService {
// 档案/报读/成绩等写入统一走 manager.getRepository见下方 helpers
// 这里只注入 Student repo查重、更新、开启事务
constructor(
@InjectRepository(Student) private readonly repo: Repository<Student>,
) {}
async batchImport(importData: StudentWorkbookImport | StudentImportRow[]) {
const data = this.normalizeImportData(importData);
// 整批导入放进同一事务:查重、建学生、写档案要么全部成功,要么全部回滚。
return this.repo.manager.transaction(async (manager) => {
const studentRepo = manager.getRepository(Student);
const organizationRepo = manager.getRepository(Organization);
// 按手机号预建报读/成绩/回访行分组(手机号是导入行与学生档案之间的关联键,
// 等价于按学生分组),避免逐学生循环内对 data.* 全量 filter。
const enrollmentsByPhone = this.groupByPhone(data.enrollments);
const examScoresByPhone = this.groupByPhone(data.examScores);
const learningRecordsByPhone = this.groupByPhone(data.learningRecords);
// 循环外按本批所有 phone/idNumber 一次性批量预取已有学生(仅两条 IN 查询),
// 内存建索引后逐行匹配,避免逐行 findOne 的 N+1。
const existingIndexes = await this.prefetchStudentIndexes(studentRepo, data.students);
let imported = 0;
let skipped = 0;
let archiveImported = 0;
let invalidDateSkipped = 0;
// 本机构 id 循环外懒加载缓存一次,避免每个学生行都查询
let hostOrganizationId: number | undefined;
for (const row of data.students) {
if (!row.name || !row.name.trim()) {
skipped++;
continue;
}
// 按 phone/idNumber 查重:先 phone 后 idNumber命中即视为已存在
if (this.matchStudentByIndexes(existingIndexes, row)) {
skipped++;
continue;
}
if (!row.organizationId && hostOrganizationId === undefined) {
hostOrganizationId = await getHostOrganizationId(organizationRepo);
}
const student = await studentRepo.save(
studentRepo.create({
name: row.name.trim(),
studentNo: row.studentNo?.trim() || undefined,
phone: row.phone?.trim() || undefined,
idNumber: row.idNumber?.trim() || undefined,
gender: row.gender || undefined,
ethnicity: row.ethnicity || undefined,
emergencyContact: row.emergencyContact || undefined,
emergencyPhone: row.emergencyPhone || undefined,
supervisor: row.supervisor || undefined,
organizationId: row.organizationId || hostOrganizationId,
}),
);
// 新保存的学生同步进内存索引:本批后续相同 phone/idNumber 行视为重复跳过,
// 保持与逐行 findOne 一致的 upsert/去重语义。
this.indexStudent(existingIndexes, student);
const archive = await this.importArchiveData(manager, student.id, row, {
enrollments: enrollmentsByPhone,
examScores: examScoresByPhone,
learningRecords: learningRecordsByPhone,
});
archiveImported += archive.imported;
invalidDateSkipped += archive.invalidDateSkipped;
imported++;
}
return {
message: `成功导入 ${imported} 名学生,导入档案相关记录 ${archiveImported} 条,跳过 ${skipped} 条(重复或空行)`,
imported,
archiveImported,
skipped,
invalidDateSkipped,
};
});
}
async matchImport(importData: StudentWorkbookImport | StudentImportRow[]) {
const data = this.normalizeImportData(importData);
// 整批匹配写入放进同一事务:更新学生资料与写档案要么全部成功,要么全部回滚。
return this.repo.manager.transaction(async (manager) => {
const studentRepo = manager.getRepository(Student);
// 与 batchImport 一致:按手机号预建分组,避免逐学生循环内全量 filter。
const enrollmentsByPhone = this.groupByPhone(data.enrollments);
const examScoresByPhone = this.groupByPhone(data.examScores);
const learningRecordsByPhone = this.groupByPhone(data.learningRecords);
// 循环外按本批所有 phone/idNumber 一次性批量预取已有学生(仅两条 IN 查询),
// 内存建索引后逐行匹配,避免逐行 findOne 的 N+1。
const existingIndexes = await this.prefetchStudentIndexes(studentRepo, data.students);
let matched = 0;
let skipped = 0;
let skippedNoFields = 0;
let skippedConflict = 0;
let archiveImported = 0;
let invalidDateSkipped = 0;
for (const row of data.students) {
// 双键命中不同学生:跳过该行,避免把 idNumber 写到错误的 student 上
if (this.hasIndexConflict(existingIndexes, row)) {
skipped++;
skippedConflict++;
continue;
}
// Match by phone first, then idNumber
const student = this.matchStudentByIndexes(existingIndexes, row);
if (!student) {
skipped++;
continue;
}
const updates: Partial<
Pick<
Student,
| 'name'
| 'studentNo'
| 'phone'
| 'idNumber'
| 'gender'
| 'ethnicity'
| 'emergencyContact'
| 'emergencyPhone'
| 'supervisor'
| 'organizationId'
>
> = {};
if (row.name?.trim()) updates.name = row.name.trim();
if (row.studentNo?.trim()) updates.studentNo = row.studentNo.trim();
// phone/idNumber 是匹配键:与命中学生一致时视为无变更(避免空 set 更新),
// 不一致时仍允许通过另一个键命中后更新。
if (row.phone?.trim() && row.phone.trim() !== student.phone) updates.phone = row.phone.trim();
if (row.idNumber?.trim() && row.idNumber.trim() !== student.idNumber) updates.idNumber = row.idNumber.trim();
if (row.gender) updates.gender = row.gender;
if (row.ethnicity) updates.ethnicity = row.ethnicity;
if (row.emergencyContact) updates.emergencyContact = row.emergencyContact;
if (row.emergencyPhone) updates.emergencyPhone = row.emergencyPhone;
if (row.supervisor) updates.supervisor = row.supervisor;
if (row.organizationId) updates.organizationId = row.organizationId;
// 按 phone/idNumber 命中已有学生但本行没有任何可写字段时,
// 跳过该学生的更新(避免 TypeORM 对空 set 报错),保留匹配/去重语义。
const hasWritableFields = Object.keys(updates).length > 0;
const archive = await this.importArchiveData(manager, student.id, row, {
enrollments: enrollmentsByPhone,
examScores: examScoresByPhone,
learningRecords: learningRecordsByPhone,
});
archiveImported += archive.imported;
invalidDateSkipped += archive.invalidDateSkipped;
if (!hasWritableFields && archive.imported === 0 && archive.invalidDateSkipped === 0) {
// 整行无任何可写字段:计入 skipped不抛错
skippedNoFields++;
skipped++;
continue;
}
if (hasWritableFields) {
await studentRepo.update(student.id, updates);
// 更新后的 phone/idNumber 同步进内存索引:后续行按新值仍可命中,
// 与逐行 findOne 在同一事务内能看到本批已更新记录的语义一致。
this.reindexStudent(existingIndexes, student, updates);
}
matched++;
}
return {
message: `更新已有学生资料 ${matched} 人,导入档案相关记录 ${archiveImported} 条,跳过 ${skipped} 条(无匹配/无更新字段/双键冲突)`,
matched,
archiveImported,
skipped,
skippedNoFields,
skippedConflict,
invalidDateSkipped,
};
});
}
/**
* 批量预取本批导入行涉及的所有 phone / idNumber 对应的已有学生:
* 只发两条 IN 查询phone In(...)、idNumber In(...)),内存建索引供逐行匹配,
* 替代逐行 findOneN+1。某类 key 为空时跳过对应查询。
*/
private async prefetchStudentIndexes(
studentRepo: Repository<Student>,
rows: StudentImportRow[],
): Promise<StudentLookupIndex> {
const phones = [
...new Set(
rows
.map((row) => row.phone?.trim())
.filter((phone): phone is string => Boolean(phone)),
),
];
const idNumbers = [
...new Set(
rows
.map((row) => row.idNumber?.trim())
.filter((idNumber): idNumber is string => Boolean(idNumber)),
),
];
const [byPhoneList, byIdNumberList] = await Promise.all([
phones.length > 0
? studentRepo.find({ where: { phone: In(phones) } })
: Promise.resolve([] as Student[]),
idNumbers.length > 0
? studentRepo.find({ where: { idNumber: In(idNumbers) } })
: Promise.resolve([] as Student[]),
]);
return {
byPhone: this.buildIndex(byPhoneList, (student) => student.phone),
byIdNumber: this.buildIndex(byIdNumberList, (student) => student.idNumber),
};
}
/** 从查询结果构建 phone / idNumber → Student 的 Mapkey 去空白,空值不建索引)。 */
private buildIndex(
students: Student[],
keyOf: (student: Student) => string | null | undefined,
): Map<string, Student> {
const index = new Map<string, Student>();
for (const student of students) {
const key = keyOf(student)?.trim();
if (key) index.set(key, student);
}
return index;
}
/**
* 双键冲突检测:同一行 phone 与 idNumber 分别命中不同的学生时,不能更新任何一个,
* 否则会把一个学生的 idNumber 写到另一个学生身上(数据错乱)。冲突行按 skipped 处理。
*/
private hasIndexConflict(indexes: StudentLookupIndex, row: StudentImportRow): boolean {
const phone = row.phone?.trim();
const idNumber = row.idNumber?.trim();
if (!phone || !idNumber) return false;
const byPhone = indexes.byPhone.get(phone);
const byIdNumber = indexes.byIdNumber.get(idNumber);
return !!byPhone && !!byIdNumber && byPhone.id !== byIdNumber.id;
}
/** 逐行匹配已有学生:优先 phone未命中再按 idNumber与 matchImport 原有语义一致)。 */
private matchStudentByIndexes(
indexes: StudentLookupIndex,
row: StudentImportRow,
): Student | undefined {
const phone = row.phone?.trim();
if (phone) {
const byPhone = indexes.byPhone.get(phone);
if (byPhone) return byPhone;
}
const idNumber = row.idNumber?.trim();
if (idNumber) {
const byIdNumber = indexes.byIdNumber.get(idNumber);
if (byIdNumber) return byIdNumber;
}
return undefined;
}
/** 新保存的学生同步进内存索引batchImport 批次内去重)。 */
private indexStudent(indexes: StudentLookupIndex, student: Student): void {
if (student.phone) indexes.byPhone.set(student.phone.trim(), student);
if (student.idNumber) indexes.byIdNumber.set(student.idNumber.trim(), student);
}
/**
* matchImport 更新学生资料后同步内存索引phone/idNumber 被改写时删除旧 key、
* 注册新 key并让内存实体与已更新值保持一致后续行按新值仍可命中
*/
private reindexStudent(
indexes: StudentLookupIndex,
student: Student,
updates: Partial<Pick<Student, 'phone' | 'idNumber'>>,
): void {
const oldPhone = student.phone?.trim();
const newPhone = updates.phone?.trim();
const oldIdNumber = student.idNumber?.trim();
const newIdNumber = updates.idNumber?.trim();
Object.assign(student, updates);
if (newPhone && newPhone !== oldPhone) {
if (oldPhone) indexes.byPhone.delete(oldPhone);
indexes.byPhone.set(newPhone, student);
}
if (newIdNumber && newIdNumber !== oldIdNumber) {
if (oldIdNumber) indexes.byIdNumber.delete(oldIdNumber);
indexes.byIdNumber.set(newIdNumber, student);
}
}
private normalizeImportData(
importData: StudentWorkbookImport | StudentImportRow[],
): StudentWorkbookImport {
if (Array.isArray(importData)) {
return { students: importData, enrollments: [], examScores: [], learningRecords: [] };
}
return importData;
}
private normalizePhone(phone?: string) {
return phone?.trim() || '';
}
private sameValue(left?: string | number | null, right?: string | number | null) {
return String(left ?? '').trim() === String(right ?? '').trim();
}
/** 按手机号预建行分组(手机号去空白后作为 key空手机号的行不参与匹配。 */
private groupByPhone<T extends { phone?: string }>(rows: T[]): Map<string, T[]> {
const byPhone = new Map<string, T[]>();
for (const row of rows) {
const phone = this.normalizePhone(row.phone);
if (!phone) continue;
const list = byPhone.get(phone);
if (list) list.push(row);
else byPhone.set(phone, [row]);
}
return byPhone;
}
/**
* 日期字段写入前的真实日历校验:先检查 YYYY-MM-DD 形状,
* 再用 common/china-time 的 addDaysToDateOnly 往返判断月/日是否真实存在
* Date.UTC 会把溢出日期归一化,如 2024-02-31 → 2024-03-02往返不一致即非法
*/
private isValidDateOnly(value: string): boolean {
const dateOnly = value.trim();
if (!/^\d{4}-\d{2}-\d{2}$/.test(dateOnly)) return false;
return addDaysToDateOnly(dateOnly, 0) === dateOnly;
}
private hasProfileData(row: StudentImportRow) {
return [
row.targetCollege,
row.targetMajor,
row.collegeSchool,
row.collegeMajor,
row.subjectDirection,
row.grade,
row.profileDate,
row.notes,
].some((value) => value !== undefined && String(value).trim() !== '');
}
private hasResultData(row: StudentImportRow) {
return [
row.cultureFinalScore,
row.professionalFinalScore,
row.admissionStatus,
row.admittedCollege,
row.admittedMajor,
].some((value) => value !== undefined && String(value).trim() !== '');
}
private async importArchiveData(
manager: EntityManager,
studentId: number,
row: StudentImportRow,
data: {
enrollments: Map<string, StudentEnrollmentImportRow[]>;
examScores: Map<string, ExamScoreImportRow[]>;
learningRecords: Map<string, LearningRecordImportRow[]>;
},
): Promise<{ imported: number; invalidDateSkipped: number }> {
const phone = this.normalizePhone(row.phone);
let imported = 0;
let invalidDateSkipped = 0;
if (this.hasProfileData(row)) {
// 非法日期不写入,跳过该字段并计数
if (row.profileDate?.trim() && !this.isValidDateOnly(row.profileDate)) {
invalidDateSkipped++;
}
await this.upsertProfileFromImport(manager, studentId, row);
imported++;
}
if (this.hasResultData(row)) {
await this.upsertResultFromImport(manager, studentId, row);
imported++;
}
if (!phone) return { imported, invalidDateSkipped };
const enrollmentRows = data.enrollments.get(phone) ?? [];
const examRows = data.examScores.get(phone) ?? [];
const learningRows = data.learningRecords.get(phone) ?? [];
// 按 studentId 批量预取既有档案行,内存匹配(保持 upsert 语义),
// 避免每个档案行单独 find 查询;新建保存后同步加入内存列表供后续行匹配。
let existingEnrollments: StudentEnrollment[] = [];
let existingExamScores: ExamScore[] = [];
let existingLearningRecords: LearningRecord[] = [];
if (enrollmentRows.length > 0) {
existingEnrollments = await manager.getRepository(StudentEnrollment).find({
where: { studentId },
});
}
if (examRows.length > 0) {
existingExamScores = await manager.getRepository(ExamScore).find({
where: { studentId },
});
}
if (learningRows.length > 0) {
existingLearningRecords = await manager.getRepository(LearningRecord).find({
where: { studentId },
});
}
const enrollmentByClassName = new Map<string, StudentEnrollment>();
for (const enrollmentRow of enrollmentRows) {
const enrollment = await this.upsertEnrollmentFromImport(
manager,
studentId,
enrollmentRow,
existingEnrollments,
);
if (!enrollment) continue;
if (enrollment.className) enrollmentByClassName.set(enrollment.className, enrollment);
imported++;
}
for (const examRow of examRows) {
// 非法日期不写入,跳过该字段并计数
if (examRow.examDate?.trim() && !this.isValidDateOnly(examRow.examDate)) {
invalidDateSkipped++;
}
if (
await this.upsertExamScoreFromImport(
manager,
studentId,
examRow,
enrollmentByClassName,
existingExamScores,
)
) {
imported++;
}
}
for (const learningRow of learningRows) {
// recordDate 必填:非法日期视为缺省,跳过整条并计数
if (learningRow.recordDate?.trim() && !this.isValidDateOnly(learningRow.recordDate)) {
invalidDateSkipped++;
continue;
}
if (await this.upsertLearningRecordFromImport(manager, studentId, learningRow, existingLearningRecords)) {
imported++;
}
}
return { imported, invalidDateSkipped };
}
private async upsertProfileFromImport(
manager: EntityManager,
studentId: number,
row: StudentImportRow,
) {
const profileRepo = manager.getRepository(StudentProfile);
const entity =
(await profileRepo.findOne({ where: { studentId } })) ||
profileRepo.create({ studentId });
if (row.targetCollege?.trim()) entity.targetCollege = row.targetCollege.trim();
if (row.targetMajor?.trim()) entity.targetMajor = row.targetMajor.trim();
if (row.collegeSchool?.trim()) entity.collegeSchool = row.collegeSchool.trim();
if (row.collegeMajor?.trim()) entity.collegeMajor = row.collegeMajor.trim();
if (row.subjectDirection?.trim()) entity.subjectDirection = row.subjectDirection.trim();
if (row.grade?.trim()) entity.grade = row.grade.trim();
if (row.profileDate?.trim() && this.isValidDateOnly(row.profileDate)) {
entity.profileDate = row.profileDate.trim();
}
if (row.notes?.trim()) entity.notes = row.notes.trim();
await profileRepo.save(entity);
}
private async upsertResultFromImport(
manager: EntityManager,
studentId: number,
row: StudentImportRow,
) {
const resultRepo = manager.getRepository(ResultArchive);
const entity =
(await resultRepo.findOne({ where: { studentId } })) ||
resultRepo.create({ studentId });
if (row.cultureFinalScore !== undefined) entity.cultureFinalScore = row.cultureFinalScore;
if (row.professionalFinalScore !== undefined)
entity.professionalFinalScore = row.professionalFinalScore;
if (row.admissionStatus?.trim()) entity.admissionStatus = row.admissionStatus.trim();
if (row.admittedCollege?.trim()) entity.admittedCollege = row.admittedCollege.trim();
if (row.admittedMajor?.trim()) entity.admittedMajor = row.admittedMajor.trim();
await resultRepo.save(entity);
}
private async upsertEnrollmentFromImport(
manager: EntityManager,
studentId: number,
row: StudentEnrollmentImportRow,
existing: StudentEnrollment[],
) {
if (!row.courseCategory?.trim() || !row.classType?.trim()) {
return null;
}
const enrollmentRepo = manager.getRepository(StudentEnrollment);
let entity = existing.find(
(item) =>
this.sameValue(item.courseCategory, row.courseCategory) &&
this.sameValue(item.classType, row.classType) &&
this.sameValue(item.className, row.className) &&
this.sameValue(item.startDate, row.startDate),
);
if (!entity) {
entity = enrollmentRepo.create({ studentId });
// 保持 upsert 语义:后续相同行可在内存列表中命中该新建实体
existing.push(entity);
}
entity.courseCategory = row.courseCategory.trim();
entity.classType = row.classType.trim();
if (row.className?.trim()) entity.className = row.className.trim();
if (row.headTeacher?.trim()) entity.headTeacher = row.headTeacher.trim();
if (row.subjectTeacher?.trim()) entity.subjectTeacher = row.subjectTeacher.trim();
if (row.startDate?.trim()) entity.startDate = row.startDate.trim();
if (row.endDate?.trim()) entity.endDate = row.endDate.trim();
if (row.status?.trim()) entity.status = row.status.trim();
else if (!entity.status) entity.status = 'active';
return enrollmentRepo.save(entity);
}
private async upsertExamScoreFromImport(
manager: EntityManager,
studentId: number,
row: ExamScoreImportRow,
enrollmentByClassName: Map<string, StudentEnrollment>,
existing: ExamScore[],
) {
if (!row.examType?.trim() || !row.subject?.trim() || row.score === undefined) return false;
const examScoreRepo = manager.getRepository(ExamScore);
let entity = existing.find(
(item) =>
this.sameValue(item.examType, row.examType) &&
this.sameValue(item.examName, row.examName) &&
this.sameValue(item.subject, row.subject) &&
this.sameValue(item.examDate, row.examDate),
);
if (!entity) {
entity = examScoreRepo.create({ studentId });
// 保持 upsert 语义:后续相同行可在内存列表中命中该新建实体
existing.push(entity);
}
entity.examType = row.examType.trim();
entity.subject = row.subject.trim();
entity.score = row.score;
if (row.examName?.trim()) entity.examName = row.examName.trim();
if (row.classAvg !== undefined) entity.classAvg = row.classAvg;
if (row.rank !== undefined) entity.rank = row.rank;
if (row.examDate?.trim() && this.isValidDateOnly(row.examDate)) {
entity.examDate = row.examDate.trim();
}
if (row.enrollmentName?.trim()) {
const enrollment = enrollmentByClassName.get(row.enrollmentName.trim());
if (enrollment) entity.enrollmentId = enrollment.id;
}
if (!entity.status) entity.status = 'active';
await examScoreRepo.save(entity);
return true;
}
private async upsertLearningRecordFromImport(
manager: EntityManager,
studentId: number,
row: LearningRecordImportRow,
existing: LearningRecord[],
) {
if (!row.recordDate?.trim() || !row.recordType?.trim() || !row.content?.trim()) return false;
const learningRecordRepo = manager.getRepository(LearningRecord);
let entity = existing.find(
(item) =>
this.sameValue(item.recordDate, row.recordDate) &&
this.sameValue(item.recordType, row.recordType) &&
this.sameValue(item.content, row.content),
);
if (!entity) {
entity = learningRecordRepo.create({ studentId });
// 保持 upsert 语义:后续相同行可在内存列表中命中该新建实体
existing.push(entity);
}
entity.recordDate = row.recordDate.trim();
entity.recordType = row.recordType.trim();
entity.content = row.content.trim();
if (row.followUpMethod?.trim()) entity.followUpMethod = row.followUpMethod.trim();
if (row.nextStep?.trim()) entity.nextStep = row.nextStep.trim();
if (!entity.status) entity.status = 'active';
await learningRecordRepo.save(entity);
return true;
}
}