From 229f930538155d2b42f0a5793cb1c32b7baa8305 Mon Sep 17 00:00:00 2001 From: wangziqi Date: Sat, 1 Aug 2026 15:18:17 +0800 Subject: [PATCH] feat(education): add secure content export jobs --- .../exportjob/ContentExportJobController.java | 45 +++++ .../vo/ContentExportCreateReqVO.java | 15 ++ .../exportjob/vo/ContentExportJobRespVO.java | 21 +++ .../exportjob/vo/ContentExportPageReqVO.java | 5 + .../exportjob/ContentExportJobDO.java | 43 +++++ .../exportjob/ContentExportJobMapper.java | 107 +++++++++++ .../education/enums/ErrorCodeConstants.java | 7 + .../job/ContentExportProcessJob.java | 17 ++ .../exportjob/ContentExportJobProjection.java | 9 + .../exportjob/ContentExportJobService.java | 11 ++ .../ContentExportJobServiceImpl.java | 167 ++++++++++++++++++ .../ContentExportProjectionMapper.java | 32 ++++ .../exportjob/ContentExportSanitizer.java | 41 +++++ .../exportjob/CreateContentExportCommand.java | 4 + .../V4469__add_secure_content_exports.sql | 66 +++++++ .../exportjob/ContentExportSanitizerTest.java | 18 ++ 16 files changed, 608 insertions(+) create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/ContentExportJobController.java create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/vo/ContentExportCreateReqVO.java create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/vo/ContentExportJobRespVO.java create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/vo/ContentExportPageReqVO.java create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/dal/dataobject/exportjob/ContentExportJobDO.java create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/dal/mysql/exportjob/ContentExportJobMapper.java create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/job/ContentExportProcessJob.java create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportJobProjection.java create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportJobService.java create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportJobServiceImpl.java create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportProjectionMapper.java create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportSanitizer.java create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/CreateContentExportCommand.java create mode 100644 yudao-module-education/src/main/resources/db/migration/education/V4469__add_secure_content_exports.sql create mode 100644 yudao-module-education/src/test/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportSanitizerTest.java diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/ContentExportJobController.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/ContentExportJobController.java new file mode 100644 index 00000000..0d3833e4 --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/ContentExportJobController.java @@ -0,0 +1,45 @@ +package cn.iocoder.yudao.module.education.controller.admin.exportjob; + +import cn.iocoder.yudao.framework.common.pojo.CommonResult; +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.framework.security.core.service.SecurityFrameworkService; +import cn.iocoder.yudao.module.education.controller.admin.exportjob.vo.*; +import cn.iocoder.yudao.module.education.service.exportjob.ContentExportJobService; +import cn.iocoder.yudao.module.education.service.exportjob.CreateContentExportCommand; +import jakarta.validation.Valid; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success; +import static cn.iocoder.yudao.framework.security.core.util.SecurityFrameworkUtils.getLoginUserId; + +@RestController +@RequestMapping("/education/content-export-jobs") +@Validated +@ConditionalOnProperty(prefix="yudao.education", name="enabled", havingValue="true") +public class ContentExportJobController { + private final ContentExportJobService service; + private final SecurityFrameworkService security; + public ContentExportJobController(ContentExportJobService service, SecurityFrameworkService security) { + this.service=service; this.security=security; + } + @PostMapping @PreAuthorize("@ss.hasPermission('education:content-export:create')") + public CommonResult create(@Valid @RequestBody ContentExportCreateReqVO req) { + return success(ContentExportJobRespVO.from(service.create(new CreateContentExportCommand(req.getCommandKey(), + req.getScope(), req.getCollectionId(), req.getFormat(), req.getAnswerMode()), getLoginUserId(), + security.hasPermission("education:content-export:answers")))); + } + @GetMapping @PreAuthorize("@ss.hasPermission('education:content-export:query')") + public CommonResult> page(@Valid ContentExportPageReqVO req) { + var page=service.page(getLoginUserId(),req.getPageNo(),req.getPageSize()); + return success(new PageResult<>(page.getList().stream().map(ContentExportJobRespVO::from).toList(),page.getTotal())); + } + @GetMapping("/{id}") @PreAuthorize("@ss.hasPermission('education:content-export:query')") + public CommonResult get(@PathVariable Long id) { + return success(ContentExportJobRespVO.from(service.get(id,getLoginUserId()))); + } + @GetMapping("/{id}/download-url") @PreAuthorize("@ss.hasPermission('education:content-export:download')") + public CommonResult downloadUrl(@PathVariable Long id) { return success(service.downloadUrl(id,getLoginUserId())); } +} diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/vo/ContentExportCreateReqVO.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/vo/ContentExportCreateReqVO.java new file mode 100644 index 00000000..446f5f6b --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/vo/ContentExportCreateReqVO.java @@ -0,0 +1,15 @@ +package cn.iocoder.yudao.module.education.controller.admin.exportjob.vo; + +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Pattern; +import lombok.Data; + +@Data +public class ContentExportCreateReqVO { + @NotBlank @Pattern(regexp = "[A-Za-z0-9._:-]{1,128}") private String commandKey; + @NotBlank private String scope; + @NotNull private Long collectionId; + @NotBlank private String format; + @NotBlank private String answerMode; +} diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/vo/ContentExportJobRespVO.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/vo/ContentExportJobRespVO.java new file mode 100644 index 00000000..f0b4ebf2 --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/vo/ContentExportJobRespVO.java @@ -0,0 +1,21 @@ +package cn.iocoder.yudao.module.education.controller.admin.exportjob.vo; + +import cn.iocoder.yudao.module.education.service.exportjob.ContentExportJobProjection; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +public class ContentExportJobRespVO { + private Long id; private String scope; private Long collectionId; private String format; private String answerMode; + private String status; private Integer attemptCount; private Integer maxAttempts; private Integer questionCount; + private String fileName; private Long fileSizeBytes; private String checksumSha256; private String failureCode; + private LocalDateTime createTime; private LocalDateTime finishedAt; + public static ContentExportJobRespVO from(ContentExportJobProjection p) { + ContentExportJobRespVO v = new ContentExportJobRespVO(); v.id=p.id(); v.scope=p.scope(); v.collectionId=p.collectionId(); + v.format=p.format(); v.answerMode=p.answerMode(); v.status=p.status(); v.attemptCount=p.attemptCount(); + v.maxAttempts=p.maxAttempts(); v.questionCount=p.questionCount(); v.fileName=p.fileName(); + v.fileSizeBytes=p.fileSizeBytes(); v.checksumSha256=p.checksumSha256(); v.failureCode=p.failureCode(); + v.createTime=p.createTime(); v.finishedAt=p.finishedAt(); return v; + } +} diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/vo/ContentExportPageReqVO.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/vo/ContentExportPageReqVO.java new file mode 100644 index 00000000..a53d0cef --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/controller/admin/exportjob/vo/ContentExportPageReqVO.java @@ -0,0 +1,5 @@ +package cn.iocoder.yudao.module.education.controller.admin.exportjob.vo; + +import cn.iocoder.yudao.framework.common.pojo.PageParam; + +public class ContentExportPageReqVO extends PageParam {} diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/dal/dataobject/exportjob/ContentExportJobDO.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/dal/dataobject/exportjob/ContentExportJobDO.java new file mode 100644 index 00000000..cbf5b14e --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/dal/dataobject/exportjob/ContentExportJobDO.java @@ -0,0 +1,43 @@ +package cn.iocoder.yudao.module.education.dal.dataobject.exportjob; + +import cn.iocoder.yudao.framework.tenant.core.db.TenantBaseDO; +import com.baomidou.mybatisplus.annotation.KeySequence; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.time.LocalDateTime; + +@TableName("education_content_export_job") +@KeySequence("education_content_export_job_seq") +@Data +@EqualsAndHashCode(callSuper = true) +public class ContentExportJobDO extends TenantBaseDO { + @TableId private Long id; + private Long actorId; + private String commandKey; + private String requestHash; + private String scope; + private Long collectionId; + private String format; + private String answerMode; + private String status; + private Integer attemptCount; + private Integer maxAttempts; + private LocalDateTime nextAttemptAt; + private String lockedBy; + private String leaseToken; + private LocalDateTime leaseExpiresAt; + private LocalDateTime lastHeartbeatAt; + private LocalDateTime startedAt; + private LocalDateTime finishedAt; + private String fileReference; + private String fileName; + private String contentType; + private Long fileSizeBytes; + private String checksumSha256; + private Integer questionCount; + private String failureCode; + private String failureMessage; +} diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/dal/mysql/exportjob/ContentExportJobMapper.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/dal/mysql/exportjob/ContentExportJobMapper.java new file mode 100644 index 00000000..0bca00c1 --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/dal/mysql/exportjob/ContentExportJobMapper.java @@ -0,0 +1,107 @@ +package cn.iocoder.yudao.module.education.dal.mysql.exportjob; + +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; +import cn.iocoder.yudao.module.education.dal.dataobject.exportjob.ContentExportJobDO; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.ibatis.annotations.*; + +import java.util.List; + +@Mapper +public interface ContentExportJobMapper extends BaseMapperX { + + default ContentExportJobDO selectTenantActorJob(Long tenantId, Long actorId, Long id) { + return selectOne(new LambdaQueryWrapperX().eq(ContentExportJobDO::getTenantId, tenantId) + .eq(ContentExportJobDO::getActorId, actorId).eq(ContentExportJobDO::getId, id)); + } + + default ContentExportJobDO selectByCommandKey(Long tenantId, Long actorId, String commandKey) { + return selectOne(new LambdaQueryWrapperX().eq(ContentExportJobDO::getTenantId, tenantId) + .eq(ContentExportJobDO::getActorId, actorId).eq(ContentExportJobDO::getCommandKey, commandKey)); + } + + default PageResult selectTenantActorPage(Long tenantId, Long actorId, int pageNo, int pageSize) { + Page page = selectPage(new Page<>(pageNo, pageSize), + new LambdaQueryWrapperX().eq(ContentExportJobDO::getTenantId, tenantId) + .eq(ContentExportJobDO::getActorId, actorId).orderByDesc(ContentExportJobDO::getId)); + return new PageResult<>(page.getRecords(), page.getTotal()); + } + + @Insert(""" + INSERT INTO education_content_export_job + (tenant_id, actor_id, command_key, request_hash, scope, collection_id, format, answer_mode, + status, attempt_count, max_attempts, next_attempt_at, creator, create_time, updater, update_time, deleted) + VALUES (#{tenantId}, #{actorId}, #{commandKey}, #{requestHash}, #{scope}, #{collectionId}, #{format}, + #{answerMode}, 'PENDING', 0, #{maxAttempts}, CURRENT_TIMESTAMP, '', CURRENT_TIMESTAMP, '', CURRENT_TIMESTAMP, FALSE) + ON CONFLICT (tenant_id, actor_id, command_key) DO NOTHING + """) + @Options(useGeneratedKeys = true, keyProperty = "id") + int insertIgnore(ContentExportJobDO job); + + @Select(""" + WITH candidate AS ( + SELECT id FROM education_content_export_job + WHERE deleted=false AND status IN ('PENDING','RENDERING') AND attempt_count < max_attempts + AND (next_attempt_at IS NULL OR next_attempt_at <= CURRENT_TIMESTAMP) + AND (lease_expires_at IS NULL OR lease_expires_at <= CURRENT_TIMESTAMP) + ORDER BY COALESCE(next_attempt_at, create_time), id + LIMIT #{limit} FOR UPDATE SKIP LOCKED + ) + UPDATE education_content_export_job job + SET status='RENDERING', locked_by=#{workerId}, lease_token=#{leaseToken}, + lease_expires_at=CURRENT_TIMESTAMP + CAST(#{leaseSeconds} || ' seconds' AS INTERVAL), + last_heartbeat_at=CURRENT_TIMESTAMP, attempt_count=job.attempt_count + 1, next_attempt_at=NULL, + started_at=COALESCE(job.started_at,CURRENT_TIMESTAMP), failure_code=NULL, failure_message=NULL, + update_time=CURRENT_TIMESTAMP + FROM candidate WHERE job.id=candidate.id RETURNING job.* + """) + List claimReady(@Param("workerId") String workerId, @Param("leaseToken") String leaseToken, + @Param("leaseSeconds") long leaseSeconds, @Param("limit") int limit); + + @Update(""" + UPDATE education_content_export_job SET + lease_expires_at=CURRENT_TIMESTAMP + CAST(#{leaseSeconds} || ' seconds' AS INTERVAL), + last_heartbeat_at=CURRENT_TIMESTAMP, update_time=CURRENT_TIMESTAMP + WHERE tenant_id=#{tenantId} AND id=#{id} AND status='RENDERING' AND lease_token=#{leaseToken} + AND lease_expires_at > CURRENT_TIMESTAMP + """) + int heartbeat(@Param("tenantId") Long tenantId, @Param("id") Long id, + @Param("leaseToken") String leaseToken, @Param("leaseSeconds") long leaseSeconds); + + @Update(""" + UPDATE education_content_export_job SET status='COMPLETED', file_reference=#{fileReference}, + file_name=#{fileName}, content_type=#{contentType}, file_size_bytes=#{fileSizeBytes}, + checksum_sha256=#{checksumSha256}, question_count=#{questionCount}, finished_at=CURRENT_TIMESTAMP, + locked_by=NULL, lease_token=NULL, lease_expires_at=NULL, last_heartbeat_at=NULL, update_time=CURRENT_TIMESTAMP + WHERE tenant_id=#{tenantId} AND id=#{id} AND status='RENDERING' AND lease_token=#{leaseToken} + AND lease_expires_at > CURRENT_TIMESTAMP + """) + int finishCompleted(@Param("tenantId") Long tenantId, @Param("id") Long id, @Param("leaseToken") String leaseToken, + @Param("fileReference") String fileReference, @Param("fileName") String fileName, + @Param("contentType") String contentType, @Param("fileSizeBytes") long fileSizeBytes, + @Param("checksumSha256") String checksumSha256, @Param("questionCount") int questionCount); + + @Update(""" + UPDATE education_content_export_job SET status=CASE WHEN attempt_count >= max_attempts THEN 'FAILED' ELSE 'PENDING' END, + next_attempt_at=CASE WHEN attempt_count >= max_attempts THEN NULL + ELSE CURRENT_TIMESTAMP + CAST(LEAST(300, POWER(2, attempt_count)::INTEGER * 5) || ' seconds' AS INTERVAL) END, + failure_code=#{failureCode}, failure_message=#{failureMessage}, + finished_at=CASE WHEN attempt_count >= max_attempts THEN CURRENT_TIMESTAMP ELSE NULL END, + locked_by=NULL, lease_token=NULL, lease_expires_at=NULL, last_heartbeat_at=NULL, update_time=CURRENT_TIMESTAMP + WHERE tenant_id=#{tenantId} AND id=#{id} AND status='RENDERING' AND lease_token=#{leaseToken} + AND lease_expires_at > CURRENT_TIMESTAMP + """) + int finishFailure(@Param("tenantId") Long tenantId, @Param("id") Long id, @Param("leaseToken") String leaseToken, + @Param("failureCode") String failureCode, @Param("failureMessage") String failureMessage); + + @Update(""" + UPDATE education_content_export_job SET status='FAILED', failure_code='ATTEMPTS_EXHAUSTED', + failure_message='Maximum export attempts exhausted', finished_at=CURRENT_TIMESTAMP, + locked_by=NULL, lease_token=NULL, lease_expires_at=NULL, last_heartbeat_at=NULL, update_time=CURRENT_TIMESTAMP + WHERE deleted=false AND status IN ('PENDING','RENDERING') AND attempt_count >= max_attempts + AND (lease_expires_at IS NULL OR lease_expires_at <= CURRENT_TIMESTAMP) + """) + int failExhausted(); +} diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/enums/ErrorCodeConstants.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/enums/ErrorCodeConstants.java index 9f0dae33..e973e575 100644 --- a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/enums/ErrorCodeConstants.java +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/enums/ErrorCodeConstants.java @@ -218,4 +218,11 @@ public interface ErrorCodeConstants { ErrorCode EDUCATION_IMPORT_ASSET_SIZE_INVALID = new ErrorCode(1_005_004_071, "导入文件为空或超过大小限制"); ErrorCode EDUCATION_IMPORT_ASSET_TYPE_INVALID = new ErrorCode(1_005_004_072, "仅支持 CSV 或 XLSX 导入文件"); ErrorCode EDUCATION_IMPORT_ASSET_CONTENT_INVALID = new ErrorCode(1_005_004_073, "导入文件内容与声明类型不匹配"); + + // ========== 内容导出任务 1-005-004-080 ~ 1-005-004-089 ========== + ErrorCode CONTENT_EXPORT_NOT_FOUND = new ErrorCode(1_005_004_082, "内容导出任务不存在或无权访问"); + ErrorCode CONTENT_EXPORT_COMMAND_CONFLICT = new ErrorCode(1_005_004_083, "导出命令幂等键已用于不同请求"); + ErrorCode CONTENT_EXPORT_INVALID_REQUEST = new ErrorCode(1_005_004_084, "内容导出请求无效"); + ErrorCode CONTENT_EXPORT_NOT_READY = new ErrorCode(1_005_004_085, "内容导出任务尚未完成"); + ErrorCode CONTENT_EXPORT_LEASE_LOST = new ErrorCode(1_005_004_086, "内容导出任务租约已失效"); } diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/job/ContentExportProcessJob.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/job/ContentExportProcessJob.java new file mode 100644 index 00000000..4b55c955 --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/job/ContentExportProcessJob.java @@ -0,0 +1,17 @@ +package cn.iocoder.yudao.module.education.job; + +import cn.iocoder.yudao.framework.quartz.core.handler.JobHandler; +import cn.iocoder.yudao.framework.tenant.core.aop.TenantIgnore; +import cn.iocoder.yudao.module.education.service.exportjob.ContentExportJobService; +import org.springframework.stereotype.Component; + +@Component +public class ContentExportProcessJob implements JobHandler { + private final ContentExportJobService service; + public ContentExportProcessJob(ContentExportJobService service) { this.service=service; } + @Override @TenantIgnore + public String execute(String param) { + int limit=param==null||param.isBlank()?10:Integer.parseInt(param); + return "processed="+service.processReady(limit); + } +} diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportJobProjection.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportJobProjection.java new file mode 100644 index 00000000..02882364 --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportJobProjection.java @@ -0,0 +1,9 @@ +package cn.iocoder.yudao.module.education.service.exportjob; + +import java.time.LocalDateTime; + +public record ContentExportJobProjection(Long id, String scope, Long collectionId, String format, String answerMode, + String status, Integer attemptCount, Integer maxAttempts, + Integer questionCount, String fileName, Long fileSizeBytes, + String checksumSha256, String failureCode, LocalDateTime createTime, + LocalDateTime finishedAt) {} diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportJobService.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportJobService.java new file mode 100644 index 00000000..54a44748 --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportJobService.java @@ -0,0 +1,11 @@ +package cn.iocoder.yudao.module.education.service.exportjob; + +import cn.iocoder.yudao.framework.common.pojo.PageResult; + +public interface ContentExportJobService { + ContentExportJobProjection create(CreateContentExportCommand command, Long actorId, boolean canIncludeAnswers); + PageResult page(Long actorId, int pageNo, int pageSize); + ContentExportJobProjection get(Long id, Long actorId); + String downloadUrl(Long id, Long actorId); + int processReady(int limit); +} diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportJobServiceImpl.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportJobServiceImpl.java new file mode 100644 index 00000000..c2ef506d --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportJobServiceImpl.java @@ -0,0 +1,167 @@ +package cn.iocoder.yudao.module.education.service.exportjob; + +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.framework.common.util.json.JsonUtils; +import cn.iocoder.yudao.framework.tenant.core.context.TenantContextHolder; +import cn.iocoder.yudao.framework.tenant.core.util.TenantUtils; +import cn.iocoder.yudao.module.education.dal.dataobject.catalog.QuestionDO; +import cn.iocoder.yudao.module.education.dal.dataobject.exportjob.ContentExportJobDO; +import cn.iocoder.yudao.module.education.dal.mysql.exportjob.ContentExportJobMapper; +import cn.iocoder.yudao.module.infra.api.file.FileApi; +import cn.iocoder.yudao.module.infra.api.file.FileContent; +import cn.iocoder.yudao.module.infra.api.file.FileDescriptor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import tools.jackson.databind.JsonNode; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.LocalDateTime; +import java.util.*; + +import static cn.iocoder.yudao.framework.common.exception.enums.GlobalErrorCodeConstants.FORBIDDEN; +import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; +import static cn.iocoder.yudao.module.education.enums.ErrorCodeConstants.*; + +@Service +public class ContentExportJobServiceImpl implements ContentExportJobService { + static final int MAX_QUESTIONS = 5000; + static final int MAX_BYTES = 10 * 1024 * 1024; + private static final int MAX_ATTEMPTS = 5; + private static final long LEASE_SECONDS = 120; + private final ContentExportJobMapper jobMapper; + private final ContentExportProjectionMapper projectionMapper; + private final FileApi fileApi; + private final ContentExportSanitizer sanitizer = new ContentExportSanitizer(); + + public ContentExportJobServiceImpl(ContentExportJobMapper jobMapper, ContentExportProjectionMapper projectionMapper, + FileApi fileApi) { + this.jobMapper = jobMapper; this.projectionMapper = projectionMapper; this.fileApi = fileApi; + } + + @Override @Transactional(rollbackFor = Exception.class) + public ContentExportJobProjection create(CreateContentExportCommand command, Long actorId, boolean canIncludeAnswers) { + validate(command, canIncludeAnswers); + Long tenantId = TenantContextHolder.getRequiredTenantId(); + String hash = sha256(JsonUtils.toJsonByte(command)); + ContentExportJobDO existing = jobMapper.selectByCommandKey(tenantId, actorId, command.commandKey()); + if (existing != null) return replay(existing, hash); + ContentExportJobDO job = new ContentExportJobDO(); + job.setTenantId(tenantId); job.setActorId(actorId); job.setCommandKey(command.commandKey()); job.setRequestHash(hash); + job.setScope(command.scope()); job.setCollectionId(command.collectionId()); job.setFormat(command.format()); + job.setAnswerMode(command.answerMode()); job.setMaxAttempts(MAX_ATTEMPTS); + jobMapper.insertIgnore(job); + return replay(Objects.requireNonNull(jobMapper.selectByCommandKey(tenantId, actorId, command.commandKey())), hash); + } + + @Override public PageResult page(Long actorId, int pageNo, int pageSize) { + PageResult page = jobMapper.selectTenantActorPage(TenantContextHolder.getRequiredTenantId(), actorId, pageNo, pageSize); + return new PageResult<>(page.getList().stream().map(this::project).toList(), page.getTotal()); + } + + @Override public ContentExportJobProjection get(Long id, Long actorId) { return project(require(id, actorId)); } + + @Override public String downloadUrl(Long id, Long actorId) { + ContentExportJobDO job = require(id, actorId); + if (!"COMPLETED".equals(job.getStatus()) || job.getFileReference() == null) throw exception(CONTENT_EXPORT_NOT_READY); + return fileApi.presignGetUrl(job.getFileReference(), 300); + } + + @Override public int processReady(int limit) { + jobMapper.failExhausted(); + String token = UUID.randomUUID().toString(); + List jobs = jobMapper.claimReady("education-content-export", token, LEASE_SECONDS, Math.min(Math.max(limit, 1), 50)); + for (ContentExportJobDO job : jobs) TenantUtils.execute(job.getTenantId(), () -> process(job, token)); + return jobs.size(); + } + + private void process(ContentExportJobDO job, String token) { + try { + List questions = TenantUtils.executeIgnore(() -> projectionMapper.selectAvailablePublishedCollection(job.getTenantId(), job.getCollectionId())); + if (questions.size() > MAX_QUESTIONS) throw new IllegalArgumentException("question limit exceeded"); + if (jobMapper.heartbeat(job.getTenantId(), job.getId(), token, LEASE_SECONDS) != 1) throw exception(CONTENT_EXPORT_LEASE_LOST); + byte[] bytes = "CSV".equals(job.getFormat()) ? renderCsv(questions, "REDACT".equals(job.getAnswerMode())) + : renderJson(job, questions, "REDACT".equals(job.getAnswerMode())); + if (bytes.length > MAX_BYTES) throw new IllegalArgumentException("output limit exceeded"); + String ext = job.getFormat().toLowerCase(Locale.ROOT); + String name = "question-collection-" + job.getCollectionId() + "." + ext; + String type = "CSV".equals(job.getFormat()) ? "text/csv;charset=UTF-8" : "application/json"; + FileDescriptor file = fileApi.createFile(new FileContent(bytes, name, + "education/content-exports/" + job.getTenantId(), type)); + if (jobMapper.finishCompleted(job.getTenantId(), job.getId(), token, file.reference(), name, type, + bytes.length, sha256(bytes), questions.size()) != 1) throw exception(CONTENT_EXPORT_LEASE_LOST); + } catch (Exception ex) { + String code = ex instanceof IllegalArgumentException ? "INVALID_EXPORT_CONTENT" : "EXPORT_PROCESSING_FAILED"; + String message = ex.getMessage() == null ? code : ex.getMessage(); + jobMapper.finishFailure(job.getTenantId(), job.getId(), token, code, message.substring(0, Math.min(500, message.length()))); + } + } + + byte[] renderJson(ContentExportJobDO job, List questions, boolean redact) { + List> rows = questions.stream().map(q -> row(q, redact)).toList(); + Map envelope = new LinkedHashMap<>(); + envelope.put("schemaVersion", 1); envelope.put("scope", "QUESTION_COLLECTION"); + envelope.put("collectionId", job.getCollectionId()); envelope.put("answerMode", job.getAnswerMode()); + envelope.put("generatedAt", LocalDateTime.now().toString()); envelope.put("questions", rows); + return JsonUtils.toJsonByte(envelope); + } + + byte[] renderCsv(List questions, boolean redact) { + StringBuilder out = new StringBuilder("id,contentVersion,stem,type,difficulty,questionContent,options,tags,correctAnswer,explanation,analysis\r\n"); + for (QuestionDO q : questions) { + Map row = row(q, redact); + List values = List.of(q.getId(), q.getContentVersion(), q.getStem(), q.getType(), nullToEmpty(q.getDifficulty()), + json(row.get("questionContent")), json(row.get("options")), json(row.get("tags")), + nullToEmpty((String) row.get("correctAnswer")), nullToEmpty((String) row.get("explanation")), + nullToEmpty((String) row.get("analysis"))); + out.append(values.stream().map(this::csv).reduce((a, b) -> a + "," + b).orElse("")).append("\r\n"); + } + return out.toString().getBytes(StandardCharsets.UTF_8); + } + + private Map row(QuestionDO q, boolean redact) { + Map row = new LinkedHashMap<>(); + row.put("id", q.getId()); row.put("contentVersion", q.getContentVersion()); row.put("stem", q.getStem()); + row.put("type", q.getType()); row.put("difficulty", q.getDifficulty()); + row.put("questionContent", sanitizer.sanitizeJson(q.getQuestionContent(), redact)); + row.put("options", sanitizer.sanitizeJson(q.getOptions(), redact)); row.put("tags", sanitizer.sanitizeJson(q.getTags(), redact)); + if (!redact) { row.put("correctAnswer", q.getCorrectAnswer()); row.put("explanation", q.getExplanation()); row.put("analysis", q.getAnalysis()); } + return row; + } + + private String csv(Object value) { + String text = String.valueOf(value == null ? "" : value); + if (!text.isEmpty() && "=+-@\t\r".indexOf(text.charAt(0)) >= 0) text = "'" + text; + return "\"" + text.replace("\"", "\"\"") + "\""; + } + + private String json(Object value) { return value == null ? "" : JsonUtils.toJsonString(value); } + private String nullToEmpty(String value) { return value == null ? "" : value; } + + private void validate(CreateContentExportCommand command, boolean canIncludeAnswers) { + if (command == null || command.commandKey() == null || !command.commandKey().matches("[A-Za-z0-9._:-]{1,128}") + || !"QUESTION_COLLECTION".equals(command.scope()) || command.collectionId() == null || command.collectionId() <= 0 + || !("JSON".equals(command.format()) || "CSV".equals(command.format())) + || !("REDACT".equals(command.answerMode()) || "INCLUDE".equals(command.answerMode()))) + throw exception(CONTENT_EXPORT_INVALID_REQUEST); + if ("INCLUDE".equals(command.answerMode()) && !canIncludeAnswers) throw exception(FORBIDDEN); + } + + private ContentExportJobProjection replay(ContentExportJobDO job, String hash) { + if (!hash.equals(job.getRequestHash())) throw exception(CONTENT_EXPORT_COMMAND_CONFLICT); + return project(job); + } + private ContentExportJobDO require(Long id, Long actorId) { + ContentExportJobDO job = jobMapper.selectTenantActorJob(TenantContextHolder.getRequiredTenantId(), actorId, id); + if (job == null) throw exception(CONTENT_EXPORT_NOT_FOUND); return job; + } + private ContentExportJobProjection project(ContentExportJobDO j) { + return new ContentExportJobProjection(j.getId(), j.getScope(), j.getCollectionId(), j.getFormat(), j.getAnswerMode(), + j.getStatus(), j.getAttemptCount(), j.getMaxAttempts(), j.getQuestionCount(), j.getFileName(), + j.getFileSizeBytes(), j.getChecksumSha256(), j.getFailureCode(), j.getCreateTime(), j.getFinishedAt()); + } + private String sha256(byte[] bytes) { + try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); } + catch (Exception ex) { throw new IllegalStateException(ex); } + } +} diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportProjectionMapper.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportProjectionMapper.java new file mode 100644 index 00000000..d61df621 --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportProjectionMapper.java @@ -0,0 +1,32 @@ +package cn.iocoder.yudao.module.education.service.exportjob; + +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.yudao.module.education.dal.dataobject.catalog.QuestionDO; +import org.apache.ibatis.annotations.*; + +import java.util.List; + +@Mapper +public interface ContentExportProjectionMapper extends BaseMapperX { + + @Select(""" + SELECT question.* FROM education_question_collection_question membership + JOIN education_question_collection collection ON collection.id=membership.collection_id + JOIN education_content_node node ON node.id=collection.node_id + JOIN education_content_entry entry ON entry.id=node.entry_id + JOIN education_question question ON question.id=membership.question_id + WHERE membership.collection_id=#{collectionId} AND membership.deleted=false + AND ((membership.tenant_id=#{tenantId} AND membership.scope='TENANT_OWNED') OR (membership.tenant_id=0 AND membership.scope='PUBLIC')) + AND collection.deleted=false AND collection.publication_status='ACTIVE' AND collection.is_active=true AND collection.is_hidden=false + AND ((collection.tenant_id=#{tenantId} AND collection.scope='TENANT_OWNED') OR (collection.tenant_id=0 AND collection.scope='PUBLIC')) + AND node.deleted=false AND node.publication_status='ACTIVE' AND node.is_active=true AND node.is_hidden=false + AND ((node.tenant_id=#{tenantId} AND node.scope='TENANT_OWNED') OR (node.tenant_id=0 AND node.scope='PUBLIC')) + AND entry.deleted=false AND entry.is_active=true AND entry.is_hidden=false + AND ((entry.tenant_id=#{tenantId} AND entry.scope='TENANT_OWNED') OR (entry.tenant_id=0 AND entry.scope='PUBLIC')) + AND question.deleted=false AND question.status='PUBLISHED' AND question.is_published=true + AND ((question.tenant_id=#{tenantId} AND question.scope='TENANT_OWNED') OR (question.tenant_id=0 AND question.scope='PUBLIC')) + ORDER BY membership.sort_order, membership.id LIMIT 5001 + """) + List selectAvailablePublishedCollection(@Param("tenantId") Long tenantId, + @Param("collectionId") Long collectionId); +} diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportSanitizer.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportSanitizer.java new file mode 100644 index 00000000..1c3e6a6b --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportSanitizer.java @@ -0,0 +1,41 @@ +package cn.iocoder.yudao.module.education.service.exportjob; + +import cn.iocoder.yudao.framework.common.util.json.JsonUtils; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.node.ArrayNode; +import tools.jackson.databind.node.ObjectNode; + +import java.util.Iterator; +import java.util.Locale; +import java.util.Set; + +public class ContentExportSanitizer { + private static final int MAX_DEPTH = 32; + private static final Set ANSWER_KEYS = Set.of("answer", "correctanswer", "explanation", "analysis", "iscorrect"); + private static final Set PRIVATE_KEYS = Set.of("tenantid", "creator", "createtime", "updater", "updatetime", "deleted", + "metadata", "accessrules", "publicationstatus", "authoringversion", "membershipversion", "placementversion"); + + public JsonNode sanitizeJson(String json, boolean redactAnswers) { + if (json == null) return null; + JsonNode node; + try { node = JsonUtils.parseTree(json); } + catch (RuntimeException ex) { throw new IllegalArgumentException("invalid export JSON", ex); } + sanitize(node, redactAnswers, 0); + return node; + } + + private void sanitize(JsonNode node, boolean redactAnswers, int depth) { + if (depth > MAX_DEPTH) throw new IllegalArgumentException("export JSON depth exceeded"); + if (node instanceof ObjectNode object) { + Iterator names = object.propertyNames().iterator(); + while (names.hasNext()) { + String name = names.next(); + String normalized = name.replace("_", "").toLowerCase(Locale.ROOT); + if (PRIVATE_KEYS.contains(normalized) || (redactAnswers && ANSWER_KEYS.contains(normalized))) names.remove(); + else sanitize(object.get(name), redactAnswers, depth + 1); + } + } else if (node instanceof ArrayNode array) { + for (JsonNode child : array) sanitize(child, redactAnswers, depth + 1); + } + } +} diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/CreateContentExportCommand.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/CreateContentExportCommand.java new file mode 100644 index 00000000..45a2b4dd --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/exportjob/CreateContentExportCommand.java @@ -0,0 +1,4 @@ +package cn.iocoder.yudao.module.education.service.exportjob; + +public record CreateContentExportCommand(String commandKey, String scope, Long collectionId, + String format, String answerMode) {} diff --git a/yudao-module-education/src/main/resources/db/migration/education/V4469__add_secure_content_exports.sql b/yudao-module-education/src/main/resources/db/migration/education/V4469__add_secure_content_exports.sql new file mode 100644 index 00000000..46aa6ba6 --- /dev/null +++ b/yudao-module-education/src/main/resources/db/migration/education/V4469__add_secure_content_exports.sql @@ -0,0 +1,66 @@ +-- EDU secure asynchronous QUESTION_COLLECTION JSON/CSV exports. + +CREATE TABLE education_content_export_job ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + tenant_id BIGINT NOT NULL, + actor_id BIGINT NOT NULL, + command_key VARCHAR(128) NOT NULL, + request_hash VARCHAR(64) NOT NULL, + scope VARCHAR(32) NOT NULL, + collection_id BIGINT NOT NULL, + format VARCHAR(8) NOT NULL, + answer_mode VARCHAR(8) NOT NULL, + status VARCHAR(16) NOT NULL DEFAULT 'PENDING', + attempt_count INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 5, + next_attempt_at TIMESTAMP, + locked_by VARCHAR(128), + lease_token VARCHAR(64), + lease_expires_at TIMESTAMP, + last_heartbeat_at TIMESTAMP, + started_at TIMESTAMP, + finished_at TIMESTAMP, + file_reference VARCHAR(1024), + file_name VARCHAR(255), + content_type VARCHAR(128), + file_size_bytes BIGINT, + checksum_sha256 VARCHAR(64), + question_count INTEGER, + failure_code VARCHAR(64), + failure_message VARCHAR(500), + creator VARCHAR(64) DEFAULT '', create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updater VARCHAR(64) DEFAULT '', update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted BOOLEAN NOT NULL DEFAULT false, + CONSTRAINT uk_education_content_export_command UNIQUE (tenant_id, actor_id, command_key), + CONSTRAINT ck_education_content_export_scope CHECK (scope='QUESTION_COLLECTION'), + CONSTRAINT ck_education_content_export_format CHECK (format IN ('JSON','CSV')), + CONSTRAINT ck_education_content_export_answer CHECK (answer_mode IN ('REDACT','INCLUDE')), + CONSTRAINT ck_education_content_export_status CHECK (status IN ('PENDING','RENDERING','COMPLETED','FAILED')), + CONSTRAINT ck_education_content_export_attempts CHECK (attempt_count>=0 AND max_attempts>0 AND attempt_count<=max_attempts), + CONSTRAINT ck_education_content_export_output CHECK (file_size_bytes IS NULL OR file_size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT ck_education_content_export_count CHECK (question_count IS NULL OR question_count BETWEEN 0 AND 5000), + CONSTRAINT ck_education_content_export_lease CHECK ((lease_token IS NULL AND locked_by IS NULL AND lease_expires_at IS NULL AND last_heartbeat_at IS NULL) + OR (lease_token IS NOT NULL AND locked_by IS NOT NULL AND lease_expires_at IS NOT NULL AND last_heartbeat_at IS NOT NULL AND status='RENDERING')), + CONSTRAINT ck_education_content_export_completed CHECK (status<>'COMPLETED' OR (file_reference IS NOT NULL AND file_name IS NOT NULL + AND content_type IS NOT NULL AND file_size_bytes IS NOT NULL AND checksum_sha256 IS NOT NULL AND question_count IS NOT NULL)), + CONSTRAINT ck_education_content_export_failed CHECK (status<>'FAILED' OR failure_code IS NOT NULL) +); +COMMENT ON TABLE education_content_export_job IS 'Tenant and actor isolated asynchronous secure content exports'; +COMMENT ON COLUMN education_content_export_job.file_reference IS 'Private Infra File reference; never returned by job APIs'; +COMMENT ON COLUMN education_content_export_job.lease_token IS 'Per-claim fencing token required by heartbeat and terminal writes'; +CREATE INDEX idx_education_content_export_claim ON education_content_export_job + (next_attempt_at,lease_expires_at,create_time,id) WHERE deleted=false AND status IN ('PENDING','RENDERING'); +CREATE INDEX idx_education_content_export_actor ON education_content_export_job (tenant_id,actor_id,id DESC) WHERE deleted=false; + +DO $$ DECLARE installed INTEGER; BEGIN + IF to_regclass('system_menu') IS NULL THEN RETURN; END IF; + INSERT INTO system_menu(id,name,permission,type,sort,parent_id,path,icon,component,component_name,status,visible,keep_alive,always_show,creator,updater) VALUES + (6817,'内容导出创建','education:content-export:create',3,17,6800,'','',NULL,NULL,0,false,true,true,'education-flyway','education-flyway'), + (6818,'内容导出查询','education:content-export:query',3,18,6800,'','',NULL,NULL,0,false,true,true,'education-flyway','education-flyway'), + (6819,'内容导出下载','education:content-export:download',3,19,6800,'','',NULL,NULL,0,false,true,true,'education-flyway','education-flyway') + ON CONFLICT(id) DO NOTHING; + SELECT count(*) INTO installed FROM system_menu WHERE deleted=0 AND status=0 AND + ((id=6817 AND permission='education:content-export:create') OR (id=6818 AND permission='education:content-export:query') + OR (id=6819 AND permission='education:content-export:download')); + IF installed<>3 THEN RAISE EXCEPTION 'Education content export RBAC seed IDs conflict' USING ERRCODE='23505'; END IF; +END $$; diff --git a/yudao-module-education/src/test/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportSanitizerTest.java b/yudao-module-education/src/test/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportSanitizerTest.java new file mode 100644 index 00000000..20ce6547 --- /dev/null +++ b/yudao-module-education/src/test/java/cn/iocoder/yudao/module/education/service/exportjob/ContentExportSanitizerTest.java @@ -0,0 +1,18 @@ +package cn.iocoder.yudao.module.education.service.exportjob; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class ContentExportSanitizerTest { + private final ContentExportSanitizer sanitizer = new ContentExportSanitizer(); + + @Test void recursivelyRedactsAnswersAndPrivateFields() { + String out=sanitizer.sanitizeJson("{\"answer\":1,\"nested\":[{\"correct_answer\":2,\"isCorrect\":true,\"safe\":3}],\"tenant_id\":9}",true).toString(); + assertEquals("{\"nested\":[{\"safe\":3}]}",out); + } + + @Test void malformedJsonFailsClosed() { + assertThrows(IllegalArgumentException.class,()->sanitizer.sanitizeJson("{",true)); + } +}