feat(education): add secure content export jobs
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
package cn.iocoder.yudao.module.education.controller.admin.exportjob;
|
||||
|
||||
import cn.iocoder.yudao.framework.common.pojo.CommonResult;
|
||||
import cn.iocoder.yudao.framework.common.pojo.PageResult;
|
||||
import cn.iocoder.yudao.framework.security.core.service.SecurityFrameworkService;
|
||||
import cn.iocoder.yudao.module.education.controller.admin.exportjob.vo.*;
|
||||
import cn.iocoder.yudao.module.education.service.exportjob.ContentExportJobService;
|
||||
import cn.iocoder.yudao.module.education.service.exportjob.CreateContentExportCommand;
|
||||
import jakarta.validation.Valid;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.security.access.prepost.PreAuthorize;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success;
|
||||
import static cn.iocoder.yudao.framework.security.core.util.SecurityFrameworkUtils.getLoginUserId;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/education/content-export-jobs")
|
||||
@Validated
|
||||
@ConditionalOnProperty(prefix="yudao.education", name="enabled", havingValue="true")
|
||||
public class ContentExportJobController {
|
||||
private final ContentExportJobService service;
|
||||
private final SecurityFrameworkService security;
|
||||
public ContentExportJobController(ContentExportJobService service, SecurityFrameworkService security) {
|
||||
this.service=service; this.security=security;
|
||||
}
|
||||
@PostMapping @PreAuthorize("@ss.hasPermission('education:content-export:create')")
|
||||
public CommonResult<ContentExportJobRespVO> create(@Valid @RequestBody ContentExportCreateReqVO req) {
|
||||
return success(ContentExportJobRespVO.from(service.create(new CreateContentExportCommand(req.getCommandKey(),
|
||||
req.getScope(), req.getCollectionId(), req.getFormat(), req.getAnswerMode()), getLoginUserId(),
|
||||
security.hasPermission("education:content-export:answers"))));
|
||||
}
|
||||
@GetMapping @PreAuthorize("@ss.hasPermission('education:content-export:query')")
|
||||
public CommonResult<PageResult<ContentExportJobRespVO>> page(@Valid ContentExportPageReqVO req) {
|
||||
var page=service.page(getLoginUserId(),req.getPageNo(),req.getPageSize());
|
||||
return success(new PageResult<>(page.getList().stream().map(ContentExportJobRespVO::from).toList(),page.getTotal()));
|
||||
}
|
||||
@GetMapping("/{id}") @PreAuthorize("@ss.hasPermission('education:content-export:query')")
|
||||
public CommonResult<ContentExportJobRespVO> get(@PathVariable Long id) {
|
||||
return success(ContentExportJobRespVO.from(service.get(id,getLoginUserId())));
|
||||
}
|
||||
@GetMapping("/{id}/download-url") @PreAuthorize("@ss.hasPermission('education:content-export:download')")
|
||||
public CommonResult<String> downloadUrl(@PathVariable Long id) { return success(service.downloadUrl(id,getLoginUserId())); }
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package cn.iocoder.yudao.module.education.controller.admin.exportjob.vo;
|
||||
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class ContentExportCreateReqVO {
|
||||
@NotBlank @Pattern(regexp = "[A-Za-z0-9._:-]{1,128}") private String commandKey;
|
||||
@NotBlank private String scope;
|
||||
@NotNull private Long collectionId;
|
||||
@NotBlank private String format;
|
||||
@NotBlank private String answerMode;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package cn.iocoder.yudao.module.education.controller.admin.exportjob.vo;
|
||||
|
||||
import cn.iocoder.yudao.module.education.service.exportjob.ContentExportJobProjection;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
public class ContentExportJobRespVO {
|
||||
private Long id; private String scope; private Long collectionId; private String format; private String answerMode;
|
||||
private String status; private Integer attemptCount; private Integer maxAttempts; private Integer questionCount;
|
||||
private String fileName; private Long fileSizeBytes; private String checksumSha256; private String failureCode;
|
||||
private LocalDateTime createTime; private LocalDateTime finishedAt;
|
||||
public static ContentExportJobRespVO from(ContentExportJobProjection p) {
|
||||
ContentExportJobRespVO v = new ContentExportJobRespVO(); v.id=p.id(); v.scope=p.scope(); v.collectionId=p.collectionId();
|
||||
v.format=p.format(); v.answerMode=p.answerMode(); v.status=p.status(); v.attemptCount=p.attemptCount();
|
||||
v.maxAttempts=p.maxAttempts(); v.questionCount=p.questionCount(); v.fileName=p.fileName();
|
||||
v.fileSizeBytes=p.fileSizeBytes(); v.checksumSha256=p.checksumSha256(); v.failureCode=p.failureCode();
|
||||
v.createTime=p.createTime(); v.finishedAt=p.finishedAt(); return v;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
package cn.iocoder.yudao.module.education.controller.admin.exportjob.vo;
|
||||
|
||||
import cn.iocoder.yudao.framework.common.pojo.PageParam;
|
||||
|
||||
public class ContentExportPageReqVO extends PageParam {}
|
||||
@@ -0,0 +1,43 @@
|
||||
package cn.iocoder.yudao.module.education.dal.dataobject.exportjob;
|
||||
|
||||
import cn.iocoder.yudao.framework.tenant.core.db.TenantBaseDO;
|
||||
import com.baomidou.mybatisplus.annotation.KeySequence;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@TableName("education_content_export_job")
|
||||
@KeySequence("education_content_export_job_seq")
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class ContentExportJobDO extends TenantBaseDO {
|
||||
@TableId private Long id;
|
||||
private Long actorId;
|
||||
private String commandKey;
|
||||
private String requestHash;
|
||||
private String scope;
|
||||
private Long collectionId;
|
||||
private String format;
|
||||
private String answerMode;
|
||||
private String status;
|
||||
private Integer attemptCount;
|
||||
private Integer maxAttempts;
|
||||
private LocalDateTime nextAttemptAt;
|
||||
private String lockedBy;
|
||||
private String leaseToken;
|
||||
private LocalDateTime leaseExpiresAt;
|
||||
private LocalDateTime lastHeartbeatAt;
|
||||
private LocalDateTime startedAt;
|
||||
private LocalDateTime finishedAt;
|
||||
private String fileReference;
|
||||
private String fileName;
|
||||
private String contentType;
|
||||
private Long fileSizeBytes;
|
||||
private String checksumSha256;
|
||||
private Integer questionCount;
|
||||
private String failureCode;
|
||||
private String failureMessage;
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package cn.iocoder.yudao.module.education.dal.mysql.exportjob;
|
||||
|
||||
import cn.iocoder.yudao.framework.common.pojo.PageResult;
|
||||
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
|
||||
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
|
||||
import cn.iocoder.yudao.module.education.dal.dataobject.exportjob.ContentExportJobDO;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import org.apache.ibatis.annotations.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface ContentExportJobMapper extends BaseMapperX<ContentExportJobDO> {
|
||||
|
||||
default ContentExportJobDO selectTenantActorJob(Long tenantId, Long actorId, Long id) {
|
||||
return selectOne(new LambdaQueryWrapperX<ContentExportJobDO>().eq(ContentExportJobDO::getTenantId, tenantId)
|
||||
.eq(ContentExportJobDO::getActorId, actorId).eq(ContentExportJobDO::getId, id));
|
||||
}
|
||||
|
||||
default ContentExportJobDO selectByCommandKey(Long tenantId, Long actorId, String commandKey) {
|
||||
return selectOne(new LambdaQueryWrapperX<ContentExportJobDO>().eq(ContentExportJobDO::getTenantId, tenantId)
|
||||
.eq(ContentExportJobDO::getActorId, actorId).eq(ContentExportJobDO::getCommandKey, commandKey));
|
||||
}
|
||||
|
||||
default PageResult<ContentExportJobDO> selectTenantActorPage(Long tenantId, Long actorId, int pageNo, int pageSize) {
|
||||
Page<ContentExportJobDO> page = selectPage(new Page<>(pageNo, pageSize),
|
||||
new LambdaQueryWrapperX<ContentExportJobDO>().eq(ContentExportJobDO::getTenantId, tenantId)
|
||||
.eq(ContentExportJobDO::getActorId, actorId).orderByDesc(ContentExportJobDO::getId));
|
||||
return new PageResult<>(page.getRecords(), page.getTotal());
|
||||
}
|
||||
|
||||
@Insert("""
|
||||
INSERT INTO education_content_export_job
|
||||
(tenant_id, actor_id, command_key, request_hash, scope, collection_id, format, answer_mode,
|
||||
status, attempt_count, max_attempts, next_attempt_at, creator, create_time, updater, update_time, deleted)
|
||||
VALUES (#{tenantId}, #{actorId}, #{commandKey}, #{requestHash}, #{scope}, #{collectionId}, #{format},
|
||||
#{answerMode}, 'PENDING', 0, #{maxAttempts}, CURRENT_TIMESTAMP, '', CURRENT_TIMESTAMP, '', CURRENT_TIMESTAMP, FALSE)
|
||||
ON CONFLICT (tenant_id, actor_id, command_key) DO NOTHING
|
||||
""")
|
||||
@Options(useGeneratedKeys = true, keyProperty = "id")
|
||||
int insertIgnore(ContentExportJobDO job);
|
||||
|
||||
@Select("""
|
||||
WITH candidate AS (
|
||||
SELECT id FROM education_content_export_job
|
||||
WHERE deleted=false AND status IN ('PENDING','RENDERING') AND attempt_count < max_attempts
|
||||
AND (next_attempt_at IS NULL OR next_attempt_at <= CURRENT_TIMESTAMP)
|
||||
AND (lease_expires_at IS NULL OR lease_expires_at <= CURRENT_TIMESTAMP)
|
||||
ORDER BY COALESCE(next_attempt_at, create_time), id
|
||||
LIMIT #{limit} FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE education_content_export_job job
|
||||
SET status='RENDERING', locked_by=#{workerId}, lease_token=#{leaseToken},
|
||||
lease_expires_at=CURRENT_TIMESTAMP + CAST(#{leaseSeconds} || ' seconds' AS INTERVAL),
|
||||
last_heartbeat_at=CURRENT_TIMESTAMP, attempt_count=job.attempt_count + 1, next_attempt_at=NULL,
|
||||
started_at=COALESCE(job.started_at,CURRENT_TIMESTAMP), failure_code=NULL, failure_message=NULL,
|
||||
update_time=CURRENT_TIMESTAMP
|
||||
FROM candidate WHERE job.id=candidate.id RETURNING job.*
|
||||
""")
|
||||
List<ContentExportJobDO> claimReady(@Param("workerId") String workerId, @Param("leaseToken") String leaseToken,
|
||||
@Param("leaseSeconds") long leaseSeconds, @Param("limit") int limit);
|
||||
|
||||
@Update("""
|
||||
UPDATE education_content_export_job SET
|
||||
lease_expires_at=CURRENT_TIMESTAMP + CAST(#{leaseSeconds} || ' seconds' AS INTERVAL),
|
||||
last_heartbeat_at=CURRENT_TIMESTAMP, update_time=CURRENT_TIMESTAMP
|
||||
WHERE tenant_id=#{tenantId} AND id=#{id} AND status='RENDERING' AND lease_token=#{leaseToken}
|
||||
AND lease_expires_at > CURRENT_TIMESTAMP
|
||||
""")
|
||||
int heartbeat(@Param("tenantId") Long tenantId, @Param("id") Long id,
|
||||
@Param("leaseToken") String leaseToken, @Param("leaseSeconds") long leaseSeconds);
|
||||
|
||||
@Update("""
|
||||
UPDATE education_content_export_job SET status='COMPLETED', file_reference=#{fileReference},
|
||||
file_name=#{fileName}, content_type=#{contentType}, file_size_bytes=#{fileSizeBytes},
|
||||
checksum_sha256=#{checksumSha256}, question_count=#{questionCount}, finished_at=CURRENT_TIMESTAMP,
|
||||
locked_by=NULL, lease_token=NULL, lease_expires_at=NULL, last_heartbeat_at=NULL, update_time=CURRENT_TIMESTAMP
|
||||
WHERE tenant_id=#{tenantId} AND id=#{id} AND status='RENDERING' AND lease_token=#{leaseToken}
|
||||
AND lease_expires_at > CURRENT_TIMESTAMP
|
||||
""")
|
||||
int finishCompleted(@Param("tenantId") Long tenantId, @Param("id") Long id, @Param("leaseToken") String leaseToken,
|
||||
@Param("fileReference") String fileReference, @Param("fileName") String fileName,
|
||||
@Param("contentType") String contentType, @Param("fileSizeBytes") long fileSizeBytes,
|
||||
@Param("checksumSha256") String checksumSha256, @Param("questionCount") int questionCount);
|
||||
|
||||
@Update("""
|
||||
UPDATE education_content_export_job SET status=CASE WHEN attempt_count >= max_attempts THEN 'FAILED' ELSE 'PENDING' END,
|
||||
next_attempt_at=CASE WHEN attempt_count >= max_attempts THEN NULL
|
||||
ELSE CURRENT_TIMESTAMP + CAST(LEAST(300, POWER(2, attempt_count)::INTEGER * 5) || ' seconds' AS INTERVAL) END,
|
||||
failure_code=#{failureCode}, failure_message=#{failureMessage},
|
||||
finished_at=CASE WHEN attempt_count >= max_attempts THEN CURRENT_TIMESTAMP ELSE NULL END,
|
||||
locked_by=NULL, lease_token=NULL, lease_expires_at=NULL, last_heartbeat_at=NULL, update_time=CURRENT_TIMESTAMP
|
||||
WHERE tenant_id=#{tenantId} AND id=#{id} AND status='RENDERING' AND lease_token=#{leaseToken}
|
||||
AND lease_expires_at > CURRENT_TIMESTAMP
|
||||
""")
|
||||
int finishFailure(@Param("tenantId") Long tenantId, @Param("id") Long id, @Param("leaseToken") String leaseToken,
|
||||
@Param("failureCode") String failureCode, @Param("failureMessage") String failureMessage);
|
||||
|
||||
@Update("""
|
||||
UPDATE education_content_export_job SET status='FAILED', failure_code='ATTEMPTS_EXHAUSTED',
|
||||
failure_message='Maximum export attempts exhausted', finished_at=CURRENT_TIMESTAMP,
|
||||
locked_by=NULL, lease_token=NULL, lease_expires_at=NULL, last_heartbeat_at=NULL, update_time=CURRENT_TIMESTAMP
|
||||
WHERE deleted=false AND status IN ('PENDING','RENDERING') AND attempt_count >= max_attempts
|
||||
AND (lease_expires_at IS NULL OR lease_expires_at <= CURRENT_TIMESTAMP)
|
||||
""")
|
||||
int failExhausted();
|
||||
}
|
||||
@@ -218,4 +218,11 @@ public interface ErrorCodeConstants {
|
||||
ErrorCode EDUCATION_IMPORT_ASSET_SIZE_INVALID = new ErrorCode(1_005_004_071, "导入文件为空或超过大小限制");
|
||||
ErrorCode EDUCATION_IMPORT_ASSET_TYPE_INVALID = new ErrorCode(1_005_004_072, "仅支持 CSV 或 XLSX 导入文件");
|
||||
ErrorCode EDUCATION_IMPORT_ASSET_CONTENT_INVALID = new ErrorCode(1_005_004_073, "导入文件内容与声明类型不匹配");
|
||||
|
||||
// ========== 内容导出任务 1-005-004-080 ~ 1-005-004-089 ==========
|
||||
ErrorCode CONTENT_EXPORT_NOT_FOUND = new ErrorCode(1_005_004_082, "内容导出任务不存在或无权访问");
|
||||
ErrorCode CONTENT_EXPORT_COMMAND_CONFLICT = new ErrorCode(1_005_004_083, "导出命令幂等键已用于不同请求");
|
||||
ErrorCode CONTENT_EXPORT_INVALID_REQUEST = new ErrorCode(1_005_004_084, "内容导出请求无效");
|
||||
ErrorCode CONTENT_EXPORT_NOT_READY = new ErrorCode(1_005_004_085, "内容导出任务尚未完成");
|
||||
ErrorCode CONTENT_EXPORT_LEASE_LOST = new ErrorCode(1_005_004_086, "内容导出任务租约已失效");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package cn.iocoder.yudao.module.education.job;
|
||||
|
||||
import cn.iocoder.yudao.framework.quartz.core.handler.JobHandler;
|
||||
import cn.iocoder.yudao.framework.tenant.core.aop.TenantIgnore;
|
||||
import cn.iocoder.yudao.module.education.service.exportjob.ContentExportJobService;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class ContentExportProcessJob implements JobHandler {
|
||||
private final ContentExportJobService service;
|
||||
public ContentExportProcessJob(ContentExportJobService service) { this.service=service; }
|
||||
@Override @TenantIgnore
|
||||
public String execute(String param) {
|
||||
int limit=param==null||param.isBlank()?10:Integer.parseInt(param);
|
||||
return "processed="+service.processReady(limit);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package cn.iocoder.yudao.module.education.service.exportjob;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
public record ContentExportJobProjection(Long id, String scope, Long collectionId, String format, String answerMode,
|
||||
String status, Integer attemptCount, Integer maxAttempts,
|
||||
Integer questionCount, String fileName, Long fileSizeBytes,
|
||||
String checksumSha256, String failureCode, LocalDateTime createTime,
|
||||
LocalDateTime finishedAt) {}
|
||||
@@ -0,0 +1,11 @@
|
||||
package cn.iocoder.yudao.module.education.service.exportjob;
|
||||
|
||||
import cn.iocoder.yudao.framework.common.pojo.PageResult;
|
||||
|
||||
public interface ContentExportJobService {
|
||||
ContentExportJobProjection create(CreateContentExportCommand command, Long actorId, boolean canIncludeAnswers);
|
||||
PageResult<ContentExportJobProjection> page(Long actorId, int pageNo, int pageSize);
|
||||
ContentExportJobProjection get(Long id, Long actorId);
|
||||
String downloadUrl(Long id, Long actorId);
|
||||
int processReady(int limit);
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package cn.iocoder.yudao.module.education.service.exportjob;
|
||||
|
||||
import cn.iocoder.yudao.framework.common.pojo.PageResult;
|
||||
import cn.iocoder.yudao.framework.common.util.json.JsonUtils;
|
||||
import cn.iocoder.yudao.framework.tenant.core.context.TenantContextHolder;
|
||||
import cn.iocoder.yudao.framework.tenant.core.util.TenantUtils;
|
||||
import cn.iocoder.yudao.module.education.dal.dataobject.catalog.QuestionDO;
|
||||
import cn.iocoder.yudao.module.education.dal.dataobject.exportjob.ContentExportJobDO;
|
||||
import cn.iocoder.yudao.module.education.dal.mysql.exportjob.ContentExportJobMapper;
|
||||
import cn.iocoder.yudao.module.infra.api.file.FileApi;
|
||||
import cn.iocoder.yudao.module.infra.api.file.FileContent;
|
||||
import cn.iocoder.yudao.module.infra.api.file.FileDescriptor;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import tools.jackson.databind.JsonNode;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.*;
|
||||
|
||||
import static cn.iocoder.yudao.framework.common.exception.enums.GlobalErrorCodeConstants.FORBIDDEN;
|
||||
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
|
||||
import static cn.iocoder.yudao.module.education.enums.ErrorCodeConstants.*;
|
||||
|
||||
@Service
|
||||
public class ContentExportJobServiceImpl implements ContentExportJobService {
|
||||
static final int MAX_QUESTIONS = 5000;
|
||||
static final int MAX_BYTES = 10 * 1024 * 1024;
|
||||
private static final int MAX_ATTEMPTS = 5;
|
||||
private static final long LEASE_SECONDS = 120;
|
||||
private final ContentExportJobMapper jobMapper;
|
||||
private final ContentExportProjectionMapper projectionMapper;
|
||||
private final FileApi fileApi;
|
||||
private final ContentExportSanitizer sanitizer = new ContentExportSanitizer();
|
||||
|
||||
public ContentExportJobServiceImpl(ContentExportJobMapper jobMapper, ContentExportProjectionMapper projectionMapper,
|
||||
FileApi fileApi) {
|
||||
this.jobMapper = jobMapper; this.projectionMapper = projectionMapper; this.fileApi = fileApi;
|
||||
}
|
||||
|
||||
@Override @Transactional(rollbackFor = Exception.class)
|
||||
public ContentExportJobProjection create(CreateContentExportCommand command, Long actorId, boolean canIncludeAnswers) {
|
||||
validate(command, canIncludeAnswers);
|
||||
Long tenantId = TenantContextHolder.getRequiredTenantId();
|
||||
String hash = sha256(JsonUtils.toJsonByte(command));
|
||||
ContentExportJobDO existing = jobMapper.selectByCommandKey(tenantId, actorId, command.commandKey());
|
||||
if (existing != null) return replay(existing, hash);
|
||||
ContentExportJobDO job = new ContentExportJobDO();
|
||||
job.setTenantId(tenantId); job.setActorId(actorId); job.setCommandKey(command.commandKey()); job.setRequestHash(hash);
|
||||
job.setScope(command.scope()); job.setCollectionId(command.collectionId()); job.setFormat(command.format());
|
||||
job.setAnswerMode(command.answerMode()); job.setMaxAttempts(MAX_ATTEMPTS);
|
||||
jobMapper.insertIgnore(job);
|
||||
return replay(Objects.requireNonNull(jobMapper.selectByCommandKey(tenantId, actorId, command.commandKey())), hash);
|
||||
}
|
||||
|
||||
@Override public PageResult<ContentExportJobProjection> page(Long actorId, int pageNo, int pageSize) {
|
||||
PageResult<ContentExportJobDO> page = jobMapper.selectTenantActorPage(TenantContextHolder.getRequiredTenantId(), actorId, pageNo, pageSize);
|
||||
return new PageResult<>(page.getList().stream().map(this::project).toList(), page.getTotal());
|
||||
}
|
||||
|
||||
@Override public ContentExportJobProjection get(Long id, Long actorId) { return project(require(id, actorId)); }
|
||||
|
||||
@Override public String downloadUrl(Long id, Long actorId) {
|
||||
ContentExportJobDO job = require(id, actorId);
|
||||
if (!"COMPLETED".equals(job.getStatus()) || job.getFileReference() == null) throw exception(CONTENT_EXPORT_NOT_READY);
|
||||
return fileApi.presignGetUrl(job.getFileReference(), 300);
|
||||
}
|
||||
|
||||
@Override public int processReady(int limit) {
|
||||
jobMapper.failExhausted();
|
||||
String token = UUID.randomUUID().toString();
|
||||
List<ContentExportJobDO> jobs = jobMapper.claimReady("education-content-export", token, LEASE_SECONDS, Math.min(Math.max(limit, 1), 50));
|
||||
for (ContentExportJobDO job : jobs) TenantUtils.execute(job.getTenantId(), () -> process(job, token));
|
||||
return jobs.size();
|
||||
}
|
||||
|
||||
private void process(ContentExportJobDO job, String token) {
|
||||
try {
|
||||
List<QuestionDO> questions = TenantUtils.executeIgnore(() -> projectionMapper.selectAvailablePublishedCollection(job.getTenantId(), job.getCollectionId()));
|
||||
if (questions.size() > MAX_QUESTIONS) throw new IllegalArgumentException("question limit exceeded");
|
||||
if (jobMapper.heartbeat(job.getTenantId(), job.getId(), token, LEASE_SECONDS) != 1) throw exception(CONTENT_EXPORT_LEASE_LOST);
|
||||
byte[] bytes = "CSV".equals(job.getFormat()) ? renderCsv(questions, "REDACT".equals(job.getAnswerMode()))
|
||||
: renderJson(job, questions, "REDACT".equals(job.getAnswerMode()));
|
||||
if (bytes.length > MAX_BYTES) throw new IllegalArgumentException("output limit exceeded");
|
||||
String ext = job.getFormat().toLowerCase(Locale.ROOT);
|
||||
String name = "question-collection-" + job.getCollectionId() + "." + ext;
|
||||
String type = "CSV".equals(job.getFormat()) ? "text/csv;charset=UTF-8" : "application/json";
|
||||
FileDescriptor file = fileApi.createFile(new FileContent(bytes, name,
|
||||
"education/content-exports/" + job.getTenantId(), type));
|
||||
if (jobMapper.finishCompleted(job.getTenantId(), job.getId(), token, file.reference(), name, type,
|
||||
bytes.length, sha256(bytes), questions.size()) != 1) throw exception(CONTENT_EXPORT_LEASE_LOST);
|
||||
} catch (Exception ex) {
|
||||
String code = ex instanceof IllegalArgumentException ? "INVALID_EXPORT_CONTENT" : "EXPORT_PROCESSING_FAILED";
|
||||
String message = ex.getMessage() == null ? code : ex.getMessage();
|
||||
jobMapper.finishFailure(job.getTenantId(), job.getId(), token, code, message.substring(0, Math.min(500, message.length())));
|
||||
}
|
||||
}
|
||||
|
||||
byte[] renderJson(ContentExportJobDO job, List<QuestionDO> questions, boolean redact) {
|
||||
List<Map<String, Object>> rows = questions.stream().map(q -> row(q, redact)).toList();
|
||||
Map<String, Object> envelope = new LinkedHashMap<>();
|
||||
envelope.put("schemaVersion", 1); envelope.put("scope", "QUESTION_COLLECTION");
|
||||
envelope.put("collectionId", job.getCollectionId()); envelope.put("answerMode", job.getAnswerMode());
|
||||
envelope.put("generatedAt", LocalDateTime.now().toString()); envelope.put("questions", rows);
|
||||
return JsonUtils.toJsonByte(envelope);
|
||||
}
|
||||
|
||||
byte[] renderCsv(List<QuestionDO> questions, boolean redact) {
|
||||
StringBuilder out = new StringBuilder("id,contentVersion,stem,type,difficulty,questionContent,options,tags,correctAnswer,explanation,analysis\r\n");
|
||||
for (QuestionDO q : questions) {
|
||||
Map<String, Object> row = row(q, redact);
|
||||
List<Object> values = List.of(q.getId(), q.getContentVersion(), q.getStem(), q.getType(), nullToEmpty(q.getDifficulty()),
|
||||
json(row.get("questionContent")), json(row.get("options")), json(row.get("tags")),
|
||||
nullToEmpty((String) row.get("correctAnswer")), nullToEmpty((String) row.get("explanation")),
|
||||
nullToEmpty((String) row.get("analysis")));
|
||||
out.append(values.stream().map(this::csv).reduce((a, b) -> a + "," + b).orElse("")).append("\r\n");
|
||||
}
|
||||
return out.toString().getBytes(StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private Map<String, Object> row(QuestionDO q, boolean redact) {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("id", q.getId()); row.put("contentVersion", q.getContentVersion()); row.put("stem", q.getStem());
|
||||
row.put("type", q.getType()); row.put("difficulty", q.getDifficulty());
|
||||
row.put("questionContent", sanitizer.sanitizeJson(q.getQuestionContent(), redact));
|
||||
row.put("options", sanitizer.sanitizeJson(q.getOptions(), redact)); row.put("tags", sanitizer.sanitizeJson(q.getTags(), redact));
|
||||
if (!redact) { row.put("correctAnswer", q.getCorrectAnswer()); row.put("explanation", q.getExplanation()); row.put("analysis", q.getAnalysis()); }
|
||||
return row;
|
||||
}
|
||||
|
||||
private String csv(Object value) {
|
||||
String text = String.valueOf(value == null ? "" : value);
|
||||
if (!text.isEmpty() && "=+-@\t\r".indexOf(text.charAt(0)) >= 0) text = "'" + text;
|
||||
return "\"" + text.replace("\"", "\"\"") + "\"";
|
||||
}
|
||||
|
||||
private String json(Object value) { return value == null ? "" : JsonUtils.toJsonString(value); }
|
||||
private String nullToEmpty(String value) { return value == null ? "" : value; }
|
||||
|
||||
private void validate(CreateContentExportCommand command, boolean canIncludeAnswers) {
|
||||
if (command == null || command.commandKey() == null || !command.commandKey().matches("[A-Za-z0-9._:-]{1,128}")
|
||||
|| !"QUESTION_COLLECTION".equals(command.scope()) || command.collectionId() == null || command.collectionId() <= 0
|
||||
|| !("JSON".equals(command.format()) || "CSV".equals(command.format()))
|
||||
|| !("REDACT".equals(command.answerMode()) || "INCLUDE".equals(command.answerMode())))
|
||||
throw exception(CONTENT_EXPORT_INVALID_REQUEST);
|
||||
if ("INCLUDE".equals(command.answerMode()) && !canIncludeAnswers) throw exception(FORBIDDEN);
|
||||
}
|
||||
|
||||
private ContentExportJobProjection replay(ContentExportJobDO job, String hash) {
|
||||
if (!hash.equals(job.getRequestHash())) throw exception(CONTENT_EXPORT_COMMAND_CONFLICT);
|
||||
return project(job);
|
||||
}
|
||||
private ContentExportJobDO require(Long id, Long actorId) {
|
||||
ContentExportJobDO job = jobMapper.selectTenantActorJob(TenantContextHolder.getRequiredTenantId(), actorId, id);
|
||||
if (job == null) throw exception(CONTENT_EXPORT_NOT_FOUND); return job;
|
||||
}
|
||||
private ContentExportJobProjection project(ContentExportJobDO j) {
|
||||
return new ContentExportJobProjection(j.getId(), j.getScope(), j.getCollectionId(), j.getFormat(), j.getAnswerMode(),
|
||||
j.getStatus(), j.getAttemptCount(), j.getMaxAttempts(), j.getQuestionCount(), j.getFileName(),
|
||||
j.getFileSizeBytes(), j.getChecksumSha256(), j.getFailureCode(), j.getCreateTime(), j.getFinishedAt());
|
||||
}
|
||||
private String sha256(byte[] bytes) {
|
||||
try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); }
|
||||
catch (Exception ex) { throw new IllegalStateException(ex); }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package cn.iocoder.yudao.module.education.service.exportjob;
|
||||
|
||||
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
|
||||
import cn.iocoder.yudao.module.education.dal.dataobject.catalog.QuestionDO;
|
||||
import org.apache.ibatis.annotations.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface ContentExportProjectionMapper extends BaseMapperX<QuestionDO> {
|
||||
|
||||
@Select("""
|
||||
SELECT question.* FROM education_question_collection_question membership
|
||||
JOIN education_question_collection collection ON collection.id=membership.collection_id
|
||||
JOIN education_content_node node ON node.id=collection.node_id
|
||||
JOIN education_content_entry entry ON entry.id=node.entry_id
|
||||
JOIN education_question question ON question.id=membership.question_id
|
||||
WHERE membership.collection_id=#{collectionId} AND membership.deleted=false
|
||||
AND ((membership.tenant_id=#{tenantId} AND membership.scope='TENANT_OWNED') OR (membership.tenant_id=0 AND membership.scope='PUBLIC'))
|
||||
AND collection.deleted=false AND collection.publication_status='ACTIVE' AND collection.is_active=true AND collection.is_hidden=false
|
||||
AND ((collection.tenant_id=#{tenantId} AND collection.scope='TENANT_OWNED') OR (collection.tenant_id=0 AND collection.scope='PUBLIC'))
|
||||
AND node.deleted=false AND node.publication_status='ACTIVE' AND node.is_active=true AND node.is_hidden=false
|
||||
AND ((node.tenant_id=#{tenantId} AND node.scope='TENANT_OWNED') OR (node.tenant_id=0 AND node.scope='PUBLIC'))
|
||||
AND entry.deleted=false AND entry.is_active=true AND entry.is_hidden=false
|
||||
AND ((entry.tenant_id=#{tenantId} AND entry.scope='TENANT_OWNED') OR (entry.tenant_id=0 AND entry.scope='PUBLIC'))
|
||||
AND question.deleted=false AND question.status='PUBLISHED' AND question.is_published=true
|
||||
AND ((question.tenant_id=#{tenantId} AND question.scope='TENANT_OWNED') OR (question.tenant_id=0 AND question.scope='PUBLIC'))
|
||||
ORDER BY membership.sort_order, membership.id LIMIT 5001
|
||||
""")
|
||||
List<QuestionDO> selectAvailablePublishedCollection(@Param("tenantId") Long tenantId,
|
||||
@Param("collectionId") Long collectionId);
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package cn.iocoder.yudao.module.education.service.exportjob;
|
||||
|
||||
import cn.iocoder.yudao.framework.common.util.json.JsonUtils;
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.node.ArrayNode;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
|
||||
public class ContentExportSanitizer {
|
||||
private static final int MAX_DEPTH = 32;
|
||||
private static final Set<String> ANSWER_KEYS = Set.of("answer", "correctanswer", "explanation", "analysis", "iscorrect");
|
||||
private static final Set<String> PRIVATE_KEYS = Set.of("tenantid", "creator", "createtime", "updater", "updatetime", "deleted",
|
||||
"metadata", "accessrules", "publicationstatus", "authoringversion", "membershipversion", "placementversion");
|
||||
|
||||
public JsonNode sanitizeJson(String json, boolean redactAnswers) {
|
||||
if (json == null) return null;
|
||||
JsonNode node;
|
||||
try { node = JsonUtils.parseTree(json); }
|
||||
catch (RuntimeException ex) { throw new IllegalArgumentException("invalid export JSON", ex); }
|
||||
sanitize(node, redactAnswers, 0);
|
||||
return node;
|
||||
}
|
||||
|
||||
private void sanitize(JsonNode node, boolean redactAnswers, int depth) {
|
||||
if (depth > MAX_DEPTH) throw new IllegalArgumentException("export JSON depth exceeded");
|
||||
if (node instanceof ObjectNode object) {
|
||||
Iterator<String> names = object.propertyNames().iterator();
|
||||
while (names.hasNext()) {
|
||||
String name = names.next();
|
||||
String normalized = name.replace("_", "").toLowerCase(Locale.ROOT);
|
||||
if (PRIVATE_KEYS.contains(normalized) || (redactAnswers && ANSWER_KEYS.contains(normalized))) names.remove();
|
||||
else sanitize(object.get(name), redactAnswers, depth + 1);
|
||||
}
|
||||
} else if (node instanceof ArrayNode array) {
|
||||
for (JsonNode child : array) sanitize(child, redactAnswers, depth + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
package cn.iocoder.yudao.module.education.service.exportjob;
|
||||
|
||||
public record CreateContentExportCommand(String commandKey, String scope, Long collectionId,
|
||||
String format, String answerMode) {}
|
||||
@@ -0,0 +1,66 @@
|
||||
-- EDU secure asynchronous QUESTION_COLLECTION JSON/CSV exports.
|
||||
|
||||
CREATE TABLE education_content_export_job (
|
||||
id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
actor_id BIGINT NOT NULL,
|
||||
command_key VARCHAR(128) NOT NULL,
|
||||
request_hash VARCHAR(64) NOT NULL,
|
||||
scope VARCHAR(32) NOT NULL,
|
||||
collection_id BIGINT NOT NULL,
|
||||
format VARCHAR(8) NOT NULL,
|
||||
answer_mode VARCHAR(8) NOT NULL,
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'PENDING',
|
||||
attempt_count INTEGER NOT NULL DEFAULT 0,
|
||||
max_attempts INTEGER NOT NULL DEFAULT 5,
|
||||
next_attempt_at TIMESTAMP,
|
||||
locked_by VARCHAR(128),
|
||||
lease_token VARCHAR(64),
|
||||
lease_expires_at TIMESTAMP,
|
||||
last_heartbeat_at TIMESTAMP,
|
||||
started_at TIMESTAMP,
|
||||
finished_at TIMESTAMP,
|
||||
file_reference VARCHAR(1024),
|
||||
file_name VARCHAR(255),
|
||||
content_type VARCHAR(128),
|
||||
file_size_bytes BIGINT,
|
||||
checksum_sha256 VARCHAR(64),
|
||||
question_count INTEGER,
|
||||
failure_code VARCHAR(64),
|
||||
failure_message VARCHAR(500),
|
||||
creator VARCHAR(64) DEFAULT '', create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updater VARCHAR(64) DEFAULT '', update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
deleted BOOLEAN NOT NULL DEFAULT false,
|
||||
CONSTRAINT uk_education_content_export_command UNIQUE (tenant_id, actor_id, command_key),
|
||||
CONSTRAINT ck_education_content_export_scope CHECK (scope='QUESTION_COLLECTION'),
|
||||
CONSTRAINT ck_education_content_export_format CHECK (format IN ('JSON','CSV')),
|
||||
CONSTRAINT ck_education_content_export_answer CHECK (answer_mode IN ('REDACT','INCLUDE')),
|
||||
CONSTRAINT ck_education_content_export_status CHECK (status IN ('PENDING','RENDERING','COMPLETED','FAILED')),
|
||||
CONSTRAINT ck_education_content_export_attempts CHECK (attempt_count>=0 AND max_attempts>0 AND attempt_count<=max_attempts),
|
||||
CONSTRAINT ck_education_content_export_output CHECK (file_size_bytes IS NULL OR file_size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT ck_education_content_export_count CHECK (question_count IS NULL OR question_count BETWEEN 0 AND 5000),
|
||||
CONSTRAINT ck_education_content_export_lease CHECK ((lease_token IS NULL AND locked_by IS NULL AND lease_expires_at IS NULL AND last_heartbeat_at IS NULL)
|
||||
OR (lease_token IS NOT NULL AND locked_by IS NOT NULL AND lease_expires_at IS NOT NULL AND last_heartbeat_at IS NOT NULL AND status='RENDERING')),
|
||||
CONSTRAINT ck_education_content_export_completed CHECK (status<>'COMPLETED' OR (file_reference IS NOT NULL AND file_name IS NOT NULL
|
||||
AND content_type IS NOT NULL AND file_size_bytes IS NOT NULL AND checksum_sha256 IS NOT NULL AND question_count IS NOT NULL)),
|
||||
CONSTRAINT ck_education_content_export_failed CHECK (status<>'FAILED' OR failure_code IS NOT NULL)
|
||||
);
|
||||
COMMENT ON TABLE education_content_export_job IS 'Tenant and actor isolated asynchronous secure content exports';
|
||||
COMMENT ON COLUMN education_content_export_job.file_reference IS 'Private Infra File reference; never returned by job APIs';
|
||||
COMMENT ON COLUMN education_content_export_job.lease_token IS 'Per-claim fencing token required by heartbeat and terminal writes';
|
||||
CREATE INDEX idx_education_content_export_claim ON education_content_export_job
|
||||
(next_attempt_at,lease_expires_at,create_time,id) WHERE deleted=false AND status IN ('PENDING','RENDERING');
|
||||
CREATE INDEX idx_education_content_export_actor ON education_content_export_job (tenant_id,actor_id,id DESC) WHERE deleted=false;
|
||||
|
||||
DO $$ DECLARE installed INTEGER; BEGIN
|
||||
IF to_regclass('system_menu') IS NULL THEN RETURN; END IF;
|
||||
INSERT INTO system_menu(id,name,permission,type,sort,parent_id,path,icon,component,component_name,status,visible,keep_alive,always_show,creator,updater) VALUES
|
||||
(6817,'内容导出创建','education:content-export:create',3,17,6800,'','',NULL,NULL,0,false,true,true,'education-flyway','education-flyway'),
|
||||
(6818,'内容导出查询','education:content-export:query',3,18,6800,'','',NULL,NULL,0,false,true,true,'education-flyway','education-flyway'),
|
||||
(6819,'内容导出下载','education:content-export:download',3,19,6800,'','',NULL,NULL,0,false,true,true,'education-flyway','education-flyway')
|
||||
ON CONFLICT(id) DO NOTHING;
|
||||
SELECT count(*) INTO installed FROM system_menu WHERE deleted=0 AND status=0 AND
|
||||
((id=6817 AND permission='education:content-export:create') OR (id=6818 AND permission='education:content-export:query')
|
||||
OR (id=6819 AND permission='education:content-export:download'));
|
||||
IF installed<>3 THEN RAISE EXCEPTION 'Education content export RBAC seed IDs conflict' USING ERRCODE='23505'; END IF;
|
||||
END $$;
|
||||
@@ -0,0 +1,18 @@
|
||||
package cn.iocoder.yudao.module.education.service.exportjob;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class ContentExportSanitizerTest {
|
||||
private final ContentExportSanitizer sanitizer = new ContentExportSanitizer();
|
||||
|
||||
@Test void recursivelyRedactsAnswersAndPrivateFields() {
|
||||
String out=sanitizer.sanitizeJson("{\"answer\":1,\"nested\":[{\"correct_answer\":2,\"isCorrect\":true,\"safe\":3}],\"tenant_id\":9}",true).toString();
|
||||
assertEquals("{\"nested\":[{\"safe\":3}]}",out);
|
||||
}
|
||||
|
||||
@Test void malformedJsonFailsClosed() {
|
||||
assertThrows(IllegalArgumentException.class,()->sanitizer.sanitizeJson("{",true));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user