From 453193e85701d6222adecfad2093cfd82dbf3200 Mon Sep 17 00:00:00 2001 From: wangziqi Date: Fri, 31 Jul 2026 22:42:20 +0800 Subject: [PATCH] feat(education): add production file scanning --- .../EducationAssetAdmissionServiceImpl.java | 2 +- .../importjob/ImportObjectScanGateway.java | 4 +- .../InfraFileImportObjectScanGateway.java | 27 ++++++++ .../QuestionImportJobServiceImpl.java | 3 +- .../InfraFileImportObjectScanGatewayTest.java | 39 +++++++++++ .../QuestionImportJobServiceImplTest.java | 27 +++++++- .../yudao/module/infra/api/file/FileApi.java | 7 +- .../module/infra/api/file/FileApiImpl.java | 5 ++ .../module/infra/api/file/FileDescriptor.java | 9 ++- .../module/infra/api/file/FileScanStatus.java | 1 + .../infra/dal/mysql/file/FileMapper.java | 6 ++ .../file/config/FileScanProperties.java | 29 ++++++++ .../config/YudaoFileAutoConfiguration.java | 8 +++ .../file/core/scan/ClamAvFileScanner.java | 50 ++++++++++++++ .../infra/service/file/FileService.java | 4 ++ .../infra/service/file/FileServiceImpl.java | 30 ++++++++ .../infra/api/file/FileApiImplTest.java | 13 ++-- .../file/core/scan/ClamAvFileScannerTest.java | 69 +++++++++++++++++++ .../service/file/FileServiceImplTest.java | 43 ++++++++++++ .../src/main/resources/application.yaml | 8 +++ 20 files changed, 371 insertions(+), 13 deletions(-) create mode 100644 yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/importjob/InfraFileImportObjectScanGateway.java create mode 100644 yudao-module-education/src/test/java/cn/iocoder/yudao/module/education/service/importjob/InfraFileImportObjectScanGatewayTest.java create mode 100644 yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/config/FileScanProperties.java create mode 100644 yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/core/scan/ClamAvFileScanner.java create mode 100644 yudao-module-infra/src/test/java/cn/iocoder/yudao/module/infra/framework/file/core/scan/ClamAvFileScannerTest.java diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/asset/EducationAssetAdmissionServiceImpl.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/asset/EducationAssetAdmissionServiceImpl.java index aa229439..8f835c8f 100644 --- a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/asset/EducationAssetAdmissionServiceImpl.java +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/asset/EducationAssetAdmissionServiceImpl.java @@ -63,7 +63,7 @@ public class EducationAssetAdmissionServiceImpl implements EducationAssetAdmissi asset.setOriginalName(name); asset.setContentType(contentType); asset.setSize((long) bytes.length); - asset.setSha256(DigestUtil.sha256Hex(bytes)); + asset.setSha256(descriptor.checksumSha256() != null ? descriptor.checksumSha256() : DigestUtil.sha256Hex(bytes)); asset.setFileReference(descriptor.reference()); asset.setScanStatus(scanStatus.name()); assetMapper.insert(asset); diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/importjob/ImportObjectScanGateway.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/importjob/ImportObjectScanGateway.java index 95ab6c1c..501e9887 100644 --- a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/importjob/ImportObjectScanGateway.java +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/importjob/ImportObjectScanGateway.java @@ -1,6 +1,6 @@ package cn.iocoder.yudao.module.education.service.importjob; public interface ImportObjectScanGateway { - enum ScanResult { CLEAN, INFECTED, ERROR } - ScanResult scan(String objectKey); + enum ScanResult { CLEAN, INFECTED, ERROR, UNAVAILABLE } + ScanResult scan(String reference, String name, String mimeType, long size, String checksumSha256); } diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/importjob/InfraFileImportObjectScanGateway.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/importjob/InfraFileImportObjectScanGateway.java new file mode 100644 index 00000000..46eb4e66 --- /dev/null +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/importjob/InfraFileImportObjectScanGateway.java @@ -0,0 +1,27 @@ +package cn.iocoder.yudao.module.education.service.importjob; + +import cn.iocoder.yudao.module.infra.api.file.FileApi; +import cn.iocoder.yudao.module.infra.api.file.FileDescriptor; +import cn.iocoder.yudao.module.infra.api.file.FileScanStatus; +import org.springframework.stereotype.Service; + +@Service +public class InfraFileImportObjectScanGateway implements ImportObjectScanGateway { + + private final FileApi fileApi; + + public InfraFileImportObjectScanGateway(FileApi fileApi) { + this.fileApi = fileApi; + } + + @Override + public ScanResult scan(String reference, String name, String mimeType, long size, String checksumSha256) { + FileScanStatus status; + try { + status = fileApi.scan(new FileDescriptor(reference, name, mimeType, size, checksumSha256)); + } catch (RuntimeException ex) { + return ScanResult.ERROR; + } + return status == null ? ScanResult.UNAVAILABLE : ScanResult.valueOf(status.name()); + } +} diff --git a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/importjob/QuestionImportJobServiceImpl.java b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/importjob/QuestionImportJobServiceImpl.java index 8a217122..1107f8dc 100644 --- a/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/importjob/QuestionImportJobServiceImpl.java +++ b/yudao-module-education/src/main/java/cn/iocoder/yudao/module/education/service/importjob/QuestionImportJobServiceImpl.java @@ -133,7 +133,8 @@ public class QuestionImportJobServiceImpl implements QuestionImportJobService { finishFailed(job, token, "SCAN_UNAVAILABLE"); throw exception(QUESTION_IMPORT_SCAN_NOT_CLEAN); } - ImportObjectScanGateway.ScanResult scanResult = gateway.scan(asset.getObjectKey()); + ImportObjectScanGateway.ScanResult scanResult = gateway.scan(asset.getObjectKey(), asset.getFileName(), + asset.getMimeType(), asset.getFileSizeBytes(), asset.getChecksumSha256()); if (scanResult != ImportObjectScanGateway.ScanResult.CLEAN) { finishFailed(job, token, "SCAN_" + scanResult.name()); throw exception(QUESTION_IMPORT_SCAN_NOT_CLEAN); diff --git a/yudao-module-education/src/test/java/cn/iocoder/yudao/module/education/service/importjob/InfraFileImportObjectScanGatewayTest.java b/yudao-module-education/src/test/java/cn/iocoder/yudao/module/education/service/importjob/InfraFileImportObjectScanGatewayTest.java new file mode 100644 index 00000000..65d44f0b --- /dev/null +++ b/yudao-module-education/src/test/java/cn/iocoder/yudao/module/education/service/importjob/InfraFileImportObjectScanGatewayTest.java @@ -0,0 +1,39 @@ +package cn.iocoder.yudao.module.education.service.importjob; + +import cn.iocoder.yudao.module.infra.api.file.FileApi; +import cn.iocoder.yudao.module.infra.api.file.FileDescriptor; +import cn.iocoder.yudao.module.infra.api.file.FileScanStatus; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.*; + +class InfraFileImportObjectScanGatewayTest { + + @Test + void passesCompleteDescriptorAndMapsEveryStatus() { + FileApi fileApi = mock(FileApi.class); + InfraFileImportObjectScanGateway gateway = new InfraFileImportObjectScanGateway(fileApi); + FileDescriptor descriptor = new FileDescriptor("ref", "questions.csv", "text/csv", 3, + "039058c6f2c0cb492c533b0a4d14ef77cc0f78abccced5287d84a1a2011cfb81"); + for (FileScanStatus status : FileScanStatus.values()) { + when(fileApi.scan(descriptor)).thenReturn(status); + assertEquals(ImportObjectScanGateway.ScanResult.valueOf(status.name()), + gateway.scan(descriptor.reference(), descriptor.name(), descriptor.contentType(), + descriptor.size(), descriptor.checksumSha256())); + } + verify(fileApi, times(FileScanStatus.values().length)).scan(descriptor); + } + + @Test + void nullAndExceptionsFailClosed() { + FileApi fileApi = mock(FileApi.class); + InfraFileImportObjectScanGateway gateway = new InfraFileImportObjectScanGateway(fileApi); + when(fileApi.scan(any())).thenReturn(null).thenThrow(new IllegalStateException("down")); + + assertEquals(ImportObjectScanGateway.ScanResult.UNAVAILABLE, + gateway.scan("ref", "x", null, 1, null)); + assertEquals(ImportObjectScanGateway.ScanResult.ERROR, + gateway.scan("ref", "x", null, 1, null)); + } +} diff --git a/yudao-module-education/src/test/java/cn/iocoder/yudao/module/education/service/importjob/QuestionImportJobServiceImplTest.java b/yudao-module-education/src/test/java/cn/iocoder/yudao/module/education/service/importjob/QuestionImportJobServiceImplTest.java index a59d4576..412411ea 100644 --- a/yudao-module-education/src/test/java/cn/iocoder/yudao/module/education/service/importjob/QuestionImportJobServiceImplTest.java +++ b/yudao-module-education/src/test/java/cn/iocoder/yudao/module/education/service/importjob/QuestionImportJobServiceImplTest.java @@ -79,7 +79,8 @@ class QuestionImportJobServiceImplTest { when(jobMapper.claimById(eq(101L), eq("PREVIEW_PENDING"), anyString(), anyString(), anyLong())) .thenReturn(claimed); when(assetMapper.selectTenantAsset(10L, 88L)).thenReturn(asset(88L)); - when(scanGateway.scan("object/10/questions.csv")).thenReturn(ImportObjectScanGateway.ScanResult.CLEAN); + when(scanGateway.scan("object/10/questions.csv", "questions.csv", "text/csv", 128L, "checksum")) + .thenReturn(ImportObjectScanGateway.ScanResult.CLEAN); when(jobMapper.finishClaim(eq(10L), eq(101L), eq("PREVIEW_PENDING"), eq("PREVIEW_READY"), anyString(), eq("CLEAN"), eq("UNAVAILABLE"), contains("METADATA_ONLY"), isNull(), isNull(), isNull(), isNull(), isNull(), isNull())).thenReturn(1); @@ -106,6 +107,28 @@ class QuestionImportJobServiceImplTest { verifyNoInteractions(parser, lifecycleService); } + @Test + void nonCleanStatusesHaveStableFailureCodesAndNeverParse() { + for (ImportObjectScanGateway.ScanResult result : List.of( + ImportObjectScanGateway.ScanResult.INFECTED, + ImportObjectScanGateway.ScanResult.ERROR, + ImportObjectScanGateway.ScanResult.UNAVAILABLE)) { + reset(jobMapper, assetMapper, scanGateway, parser, lifecycleService); + when(jobMapper.claimById(eq(101L), eq("PREVIEW_PENDING"), anyString(), anyString(), anyLong())) + .thenReturn(job(101L, "PREVIEW_PENDING")); + when(assetMapper.selectTenantAsset(10L, 88L)).thenReturn(asset(88L)); + when(scanGateway.scan("object/10/questions.csv", "questions.csv", "text/csv", 128L, "checksum")) + .thenReturn(result); + + assertThrows(ServiceException.class, () -> service.preview(101L)); + + verify(jobMapper).finishClaim(eq(10L), eq(101L), eq("PREVIEW_PENDING"), eq("FAILED"), + anyString(), isNull(), isNull(), isNull(), isNull(), isNull(), isNull(), isNull(), + eq("SCAN_" + result.name()), eq("SCAN_" + result.name())); + verifyNoInteractions(parser, lifecycleService); + } + } + @Test void executePropagatesClaimedTenantAndCreatesDrafts() { TenantContextHolder.clear(); @@ -131,7 +154,7 @@ class QuestionImportJobServiceImplTest { private ContentImportAssetDO asset(Long id) { ContentImportAssetDO asset = new ContentImportAssetDO(); asset.setId(id); asset.setTenantId(10L); asset.setObjectKey("object/10/questions.csv"); asset.setFileName("questions.csv"); - asset.setMimeType("text/csv"); asset.setFileSizeBytes(128L); return asset; + asset.setMimeType("text/csv"); asset.setFileSizeBytes(128L); asset.setChecksumSha256("checksum"); return asset; } private ContentImportJobDO job(Long id, String status) { diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileApi.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileApi.java index 0a6f6445..7f094b28 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileApi.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileApi.java @@ -1,5 +1,6 @@ package cn.iocoder.yudao.module.infra.api.file; +import cn.hutool.crypto.digest.DigestUtil; import jakarta.validation.constraints.NotEmpty; /** @@ -46,8 +47,10 @@ public interface FileApi { * 保存有界文件内容，返回不泄露存储实现的公开描述。 */ default FileDescriptor createFile(FileContent content) { - String reference = createFile(content.bytes(), content.name(), content.directory(), content.contentType()); - return new FileDescriptor(reference, content.name(), content.contentType(), content.bytes().length); + byte[] bytes = content.bytes(); + String reference = createFile(bytes, content.name(), content.directory(), content.contentType()); + return new FileDescriptor(reference, content.name(), content.contentType(), bytes.length, + DigestUtil.sha256Hex(bytes)); } /** diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileApiImpl.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileApiImpl.java index f5d8da82..d49a30ba 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileApiImpl.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileApiImpl.java @@ -26,6 +26,11 @@ public class FileApiImpl implements FileApi { return fileService.createFile(content, name, directory, type); } + @Override + public FileScanStatus scan(FileDescriptor file) { + return fileService.scan(file); + } + @Override public String presignGetUrl(String url, Integer expirationSeconds) { return fileService.presignGetUrl(url, expirationSeconds); diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileDescriptor.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileDescriptor.java index 0eb575df..686c2365 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileDescriptor.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileDescriptor.java @@ -1,7 +1,11 @@ package cn.iocoder.yudao.module.infra.api.file; /** Public, storage-agnostic description of an accepted file. */ -public record FileDescriptor(String reference, String name, String contentType, long size) { +public record FileDescriptor(String reference, String name, String contentType, long size, String checksumSha256) { + + public FileDescriptor(String reference, String name, String contentType, long size) { + this(reference, name, contentType, size, null); + } public FileDescriptor { if (reference == null || reference.isBlank()) { @@ -10,5 +14,8 @@ public record FileDescriptor(String reference, String name, String contentType, if (size < 0 || size > FileContent.MAX_CONTENT_BYTES) { throw new IllegalArgumentException("file size is outside public contract bounds"); } + if (checksumSha256 != null && !checksumSha256.matches("(?i)[0-9a-f]{64}")) { + throw new IllegalArgumentException("file SHA-256 checksum is invalid"); + } } } diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileScanStatus.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileScanStatus.java index 5538130f..9e0838b3 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileScanStatus.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/api/file/FileScanStatus.java @@ -4,5 +4,6 @@ package cn.iocoder.yudao.module.infra.api.file; public enum FileScanStatus { CLEAN, INFECTED, + ERROR, UNAVAILABLE } diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/dal/mysql/file/FileMapper.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/dal/mysql/file/FileMapper.java index b64df52e..4bbc9521 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/dal/mysql/file/FileMapper.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/dal/mysql/file/FileMapper.java @@ -23,6 +23,12 @@ public interface FileMapper extends BaseMapperX { .orderByDesc(FileDO::getId)); } + default FileDO selectLatestByUrl(String url) { + return selectLastOne(new LambdaQueryWrapperX() + .eq(FileDO::getUrl, url) + .orderByAsc(FileDO::getId)); + } + default FileDO selectLatestByConfigIdAndPath(Long configId, String path) { return selectLastOne(new LambdaQueryWrapperX() .eq(FileDO::getConfigId, configId) diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/config/FileScanProperties.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/config/FileScanProperties.java new file mode 100644 index 00000000..3b7b99b4 --- /dev/null +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/config/FileScanProperties.java @@ -0,0 +1,29 @@ +package cn.iocoder.yudao.module.infra.framework.file.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.time.Duration; + +@ConfigurationProperties(prefix = "yudao.infra.file-scan") +public class FileScanProperties { + + private boolean enabled; + private String host = "127.0.0.1"; + private int port = 3310; + private Duration connectTimeout = Duration.ofSeconds(3); + private Duration readTimeout = Duration.ofSeconds(30); + private int chunkSize = 8192; + + public boolean isEnabled() { return enabled; } + public void setEnabled(boolean enabled) { this.enabled = enabled; } + public String getHost() { return host; } + public void setHost(String host) { this.host = host; } + public int getPort() { return port; } + public void setPort(int port) { this.port = port; } + public Duration getConnectTimeout() { return connectTimeout; } + public void setConnectTimeout(Duration connectTimeout) { this.connectTimeout = connectTimeout; } + public Duration getReadTimeout() { return readTimeout; } + public void setReadTimeout(Duration readTimeout) { this.readTimeout = readTimeout; } + public int getChunkSize() { return chunkSize; } + public void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; } +} diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/config/YudaoFileAutoConfiguration.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/config/YudaoFileAutoConfiguration.java index 8e3f506f..0d34e8d7 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/config/YudaoFileAutoConfiguration.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/config/YudaoFileAutoConfiguration.java @@ -2,6 +2,8 @@ package cn.iocoder.yudao.module.infra.framework.file.config; import cn.iocoder.yudao.module.infra.framework.file.core.client.FileClientFactory; import cn.iocoder.yudao.module.infra.framework.file.core.client.FileClientFactoryImpl; +import cn.iocoder.yudao.module.infra.framework.file.core.scan.ClamAvFileScanner; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -11,6 +13,7 @@ import org.springframework.context.annotation.Configuration; * @author 恭学教育 */ @Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties(FileScanProperties.class) public class YudaoFileAutoConfiguration { @Bean @@ -18,4 +21,9 @@ public class YudaoFileAutoConfiguration { return new FileClientFactoryImpl(); } + @Bean + public ClamAvFileScanner clamAvFileScanner(FileScanProperties properties) { + return new ClamAvFileScanner(properties); + } + } diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/core/scan/ClamAvFileScanner.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/core/scan/ClamAvFileScanner.java new file mode 100644 index 00000000..571a96de --- /dev/null +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/core/scan/ClamAvFileScanner.java @@ -0,0 +1,50 @@ +package cn.iocoder.yudao.module.infra.framework.file.core.scan; + +import cn.iocoder.yudao.module.infra.api.file.FileScanStatus; +import cn.iocoder.yudao.module.infra.framework.file.config.FileScanProperties; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.charset.StandardCharsets; + +public class ClamAvFileScanner { + + private final FileScanProperties properties; + + public ClamAvFileScanner(FileScanProperties properties) { + this.properties = properties; + } + + public FileScanStatus scan(byte[] content) { + if (!properties.isEnabled()) return FileScanStatus.UNAVAILABLE; + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(properties.getHost(), properties.getPort()), + Math.toIntExact(properties.getConnectTimeout().toMillis())); + socket.setSoTimeout(Math.toIntExact(properties.getReadTimeout().toMillis())); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + output.write("zINSTREAM\0".getBytes(StandardCharsets.US_ASCII)); + for (int offset = 0; offset < content.length; offset += properties.getChunkSize()) { + int length = Math.min(properties.getChunkSize(), content.length - offset); + output.writeInt(length); + output.write(content, offset, length); + } + output.writeInt(0); + output.flush(); + String response = readResponse(socket.getInputStream()); + if (response.endsWith(" OK")) return FileScanStatus.CLEAN; + if (response.endsWith(" FOUND")) return FileScanStatus.INFECTED; + return FileScanStatus.ERROR; + } catch (Exception ex) { + return FileScanStatus.UNAVAILABLE; + } + } + + private String readResponse(InputStream input) throws Exception { + ByteArrayOutputStream response = new ByteArrayOutputStream(); + for (int value; (value = input.read()) != -1 && value != 0 && value != '\n';) response.write(value); + return response.toString(StandardCharsets.US_ASCII); + } +} diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/file/FileService.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/file/FileService.java index 23603d35..53cfc8ea 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/file/FileService.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/file/FileService.java @@ -1,6 +1,8 @@ package cn.iocoder.yudao.module.infra.service.file; import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.module.infra.api.file.FileDescriptor; +import cn.iocoder.yudao.module.infra.api.file.FileScanStatus; import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FileCreateReqVO; import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePageReqVO; import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePresignedUrlRespVO; @@ -36,6 +38,8 @@ public interface FileService { String createFile(@NotEmpty(message = "文件内容不能为空") byte[] content, String name, String directory, String type); + FileScanStatus scan(FileDescriptor file); + /** * 生成文件预签名地址信息，用于上传 * diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/file/FileServiceImpl.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/file/FileServiceImpl.java index 3ff0777a..d2585ee5 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/file/FileServiceImpl.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/file/FileServiceImpl.java @@ -9,12 +9,15 @@ import cn.hutool.crypto.digest.DigestUtil; import cn.iocoder.yudao.framework.common.pojo.PageResult; import cn.iocoder.yudao.framework.common.util.http.HttpUtils; import cn.iocoder.yudao.framework.common.util.object.BeanUtils; +import cn.iocoder.yudao.module.infra.api.file.FileDescriptor; +import cn.iocoder.yudao.module.infra.api.file.FileScanStatus; import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FileCreateReqVO; import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePageReqVO; import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePresignedUrlRespVO; import cn.iocoder.yudao.module.infra.dal.dataobject.file.FileDO; import cn.iocoder.yudao.module.infra.dal.mysql.file.FileMapper; import cn.iocoder.yudao.module.infra.framework.file.core.client.FileClient; +import cn.iocoder.yudao.module.infra.framework.file.core.scan.ClamAvFileScanner; import cn.iocoder.yudao.module.infra.framework.file.core.utils.FilePathUtils; import cn.iocoder.yudao.module.infra.framework.file.core.utils.FileTypeUtils; import com.google.common.annotations.VisibleForTesting; @@ -22,6 +25,7 @@ import jakarta.annotation.Resource; import lombok.SneakyThrows; import org.springframework.stereotype.Service; +import java.security.MessageDigest; import java.util.List; import static cn.hutool.core.date.DatePattern.PURE_DATE_PATTERN; @@ -63,6 +67,9 @@ public class FileServiceImpl implements FileService { @Resource private FileMapper fileMapper; + @Resource + private ClamAvFileScanner fileScanner; + @Override public PageResult getFilePage(FilePageReqVO pageReqVO) { return fileMapper.selectPage(pageReqVO); @@ -104,6 +111,29 @@ public class FileServiceImpl implements FileService { return url; } + @Override + public FileScanStatus scan(FileDescriptor descriptor) { + FileDO file = fileMapper.selectLatestByUrl(descriptor.reference()); + if (file == null || !descriptor.reference().equals(file.getUrl())) return FileScanStatus.ERROR; + FileClient client = fileConfigService.getFileClient(file.getConfigId()); + if (client == null) return FileScanStatus.UNAVAILABLE; + byte[] content; + try { + content = client.getContent(file.getPath()); + } catch (Exception ex) { + return FileScanStatus.UNAVAILABLE; + } + if (content.length != descriptor.size() || file.getSize() == null || content.length != file.getSize()) { + return FileScanStatus.ERROR; + } + if (descriptor.checksumSha256() != null && !MessageDigest.isEqual( + DigestUtil.sha256Hex(content).getBytes(java.nio.charset.StandardCharsets.US_ASCII), + descriptor.checksumSha256().toLowerCase().getBytes(java.nio.charset.StandardCharsets.US_ASCII))) { + return FileScanStatus.ERROR; + } + return fileScanner.scan(content); + } + @VisibleForTesting String generateUploadPath(String name, String directory) { // 1.1 处理 name 和 directory 的合法性 diff --git a/yudao-module-infra/src/test/java/cn/iocoder/yudao/module/infra/api/file/FileApiImplTest.java b/yudao-module-infra/src/test/java/cn/iocoder/yudao/module/infra/api/file/FileApiImplTest.java index 753054de..1d7425b8 100644 --- a/yudao-module-infra/src/test/java/cn/iocoder/yudao/module/infra/api/file/FileApiImplTest.java +++ b/yudao-module-infra/src/test/java/cn/iocoder/yudao/module/infra/api/file/FileApiImplTest.java @@ -25,15 +25,20 @@ class FileApiImplTest { assertEquals("questions.csv", descriptor.name()); assertEquals("text/csv", descriptor.contentType()); assertEquals(3, descriptor.size()); + assertEquals("039058c6f2c0cb492c533b0a4d14ef77cc0f78abccced5287d84a1a2011cfb81", + descriptor.checksumSha256()); verify(fileService).createFile(new byte[]{1, 2, 3}, "questions.csv", "education/imports", "text/csv"); } @Test - void scannerUnavailableNeverReportsClean() { - FileApiImpl api = new FileApiImpl(mock(FileService.class)); + void scannerDelegatesToFileService() { + FileService fileService = mock(FileService.class); + FileDescriptor descriptor = new FileDescriptor("opaque", "questions.csv", "text/csv", 3); + when(fileService.scan(descriptor)).thenReturn(FileScanStatus.CLEAN); + FileApiImpl api = new FileApiImpl(fileService); - assertEquals(FileScanStatus.UNAVAILABLE, - api.scan(new FileDescriptor("opaque", "questions.csv", "text/csv", 3))); + assertEquals(FileScanStatus.CLEAN, api.scan(descriptor)); + verify(fileService).scan(descriptor); } @Test diff --git a/yudao-module-infra/src/test/java/cn/iocoder/yudao/module/infra/framework/file/core/scan/ClamAvFileScannerTest.java b/yudao-module-infra/src/test/java/cn/iocoder/yudao/module/infra/framework/file/core/scan/ClamAvFileScannerTest.java new file mode 100644 index 00000000..3b0d41fd --- /dev/null +++ b/yudao-module-infra/src/test/java/cn/iocoder/yudao/module/infra/framework/file/core/scan/ClamAvFileScannerTest.java @@ -0,0 +1,69 @@ +package cn.iocoder.yudao.module.infra.framework.file.core.scan; + +import cn.iocoder.yudao.module.infra.api.file.FileScanStatus; +import cn.iocoder.yudao.module.infra.framework.file.config.FileScanProperties; +import org.junit.jupiter.api.Test; + +import java.io.DataInputStream; +import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CompletableFuture; + +import static org.junit.jupiter.api.Assertions.*; + +class ClamAvFileScannerTest { + + @Test + void sendsClamdInstreamProtocol() throws Exception { + try (ServerSocket server = new ServerSocket(0)) { + CompletableFuture received = CompletableFuture.supplyAsync(() -> { + try (var socket = server.accept()) { + DataInputStream input = new DataInputStream(socket.getInputStream()); + assertArrayEquals("zINSTREAM\0".getBytes(StandardCharsets.US_ASCII), input.readNBytes(10)); + int length = input.readInt(); + byte[] content = input.readNBytes(length); + assertEquals(0, input.readInt()); + socket.getOutputStream().write("stream: OK\0".getBytes(StandardCharsets.US_ASCII)); + return content; + } catch (Exception ex) { + throw new RuntimeException(ex); + } + }); + FileScanProperties properties = properties(server.getLocalPort()); + + assertEquals(FileScanStatus.CLEAN, new ClamAvFileScanner(properties).scan(new byte[]{1, 2, 3})); + assertArrayEquals(new byte[]{1, 2, 3}, received.get()); + } + } + + @Test + void mapsFoundErrorAndDisabledFailClosed() throws Exception { + assertEquals(FileScanStatus.UNAVAILABLE, new ClamAvFileScanner(new FileScanProperties()).scan(new byte[]{1})); + assertEquals(FileScanStatus.INFECTED, scanResponse("stream: Eicar-Signature FOUND\0")); + assertEquals(FileScanStatus.ERROR, scanResponse("stream: size limit exceeded ERROR\0")); + } + + private FileScanStatus scanResponse(String response) throws Exception { + try (ServerSocket server = new ServerSocket(0)) { + CompletableFuture.runAsync(() -> { + try (var socket = server.accept()) { + DataInputStream input = new DataInputStream(socket.getInputStream()); + input.readNBytes(10); + for (int length; (length = input.readInt()) != 0;) input.readNBytes(length); + socket.getOutputStream().write(response.getBytes(StandardCharsets.US_ASCII)); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + }); + return new ClamAvFileScanner(properties(server.getLocalPort())).scan(new byte[]{1}); + } + } + + private FileScanProperties properties(int port) { + FileScanProperties properties = new FileScanProperties(); + properties.setEnabled(true); + properties.setHost("127.0.0.1"); + properties.setPort(port); + return properties; + } +} diff --git a/yudao-module-infra/src/test/java/cn/iocoder/yudao/module/infra/service/file/FileServiceImplTest.java b/yudao-module-infra/src/test/java/cn/iocoder/yudao/module/infra/service/file/FileServiceImplTest.java index 4b14aec0..5c169a77 100644 --- a/yudao-module-infra/src/test/java/cn/iocoder/yudao/module/infra/service/file/FileServiceImplTest.java +++ b/yudao-module-infra/src/test/java/cn/iocoder/yudao/module/infra/service/file/FileServiceImplTest.java @@ -5,11 +5,14 @@ import cn.iocoder.yudao.framework.common.pojo.PageResult; import cn.iocoder.yudao.framework.common.util.object.ObjectUtils; import cn.iocoder.yudao.framework.test.core.ut.BaseDbUnitTest; import cn.iocoder.yudao.framework.test.core.util.AssertUtils; +import cn.iocoder.yudao.module.infra.api.file.FileDescriptor; +import cn.iocoder.yudao.module.infra.api.file.FileScanStatus; import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FileCreateReqVO; import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePageReqVO; import cn.iocoder.yudao.module.infra.dal.dataobject.file.FileDO; import cn.iocoder.yudao.module.infra.dal.mysql.file.FileMapper; import cn.iocoder.yudao.module.infra.framework.file.core.client.FileClient; +import cn.iocoder.yudao.module.infra.framework.file.core.scan.ClamAvFileScanner; import jakarta.annotation.Resource; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -40,6 +43,9 @@ public class FileServiceImplTest extends BaseDbUnitTest { @MockitoBean private FileConfigService fileConfigService; + @MockitoBean + private ClamAvFileScanner fileScanner; + @BeforeEach public void setUp() { FileServiceImpl.PATH_PREFIX_DATE_ENABLE = true; @@ -146,6 +152,43 @@ public class FileServiceImplTest extends BaseDbUnitTest { assertEquals(content.length, file.getSize()); } + @Test + public void scanUsesOnlyRegisteredFileClientAndValidatesIntegrity() throws Exception { + byte[] content = {1, 2, 3}; + FileDO file = new FileDO().setConfigId(10L).setPath("safe/file.csv") + .setUrl("https://storage.example/file.csv").setSize(3L); + fileMapper.insert(file); + FileClient client = mock(FileClient.class); + when(fileConfigService.getFileClient(10L)).thenReturn(client); + when(client.getContent("safe/file.csv")).thenReturn(content); + when(fileScanner.scan(content)).thenReturn(FileScanStatus.CLEAN); + + FileScanStatus result = fileService.scan(new FileDescriptor(file.getUrl(), "file.csv", "text/csv", 3, + "039058c6f2c0cb492c533b0a4d14ef77cc0f78abccced5287d84a1a2011cfb81")); + + assertEquals(FileScanStatus.CLEAN, result); + verify(client).getContent("safe/file.csv"); + verify(fileScanner).scan(content); + } + + @Test + public void scanRejectsArbitraryUrlAndIntegrityMismatch() throws Exception { + assertEquals(FileScanStatus.ERROR, fileService.scan( + new FileDescriptor("http://127.0.0.1:8080/admin", "x", null, 1))); + verifyNoInteractions(fileConfigService, fileScanner); + + FileDO file = new FileDO().setConfigId(10L).setPath("safe/file.csv") + .setUrl("https://storage.example/file.csv").setSize(3L); + fileMapper.insert(file); + FileClient client = mock(FileClient.class); + when(fileConfigService.getFileClient(10L)).thenReturn(client); + when(client.getContent("safe/file.csv")).thenReturn(new byte[]{1, 2}); + + assertEquals(FileScanStatus.ERROR, fileService.scan( + new FileDescriptor(file.getUrl(), "file.csv", "text/csv", 3))); + verifyNoInteractions(fileScanner); + } + @Test public void testDeleteFile_success() throws Exception { // mock 数据 diff --git a/yudao-server/src/main/resources/application.yaml b/yudao-server/src/main/resources/application.yaml index 364badbc..7fd24eb6 100644 --- a/yudao-server/src/main/resources/application.yaml +++ b/yudao-server/src/main/resources/application.yaml @@ -260,6 +260,14 @@ yudao: --- #################### 项目相关配置 #################### yudao: + infra: + file-scan: + enabled: ${FILE_SCAN_ENABLED:false} + host: ${FILE_SCAN_HOST:127.0.0.1} + port: ${FILE_SCAN_PORT:3310} + connect-timeout: ${FILE_SCAN_CONNECT_TIMEOUT:3s} + read-timeout: ${FILE_SCAN_READ_TIMEOUT:30s} + chunk-size: ${FILE_SCAN_CHUNK_SIZE:8192} info: version: 1.0.0 base-package: cn.iocoder.yudao