forked from wangziqi/ruoyi-vue-pro
feat(education): add production file scanning
This commit is contained in:
@@ -63,7 +63,7 @@ public class EducationAssetAdmissionServiceImpl implements EducationAssetAdmissi
|
|||||||
asset.setOriginalName(name);
|
asset.setOriginalName(name);
|
||||||
asset.setContentType(contentType);
|
asset.setContentType(contentType);
|
||||||
asset.setSize((long) bytes.length);
|
asset.setSize((long) bytes.length);
|
||||||
asset.setSha256(DigestUtil.sha256Hex(bytes));
|
asset.setSha256(descriptor.checksumSha256() != null ? descriptor.checksumSha256() : DigestUtil.sha256Hex(bytes));
|
||||||
asset.setFileReference(descriptor.reference());
|
asset.setFileReference(descriptor.reference());
|
||||||
asset.setScanStatus(scanStatus.name());
|
asset.setScanStatus(scanStatus.name());
|
||||||
assetMapper.insert(asset);
|
assetMapper.insert(asset);
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
package cn.iocoder.yudao.module.education.service.importjob;
|
package cn.iocoder.yudao.module.education.service.importjob;
|
||||||
|
|
||||||
public interface ImportObjectScanGateway {
|
public interface ImportObjectScanGateway {
|
||||||
enum ScanResult { CLEAN, INFECTED, ERROR }
|
enum ScanResult { CLEAN, INFECTED, ERROR, UNAVAILABLE }
|
||||||
ScanResult scan(String objectKey);
|
ScanResult scan(String reference, String name, String mimeType, long size, String checksumSha256);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package cn.iocoder.yudao.module.education.service.importjob;
|
||||||
|
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileApi;
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileDescriptor;
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileScanStatus;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
public class InfraFileImportObjectScanGateway implements ImportObjectScanGateway {
|
||||||
|
|
||||||
|
private final FileApi fileApi;
|
||||||
|
|
||||||
|
public InfraFileImportObjectScanGateway(FileApi fileApi) {
|
||||||
|
this.fileApi = fileApi;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ScanResult scan(String reference, String name, String mimeType, long size, String checksumSha256) {
|
||||||
|
FileScanStatus status;
|
||||||
|
try {
|
||||||
|
status = fileApi.scan(new FileDescriptor(reference, name, mimeType, size, checksumSha256));
|
||||||
|
} catch (RuntimeException ex) {
|
||||||
|
return ScanResult.ERROR;
|
||||||
|
}
|
||||||
|
return status == null ? ScanResult.UNAVAILABLE : ScanResult.valueOf(status.name());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -133,7 +133,8 @@ public class QuestionImportJobServiceImpl implements QuestionImportJobService {
|
|||||||
finishFailed(job, token, "SCAN_UNAVAILABLE");
|
finishFailed(job, token, "SCAN_UNAVAILABLE");
|
||||||
throw exception(QUESTION_IMPORT_SCAN_NOT_CLEAN);
|
throw exception(QUESTION_IMPORT_SCAN_NOT_CLEAN);
|
||||||
}
|
}
|
||||||
ImportObjectScanGateway.ScanResult scanResult = gateway.scan(asset.getObjectKey());
|
ImportObjectScanGateway.ScanResult scanResult = gateway.scan(asset.getObjectKey(), asset.getFileName(),
|
||||||
|
asset.getMimeType(), asset.getFileSizeBytes(), asset.getChecksumSha256());
|
||||||
if (scanResult != ImportObjectScanGateway.ScanResult.CLEAN) {
|
if (scanResult != ImportObjectScanGateway.ScanResult.CLEAN) {
|
||||||
finishFailed(job, token, "SCAN_" + scanResult.name());
|
finishFailed(job, token, "SCAN_" + scanResult.name());
|
||||||
throw exception(QUESTION_IMPORT_SCAN_NOT_CLEAN);
|
throw exception(QUESTION_IMPORT_SCAN_NOT_CLEAN);
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package cn.iocoder.yudao.module.education.service.importjob;
|
||||||
|
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileApi;
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileDescriptor;
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileScanStatus;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.mockito.Mockito.*;
|
||||||
|
|
||||||
|
class InfraFileImportObjectScanGatewayTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void passesCompleteDescriptorAndMapsEveryStatus() {
|
||||||
|
FileApi fileApi = mock(FileApi.class);
|
||||||
|
InfraFileImportObjectScanGateway gateway = new InfraFileImportObjectScanGateway(fileApi);
|
||||||
|
FileDescriptor descriptor = new FileDescriptor("ref", "questions.csv", "text/csv", 3,
|
||||||
|
"039058c6f2c0cb492c533b0a4d14ef77cc0f78abccced5287d84a1a2011cfb81");
|
||||||
|
for (FileScanStatus status : FileScanStatus.values()) {
|
||||||
|
when(fileApi.scan(descriptor)).thenReturn(status);
|
||||||
|
assertEquals(ImportObjectScanGateway.ScanResult.valueOf(status.name()),
|
||||||
|
gateway.scan(descriptor.reference(), descriptor.name(), descriptor.contentType(),
|
||||||
|
descriptor.size(), descriptor.checksumSha256()));
|
||||||
|
}
|
||||||
|
verify(fileApi, times(FileScanStatus.values().length)).scan(descriptor);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void nullAndExceptionsFailClosed() {
|
||||||
|
FileApi fileApi = mock(FileApi.class);
|
||||||
|
InfraFileImportObjectScanGateway gateway = new InfraFileImportObjectScanGateway(fileApi);
|
||||||
|
when(fileApi.scan(any())).thenReturn(null).thenThrow(new IllegalStateException("down"));
|
||||||
|
|
||||||
|
assertEquals(ImportObjectScanGateway.ScanResult.UNAVAILABLE,
|
||||||
|
gateway.scan("ref", "x", null, 1, null));
|
||||||
|
assertEquals(ImportObjectScanGateway.ScanResult.ERROR,
|
||||||
|
gateway.scan("ref", "x", null, 1, null));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -79,7 +79,8 @@ class QuestionImportJobServiceImplTest {
|
|||||||
when(jobMapper.claimById(eq(101L), eq("PREVIEW_PENDING"), anyString(), anyString(), anyLong()))
|
when(jobMapper.claimById(eq(101L), eq("PREVIEW_PENDING"), anyString(), anyString(), anyLong()))
|
||||||
.thenReturn(claimed);
|
.thenReturn(claimed);
|
||||||
when(assetMapper.selectTenantAsset(10L, 88L)).thenReturn(asset(88L));
|
when(assetMapper.selectTenantAsset(10L, 88L)).thenReturn(asset(88L));
|
||||||
when(scanGateway.scan("object/10/questions.csv")).thenReturn(ImportObjectScanGateway.ScanResult.CLEAN);
|
when(scanGateway.scan("object/10/questions.csv", "questions.csv", "text/csv", 128L, "checksum"))
|
||||||
|
.thenReturn(ImportObjectScanGateway.ScanResult.CLEAN);
|
||||||
when(jobMapper.finishClaim(eq(10L), eq(101L), eq("PREVIEW_PENDING"), eq("PREVIEW_READY"),
|
when(jobMapper.finishClaim(eq(10L), eq(101L), eq("PREVIEW_PENDING"), eq("PREVIEW_READY"),
|
||||||
anyString(), eq("CLEAN"), eq("UNAVAILABLE"), contains("METADATA_ONLY"), isNull(), isNull(),
|
anyString(), eq("CLEAN"), eq("UNAVAILABLE"), contains("METADATA_ONLY"), isNull(), isNull(),
|
||||||
isNull(), isNull(), isNull(), isNull())).thenReturn(1);
|
isNull(), isNull(), isNull(), isNull())).thenReturn(1);
|
||||||
@@ -106,6 +107,28 @@ class QuestionImportJobServiceImplTest {
|
|||||||
verifyNoInteractions(parser, lifecycleService);
|
verifyNoInteractions(parser, lifecycleService);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void nonCleanStatusesHaveStableFailureCodesAndNeverParse() {
|
||||||
|
for (ImportObjectScanGateway.ScanResult result : List.of(
|
||||||
|
ImportObjectScanGateway.ScanResult.INFECTED,
|
||||||
|
ImportObjectScanGateway.ScanResult.ERROR,
|
||||||
|
ImportObjectScanGateway.ScanResult.UNAVAILABLE)) {
|
||||||
|
reset(jobMapper, assetMapper, scanGateway, parser, lifecycleService);
|
||||||
|
when(jobMapper.claimById(eq(101L), eq("PREVIEW_PENDING"), anyString(), anyString(), anyLong()))
|
||||||
|
.thenReturn(job(101L, "PREVIEW_PENDING"));
|
||||||
|
when(assetMapper.selectTenantAsset(10L, 88L)).thenReturn(asset(88L));
|
||||||
|
when(scanGateway.scan("object/10/questions.csv", "questions.csv", "text/csv", 128L, "checksum"))
|
||||||
|
.thenReturn(result);
|
||||||
|
|
||||||
|
assertThrows(ServiceException.class, () -> service.preview(101L));
|
||||||
|
|
||||||
|
verify(jobMapper).finishClaim(eq(10L), eq(101L), eq("PREVIEW_PENDING"), eq("FAILED"),
|
||||||
|
anyString(), isNull(), isNull(), isNull(), isNull(), isNull(), isNull(), isNull(),
|
||||||
|
eq("SCAN_" + result.name()), eq("SCAN_" + result.name()));
|
||||||
|
verifyNoInteractions(parser, lifecycleService);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void executePropagatesClaimedTenantAndCreatesDrafts() {
|
void executePropagatesClaimedTenantAndCreatesDrafts() {
|
||||||
TenantContextHolder.clear();
|
TenantContextHolder.clear();
|
||||||
@@ -131,7 +154,7 @@ class QuestionImportJobServiceImplTest {
|
|||||||
private ContentImportAssetDO asset(Long id) {
|
private ContentImportAssetDO asset(Long id) {
|
||||||
ContentImportAssetDO asset = new ContentImportAssetDO(); asset.setId(id); asset.setTenantId(10L);
|
ContentImportAssetDO asset = new ContentImportAssetDO(); asset.setId(id); asset.setTenantId(10L);
|
||||||
asset.setObjectKey("object/10/questions.csv"); asset.setFileName("questions.csv");
|
asset.setObjectKey("object/10/questions.csv"); asset.setFileName("questions.csv");
|
||||||
asset.setMimeType("text/csv"); asset.setFileSizeBytes(128L); return asset;
|
asset.setMimeType("text/csv"); asset.setFileSizeBytes(128L); asset.setChecksumSha256("checksum"); return asset;
|
||||||
}
|
}
|
||||||
|
|
||||||
private ContentImportJobDO job(Long id, String status) {
|
private ContentImportJobDO job(Long id, String status) {
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
package cn.iocoder.yudao.module.infra.api.file;
|
package cn.iocoder.yudao.module.infra.api.file;
|
||||||
|
|
||||||
|
import cn.hutool.crypto.digest.DigestUtil;
|
||||||
import jakarta.validation.constraints.NotEmpty;
|
import jakarta.validation.constraints.NotEmpty;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -46,8 +47,10 @@ public interface FileApi {
|
|||||||
* 保存有界文件内容,返回不泄露存储实现的公开描述。
|
* 保存有界文件内容,返回不泄露存储实现的公开描述。
|
||||||
*/
|
*/
|
||||||
default FileDescriptor createFile(FileContent content) {
|
default FileDescriptor createFile(FileContent content) {
|
||||||
String reference = createFile(content.bytes(), content.name(), content.directory(), content.contentType());
|
byte[] bytes = content.bytes();
|
||||||
return new FileDescriptor(reference, content.name(), content.contentType(), content.bytes().length);
|
String reference = createFile(bytes, content.name(), content.directory(), content.contentType());
|
||||||
|
return new FileDescriptor(reference, content.name(), content.contentType(), bytes.length,
|
||||||
|
DigestUtil.sha256Hex(bytes));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -26,6 +26,11 @@ public class FileApiImpl implements FileApi {
|
|||||||
return fileService.createFile(content, name, directory, type);
|
return fileService.createFile(content, name, directory, type);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public FileScanStatus scan(FileDescriptor file) {
|
||||||
|
return fileService.scan(file);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String presignGetUrl(String url, Integer expirationSeconds) {
|
public String presignGetUrl(String url, Integer expirationSeconds) {
|
||||||
return fileService.presignGetUrl(url, expirationSeconds);
|
return fileService.presignGetUrl(url, expirationSeconds);
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
package cn.iocoder.yudao.module.infra.api.file;
|
package cn.iocoder.yudao.module.infra.api.file;
|
||||||
|
|
||||||
/** Public, storage-agnostic description of an accepted file. */
|
/** Public, storage-agnostic description of an accepted file. */
|
||||||
public record FileDescriptor(String reference, String name, String contentType, long size) {
|
public record FileDescriptor(String reference, String name, String contentType, long size, String checksumSha256) {
|
||||||
|
|
||||||
|
public FileDescriptor(String reference, String name, String contentType, long size) {
|
||||||
|
this(reference, name, contentType, size, null);
|
||||||
|
}
|
||||||
|
|
||||||
public FileDescriptor {
|
public FileDescriptor {
|
||||||
if (reference == null || reference.isBlank()) {
|
if (reference == null || reference.isBlank()) {
|
||||||
@@ -10,5 +14,8 @@ public record FileDescriptor(String reference, String name, String contentType,
|
|||||||
if (size < 0 || size > FileContent.MAX_CONTENT_BYTES) {
|
if (size < 0 || size > FileContent.MAX_CONTENT_BYTES) {
|
||||||
throw new IllegalArgumentException("file size is outside public contract bounds");
|
throw new IllegalArgumentException("file size is outside public contract bounds");
|
||||||
}
|
}
|
||||||
|
if (checksumSha256 != null && !checksumSha256.matches("(?i)[0-9a-f]{64}")) {
|
||||||
|
throw new IllegalArgumentException("file SHA-256 checksum is invalid");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,5 +4,6 @@ package cn.iocoder.yudao.module.infra.api.file;
|
|||||||
public enum FileScanStatus {
|
public enum FileScanStatus {
|
||||||
CLEAN,
|
CLEAN,
|
||||||
INFECTED,
|
INFECTED,
|
||||||
|
ERROR,
|
||||||
UNAVAILABLE
|
UNAVAILABLE
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,6 +23,12 @@ public interface FileMapper extends BaseMapperX<FileDO> {
|
|||||||
.orderByDesc(FileDO::getId));
|
.orderByDesc(FileDO::getId));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
default FileDO selectLatestByUrl(String url) {
|
||||||
|
return selectLastOne(new LambdaQueryWrapperX<FileDO>()
|
||||||
|
.eq(FileDO::getUrl, url)
|
||||||
|
.orderByAsc(FileDO::getId));
|
||||||
|
}
|
||||||
|
|
||||||
default FileDO selectLatestByConfigIdAndPath(Long configId, String path) {
|
default FileDO selectLatestByConfigIdAndPath(Long configId, String path) {
|
||||||
return selectLastOne(new LambdaQueryWrapperX<FileDO>()
|
return selectLastOne(new LambdaQueryWrapperX<FileDO>()
|
||||||
.eq(FileDO::getConfigId, configId)
|
.eq(FileDO::getConfigId, configId)
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package cn.iocoder.yudao.module.infra.framework.file.config;
|
||||||
|
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
|
||||||
|
@ConfigurationProperties(prefix = "yudao.infra.file-scan")
|
||||||
|
public class FileScanProperties {
|
||||||
|
|
||||||
|
private boolean enabled;
|
||||||
|
private String host = "127.0.0.1";
|
||||||
|
private int port = 3310;
|
||||||
|
private Duration connectTimeout = Duration.ofSeconds(3);
|
||||||
|
private Duration readTimeout = Duration.ofSeconds(30);
|
||||||
|
private int chunkSize = 8192;
|
||||||
|
|
||||||
|
public boolean isEnabled() { return enabled; }
|
||||||
|
public void setEnabled(boolean enabled) { this.enabled = enabled; }
|
||||||
|
public String getHost() { return host; }
|
||||||
|
public void setHost(String host) { this.host = host; }
|
||||||
|
public int getPort() { return port; }
|
||||||
|
public void setPort(int port) { this.port = port; }
|
||||||
|
public Duration getConnectTimeout() { return connectTimeout; }
|
||||||
|
public void setConnectTimeout(Duration connectTimeout) { this.connectTimeout = connectTimeout; }
|
||||||
|
public Duration getReadTimeout() { return readTimeout; }
|
||||||
|
public void setReadTimeout(Duration readTimeout) { this.readTimeout = readTimeout; }
|
||||||
|
public int getChunkSize() { return chunkSize; }
|
||||||
|
public void setChunkSize(int chunkSize) { this.chunkSize = chunkSize; }
|
||||||
|
}
|
||||||
@@ -2,6 +2,8 @@ package cn.iocoder.yudao.module.infra.framework.file.config;
|
|||||||
|
|
||||||
import cn.iocoder.yudao.module.infra.framework.file.core.client.FileClientFactory;
|
import cn.iocoder.yudao.module.infra.framework.file.core.client.FileClientFactory;
|
||||||
import cn.iocoder.yudao.module.infra.framework.file.core.client.FileClientFactoryImpl;
|
import cn.iocoder.yudao.module.infra.framework.file.core.client.FileClientFactoryImpl;
|
||||||
|
import cn.iocoder.yudao.module.infra.framework.file.core.scan.ClamAvFileScanner;
|
||||||
|
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||||
import org.springframework.context.annotation.Bean;
|
import org.springframework.context.annotation.Bean;
|
||||||
import org.springframework.context.annotation.Configuration;
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
|
||||||
@@ -11,6 +13,7 @@ import org.springframework.context.annotation.Configuration;
|
|||||||
* @author 恭学教育
|
* @author 恭学教育
|
||||||
*/
|
*/
|
||||||
@Configuration(proxyBeanMethods = false)
|
@Configuration(proxyBeanMethods = false)
|
||||||
|
@EnableConfigurationProperties(FileScanProperties.class)
|
||||||
public class YudaoFileAutoConfiguration {
|
public class YudaoFileAutoConfiguration {
|
||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
@@ -18,4 +21,9 @@ public class YudaoFileAutoConfiguration {
|
|||||||
return new FileClientFactoryImpl();
|
return new FileClientFactoryImpl();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public ClamAvFileScanner clamAvFileScanner(FileScanProperties properties) {
|
||||||
|
return new ClamAvFileScanner(properties);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
package cn.iocoder.yudao.module.infra.framework.file.core.scan;
|
||||||
|
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileScanStatus;
|
||||||
|
import cn.iocoder.yudao.module.infra.framework.file.config.FileScanProperties;
|
||||||
|
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.io.DataOutputStream;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
import java.net.Socket;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
|
||||||
|
public class ClamAvFileScanner {
|
||||||
|
|
||||||
|
private final FileScanProperties properties;
|
||||||
|
|
||||||
|
public ClamAvFileScanner(FileScanProperties properties) {
|
||||||
|
this.properties = properties;
|
||||||
|
}
|
||||||
|
|
||||||
|
public FileScanStatus scan(byte[] content) {
|
||||||
|
if (!properties.isEnabled()) return FileScanStatus.UNAVAILABLE;
|
||||||
|
try (Socket socket = new Socket()) {
|
||||||
|
socket.connect(new InetSocketAddress(properties.getHost(), properties.getPort()),
|
||||||
|
Math.toIntExact(properties.getConnectTimeout().toMillis()));
|
||||||
|
socket.setSoTimeout(Math.toIntExact(properties.getReadTimeout().toMillis()));
|
||||||
|
DataOutputStream output = new DataOutputStream(socket.getOutputStream());
|
||||||
|
output.write("zINSTREAM\0".getBytes(StandardCharsets.US_ASCII));
|
||||||
|
for (int offset = 0; offset < content.length; offset += properties.getChunkSize()) {
|
||||||
|
int length = Math.min(properties.getChunkSize(), content.length - offset);
|
||||||
|
output.writeInt(length);
|
||||||
|
output.write(content, offset, length);
|
||||||
|
}
|
||||||
|
output.writeInt(0);
|
||||||
|
output.flush();
|
||||||
|
String response = readResponse(socket.getInputStream());
|
||||||
|
if (response.endsWith(" OK")) return FileScanStatus.CLEAN;
|
||||||
|
if (response.endsWith(" FOUND")) return FileScanStatus.INFECTED;
|
||||||
|
return FileScanStatus.ERROR;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
return FileScanStatus.UNAVAILABLE;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String readResponse(InputStream input) throws Exception {
|
||||||
|
ByteArrayOutputStream response = new ByteArrayOutputStream();
|
||||||
|
for (int value; (value = input.read()) != -1 && value != 0 && value != '\n';) response.write(value);
|
||||||
|
return response.toString(StandardCharsets.US_ASCII);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
package cn.iocoder.yudao.module.infra.service.file;
|
package cn.iocoder.yudao.module.infra.service.file;
|
||||||
|
|
||||||
import cn.iocoder.yudao.framework.common.pojo.PageResult;
|
import cn.iocoder.yudao.framework.common.pojo.PageResult;
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileDescriptor;
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileScanStatus;
|
||||||
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FileCreateReqVO;
|
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FileCreateReqVO;
|
||||||
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePageReqVO;
|
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePageReqVO;
|
||||||
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePresignedUrlRespVO;
|
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePresignedUrlRespVO;
|
||||||
@@ -36,6 +38,8 @@ public interface FileService {
|
|||||||
String createFile(@NotEmpty(message = "文件内容不能为空") byte[] content,
|
String createFile(@NotEmpty(message = "文件内容不能为空") byte[] content,
|
||||||
String name, String directory, String type);
|
String name, String directory, String type);
|
||||||
|
|
||||||
|
FileScanStatus scan(FileDescriptor file);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 生成文件预签名地址信息,用于上传
|
* 生成文件预签名地址信息,用于上传
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -9,12 +9,15 @@ import cn.hutool.crypto.digest.DigestUtil;
|
|||||||
import cn.iocoder.yudao.framework.common.pojo.PageResult;
|
import cn.iocoder.yudao.framework.common.pojo.PageResult;
|
||||||
import cn.iocoder.yudao.framework.common.util.http.HttpUtils;
|
import cn.iocoder.yudao.framework.common.util.http.HttpUtils;
|
||||||
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
|
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileDescriptor;
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileScanStatus;
|
||||||
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FileCreateReqVO;
|
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FileCreateReqVO;
|
||||||
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePageReqVO;
|
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePageReqVO;
|
||||||
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePresignedUrlRespVO;
|
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePresignedUrlRespVO;
|
||||||
import cn.iocoder.yudao.module.infra.dal.dataobject.file.FileDO;
|
import cn.iocoder.yudao.module.infra.dal.dataobject.file.FileDO;
|
||||||
import cn.iocoder.yudao.module.infra.dal.mysql.file.FileMapper;
|
import cn.iocoder.yudao.module.infra.dal.mysql.file.FileMapper;
|
||||||
import cn.iocoder.yudao.module.infra.framework.file.core.client.FileClient;
|
import cn.iocoder.yudao.module.infra.framework.file.core.client.FileClient;
|
||||||
|
import cn.iocoder.yudao.module.infra.framework.file.core.scan.ClamAvFileScanner;
|
||||||
import cn.iocoder.yudao.module.infra.framework.file.core.utils.FilePathUtils;
|
import cn.iocoder.yudao.module.infra.framework.file.core.utils.FilePathUtils;
|
||||||
import cn.iocoder.yudao.module.infra.framework.file.core.utils.FileTypeUtils;
|
import cn.iocoder.yudao.module.infra.framework.file.core.utils.FileTypeUtils;
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
@@ -22,6 +25,7 @@ import jakarta.annotation.Resource;
|
|||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.security.MessageDigest;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import static cn.hutool.core.date.DatePattern.PURE_DATE_PATTERN;
|
import static cn.hutool.core.date.DatePattern.PURE_DATE_PATTERN;
|
||||||
@@ -63,6 +67,9 @@ public class FileServiceImpl implements FileService {
|
|||||||
@Resource
|
@Resource
|
||||||
private FileMapper fileMapper;
|
private FileMapper fileMapper;
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private ClamAvFileScanner fileScanner;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public PageResult<FileDO> getFilePage(FilePageReqVO pageReqVO) {
|
public PageResult<FileDO> getFilePage(FilePageReqVO pageReqVO) {
|
||||||
return fileMapper.selectPage(pageReqVO);
|
return fileMapper.selectPage(pageReqVO);
|
||||||
@@ -104,6 +111,29 @@ public class FileServiceImpl implements FileService {
|
|||||||
return url;
|
return url;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public FileScanStatus scan(FileDescriptor descriptor) {
|
||||||
|
FileDO file = fileMapper.selectLatestByUrl(descriptor.reference());
|
||||||
|
if (file == null || !descriptor.reference().equals(file.getUrl())) return FileScanStatus.ERROR;
|
||||||
|
FileClient client = fileConfigService.getFileClient(file.getConfigId());
|
||||||
|
if (client == null) return FileScanStatus.UNAVAILABLE;
|
||||||
|
byte[] content;
|
||||||
|
try {
|
||||||
|
content = client.getContent(file.getPath());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
return FileScanStatus.UNAVAILABLE;
|
||||||
|
}
|
||||||
|
if (content.length != descriptor.size() || file.getSize() == null || content.length != file.getSize()) {
|
||||||
|
return FileScanStatus.ERROR;
|
||||||
|
}
|
||||||
|
if (descriptor.checksumSha256() != null && !MessageDigest.isEqual(
|
||||||
|
DigestUtil.sha256Hex(content).getBytes(java.nio.charset.StandardCharsets.US_ASCII),
|
||||||
|
descriptor.checksumSha256().toLowerCase().getBytes(java.nio.charset.StandardCharsets.US_ASCII))) {
|
||||||
|
return FileScanStatus.ERROR;
|
||||||
|
}
|
||||||
|
return fileScanner.scan(content);
|
||||||
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
String generateUploadPath(String name, String directory) {
|
String generateUploadPath(String name, String directory) {
|
||||||
// 1.1 处理 name 和 directory 的合法性
|
// 1.1 处理 name 和 directory 的合法性
|
||||||
|
|||||||
@@ -25,15 +25,20 @@ class FileApiImplTest {
|
|||||||
assertEquals("questions.csv", descriptor.name());
|
assertEquals("questions.csv", descriptor.name());
|
||||||
assertEquals("text/csv", descriptor.contentType());
|
assertEquals("text/csv", descriptor.contentType());
|
||||||
assertEquals(3, descriptor.size());
|
assertEquals(3, descriptor.size());
|
||||||
|
assertEquals("039058c6f2c0cb492c533b0a4d14ef77cc0f78abccced5287d84a1a2011cfb81",
|
||||||
|
descriptor.checksumSha256());
|
||||||
verify(fileService).createFile(new byte[]{1, 2, 3}, "questions.csv", "education/imports", "text/csv");
|
verify(fileService).createFile(new byte[]{1, 2, 3}, "questions.csv", "education/imports", "text/csv");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void scannerUnavailableNeverReportsClean() {
|
void scannerDelegatesToFileService() {
|
||||||
FileApiImpl api = new FileApiImpl(mock(FileService.class));
|
FileService fileService = mock(FileService.class);
|
||||||
|
FileDescriptor descriptor = new FileDescriptor("opaque", "questions.csv", "text/csv", 3);
|
||||||
|
when(fileService.scan(descriptor)).thenReturn(FileScanStatus.CLEAN);
|
||||||
|
FileApiImpl api = new FileApiImpl(fileService);
|
||||||
|
|
||||||
assertEquals(FileScanStatus.UNAVAILABLE,
|
assertEquals(FileScanStatus.CLEAN, api.scan(descriptor));
|
||||||
api.scan(new FileDescriptor("opaque", "questions.csv", "text/csv", 3)));
|
verify(fileService).scan(descriptor);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package cn.iocoder.yudao.module.infra.framework.file.core.scan;
|
||||||
|
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileScanStatus;
|
||||||
|
import cn.iocoder.yudao.module.infra.framework.file.config.FileScanProperties;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.io.DataInputStream;
|
||||||
|
import java.net.ServerSocket;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.concurrent.CompletableFuture;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
|
||||||
|
class ClamAvFileScannerTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void sendsClamdInstreamProtocol() throws Exception {
|
||||||
|
try (ServerSocket server = new ServerSocket(0)) {
|
||||||
|
CompletableFuture<byte[]> received = CompletableFuture.supplyAsync(() -> {
|
||||||
|
try (var socket = server.accept()) {
|
||||||
|
DataInputStream input = new DataInputStream(socket.getInputStream());
|
||||||
|
assertArrayEquals("zINSTREAM\0".getBytes(StandardCharsets.US_ASCII), input.readNBytes(10));
|
||||||
|
int length = input.readInt();
|
||||||
|
byte[] content = input.readNBytes(length);
|
||||||
|
assertEquals(0, input.readInt());
|
||||||
|
socket.getOutputStream().write("stream: OK\0".getBytes(StandardCharsets.US_ASCII));
|
||||||
|
return content;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new RuntimeException(ex);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
FileScanProperties properties = properties(server.getLocalPort());
|
||||||
|
|
||||||
|
assertEquals(FileScanStatus.CLEAN, new ClamAvFileScanner(properties).scan(new byte[]{1, 2, 3}));
|
||||||
|
assertArrayEquals(new byte[]{1, 2, 3}, received.get());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void mapsFoundErrorAndDisabledFailClosed() throws Exception {
|
||||||
|
assertEquals(FileScanStatus.UNAVAILABLE, new ClamAvFileScanner(new FileScanProperties()).scan(new byte[]{1}));
|
||||||
|
assertEquals(FileScanStatus.INFECTED, scanResponse("stream: Eicar-Signature FOUND\0"));
|
||||||
|
assertEquals(FileScanStatus.ERROR, scanResponse("stream: size limit exceeded ERROR\0"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private FileScanStatus scanResponse(String response) throws Exception {
|
||||||
|
try (ServerSocket server = new ServerSocket(0)) {
|
||||||
|
CompletableFuture.runAsync(() -> {
|
||||||
|
try (var socket = server.accept()) {
|
||||||
|
DataInputStream input = new DataInputStream(socket.getInputStream());
|
||||||
|
input.readNBytes(10);
|
||||||
|
for (int length; (length = input.readInt()) != 0;) input.readNBytes(length);
|
||||||
|
socket.getOutputStream().write(response.getBytes(StandardCharsets.US_ASCII));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new RuntimeException(ex);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return new ClamAvFileScanner(properties(server.getLocalPort())).scan(new byte[]{1});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private FileScanProperties properties(int port) {
|
||||||
|
FileScanProperties properties = new FileScanProperties();
|
||||||
|
properties.setEnabled(true);
|
||||||
|
properties.setHost("127.0.0.1");
|
||||||
|
properties.setPort(port);
|
||||||
|
return properties;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,11 +5,14 @@ import cn.iocoder.yudao.framework.common.pojo.PageResult;
|
|||||||
import cn.iocoder.yudao.framework.common.util.object.ObjectUtils;
|
import cn.iocoder.yudao.framework.common.util.object.ObjectUtils;
|
||||||
import cn.iocoder.yudao.framework.test.core.ut.BaseDbUnitTest;
|
import cn.iocoder.yudao.framework.test.core.ut.BaseDbUnitTest;
|
||||||
import cn.iocoder.yudao.framework.test.core.util.AssertUtils;
|
import cn.iocoder.yudao.framework.test.core.util.AssertUtils;
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileDescriptor;
|
||||||
|
import cn.iocoder.yudao.module.infra.api.file.FileScanStatus;
|
||||||
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FileCreateReqVO;
|
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FileCreateReqVO;
|
||||||
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePageReqVO;
|
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FilePageReqVO;
|
||||||
import cn.iocoder.yudao.module.infra.dal.dataobject.file.FileDO;
|
import cn.iocoder.yudao.module.infra.dal.dataobject.file.FileDO;
|
||||||
import cn.iocoder.yudao.module.infra.dal.mysql.file.FileMapper;
|
import cn.iocoder.yudao.module.infra.dal.mysql.file.FileMapper;
|
||||||
import cn.iocoder.yudao.module.infra.framework.file.core.client.FileClient;
|
import cn.iocoder.yudao.module.infra.framework.file.core.client.FileClient;
|
||||||
|
import cn.iocoder.yudao.module.infra.framework.file.core.scan.ClamAvFileScanner;
|
||||||
import jakarta.annotation.Resource;
|
import jakarta.annotation.Resource;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
@@ -40,6 +43,9 @@ public class FileServiceImplTest extends BaseDbUnitTest {
|
|||||||
@MockitoBean
|
@MockitoBean
|
||||||
private FileConfigService fileConfigService;
|
private FileConfigService fileConfigService;
|
||||||
|
|
||||||
|
@MockitoBean
|
||||||
|
private ClamAvFileScanner fileScanner;
|
||||||
|
|
||||||
@BeforeEach
|
@BeforeEach
|
||||||
public void setUp() {
|
public void setUp() {
|
||||||
FileServiceImpl.PATH_PREFIX_DATE_ENABLE = true;
|
FileServiceImpl.PATH_PREFIX_DATE_ENABLE = true;
|
||||||
@@ -146,6 +152,43 @@ public class FileServiceImplTest extends BaseDbUnitTest {
|
|||||||
assertEquals(content.length, file.getSize());
|
assertEquals(content.length, file.getSize());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void scanUsesOnlyRegisteredFileClientAndValidatesIntegrity() throws Exception {
|
||||||
|
byte[] content = {1, 2, 3};
|
||||||
|
FileDO file = new FileDO().setConfigId(10L).setPath("safe/file.csv")
|
||||||
|
.setUrl("https://storage.example/file.csv").setSize(3L);
|
||||||
|
fileMapper.insert(file);
|
||||||
|
FileClient client = mock(FileClient.class);
|
||||||
|
when(fileConfigService.getFileClient(10L)).thenReturn(client);
|
||||||
|
when(client.getContent("safe/file.csv")).thenReturn(content);
|
||||||
|
when(fileScanner.scan(content)).thenReturn(FileScanStatus.CLEAN);
|
||||||
|
|
||||||
|
FileScanStatus result = fileService.scan(new FileDescriptor(file.getUrl(), "file.csv", "text/csv", 3,
|
||||||
|
"039058c6f2c0cb492c533b0a4d14ef77cc0f78abccced5287d84a1a2011cfb81"));
|
||||||
|
|
||||||
|
assertEquals(FileScanStatus.CLEAN, result);
|
||||||
|
verify(client).getContent("safe/file.csv");
|
||||||
|
verify(fileScanner).scan(content);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void scanRejectsArbitraryUrlAndIntegrityMismatch() throws Exception {
|
||||||
|
assertEquals(FileScanStatus.ERROR, fileService.scan(
|
||||||
|
new FileDescriptor("http://127.0.0.1:8080/admin", "x", null, 1)));
|
||||||
|
verifyNoInteractions(fileConfigService, fileScanner);
|
||||||
|
|
||||||
|
FileDO file = new FileDO().setConfigId(10L).setPath("safe/file.csv")
|
||||||
|
.setUrl("https://storage.example/file.csv").setSize(3L);
|
||||||
|
fileMapper.insert(file);
|
||||||
|
FileClient client = mock(FileClient.class);
|
||||||
|
when(fileConfigService.getFileClient(10L)).thenReturn(client);
|
||||||
|
when(client.getContent("safe/file.csv")).thenReturn(new byte[]{1, 2});
|
||||||
|
|
||||||
|
assertEquals(FileScanStatus.ERROR, fileService.scan(
|
||||||
|
new FileDescriptor(file.getUrl(), "file.csv", "text/csv", 3)));
|
||||||
|
verifyNoInteractions(fileScanner);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testDeleteFile_success() throws Exception {
|
public void testDeleteFile_success() throws Exception {
|
||||||
// mock 数据
|
// mock 数据
|
||||||
|
|||||||
@@ -260,6 +260,14 @@ yudao:
|
|||||||
--- #################### 项目相关配置 ####################
|
--- #################### 项目相关配置 ####################
|
||||||
|
|
||||||
yudao:
|
yudao:
|
||||||
|
infra:
|
||||||
|
file-scan:
|
||||||
|
enabled: ${FILE_SCAN_ENABLED:false}
|
||||||
|
host: ${FILE_SCAN_HOST:127.0.0.1}
|
||||||
|
port: ${FILE_SCAN_PORT:3310}
|
||||||
|
connect-timeout: ${FILE_SCAN_CONNECT_TIMEOUT:3s}
|
||||||
|
read-timeout: ${FILE_SCAN_READ_TIMEOUT:30s}
|
||||||
|
chunk-size: ${FILE_SCAN_CHUNK_SIZE:8192}
|
||||||
info:
|
info:
|
||||||
version: 1.0.0
|
version: 1.0.0
|
||||||
base-package: cn.iocoder.yudao
|
base-package: cn.iocoder.yudao
|
||||||
|
|||||||
Reference in New Issue
Block a user