From a517ffc6a7d3461ddfa3a580cbac779a1561c8dd Mon Sep 17 00:00:00 2001 From: xiong Date: Tue, 4 Aug 2026 13:27:25 +0800 Subject: [PATCH] feat(dev): containerize local dependencies --- Directory.Packages.props | 3 +- README.md | 15 +- .../ExternalServiceOptionsExtensions.cs | 36 ++- Tiku.Api/appsettings.Development.json | 8 + .../Storage/AliyunOssObjectStorageService.cs | 224 ++++++++++++++---- .../Storage/S3CompatibleOptions.cs | 17 ++ .../Tiku.Infrastructure.csproj | 1 + .../AliyunOssObjectStorageServiceTests.cs | 57 ++++- Tiku.Worker/WorkerDependencyInjection.cs | 38 ++- Tiku.Worker/WorkerServices.cs | 5 +- Tiku.Worker/appsettings.Development.json | 10 + compose.yaml | 20 ++ docs/architecture/overview.md | 2 +- docs/operations.md | 2 +- docs/quickstart.md | 63 +++-- 15 files changed, 428 insertions(+), 73 deletions(-) create mode 100644 Tiku.Infrastructure/Storage/S3CompatibleOptions.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 89162b3..5624c75 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -36,6 +36,7 @@ + @@ -61,4 +62,4 @@ - \ No newline at end of file + diff --git a/README.md b/README.md index f901abc..6dee71b 100644 --- a/README.md +++ b/README.md @@ -32,17 +32,24 @@ Tiku.IntegrationTests API、授权、EF 模型、迁移和真实 PostgreSQL 测 ## 快速启动 -需要 .NET 10 SDK 和 PostgreSQL。Development 默认连接本机 `tiku` 数据库，也可以通过 `DATABASE_URL` 覆盖。 +需要 .NET 10 SDK、Node.js 24+ 和 Docker Desktop。本地 PostgreSQL、Redis 与 S3 兼容对象存储统一由根目录的 `compose.yaml` 提供： ```bash -createdb -h 127.0.0.1 -U "$(whoami)" tiku +docker compose up -d --wait + +export ConnectionStrings__Database='Host=127.0.0.1;Port=5432;Database=tiku;Username=tiku;Password=tiku_dev' +export ConnectionStrings__Redis='127.0.0.1:6379,abortConnect=false' + dotnet restore TIKU-BACKEND.slnx dotnet build TIKU-BACKEND.slnx --no-restore ASPNETCORE_ENVIRONMENT=Development dotnet run --project Tiku.DbMigrator -dotnet run --project Tiku.Api -dotnet run --project Tiku.Worker +ASPNETCORE_ENVIRONMENT=Development dotnet run --project Tiku.Api --launch-profile http +# 另开终端并设置相同连接串后启动 Worker +DOTNET_ENVIRONMENT=Development dotnet run --project Tiku.Worker ``` +`docker compose ps` 应显示 PostgreSQL、Redis 与 MinIO 均为 `healthy`。Development 未配置阿里云 OSS 时，`local_dev` Provider 会把对象实际保存到 MinIO，而不是返回占位结果。默认账号和密码只用于本机开发，不能用于共享或生产环境。完整配置、停止和排障步骤见[本地开发快速上手](docs/quickstart.md)。 + 默认 Development seed 会在尚无平台角色绑定时创建平台管理员 `admin@tiku.local` 和演示数据；随机临时密码只在首次创建时输出。日常步骤见[本地开发快速上手](docs/quickstart.md)，不含演示数据的完整 SaaS 验收见[空数据库到租户建站验收](docs/tenant-provisioning.md)。 默认开发入口： diff --git a/Tiku.Api/Configuration/ExternalServiceOptionsExtensions.cs b/Tiku.Api/Configuration/ExternalServiceOptionsExtensions.cs index ad0ac08..8cd9579 100644 --- a/Tiku.Api/Configuration/ExternalServiceOptionsExtensions.cs +++ b/Tiku.Api/Configuration/ExternalServiceOptionsExtensions.cs @@ -18,9 +18,15 @@ internal static class ExternalServiceOptionsExtensions configuration.GetSection(ObjectStorageOptions.SectionName)); services.Configure( configuration.GetSection(AliyunOssOptions.SectionName)); + services.Configure( + configuration.GetSection(S3CompatibleOptions.SectionName)); services.PostConfigure(options => { options.DefaultProvider = configuration["STORAGE_DEFAULT_PROVIDER"] ?? options.DefaultProvider; + if (!environment.IsProduction() && + options.DefaultProvider == ObjectStorageProviders.AliyunOss && + !HasConfiguredAliyunOss(configuration)) + options.DefaultProvider = ObjectStorageProviders.LocalDev; options.DefaultBucket = configuration["STORAGE_DEFAULT_BUCKET"] ?? options.DefaultBucket; options.PublicBaseUrl = configuration["STORAGE_PUBLIC_BASE_URL"] ?? options.PublicBaseUrl; options.AllowedMimePrefixes = SplitLegacyList( @@ -53,6 +59,16 @@ internal static class ExternalServiceOptionsExtensions ? useInternalEndpoint : options.UseInternalEndpoint; }); + services.PostConfigure(options => + { + options.Endpoint = configuration["S3_ENDPOINT"] ?? options.Endpoint; + options.AccessKey = configuration["S3_ACCESS_KEY"] ?? options.AccessKey; + options.SecretKey = configuration["S3_SECRET_KEY"] ?? options.SecretKey; + options.Region = configuration["S3_REGION"] ?? options.Region; + options.Secure = bool.TryParse(configuration["S3_SECURE"], out var secure) + ? secure + : options.Secure; + }); services.AddOptions() .Validate( options => !environment.IsProduction() || @@ -67,6 +83,12 @@ internal static class ExternalServiceOptionsExtensions aliyun.IsConfigured, "Aliyun OSS credentials and region or endpoint are required when it is the default provider.") .ValidateOnStart(); + services.AddOptions() + .Validate>( + (s3, storage) => + storage.Value.DefaultProvider != ObjectStorageProviders.LocalDev || s3.IsConfigured, + "S3-compatible endpoint and credentials are required when local_dev is the default provider.") + .ValidateOnStart(); services.AddOptions() .Bind(configuration.GetSection(ClamAvOptions.SectionName)) .Validate(ClamAvOptions.BeValid, "ClamAV settings are invalid.") @@ -112,4 +134,16 @@ internal static class ExternalServiceOptionsExtensions ? fallback : value.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); } -} \ No newline at end of file + + private static bool HasConfiguredAliyunOss(IConfiguration configuration) + { + return !string.IsNullOrWhiteSpace(configuration["ALIYUN_OSS_ACCESS_KEY_ID"] ?? + configuration["Storage:AliyunOss:AccessKeyId"]) && + !string.IsNullOrWhiteSpace(configuration["ALIYUN_OSS_ACCESS_KEY_SECRET"] ?? + configuration["Storage:AliyunOss:AccessKeySecret"]) && + (!string.IsNullOrWhiteSpace(configuration["ALIYUN_OSS_REGION"] ?? + configuration["Storage:AliyunOss:Region"]) || + !string.IsNullOrWhiteSpace(configuration["ALIYUN_OSS_ENDPOINT"] ?? + configuration["Storage:AliyunOss:Endpoint"])); + } +} diff --git a/Tiku.Api/appsettings.Development.json b/Tiku.Api/appsettings.Development.json index 03daa17..1935da6 100644 --- a/Tiku.Api/appsettings.Development.json +++ b/Tiku.Api/appsettings.Development.json @@ -47,6 +47,14 @@ } }, "Storage": { + "DefaultProvider": "local_dev", + "S3Compatible": { + "Endpoint": "127.0.0.1:9000", + "AccessKey": "tiku", + "SecretKey": "tiku_minio_dev", + "Region": "us-east-1", + "Secure": false + }, "AliyunOss": { "Region": "cn-hangzhou", "Endpoint": "", diff --git a/Tiku.Infrastructure/Storage/AliyunOssObjectStorageService.cs b/Tiku.Infrastructure/Storage/AliyunOssObjectStorageService.cs index 7092517..901c5da 100644 --- a/Tiku.Infrastructure/Storage/AliyunOssObjectStorageService.cs +++ b/Tiku.Infrastructure/Storage/AliyunOssObjectStorageService.cs @@ -4,6 +4,9 @@ using System.Text.RegularExpressions; using AlibabaCloud.OSS.V2.Credentials; using AlibabaCloud.OSS.V2.Models; using Microsoft.Extensions.Options; +using Minio; +using Minio.DataModel.Args; +using Minio.Exceptions; using Tiku.Application.Storage; using Oss = AlibabaCloud.OSS.V2; @@ -11,7 +14,8 @@ namespace Tiku.Infrastructure.Storage; public sealed partial class AliyunOssObjectStorageService( IOptions storageOptions, - IOptions aliyunOptions) + IOptions aliyunOptions, + IOptions s3Options) : IObjectStorageService, IDisposable { private static readonly HashSet SupportedProviders = @@ -31,6 +35,7 @@ public sealed partial class AliyunOssObjectStorageService( private readonly object clientLock = new(); private Oss.Client? client; + private IMinioClient? s3Client; private bool disposed; public void Dispose() @@ -41,6 +46,8 @@ public sealed partial class AliyunOssObjectStorageService( { client?.Dispose(); client = null; + s3Client?.Dispose(); + s3Client = null; disposed = true; } } @@ -138,14 +145,14 @@ public sealed partial class AliyunOssObjectStorageService( public void AssertWritableLocation(StorageAssetLocation location) { var provider = NormalizeProvider(location.Provider); - if (provider == ObjectStorageProviders.AliyunOss && + if (provider is ObjectStorageProviders.AliyunOss or ObjectStorageProviders.LocalDev && (string.IsNullOrWhiteSpace(location.Bucket) || string.IsNullOrWhiteSpace(location.ObjectKey))) throw new ObjectStorageException( $"{provider} asset requires bucket and objectKey.", "ASSET_OBJECT_LOCATION_REQUIRED"); } - public Task SignUploadAsync( + public async Task SignUploadAsync( ObjectStorageUploadSignRequest request, CancellationToken cancellationToken = default) { @@ -156,29 +163,39 @@ public sealed partial class AliyunOssObjectStorageService( var mimeType = ValidateMimeType(request.MimeType); ValidateFileSize(request.FileSizeBytes); - return provider == ObjectStorageProviders.LocalDev - ? Task.FromResult(LocalSignedUrl( + if (provider == ObjectStorageProviders.LocalDev) + { + await EnsureS3BucketAsync(request.Bucket, cancellationToken); + var url = await GetS3Client().PresignedPutObjectAsync(new PresignedPutObjectArgs() + .WithBucket(request.Bucket) + .WithObject(objectKey) + .WithExpiry(ExpirySeconds(request.ExpiresIn))); + return new ObjectStorageSignedUrl( provider, request.Bucket, objectKey, "PUT", - request.ExpiresIn, - "attachment", + new Uri(url, UriKind.Absolute), new Dictionary(StringComparer.OrdinalIgnoreCase) { ["content-type"] = mimeType - })) - : Task.FromResult(SignAliyunOss( + }, + DateTimeOffset.UtcNow.Add(request.ExpiresIn), + request.ExpiresIn, + "s3-compatible-presigned-put"); + } + + return SignAliyunOss( request.Bucket, objectKey, "PUT", request.ExpiresIn, mimeType, null, - null)); + null); } - public Task SignDownloadAsync( + public async Task SignDownloadAsync( ObjectStorageDownloadSignRequest request, CancellationToken cancellationToken = default) { @@ -186,7 +203,7 @@ public sealed partial class AliyunOssObjectStorageService( var provider = NormalizeProvider(request.Provider); if (!string.IsNullOrWhiteSpace(request.CdnUrl)) - return Task.FromResult(new ObjectStorageSignedUrl( + return new ObjectStorageSignedUrl( provider, request.Bucket, request.ObjectKey, @@ -195,20 +212,36 @@ public sealed partial class AliyunOssObjectStorageService( new Dictionary(), DateTimeOffset.UtcNow.Add(request.ExpiresIn), request.ExpiresIn, - "public-or-provider-managed")); + "public-or-provider-managed"); if (string.IsNullOrWhiteSpace(request.ObjectKey)) throw new ObjectStorageException("Asset requires objectKey or cdnUrl.", "ASSET_LOCATION_REQUIRED"); var objectKey = ValidateObjectKey(request.TenantId, request.ObjectKey); - if (provider is ObjectStorageProviders.LocalDev or ObjectStorageProviders.QiniuKodo) - return Task.FromResult(LocalSignedUrl( + if (provider == ObjectStorageProviders.LocalDev) + { + if (string.IsNullOrWhiteSpace(request.Bucket)) + throw new ObjectStorageException("S3-compatible asset requires bucket.", + "ASSET_OBJECT_LOCATION_REQUIRED"); + var url = await GetS3Client().PresignedGetObjectAsync(new PresignedGetObjectArgs() + .WithBucket(request.Bucket) + .WithObject(objectKey) + .WithExpiry(ExpirySeconds(request.ExpiresIn))); + return new ObjectStorageSignedUrl( provider, request.Bucket, objectKey, "GET", + new Uri(url, UriKind.Absolute), + new Dictionary(), + DateTimeOffset.UtcNow.Add(request.ExpiresIn), request.ExpiresIn, - request.Disposition)); + "s3-compatible-presigned-get"); + } + + if (provider == ObjectStorageProviders.QiniuKodo) + return LocalSignedUrl(provider, request.Bucket, objectKey, "GET", request.ExpiresIn, + request.Disposition); if (provider != ObjectStorageProviders.AliyunOss) throw new ObjectStorageException($"{provider} asset requires cdnUrl.", "ASSET_LOCATION_REQUIRED"); @@ -216,14 +249,14 @@ public sealed partial class AliyunOssObjectStorageService( if (string.IsNullOrWhiteSpace(request.Bucket)) throw new ObjectStorageException("Aliyun OSS asset requires bucket.", "ASSET_OBJECT_LOCATION_REQUIRED"); - return Task.FromResult(SignAliyunOss( + return SignAliyunOss( request.Bucket, objectKey, "GET", request.ExpiresIn, null, request.FileName, - request.Disposition)); + request.Disposition); } public async Task HeadObjectAsync( @@ -241,18 +274,34 @@ public sealed partial class AliyunOssObjectStorageService( var checksumSha256 = request.DeclaredChecksumSha256?.Trim().ToLowerInvariant(); if (provider == ObjectStorageProviders.LocalDev) - return new ObjectStorageMetadata( - provider, - request.Bucket, - objectKey, - true, - fileSizeBytes, - mimeType, - checksumSha256, - checksumSha256, - DateTimeOffset.UtcNow.ToString("O"), - new Dictionary(), - "local-dev-declared-metadata"); + { + if (string.IsNullOrWhiteSpace(request.Bucket)) + throw new ObjectStorageException("S3-compatible asset requires bucket.", + "ASSET_OBJECT_LOCATION_REQUIRED"); + try + { + var result = await GetS3Client().StatObjectAsync(new StatObjectArgs() + .WithBucket(request.Bucket) + .WithObject(objectKey), cancellationToken); + return new ObjectStorageMetadata( + provider, + request.Bucket, + objectKey, + true, + result.Size, + result.ContentType, + checksumSha256, + result.ETag, + result.LastModified.ToString("O"), + new Dictionary(), + "s3-compatible-stat-object"); + } + catch (ObjectNotFoundException) + { + throw new ObjectStorageException("Uploaded object was not found in S3-compatible storage.", + "STORAGE_OBJECT_NOT_FOUND"); + } + } if (provider != ObjectStorageProviders.AliyunOss) throw new ObjectStorageException( @@ -298,17 +347,47 @@ public sealed partial class AliyunOssObjectStorageService( throw new ObjectStorageException("Writable asset requires bucket.", "ASSET_OBJECT_LOCATION_REQUIRED"); if (provider == ObjectStorageProviders.LocalDev) - return new ObjectStorageWriteResult( - provider, - request.Bucket, - objectKey, - BuildPublicUrl(request.Bucket, objectKey), - fileSizeBytes, - mimeType, - checksumSha256 ?? await ComputeSha256Async(request.Content, cancellationToken), - null, - new Dictionary(), - "local-dev-write-placeholder"); + { + await EnsureS3BucketAsync(request.Bucket, cancellationToken); + MemoryStream? bufferedContent = null; + var uploadContent = request.Content; + if (checksumSha256 is null && !request.Content.CanSeek) + { + bufferedContent = new MemoryStream(); + await request.Content.CopyToAsync(bufferedContent, cancellationToken); + bufferedContent.Position = 0; + uploadContent = bufferedContent; + } + + var resolvedChecksum = checksumSha256 ?? await ComputeSha256Async(uploadContent, cancellationToken); + var objectSize = fileSizeBytes ?? (uploadContent.CanSeek + ? uploadContent.Length - uploadContent.Position + : -1); + try + { + await GetS3Client().PutObjectAsync(new PutObjectArgs() + .WithBucket(request.Bucket) + .WithObject(objectKey) + .WithStreamData(uploadContent) + .WithObjectSize(objectSize) + .WithContentType(mimeType), cancellationToken); + return new ObjectStorageWriteResult( + provider, + request.Bucket, + objectKey, + BuildPublicUrl(request.Bucket, objectKey), + fileSizeBytes ?? objectSize, + mimeType, + resolvedChecksum, + null, + new Dictionary(), + "s3-compatible-put-object"); + } + finally + { + if (bufferedContent is not null) await bufferedContent.DisposeAsync(); + } + } if (provider != ObjectStorageProviders.AliyunOss) throw new ObjectStorageException( @@ -353,6 +432,30 @@ public sealed partial class AliyunOssObjectStorageService( { var provider = NormalizeProvider(request.Provider); var objectKey = ValidateObjectKey(request.TenantId, request.ObjectKey); + if (provider == ObjectStorageProviders.LocalDev) + { + if (string.IsNullOrWhiteSpace(request.Bucket)) + throw new ObjectStorageException("S3-compatible asset requires bucket.", + "ASSET_OBJECT_LOCATION_REQUIRED"); + var content = new MemoryStream(); + try + { + await GetS3Client().GetObjectAsync(new GetObjectArgs() + .WithBucket(request.Bucket) + .WithObject(objectKey) + .WithCallbackStream(async (stream, token) => + await stream.CopyToAsync(content, token)), cancellationToken); + content.Position = 0; + return content; + } + catch (ObjectNotFoundException) + { + await content.DisposeAsync(); + throw new ObjectStorageException("Object was not found in S3-compatible storage.", + "STORAGE_OBJECT_NOT_FOUND"); + } + } + if (provider != ObjectStorageProviders.AliyunOss) throw new ObjectStorageException( $"{provider} does not support managed server-side reads.", @@ -457,6 +560,43 @@ public sealed partial class AliyunOssObjectStorageService( } } + private IMinioClient GetS3Client() + { + ObjectDisposedException.ThrowIf(disposed, this); + if (s3Client is not null) return s3Client; + + lock (clientLock) + { + if (s3Client is not null) return s3Client; + var options = s3Options.Value; + if (!options.IsConfigured) + throw new ObjectStorageNotConfiguredException( + "S3-compatible storage is not configured: S3_ENDPOINT, S3_ACCESS_KEY and S3_SECRET_KEY are required."); + + var builder = new MinioClient() + .WithEndpoint(options.Endpoint!.Trim()) + .WithCredentials(options.AccessKey!.Trim(), options.SecretKey!.Trim()) + .WithSSL(options.Secure); + if (!string.IsNullOrWhiteSpace(options.Region)) builder = builder.WithRegion(options.Region.Trim()); + s3Client = builder.Build(); + return s3Client; + } + } + + private async Task EnsureS3BucketAsync(string bucket, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(bucket); + var client = GetS3Client(); + var exists = await client.BucketExistsAsync(new BucketExistsArgs().WithBucket(bucket), cancellationToken); + if (!exists) + await client.MakeBucketAsync(new MakeBucketArgs().WithBucket(bucket), cancellationToken); + } + + private static int ExpirySeconds(TimeSpan expiresIn) + { + return Math.Clamp((int)Math.Ceiling(expiresIn.TotalSeconds), 1, 7 * 24 * 60 * 60); + } + private static Oss.Client CreateClient(AliyunOssOptions currentOptions) { if (!currentOptions.IsConfigured) @@ -625,4 +765,4 @@ public sealed partial class AliyunOssObjectStorageService( [GeneratedRegex("/{2,}")] private static partial Regex ConsecutiveSlashRegex(); -} \ No newline at end of file +} diff --git a/Tiku.Infrastructure/Storage/S3CompatibleOptions.cs b/Tiku.Infrastructure/Storage/S3CompatibleOptions.cs new file mode 100644 index 0000000..487f4eb --- /dev/null +++ b/Tiku.Infrastructure/Storage/S3CompatibleOptions.cs @@ -0,0 +1,17 @@ +namespace Tiku.Infrastructure.Storage; + +public sealed class S3CompatibleOptions +{ + public const string SectionName = "Storage:S3Compatible"; + + public string? Endpoint { get; set; } + public string? AccessKey { get; set; } + public string? SecretKey { get; set; } + public string? Region { get; set; } + public bool Secure { get; set; } + + public bool IsConfigured => + !string.IsNullOrWhiteSpace(Endpoint) && + !string.IsNullOrWhiteSpace(AccessKey) && + !string.IsNullOrWhiteSpace(SecretKey); +} diff --git a/Tiku.Infrastructure/Tiku.Infrastructure.csproj b/Tiku.Infrastructure/Tiku.Infrastructure.csproj index 26bff87..5af1fdf 100644 --- a/Tiku.Infrastructure/Tiku.Infrastructure.csproj +++ b/Tiku.Infrastructure/Tiku.Infrastructure.csproj @@ -19,6 +19,7 @@ + diff --git a/Tiku.UnitTests/Storage/AliyunOssObjectStorageServiceTests.cs b/Tiku.UnitTests/Storage/AliyunOssObjectStorageServiceTests.cs index 727c2e4..e8e0ca9 100644 --- a/Tiku.UnitTests/Storage/AliyunOssObjectStorageServiceTests.cs +++ b/Tiku.UnitTests/Storage/AliyunOssObjectStorageServiceTests.cs @@ -120,6 +120,51 @@ public sealed class AliyunOssObjectStorageServiceTests })); } + [Fact] + public async Task Local_dev_round_trips_through_configured_s3_endpoint() + { + var endpoint = Environment.GetEnvironmentVariable("TIKU_S3_TEST_ENDPOINT"); + if (string.IsNullOrWhiteSpace(endpoint)) return; + + using var service = new AliyunOssObjectStorageService( + Options.Create(new ObjectStorageOptions { DefaultProvider = ObjectStorageProviders.LocalDev }), + Options.Create(new AliyunOssOptions()), + Options.Create(new S3CompatibleOptions + { + Endpoint = endpoint, + AccessKey = Environment.GetEnvironmentVariable("TIKU_S3_TEST_ACCESS_KEY") ?? "tiku", + SecretKey = Environment.GetEnvironmentVariable("TIKU_S3_TEST_SECRET_KEY") ?? "tiku_minio_dev", + Region = "us-east-1" + })); + var objectKey = $"{TenantId:N}/tests/{Guid.NewGuid():N}.json"; + const string payload = "{\"storage\":\"minio\"}"; + await using var input = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(payload)); + + var write = await service.WriteObjectAsync(new ObjectStorageWriteRequest( + TenantId, + ObjectStorageProviders.LocalDev, + "tiku-tests", + objectKey, + "application/json", + input, + input.Length)); + var metadata = await service.HeadObjectAsync(new ObjectStorageHeadRequest( + TenantId, + ObjectStorageProviders.LocalDev, + "tiku-tests", + objectKey)); + await using var output = await service.OpenReadAsync(new ObjectStorageReadRequest( + TenantId, + ObjectStorageProviders.LocalDev, + "tiku-tests", + objectKey)); + using var reader = new StreamReader(output); + + Assert.Equal("s3-compatible-put-object", write.VerificationSource); + Assert.Equal(input.Length, metadata.SizeBytes); + Assert.Equal(payload, await reader.ReadToEndAsync()); + } + private static AliyunOssObjectStorageService CreateConfiguredService(ObjectStorageOptions? storageOptions = null) { return new AliyunOssObjectStorageService( @@ -129,12 +174,20 @@ public sealed class AliyunOssObjectStorageServiceTests Region = "cn-hangzhou", AccessKeyId = "test-access-key-id", AccessKeySecret = "test-access-key-secret" + }), + Options.Create(new S3CompatibleOptions + { + Endpoint = "127.0.0.1:9000", + AccessKey = "test-access-key", + SecretKey = "test-secret-key", + Region = "us-east-1" })); } private static AliyunOssObjectStorageService CreateUnconfiguredService() { return new AliyunOssObjectStorageService(Options.Create(new ObjectStorageOptions()), - Options.Create(new AliyunOssOptions())); + Options.Create(new AliyunOssOptions()), + Options.Create(new S3CompatibleOptions())); } -} \ No newline at end of file +} diff --git a/Tiku.Worker/WorkerDependencyInjection.cs b/Tiku.Worker/WorkerDependencyInjection.cs index 0fe69c7..9693bda 100644 --- a/Tiku.Worker/WorkerDependencyInjection.cs +++ b/Tiku.Worker/WorkerDependencyInjection.cs @@ -31,6 +31,8 @@ internal static class WorkerDependencyInjection : throw new InvalidOperationException( "Database connection is required outside Development. Configure ConnectionStrings:Database or DATABASE_URL.")); builder.Services.AddApplication(); + builder.Services.AddAuthentication(); + builder.Services.AddDataProtection(); builder.Services.AddInfrastructure(connectionString); var redisConnectionString = builder.Configuration.GetConnectionString("Redis") ?? builder.Configuration["REDIS_URL"]; @@ -49,9 +51,15 @@ internal static class WorkerDependencyInjection builder.Services.Configure( builder.Configuration.GetSection(ObjectStorageOptions.SectionName)); builder.Services.Configure(builder.Configuration.GetSection(AliyunOssOptions.SectionName)); + builder.Services.Configure( + builder.Configuration.GetSection(S3CompatibleOptions.SectionName)); builder.Services.PostConfigure(options => { options.DefaultProvider = builder.Configuration["STORAGE_DEFAULT_PROVIDER"] ?? options.DefaultProvider; + if (!builder.Environment.IsProduction() && + options.DefaultProvider == ObjectStorageProviders.AliyunOss && + !HasConfiguredAliyunOss(builder.Configuration)) + options.DefaultProvider = ObjectStorageProviders.LocalDev; options.DefaultBucket = builder.Configuration["STORAGE_DEFAULT_BUCKET"] ?? options.DefaultBucket; options.PublicBaseUrl = builder.Configuration["STORAGE_PUBLIC_BASE_URL"] ?? options.PublicBaseUrl; options.AllowedMimePrefixes = SplitLegacyList( @@ -84,6 +92,16 @@ internal static class WorkerDependencyInjection ? useInternalEndpoint : options.UseInternalEndpoint; }); + builder.Services.PostConfigure(options => + { + options.Endpoint = builder.Configuration["S3_ENDPOINT"] ?? options.Endpoint; + options.AccessKey = builder.Configuration["S3_ACCESS_KEY"] ?? options.AccessKey; + options.SecretKey = builder.Configuration["S3_SECRET_KEY"] ?? options.SecretKey; + options.Region = builder.Configuration["S3_REGION"] ?? options.Region; + options.Secure = bool.TryParse(builder.Configuration["S3_SECURE"], out var secure) + ? secure + : options.Secure; + }); builder.Services.AddOptions() .Validate( options => !builder.Environment.IsProduction() || @@ -98,6 +116,12 @@ internal static class WorkerDependencyInjection aliyun.IsConfigured, "Aliyun OSS credentials and region or endpoint are required when it is the default provider.") .ValidateOnStart(); + builder.Services.AddOptions() + .Validate>( + (s3, storage) => + storage.Value.DefaultProvider != ObjectStorageProviders.LocalDev || s3.IsConfigured, + "S3-compatible endpoint and credentials are required when local_dev is the default provider.") + .ValidateOnStart(); builder.Services.AddOptions() .Bind(builder.Configuration.GetSection(ClamAvOptions.SectionName)) .Validate(ClamAvOptions.BeValid, "ClamAV settings are invalid.") @@ -137,4 +161,16 @@ internal static class WorkerDependencyInjection ? fallback : value.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); } -} \ No newline at end of file + + private static bool HasConfiguredAliyunOss(IConfiguration configuration) + { + return !string.IsNullOrWhiteSpace(configuration["ALIYUN_OSS_ACCESS_KEY_ID"] ?? + configuration["Storage:AliyunOss:AccessKeyId"]) && + !string.IsNullOrWhiteSpace(configuration["ALIYUN_OSS_ACCESS_KEY_SECRET"] ?? + configuration["Storage:AliyunOss:AccessKeySecret"]) && + (!string.IsNullOrWhiteSpace(configuration["ALIYUN_OSS_REGION"] ?? + configuration["Storage:AliyunOss:Region"]) || + !string.IsNullOrWhiteSpace(configuration["ALIYUN_OSS_ENDPOINT"] ?? + configuration["Storage:AliyunOss:Endpoint"])); + } +} diff --git a/Tiku.Worker/WorkerServices.cs b/Tiku.Worker/WorkerServices.cs index 1a2728a..b94b76d 100644 --- a/Tiku.Worker/WorkerServices.cs +++ b/Tiku.Worker/WorkerServices.cs @@ -2,6 +2,7 @@ using System.Data; using System.Diagnostics; using Microsoft.Extensions.Options; using Npgsql; +using NpgsqlTypes; using Tiku.Application.Jobs; using Tiku.Application.PlatformAdmin; using Tiku.Application.PlatformBilling; @@ -80,7 +81,7 @@ internal sealed class WorkerStateReporter(NpgsqlDataSource dataSource) command.Parameters.AddWithValue("started_at", startedAt); command.Parameters.AddWithValue("now", now); command.Parameters.AddWithValue("running", running); - command.Parameters.AddWithValue("error", (object?)error ?? DBNull.Value); + command.Parameters.Add("error", NpgsqlDbType.Text).Value = (object?)error ?? DBNull.Value; await command.ExecuteNonQueryAsync(cancellationToken); } } @@ -323,4 +324,4 @@ internal sealed class PlatformApprovalWorker( return await scope.ServiceProvider.GetRequiredService() .ProcessApprovedAsync(cancellationToken: cancellationToken); } -} \ No newline at end of file +} diff --git a/Tiku.Worker/appsettings.Development.json b/Tiku.Worker/appsettings.Development.json index a576829..432f113 100644 --- a/Tiku.Worker/appsettings.Development.json +++ b/Tiku.Worker/appsettings.Development.json @@ -3,5 +3,15 @@ "MinimumLevel": { "Default": "Debug" } + }, + "Storage": { + "DefaultProvider": "local_dev", + "S3Compatible": { + "Endpoint": "127.0.0.1:9000", + "AccessKey": "tiku", + "SecretKey": "tiku_minio_dev", + "Region": "us-east-1", + "Secure": false + } } } diff --git a/compose.yaml b/compose.yaml index d8418ba..b227de5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -34,6 +34,26 @@ services: retries: 10 start_period: 5s + minio: + image: minio/minio:latest + restart: unless-stopped + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: ${TIKU_MINIO_ROOT_USER:-tiku} + MINIO_ROOT_PASSWORD: ${TIKU_MINIO_ROOT_PASSWORD:-tiku_minio_dev} + ports: + - "127.0.0.1:${TIKU_MINIO_API_PORT:-9000}:9000" + - "127.0.0.1:${TIKU_MINIO_CONSOLE_PORT:-9001}:9001" + volumes: + - minio_data:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + volumes: postgres_data: redis_data: + minio_data: diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index e3445d1..f372a4e 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -100,6 +100,6 @@ API 不自动迁移数据库。 ## 外部服务边界 -Application 通过接口表达身份、短信、对象存储、支付、通知、域名和 AI 能力；Infrastructure 当前包含自托管身份、阿里云短信/OSS、微信、支付宝、站内通知、DNS JSON 查询和 HTTP 网关实现。 +Application 通过接口表达身份、短信、对象存储、支付、通知、域名和 AI 能力；Infrastructure 当前包含自托管身份、阿里云短信/OSS、S3 兼容本地对象存储、微信、支付宝、站内通知、DNS JSON 查询和 HTTP 网关实现。 租户级 Provider 元数据和密钥分别存入 `TenantExternalProvider` 与 `TenantSecret`。密钥由 32 字节 master key 加密，API 不应把明文、`SecretRef` 或 Provider 内部 payload 返回给客户端。 diff --git a/docs/operations.md b/docs/operations.md index 3584291..cc74802 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -160,7 +160,7 @@ Production 启动至少需要核对： ## 对象存储与外部 Provider -对象存储读取 `Storage` / `Storage:AliyunOss`，API 与 Worker 都支持 `STORAGE_*` 与 `ALIYUN_OSS_*` 环境变量。当前默认实现是阿里云 OSS，并强制租户 key 前缀、上传大小和 MIME allowlist。租户导出使用 `application/gzip`，该类型不能从 allowlist 删除。 +对象存储读取 `Storage`、`Storage:AliyunOss` 与 `Storage:S3Compatible`。API 与 Worker 支持 `STORAGE_*`、`ALIYUN_OSS_*` 以及 `S3_ENDPOINT`、`S3_ACCESS_KEY`、`S3_SECRET_KEY`、`S3_REGION`、`S3_SECURE` 环境变量。Production 默认并强制使用已配置的阿里云 OSS；非 Production 没有完整 OSS 凭据时回退到 `local_dev`，该 Provider 使用 S3 兼容存储，快速部署环境由 MinIO 提供。所有托管 Provider 都强制租户 key 前缀、上传大小和 MIME allowlist。租户导出使用 `application/gzip`，该类型不能从 allowlist 删除。 ## ClamAV 资源安全扫描 diff --git a/docs/quickstart.md b/docs/quickstart.md index da423df..7c7c05a 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -6,14 +6,14 @@ - .NET 10 SDK - Node.js 24+、npm 11+ -- PostgreSQL -- Redis（Development 可不配置；涉及分布式缓存、频控或完整验收时应启动） +- Docker Desktop（通过 Docker Compose 提供 PostgreSQL 18、Redis 7 和 MinIO） ```bash dotnet --version node --version npm --version -pg_isready -h 127.0.0.1 -p 5432 +docker info +docker compose version ``` 首次拉取代码后恢复依赖： @@ -23,27 +23,54 @@ dotnet restore TIKU-BACKEND.slnx npm --prefix Tiku.PlatformAdmin.Web install ``` -## 配置数据库 +## 启动 PostgreSQL、Redis 与对象存储 -Development 未配置连接串时，默认使用当前系统用户连接本机 `tiku` 数据库： +在仓库根目录启动依赖并等待健康检查通过： + +```bash +docker compose up -d --wait +docker compose ps +docker compose exec postgres pg_isready -U tiku -d tiku +docker compose exec redis redis-cli ping +curl --fail http://127.0.0.1:9000/minio/health/live +``` + +默认只监听本机回环地址： ```text -Host=localhost;Database=tiku;Username=<当前系统用户> +PostgreSQL: 127.0.0.1:5432，数据库 tiku，用户 tiku，密码 tiku_dev +Redis: 127.0.0.1:6379 +MinIO API: 127.0.0.1:9000，Access Key tiku，Secret Key tiku_minio_dev +MinIO 控制台: http://127.0.0.1:9001 ``` -首次使用可创建数据库： +为 DbMigrator、API 和 Worker 设置相同的连接串： ```bash -createdb -h 127.0.0.1 -U <数据库用户> tiku +export ConnectionStrings__Database='Host=127.0.0.1;Port=5432;Database=tiku;Username=tiku;Password=tiku_dev' +export ConnectionStrings__Redis='127.0.0.1:6379,abortConnect=false' ``` -使用其他地址、端口或账号时，通过环境变量覆盖： +Development 配置会把 `local_dev` 对象存储指向该 MinIO；没有完整阿里云 OSS 凭据时也会自动回退到 `local_dev`。文件内容保存在 MinIO 数据卷，PostgreSQL 只保存业务元数据和对象引用。 + +这些默认凭据仅用于本机开发，不能用于共享或生产环境。需要自定义数据库、MinIO 凭据或宿主机端口时，应在首次创建数据卷前设置 Compose 变量，并同步修改应用连接串或 `Storage:S3Compatible` 配置： ```bash -export ConnectionStrings__Database='Host=127.0.0.1;Port=5432;Database=tiku;Username=<数据库用户>;Password=<本地密码>' +export TIKU_POSTGRES_DB='<数据库名>' +export TIKU_POSTGRES_USER='<数据库用户>' +export TIKU_POSTGRES_PASSWORD='<本地密码>' +export TIKU_POSTGRES_PORT='<宿主机端口>' +export TIKU_REDIS_PORT='<宿主机端口>' +export TIKU_MINIO_ROOT_USER='' +export TIKU_MINIO_ROOT_PASSWORD='' +export TIKU_MINIO_API_PORT='<宿主机 API 端口>' +export TIKU_MINIO_CONSOLE_PORT='<宿主机控制台端口>' +docker compose up -d --wait ``` -不要把连接串、密码、Token 或私钥写入仓库。 +PostgreSQL 初始化变量不会修改已有数据卷中的账号或数据库。不要把自定义连接串、密码、Token 或私钥写入仓库。 + +日常暂停使用 `docker compose stop`，恢复使用 `docker compose start`。`docker compose down` 会删除容器和网络但保留命名数据卷；`docker compose down -v` 会永久删除本地 PostgreSQL、Redis 与 MinIO 数据，只能在明确不需要数据时使用。 ## 迁移与初始化 @@ -90,14 +117,10 @@ ASPNETCORE_ENVIRONMENT=Development dotnet run --project Tiku.Api --launch-profil 后台循环不在 API 内运行。需要处理域名、订阅、任务队列、授权缓存失效或商业账务时，另开终端启动 Worker： ```bash -ASPNETCORE_ENVIRONMENT=Development dotnet run --project Tiku.Worker +DOTNET_ENVIRONMENT=Development dotnet run --project Tiku.Worker ``` -如需 Redis，API 与 Worker 应使用同一实例： - -```bash -export ConnectionStrings__Redis='localhost:6379,abortConnect=false' -``` +API 与 Worker 必须使用前面设置的同一 PostgreSQL、Redis 和对象存储。Worker 是通用 Host，环境名使用 `DOTNET_ENVIRONMENT`；每个新终端都需要重新设置连接串，或通过本机未跟踪的安全配置注入。 ## 验证修改 @@ -119,7 +142,7 @@ PostgreSQL 特有的 Migration、事务、约束和租户隔离行为必须由 ### API 报数据库不可用 -确认 PostgreSQL 已启动、数据库存在，并核对 `ConnectionStrings__Database` 或 `DATABASE_URL`。非 Development 环境没有本地默认连接串。 +先运行 `docker compose ps`，确认 PostgreSQL 为 `healthy`，再核对 `ConnectionStrings__Database` 或 `DATABASE_URL`。如果 `5432` 端口已被本机 PostgreSQL 占用，应停止该服务或通过 `TIKU_POSTGRES_PORT` 改用其他宿主机端口；非 Development 环境没有本地默认连接串。 ### 找不到平台管理员临时密码 @@ -129,6 +152,10 @@ PostgreSQL 特有的 Migration、事务、约束和租户隔离行为必须由 `/api/system/health/ready` 会检查 PostgreSQL 和已配置的 Redis。先验证数据库连接；配置 Redis 后还需确认 Redis 可访问。匿名响应不会暴露依赖详情。 +### 上传或导出提示对象存储不可用 + +运行 `docker compose ps` 并确认 MinIO 为 `healthy`，再检查 `http://127.0.0.1:9000/minio/health/live`。自定义 MinIO 凭据或端口后，必须同步设置 `Storage:S3Compatible` 或 `S3_ENDPOINT`、`S3_ACCESS_KEY`、`S3_SECRET_KEY` 与 `S3_SECURE`。 + ### API 启动了但后台任务不执行 Production 和常规 Development 都需要独立运行 `Tiku.Worker`。Development 仅额外在 API 中注册本地域名生命周期旁路，不代表 API 承载全部 Worker 循环。