245 lines
11 KiB
C#
245 lines
11 KiB
C#
using System.Formats.Tar;
|
|
using System.IO.Compression;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Tiku.Application.Jobs;
|
|
using Tiku.Application.Storage;
|
|
using Tiku.Domain.Content;
|
|
using Tiku.Domain.Operations;
|
|
using Tiku.Domain.Tenancy;
|
|
using Tiku.Infrastructure.Jobs;
|
|
using Tiku.Infrastructure.Persistence;
|
|
|
|
namespace Tiku.Infrastructure.PlatformAdmin.TenantProvisioning;
|
|
|
|
internal sealed class TenantExportJobHandler(
|
|
TikuDbContext dbContext,
|
|
IObjectStorageService storage) : IBackgroundJobHandler
|
|
{
|
|
public string JobType => "tenant_export";
|
|
|
|
public async Task<BackgroundJobHandlerResult> HandleAsync(
|
|
BackgroundJobExecutionContext context,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var operationId = BackgroundJobPayload.GetGuid(context.Payload, "operationId") ??
|
|
throw new InvalidOperationException("tenant_export job requires operationId.");
|
|
var operation = await dbContext.TenantLifecycleOperations.SingleOrDefaultAsync(item =>
|
|
item.TenantId == context.TenantId && item.Id == operationId &&
|
|
item.OperationType == TenantLifecycleOperationType.Export,
|
|
cancellationToken) ?? throw new InvalidOperationException("Tenant export operation was not found.");
|
|
operation.Status = TenantLifecycleOperationStatus.Processing;
|
|
operation.StartedAt ??= DateTimeOffset.UtcNow;
|
|
operation.LastError = null;
|
|
await dbContext.SaveChangesAsync(cancellationToken);
|
|
|
|
var temporaryPath = Path.Combine(Path.GetTempPath(), $"tiku-tenant-export-{operation.Id:N}.tar.gz");
|
|
try
|
|
{
|
|
var tenant = await dbContext.Tenants.AsNoTracking()
|
|
.SingleAsync(item => item.Id == context.TenantId, cancellationToken);
|
|
var memberships = await dbContext.TenantMemberships.AsNoTracking()
|
|
.Where(item => item.TenantId == context.TenantId)
|
|
.Select(item => new { item.UserId, item.Role, item.Status, item.CreatedAt, item.UpdatedAt })
|
|
.ToArrayAsync(cancellationToken);
|
|
var domains = await dbContext.TenantDomains.AsNoTracking()
|
|
.Where(item => item.TenantId == context.TenantId)
|
|
.Select(item => new
|
|
{
|
|
item.Id,
|
|
item.Host,
|
|
item.DomainType,
|
|
item.Status,
|
|
item.IsPrimary,
|
|
item.CreatedAt,
|
|
item.UpdatedAt
|
|
})
|
|
.ToArrayAsync(cancellationToken);
|
|
var assets = await dbContext.ContentAssets.AsNoTracking()
|
|
.Where(item => item.TenantId == context.TenantId && item.Status == ContentStatus.Active)
|
|
.ToArrayAsync(cancellationToken);
|
|
|
|
await using (var file = new FileStream(temporaryPath, FileMode.CreateNew, FileAccess.Write, FileShare.None,
|
|
128 * 1024, FileOptions.Asynchronous))
|
|
await using (var gzip = new GZipStream(file, CompressionLevel.Fastest, false))
|
|
await using (var archive = new TarWriter(gzip, TarEntryFormat.Pax, false))
|
|
{
|
|
await WriteJsonEntryAsync(archive, "manifest.json", new
|
|
{
|
|
format = "tiku-tenant-export",
|
|
version = 1,
|
|
tenantId = context.TenantId,
|
|
operationId,
|
|
generatedAt = DateTimeOffset.UtcNow,
|
|
exclusions = new[]
|
|
{
|
|
"password_hashes", "auth_tokens", "refresh_tokens", "secret_plaintext",
|
|
"data_protection_keys", "global_platform_data"
|
|
},
|
|
tables = new[] { "tenant", "tenant_memberships", "tenant_domains", "content_assets" }
|
|
}, cancellationToken);
|
|
await WriteJsonLinesEntryAsync(archive, "data/tenant.jsonl", new[]
|
|
{
|
|
new
|
|
{
|
|
tenant.Id, tenant.Slug, tenant.Name, tenant.LegalName, tenant.Status, tenant.Mode,
|
|
tenant.BillingStatus, tenant.OwnerUserId, tenant.CreatedAt, tenant.UpdatedAt
|
|
}
|
|
}, cancellationToken);
|
|
await WriteJsonLinesEntryAsync(archive, "data/tenant_memberships.jsonl", memberships,
|
|
cancellationToken);
|
|
await WriteJsonLinesEntryAsync(archive, "data/tenant_domains.jsonl", domains, cancellationToken);
|
|
await WriteJsonLinesEntryAsync(archive, "data/content_assets.jsonl", assets.Select(item => new
|
|
{
|
|
item.Id,
|
|
item.AssetKey,
|
|
item.Title,
|
|
item.FileName,
|
|
item.StorageProvider,
|
|
item.Bucket,
|
|
item.ObjectKey,
|
|
item.MimeType,
|
|
item.FileSizeBytes,
|
|
item.ChecksumSha256,
|
|
item.UploadStatus,
|
|
item.SecurityScanStatus,
|
|
item.CreatedAt,
|
|
item.UpdatedAt
|
|
}), cancellationToken);
|
|
|
|
foreach (var asset in assets.Where(item =>
|
|
item.UploadStatus == AssetUploadStatus.Verified &&
|
|
item.SecurityScanStatus is AssetSecurityScanStatus.Passed
|
|
or AssetSecurityScanStatus.NotRequired &&
|
|
!string.IsNullOrWhiteSpace(item.Bucket) &&
|
|
!string.IsNullOrWhiteSpace(item.ObjectKey)))
|
|
{
|
|
var provider = asset.StorageProvider switch
|
|
{
|
|
AssetStorageProvider.AliyunOss => ObjectStorageProviders.AliyunOss,
|
|
AssetStorageProvider.LocalDev => ObjectStorageProviders.LocalDev,
|
|
_ => null
|
|
};
|
|
if (provider is null) continue;
|
|
await using var content = await storage.OpenReadAsync(
|
|
new ObjectStorageReadRequest(
|
|
context.TenantId, provider, asset.Bucket!, asset.ObjectKey!),
|
|
cancellationToken);
|
|
var name = SanitizeTarPath(asset.FileName ?? asset.Id.ToString("N"));
|
|
archive.WriteEntry(new PaxTarEntry(TarEntryType.RegularFile, $"assets/{asset.Id:N}/{name}")
|
|
{
|
|
DataStream = content
|
|
});
|
|
}
|
|
}
|
|
|
|
await using var upload = new FileStream(temporaryPath, FileMode.Open, FileAccess.Read, FileShare.Read,
|
|
128 * 1024, FileOptions.Asynchronous);
|
|
var providerName = storage.ConfiguredDefaultProvider();
|
|
var bucket = storage.ConfiguredDefaultBucket();
|
|
var objectKey =
|
|
storage.ValidateObjectKey(context.TenantId,
|
|
$"{context.TenantId:N}/tenant-exports/{operation.Id:N}.tar.gz");
|
|
var written = await storage.WriteObjectAsync(
|
|
new ObjectStorageWriteRequest(
|
|
context.TenantId,
|
|
providerName,
|
|
bucket,
|
|
objectKey,
|
|
"application/gzip",
|
|
upload,
|
|
upload.Length,
|
|
Upsert: false),
|
|
cancellationToken);
|
|
var exportAsset = new ContentAsset
|
|
{
|
|
TenantId = context.TenantId,
|
|
Title = "Tenant export archive",
|
|
FileName = $"tenant-export-{operation.Id:N}.tar.gz",
|
|
AssetType = ContentAssetType.Document,
|
|
StorageProvider = providerName switch
|
|
{
|
|
ObjectStorageProviders.AliyunOss => AssetStorageProvider.AliyunOss,
|
|
ObjectStorageProviders.LocalDev => AssetStorageProvider.LocalDev,
|
|
_ => AssetStorageProvider.ExternalUrl
|
|
},
|
|
Bucket = written.Bucket,
|
|
ObjectKey = written.ObjectKey,
|
|
MimeType = "application/gzip",
|
|
FileSizeBytes = written.SizeBytes,
|
|
ChecksumSha256 = written.ChecksumSha256,
|
|
UploadStatus = AssetUploadStatus.Verified,
|
|
VerifiedAt = DateTimeOffset.UtcNow,
|
|
VerifiedSizeBytes = written.SizeBytes,
|
|
SecurityScanStatus = AssetSecurityScanStatus.NotRequired,
|
|
Source = "tenant_export"
|
|
};
|
|
dbContext.ContentAssets.Add(exportAsset);
|
|
operation.ExportAssetId = exportAsset.Id;
|
|
operation.Status = TenantLifecycleOperationStatus.Succeeded;
|
|
operation.CompletedAt = DateTimeOffset.UtcNow;
|
|
operation.Result = JsonSerializer.SerializeToElement(new
|
|
{
|
|
exportAssetId = exportAsset.Id,
|
|
written.SizeBytes,
|
|
assetCount = assets.Length
|
|
});
|
|
await dbContext.SaveChangesAsync(cancellationToken);
|
|
return new BackgroundJobHandlerResult(operation.Result, exportAsset.Id);
|
|
}
|
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
|
{
|
|
operation.Status = TenantLifecycleOperationStatus.Failed;
|
|
operation.LastError = exception.Message;
|
|
operation.CompletedAt = DateTimeOffset.UtcNow;
|
|
await dbContext.SaveChangesAsync(cancellationToken);
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
if (File.Exists(temporaryPath)) File.Delete(temporaryPath);
|
|
}
|
|
}
|
|
|
|
private static async Task WriteJsonEntryAsync<T>(
|
|
TarWriter archive,
|
|
string name,
|
|
T value,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var stream = new MemoryStream();
|
|
await JsonSerializer.SerializeAsync(stream, value, cancellationToken: cancellationToken);
|
|
stream.Position = 0;
|
|
archive.WriteEntry(new PaxTarEntry(TarEntryType.RegularFile, name) { DataStream = stream });
|
|
await stream.DisposeAsync();
|
|
}
|
|
|
|
private static async Task WriteJsonLinesEntryAsync<T>(
|
|
TarWriter archive,
|
|
string name,
|
|
IEnumerable<T> values,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var stream = new MemoryStream();
|
|
await using (var writer = new StreamWriter(stream, new UTF8Encoding(false), leaveOpen: true))
|
|
{
|
|
foreach (var value in values)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
await writer.WriteLineAsync(JsonSerializer.Serialize(value));
|
|
}
|
|
}
|
|
|
|
stream.Position = 0;
|
|
archive.WriteEntry(new PaxTarEntry(TarEntryType.RegularFile, name) { DataStream = stream });
|
|
await stream.DisposeAsync();
|
|
}
|
|
|
|
private static string SanitizeTarPath(string value)
|
|
{
|
|
var name = Path.GetFileName(value).Replace('\\', '_').Replace('/', '_');
|
|
return string.IsNullOrWhiteSpace(name) ? "asset.bin" : name;
|
|
}
|
|
}
|