496 lines
19 KiB
C#
496 lines
19 KiB
C#
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using System.Text.Json;
|
|
using Tiku.Application.Content;
|
|
using Tiku.Application.Jobs;
|
|
using Tiku.Application.Security;
|
|
using Tiku.Application.Tenancy;
|
|
using Tiku.Domain.Commerce;
|
|
using Tiku.Domain.Common;
|
|
using Tiku.Domain.Content;
|
|
using Tiku.Domain.Operations;
|
|
using Tiku.Infrastructure.Persistence;
|
|
|
|
namespace Tiku.Infrastructure.Jobs;
|
|
|
|
internal sealed class BackgroundJobService(
|
|
TikuDbContext dbContext,
|
|
ITenantExecutionScope tenantExecutionScope,
|
|
ICapabilityAccessEvaluator capabilityAccessEvaluator,
|
|
IBackgroundJobDispatcher backgroundJobDispatcher) : IBackgroundJobService
|
|
{
|
|
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(5);
|
|
|
|
public async Task<BackgroundJobItem> EnqueueAsync(
|
|
CreateBackgroundJobCommand command,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var normalizedJobType = NormalizeJobType(command.JobType);
|
|
if (!await capabilityAccessEvaluator.IsAllowedAsync(
|
|
command.TenantId,
|
|
ResolveCapabilityModule(normalizedJobType),
|
|
CapabilityOperation.Write,
|
|
cancellationToken))
|
|
{
|
|
throw new InvalidOperationException("Tenant capability does not allow this background job.");
|
|
}
|
|
var job = new BackgroundJob
|
|
{
|
|
TenantId = command.TenantId,
|
|
JobType = normalizedJobType,
|
|
Payload = command.Payload,
|
|
RunAfter = command.RunAfter,
|
|
MaxRetries = Math.Clamp(command.MaxRetries, 0, 20)
|
|
};
|
|
dbContext.BackgroundJobs.Add(job);
|
|
if (job.RunAfter is null && backgroundJobDispatcher.IsEnabled)
|
|
{
|
|
await backgroundJobDispatcher.DispatchAsync(
|
|
job.Id,
|
|
job.TenantId,
|
|
job.JobType,
|
|
job.Id.ToString("N"),
|
|
cancellationToken);
|
|
}
|
|
await dbContext.SaveChangesAsync(cancellationToken);
|
|
return ToItem(job);
|
|
}
|
|
|
|
public async Task<int> ProcessPendingAsync(
|
|
string workerId,
|
|
int batchSize,
|
|
bool includeImmediateJobs = true,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var now = DateTimeOffset.UtcNow;
|
|
var jobs = await dbContext.BackgroundJobs
|
|
.Where(job =>
|
|
job.Status == BackgroundJobStatus.Pending &&
|
|
(includeImmediateJobs || job.RunAfter != null) &&
|
|
(job.RunAfter == null || job.RunAfter <= now))
|
|
.OrderBy(job => job.CreatedAt)
|
|
.Take(Math.Clamp(batchSize, 1, 100))
|
|
.ToArrayAsync(cancellationToken);
|
|
|
|
var processed = 0;
|
|
foreach (var job in jobs)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
if (await ProcessJobAsync(job, workerId, cancellationToken)) processed++;
|
|
}
|
|
|
|
return processed;
|
|
}
|
|
|
|
public async Task<bool> ProcessRequestedAsync(
|
|
Guid jobId,
|
|
Guid tenantId,
|
|
string jobType,
|
|
string workerId,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var normalizedJobType = NormalizeJobType(jobType);
|
|
var job = await dbContext.BackgroundJobs.SingleOrDefaultAsync(
|
|
item => item.Id == jobId && item.TenantId == tenantId,
|
|
cancellationToken);
|
|
if (job is null)
|
|
{
|
|
throw new InvalidOperationException("The requested background job does not exist in the target tenant.");
|
|
}
|
|
if (!string.Equals(job.JobType, normalizedJobType, StringComparison.Ordinal))
|
|
{
|
|
throw new InvalidOperationException("The requested background job type does not match the persisted job.");
|
|
}
|
|
if (job.Status != BackgroundJobStatus.Pending || job.RunAfter is not null)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
return await ProcessJobAsync(job, workerId, cancellationToken);
|
|
}
|
|
|
|
private async Task<bool> ProcessJobAsync(
|
|
BackgroundJob job,
|
|
string workerId,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
if (job.Status != BackgroundJobStatus.Pending)
|
|
{
|
|
return false;
|
|
}
|
|
if (!await capabilityAccessEvaluator.IsAllowedAsync(
|
|
job.TenantId,
|
|
ResolveCapabilityModule(job.JobType),
|
|
CapabilityOperation.Write,
|
|
cancellationToken))
|
|
{
|
|
job.Status = BackgroundJobStatus.Failed;
|
|
job.CompletedAt = DateTimeOffset.UtcNow;
|
|
job.LastError = "Tenant capability was revoked before job execution.";
|
|
await dbContext.SaveChangesAsync(cancellationToken);
|
|
return true;
|
|
}
|
|
|
|
var now = DateTimeOffset.UtcNow;
|
|
job.Status = BackgroundJobStatus.Processing;
|
|
job.LockedBy = workerId;
|
|
job.LockExpiresAt = now.Add(LeaseDuration);
|
|
job.StartedAt = now;
|
|
await dbContext.SaveChangesAsync(cancellationToken);
|
|
|
|
try
|
|
{
|
|
var result = await tenantExecutionScope.ExecuteAsync(
|
|
new SystemScopeRequest(
|
|
job.TenantId, SystemScopeCallerType.Worker, workerId,
|
|
$"Background job {job.JobType}", job.Id.ToString("N")),
|
|
(provider, token) => ProcessCoreAsync(provider, job, token),
|
|
cancellationToken);
|
|
job.Status = BackgroundJobStatus.Succeeded;
|
|
job.CompletedAt = DateTimeOffset.UtcNow;
|
|
job.LastError = null;
|
|
job.Result = result;
|
|
}
|
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
|
{
|
|
job.RetryCount++;
|
|
job.LastError = exception.Message;
|
|
job.Status = job.RetryCount > job.MaxRetries
|
|
? BackgroundJobStatus.Failed
|
|
: BackgroundJobStatus.Pending;
|
|
job.RunAfter = job.Status == BackgroundJobStatus.Pending
|
|
? DateTimeOffset.UtcNow.AddSeconds(Math.Min(300, 10 * job.RetryCount))
|
|
: null;
|
|
}
|
|
finally
|
|
{
|
|
job.LockedBy = null;
|
|
job.LockExpiresAt = null;
|
|
await dbContext.SaveChangesAsync(cancellationToken);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
public async Task<IReadOnlyCollection<BackgroundJobItem>> ListAsync(
|
|
Guid tenantId,
|
|
string? jobType = null,
|
|
int limit = 50,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var query = dbContext.BackgroundJobs.AsNoTracking()
|
|
.Where(job => job.TenantId == tenantId);
|
|
if (!string.IsNullOrWhiteSpace(jobType))
|
|
{
|
|
var normalized = NormalizeJobType(jobType);
|
|
query = query.Where(job => job.JobType == normalized);
|
|
}
|
|
|
|
var jobs = await query
|
|
.OrderByDescending(job => job.CreatedAt)
|
|
.Take(Math.Clamp(limit, 1, 200))
|
|
.ToArrayAsync(cancellationToken);
|
|
return jobs.Select(ToItem).ToArray();
|
|
}
|
|
|
|
private async Task<JsonElement> ProcessCoreAsync(
|
|
IServiceProvider scopedProvider,
|
|
BackgroundJob job,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
var scopedDbContext = scopedProvider.GetRequiredService<TikuDbContext>();
|
|
return job.JobType switch
|
|
{
|
|
"content_export" => await ProcessContentExportAsync(scopedDbContext, job, cancellationToken),
|
|
"content_import" => await ProcessContentImportAsync(scopedProvider, job, cancellationToken),
|
|
"asset_security_scan" => throw new NotSupportedException("asset_security_scan requires a configured scanner provider before it can write scan results."),
|
|
"statistics_aggregation" => await ProcessStatisticsAggregationAsync(scopedDbContext, job, cancellationToken),
|
|
"commerce_reconciliation" => await ProcessCommerceReconciliationAsync(scopedDbContext, job, cancellationToken),
|
|
"tenant_domain_recheck" => await ProcessTenantDomainRecheckAsync(scopedProvider, cancellationToken),
|
|
_ => throw new InvalidOperationException($"Unsupported background job type '{job.JobType}'.")
|
|
};
|
|
}
|
|
|
|
private static async Task<JsonElement> ProcessContentImportAsync(
|
|
IServiceProvider scopedProvider,
|
|
BackgroundJob job,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var directContentService = scopedProvider.GetRequiredService<IDirectContentService>();
|
|
var createdBy = GetJsonGuid(job.Payload, "createdBy") ?? Guid.Empty;
|
|
if (createdBy == Guid.Empty)
|
|
{
|
|
throw new InvalidOperationException("content_import job requires createdBy.");
|
|
}
|
|
|
|
var command = new SimpleImportCommand(
|
|
GetJsonString(job.Payload, "importType") ?? throw new InvalidOperationException("content_import job requires importType."),
|
|
GetJsonString(job.Payload, "sourceFormat"),
|
|
GetJsonString(job.Payload, "sourceName"),
|
|
GetJsonGuid(job.Payload, "regionId"),
|
|
GetJsonGuid(job.Payload, "entryId"),
|
|
GetJsonGuid(job.Payload, "contentNodeId"),
|
|
GetJsonGuid(job.Payload, "subjectId"),
|
|
GetJsonGuid(job.Payload, "categoryId"),
|
|
GetJsonGuid(job.Payload, "questionBankId"),
|
|
GetJsonGuid(job.Payload, "collectionId"),
|
|
GetJsonArray(job.Payload, "items"),
|
|
false);
|
|
var result = await directContentService.ExecuteImportAsync(
|
|
new DirectContentActor(job.TenantId, createdBy),
|
|
command,
|
|
cancellationToken);
|
|
return JsonSerializer.SerializeToElement(new
|
|
{
|
|
importJobId = result.Job.Id,
|
|
result.Job.ImportType,
|
|
result.Job.Status,
|
|
result.Job.TotalCount,
|
|
result.Job.InsertedCount,
|
|
result.Job.UpdatedCount,
|
|
result.Job.ErrorCount
|
|
});
|
|
}
|
|
|
|
private async Task<JsonElement> ProcessContentExportAsync(
|
|
TikuDbContext scopedDbContext,
|
|
BackgroundJob job,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var exportType = GetJsonString(job.Payload, "exportType") ?? "summary";
|
|
var assetKey = $"background-jobs/{job.Id:N}/content-export.json";
|
|
var asset = await scopedDbContext.ContentAssets.SingleOrDefaultAsync(
|
|
item => item.TenantId == job.TenantId && item.AssetKey == assetKey,
|
|
cancellationToken);
|
|
if (asset is null)
|
|
{
|
|
asset = new ContentAsset
|
|
{
|
|
TenantId = job.TenantId,
|
|
AssetKey = assetKey,
|
|
AssetType = ContentAssetType.Document,
|
|
StorageProvider = AssetStorageProvider.ExternalUrl,
|
|
UploadStatus = AssetUploadStatus.Verified,
|
|
SecurityScanStatus = AssetSecurityScanStatus.NotRequired,
|
|
Source = "background_job"
|
|
};
|
|
scopedDbContext.ContentAssets.Add(asset);
|
|
}
|
|
|
|
var questionBankCount = await scopedDbContext.QuestionBanks.CountAsync(item => item.TenantId == job.TenantId, cancellationToken);
|
|
var questionCount = await scopedDbContext.Questions.CountAsync(item => item.TenantId == job.TenantId, cancellationToken);
|
|
var studentCount = await scopedDbContext.StudentProfiles.CountAsync(item => item.TenantId == job.TenantId, cancellationToken);
|
|
asset.FileName = $"content-export-{DateTimeOffset.UtcNow:yyyyMMddHHmmss}.json";
|
|
asset.Title = "Content export manifest";
|
|
asset.Description = $"Generated content export manifest for {exportType}.";
|
|
asset.ObjectKey = assetKey;
|
|
asset.MimeType = "application/json";
|
|
asset.Metadata = JsonSerializer.SerializeToElement(new
|
|
{
|
|
exportType,
|
|
generatedAt = DateTimeOffset.UtcNow,
|
|
questionBankCount,
|
|
questionCount,
|
|
studentCount,
|
|
payload = job.Payload
|
|
});
|
|
await scopedDbContext.SaveChangesAsync(cancellationToken);
|
|
job.OutputAssetId = asset.Id;
|
|
return JsonSerializer.SerializeToElement(new
|
|
{
|
|
outputAssetId = asset.Id,
|
|
asset.AssetKey,
|
|
questionBankCount,
|
|
questionCount,
|
|
studentCount
|
|
});
|
|
}
|
|
|
|
private async Task<JsonElement> ProcessCommerceReconciliationAsync(
|
|
TikuDbContext scopedDbContext,
|
|
BackgroundJob job,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var provider = NormalizeProvider(GetJsonString(job.Payload, "provider"));
|
|
var hasProviderConfig = await scopedDbContext.TenantExternalProviders.AnyAsync(
|
|
item =>
|
|
item.TenantId == job.TenantId &&
|
|
item.Capability == Tiku.Domain.Tenancy.TenantExternalProviderCapability.Payment &&
|
|
item.Provider == provider &&
|
|
item.Status == Tiku.Domain.Tenancy.TenantExternalProviderStatus.Active,
|
|
cancellationToken);
|
|
if (!hasProviderConfig)
|
|
{
|
|
throw new InvalidOperationException($"Active payment provider '{provider}' is required for commerce reconciliation job.");
|
|
}
|
|
|
|
var billDate = GetJsonDateOnly(job.Payload, "billDate") ?? DateOnly.FromDateTime(DateTime.UtcNow.Date);
|
|
var billType = GetJsonEnum(job.Payload, "billType", ReconciliationBillType.Combined);
|
|
var sourceHash = $"background-job:{job.Id:N}";
|
|
var batch = await scopedDbContext.CommerceReconciliationBatches.SingleOrDefaultAsync(
|
|
item =>
|
|
item.TenantId == job.TenantId &&
|
|
item.Provider == provider &&
|
|
item.Source == ReconciliationSource.ProviderDownload &&
|
|
item.SourceHash == sourceHash,
|
|
cancellationToken);
|
|
if (batch is null)
|
|
{
|
|
batch = new CommerceReconciliationBatch
|
|
{
|
|
TenantId = job.TenantId,
|
|
Provider = provider,
|
|
BillDate = billDate,
|
|
BillType = billType,
|
|
Source = ReconciliationSource.ProviderDownload,
|
|
SourceName = $"provider-bill:{provider}:{billDate:yyyyMMdd}",
|
|
SourceHash = sourceHash,
|
|
Status = ReconciliationBatchStatus.Pending,
|
|
Metadata = JsonSerializer.SerializeToElement(new
|
|
{
|
|
jobId = job.Id,
|
|
note = "Provider bill job created the reconciliation batch; provider download/parser is handled by a dedicated provider processor."
|
|
})
|
|
};
|
|
scopedDbContext.CommerceReconciliationBatches.Add(batch);
|
|
await scopedDbContext.SaveChangesAsync(cancellationToken);
|
|
}
|
|
|
|
return JsonSerializer.SerializeToElement(new
|
|
{
|
|
batchId = batch.Id,
|
|
provider,
|
|
billDate,
|
|
billType = billType.ToString(),
|
|
status = batch.Status.ToString()
|
|
});
|
|
}
|
|
|
|
private static async Task<JsonElement> ProcessTenantDomainRecheckAsync(
|
|
IServiceProvider scopedProvider,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var lifecycleService = scopedProvider.GetRequiredService<ITenantDomainLifecycleService>();
|
|
var processed = await lifecycleService.ProcessPendingAsync(cancellationToken);
|
|
return JsonSerializer.SerializeToElement(new
|
|
{
|
|
processed
|
|
});
|
|
}
|
|
|
|
private async Task<JsonElement> ProcessStatisticsAggregationAsync(
|
|
TikuDbContext scopedDbContext,
|
|
BackgroundJob job,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var since = DateTimeOffset.UtcNow.AddDays(-7);
|
|
var activeLearnerCount = await scopedDbContext.PracticeSessions
|
|
.Where(item => item.TenantId == job.TenantId && item.StartedAt >= since)
|
|
.Select(item => item.UserId)
|
|
.Distinct()
|
|
.CountAsync(cancellationToken);
|
|
var paidOrderCount = await scopedDbContext.Orders
|
|
.CountAsync(item => item.TenantId == job.TenantId && item.Status == OrderStatus.Paid, cancellationToken);
|
|
var revenueCents = await scopedDbContext.Orders
|
|
.Where(item => item.TenantId == job.TenantId &&
|
|
(item.Status == OrderStatus.Paid ||
|
|
item.Status == OrderStatus.PartiallyRefunded ||
|
|
item.Status == OrderStatus.Refunded))
|
|
.SumAsync(item => item.AmountCents - item.RefundedAmountCents, cancellationToken);
|
|
return JsonSerializer.SerializeToElement(new
|
|
{
|
|
since,
|
|
activeLearnerCount,
|
|
paidOrderCount,
|
|
revenueCents
|
|
});
|
|
}
|
|
|
|
private static string NormalizeJobType(string jobType)
|
|
{
|
|
return jobType.Trim().ToLowerInvariant();
|
|
}
|
|
|
|
private static string ResolveCapabilityModule(string jobType) => jobType switch
|
|
{
|
|
"content_export" or "content_import" or "asset_security_scan" => "content",
|
|
"statistics_aggregation" => "dashboard",
|
|
"commerce_reconciliation" => "commerce",
|
|
"tenant_domain_recheck" => "settings",
|
|
_ => "job"
|
|
};
|
|
|
|
private static string NormalizeProvider(string? provider)
|
|
{
|
|
var normalized = (provider ?? string.Empty).Trim().ToLowerInvariant();
|
|
return string.IsNullOrWhiteSpace(normalized)
|
|
? throw new InvalidOperationException("Background job provider is required.")
|
|
: normalized;
|
|
}
|
|
|
|
private static string? GetJsonString(JsonElement element, string propertyName)
|
|
{
|
|
return element.ValueKind == JsonValueKind.Object &&
|
|
element.TryGetProperty(propertyName, out var property) &&
|
|
property.ValueKind == JsonValueKind.String
|
|
? property.GetString()
|
|
: null;
|
|
}
|
|
|
|
private static Guid? GetJsonGuid(JsonElement element, string propertyName)
|
|
{
|
|
if (element.ValueKind != JsonValueKind.Object ||
|
|
!element.TryGetProperty(propertyName, out var property))
|
|
{
|
|
return null;
|
|
}
|
|
|
|
return property.ValueKind == JsonValueKind.String && Guid.TryParse(property.GetString(), out var value)
|
|
? value
|
|
: null;
|
|
}
|
|
|
|
private static IReadOnlyCollection<JsonElement> GetJsonArray(JsonElement element, string propertyName)
|
|
{
|
|
if (element.ValueKind != JsonValueKind.Object ||
|
|
!element.TryGetProperty(propertyName, out var property) ||
|
|
property.ValueKind != JsonValueKind.Array)
|
|
{
|
|
return [];
|
|
}
|
|
|
|
return property.EnumerateArray().Select(item => item.Clone()).ToArray();
|
|
}
|
|
|
|
private static DateOnly? GetJsonDateOnly(JsonElement element, string propertyName)
|
|
{
|
|
var value = GetJsonString(element, propertyName);
|
|
return DateOnly.TryParse(value, out var parsed) ? parsed : null;
|
|
}
|
|
|
|
private static TEnum GetJsonEnum<TEnum>(JsonElement element, string propertyName, TEnum fallback)
|
|
where TEnum : struct
|
|
{
|
|
var value = GetJsonString(element, propertyName);
|
|
return Enum.TryParse<TEnum>(value, true, out var parsed) ? parsed : fallback;
|
|
}
|
|
|
|
private static BackgroundJobItem ToItem(BackgroundJob job)
|
|
{
|
|
return new BackgroundJobItem(
|
|
job.Id,
|
|
job.TenantId,
|
|
job.JobType,
|
|
job.Status,
|
|
job.RetryCount,
|
|
job.MaxRetries,
|
|
job.RunAfter,
|
|
job.StartedAt,
|
|
job.CompletedAt,
|
|
job.LastError,
|
|
job.OutputAssetId,
|
|
job.Result);
|
|
}
|
|
}
|