perf: optimize authorization scoreline and workers

This commit is contained in:
2026-07-30 09:12:27 +08:00
parent 4bea745b79
commit bedc77bffd
52 changed files with 21911 additions and 347 deletions

View File

@@ -1,4 +1,5 @@
using System.Text.Json;
using System.Text;
using System.Text.RegularExpressions;
using Microsoft.EntityFrameworkCore;
using Tiku.Application.Catalog;
@@ -8,7 +9,9 @@ using Tiku.Infrastructure.Persistence;
namespace Tiku.Infrastructure.Scoreline;
public sealed partial class ScorelineQueryService(TikuDbContext dbContext) : IScorelineQueryService
public sealed partial class ScorelineQueryService(
TikuDbContext dbContext,
ScorelineRecordQuery recordQuery) : IScorelineQueryService
{
public async Task<CatalogList<ScorelineFieldItem>> GetFieldsAsync(
ScorelineFilter filter,
@@ -36,33 +39,37 @@ public sealed partial class ScorelineQueryService(TikuDbContext dbContext) : ISc
{
var page = Math.Max(filter.Page ?? 1, 1);
var pageSize = Math.Clamp(filter.PageSize ?? 20, 1, 200);
var records = await BuildRecordQuery(filter)
.OrderByDescending(record => record.Year)
.ThenBy(record => record.SchoolName)
.ThenBy(record => record.MajorName)
.ToArrayAsync(cancellationToken);
ValidateDynamicFilterKeys(filter.DynamicFilters);
var result = await recordQuery.ExecutePageAsync(filter, page, pageSize, cancellationToken);
return new ScorelineRecordPage(page, pageSize, result.Total ?? 0, result.Items);
}
var filtered = await ApplyDynamicFiltersAsync(filter, records, cancellationToken);
var items = filtered
.Skip((page - 1) * pageSize)
.Take(pageSize)
.Select(ToRecordItem)
.ToArray();
return new ScorelineRecordPage(page, pageSize, filtered.Count, items);
public async Task<ScorelineRecordCursorPage> GetRecordsCursorAsync(
ScorelineFilter filter,
string? cursor,
CancellationToken cancellationToken = default)
{
ValidateDynamicFilterKeys(filter.DynamicFilters);
var pageSize = Math.Clamp(filter.PageSize ?? 20, 1, 200);
var position = DecodeCursor(cursor);
var result = await recordQuery.ExecuteCursorAsync(filter, pageSize, position, cancellationToken);
var hasMore = result.Items.Count > pageSize;
var items = result.Items.Take(pageSize).ToArray();
var nextCursor = hasMore && items.Length > 0 ? EncodeCursor(items[^1]) : null;
return new ScorelineRecordCursorPage(pageSize, hasMore, nextCursor, items);
}
public async Task<CatalogList<ScorelineRecordItem>> GetTrendAsync(
ScorelineFilter filter,
CancellationToken cancellationToken = default)
{
var records = await BuildRecordQuery(filter)
.OrderBy(record => record.Year)
.ThenBy(record => record.MajorName)
.Take(ResolveLimit(filter.Limit, 100))
.ToArrayAsync(cancellationToken);
var filtered = await ApplyDynamicFiltersAsync(filter, records, cancellationToken);
return new CatalogList<ScorelineRecordItem>(filtered.Select(ToRecordItem).ToArray());
ValidateDynamicFilterKeys(filter.DynamicFilters);
var trendFilter = filter with { Page = 1, PageSize = ResolveLimit(filter.Limit, 100) };
var result = await recordQuery.ExecutePageAsync(trendFilter, 1, trendFilter.PageSize!.Value, cancellationToken);
return new CatalogList<ScorelineRecordItem>(result.Items
.OrderBy(item => item.Year)
.ThenBy(item => item.MajorName)
.ToArray());
}
public async Task<CatalogList<int>> GetYearsAsync(
@@ -114,111 +121,73 @@ public sealed partial class ScorelineQueryService(TikuDbContext dbContext) : ISc
return query;
}
private async Task<IReadOnlyList<ScorelineRecord>> ApplyDynamicFiltersAsync(
ScorelineFilter filter,
IReadOnlyCollection<ScorelineRecord> records,
CancellationToken cancellationToken)
private static void ValidateDynamicFilterKeys(IReadOnlyCollection<ScorelineDynamicFilter>? filters)
{
if (filter.DynamicFilters is null || filter.DynamicFilters.Count == 0)
{
return records.ToArray();
}
var allowedFields = await dbContext.ScorelineFields.AsNoTracking()
.Where(field =>
field.TenantId == filter.TenantId &&
field.IsFilter &&
(filter.RegionId == null || field.RegionId == filter.RegionId || field.RegionId == null))
.ToArrayAsync(cancellationToken);
var allowed = allowedFields
.GroupBy(field => field.FieldKey, StringComparer.Ordinal)
.ToDictionary(group => group.Key, group => group.OrderBy(field => field.SortOrder).First(), StringComparer.Ordinal);
foreach (var dynamicFilter in filter.DynamicFilters)
foreach (var dynamicFilter in filters ?? [])
{
if (!FieldKeyRegex().IsMatch(dynamicFilter.FieldKey))
{
throw new ScorelineQueryException("Scoreline field filter key is invalid.", "scoreline_field_filter_key_invalid");
}
if (!allowed.ContainsKey(dynamicFilter.FieldKey))
if (dynamicFilter.Operator is not ("field" or "min" or "max"))
{
throw new ScorelineQueryException("Scoreline field filter is not enabled.", "scoreline_field_filter_not_allowed");
throw new ScorelineQueryException("Scoreline field filter operator is invalid.", "scoreline_field_filter_operator_invalid");
}
}
return records
.Where(record => filter.DynamicFilters.All(dynamicFilter => MatchDynamicFilter(record, allowed[dynamicFilter.FieldKey], dynamicFilter)))
.ToArray();
}
private static bool MatchDynamicFilter(
ScorelineRecord record,
ScorelineField field,
ScorelineDynamicFilter filter)
private static string EncodeCursor(ScorelineRecordItem item)
{
if (record.FieldValues.ValueKind != JsonValueKind.Object ||
!record.FieldValues.TryGetProperty(filter.FieldKey, out var value))
var json = JsonSerializer.SerializeToUtf8Bytes(new
{
return false;
}
var fieldType = field.FieldType.Trim().ToLowerInvariant();
if (filter.Operator is "min" or "max")
{
if (!IsNumericType(fieldType))
{
throw new ScorelineQueryException("Scoreline range filter requires a numeric field.", "scoreline_field_range_type_invalid");
}
if (!TryGetDecimal(value, out var actual) || !decimal.TryParse(filter.Value, out var expected))
{
throw new ScorelineQueryException("Scoreline range filter value must be numeric.", "scoreline_field_range_value_invalid");
}
return filter.Operator == "min" ? actual >= expected : actual <= expected;
}
if (IsNumericType(fieldType))
{
if (!TryGetDecimal(value, out var actual) || !decimal.TryParse(filter.Value, out var expected))
{
throw new ScorelineQueryException("Scoreline numeric filter value must be numeric.", "scoreline_field_value_invalid");
}
return actual == expected;
}
var actualText = value.ValueKind == JsonValueKind.String ? value.GetString() : value.ToString();
return fieldType switch
{
"text" or "textarea" or "string" => actualText?.Contains(filter.Value, StringComparison.OrdinalIgnoreCase) == true,
"boolean" or "bool" => string.Equals(actualText, filter.Value, StringComparison.OrdinalIgnoreCase),
_ => string.Equals(actualText, filter.Value, StringComparison.OrdinalIgnoreCase)
};
v = 1,
year = item.Year,
schoolName = item.SchoolName,
majorName = item.MajorName,
id = item.Id
});
return Convert.ToBase64String(json).TrimEnd('=').Replace('+', '-').Replace('/', '_');
}
private static bool IsNumericType(string fieldType)
private static ScorelineCursorPosition? DecodeCursor(string? cursor)
{
return fieldType is "number" or "integer" or "decimal" or "float";
}
private static bool TryGetDecimal(JsonElement value, out decimal result)
{
if (value.ValueKind == JsonValueKind.Number)
if (string.IsNullOrWhiteSpace(cursor))
{
return value.TryGetDecimal(out result);
return null;
}
if (value.ValueKind == JsonValueKind.String)
try
{
return decimal.TryParse(value.GetString(), out result);
}
var normalized = cursor.Replace('-', '+').Replace('_', '/');
normalized = normalized.PadRight(normalized.Length + ((4 - normalized.Length % 4) % 4), '=');
using var document = JsonDocument.Parse(Convert.FromBase64String(normalized));
var root = document.RootElement;
if (root.GetProperty("v").GetInt32() != 1 ||
!root.TryGetProperty("year", out var year) ||
!root.TryGetProperty("id", out var id) ||
!Guid.TryParse(id.GetString(), out var recordId))
{
throw new FormatException();
}
result = 0;
return false;
return new ScorelineCursorPosition(
year.GetInt32(),
ReadNullableString(root, "schoolName"),
ReadNullableString(root, "majorName"),
recordId);
}
catch (Exception exception) when (exception is FormatException or JsonException or InvalidOperationException or KeyNotFoundException)
{
throw new ScorelineQueryException("Scoreline cursor is invalid or unsupported.", "scoreline_cursor_invalid");
}
}
private static string? ReadNullableString(JsonElement root, string name) =>
root.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String
? value.GetString()
: null;
private static ScorelineFieldItem ToFieldItem(ScorelineField field)
{
return new ScorelineFieldItem(

View File

@@ -0,0 +1,292 @@
using System.Data;
using System.Globalization;
using System.Text;
using System.Text.Json;
using Npgsql;
using NpgsqlTypes;
using Tiku.Application.Scoreline;
namespace Tiku.Infrastructure.Scoreline;
internal sealed record ScorelineCursorPosition(int Year, string? SchoolName, string? MajorName, Guid Id);
internal sealed record ScorelineQueryResult(int? Total, IReadOnlyList<ScorelineRecordItem> Items);
public sealed class ScorelineRecordQuery(NpgsqlDataSource dataSource)
{
internal async Task<ScorelineQueryResult> ExecutePageAsync(
ScorelineFilter filter,
int page,
int pageSize,
CancellationToken cancellationToken)
{
var offset = checked((page - 1) * pageSize);
return await ExecuteAsync(filter, pageSize, offset, null, true, cancellationToken);
}
internal Task<ScorelineQueryResult> ExecuteCursorAsync(
ScorelineFilter filter,
int pageSize,
ScorelineCursorPosition? cursor,
CancellationToken cancellationToken) =>
ExecuteAsync(filter, pageSize + 1, null, cursor, false, cancellationToken);
private async Task<ScorelineQueryResult> ExecuteAsync(
ScorelineFilter filter,
int limit,
int? offset,
ScorelineCursorPosition? cursor,
bool includeCount,
CancellationToken cancellationToken)
{
await using var connection = await dataSource.OpenConnectionAsync(cancellationToken);
await using var command = connection.CreateCommand();
AddParameters(command, filter, limit, offset, cursor);
command.CommandText = BuildSql(filter.DynamicFilters ?? [], includeCount, cursor is not null, offset.HasValue);
await using var reader = await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken);
await ValidateFiltersAsync(reader, filter.DynamicFilters ?? [], cancellationToken);
int? total = null;
if (includeCount)
{
if (!await reader.NextResultAsync(cancellationToken) || !await reader.ReadAsync(cancellationToken))
{
throw new InvalidOperationException("Scoreline count result was missing.");
}
total = reader.GetInt32(0);
}
if (!await reader.NextResultAsync(cancellationToken))
{
throw new InvalidOperationException("Scoreline item result was missing.");
}
var items = new List<ScorelineRecordItem>(limit);
while (await reader.ReadAsync(cancellationToken))
{
items.Add(new ScorelineRecordItem(
reader.GetGuid(0),
reader.IsDBNull(1) ? null : reader.GetString(1),
reader.IsDBNull(2) ? null : reader.GetGuid(2),
reader.IsDBNull(3) ? null : reader.GetGuid(3),
reader.IsDBNull(4) ? null : reader.GetGuid(4),
reader.GetInt32(5),
reader.IsDBNull(6) ? null : reader.GetString(6),
reader.IsDBNull(7) ? null : reader.GetString(7),
JsonDocument.Parse(reader.GetString(8)).RootElement.Clone()));
}
return new ScorelineQueryResult(total, items);
}
private static async Task ValidateFiltersAsync(
NpgsqlDataReader reader,
IReadOnlyCollection<ScorelineDynamicFilter> filters,
CancellationToken cancellationToken)
{
var validated = 0;
while (await reader.ReadAsync(cancellationToken))
{
validated++;
var operation = reader.GetString(0);
var value = reader.GetString(2);
if (reader.IsDBNull(3))
{
throw new ScorelineQueryException("Scoreline field filter is not enabled.", "scoreline_field_filter_not_allowed");
}
var fieldType = reader.GetString(3).Trim().ToLowerInvariant();
if (operation is "min" or "max")
{
if (!IsNumericType(fieldType))
{
throw new ScorelineQueryException("Scoreline range filter requires a numeric field.", "scoreline_field_range_type_invalid");
}
if (!decimal.TryParse(value, NumberStyles.Number, CultureInfo.InvariantCulture, out _))
{
throw new ScorelineQueryException("Scoreline range filter value must be numeric.", "scoreline_field_range_value_invalid");
}
}
else if (IsNumericType(fieldType) &&
!decimal.TryParse(value, NumberStyles.Number, CultureInfo.InvariantCulture, out _))
{
throw new ScorelineQueryException("Scoreline numeric filter value must be numeric.", "scoreline_field_value_invalid");
}
else if (fieldType is "boolean" or "bool" && !bool.TryParse(value, out _))
{
throw new ScorelineQueryException("Scoreline boolean filter value must be true or false.", "scoreline_field_value_invalid");
}
}
if (validated != filters.Count)
{
throw new InvalidOperationException("Scoreline filter validation result was incomplete.");
}
}
private static string BuildSql(
IReadOnlyCollection<ScorelineDynamicFilter> filters,
bool includeCount,
bool includeCursor,
bool includeOffset)
{
var validationCte = BuildFilterCte(filters);
var baseWhere = BuildBaseWhere(includeCursor);
var dynamicWhere = filters.Count == 0 ? string.Empty : DynamicWhere;
var sql = new StringBuilder();
sql.Append(validationCte).AppendLine()
.AppendLine("SELECT operator, field_key, value, field_type FROM allowed_filters ORDER BY ordinal;");
if (includeCount)
{
sql.Append(validationCte).AppendLine()
.Append("SELECT count(*)::integer FROM scoreline_records r WHERE ")
.Append(baseWhere).Append(dynamicWhere).AppendLine(";");
}
sql.Append(validationCte).AppendLine()
.Append("SELECT r.id, r.legacy_id, r.region_id, r.school_id, r.major_id, r.year, r.school_name, r.major_name, r.field_values::text ")
.Append("FROM scoreline_records r WHERE ").Append(baseWhere).Append(dynamicWhere)
.AppendLine(" ORDER BY r.year DESC, r.school_name ASC NULLS LAST, r.major_name ASC NULLS LAST, r.id ASC")
.Append(" LIMIT @limit");
if (includeOffset)
{
sql.Append(" OFFSET @offset");
}
sql.Append(';');
return sql.ToString();
}
private static string BuildFilterCte(IReadOnlyCollection<ScorelineDynamicFilter> filters)
{
var requested = filters.Count == 0
? "SELECT NULL::text AS operator, NULL::text AS field_key, NULL::text AS value, NULL::integer AS ordinal WHERE FALSE"
: "VALUES " + string.Join(", ", filters.Select((_, index) => $"(@filter_operator_{index}, @filter_key_{index}, @filter_value_{index}, {index})"));
return $$"""
WITH requested_filters(operator, field_key, value, ordinal) AS ({{requested}}),
allowed_filters AS (
SELECT requested.operator, requested.field_key, requested.value, requested.ordinal, configured.field_type
FROM requested_filters requested
LEFT JOIN LATERAL (
SELECT field.field_type
FROM scoreline_fields field
WHERE field.tenant_id = @tenant_id
AND field.is_filter
AND field.field_key = requested.field_key
AND (@region_id IS NULL OR field.region_id = @region_id OR field.region_id IS NULL)
ORDER BY (field.region_id = @region_id) DESC NULLS LAST, field.sort_order, field.id
LIMIT 1
) configured ON TRUE
)
""";
}
private static string BuildBaseWhere(bool includeCursor)
{
var where = """
r.tenant_id = @tenant_id
AND (@region_id IS NULL OR r.region_id = @region_id)
AND (@school_id IS NULL OR r.school_id = @school_id)
AND (@major_id IS NULL OR r.major_id = @major_id)
AND (@year IS NULL OR r.year = @year)
AND (@keyword IS NULL OR r.school_name ILIKE @keyword ESCAPE '\' OR r.major_name ILIKE @keyword ESCAPE '\')
""";
if (!includeCursor)
{
return where;
}
return where + """
AND (
r.year < @cursor_year OR
(r.year = @cursor_year AND (
(@cursor_school IS NOT NULL AND (r.school_name > @cursor_school OR r.school_name IS NULL)) OR
(r.school_name IS NOT DISTINCT FROM @cursor_school AND (
(@cursor_major IS NOT NULL AND (r.major_name > @cursor_major OR r.major_name IS NULL)) OR
(r.major_name IS NOT DISTINCT FROM @cursor_major AND r.id > @cursor_id)
))
))
)
""";
}
private const string DynamicWhere = """
AND NOT EXISTS (
SELECT 1
FROM allowed_filters filter
WHERE filter.field_type IS NULL OR NOT (
CASE
WHEN lower(filter.field_type) IN ('number', 'integer', 'decimal', 'float') THEN
jsonb_typeof(r.field_values -> filter.field_key) = 'number' AND
CASE filter.operator
WHEN 'min' THEN (r.field_values ->> filter.field_key)::numeric >= filter.value::numeric
WHEN 'max' THEN (r.field_values ->> filter.field_key)::numeric <= filter.value::numeric
ELSE (r.field_values ->> filter.field_key)::numeric = filter.value::numeric
END
WHEN lower(filter.field_type) IN ('boolean', 'bool') THEN
filter.operator = 'field' AND
jsonb_typeof(r.field_values -> filter.field_key) = 'boolean' AND
(r.field_values ->> filter.field_key)::boolean = filter.value::boolean
WHEN lower(filter.field_type) IN ('text', 'textarea', 'string') THEN
filter.operator = 'field' AND
jsonb_typeof(r.field_values -> filter.field_key) = 'string' AND
strpos(lower(r.field_values ->> filter.field_key), lower(filter.value)) > 0
ELSE
filter.operator = 'field' AND
lower(r.field_values ->> filter.field_key) = lower(filter.value)
END
)
)
""";
private static void AddParameters(
NpgsqlCommand command,
ScorelineFilter filter,
int limit,
int? offset,
ScorelineCursorPosition? cursor)
{
command.Parameters.AddWithValue("tenant_id", filter.TenantId);
command.Parameters.Add("region_id", NpgsqlDbType.Uuid).Value = (object?)filter.RegionId ?? DBNull.Value;
command.Parameters.Add("school_id", NpgsqlDbType.Uuid).Value = (object?)filter.SchoolId ?? DBNull.Value;
command.Parameters.Add("major_id", NpgsqlDbType.Uuid).Value = (object?)filter.MajorId ?? DBNull.Value;
command.Parameters.Add("year", NpgsqlDbType.Integer).Value = (object?)filter.Year ?? DBNull.Value;
command.Parameters.Add("keyword", NpgsqlDbType.Text).Value = string.IsNullOrWhiteSpace(filter.Keyword)
? DBNull.Value
: $"%{EscapeLike(filter.Keyword.Trim())}%";
command.Parameters.AddWithValue("limit", limit);
if (offset.HasValue)
{
command.Parameters.AddWithValue("offset", offset.Value);
}
if (cursor is not null)
{
command.Parameters.AddWithValue("cursor_year", cursor.Year);
command.Parameters.Add("cursor_school", NpgsqlDbType.Text).Value = (object?)cursor.SchoolName ?? DBNull.Value;
command.Parameters.Add("cursor_major", NpgsqlDbType.Text).Value = (object?)cursor.MajorName ?? DBNull.Value;
command.Parameters.AddWithValue("cursor_id", cursor.Id);
}
foreach (var (filterValue, index) in (filter.DynamicFilters ?? []).Select((value, index) => (value, index)))
{
command.Parameters.AddWithValue($"filter_operator_{index}", filterValue.Operator);
command.Parameters.AddWithValue($"filter_key_{index}", filterValue.FieldKey);
command.Parameters.AddWithValue($"filter_value_{index}", filterValue.Value);
}
}
private static string EscapeLike(string value) =>
value.Replace("\\", "\\\\", StringComparison.Ordinal)
.Replace("%", "\\%", StringComparison.Ordinal)
.Replace("_", "\\_", StringComparison.Ordinal);
private static bool IsNumericType(string fieldType) =>
fieldType is "number" or "integer" or "decimal" or "float";
}