perf: optimize authorization scoreline and workers
This commit is contained in:
292
Tiku.Infrastructure/Scoreline/ScorelineRecordQuery.cs
Normal file
292
Tiku.Infrastructure/Scoreline/ScorelineRecordQuery.cs
Normal file
@@ -0,0 +1,292 @@
|
||||
using System.Data;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Npgsql;
|
||||
using NpgsqlTypes;
|
||||
using Tiku.Application.Scoreline;
|
||||
|
||||
namespace Tiku.Infrastructure.Scoreline;
|
||||
|
||||
internal sealed record ScorelineCursorPosition(int Year, string? SchoolName, string? MajorName, Guid Id);
|
||||
|
||||
internal sealed record ScorelineQueryResult(int? Total, IReadOnlyList<ScorelineRecordItem> Items);
|
||||
|
||||
public sealed class ScorelineRecordQuery(NpgsqlDataSource dataSource)
|
||||
{
|
||||
internal async Task<ScorelineQueryResult> ExecutePageAsync(
|
||||
ScorelineFilter filter,
|
||||
int page,
|
||||
int pageSize,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var offset = checked((page - 1) * pageSize);
|
||||
return await ExecuteAsync(filter, pageSize, offset, null, true, cancellationToken);
|
||||
}
|
||||
|
||||
internal Task<ScorelineQueryResult> ExecuteCursorAsync(
|
||||
ScorelineFilter filter,
|
||||
int pageSize,
|
||||
ScorelineCursorPosition? cursor,
|
||||
CancellationToken cancellationToken) =>
|
||||
ExecuteAsync(filter, pageSize + 1, null, cursor, false, cancellationToken);
|
||||
|
||||
private async Task<ScorelineQueryResult> ExecuteAsync(
|
||||
ScorelineFilter filter,
|
||||
int limit,
|
||||
int? offset,
|
||||
ScorelineCursorPosition? cursor,
|
||||
bool includeCount,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync(cancellationToken);
|
||||
await using var command = connection.CreateCommand();
|
||||
AddParameters(command, filter, limit, offset, cursor);
|
||||
command.CommandText = BuildSql(filter.DynamicFilters ?? [], includeCount, cursor is not null, offset.HasValue);
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken);
|
||||
await ValidateFiltersAsync(reader, filter.DynamicFilters ?? [], cancellationToken);
|
||||
|
||||
int? total = null;
|
||||
if (includeCount)
|
||||
{
|
||||
if (!await reader.NextResultAsync(cancellationToken) || !await reader.ReadAsync(cancellationToken))
|
||||
{
|
||||
throw new InvalidOperationException("Scoreline count result was missing.");
|
||||
}
|
||||
|
||||
total = reader.GetInt32(0);
|
||||
}
|
||||
|
||||
if (!await reader.NextResultAsync(cancellationToken))
|
||||
{
|
||||
throw new InvalidOperationException("Scoreline item result was missing.");
|
||||
}
|
||||
|
||||
var items = new List<ScorelineRecordItem>(limit);
|
||||
while (await reader.ReadAsync(cancellationToken))
|
||||
{
|
||||
items.Add(new ScorelineRecordItem(
|
||||
reader.GetGuid(0),
|
||||
reader.IsDBNull(1) ? null : reader.GetString(1),
|
||||
reader.IsDBNull(2) ? null : reader.GetGuid(2),
|
||||
reader.IsDBNull(3) ? null : reader.GetGuid(3),
|
||||
reader.IsDBNull(4) ? null : reader.GetGuid(4),
|
||||
reader.GetInt32(5),
|
||||
reader.IsDBNull(6) ? null : reader.GetString(6),
|
||||
reader.IsDBNull(7) ? null : reader.GetString(7),
|
||||
JsonDocument.Parse(reader.GetString(8)).RootElement.Clone()));
|
||||
}
|
||||
|
||||
return new ScorelineQueryResult(total, items);
|
||||
}
|
||||
|
||||
private static async Task ValidateFiltersAsync(
|
||||
NpgsqlDataReader reader,
|
||||
IReadOnlyCollection<ScorelineDynamicFilter> filters,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var validated = 0;
|
||||
while (await reader.ReadAsync(cancellationToken))
|
||||
{
|
||||
validated++;
|
||||
var operation = reader.GetString(0);
|
||||
var value = reader.GetString(2);
|
||||
if (reader.IsDBNull(3))
|
||||
{
|
||||
throw new ScorelineQueryException("Scoreline field filter is not enabled.", "scoreline_field_filter_not_allowed");
|
||||
}
|
||||
|
||||
var fieldType = reader.GetString(3).Trim().ToLowerInvariant();
|
||||
if (operation is "min" or "max")
|
||||
{
|
||||
if (!IsNumericType(fieldType))
|
||||
{
|
||||
throw new ScorelineQueryException("Scoreline range filter requires a numeric field.", "scoreline_field_range_type_invalid");
|
||||
}
|
||||
|
||||
if (!decimal.TryParse(value, NumberStyles.Number, CultureInfo.InvariantCulture, out _))
|
||||
{
|
||||
throw new ScorelineQueryException("Scoreline range filter value must be numeric.", "scoreline_field_range_value_invalid");
|
||||
}
|
||||
}
|
||||
else if (IsNumericType(fieldType) &&
|
||||
!decimal.TryParse(value, NumberStyles.Number, CultureInfo.InvariantCulture, out _))
|
||||
{
|
||||
throw new ScorelineQueryException("Scoreline numeric filter value must be numeric.", "scoreline_field_value_invalid");
|
||||
}
|
||||
else if (fieldType is "boolean" or "bool" && !bool.TryParse(value, out _))
|
||||
{
|
||||
throw new ScorelineQueryException("Scoreline boolean filter value must be true or false.", "scoreline_field_value_invalid");
|
||||
}
|
||||
}
|
||||
|
||||
if (validated != filters.Count)
|
||||
{
|
||||
throw new InvalidOperationException("Scoreline filter validation result was incomplete.");
|
||||
}
|
||||
}
|
||||
|
||||
private static string BuildSql(
|
||||
IReadOnlyCollection<ScorelineDynamicFilter> filters,
|
||||
bool includeCount,
|
||||
bool includeCursor,
|
||||
bool includeOffset)
|
||||
{
|
||||
var validationCte = BuildFilterCte(filters);
|
||||
var baseWhere = BuildBaseWhere(includeCursor);
|
||||
var dynamicWhere = filters.Count == 0 ? string.Empty : DynamicWhere;
|
||||
var sql = new StringBuilder();
|
||||
sql.Append(validationCte).AppendLine()
|
||||
.AppendLine("SELECT operator, field_key, value, field_type FROM allowed_filters ORDER BY ordinal;");
|
||||
|
||||
if (includeCount)
|
||||
{
|
||||
sql.Append(validationCte).AppendLine()
|
||||
.Append("SELECT count(*)::integer FROM scoreline_records r WHERE ")
|
||||
.Append(baseWhere).Append(dynamicWhere).AppendLine(";");
|
||||
}
|
||||
|
||||
sql.Append(validationCte).AppendLine()
|
||||
.Append("SELECT r.id, r.legacy_id, r.region_id, r.school_id, r.major_id, r.year, r.school_name, r.major_name, r.field_values::text ")
|
||||
.Append("FROM scoreline_records r WHERE ").Append(baseWhere).Append(dynamicWhere)
|
||||
.AppendLine(" ORDER BY r.year DESC, r.school_name ASC NULLS LAST, r.major_name ASC NULLS LAST, r.id ASC")
|
||||
.Append(" LIMIT @limit");
|
||||
if (includeOffset)
|
||||
{
|
||||
sql.Append(" OFFSET @offset");
|
||||
}
|
||||
|
||||
sql.Append(';');
|
||||
return sql.ToString();
|
||||
}
|
||||
|
||||
private static string BuildFilterCte(IReadOnlyCollection<ScorelineDynamicFilter> filters)
|
||||
{
|
||||
var requested = filters.Count == 0
|
||||
? "SELECT NULL::text AS operator, NULL::text AS field_key, NULL::text AS value, NULL::integer AS ordinal WHERE FALSE"
|
||||
: "VALUES " + string.Join(", ", filters.Select((_, index) => $"(@filter_operator_{index}, @filter_key_{index}, @filter_value_{index}, {index})"));
|
||||
|
||||
return $$"""
|
||||
WITH requested_filters(operator, field_key, value, ordinal) AS ({{requested}}),
|
||||
allowed_filters AS (
|
||||
SELECT requested.operator, requested.field_key, requested.value, requested.ordinal, configured.field_type
|
||||
FROM requested_filters requested
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT field.field_type
|
||||
FROM scoreline_fields field
|
||||
WHERE field.tenant_id = @tenant_id
|
||||
AND field.is_filter
|
||||
AND field.field_key = requested.field_key
|
||||
AND (@region_id IS NULL OR field.region_id = @region_id OR field.region_id IS NULL)
|
||||
ORDER BY (field.region_id = @region_id) DESC NULLS LAST, field.sort_order, field.id
|
||||
LIMIT 1
|
||||
) configured ON TRUE
|
||||
)
|
||||
""";
|
||||
}
|
||||
|
||||
private static string BuildBaseWhere(bool includeCursor)
|
||||
{
|
||||
var where = """
|
||||
r.tenant_id = @tenant_id
|
||||
AND (@region_id IS NULL OR r.region_id = @region_id)
|
||||
AND (@school_id IS NULL OR r.school_id = @school_id)
|
||||
AND (@major_id IS NULL OR r.major_id = @major_id)
|
||||
AND (@year IS NULL OR r.year = @year)
|
||||
AND (@keyword IS NULL OR r.school_name ILIKE @keyword ESCAPE '\' OR r.major_name ILIKE @keyword ESCAPE '\')
|
||||
""";
|
||||
if (!includeCursor)
|
||||
{
|
||||
return where;
|
||||
}
|
||||
|
||||
return where + """
|
||||
AND (
|
||||
r.year < @cursor_year OR
|
||||
(r.year = @cursor_year AND (
|
||||
(@cursor_school IS NOT NULL AND (r.school_name > @cursor_school OR r.school_name IS NULL)) OR
|
||||
(r.school_name IS NOT DISTINCT FROM @cursor_school AND (
|
||||
(@cursor_major IS NOT NULL AND (r.major_name > @cursor_major OR r.major_name IS NULL)) OR
|
||||
(r.major_name IS NOT DISTINCT FROM @cursor_major AND r.id > @cursor_id)
|
||||
))
|
||||
))
|
||||
)
|
||||
""";
|
||||
}
|
||||
|
||||
private const string DynamicWhere = """
|
||||
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM allowed_filters filter
|
||||
WHERE filter.field_type IS NULL OR NOT (
|
||||
CASE
|
||||
WHEN lower(filter.field_type) IN ('number', 'integer', 'decimal', 'float') THEN
|
||||
jsonb_typeof(r.field_values -> filter.field_key) = 'number' AND
|
||||
CASE filter.operator
|
||||
WHEN 'min' THEN (r.field_values ->> filter.field_key)::numeric >= filter.value::numeric
|
||||
WHEN 'max' THEN (r.field_values ->> filter.field_key)::numeric <= filter.value::numeric
|
||||
ELSE (r.field_values ->> filter.field_key)::numeric = filter.value::numeric
|
||||
END
|
||||
WHEN lower(filter.field_type) IN ('boolean', 'bool') THEN
|
||||
filter.operator = 'field' AND
|
||||
jsonb_typeof(r.field_values -> filter.field_key) = 'boolean' AND
|
||||
(r.field_values ->> filter.field_key)::boolean = filter.value::boolean
|
||||
WHEN lower(filter.field_type) IN ('text', 'textarea', 'string') THEN
|
||||
filter.operator = 'field' AND
|
||||
jsonb_typeof(r.field_values -> filter.field_key) = 'string' AND
|
||||
strpos(lower(r.field_values ->> filter.field_key), lower(filter.value)) > 0
|
||||
ELSE
|
||||
filter.operator = 'field' AND
|
||||
lower(r.field_values ->> filter.field_key) = lower(filter.value)
|
||||
END
|
||||
)
|
||||
)
|
||||
""";
|
||||
|
||||
private static void AddParameters(
|
||||
NpgsqlCommand command,
|
||||
ScorelineFilter filter,
|
||||
int limit,
|
||||
int? offset,
|
||||
ScorelineCursorPosition? cursor)
|
||||
{
|
||||
command.Parameters.AddWithValue("tenant_id", filter.TenantId);
|
||||
command.Parameters.Add("region_id", NpgsqlDbType.Uuid).Value = (object?)filter.RegionId ?? DBNull.Value;
|
||||
command.Parameters.Add("school_id", NpgsqlDbType.Uuid).Value = (object?)filter.SchoolId ?? DBNull.Value;
|
||||
command.Parameters.Add("major_id", NpgsqlDbType.Uuid).Value = (object?)filter.MajorId ?? DBNull.Value;
|
||||
command.Parameters.Add("year", NpgsqlDbType.Integer).Value = (object?)filter.Year ?? DBNull.Value;
|
||||
command.Parameters.Add("keyword", NpgsqlDbType.Text).Value = string.IsNullOrWhiteSpace(filter.Keyword)
|
||||
? DBNull.Value
|
||||
: $"%{EscapeLike(filter.Keyword.Trim())}%";
|
||||
command.Parameters.AddWithValue("limit", limit);
|
||||
if (offset.HasValue)
|
||||
{
|
||||
command.Parameters.AddWithValue("offset", offset.Value);
|
||||
}
|
||||
|
||||
if (cursor is not null)
|
||||
{
|
||||
command.Parameters.AddWithValue("cursor_year", cursor.Year);
|
||||
command.Parameters.Add("cursor_school", NpgsqlDbType.Text).Value = (object?)cursor.SchoolName ?? DBNull.Value;
|
||||
command.Parameters.Add("cursor_major", NpgsqlDbType.Text).Value = (object?)cursor.MajorName ?? DBNull.Value;
|
||||
command.Parameters.AddWithValue("cursor_id", cursor.Id);
|
||||
}
|
||||
|
||||
foreach (var (filterValue, index) in (filter.DynamicFilters ?? []).Select((value, index) => (value, index)))
|
||||
{
|
||||
command.Parameters.AddWithValue($"filter_operator_{index}", filterValue.Operator);
|
||||
command.Parameters.AddWithValue($"filter_key_{index}", filterValue.FieldKey);
|
||||
command.Parameters.AddWithValue($"filter_value_{index}", filterValue.Value);
|
||||
}
|
||||
}
|
||||
|
||||
private static string EscapeLike(string value) =>
|
||||
value.Replace("\\", "\\\\", StringComparison.Ordinal)
|
||||
.Replace("%", "\\%", StringComparison.Ordinal)
|
||||
.Replace("_", "\\_", StringComparison.Ordinal);
|
||||
|
||||
private static bool IsNumericType(string fieldType) =>
|
||||
fieldType is "number" or "integer" or "decimal" or "float";
|
||||
}
|
||||
Reference in New Issue
Block a user