Backend Scalability & Resilience

// Program.cs - Rate Limiting Configuration (.NET 10)
builder.Services.AddRateLimiter(options =>
{
    // Per-user rate limit
    options.AddPolicy("PerUser", context =>
        RateLimitPartition.GetFixedWindowLimiter(
            partitionKey: context.User?.Identity?.Name ?? context.Connection.RemoteIpAddress?.ToString(),
            factory: _ => new FixedWindowRateLimiterOptions
            {
                PermitLimit = 100,
                Window = TimeSpan.FromMinutes(1),
                QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
                QueueLimit = 10
            }));

    // Per-tenant rate limit
    options.AddPolicy("PerTenant", context =>
        RateLimitPartition.GetFixedWindowLimiter(
            partitionKey: context.User?.FindFirst("tenant_id")?.Value ?? "anonymous",
            factory: _ => new FixedWindowRateLimiterOptions
            {
                PermitLimit = 1000,
                Window = TimeSpan.FromMinutes(1)
            }));

    options.OnRejected = async (context, token) =>
    {
        context.HttpContext.Response.StatusCode = 429;
        await context.HttpContext.Response.WriteAsync("Too many requests. Please try again later.");
    };
});

// Polly Circuit Breaker with HttpClient
builder.Services.AddHttpClient("BrevoClient", client =>
{
    client.BaseAddress = new Uri("https://api.brevo.com");
})
.AddPolicyHandler(Policy
    .Handle<HttpRequestException>()
    .OrResult<HttpResponseMessage>(r => !r.IsSuccessStatusCode)
    .CircuitBreakerAsync(
        handledEventsAllowedBeforeBreaking: 5,
        durationOfBreak: TimeSpan.FromSeconds(30),
        onBreak: (result, duration) =>
        {
            Log.Warning("Circuit OPEN for Brevo. Duration: {Duration}", duration);
        },
        onReset: () =>
        {
            Log.Information("Circuit CLOSED for Brevo. Service recovered.");
        }
    ))
.AddPolicyHandler(Policy
    .Handle<HttpRequestException>()
    .WaitAndRetryAsync(3, retryAttempt =>
        TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)) // 2s, 4s, 8s
    ));

// Bulkhead for Report Generation
private static readonly SemaphoreSlim _reportSemaphore = new(10, 10);

public async Task<ReportResult> GenerateReportAsync(ReportRequest request)
{
    if (!await _reportSemaphore.WaitAsync(TimeSpan.FromSeconds(30)))
    {
        throw new ServiceUnavailableException("Report generation queue full. Please try again.");
    }

    try
    {
        return await _reportService.GenerateAsync(request);
    }
    finally
    {
        _reportSemaphore.Release();
    }
}

// Retry Policy with Jitter (prevents thundering herd)
var retryPolicy = Policy
    .Handle<HttpRequestException>()
    .OrResult<HttpResponseMessage>(r => r.StatusCode == HttpStatusCode.ServiceUnavailable)
    .WaitAndRetryAsync(
        retryCount: 5,
        sleepDurationProvider: (retryAttempt, context) =>
        {
            var baseDelay = TimeSpan.FromSeconds(Math.Pow(2, retryAttempt));
            var jitter = TimeSpan.FromMilliseconds(Random.Shared.Next(0, 1000));
            var totalDelay = baseDelay + jitter;
            return totalDelay > TimeSpan.FromSeconds(30)
                ? TimeSpan.FromSeconds(30)
                : totalDelay;
        },
        onRetry: (outcome, timespan, retryAttempt, context) =>
        {
            Log.Warning("Retry {Attempt} after {Delay}ms", retryAttempt, timespan.TotalMilliseconds);
        }
    );

// Azure Service Bus Producer
public class EmailQueueService
{
    private readonly ServiceBusSender _sender;

    public async Task QueueEmailAsync(EmailMessage email)
    {
        var message = new ServiceBusMessage(JsonSerializer.Serialize(email))
        {
            MessageId = Guid.NewGuid().ToString(),
            ContentType = "application/json",
            Subject = email.TemplateCode,
            ScheduledEnqueueTime = email.ScheduledTime ?? DateTimeOffset.UtcNow
        };

        await _sender.SendMessageAsync(message);
    }
}

// Azure Service Bus Consumer (Hangfire Worker)
public class EmailQueueProcessor
{
    [AutomaticRetry(Attempts = 5, DelaysInSeconds = new[] { 10, 30, 60, 120, 300 })]
    public async Task ProcessBatchAsync()
    {
        var messages = await _receiver.ReceiveMessagesAsync(maxMessages: 100);

        foreach (var message in messages)
        {
            var email = JsonSerializer.Deserialize<EmailMessage>(message.Body);
            await _brevoService.SendAsync(email);
            await _receiver.CompleteMessageAsync(message);
        }
    }
}