Zero-Downtime Deploys in ASP.NET Core: Graceful Shutdown, Draining, and Kubernetes

var builder = WebApplication.CreateBuilder(args);
var app = builder.Build();

// ── Inject IHostApplicationLifetime ───────────────────────────────────────
// IHostApplicationLifetime is registered by the host automatically.
// Resolve it from the DI container after Build() — do not inject into
// middleware or services that need to trigger application shutdown
// from within a request handler (use IHostApplicationLifetime there instead).
var lifetime = app.Services.GetRequiredService<IHostApplicationLifetime>();
var logger   = app.Services.GetRequiredService<ILogger<Program>>();

// ── ApplicationStarted ────────────────────────────────────────────────────
// Fires after all IHostedService.StartAsync calls complete and Kestrel
// is listening. The app is fully ready to receive traffic at this point.
// Use for: post-startup notifications, readiness signal to a service mesh,
//          warming up caches that depend on the full DI container being ready.
lifetime.ApplicationStarted.Register(() =>
{
    logger.LogInformation(
        "Application started. Environment: {Environment}. Process: {PID}",
        builder.Environment.EnvironmentName,
        Environment.ProcessId);
});

// ── ApplicationStopping ───────────────────────────────────────────────────
// Fires when SIGTERM (or Ctrl+C) is received.
// Hosted services have NOT yet been stopped.
// Kestrel has NOT yet stopped accepting connections.
// This is the earliest point you can react to an impending shutdown.
//
// Use for:
//   ✓ Stopping message queue consumers (stop pulling new messages)
//   ✓ Signalling background workers to finish current work and exit
//   ✓ Flushing in-memory write buffers to durable storage
//   ✓ Marking the instance as draining in a service registry
//   ✗ NOT for work that requires Kestrel to still be running — it is,
//     but you cannot guarantee how long it will remain so
lifetime.ApplicationStopping.Register(() =>
{
    logger.LogInformation(
        "Application stopping. Beginning pre-shutdown cleanup. PID: {PID}",
        Environment.ProcessId);

    // Example: signal a background message consumer to stop pulling.
    // The consumer's current in-progress message will complete;
    // no new messages will be pulled after this point.
    // app.Services.GetRequiredService<IMessageConsumer>().SignalStop();
});

// ── ApplicationStopped ────────────────────────────────────────────────────
// Fires after ALL hosted services have stopped and Kestrel has
// finished draining connections. The process is about to exit.
//
// Use for:
//   ✓ Flushing telemetry to a remote endpoint (OpenTelemetry, Sentry)
//   ✓ Final log flush for log sinks that buffer writes
//   ✓ Disposing resources that must be released after all services stop
//   ✗ NOT for any work that depends on the DI container being available —
//     the container may have been partially disposed by this point
lifetime.ApplicationStopped.Register(() =>
{
    logger.LogInformation(
        "Application stopped. All hosted services have completed. PID: {PID}",
        Environment.ProcessId);
    // Flush structured log sink (e.g., Serilog)
    // Log.CloseAndFlush();
});

app.Run();

// ── Option A: Code-based configuration ───────────────────────────────────
// UseShutdownTimeout sets the maximum time the host waits for:
//   1. All IHostedService.StopAsync calls to complete
//   2. Kestrel to finish draining active connections
// After this window, remaining work is abandoned and the process exits.

var builder = WebApplication.CreateBuilder(args);

builder.WebHost.UseShutdownTimeout(TimeSpan.FromSeconds(30));
// 30 seconds is a reasonable starting point for services with:
//   - Active DB connections (EF Core query completion)
//   - Message queue consumers (current message processing)
//   - HTTP client calls to downstream services
// Increase if your background services need longer to drain.

// ── Option B: Environment variable (preferred for containerised workloads) ─
// DOTNET_SHUTDOWNTIMEOUTSECONDS=30
// Set in your Kubernetes Deployment env block — no code change required
// when tuning per environment. Takes precedence over UseShutdownTimeout.

// ── Option C: appsettings.json / appsettings.Production.json ─────────────
/*
{
  "ShutdownTimeoutSeconds": 30
}
*/
// Then read in Program.cs:
var shutdownSeconds = builder.Configuration.GetValue("ShutdownTimeoutSeconds", 30);
builder.WebHost.UseShutdownTimeout(TimeSpan.FromSeconds(shutdownSeconds));

// ── BackgroundService: honouring the shutdown timeout ────────────────────
// Every BackgroundService must respect the stoppingToken passed to ExecuteAsync.
// When the host begins shutdown, StopAsync is called with a CancellationToken
// that is cancelled after the shutdown timeout elapses.
// A service that ignores stoppingToken will be forcibly terminated.

public class OrderProcessingService(ILogger<OrderProcessingService> logger)
    : BackgroundService
{
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        logger.LogInformation("Order processing service started.");

        // stoppingToken is cancelled when ApplicationStopping fires
        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                await ProcessNextOrderAsync(stoppingToken);
            }
            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
            {
                // Clean exit — shutdown was requested. Do not log as an error.
                break;
            }
        }

        logger.LogInformation("Order processing service stopped cleanly.");
        // ↑ This log line only appears if shutdown completed within the timeout.
        //   If it is absent in your logs, the service was forcibly terminated —
        //   a clear signal that your shutdown timeout needs to be increased.
    }

    private static async Task ProcessNextOrderAsync(CancellationToken ct)
    {
        // Pass ct through to every async call so cancellation propagates
        // to DB queries, HTTP calls, and message queue operations.
        await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
    }
}

// ── Timeout sizing guide ──────────────────────────────────────────────────
// Minimum recommended timeout = max expected single-operation duration
//   + connection drain time (typically 5–10s for most workloads)
//   + 5s safety buffer
//
// Example for a service with 15s max DB query + 10s drain + 5s buffer:
// UseShutdownTimeout(TimeSpan.FromSeconds(30))
//
// Kubernetes terminationGracePeriodSeconds:
//   preStop sleep (10s) + .NET shutdown timeout (30s) + buffer (10s) = 50s

// ── deployment.yaml ───────────────────────────────────────────────────────
/*
apiVersion: apps/v1
kind: Deployment
metadata:
  name: api
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 0      # never take a pod offline before a new one is ready
      maxSurge: 1            # spin up one extra pod during the rollout
  template:
    spec:
      # terminationGracePeriodSeconds MUST be larger than:
      #   preStop sleep (10s) + .NET shutdown timeout (30s) + buffer (10s) = 50s
      terminationGracePeriodSeconds: 60

      containers:
      - name: api
        image: yourcompany/api:latest
        lifecycle:
          preStop:
            exec:
              # preStop runs BEFORE Kubernetes sends SIGTERM.
              # This sleep gives kube-proxy and Ingress controllers time to
              # propagate the endpoint removal to all load balancers.
              # During these 10 seconds the pod is still healthy and serving
              # traffic — no requests are dropped.
              # After the sleep, Kubernetes sends SIGTERM and the .NET host
              # begins its graceful shutdown sequence.
              command: ["/bin/sh", "-c", "sleep 10"]

        # Readiness probe stops routing new requests to the pod when it fails.
        # This is separate from graceful shutdown but complements it —
        # a pod that is shutting down will fail readiness checks and be
        # removed from the load balancer rotation as a secondary safety net.
        readinessProbe:
          httpGet:
            path: /healthz/ready
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 5
          failureThreshold: 3

        # Liveness probe — do NOT make this too aggressive during shutdown.
        # A liveness failure during graceful shutdown causes Kubernetes to
        # restart the pod rather than allowing it to drain cleanly.
        livenessProbe:
          httpGet:
            path: /healthz/live
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 10
          failureThreshold: 6   # allow 60s of liveness failures before restart
*/

// ── Program.cs — Kestrel connection drain configuration ──────────────────
var builder = WebApplication.CreateBuilder(args);

builder.WebHost.ConfigureKestrel(options =>
{
    // AllowSynchronousIO is unrelated to shutdown but commonly misconfigured.
    // The key shutdown-related setting is the overall request timeout.

    options.Limits.KeepAliveTimeout = TimeSpan.FromSeconds(120);
    // KeepAliveTimeout controls how long Kestrel keeps an idle HTTP/1.1
    // connection open. During graceful shutdown, Kestrel honours this
    // for existing keep-alive connections — reducing it means idle
    // connections close faster during drain, speeding up shutdown.
    // For HTTP/2, stream multiplexing means connection draining works
    // differently — Kestrel sends a GOAWAY frame to signal shutdown.

    options.Limits.RequestHeadersTimeout = TimeSpan.FromSeconds(30);
});

// UseShutdownTimeout covers the full drain window —
// Kestrel will wait up to this duration for in-flight requests to complete.
builder.WebHost.UseShutdownTimeout(TimeSpan.FromSeconds(30));

var app = builder.Build();

// ── Health check endpoints ────────────────────────────────────────────────
// Map readiness and liveness to separate paths so Kubernetes can
// distinguish between "not ready to receive traffic" and "needs restart".
// Stop reporting ready during shutdown — this accelerates load balancer
// removal as a secondary signal alongside the preStop hook.

var appLifetime = app.Services.GetRequiredService<IHostApplicationLifetime>();
var isReady     = true;

appLifetime.ApplicationStopping.Register(() => { isReady = false; });
// When ApplicationStopping fires, the readiness probe immediately starts
// failing. Kubernetes removes the pod from the endpoint slice.
// Combined with the preStop sleep, this provides two complementary
// mechanisms for draining traffic before the pod exits.

app.MapGet("/healthz/live",  () => Results.Ok("live"));
app.MapGet("/healthz/ready", () => isReady
    ? Results.Ok("ready")
    : Results.StatusCode(StatusCodes.Status503ServiceUnavailable));

app.Run();


// ── The complete SIGTERM-to-exit timeline ─────────────────────────────────
//
// T+0s   Kubernetes decides to terminate the pod (new deploy, scale-down)
// T+0s   preStop hook starts: /bin/sh -c "sleep 10"
//        Pod is still healthy, serving traffic normally
// T+10s  preStop hook completes. Kubernetes sends SIGTERM to the container.
//        All load balancers have had 10s to propagate endpoint removal.
// T+10s  .NET host receives SIGTERM. ApplicationStopping fires.
//        isReady = false → readiness probe begins failing (belt-and-braces)
//        Background services receive stoppingToken cancellation
//        Kestrel stops accepting new connections
//        Kestrel drains in-flight requests (up to shutdownTimeout)
// T+40s  All in-flight requests complete. All BackgroundServices stopped.
//        ApplicationStopped fires. Final telemetry flush.
//        Process exits with code 0.
// T+60s  terminationGracePeriodSeconds elapses. Kubernetes would send SIGKILL
//        here if the process had not already exited. It has — clean shutdown.

public sealed class MessageConsumerService(
    IMessageQueue       queue,
    ILogger<MessageConsumerService> logger)
    : BackgroundService
{
    // ── ExecuteAsync: the main processing loop ─────────────────────────────
    // stoppingToken is cancelled when IHostApplicationLifetime.ApplicationStopping fires.
    // This is the signal to stop pulling new work and exit cleanly.
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        logger.LogInformation("Message consumer started.");

        while (!stoppingToken.IsCancellationRequested)
        {
            MessageEnvelope? message = null;

            try
            {
                // ── Pull phase: pass stoppingToken so the receive call unblocks ──
                // If no message arrives within the timeout, the loop checks
                // stoppingToken and exits cleanly on the next iteration.
                message = await queue.ReceiveAsync(
                    timeout: TimeSpan.FromSeconds(5),
                    cancellationToken: stoppingToken);

                if (message is null)
                    continue;

                // ── Process phase: use CancellationToken.None ──────────────────
                // Do NOT pass stoppingToken to the processing call.
                // If shutdown is requested mid-processing, you want this message
                // to complete — not be cancelled and left unacknowledged.
                // The shutdown timeout gives you the window to finish.
                await ProcessMessageAsync(message, CancellationToken.None);

                // ── Acknowledge: only after successful processing ──────────────
                await queue.AcknowledgeAsync(message.Id, CancellationToken.None);

                logger.LogDebug("Message {MessageId} processed and acknowledged.", message.Id);
            }
            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
            {
                // Normal shutdown path — stoppingToken was cancelled during ReceiveAsync.
                // The current message (if any) will be requeued automatically
                // by the broker's visibility timeout — we did not acknowledge it.
                logger.LogInformation("Consumer shutting down — receive cancelled.");
                break;
            }
            catch (Exception ex)
            {
                // Processing error — log and continue. Do not acknowledge.
                // The broker will redeliver after the visibility timeout.
                logger.LogError(ex,
                    "Failed to process message {MessageId}. Message will be redelivered.",
                    message?.Id);

                // Back off before retrying to avoid a tight error loop
                await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
            }
        }

        logger.LogInformation("Message consumer stopped cleanly.");
        // If this line appears in logs after every deployment: shutdown is working.
        // If it is absent: the shutdown timeout is too short — the consumer
        // was abandoned mid-message. Increase UseShutdownTimeout.
    }

    private static async Task ProcessMessageAsync(
        MessageEnvelope message,
        CancellationToken ct)
    {
        // Business logic here — DB writes, HTTP calls, event publishing.
        // All using CancellationToken.None so shutdown cannot interrupt processing.
        await Task.Delay(TimeSpan.FromMilliseconds(200), ct);
    }
}

// ── Registration in Program.cs ────────────────────────────────────────────
// builder.Services.AddHostedService<MessageConsumerService>();
//
// The host calls StopAsync on all IHostedService instances during shutdown.
// StopAsync triggers cancellation of stoppingToken, which unblocks ReceiveAsync
// and causes the while loop to exit after the current message completes.
// The host waits up to shutdownTimeout for StopAsync to return before
// abandoning the service — size your timeout to match your worst-case
// single-message processing duration.

// ════════════════════════════════════════════════════════════════════════════
// COMPLETE GRACEFUL SHUTDOWN CONFIGURATION — ASP.NET CORE + KUBERNETES
// ════════════════════════════════════════════════════════════════════════════

var builder = WebApplication.CreateBuilder(args);

// ── 1. Shutdown timeout ───────────────────────────────────────────────────
// Must be less than: terminationGracePeriodSeconds − preStop sleep duration
// Formula: shutdownTimeout < terminationGracePeriodSeconds − preStopSleep
// Example: 30s < 60s − 10s = 50s ✓
builder.WebHost.UseShutdownTimeout(TimeSpan.FromSeconds(30));

// ── 2. Kestrel limits ────────────────────────────────────────────────────
builder.WebHost.ConfigureKestrel(k =>
{
    k.Limits.KeepAliveTimeout      = TimeSpan.FromSeconds(120);
    k.Limits.RequestHeadersTimeout = TimeSpan.FromSeconds(30);
    // HTTP/2: Kestrel automatically sends GOAWAY on shutdown —
    // no additional configuration needed for HTTP/2 connection draining.
});

// ── 3. Register background services ──────────────────────────────────────
builder.Services.AddHostedService<MessageConsumerService>();
builder.Services.AddHostedService<OutboxProcessorService>();
// Each must honour stoppingToken in its ExecuteAsync loop (see previous section).

// ── 4. Health checks ─────────────────────────────────────────────────────
builder.Services.AddHealthChecks()
    .AddDbContextCheck<AppDbContext>("database")
    .AddCheck("self", () => HealthCheckResult.Healthy());

var app = builder.Build();

// ── 5. Lifetime hooks ────────────────────────────────────────────────────
var lifetime = app.Services.GetRequiredService<IHostApplicationLifetime>();
var logger   = app.Services.GetRequiredService<ILogger<Program>>();
var isReady  = true;

lifetime.ApplicationStarted.Register(() =>
    logger.LogInformation("✓ Application started. PID {PID}", Environment.ProcessId));

lifetime.ApplicationStopping.Register(() =>
{
    isReady = false;   // fail readiness probe immediately on shutdown signal
    logger.LogInformation("⚡ Shutdown signal received. Beginning drain. PID {PID}",
        Environment.ProcessId);
});

lifetime.ApplicationStopped.Register(() =>
{
    logger.LogInformation("✓ Application stopped cleanly. PID {PID}", Environment.ProcessId);
    // Log.CloseAndFlush();   // flush Serilog or other buffering sinks here
});

// ── 6. Middleware pipeline ────────────────────────────────────────────────
app.UseRouting();
app.UseAuthentication();
app.UseAuthorization();

// ── 7. Health check endpoints ────────────────────────────────────────────
// Separate liveness and readiness — Kubernetes treats failures differently.
app.MapGet("/healthz/live",  () => Results.Ok(new { status = "live" }))
   .WithMetadata(new DisableRateLimitingAttribute());   // never rate-limit probes

app.MapGet("/healthz/ready", () => isReady
    ? Results.Ok(new { status = "ready" })
    : Results.StatusCode(StatusCodes.Status503ServiceUnavailable));

app.MapHealthChecks("/healthz/detail", new HealthCheckOptions
{
    ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse
});

app.MapControllers();

// ── 8. Kubernetes Deployment checklist ───────────────────────────────────
//
// □ terminationGracePeriodSeconds: 60         (preStop 10s + timeout 30s + buffer 20s)
// □ preStop exec: sleep 10                    (absorb load balancer propagation delay)
// □ readinessProbe path: /healthz/ready       (stops routing on failure during drain)
// □ livenessProbe failureThreshold: 6+        (do not restart during drain)
// □ rollingUpdate maxUnavailable: 0           (never remove pod before replacement ready)
// □ rollingUpdate maxSurge: 1                 (spin up new pod before removing old one)
// □ DOTNET_SHUTDOWNTIMEOUTSECONDS env var     (overrides UseShutdownTimeout if set)
//
// □ Every BackgroundService honours stoppingToken in its processing loop
// □ Every BackgroundService uses CancellationToken.None for in-progress work
// □ "Stopped cleanly" log line appears after every deployment in production logs

app.Run();