Chapter 28: Phase 4 - Production Implementation
Overview
Phase 4 prepares the POS platform for production deployment. This 2-week phase (Weeks 15-16) covers monitoring, security hardening, deployment procedures, and go-live operations. Every step is critical for a successful production launch.
Week 15: Monitoring and Alerting
Day 1: Structured Logging with Serilog
Objective: Implement structured, queryable logging with correlation IDs.
Claude Command:
/dev-team implement structured logging with Serilog and correlation IDs
Implementation:
// Install packages
// dotnet add package Serilog.AspNetCore
// dotnet add package Serilog.Sinks.Console
// dotnet add package Serilog.Sinks.File
// dotnet add package Serilog.Enrichers.Environment
// dotnet add package Serilog.Enrichers.Thread
// src/PosPlatform.Api/Program.cs - Logging configuration
using Serilog;
using Serilog.Events;
var builder = WebApplication.CreateBuilder(args);
// Configure Serilog
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Information()
.MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
.MinimumLevel.Override("Microsoft.EntityFrameworkCore", LogEventLevel.Warning)
.Enrich.FromLogContext()
.Enrich.WithEnvironmentName()
.Enrich.WithMachineName()
.Enrich.WithThreadId()
.Enrich.WithProperty("Application", "PosPlatform.Api")
.WriteTo.Console(
outputTemplate: "[{Timestamp:HH:mm:ss} {Level:u3}] {CorrelationId} {Message:lj}{NewLine}{Exception}")
.WriteTo.File(
path: "logs/pos-api-.log",
rollingInterval: RollingInterval.Day,
retainedFileCountLimit: 30,
outputTemplate: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {CorrelationId} {TenantCode} {UserId} {Message:lj}{NewLine}{Exception}")
.CreateLogger();
builder.Host.UseSerilog();
// ... rest of configuration
// src/PosPlatform.Api/Middleware/CorrelationIdMiddleware.cs
using Serilog.Context;
namespace PosPlatform.Api.Middleware;
public class CorrelationIdMiddleware
{
private const string CorrelationIdHeader = "X-Correlation-Id";
private readonly RequestDelegate _next;
public CorrelationIdMiddleware(RequestDelegate next)
{
_next = next;
}
public async Task InvokeAsync(HttpContext context)
{
var correlationId = context.Request.Headers[CorrelationIdHeader].FirstOrDefault()
?? Guid.NewGuid().ToString();
context.Response.Headers[CorrelationIdHeader] = correlationId;
using (LogContext.PushProperty("CorrelationId", correlationId))
{
await _next(context);
}
}
}
// src/PosPlatform.Api/Middleware/RequestLoggingMiddleware.cs
using System.Diagnostics;
namespace PosPlatform.Api.Middleware;
public class RequestLoggingMiddleware
{
private readonly RequestDelegate _next;
private readonly ILogger<RequestLoggingMiddleware> _logger;
public RequestLoggingMiddleware(RequestDelegate next, ILogger<RequestLoggingMiddleware> logger)
{
_next = next;
_logger = logger;
}
public async Task InvokeAsync(HttpContext context)
{
var stopwatch = Stopwatch.StartNew();
try
{
await _next(context);
stopwatch.Stop();
_logger.LogInformation(
"HTTP {Method} {Path} responded {StatusCode} in {ElapsedMs}ms",
context.Request.Method,
context.Request.Path,
context.Response.StatusCode,
stopwatch.ElapsedMilliseconds);
}
catch (Exception ex)
{
stopwatch.Stop();
_logger.LogError(ex,
"HTTP {Method} {Path} failed after {ElapsedMs}ms",
context.Request.Method,
context.Request.Path,
stopwatch.ElapsedMilliseconds);
throw;
}
}
}
Day 2: Prometheus Metrics
Objective: Expose application metrics for Prometheus scraping.
Claude Command:
/dev-team add Prometheus metrics endpoints for key operations
Implementation:
// Install packages
// dotnet add package prometheus-net.AspNetCore
// src/PosPlatform.Api/Metrics/PosMetrics.cs
using Prometheus;
namespace PosPlatform.Api.Metrics;
public static class PosMetrics
{
// HTTP request metrics (auto-collected by prometheus-net)
public static readonly Counter HttpRequestsTotal = Prometheus.Metrics
.CreateCounter("pos_http_requests_total", "Total HTTP requests",
new CounterConfiguration
{
LabelNames = new[] { "method", "endpoint", "status_code" }
});
// Business metrics
public static readonly Counter SalesTotal = Prometheus.Metrics
.CreateCounter("pos_sales_total", "Total sales completed",
new CounterConfiguration
{
LabelNames = new[] { "tenant", "location" }
});
public static readonly Counter SalesAmount = Prometheus.Metrics
.CreateCounter("pos_sales_amount_total", "Total sales amount in cents",
new CounterConfiguration
{
LabelNames = new[] { "tenant", "location", "payment_method" }
});
public static readonly Histogram SaleProcessingDuration = Prometheus.Metrics
.CreateHistogram("pos_sale_processing_seconds", "Sale processing duration",
new HistogramConfiguration
{
LabelNames = new[] { "tenant" },
Buckets = new[] { 0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0 }
});
public static readonly Gauge ActiveSales = Prometheus.Metrics
.CreateGauge("pos_active_sales", "Currently active sales",
new GaugeConfiguration
{
LabelNames = new[] { "tenant", "location" }
});
// Inventory metrics
public static readonly Gauge InventoryLowStock = Prometheus.Metrics
.CreateGauge("pos_inventory_low_stock_items", "Items below reorder point",
new GaugeConfiguration
{
LabelNames = new[] { "tenant", "location" }
});
public static readonly Counter InventoryAdjustments = Prometheus.Metrics
.CreateCounter("pos_inventory_adjustments_total", "Inventory adjustments",
new CounterConfiguration
{
LabelNames = new[] { "tenant", "location", "reason" }
});
// System metrics
public static readonly Gauge DatabaseConnections = Prometheus.Metrics
.CreateGauge("pos_database_connections", "Active database connections");
public static readonly Counter OfflineTransactions = Prometheus.Metrics
.CreateCounter("pos_offline_transactions_total", "Transactions processed offline",
new CounterConfiguration
{
LabelNames = new[] { "tenant", "type" }
});
public static readonly Gauge SyncQueueDepth = Prometheus.Metrics
.CreateGauge("pos_sync_queue_depth", "Pending sync queue depth",
new GaugeConfiguration
{
LabelNames = new[] { "tenant" }
});
}
// Program.cs additions
using Prometheus;
var builder = WebApplication.CreateBuilder(args);
// Add metrics
builder.Services.AddSingleton<IMetricServer>(sp =>
new MetricServer(port: 9090));
var app = builder.Build();
// Enable metrics endpoint
app.UseHttpMetrics();
// Metrics endpoint for Prometheus scraping
app.MapMetrics("/metrics");
// src/PosPlatform.Core/Services/SaleCompletionService.cs - Metrics integration
public async Task<SaleCompletionResult> CompleteSaleAsync(
Guid saleId,
CancellationToken ct = default)
{
using var timer = PosMetrics.SaleProcessingDuration
.WithLabels(_tenantContext.TenantCode!)
.NewTimer();
var sale = await _saleRepository.GetByIdAsync(saleId, ct);
// ... processing
// Record metrics on success
PosMetrics.SalesTotal
.WithLabels(_tenantContext.TenantCode!, sale.LocationId.ToString())
.Inc();
foreach (var payment in sale.Payments)
{
PosMetrics.SalesAmount
.WithLabels(
_tenantContext.TenantCode!,
sale.LocationId.ToString(),
payment.Method.ToString())
.Inc((long)(payment.Amount * 100)); // Store as cents
}
return result;
}
Day 3: Health Checks
Objective: Implement comprehensive health checks for all dependencies.
Claude Command:
/dev-team create health check endpoints for database, Redis, and RabbitMQ
Implementation:
// Install packages
// dotnet add package AspNetCore.HealthChecks.NpgSql
// dotnet add package AspNetCore.HealthChecks.Redis
// dotnet add package AspNetCore.HealthChecks.RabbitMQ
// src/PosPlatform.Api/HealthChecks/ServiceHealthCheck.cs
using Microsoft.Extensions.Diagnostics.HealthChecks;
namespace PosPlatform.Api.HealthChecks;
public class ServiceHealthCheck : IHealthCheck
{
private readonly IServiceProvider _serviceProvider;
public ServiceHealthCheck(IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
}
public async Task<HealthCheckResult> CheckHealthAsync(
HealthCheckContext context,
CancellationToken ct = default)
{
try
{
// Check critical services are resolvable
using var scope = _serviceProvider.CreateScope();
var saleService = scope.ServiceProvider.GetService<ISaleRepository>();
if (saleService == null)
return HealthCheckResult.Degraded("Sale service unavailable");
var inventoryService = scope.ServiceProvider.GetService<IInventoryRepository>();
if (inventoryService == null)
return HealthCheckResult.Degraded("Inventory service unavailable");
return HealthCheckResult.Healthy("All services operational");
}
catch (Exception ex)
{
return HealthCheckResult.Unhealthy("Service check failed", ex);
}
}
}
// Program.cs - Health check configuration
builder.Services.AddHealthChecks()
// Database
.AddNpgSql(
builder.Configuration.GetConnectionString("DefaultConnection")!,
name: "postgresql",
tags: new[] { "db", "critical" })
// Redis
.AddRedis(
builder.Configuration["Redis:ConnectionString"]!,
name: "redis",
tags: new[] { "cache", "critical" })
// RabbitMQ
.AddRabbitMQ(
builder.Configuration["RabbitMQ:ConnectionString"]!,
name: "rabbitmq",
tags: new[] { "messaging" })
// Custom service check
.AddCheck<ServiceHealthCheck>(
"services",
tags: new[] { "services" });
// Health endpoints
app.MapHealthChecks("/health", new HealthCheckOptions
{
ResponseWriter = WriteHealthResponse,
Predicate = _ => true
});
app.MapHealthChecks("/health/ready", new HealthCheckOptions
{
ResponseWriter = WriteHealthResponse,
Predicate = check => check.Tags.Contains("critical")
});
app.MapHealthChecks("/health/live", new HealthCheckOptions
{
Predicate = _ => false // Just returns 200 if app is running
});
static Task WriteHealthResponse(HttpContext context, HealthReport report)
{
context.Response.ContentType = "application/json";
var result = new
{
status = report.Status.ToString(),
duration = report.TotalDuration.TotalMilliseconds,
checks = report.Entries.Select(e => new
{
name = e.Key,
status = e.Value.Status.ToString(),
duration = e.Value.Duration.TotalMilliseconds,
description = e.Value.Description,
error = e.Value.Exception?.Message
})
};
return context.Response.WriteAsJsonAsync(result);
}
Day 4: Grafana Dashboards
Claude Command:
/devops-team create Grafana dashboards for POS metrics
Dashboard Configuration:
// grafana/dashboards/pos-overview.json
{
"title": "POS Platform Overview",
"uid": "pos-overview",
"panels": [
{
"title": "Sales Per Minute",
"type": "graph",
"targets": [
{
"expr": "rate(pos_sales_total[1m])",
"legendFormat": "{{tenant}} - {{location}}"
}
],
"gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 }
},
{
"title": "Revenue (Last Hour)",
"type": "stat",
"targets": [
{
"expr": "sum(increase(pos_sales_amount_total[1h])) / 100",
"legendFormat": "Revenue"
}
],
"gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 }
},
{
"title": "Active Sales",
"type": "gauge",
"targets": [
{
"expr": "sum(pos_active_sales)"
}
],
"gridPos": { "x": 18, "y": 0, "w": 6, "h": 4 }
},
{
"title": "Sale Processing Time (p95)",
"type": "graph",
"targets": [
{
"expr": "histogram_quantile(0.95, rate(pos_sale_processing_seconds_bucket[5m]))",
"legendFormat": "{{tenant}}"
}
],
"gridPos": { "x": 0, "y": 8, "w": 12, "h": 8 }
},
{
"title": "Low Stock Alerts",
"type": "stat",
"targets": [
{
"expr": "sum(pos_inventory_low_stock_items)"
}
],
"gridPos": { "x": 12, "y": 4, "w": 6, "h": 4 }
},
{
"title": "Sync Queue Depth",
"type": "gauge",
"targets": [
{
"expr": "sum(pos_sync_queue_depth)"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 10 },
{ "color": "red", "value": 50 }
]
},
"gridPos": { "x": 18, "y": 4, "w": 6, "h": 4 }
},
{
"title": "HTTP Request Rate",
"type": "graph",
"targets": [
{
"expr": "rate(http_requests_total[1m])",
"legendFormat": "{{method}} {{status}}"
}
],
"gridPos": { "x": 0, "y": 16, "w": 12, "h": 8 }
},
{
"title": "Database Connections",
"type": "gauge",
"targets": [
{
"expr": "pos_database_connections"
}
],
"gridPos": { "x": 12, "y": 8, "w": 6, "h": 4 }
}
]
}
Day 5: Alerting Rules
Claude Command:
/devops-team configure alerting rules for critical conditions
Prometheus Alert Rules:
# prometheus/alerts/pos-alerts.yml
groups:
- name: pos-critical
rules:
- alert: POSSalesDown
expr: rate(pos_sales_total[5m]) == 0 and hour() >= 9 and hour() <= 21
for: 5m
labels:
severity: critical
annotations:
summary: "No sales in last 5 minutes during business hours"
description: "{{ $labels.tenant }} at {{ $labels.location }} has no sales"
- alert: POSHighLatency
expr: histogram_quantile(0.95, rate(pos_sale_processing_seconds_bucket[5m])) > 5
for: 2m
labels:
severity: warning
annotations:
summary: "Sale processing latency is high"
description: "95th percentile latency is {{ $value }}s for {{ $labels.tenant }}"
- alert: POSSyncQueueBacklog
expr: pos_sync_queue_depth > 100
for: 10m
labels:
severity: critical
annotations:
summary: "Offline sync queue is backed up"
description: "{{ $value }} transactions pending sync for {{ $labels.tenant }}"
- alert: POSLowStockCritical
expr: pos_inventory_low_stock_items > 50
for: 30m
labels:
severity: warning
annotations:
summary: "Many items below reorder point"
description: "{{ $value }} items need reorder at {{ $labels.location }}"
- name: pos-infrastructure
rules:
- alert: POSDatabaseDown
expr: up{job="postgresql"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: "PostgreSQL database is down"
description: "Database connection failed"
- alert: POSRedisDown
expr: up{job="redis"} == 0
for: 2m
labels:
severity: critical
annotations:
summary: "Redis cache is down"
description: "Redis connection failed"
- alert: POSHighMemory
expr: process_resident_memory_bytes / 1024 / 1024 > 500
for: 5m
labels:
severity: warning
annotations:
summary: "High memory usage"
description: "Memory usage is {{ $value }}MB"
Alertmanager Configuration:
# alertmanager/config.yml
global:
smtp_smarthost: 'smtp.example.com:587'
smtp_from: 'pos-alerts@example.com'
slack_api_url: 'https://hooks.slack.com/services/xxx/xxx/xxx'
route:
group_by: ['alertname', 'tenant']
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
receiver: 'default'
routes:
- match:
severity: critical
receiver: 'pagerduty-critical'
- match:
severity: warning
receiver: 'slack-warnings'
receivers:
- name: 'default'
email_configs:
- to: 'ops@example.com'
- name: 'pagerduty-critical'
pagerduty_configs:
- service_key: 'your-pagerduty-key'
severity: critical
- name: 'slack-warnings'
slack_configs:
- channel: '#pos-alerts'
send_resolved: true
title: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}'
text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'
Week 15: Security Hardening
Day 1: Input Validation
Claude Command:
/security-team review and enhance input validation coverage
Implementation:
// src/PosPlatform.Api/Validation/ValidatorExtensions.cs
using FluentValidation;
namespace PosPlatform.Api.Validation;
public static class ValidatorExtensions
{
public static IRuleBuilderOptions<T, string> SafeString<T>(
this IRuleBuilder<T, string> ruleBuilder,
int maxLength = 255)
{
return ruleBuilder
.MaximumLength(maxLength)
.Must(s => s == null || !ContainsDangerousCharacters(s))
.WithMessage("Input contains invalid characters");
}
public static IRuleBuilderOptions<T, string> Sku<T>(
this IRuleBuilder<T, string> ruleBuilder)
{
return ruleBuilder
.NotEmpty()
.MaximumLength(50)
.Matches(@"^[A-Za-z0-9\-_]+$")
.WithMessage("SKU must contain only alphanumeric characters, hyphens, and underscores");
}
public static IRuleBuilderOptions<T, string> Email<T>(
this IRuleBuilder<T, string> ruleBuilder)
{
return ruleBuilder
.EmailAddress()
.MaximumLength(255)
.Must(e => e == null || !e.Contains(".."))
.WithMessage("Invalid email format");
}
public static IRuleBuilderOptions<T, decimal> MoneyAmount<T>(
this IRuleBuilder<T, decimal> ruleBuilder)
{
return ruleBuilder
.GreaterThanOrEqualTo(0)
.LessThanOrEqualTo(999999.99m)
.PrecisionScale(10, 2, true)
.WithMessage("Invalid monetary amount");
}
private static bool ContainsDangerousCharacters(string input)
{
var dangerous = new[] { "<script", "javascript:", "onclick", "onerror", "--", "/*", "*/" };
return dangerous.Any(d => input.Contains(d, StringComparison.OrdinalIgnoreCase));
}
}
// Validators for key DTOs
public class CreateProductRequestValidator : AbstractValidator<CreateProductRequest>
{
public CreateProductRequestValidator()
{
RuleFor(x => x.Sku).Sku();
RuleFor(x => x.Name).SafeString(255).NotEmpty();
RuleFor(x => x.Description).SafeString(2000);
RuleFor(x => x.BasePrice).MoneyAmount();
RuleFor(x => x.Cost).MoneyAmount();
}
}
public class CreateSaleRequestValidator : AbstractValidator<CreateSaleRequest>
{
public CreateSaleRequestValidator()
{
RuleFor(x => x.LocationId).NotEmpty();
RuleFor(x => x.CashierId).NotEmpty();
RuleForEach(x => x.Items).SetValidator(new SaleItemValidator());
}
}
public class SaleItemValidator : AbstractValidator<SaleItemRequest>
{
public SaleItemValidator()
{
RuleFor(x => x.Sku).Sku();
RuleFor(x => x.Quantity).GreaterThan(0).LessThanOrEqualTo(1000);
RuleFor(x => x.UnitPrice).MoneyAmount();
}
}
// Program.cs - Register validators
builder.Services.AddValidatorsFromAssemblyContaining<CreateProductRequestValidator>();
builder.Services.AddFluentValidationAutoValidation();
Day 2: Rate Limiting
Claude Command:
/dev-team implement rate limiting middleware
Implementation:
// Install package
// dotnet add package AspNetCoreRateLimit
// Program.cs
using AspNetCoreRateLimit;
builder.Services.AddMemoryCache();
builder.Services.Configure<IpRateLimitOptions>(options =>
{
options.EnableEndpointRateLimiting = true;
options.StackBlockedRequests = false;
options.HttpStatusCode = 429;
options.RealIpHeader = "X-Real-IP";
options.GeneralRules = new List<RateLimitRule>
{
// General API limit
new RateLimitRule
{
Endpoint = "*",
Period = "1m",
Limit = 100
},
// Login endpoints - stricter
new RateLimitRule
{
Endpoint = "*:/api/auth/*",
Period = "1m",
Limit = 10
},
// Sales endpoints
new RateLimitRule
{
Endpoint = "POST:/api/sales",
Period = "1s",
Limit = 5
}
};
});
builder.Services.AddSingleton<IRateLimitConfiguration, RateLimitConfiguration>();
builder.Services.AddInMemoryRateLimiting();
// In pipeline
app.UseIpRateLimiting();
// src/PosPlatform.Api/Middleware/LoginRateLimitMiddleware.cs
public class LoginRateLimitMiddleware
{
private readonly RequestDelegate _next;
private readonly IMemoryCache _cache;
private readonly ILogger<LoginRateLimitMiddleware> _logger;
private const int MaxAttempts = 5;
private const int LockoutMinutes = 15;
public LoginRateLimitMiddleware(
RequestDelegate next,
IMemoryCache cache,
ILogger<LoginRateLimitMiddleware> logger)
{
_next = next;
_cache = cache;
_logger = logger;
}
public async Task InvokeAsync(HttpContext context)
{
if (!IsLoginEndpoint(context))
{
await _next(context);
return;
}
var key = GetRateLimitKey(context);
if (IsLockedOut(key))
{
_logger.LogWarning("Login attempt blocked due to rate limiting: {Key}", key);
context.Response.StatusCode = 429;
await context.Response.WriteAsJsonAsync(new
{
error = "Too many login attempts. Please try again later.",
retryAfter = LockoutMinutes * 60
});
return;
}
await _next(context);
// Check if login failed (401 response)
if (context.Response.StatusCode == 401)
{
IncrementFailedAttempts(key);
}
else if (context.Response.StatusCode == 200)
{
ClearFailedAttempts(key);
}
}
private bool IsLoginEndpoint(HttpContext context)
=> context.Request.Path.StartsWithSegments("/api/auth/login") &&
context.Request.Method == "POST";
private string GetRateLimitKey(HttpContext context)
{
var ip = context.Connection.RemoteIpAddress?.ToString() ?? "unknown";
return $"login_attempts:{ip}";
}
private bool IsLockedOut(string key)
{
var attempts = _cache.Get<int>(key);
return attempts >= MaxAttempts;
}
private void IncrementFailedAttempts(string key)
{
var attempts = _cache.Get<int>(key) + 1;
_cache.Set(key, attempts, TimeSpan.FromMinutes(LockoutMinutes));
}
private void ClearFailedAttempts(string key)
{
_cache.Remove(key);
}
}
Day 3: Secrets Management
Claude Command:
/devops-team configure secrets management with HashiCorp Vault
Implementation:
// Install package
// dotnet add package VaultSharp
// src/PosPlatform.Infrastructure/Secrets/VaultSecretProvider.cs
using VaultSharp;
using VaultSharp.V1.AuthMethods.Token;
namespace PosPlatform.Infrastructure.Secrets;
public interface ISecretProvider
{
Task<string> GetSecretAsync(string path, string key);
Task<Dictionary<string, string>> GetSecretsAsync(string path);
}
public class VaultSecretProvider : ISecretProvider
{
private readonly IVaultClient _client;
private readonly ILogger<VaultSecretProvider> _logger;
public VaultSecretProvider(
IConfiguration configuration,
ILogger<VaultSecretProvider> logger)
{
_logger = logger;
var vaultAddr = configuration["Vault:Address"];
var vaultToken = configuration["Vault:Token"];
var authMethod = new TokenAuthMethodInfo(vaultToken);
var settings = new VaultClientSettings(vaultAddr, authMethod);
_client = new VaultClient(settings);
}
public async Task<string> GetSecretAsync(string path, string key)
{
var secrets = await GetSecretsAsync(path);
return secrets.TryGetValue(key, out var value) ? value : string.Empty;
}
public async Task<Dictionary<string, string>> GetSecretsAsync(string path)
{
try
{
var secret = await _client.V1.Secrets.KeyValue.V2.ReadSecretAsync(
path: path,
mountPoint: "secret");
return secret.Data.Data
.ToDictionary(
kv => kv.Key,
kv => kv.Value?.ToString() ?? string.Empty);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to retrieve secrets from path: {Path}", path);
throw;
}
}
}
// Startup configuration for secrets
public static class SecretConfigurationExtensions
{
public static IHostBuilder ConfigureSecrets(this IHostBuilder hostBuilder)
{
return hostBuilder.ConfigureAppConfiguration((context, config) =>
{
var settings = config.Build();
var vaultEnabled = settings.GetValue<bool>("Vault:Enabled");
if (vaultEnabled)
{
var vaultAddress = settings["Vault:Address"];
var vaultToken = Environment.GetEnvironmentVariable("VAULT_TOKEN");
config.AddVaultConfiguration(vaultAddress, vaultToken, new[]
{
"secret/data/pos/database",
"secret/data/pos/jwt",
"secret/data/pos/payment-gateway"
});
}
});
}
}
Day 4: Security Headers
Claude Command:
/dev-team add security headers middleware
Implementation:
// src/PosPlatform.Api/Middleware/SecurityHeadersMiddleware.cs
namespace PosPlatform.Api.Middleware;
public class SecurityHeadersMiddleware
{
private readonly RequestDelegate _next;
public SecurityHeadersMiddleware(RequestDelegate next)
{
_next = next;
}
public async Task InvokeAsync(HttpContext context)
{
// Prevent clickjacking
context.Response.Headers["X-Frame-Options"] = "DENY";
// Prevent MIME type sniffing
context.Response.Headers["X-Content-Type-Options"] = "nosniff";
// Enable XSS protection
context.Response.Headers["X-XSS-Protection"] = "1; mode=block";
// Content Security Policy
context.Response.Headers["Content-Security-Policy"] =
"default-src 'self'; " +
"script-src 'self' 'unsafe-inline' 'unsafe-eval'; " +
"style-src 'self' 'unsafe-inline'; " +
"img-src 'self' data: https:; " +
"font-src 'self'; " +
"connect-src 'self' wss: https:; " +
"frame-ancestors 'none'";
// Strict Transport Security (HTTPS only)
context.Response.Headers["Strict-Transport-Security"] =
"max-age=31536000; includeSubDomains";
// Referrer Policy
context.Response.Headers["Referrer-Policy"] = "strict-origin-when-cross-origin";
// Permissions Policy
context.Response.Headers["Permissions-Policy"] =
"accelerometer=(), camera=(), geolocation=(), gyroscope=(), " +
"magnetometer=(), microphone=(), payment=(), usb=()";
await _next(context);
}
}
// Extension method
public static class SecurityHeadersExtensions
{
public static IApplicationBuilder UseSecurityHeaders(this IApplicationBuilder app)
{
return app.UseMiddleware<SecurityHeadersMiddleware>();
}
}
Day 5: Security Scanning
Claude Command:
/security-team run security vulnerability scan and remediate findings
Security Scan Configuration:
# .github/workflows/security-scan.yml
name: Security Scan
on:
push:
branches: [main, develop]
pull_request:
branches: [main]
schedule:
- cron: '0 6 * * 1' # Weekly on Monday
jobs:
dependency-scan:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: '8.0.x'
- name: Restore dependencies
run: dotnet restore
- name: Run security audit
run: dotnet list package --vulnerable --include-transitive
code-scan:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: csharp
- name: Build
run: dotnet build
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
container-scan:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Build image
run: docker build -t pos-api:scan -f docker/Dockerfile .
- name: Run Trivy scan
uses: aquasecurity/trivy-action@master
with:
image-ref: 'pos-api:scan'
format: 'sarif'
output: 'trivy-results.sarif'
- name: Upload Trivy scan results
uses: github/codeql-action/upload-sarif@v3
with:
sarif_file: 'trivy-results.sarif'
Week 16: Production Deployment
Day 1-2: Production Infrastructure
Claude Command:
/devops-team provision production infrastructure with Kubernetes
Kubernetes Manifests:
# k8s/namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
name: pos-platform
labels:
name: pos-platform
---
# k8s/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: pos-api
namespace: pos-platform
spec:
replicas: 3
selector:
matchLabels:
app: pos-api
template:
metadata:
labels:
app: pos-api
spec:
containers:
- name: pos-api
image: pos-platform/api:latest
ports:
- containerPort: 8080
env:
- name: ASPNETCORE_ENVIRONMENT
value: "Production"
- name: ConnectionStrings__DefaultConnection
valueFrom:
secretKeyRef:
name: pos-secrets
key: database-connection
resources:
requests:
memory: "256Mi"
cpu: "250m"
limits:
memory: "512Mi"
cpu: "500m"
livenessProbe:
httpGet:
path: /health/live
port: 8080
initialDelaySeconds: 10
periodSeconds: 10
readinessProbe:
httpGet:
path: /health/ready
port: 8080
initialDelaySeconds: 5
periodSeconds: 5
imagePullSecrets:
- name: registry-credentials
---
# k8s/service.yaml
apiVersion: v1
kind: Service
metadata:
name: pos-api
namespace: pos-platform
spec:
selector:
app: pos-api
ports:
- protocol: TCP
port: 80
targetPort: 8080
type: ClusterIP
---
# k8s/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: pos-api
namespace: pos-platform
annotations:
kubernetes.io/ingress.class: nginx
cert-manager.io/cluster-issuer: letsencrypt-prod
spec:
tls:
- hosts:
- api.posplatform.com
secretName: pos-api-tls
rules:
- host: api.posplatform.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: pos-api
port:
number: 80
Day 3: Database Migration
Claude Command:
/devops-team run database migrations for production
Migration Script:
#!/bin/bash
# scripts/deploy-database.sh
set -e
echo "=== POS Platform Database Deployment ==="
# Configuration
DB_HOST=${DB_HOST:-"localhost"}
DB_PORT=${DB_PORT:-"5432"}
DB_NAME=${DB_NAME:-"pos_platform"}
DB_USER=${DB_USER:-"pos_admin"}
BACKUP_DIR="/backups/$(date +%Y%m%d_%H%M%S)"
# Create backup directory
mkdir -p "$BACKUP_DIR"
echo "1. Creating pre-migration backup..."
pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
-F c -f "$BACKUP_DIR/pre-migration.dump"
echo " Backup saved to: $BACKUP_DIR/pre-migration.dump"
echo "2. Running EF Core migrations..."
cd /app
dotnet ef database update --connection "Host=$DB_HOST;Port=$DB_PORT;Database=$DB_NAME;Username=$DB_USER;Password=$DB_PASSWORD"
echo "3. Verifying migration..."
TABLES=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -t -c \
"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'shared'")
if [ "$TABLES" -gt 0 ]; then
echo " Migration verified: $TABLES tables in shared schema"
else
echo " ERROR: No tables found in shared schema!"
echo " Rolling back..."
pg_restore -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
-c "$BACKUP_DIR/pre-migration.dump"
exit 1
fi
echo "4. Creating post-migration backup..."
pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
-F c -f "$BACKUP_DIR/post-migration.dump"
echo "=== Database deployment complete ==="
Day 4: Blue-Green Deployment
Claude Command:
/devops-team execute blue-green deployment for zero downtime
Deployment Script:
#!/bin/bash
# scripts/blue-green-deploy.sh
set -e
echo "=== POS Platform Blue-Green Deployment ==="
NAMESPACE="pos-platform"
NEW_VERSION=$1
CURRENT_COLOR=$(kubectl get svc pos-api -n $NAMESPACE -o jsonpath='{.spec.selector.color}')
if [ "$CURRENT_COLOR" == "blue" ]; then
NEW_COLOR="green"
else
NEW_COLOR="blue"
fi
echo "Current: $CURRENT_COLOR -> New: $NEW_COLOR"
echo "Deploying version: $NEW_VERSION"
# Step 1: Deploy new version
echo "1. Deploying $NEW_COLOR environment..."
kubectl apply -f - <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: pos-api-$NEW_COLOR
namespace: $NAMESPACE
spec:
replicas: 3
selector:
matchLabels:
app: pos-api
color: $NEW_COLOR
template:
metadata:
labels:
app: pos-api
color: $NEW_COLOR
version: $NEW_VERSION
spec:
containers:
- name: pos-api
image: pos-platform/api:$NEW_VERSION
ports:
- containerPort: 8080
readinessProbe:
httpGet:
path: /health/ready
port: 8080
initialDelaySeconds: 5
periodSeconds: 5
EOF
# Step 2: Wait for rollout
echo "2. Waiting for $NEW_COLOR rollout..."
kubectl rollout status deployment/pos-api-$NEW_COLOR -n $NAMESPACE --timeout=5m
# Step 3: Run smoke tests
echo "3. Running smoke tests against $NEW_COLOR..."
NEW_POD=$(kubectl get pod -n $NAMESPACE -l color=$NEW_COLOR -o jsonpath='{.items[0].metadata.name}')
kubectl exec -n $NAMESPACE $NEW_POD -- curl -s http://localhost:8080/health
# Step 4: Switch traffic
echo "4. Switching traffic to $NEW_COLOR..."
kubectl patch svc pos-api -n $NAMESPACE -p "{\"spec\":{\"selector\":{\"color\":\"$NEW_COLOR\"}}}"
# Step 5: Monitor
echo "5. Monitoring new deployment for 2 minutes..."
sleep 120
# Check error rate
ERROR_RATE=$(kubectl logs -n $NAMESPACE -l color=$NEW_COLOR --since=2m | grep -c "ERROR" || true)
if [ "$ERROR_RATE" -gt 10 ]; then
echo "ERROR: High error rate detected. Rolling back..."
kubectl patch svc pos-api -n $NAMESPACE -p "{\"spec\":{\"selector\":{\"color\":\"$CURRENT_COLOR\"}}}"
exit 1
fi
# Step 6: Scale down old
echo "6. Scaling down $CURRENT_COLOR..."
kubectl scale deployment pos-api-$CURRENT_COLOR -n $NAMESPACE --replicas=0
echo "=== Deployment complete ==="
echo "Version $NEW_VERSION is now live on $NEW_COLOR"
Day 5: Go-Live
Go-Live Checklist:
# POS Platform Go-Live Checklist
## Pre-Deployment (D-1)
- [ ] All features tested in staging environment
- [ ] Load testing completed (target: 100 concurrent users)
- [ ] Security scan passed with no critical issues
- [ ] Database backup verified and tested
- [ ] Rollback procedure documented and tested
- [ ] On-call rotation confirmed for launch weekend
- [ ] Customer support team briefed on new system
- [ ] DNS TTL reduced to 5 minutes
## Deployment Day (D-Day)
### Morning (Before Store Opens)
- [ ] Team standup at 6:00 AM
- [ ] Final database backup
- [ ] Deploy to production (blue-green)
- [ ] Verify health checks passing
- [ ] Run smoke tests
- [ ] Check monitoring dashboards
### Store Opening
- [ ] Process first test transaction
- [ ] Verify receipt printing
- [ ] Confirm payment processing
- [ ] Check inventory updates
### Throughout Day
- [ ] Monitor error rates (target: <0.1%)
- [ ] Monitor latency (target: p95 <500ms)
- [ ] Check offline sync queue
- [ ] Respond to support tickets within 15 minutes
## Post-Deployment (D+1)
- [ ] Review overnight logs
- [ ] Check daily sales reports
- [ ] Verify nightly backups ran
- [ ] Team retrospective meeting
- [ ] Document any issues encountered
## Success Criteria
- [ ] Zero data loss
- [ ] No customer-facing downtime during deployment
- [ ] All stores operational within 30 minutes of go-live
- [ ] Error rate below 0.1%
- [ ] All payments processed successfully
Rollback Procedures
Immediate Rollback
#!/bin/bash
# scripts/rollback.sh
set -e
NAMESPACE="pos-platform"
CURRENT_COLOR=$(kubectl get svc pos-api -n $NAMESPACE -o jsonpath='{.spec.selector.color}')
if [ "$CURRENT_COLOR" == "blue" ]; then
ROLLBACK_COLOR="green"
else
ROLLBACK_COLOR="blue"
fi
echo "!!! ROLLBACK INITIATED !!!"
echo "Switching from $CURRENT_COLOR to $ROLLBACK_COLOR"
# Scale up rollback environment
kubectl scale deployment pos-api-$ROLLBACK_COLOR -n $NAMESPACE --replicas=3
# Wait for pods
kubectl rollout status deployment/pos-api-$ROLLBACK_COLOR -n $NAMESPACE --timeout=2m
# Switch traffic
kubectl patch svc pos-api -n $NAMESPACE -p "{\"spec\":{\"selector\":{\"color\":\"$ROLLBACK_COLOR\"}}}"
echo "Rollback complete. Traffic now on $ROLLBACK_COLOR"
# Scale down failed deployment
kubectl scale deployment pos-api-$CURRENT_COLOR -n $NAMESPACE --replicas=0
Database Rollback
#!/bin/bash
# scripts/rollback-database.sh
BACKUP_FILE=$1
if [ -z "$BACKUP_FILE" ]; then
echo "Usage: rollback-database.sh <backup-file>"
exit 1
fi
echo "!!! DATABASE ROLLBACK !!!"
echo "Restoring from: $BACKUP_FILE"
# Stop application
kubectl scale deployment --all -n pos-platform --replicas=0
# Restore database
pg_restore -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
--clean --if-exists "$BACKUP_FILE"
# Restart application
kubectl scale deployment pos-api-blue -n pos-platform --replicas=3
echo "Database rollback complete"
Post Go-Live Operations
Daily Operations Checklist
## Daily POS Operations Checklist
### Morning (Before Store Opens)
- [ ] Check overnight error logs
- [ ] Verify backup completed successfully
- [ ] Review offline sync queue (should be empty)
- [ ] Check low stock alerts
- [ ] Verify all terminals connected
### Throughout Day
- [ ] Monitor sales dashboard
- [ ] Check response times
- [ ] Review any support tickets
- [ ] Monitor cash drawer sessions
### Evening (After Store Closes)
- [ ] Review daily sales summary
- [ ] Check inventory discrepancies
- [ ] Verify cash drawer reconciliations
- [ ] Confirm backup initiated
Next Steps
With production deployment complete:
- Review Appendix A for API reference documentation
- Consult Appendix B for troubleshooting guides
- Reference Appendix C for operational procedures
- Plan ongoing maintenance and feature development
Chapter 28 Complete - Phase 4 Production Implementation
Congratulations!
You have completed the POS Platform Implementation Guide. The system is now:
- Multi-tenant with complete data isolation
- Fully authenticated with JWT and PIN support
- Processing sales with event-sourced transactions
- Managing inventory across multiple locations
- Supporting customer loyalty programs
- Operating offline with sync capabilities
- Monitored with comprehensive observability
- Secured with defense-in-depth measures
- Deployed with zero-downtime capability
Total Implementation Time: 16 weeks Lines of Code: ~15,000+ Database Tables: 30+ API Endpoints: 50+
Welcome to production!