using Microsoft.EntityFrameworkCore; using SqrtSpace.SpaceTime.Core; using SqrtSpace.SpaceTime.EntityFramework; using SqrtSpace.SpaceTime.Linq; using SampleWebApi.Data; using SampleWebApi.Models; using System.Text; namespace SampleWebApi.Services; public interface IProductService { Task> GetProductsPagedAsync(int page, int pageSize); IAsyncEnumerable StreamProductsAsync(string? category, decimal? minPrice); Task> SearchProductsAsync(string query, string sortBy, bool descending); Task BulkUpdatePricesAsync(string? categoryFilter, decimal priceMultiplier, string operationId, CheckpointManager? checkpoint); Task ExportToCsvAsync(Stream outputStream, string? category); Task GetStatisticsAsync(string? category); } public class ProductService : IProductService { private readonly SampleDbContext _context; private readonly ILogger _logger; public ProductService(SampleDbContext context, ILogger logger) { _context = context; _logger = logger; } public async Task> GetProductsPagedAsync(int page, int pageSize) { var query = _context.Products.AsQueryable(); var totalCount = await query.CountAsync(); var items = await query .Skip((page - 1) * pageSize) .Take(pageSize) .ToListAsync(); return new PagedResult { Items = items, Page = page, PageSize = pageSize, TotalCount = totalCount }; } public async IAsyncEnumerable StreamProductsAsync(string? category, decimal? minPrice) { var query = _context.Products.AsQueryable(); if (!string.IsNullOrEmpty(category)) { query = query.Where(p => p.Category == category); } if (minPrice.HasValue) { query = query.Where(p => p.Price >= minPrice.Value); } // Use BatchBySqrtN to process in memory-efficient chunks await foreach (var batch in query.BatchBySqrtNAsync()) { foreach (var product in batch) { yield return product; } } } public async Task> SearchProductsAsync(string query, string sortBy, bool descending) { var searchQuery = _context.Products .Where(p => p.Name.Contains(query) || p.Description.Contains(query)); // Count to determine if we need external sorting var count = await searchQuery.CountAsync(); _logger.LogInformation("Search found {count} products for query '{query}'", count, query); IQueryable sortedQuery = sortBy.ToLower() switch { "price" => descending ? searchQuery.OrderByDescending(p => p.Price) : searchQuery.OrderBy(p => p.Price), "category" => descending ? searchQuery.OrderByDescending(p => p.Category) : searchQuery.OrderBy(p => p.Category), _ => descending ? searchQuery.OrderByDescending(p => p.Name) : searchQuery.OrderBy(p => p.Name) }; // Use external sorting for large result sets if (count > 10000) { _logger.LogInformation("Using external sorting for {count} products", count); sortedQuery = sortedQuery.UseExternalSorting(); } return await sortedQuery.ToListAsync(); } public async Task BulkUpdatePricesAsync( string? categoryFilter, decimal priceMultiplier, string operationId, CheckpointManager? checkpoint) { var state = new BulkUpdateState { OperationId = operationId }; // Try to restore from checkpoint if (checkpoint != null) { var previousState = await checkpoint.RestoreLatestCheckpointAsync(); if (previousState != null) { state = previousState; _logger.LogInformation("Resuming bulk update from checkpoint. Already processed: {count}", state.ProcessedCount); } } var query = _context.Products.AsQueryable(); if (!string.IsNullOrEmpty(categoryFilter)) { query = query.Where(p => p.Category == categoryFilter); } var totalProducts = await query.CountAsync(); var products = query.Skip(state.ProcessedCount); // Process in batches using √n strategy await foreach (var batch in products.BatchBySqrtNAsync()) { try { foreach (var product in batch) { product.Price *= priceMultiplier; product.UpdatedAt = DateTime.UtcNow; state.ProcessedCount++; state.UpdatedCount++; } await _context.SaveChangesAsync(); // Save checkpoint if (checkpoint?.ShouldCheckpoint() == true) { state.LastCheckpoint = DateTime.UtcNow; await checkpoint.CreateCheckpointAsync(state); _logger.LogInformation("Checkpoint saved. Processed: {count}/{total}", state.ProcessedCount, totalProducts); } } catch (Exception ex) { _logger.LogError(ex, "Error updating batch. Processed so far: {count}", state.ProcessedCount); state.FailedCount += batch.Count - (state.ProcessedCount % batch.Count); // Save checkpoint on error if (checkpoint != null) { await checkpoint.CreateCheckpointAsync(state); } throw; } } return new BulkUpdateResult { OperationId = operationId, TotalProducts = totalProducts, UpdatedProducts = state.UpdatedCount, FailedProducts = state.FailedCount, Completed = true, CheckpointId = state.LastCheckpoint.ToString("O") }; } public async Task ExportToCsvAsync(Stream outputStream, string? category) { using var writer = new StreamWriter(outputStream, Encoding.UTF8); // Write header await writer.WriteLineAsync("Id,Name,Category,Price,StockQuantity,CreatedAt,UpdatedAt"); var query = _context.Products.AsQueryable(); if (!string.IsNullOrEmpty(category)) { query = query.Where(p => p.Category == category); } // Stream products in batches to minimize memory usage await foreach (var batch in query.BatchBySqrtNAsync()) { foreach (var product in batch) { await writer.WriteLineAsync( $"{product.Id}," + $"\"{product.Name.Replace("\"", "\"\"")}\"," + $"\"{product.Category}\"," + $"{product.Price}," + $"{product.StockQuantity}," + $"{product.CreatedAt:yyyy-MM-dd HH:mm:ss}," + $"{product.UpdatedAt:yyyy-MM-dd HH:mm:ss}"); } await writer.FlushAsync(); } } public async Task GetStatisticsAsync(string? category) { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var query = _context.Products.AsQueryable(); if (!string.IsNullOrEmpty(category)) { query = query.Where(p => p.Category == category); } var totalCount = await query.CountAsync(); var computationMethod = totalCount > 100000 ? "External" : "InMemory"; ProductStatistics stats; if (computationMethod == "External") { _logger.LogInformation("Using external aggregation for {count} products", totalCount); // For large datasets, compute statistics in batches decimal totalPrice = 0; decimal minPrice = decimal.MaxValue; decimal maxPrice = decimal.MinValue; var categoryStats = new Dictionary(); await foreach (var batch in query.BatchBySqrtNAsync()) { foreach (var product in batch) { totalPrice += product.Price; minPrice = Math.Min(minPrice, product.Price); maxPrice = Math.Max(maxPrice, product.Price); if (!categoryStats.ContainsKey(product.Category)) { categoryStats[product.Category] = (0, 0); } var current = categoryStats[product.Category]; categoryStats[product.Category] = (current.count + 1, current.totalPrice + product.Price); } } stats = new ProductStatistics { TotalProducts = totalCount, AveragePrice = totalCount > 0 ? totalPrice / totalCount : 0, MinPrice = minPrice == decimal.MaxValue ? 0 : minPrice, MaxPrice = maxPrice == decimal.MinValue ? 0 : maxPrice, ProductsByCategory = categoryStats.ToDictionary(k => k.Key, v => v.Value.count), AveragePriceByCategory = categoryStats.ToDictionary( k => k.Key, v => v.Value.count > 0 ? v.Value.totalPrice / v.Value.count : 0) }; } else { // For smaller datasets, use in-memory aggregation var products = await query.ToListAsync(); stats = new ProductStatistics { TotalProducts = products.Count, AveragePrice = products.Any() ? products.Average(p => p.Price) : 0, MinPrice = products.Any() ? products.Min(p => p.Price) : 0, MaxPrice = products.Any() ? products.Max(p => p.Price) : 0, ProductsByCategory = products.GroupBy(p => p.Category) .ToDictionary(g => g.Key, g => g.Count()), AveragePriceByCategory = products.GroupBy(p => p.Category) .ToDictionary(g => g.Key, g => g.Average(p => p.Price)) }; } stats.ComputationTimeMs = stopwatch.ElapsedMilliseconds; stats.ComputationMethod = computationMethod; return stats; } }