using System.Linq.Expressions; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Query; using SqrtSpace.SpaceTime.Core; namespace SqrtSpace.SpaceTime.EntityFramework; /// /// Extended query extensions for SpaceTime optimizations /// public static class SpaceTimeQueryExtensions { /// /// Configures the query to use external sorting for large datasets /// public static IQueryable UseExternalSorting(this IQueryable query) where T : class { // Mark the query for external sorting return query.TagWith("SpaceTime:UseExternalSorting"); } /// /// Streams query results asynchronously for memory efficiency /// public static async IAsyncEnumerable StreamQueryResultsAsync( this IQueryable query, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) where T : class { var context = GetDbContext(query); if (context == null) { // Fallback to regular async enumeration await foreach (var item in query.AsAsyncEnumerable().WithCancellation(cancellationToken)) { yield return item; } yield break; } // Get total count for batch size calculation var totalCount = await query.CountAsync(cancellationToken); var batchSize = SpaceTimeCalculator.CalculateSqrtInterval(totalCount); // Stream in batches for (int offset = 0; offset < totalCount; offset += batchSize) { var batch = await query .Skip(offset) .Take(batchSize) .ToListAsync(cancellationToken); foreach (var item in batch) { yield return item; } // Clear change tracker to prevent memory buildup context.ChangeTracker.Clear(); } } /// /// Processes query results in √n-sized batches with checkpoint support /// public static async IAsyncEnumerable> BatchBySqrtNAsync( this IQueryable query, string? checkpointId = null, bool resumeFromCheckpoint = false, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) where T : class { var context = GetDbContext(query); var options = context?.GetService(); CheckpointManager? checkpointManager = null; if (!string.IsNullOrEmpty(checkpointId) && options?.EnableQueryCheckpointing == true) { checkpointManager = new CheckpointManager(options.CheckpointDirectory); } try { var totalCount = await query.CountAsync(cancellationToken); var batchSize = SpaceTimeCalculator.CalculateSqrtInterval(totalCount); var startOffset = 0; // Resume from checkpoint if requested if (resumeFromCheckpoint && checkpointManager != null) { var checkpoint = await checkpointManager.RestoreCheckpointAsync(checkpointId!); if (checkpoint != null) { startOffset = checkpoint.ProcessedCount; } } for (int offset = startOffset; offset < totalCount; offset += batchSize) { var batch = await query .Skip(offset) .Take(batchSize) .ToListAsync(cancellationToken); if (batch.Count == 0) yield break; yield return batch; // Save checkpoint if (checkpointManager != null && !string.IsNullOrEmpty(checkpointId)) { await checkpointManager.CreateCheckpointAsync(new QueryCheckpoint { ProcessedCount = offset + batch.Count, TotalCount = totalCount }, checkpointId); } // Clear change tracker if enabled if (context != null && options?.EnableSqrtNChangeTracking == true) { var trackedCount = context.ChangeTracker.Entries().Count(); if (trackedCount > (options.MaxTrackedEntities ?? batchSize)) { context.ChangeTracker.Clear(); } } } } finally { checkpointManager?.Dispose(); } } private static DbContext? GetDbContext(IQueryable query) { if (query.Provider is IInfrastructure infrastructure) { var context = infrastructure.Instance.GetService(typeof(DbContext)) as DbContext; return context; } // Fallback: try reflection var provider = query.Provider; var contextProperty = provider.GetType().GetProperty("Context"); return contextProperty?.GetValue(provider) as DbContext; } private class QueryCheckpoint { public int ProcessedCount { get; set; } public int TotalCount { get; set; } } } /// /// Extension methods for DbContext bulk operations /// public static class SpaceTimeDbContextExtensions { /// /// Performs bulk insert with √n buffering for memory efficiency /// public static async Task BulkInsertWithSqrtNBufferingAsync( this DbContext context, IEnumerable entities, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(context); ArgumentNullException.ThrowIfNull(entities); var options = context.GetService(); var entityList = entities as IList ?? entities.ToList(); var totalCount = entityList.Count; var batchSize = SpaceTimeCalculator.CalculateSqrtInterval(totalCount); // Disable auto-detect changes for performance var originalAutoDetectChanges = context.ChangeTracker.AutoDetectChangesEnabled; context.ChangeTracker.AutoDetectChangesEnabled = false; try { for (int i = 0; i < totalCount; i += batchSize) { var batch = entityList.Skip(i).Take(batchSize); await context.AddRangeAsync(batch, cancellationToken); await context.SaveChangesAsync(cancellationToken); // Clear change tracker after each batch to prevent memory buildup context.ChangeTracker.Clear(); } } finally { context.ChangeTracker.AutoDetectChangesEnabled = originalAutoDetectChanges; } } /// /// Performs bulk update with √n buffering for memory efficiency /// public static async Task BulkUpdateWithSqrtNBufferingAsync( this DbContext context, IEnumerable entities, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(context); ArgumentNullException.ThrowIfNull(entities); var entityList = entities as IList ?? entities.ToList(); var totalCount = entityList.Count; var batchSize = SpaceTimeCalculator.CalculateSqrtInterval(totalCount); // Disable auto-detect changes for performance var originalAutoDetectChanges = context.ChangeTracker.AutoDetectChangesEnabled; context.ChangeTracker.AutoDetectChangesEnabled = false; try { for (int i = 0; i < totalCount; i += batchSize) { var batch = entityList.Skip(i).Take(batchSize); context.UpdateRange(batch); await context.SaveChangesAsync(cancellationToken); // Clear change tracker after each batch context.ChangeTracker.Clear(); } } finally { context.ChangeTracker.AutoDetectChangesEnabled = originalAutoDetectChanges; } } /// /// Performs bulk delete with √n buffering for memory efficiency /// public static async Task BulkDeleteWithSqrtNBufferingAsync( this DbContext context, IEnumerable entities, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(context); ArgumentNullException.ThrowIfNull(entities); var entityList = entities as IList ?? entities.ToList(); var totalCount = entityList.Count; var batchSize = SpaceTimeCalculator.CalculateSqrtInterval(totalCount); // Disable auto-detect changes for performance var originalAutoDetectChanges = context.ChangeTracker.AutoDetectChangesEnabled; context.ChangeTracker.AutoDetectChangesEnabled = false; try { for (int i = 0; i < totalCount; i += batchSize) { var batch = entityList.Skip(i).Take(batchSize); context.RemoveRange(batch); await context.SaveChangesAsync(cancellationToken); // Clear change tracker after each batch context.ChangeTracker.Clear(); } } finally { context.ChangeTracker.AutoDetectChangesEnabled = originalAutoDetectChanges; } } }