David Friedel
45d6680396
- Update paper repository URL to git.marketally.com - Update contributing guide with Gitea URLs and instructions - Update repository links to point to new Gitea location
726 lines
19 KiB
Markdown
726 lines
19 KiB
Markdown
# SqrtSpace SpaceTime for .NET
|
|
|
|
[](https://www.nuget.org/packages/SqrtSpace.SpaceTime.Core/)
|
|
[](LICENSE)
|
|
|
|
Memory-efficient algorithms and data structures for .NET using Williams' √n space-time tradeoffs. Reduce memory usage by 90-99% with minimal performance impact.
|
|
|
|
**Paper Repository**: [git.marketally.com/sqrtspace/sqrtspace-paper](https://git.marketally.com/sqrtspace/sqrtspace-paper)
|
|
|
|
## Quick Start
|
|
|
|
```bash
|
|
# Core functionality
|
|
dotnet add package SqrtSpace.SpaceTime.Core
|
|
|
|
# LINQ extensions
|
|
dotnet add package SqrtSpace.SpaceTime.Linq
|
|
|
|
# Adaptive collections
|
|
dotnet add package SqrtSpace.SpaceTime.Collections
|
|
|
|
# Entity Framework Core integration
|
|
dotnet add package SqrtSpace.SpaceTime.EntityFramework
|
|
|
|
# ASP.NET Core middleware
|
|
dotnet add package SqrtSpace.SpaceTime.AspNetCore
|
|
|
|
# Roslyn analyzers
|
|
dotnet add package SqrtSpace.SpaceTime.Analyzers
|
|
|
|
# Additional packages
|
|
dotnet add package SqrtSpace.SpaceTime.Caching
|
|
dotnet add package SqrtSpace.SpaceTime.Distributed
|
|
dotnet add package SqrtSpace.SpaceTime.Diagnostics
|
|
dotnet add package SqrtSpace.SpaceTime.Scheduling
|
|
dotnet add package SqrtSpace.SpaceTime.Pipeline
|
|
dotnet add package SqrtSpace.SpaceTime.Configuration
|
|
dotnet add package SqrtSpace.SpaceTime.Serialization
|
|
dotnet add package SqrtSpace.SpaceTime.MemoryManagement
|
|
```
|
|
|
|
## What's Included
|
|
|
|
### 1. Core Library
|
|
|
|
Foundation for all SpaceTime optimizations:
|
|
|
|
```csharp
|
|
using SqrtSpace.SpaceTime.Core;
|
|
|
|
// Calculate optimal buffer sizes
|
|
int bufferSize = SpaceTimeCalculator.CalculateSqrtInterval(dataSize);
|
|
|
|
// Get memory hierarchy information
|
|
var hierarchy = MemoryHierarchy.GetCurrent();
|
|
Console.WriteLine($"L1 Cache: {hierarchy.L1CacheSize:N0} bytes");
|
|
Console.WriteLine($"L2 Cache: {hierarchy.L2CacheSize:N0} bytes");
|
|
Console.WriteLine($"Available RAM: {hierarchy.AvailableMemory:N0} bytes");
|
|
|
|
// Use external storage for large data
|
|
using var storage = new ExternalStorage<Record>("data.tmp");
|
|
await storage.AppendAsync(records);
|
|
```
|
|
|
|
### 2. Memory-Aware LINQ Extensions
|
|
|
|
Transform memory-hungry LINQ operations:
|
|
|
|
```csharp
|
|
using SqrtSpace.SpaceTime.Linq;
|
|
|
|
// Standard LINQ - loads all 10M items into memory
|
|
var sorted = millionItems
|
|
.OrderBy(x => x.Date)
|
|
.ToList(); // 800MB memory
|
|
|
|
// SpaceTime LINQ - uses √n memory
|
|
var sorted = millionItems
|
|
.OrderByExternal(x => x.Date)
|
|
.ToList(); // 25MB memory (97% less!)
|
|
|
|
// Process in optimal batches
|
|
await foreach (var batch in largeQuery.BatchBySqrtNAsync())
|
|
{
|
|
await ProcessBatch(batch);
|
|
}
|
|
|
|
// External joins for large datasets
|
|
var results = customers
|
|
.JoinExternal(orders, c => c.Id, o => o.CustomerId,
|
|
(c, o) => new { Customer = c, Order = o })
|
|
.ToList();
|
|
```
|
|
|
|
### 3. Adaptive Collections
|
|
|
|
Collections that automatically switch implementations based on size:
|
|
|
|
```csharp
|
|
using SqrtSpace.SpaceTime.Collections;
|
|
|
|
// Automatically adapts: Array → Dictionary → B-Tree → External storage
|
|
var adaptiveMap = new AdaptiveDictionary<string, Customer>();
|
|
|
|
// Starts as array (< 16 items)
|
|
adaptiveMap["user1"] = customer1;
|
|
|
|
// Switches to Dictionary (< 10K items)
|
|
for (int i = 0; i < 5000; i++)
|
|
adaptiveMap[$"user{i}"] = customers[i];
|
|
|
|
// Switches to B-Tree (< 1M items)
|
|
// Then to external storage (> 1M items) with √n memory
|
|
|
|
// Adaptive lists with external sorting
|
|
var list = new AdaptiveList<Order>();
|
|
list.AddRange(millionOrders);
|
|
list.Sort(); // Automatically uses external sort if needed
|
|
```
|
|
|
|
### 4. Entity Framework Core Optimizations
|
|
|
|
Optimize EF Core for large datasets:
|
|
|
|
```csharp
|
|
services.AddDbContext<AppDbContext>(options =>
|
|
{
|
|
options.UseSqlServer(connectionString)
|
|
.UseSpaceTimeOptimizer(opt =>
|
|
{
|
|
opt.EnableSqrtNChangeTracking = true;
|
|
opt.BufferPoolStrategy = BufferPoolStrategy.SqrtN;
|
|
opt.EnableQueryCheckpointing = true;
|
|
});
|
|
});
|
|
|
|
// Query with √n memory usage
|
|
var results = await dbContext.Orders
|
|
.Where(o => o.Status == "Pending")
|
|
.ToListWithSqrtNMemoryAsync();
|
|
|
|
// Process in optimal batches
|
|
await foreach (var batch in dbContext.Customers.BatchBySqrtNAsync())
|
|
{
|
|
await ProcessCustomerBatch(batch);
|
|
}
|
|
|
|
// Optimized change tracking
|
|
using (dbContext.BeginSqrtNTracking())
|
|
{
|
|
// Make changes to thousands of entities
|
|
await dbContext.BulkUpdateAsync(entities);
|
|
}
|
|
```
|
|
|
|
### 5. ASP.NET Core Streaming
|
|
|
|
Stream large responses efficiently:
|
|
|
|
```csharp
|
|
[HttpGet("large-dataset")]
|
|
[SpaceTimeStreaming(ChunkStrategy = ChunkStrategy.SqrtN)]
|
|
public async IAsyncEnumerable<DataItem> GetLargeDataset()
|
|
{
|
|
// Automatically chunks response using √n sizing
|
|
await foreach (var item in repository.GetAllAsync())
|
|
{
|
|
yield return item;
|
|
}
|
|
}
|
|
|
|
// In Program.cs
|
|
builder.Services.AddSpaceTime(options =>
|
|
{
|
|
options.EnableCheckpointing = true;
|
|
options.EnableStreaming = true;
|
|
options.DefaultChunkSize = SpaceTimeDefaults.SqrtN;
|
|
});
|
|
|
|
app.UseSpaceTime();
|
|
app.UseSpaceTimeEndpoints();
|
|
```
|
|
|
|
### 6. Memory-Aware Caching
|
|
|
|
Intelligent caching with hot/cold storage:
|
|
|
|
```csharp
|
|
using SqrtSpace.SpaceTime.Caching;
|
|
|
|
// Configure caching
|
|
services.AddSpaceTimeCaching(options =>
|
|
{
|
|
options.MaxHotMemory = 100 * 1024 * 1024; // 100MB hot cache
|
|
options.EnableColdStorage = true;
|
|
options.ColdStoragePath = "/tmp/cache";
|
|
options.EvictionStrategy = EvictionStrategy.SqrtN;
|
|
});
|
|
|
|
// Use the cache
|
|
public class ProductService
|
|
{
|
|
private readonly ISpaceTimeCache<string, Product> _cache;
|
|
|
|
public async Task<Product> GetProductAsync(string id)
|
|
{
|
|
return await _cache.GetOrAddAsync(id, async () =>
|
|
{
|
|
// Expensive database query
|
|
return await _repository.GetProductAsync(id);
|
|
});
|
|
}
|
|
}
|
|
```
|
|
|
|
### 7. Distributed Processing
|
|
|
|
Coordinate work across multiple nodes:
|
|
|
|
```csharp
|
|
using SqrtSpace.SpaceTime.Distributed;
|
|
|
|
// Configure distributed coordinator
|
|
services.AddSpaceTimeDistributed(options =>
|
|
{
|
|
options.NodeId = Environment.MachineName;
|
|
options.CoordinationEndpoint = "redis://coordinator:6379";
|
|
});
|
|
|
|
// Use distributed processing
|
|
public class DataProcessor
|
|
{
|
|
private readonly ISpaceTimeCoordinator _coordinator;
|
|
|
|
public async Task ProcessLargeDatasetAsync(string datasetId)
|
|
{
|
|
// Get optimal partition for this node
|
|
var partition = await _coordinator.RequestPartitionAsync(
|
|
datasetId, estimatedSize: 10_000_000);
|
|
|
|
// Process only this node's portion
|
|
await foreach (var item in GetPartitionData(partition))
|
|
{
|
|
await ProcessItem(item);
|
|
await _coordinator.ReportProgressAsync(partition.Id, 1);
|
|
}
|
|
}
|
|
}
|
|
```
|
|
|
|
### 8. Diagnostics and Monitoring
|
|
|
|
Comprehensive diagnostics with OpenTelemetry:
|
|
|
|
```csharp
|
|
using SqrtSpace.SpaceTime.Diagnostics;
|
|
|
|
// Configure diagnostics
|
|
services.AddSpaceTimeDiagnostics(options =>
|
|
{
|
|
options.EnableMetrics = true;
|
|
options.EnableTracing = true;
|
|
options.EnableMemoryTracking = true;
|
|
});
|
|
|
|
// Monitor operations
|
|
public class ImportService
|
|
{
|
|
private readonly ISpaceTimeDiagnostics _diagnostics;
|
|
|
|
public async Task ImportDataAsync(string filePath)
|
|
{
|
|
using var operation = _diagnostics.StartOperation(
|
|
"DataImport", OperationType.BatchProcessing);
|
|
|
|
operation.SetTag("file.path", filePath);
|
|
operation.SetTag("file.size", new FileInfo(filePath).Length);
|
|
|
|
try
|
|
{
|
|
await ProcessFile(filePath);
|
|
operation.RecordSuccess();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
operation.RecordError(ex);
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
```
|
|
|
|
### 9. Memory-Aware Task Scheduling
|
|
|
|
Schedule tasks based on memory availability:
|
|
|
|
```csharp
|
|
using SqrtSpace.SpaceTime.Scheduling;
|
|
|
|
// Configure scheduler
|
|
services.AddSpaceTimeScheduling(options =>
|
|
{
|
|
options.MaxMemoryPerTask = 50 * 1024 * 1024; // 50MB per task
|
|
options.EnableMemoryThrottling = true;
|
|
});
|
|
|
|
// Schedule memory-intensive tasks
|
|
public class BatchProcessor
|
|
{
|
|
private readonly ISpaceTimeTaskScheduler _scheduler;
|
|
|
|
public async Task ProcessBatchesAsync(IEnumerable<Batch> batches)
|
|
{
|
|
var tasks = batches.Select(batch =>
|
|
_scheduler.ScheduleAsync(async () =>
|
|
{
|
|
await ProcessBatch(batch);
|
|
},
|
|
estimatedMemory: batch.EstimatedMemoryUsage,
|
|
priority: TaskPriority.Normal));
|
|
|
|
await Task.WhenAll(tasks);
|
|
}
|
|
}
|
|
```
|
|
|
|
### 10. Data Pipeline Framework
|
|
|
|
Build memory-efficient data pipelines:
|
|
|
|
```csharp
|
|
using SqrtSpace.SpaceTime.Pipeline;
|
|
|
|
// Build a pipeline
|
|
var pipeline = pipelineFactory.CreatePipeline<InputData, OutputData>("ImportPipeline")
|
|
.AddTransform("Parse", async (input, ct) =>
|
|
await ParseData(input))
|
|
.AddBatch("Validate", async (batch, ct) =>
|
|
await ValidateBatch(batch))
|
|
.AddFilter("FilterInvalid", data =>
|
|
data.IsValid)
|
|
.AddCheckpoint("SaveProgress")
|
|
.AddParallel("Enrich", async (data, ct) =>
|
|
await EnrichData(data), maxConcurrency: 4)
|
|
.Build();
|
|
|
|
// Execute pipeline
|
|
var result = await pipeline.ExecuteAsync(inputData);
|
|
```
|
|
|
|
### 11. Configuration and Policy System
|
|
|
|
Centralized configuration management:
|
|
|
|
```csharp
|
|
using SqrtSpace.SpaceTime.Configuration;
|
|
|
|
// Configure SpaceTime
|
|
services.AddSpaceTimeConfiguration(configuration);
|
|
|
|
// Define policies
|
|
services.Configure<SpaceTimeConfiguration>(options =>
|
|
{
|
|
options.Memory.MaxMemory = 1_073_741_824; // 1GB
|
|
options.Memory.ExternalAlgorithmThreshold = 0.7; // Switch at 70%
|
|
options.Algorithms.Policies["Sort"] = new AlgorithmPolicy
|
|
{
|
|
PreferExternal = true,
|
|
SizeThreshold = 1_000_000
|
|
};
|
|
});
|
|
|
|
// Use policy engine
|
|
public class DataService
|
|
{
|
|
private readonly IPolicyEngine _policyEngine;
|
|
|
|
public async Task<ProcessingStrategy> DetermineStrategyAsync(long dataSize)
|
|
{
|
|
var context = new PolicyContext
|
|
{
|
|
OperationType = "DataProcessing",
|
|
DataSize = dataSize,
|
|
AvailableMemory = GC.GetTotalMemory(false)
|
|
};
|
|
|
|
var result = await _policyEngine.EvaluateAsync(context);
|
|
return result.ShouldProceed
|
|
? ProcessingStrategy.Continue
|
|
: ProcessingStrategy.Defer;
|
|
}
|
|
}
|
|
```
|
|
|
|
### 12. Serialization Optimizers
|
|
|
|
Memory-efficient serialization with streaming:
|
|
|
|
```csharp
|
|
using SqrtSpace.SpaceTime.Serialization;
|
|
|
|
// Configure serialization
|
|
services.AddSpaceTimeSerialization(builder =>
|
|
{
|
|
builder.UseFormat(SerializationFormat.MessagePack)
|
|
.ConfigureCompression(enable: true, level: 6)
|
|
.ConfigureMemoryLimits(100 * 1024 * 1024); // 100MB
|
|
});
|
|
|
|
// Stream large collections
|
|
public class ExportService
|
|
{
|
|
private readonly StreamingSerializer<Customer> _serializer;
|
|
|
|
public async Task ExportCustomersAsync(string filePath)
|
|
{
|
|
await _serializer.SerializeToFileAsync(
|
|
GetCustomersAsync(),
|
|
filePath,
|
|
options: new SerializationOptions
|
|
{
|
|
EnableCheckpointing = true,
|
|
BufferSize = 0 // Auto √n sizing
|
|
},
|
|
progress: new Progress<SerializationProgress>(p =>
|
|
{
|
|
Console.WriteLine($"Exported {p.ItemsProcessed:N0} items");
|
|
}));
|
|
}
|
|
}
|
|
```
|
|
|
|
### 13. Memory Pressure Handling
|
|
|
|
Automatic response to memory pressure:
|
|
|
|
```csharp
|
|
using SqrtSpace.SpaceTime.MemoryManagement;
|
|
|
|
// Configure memory management
|
|
services.AddSpaceTimeMemoryManagement(options =>
|
|
{
|
|
options.EnableAutomaticHandling = true;
|
|
options.CheckInterval = TimeSpan.FromSeconds(5);
|
|
});
|
|
|
|
// Add custom handler
|
|
services.AddMemoryPressureHandler<CustomCacheEvictionHandler>();
|
|
|
|
// Monitor memory pressure
|
|
public class MemoryAwareService
|
|
{
|
|
private readonly IMemoryPressureMonitor _monitor;
|
|
|
|
public MemoryAwareService(IMemoryPressureMonitor monitor)
|
|
{
|
|
_monitor = monitor;
|
|
_monitor.PressureEvents.Subscribe(OnMemoryPressure);
|
|
}
|
|
|
|
private void OnMemoryPressure(MemoryPressureEvent e)
|
|
{
|
|
if (e.CurrentLevel >= MemoryPressureLevel.High)
|
|
{
|
|
// Reduce memory usage
|
|
TrimCaches();
|
|
ForceGarbageCollection();
|
|
}
|
|
}
|
|
}
|
|
```
|
|
|
|
### 14. Checkpointing for Fault Tolerance
|
|
|
|
Add automatic checkpointing to long-running operations:
|
|
|
|
```csharp
|
|
[EnableCheckpoint(Strategy = CheckpointStrategy.SqrtN)]
|
|
public async Task<ImportResult> ImportLargeDataset(string filePath)
|
|
{
|
|
var checkpoint = HttpContext.Features.Get<ICheckpointFeature>();
|
|
var results = new List<Record>();
|
|
|
|
await foreach (var record in ReadRecordsAsync(filePath))
|
|
{
|
|
var processed = await ProcessRecord(record);
|
|
results.Add(processed);
|
|
|
|
// Automatically checkpoints every √n iterations
|
|
if (checkpoint.ShouldCheckpoint())
|
|
{
|
|
await checkpoint.SaveStateAsync(results);
|
|
}
|
|
}
|
|
|
|
return new ImportResult(results);
|
|
}
|
|
```
|
|
|
|
### 15. Roslyn Analyzers
|
|
|
|
Get compile-time suggestions for memory optimizations:
|
|
|
|
```csharp
|
|
// Analyzer warning: ST001 - Large allocation detected
|
|
var allOrders = await dbContext.Orders.ToListAsync(); // Warning
|
|
|
|
// Quick fix applied:
|
|
var allOrders = await dbContext.Orders.ToListWithSqrtNMemoryAsync(); // Fixed
|
|
|
|
// Analyzer warning: ST002 - Inefficient LINQ operation
|
|
var sorted = items.OrderBy(x => x.Id).ToList(); // Warning
|
|
|
|
// Quick fix applied:
|
|
var sorted = items.OrderByExternal(x => x.Id).ToList(); // Fixed
|
|
```
|
|
|
|
## Real-World Performance
|
|
|
|
Benchmarks on .NET 8.0:
|
|
|
|
| Operation | Standard | SpaceTime | Memory Reduction | Time Overhead |
|
|
|-----------|----------|-----------|------------------|---------------|
|
|
| Sort 10M items | 800MB, 1.2s | 25MB, 1.8s | **97%** | 50% |
|
|
| LINQ GroupBy 1M | 120MB, 0.8s | 3.5MB, 1.1s | **97%** | 38% |
|
|
| EF Core Query 100K | 200MB, 2.1s | 14MB, 2.4s | **93%** | 14% |
|
|
| Stream 1GB JSON | 1GB, 5s | 32MB, 5.5s | **97%** | 10% |
|
|
| Cache 1M items | 400MB | 35MB hot + disk | **91%** | 5% |
|
|
| Distributed sort | N/A | 50MB per node | **95%** | 20% |
|
|
|
|
## When to Use
|
|
|
|
### Perfect for:
|
|
- Large dataset processing (> 100K items)
|
|
- Memory-constrained environments (containers, serverless)
|
|
- Reducing cloud costs (smaller instances)
|
|
- Import/export operations
|
|
- Batch processing
|
|
- Real-time systems with predictable memory
|
|
- Distributed data processing
|
|
- Long-running operations requiring fault tolerance
|
|
|
|
### Not ideal for:
|
|
- Small datasets (< 1000 items)
|
|
- Ultra-low latency requirements (< 10ms)
|
|
- Simple CRUD operations
|
|
- CPU-bound calculations without memory pressure
|
|
|
|
## Configuration
|
|
|
|
### Global Configuration
|
|
|
|
```csharp
|
|
// In Program.cs
|
|
services.Configure<SpaceTimeConfiguration>(config =>
|
|
{
|
|
// Memory settings
|
|
config.Memory.MaxMemory = 1_073_741_824; // 1GB
|
|
config.Memory.BufferSizeStrategy = BufferSizeStrategy.Sqrt;
|
|
|
|
// Algorithm selection
|
|
config.Algorithms.EnableAdaptiveSelection = true;
|
|
config.Algorithms.MinExternalAlgorithmSize = 10_000_000; // 10MB
|
|
|
|
// Performance tuning
|
|
config.Performance.EnableParallelism = true;
|
|
config.Performance.MaxDegreeOfParallelism = Environment.ProcessorCount;
|
|
|
|
// Storage settings
|
|
config.Storage.DefaultStorageDirectory = "/tmp/spacetime";
|
|
config.Storage.EnableCompression = true;
|
|
|
|
// Features
|
|
config.Features.EnableCheckpointing = true;
|
|
config.Features.EnableAdaptiveDataStructures = true;
|
|
});
|
|
```
|
|
|
|
### Environment Variables
|
|
|
|
Configure via environment variables:
|
|
|
|
```bash
|
|
# Memory settings
|
|
SPACETIME_MAX_MEMORY=1073741824
|
|
SPACETIME_MEMORY_THRESHOLD=0.7
|
|
|
|
# Performance settings
|
|
SPACETIME_ENABLE_PARALLEL=true
|
|
SPACETIME_MAX_PARALLELISM=8
|
|
|
|
# Storage settings
|
|
SPACETIME_STORAGE_DIR=/tmp/spacetime
|
|
SPACETIME_ENABLE_COMPRESSION=true
|
|
```
|
|
|
|
### Per-Operation Configuration
|
|
|
|
```csharp
|
|
// Custom buffer size
|
|
var sorted = data.OrderByExternal(x => x.Id, bufferSize: 10000);
|
|
|
|
// Custom checkpoint interval
|
|
var checkpoint = new CheckpointManager(strategy: CheckpointStrategy.Linear);
|
|
|
|
// Force specific implementation
|
|
var list = new AdaptiveList<Order>(strategy: AdaptiveStrategy.ForceExternal);
|
|
|
|
// Configure pipeline
|
|
var pipeline = builder.Configure(config =>
|
|
{
|
|
config.ExpectedItemCount = 1_000_000;
|
|
config.EnableCheckpointing = true;
|
|
config.DefaultTimeout = TimeSpan.FromMinutes(30);
|
|
});
|
|
```
|
|
|
|
## How It Works
|
|
|
|
Based on Williams' theoretical result that TIME[t] ⊆ SPACE[√(t log t)]:
|
|
|
|
1. **Memory Reduction**: Use O(√n) memory instead of O(n)
|
|
2. **External Storage**: Spill to disk when memory limit reached
|
|
3. **Optimal Chunking**: Process data in √n-sized chunks
|
|
4. **Adaptive Strategies**: Switch algorithms based on data size
|
|
5. **Distributed Coordination**: Split work across nodes
|
|
6. **Memory Pressure Handling**: Automatic response to low memory
|
|
|
|
## Examples
|
|
|
|
### Processing Large CSV
|
|
|
|
```csharp
|
|
[HttpPost("import-csv")]
|
|
[EnableCheckpoint]
|
|
public async Task<IActionResult> ImportCsv(IFormFile file)
|
|
{
|
|
var pipeline = _pipelineFactory.CreatePipeline<string, Record>("CsvImport")
|
|
.AddTransform("Parse", line => ParseCsvLine(line))
|
|
.AddBatch("Validate", async batch => await ValidateRecords(batch))
|
|
.AddCheckpoint("Progress")
|
|
.AddTransform("Save", async record => await SaveRecord(record))
|
|
.Build();
|
|
|
|
var lines = ReadCsvLines(file.OpenReadStream());
|
|
var result = await pipeline.ExecuteAsync(lines);
|
|
|
|
return Ok(new { ProcessedCount = result.ProcessedCount });
|
|
}
|
|
```
|
|
|
|
### Optimized Data Export
|
|
|
|
```csharp
|
|
[HttpGet("export")]
|
|
[SpaceTimeStreaming]
|
|
public async IAsyncEnumerable<CustomerExport> ExportCustomers()
|
|
{
|
|
// Process customers in √n batches with progress
|
|
var totalCount = await dbContext.Customers.CountAsync();
|
|
var batchSize = SpaceTimeCalculator.CalculateSqrtInterval(totalCount);
|
|
|
|
await foreach (var batch in dbContext.Customers
|
|
.OrderBy(c => c.Id)
|
|
.BatchAsync(batchSize))
|
|
{
|
|
foreach (var customer in batch)
|
|
{
|
|
yield return new CustomerExport
|
|
{
|
|
Id = customer.Id,
|
|
Name = customer.Name,
|
|
TotalOrders = await GetOrderCount(customer.Id)
|
|
};
|
|
}
|
|
}
|
|
}
|
|
```
|
|
|
|
### Memory-Aware Background Job
|
|
|
|
```csharp
|
|
public class DataProcessingJob : IHostedService
|
|
{
|
|
private readonly ISpaceTimeTaskScheduler _scheduler;
|
|
private readonly IMemoryPressureMonitor _memoryMonitor;
|
|
|
|
public async Task ExecuteAsync(CancellationToken cancellationToken)
|
|
{
|
|
// Schedule based on memory availability
|
|
await _scheduler.ScheduleAsync(async () =>
|
|
{
|
|
if (_memoryMonitor.CurrentPressureLevel > MemoryPressureLevel.Medium)
|
|
{
|
|
// Use external algorithms
|
|
await ProcessDataExternal();
|
|
}
|
|
else
|
|
{
|
|
// Use in-memory algorithms
|
|
await ProcessDataInMemory();
|
|
}
|
|
},
|
|
estimatedMemory: 100 * 1024 * 1024, // 100MB
|
|
priority: TaskPriority.Low);
|
|
}
|
|
}
|
|
```
|
|
|
|
## Contributing
|
|
|
|
We welcome contributions! Please see our [Contributing Guide](CONTRIBUTING.md).
|
|
|
|
## License
|
|
|
|
Apache 2.0 - See [LICENSE](LICENSE) for details.
|
|
|
|
## Links
|
|
|
|
- [NuGet Packages](https://www.nuget.org/profiles/sqrtspace)
|
|
- [Git Repository](https://git.marketally.com/sqrtspace/sqrtspace-dotnet)
|
|
|
|
---
|
|
|
|
*Making theoretical computer science practical for .NET developers*
|