Initial commit - ViewEngine.Client library
This commit is contained in:
commit
6571247025
21
.gitignore
vendored
Normal file
21
.gitignore
vendored
Normal file
@ -0,0 +1,21 @@
|
||||
# Build results
|
||||
bin/
|
||||
obj/
|
||||
[Dd]ebug/
|
||||
[Rr]elease/
|
||||
|
||||
# NuGet packages
|
||||
*.nupkg
|
||||
*.snupkg
|
||||
|
||||
# Visual Studio
|
||||
.vs/
|
||||
*.user
|
||||
*.suo
|
||||
|
||||
# JetBrains Rider
|
||||
.idea/
|
||||
|
||||
# OS files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
42
Configuration/ViewEngineOptions.cs
Normal file
42
Configuration/ViewEngineOptions.cs
Normal file
@ -0,0 +1,42 @@
|
||||
namespace ViewEngine.Client.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for the ViewEngine client
|
||||
/// </summary>
|
||||
public class ViewEngineOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration section name
|
||||
/// </summary>
|
||||
public const string SectionName = "ViewEngine";
|
||||
|
||||
/// <summary>
|
||||
/// API key for authentication (required)
|
||||
/// </summary>
|
||||
public string ApiKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Base URL for the ViewEngine API (default: https://www.viewengine.io)
|
||||
/// </summary>
|
||||
public string BaseUrl { get; set; } = "https://www.viewengine.io";
|
||||
|
||||
/// <summary>
|
||||
/// HTTP timeout in seconds (default: 120)
|
||||
/// </summary>
|
||||
public int TimeoutSeconds { get; set; } = 120;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of retry attempts for failed requests (default: 3)
|
||||
/// </summary>
|
||||
public int MaxRetries { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Base delay in milliseconds for exponential backoff (default: 1000)
|
||||
/// </summary>
|
||||
public int BaseDelayMs { get; set; } = 1000;
|
||||
|
||||
/// <summary>
|
||||
/// Default polling interval in milliseconds when checking job status (default: 2000)
|
||||
/// </summary>
|
||||
public int DefaultPollingIntervalMs { get; set; } = 2000;
|
||||
}
|
||||
464
Examples.md
Normal file
464
Examples.md
Normal file
@ -0,0 +1,464 @@
|
||||
# ViewEngine.Client Usage Examples
|
||||
|
||||
## Table of Contents
|
||||
- [Basic Usage](#basic-usage)
|
||||
- [ASP.NET Core Integration](#aspnet-core-integration)
|
||||
- [Console Application](#console-application)
|
||||
- [Advanced Scenarios](#advanced-scenarios)
|
||||
|
||||
## Basic Usage
|
||||
|
||||
### Simple Web Page Retrieval
|
||||
|
||||
```csharp
|
||||
using ViewEngine.Client;
|
||||
using ViewEngine.Client.Models;
|
||||
|
||||
// Create client
|
||||
var client = new ViewEngineClient("ak_your-api-key-here");
|
||||
|
||||
// Retrieve a page
|
||||
var request = new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com"
|
||||
};
|
||||
|
||||
var pageData = await client.RetrieveAndWaitAsync(request);
|
||||
|
||||
Console.WriteLine($"Title: {pageData.Title}");
|
||||
Console.WriteLine($"Body: {pageData.Body}");
|
||||
Console.WriteLine($"Found {pageData.Routes.Count} navigation links");
|
||||
```
|
||||
|
||||
### Manual Polling
|
||||
|
||||
```csharp
|
||||
// Submit request
|
||||
var submitResponse = await client.SubmitRetrievalAsync(new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com",
|
||||
TimeoutSeconds = 60,
|
||||
Priority = 5
|
||||
});
|
||||
|
||||
Console.WriteLine($"Request ID: {submitResponse.RequestId}");
|
||||
Console.WriteLine($"Status: {submitResponse.Status}");
|
||||
|
||||
// Poll for completion
|
||||
while (true)
|
||||
{
|
||||
var status = await client.GetRetrievalStatusAsync(submitResponse.RequestId);
|
||||
|
||||
Console.WriteLine($"Current status: {status.Status}");
|
||||
|
||||
if (status.Status == "complete")
|
||||
{
|
||||
var content = await client.GetPageContentAsync(submitResponse.RequestId);
|
||||
Console.WriteLine($"Title: {content.Title}");
|
||||
break;
|
||||
}
|
||||
else if (status.Status == "failed")
|
||||
{
|
||||
Console.WriteLine($"Failed: {status.Error}");
|
||||
break;
|
||||
}
|
||||
|
||||
await Task.Delay(2000); // Poll every 2 seconds
|
||||
}
|
||||
```
|
||||
|
||||
## ASP.NET Core Integration
|
||||
|
||||
### Minimal API Example
|
||||
|
||||
**Program.cs:**
|
||||
```csharp
|
||||
using ViewEngine.Client;
|
||||
using ViewEngine.Client.Extensions;
|
||||
using ViewEngine.Client.Models;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
// Add ViewEngine client from configuration
|
||||
builder.Services.AddViewEngineClient(builder.Configuration);
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
// Simple endpoint
|
||||
app.MapGet("/retrieve/{url}", async (string url, ViewEngineClient client) =>
|
||||
{
|
||||
var request = new SubmitRetrievalRequest { Url = url };
|
||||
var pageData = await client.RetrieveAndWaitAsync(request);
|
||||
|
||||
return Results.Ok(new
|
||||
{
|
||||
pageData.Title,
|
||||
pageData.Url,
|
||||
LinkCount = pageData.Routes.Count + pageData.BodyRoutes.Count
|
||||
});
|
||||
});
|
||||
|
||||
// Endpoint with custom parameters
|
||||
app.MapPost("/retrieve", async (SubmitRetrievalRequest request, ViewEngineClient client) =>
|
||||
{
|
||||
var pageData = await client.RetrieveAndWaitAsync(request);
|
||||
return Results.Ok(pageData);
|
||||
});
|
||||
|
||||
app.Run();
|
||||
```
|
||||
|
||||
**appsettings.json:**
|
||||
```json
|
||||
{
|
||||
"ViewEngine": {
|
||||
"ApiKey": "ak_your-api-key-here",
|
||||
"BaseUrl": "https://www.viewengine.io/api/v1",
|
||||
"TimeoutSeconds": 120,
|
||||
"MaxRetries": 3,
|
||||
"DefaultPollingIntervalMs": 2000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Controller Example
|
||||
|
||||
```csharp
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using ViewEngine.Client;
|
||||
using ViewEngine.Client.Models;
|
||||
|
||||
[ApiController]
|
||||
[Route("api/[controller]")]
|
||||
public class WebPagesController : ControllerBase
|
||||
{
|
||||
private readonly ViewEngineClient _client;
|
||||
private readonly ILogger<WebPagesController> _logger;
|
||||
|
||||
public WebPagesController(ViewEngineClient client, ILogger<WebPagesController> logger)
|
||||
{
|
||||
_client = client;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
[HttpPost("retrieve")]
|
||||
public async Task<ActionResult<PageData>> RetrievePage([FromBody] SubmitRetrievalRequest request)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("Retrieving page: {Url}", request.Url);
|
||||
|
||||
var pageData = await _client.RetrieveAndWaitAsync(request);
|
||||
|
||||
return Ok(pageData);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to retrieve page: {Url}", request.Url);
|
||||
return StatusCode(500, new { error = ex.Message });
|
||||
}
|
||||
}
|
||||
|
||||
[HttpGet("status/{requestId}")]
|
||||
public async Task<ActionResult<RetrievalStatusResponse>> GetStatus(Guid requestId)
|
||||
{
|
||||
var status = await _client.GetRetrievalStatusAsync(requestId);
|
||||
return Ok(status);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Console Application
|
||||
|
||||
### Full Featured Console App
|
||||
|
||||
```csharp
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using ViewEngine.Client;
|
||||
using ViewEngine.Client.Models;
|
||||
|
||||
class Program
|
||||
{
|
||||
static async Task Main(string[] args)
|
||||
{
|
||||
// Get API key from environment variable or args
|
||||
var apiKey = Environment.GetEnvironmentVariable("VIEWENGINE_API_KEY")
|
||||
?? args.FirstOrDefault()
|
||||
?? throw new Exception("API key required");
|
||||
|
||||
using var client = new ViewEngineClient(apiKey);
|
||||
|
||||
Console.WriteLine("ViewEngine Client Demo");
|
||||
Console.WriteLine("======================");
|
||||
|
||||
while (true)
|
||||
{
|
||||
Console.Write("\nEnter URL to retrieve (or 'quit' to exit): ");
|
||||
var url = Console.ReadLine();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(url) || url.Equals("quit", StringComparison.OrdinalIgnoreCase))
|
||||
break;
|
||||
|
||||
await RetrievePageAsync(client, url);
|
||||
}
|
||||
}
|
||||
|
||||
static async Task RetrievePageAsync(ViewEngineClient client, string url)
|
||||
{
|
||||
try
|
||||
{
|
||||
Console.WriteLine($"\nRetrieving: {url}");
|
||||
|
||||
var request = new SubmitRetrievalRequest
|
||||
{
|
||||
Url = url,
|
||||
TimeoutSeconds = 60,
|
||||
Priority = 5
|
||||
};
|
||||
|
||||
var submitResponse = await client.SubmitRetrievalAsync(request);
|
||||
Console.WriteLine($"Request ID: {submitResponse.RequestId}");
|
||||
Console.WriteLine($"Status: {submitResponse.Status}");
|
||||
|
||||
// Poll for completion
|
||||
RetrievalStatusResponse status;
|
||||
do
|
||||
{
|
||||
await Task.Delay(2000);
|
||||
status = await client.GetRetrievalStatusAsync(submitResponse.RequestId);
|
||||
Console.WriteLine($"Status: {status.Status} - {status.Message}");
|
||||
}
|
||||
while (status.Status == "queued" || status.Status == "processing");
|
||||
|
||||
if (status.Status == "complete")
|
||||
{
|
||||
var pageData = await client.GetPageContentAsync(submitResponse.RequestId);
|
||||
|
||||
Console.WriteLine($"\n--- Page Data ---");
|
||||
Console.WriteLine($"Title: {pageData.Title}");
|
||||
Console.WriteLine($"URL: {pageData.Url}");
|
||||
Console.WriteLine($"Meta Description: {pageData.MetaDescription}");
|
||||
Console.WriteLine($"Body Length: {pageData.Body?.Length ?? 0} characters");
|
||||
|
||||
Console.WriteLine($"\n--- Navigation Links ({pageData.Routes.Count}) ---");
|
||||
foreach (var link in pageData.Routes.Take(5))
|
||||
{
|
||||
Console.WriteLine($" [{link.Rank}] {link.Text} -> {link.Url}");
|
||||
}
|
||||
|
||||
Console.WriteLine($"\n--- Body Links ({pageData.BodyRoutes.Count}) ---");
|
||||
foreach (var link in pageData.BodyRoutes.Take(5))
|
||||
{
|
||||
var adMarker = link.IsPotentialAd ? "[AD] " : "";
|
||||
Console.WriteLine($" {adMarker}[{link.Rank}] {link.Text} -> {link.Url}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"\nRetrieval failed: {status.Error}");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"\nError: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Advanced Scenarios
|
||||
|
||||
### Client Management
|
||||
|
||||
```csharp
|
||||
// Add a new client
|
||||
var addRequest = new AddClientRequest
|
||||
{
|
||||
Email = "feeder@example.com",
|
||||
Alias = "Production Feeder #1",
|
||||
CustomUserId = "prod-feeder-001",
|
||||
DailyMaximum = 1000
|
||||
};
|
||||
|
||||
var client = await viewEngineClient.AddClientAsync(addRequest);
|
||||
Console.WriteLine($"Added client: {client.Id}");
|
||||
|
||||
// Route job to this specific client
|
||||
var retrievalRequest = new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com",
|
||||
ClientId = client.Id // Use the client's ID
|
||||
};
|
||||
|
||||
// Or use custom user ID
|
||||
var retrievalRequest2 = new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com",
|
||||
CustomUserId = "prod-feeder-001"
|
||||
};
|
||||
|
||||
// List all clients
|
||||
var clients = await viewEngineClient.GetClientsAsync();
|
||||
foreach (var c in clients)
|
||||
{
|
||||
Console.WriteLine($"{c.Alias}: {(c.FeederOnline ? "Online" : "Offline")}");
|
||||
Console.WriteLine($" Jobs today: {c.JobsProcessedToday}/{c.DailyMaximum}");
|
||||
}
|
||||
|
||||
// Get client statistics
|
||||
var stats = await viewEngineClient.GetClientStatsAsync(client.Id);
|
||||
Console.WriteLine($"Total jobs: {stats.TotalJobsProcessed}");
|
||||
Console.WriteLine($"Success rate: {stats.SuccessRate:F2}%");
|
||||
Console.WriteLine($"Avg processing time: {stats.AverageProcessingTimeMs:F0}ms");
|
||||
|
||||
// Update client settings
|
||||
await viewEngineClient.UpdateClientAsync(client.Id, new UpdateClientRequest
|
||||
{
|
||||
DailyMaximum = 2000,
|
||||
Alias = "Production Feeder #1 (Updated)"
|
||||
});
|
||||
|
||||
// Suspend client
|
||||
await viewEngineClient.SuspendClientAsync(client.Id);
|
||||
|
||||
// Reactivate
|
||||
await viewEngineClient.ActivateClientAsync(client.Id);
|
||||
|
||||
// Delete client
|
||||
await viewEngineClient.DeleteClientAsync(client.Id);
|
||||
```
|
||||
|
||||
### Platform-Specific Retrieval
|
||||
|
||||
```csharp
|
||||
// Request page to be rendered on specific platform
|
||||
var request = new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com",
|
||||
PreferredPlatform = "Windows" // or "Android", "iOS"
|
||||
};
|
||||
|
||||
var pageData = await client.RetrieveAndWaitAsync(request);
|
||||
```
|
||||
|
||||
### High Priority Jobs
|
||||
|
||||
```csharp
|
||||
var request = new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com",
|
||||
Priority = 10, // Highest priority (1-10)
|
||||
TimeoutSeconds = 300 // Max 5 minutes
|
||||
};
|
||||
```
|
||||
|
||||
### Batch Processing
|
||||
|
||||
```csharp
|
||||
var urls = new[]
|
||||
{
|
||||
"https://example.com",
|
||||
"https://another-example.com",
|
||||
"https://third-example.com"
|
||||
};
|
||||
|
||||
// Submit all requests
|
||||
var submitTasks = urls.Select(url => client.SubmitRetrievalAsync(new SubmitRetrievalRequest
|
||||
{
|
||||
Url = url,
|
||||
Priority = 5
|
||||
}));
|
||||
|
||||
var submitResponses = await Task.WhenAll(submitTasks);
|
||||
|
||||
// Wait for all to complete
|
||||
var tasks = submitResponses.Select(async response =>
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var status = await client.GetRetrievalStatusAsync(response.RequestId);
|
||||
|
||||
if (status.Status == "complete")
|
||||
{
|
||||
return await client.GetPageContentAsync(response.RequestId);
|
||||
}
|
||||
else if (status.Status == "failed")
|
||||
{
|
||||
throw new Exception($"Failed to retrieve {status.Url}");
|
||||
}
|
||||
|
||||
await Task.Delay(2000);
|
||||
}
|
||||
});
|
||||
|
||||
var results = await Task.WhenAll(tasks);
|
||||
|
||||
foreach (var pageData in results)
|
||||
{
|
||||
Console.WriteLine($"{pageData.Title} - {pageData.Url}");
|
||||
}
|
||||
```
|
||||
|
||||
### Error Handling
|
||||
|
||||
```csharp
|
||||
try
|
||||
{
|
||||
var pageData = await client.RetrieveAndWaitAsync(request);
|
||||
}
|
||||
catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Unauthorized)
|
||||
{
|
||||
Console.WriteLine("Invalid API key");
|
||||
}
|
||||
catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
Console.WriteLine("Rate limit exceeded");
|
||||
}
|
||||
catch (InvalidOperationException ex) when (ex.Message.Contains("failed"))
|
||||
{
|
||||
Console.WriteLine($"Retrieval failed: {ex.Message}");
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
Console.WriteLine("Request was canceled");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Unexpected error: {ex.Message}");
|
||||
}
|
||||
```
|
||||
|
||||
### Custom Timeout and Retry Settings
|
||||
|
||||
```csharp
|
||||
var options = new ViewEngineOptions
|
||||
{
|
||||
ApiKey = "ak_your-api-key-here",
|
||||
TimeoutSeconds = 180, // 3 minutes
|
||||
MaxRetries = 5,
|
||||
BaseDelayMs = 500,
|
||||
DefaultPollingIntervalMs = 1000 // Poll every second
|
||||
};
|
||||
|
||||
using var client = new ViewEngineClient(options);
|
||||
```
|
||||
|
||||
### Dependency Injection with Multiple Clients
|
||||
|
||||
```csharp
|
||||
// Configure multiple named clients
|
||||
builder.Services.AddHttpClient<ViewEngineClient>("Primary")
|
||||
.ConfigureHttpClient((sp, client) =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://www.viewengine.io/api/v1");
|
||||
client.DefaultRequestHeaders.Add("X-API-Key", "ak_primary_key");
|
||||
});
|
||||
|
||||
builder.Services.AddHttpClient<ViewEngineClient>("Secondary")
|
||||
.ConfigureHttpClient((sp, client) =>
|
||||
{
|
||||
client.BaseAddress = new Uri("https://www.viewengine.io/api/v1");
|
||||
client.DefaultRequestHeaders.Add("X-API-Key", "ak_secondary_key");
|
||||
});
|
||||
```
|
||||
67
Extensions/ServiceCollectionExtensions.cs
Normal file
67
Extensions/ServiceCollectionExtensions.cs
Normal file
@ -0,0 +1,67 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ViewEngine.Client.Configuration;
|
||||
|
||||
namespace ViewEngine.Client.Extensions;
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for registering ViewEngine client services
|
||||
/// </summary>
|
||||
public static class ServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Adds ViewEngine client services to the dependency injection container
|
||||
/// </summary>
|
||||
/// <param name="services">The service collection</param>
|
||||
/// <param name="configuration">The configuration</param>
|
||||
/// <returns>The service collection for chaining</returns>
|
||||
public static IServiceCollection AddViewEngineClient(this IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
// Bind configuration
|
||||
services.Configure<ViewEngineOptions>(configuration.GetSection(ViewEngineOptions.SectionName));
|
||||
|
||||
// Register the client as a typed HttpClient
|
||||
services.AddHttpClient<ViewEngineClient>((serviceProvider, httpClient) =>
|
||||
{
|
||||
var options = configuration.GetSection(ViewEngineOptions.SectionName).Get<ViewEngineOptions>()
|
||||
?? throw new InvalidOperationException("ViewEngine configuration is missing");
|
||||
|
||||
if (string.IsNullOrWhiteSpace(options.ApiKey))
|
||||
throw new InvalidOperationException("ViewEngine API key is required");
|
||||
|
||||
httpClient.BaseAddress = new Uri(options.BaseUrl);
|
||||
httpClient.Timeout = TimeSpan.FromSeconds(options.TimeoutSeconds);
|
||||
httpClient.DefaultRequestHeaders.Add("X-API-Key", options.ApiKey);
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds ViewEngine client services to the dependency injection container with custom configuration
|
||||
/// </summary>
|
||||
/// <param name="services">The service collection</param>
|
||||
/// <param name="configureOptions">Action to configure options</param>
|
||||
/// <returns>The service collection for chaining</returns>
|
||||
public static IServiceCollection AddViewEngineClient(this IServiceCollection services, Action<ViewEngineOptions> configureOptions)
|
||||
{
|
||||
// Configure options
|
||||
services.Configure(configureOptions);
|
||||
|
||||
// Register the client as a typed HttpClient
|
||||
services.AddHttpClient<ViewEngineClient>((serviceProvider, httpClient) =>
|
||||
{
|
||||
var options = new ViewEngineOptions();
|
||||
configureOptions(options);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(options.ApiKey))
|
||||
throw new InvalidOperationException("ViewEngine API key is required");
|
||||
|
||||
httpClient.BaseAddress = new Uri(options.BaseUrl);
|
||||
httpClient.Timeout = TimeSpan.FromSeconds(options.TimeoutSeconds);
|
||||
httpClient.DefaultRequestHeaders.Add("X-API-Key", options.ApiKey);
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
160
Models/ClientManagementModels.cs
Normal file
160
Models/ClientManagementModels.cs
Normal file
@ -0,0 +1,160 @@
|
||||
namespace ViewEngine.Client.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Request to add a client programmatically
|
||||
/// </summary>
|
||||
public class AddClientRequest
|
||||
{
|
||||
/// <summary>
|
||||
/// Email address of the user to add as client
|
||||
/// </summary>
|
||||
public string Email { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Friendly name for this client (e.g., "John's Feeder")
|
||||
/// </summary>
|
||||
public string? Alias { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Your custom identifier for this client (alternative to using clientId)
|
||||
/// </summary>
|
||||
public string? CustomUserId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Max jobs per day for this client (0 = unlimited, default: 0)
|
||||
/// </summary>
|
||||
public int DailyMaximum { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Request to update client settings
|
||||
/// </summary>
|
||||
public class UpdateClientRequest
|
||||
{
|
||||
/// <summary>
|
||||
/// Friendly name for this client
|
||||
/// </summary>
|
||||
public string? Alias { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Your custom identifier for this client
|
||||
/// </summary>
|
||||
public string? CustomUserId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Max jobs per day for this client (0 = unlimited)
|
||||
/// </summary>
|
||||
public int? DailyMaximum { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Information about a client
|
||||
/// </summary>
|
||||
public class ClientInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// Client ID
|
||||
/// </summary>
|
||||
public Guid Id { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// User ID of the client
|
||||
/// </summary>
|
||||
public Guid UserId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Client's email address
|
||||
/// </summary>
|
||||
public string Email { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Client's display name
|
||||
/// </summary>
|
||||
public string? DisplayName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Friendly alias for the client
|
||||
/// </summary>
|
||||
public string? Alias { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Your custom user identifier
|
||||
/// </summary>
|
||||
public string? CustomUserId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Client status: Active, Suspended, etc.
|
||||
/// </summary>
|
||||
public string Status { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum jobs per day
|
||||
/// </summary>
|
||||
public int DailyMaximum { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the client was added
|
||||
/// </summary>
|
||||
public DateTime CreatedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the client was last active
|
||||
/// </summary>
|
||||
public DateTime? LastActiveAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the client's feeder is currently online
|
||||
/// </summary>
|
||||
public bool FeederOnline { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of jobs processed today
|
||||
/// </summary>
|
||||
public int JobsProcessedToday { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Statistics for a client
|
||||
/// </summary>
|
||||
public class ClientStats
|
||||
{
|
||||
/// <summary>
|
||||
/// Client ID
|
||||
/// </summary>
|
||||
public Guid ClientId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Total jobs processed all time
|
||||
/// </summary>
|
||||
public int TotalJobsProcessed { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Jobs processed today
|
||||
/// </summary>
|
||||
public int JobsProcessedToday { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Jobs processed this month
|
||||
/// </summary>
|
||||
public int JobsProcessedThisMonth { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Average processing time in milliseconds
|
||||
/// </summary>
|
||||
public double AverageProcessingTimeMs { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Success rate (0-100)
|
||||
/// </summary>
|
||||
public double SuccessRate { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the feeder was last online
|
||||
/// </summary>
|
||||
public DateTime? LastOnlineAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Current feeder status
|
||||
/// </summary>
|
||||
public bool IsOnline { get; set; }
|
||||
}
|
||||
312
Models/RetrievalModels.cs
Normal file
312
Models/RetrievalModels.cs
Normal file
@ -0,0 +1,312 @@
|
||||
namespace ViewEngine.Client.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Request to submit a web page retrieval job
|
||||
/// </summary>
|
||||
public class SubmitRetrievalRequest
|
||||
{
|
||||
/// <summary>
|
||||
/// The URL of the web page to retrieve (scheme optional, will default to https://)
|
||||
/// </summary>
|
||||
public string Url { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum time to wait in seconds (default: 60, max: 300)
|
||||
/// </summary>
|
||||
public int TimeoutSeconds { get; set; } = 60;
|
||||
|
||||
/// <summary>
|
||||
/// Force fresh retrieval, bypassing cache (default: false)
|
||||
/// </summary>
|
||||
public bool ForceRefresh { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of feeders that must agree (Community mode only, 1-10)
|
||||
/// </summary>
|
||||
public int? RequiredQuorum { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Job priority (1=lowest, 10=highest, default: 5). Higher priority costs more.
|
||||
/// </summary>
|
||||
public int Priority { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Route job to specific client's feeder (requires Client Management setup)
|
||||
/// </summary>
|
||||
public Guid? ClientId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Route job using your custom client identifier (alternative to clientId)
|
||||
/// </summary>
|
||||
public string? CustomUserId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Restrict processing to a specific platform: "Android", "iOS", or "Windows" (Community mode only)
|
||||
/// </summary>
|
||||
public string? PreferredPlatform { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// If true, generate an AI summary of the page content (default: false)
|
||||
/// </summary>
|
||||
public bool GenerateSummary { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Target language code for translation (ISO 639-1, e.g., "en", "es", "de", "zh").
|
||||
/// If specified and the page is in a different language, content will be translated.
|
||||
/// Null = no translation, return content as-is.
|
||||
/// </summary>
|
||||
public string? TargetLanguage { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Response from submitting a retrieval request
|
||||
/// </summary>
|
||||
public class RetrievalResponse
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique identifier for this retrieval request
|
||||
/// </summary>
|
||||
public Guid RequestId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Current status of the request
|
||||
/// </summary>
|
||||
public string Status { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// User-friendly message describing the status
|
||||
/// </summary>
|
||||
public string Message { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Estimated wait time in seconds
|
||||
/// </summary>
|
||||
public int? EstimatedWaitTimeSeconds { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Status response for a retrieval request
|
||||
/// </summary>
|
||||
public class RetrievalStatusResponse
|
||||
{
|
||||
/// <summary>
|
||||
/// The request ID
|
||||
/// </summary>
|
||||
public Guid RequestId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The URL being retrieved
|
||||
/// </summary>
|
||||
public string Url { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Current status: queued, processing, validating, complete, failed, canceled
|
||||
/// </summary>
|
||||
public string Status { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// User-friendly status message
|
||||
/// </summary>
|
||||
public string Message { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Content information (available when status is complete)
|
||||
/// </summary>
|
||||
public ContentInfo? Content { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Error message if status is failed
|
||||
/// </summary>
|
||||
public string? Error { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the request was created
|
||||
/// </summary>
|
||||
public DateTime CreatedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the request was completed
|
||||
/// </summary>
|
||||
public DateTime? CompletedAt { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Content information for a completed retrieval
|
||||
/// </summary>
|
||||
public class ContentInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// URL to download the decrypted page content
|
||||
/// </summary>
|
||||
public string PageDataUrl { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Hash of the retrieved content
|
||||
/// </summary>
|
||||
public string? ContentHash { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Artifacts such as screenshots and thumbnails
|
||||
/// </summary>
|
||||
public ArtifactsInfo? Artifacts { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Performance and quality metrics
|
||||
/// </summary>
|
||||
public Dictionary<string, object>? Metrics { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Artifacts from page retrieval
|
||||
/// </summary>
|
||||
public class ArtifactsInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// URL to full page screenshot
|
||||
/// </summary>
|
||||
public string? Screenshot { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// URL to page thumbnail
|
||||
/// </summary>
|
||||
public string? Thumbnail { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Page content data returned from the API
|
||||
/// </summary>
|
||||
public class PageData
|
||||
{
|
||||
/// <summary>
|
||||
/// Page title
|
||||
/// </summary>
|
||||
public string Title { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Full text content of the page
|
||||
/// </summary>
|
||||
public string Body { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Meta description tag content
|
||||
/// </summary>
|
||||
public string? MetaDescription { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Final URL after any redirects
|
||||
/// </summary>
|
||||
public string Url { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// URL to the page's favicon
|
||||
/// </summary>
|
||||
public string? FaviconUrl { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Base64-encoded PNG screenshot of the page (typically 320x180)
|
||||
/// </summary>
|
||||
public string? Thumbnail { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Links found in navigation/header areas
|
||||
/// </summary>
|
||||
public List<LinkInfo> Routes { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Links found in page body content
|
||||
/// </summary>
|
||||
public List<LinkInfo> BodyRoutes { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Optional AI-generated summary of the page content
|
||||
/// </summary>
|
||||
public string? Summary { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// HTTP status code from the page retrieval.
|
||||
/// Null if status code is not available.
|
||||
/// Common values: 200 (OK), 404 (Not Found), 500 (Server Error).
|
||||
/// </summary>
|
||||
public int? HttpStatusCode { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if the HTTP status code indicates success (2xx) or is not available.
|
||||
/// </summary>
|
||||
public bool IsSuccess => HttpStatusCode == null || (HttpStatusCode >= 200 && HttpStatusCode < 300);
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if the HTTP status code indicates a client error (4xx).
|
||||
/// </summary>
|
||||
public bool IsClientError => HttpStatusCode >= 400 && HttpStatusCode < 500;
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if the HTTP status code indicates a server error (5xx).
|
||||
/// </summary>
|
||||
public bool IsServerError => HttpStatusCode >= 500;
|
||||
|
||||
/// <summary>
|
||||
/// Language of the page as declared in the HTML lang attribute (e.g., "en", "es", "zh-CN").
|
||||
/// Null if not specified in the HTML.
|
||||
/// </summary>
|
||||
public string? HtmlLang { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Detected/confirmed language of the content (ISO 639-1 code, e.g., "en", "es", "zh").
|
||||
/// May differ from HtmlLang if automatic language detection was performed.
|
||||
/// </summary>
|
||||
public string? DetectedLanguage { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// If content was translated, this is the target language code.
|
||||
/// Null if no translation was performed.
|
||||
/// </summary>
|
||||
public string? TranslatedTo { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Original body text before translation.
|
||||
/// Only populated if translation occurred.
|
||||
/// </summary>
|
||||
public string? OriginalBody { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Original title before translation.
|
||||
/// Only populated if translation occurred.
|
||||
/// </summary>
|
||||
public string? OriginalTitle { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Information about a link on the page
|
||||
/// </summary>
|
||||
public class LinkInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// The link URL
|
||||
/// </summary>
|
||||
public string Url { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The anchor text (what the link says)
|
||||
/// </summary>
|
||||
public string Text { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Importance ranking (higher = more important, typically 1-10)
|
||||
/// </summary>
|
||||
public int Rank { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// How many times this link appears on the page
|
||||
/// </summary>
|
||||
public int Occurrences { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this link is likely an advertisement
|
||||
/// </summary>
|
||||
public bool IsPotentialAd { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Explanation of why the link was flagged as an ad (null if not an ad)
|
||||
/// </summary>
|
||||
public string? AdReason { get; set; }
|
||||
}
|
||||
244
PROJECT_SUMMARY.md
Normal file
244
PROJECT_SUMMARY.md
Normal file
@ -0,0 +1,244 @@
|
||||
# ViewEngine.Client - Project Summary
|
||||
|
||||
## Overview
|
||||
|
||||
The **ViewEngine.Client** is a production-ready .NET 9 NuGet package that provides a clean, type-safe client library for consuming the ViewEngine REST API. It enables developers to easily retrieve web pages, extract content, and manage distributed web scraping operations.
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
ViewEngine.Client/
|
||||
├── Configuration/
|
||||
│ └── ViewEngineOptions.cs # Configuration options for the client
|
||||
├── Extensions/
|
||||
│ └── ServiceCollectionExtensions.cs # Dependency injection helpers
|
||||
├── Models/
|
||||
│ ├── ClientManagementModels.cs # DTOs for client management API
|
||||
│ └── RetrievalModels.cs # DTOs for retrieval API
|
||||
├── ViewEngineClient.cs # Main client implementation
|
||||
├── ViewEngine.Client.csproj # Project file with NuGet metadata
|
||||
├── README.md # Comprehensive documentation
|
||||
├── QUICKSTART.md # Quick reference guide
|
||||
├── Examples.md # Detailed usage examples
|
||||
└── PROJECT_SUMMARY.md # This file
|
||||
```
|
||||
|
||||
## Key Features
|
||||
|
||||
### Core Functionality
|
||||
- ✅ Submit web page retrieval requests via MCP API
|
||||
- ✅ Poll for job status and retrieve results
|
||||
- ✅ Download decrypted page content with full text extraction
|
||||
- ✅ Access screenshots, thumbnails, and artifacts
|
||||
- ✅ Manage clients and route jobs to specific feeders
|
||||
- ✅ Built-in retry logic with exponential backoff
|
||||
- ✅ Full support for API key authentication
|
||||
- ✅ Type-safe request/response models
|
||||
- ✅ Async/await support throughout
|
||||
- ✅ Dependency injection integration
|
||||
|
||||
### Advanced Features
|
||||
- ✅ Rate limit handling with retry-after support
|
||||
- ✅ Exponential backoff for transient failures
|
||||
- ✅ Server error retry logic
|
||||
- ✅ Platform-specific retrieval (Android, iOS, Windows)
|
||||
- ✅ Priority-based job scheduling
|
||||
- ✅ Client routing with ClientId or CustomUserId
|
||||
- ✅ Force refresh option to bypass cache
|
||||
- ✅ Configurable timeouts and polling intervals
|
||||
|
||||
### Client Management
|
||||
- ✅ Add clients programmatically
|
||||
- ✅ List and filter clients
|
||||
- ✅ Update client settings
|
||||
- ✅ Suspend/activate clients
|
||||
- ✅ Delete clients
|
||||
- ✅ Get detailed client statistics
|
||||
- ✅ Monitor feeder online status
|
||||
- ✅ Track job processing metrics
|
||||
|
||||
## API Coverage
|
||||
|
||||
### MCP Retrieval API (/mcp/*)
|
||||
- `POST /mcp/retrieve` - Submit retrieval request
|
||||
- `GET /mcp/retrieve/{requestId}` - Get retrieval status
|
||||
- `GET /mcp/retrieve/{requestId}/content` - Download page content
|
||||
- `GET /mcp/tools` - List available tools
|
||||
|
||||
### Client Management API (/v1/clients/*)
|
||||
- `POST /v1/clients` - Add client
|
||||
- `GET /v1/clients` - List clients
|
||||
- `PUT /v1/clients/{id}` - Update client
|
||||
- `PUT /v1/clients/{id}/suspend` - Suspend client
|
||||
- `PUT /v1/clients/{id}/activate` - Activate client
|
||||
- `DELETE /v1/clients/{id}` - Delete client
|
||||
- `GET /v1/clients/{id}/stats` - Get client stats
|
||||
|
||||
## NuGet Package Details
|
||||
|
||||
### Package Metadata
|
||||
- **Package ID**: `ViewEngine.Client`
|
||||
- **Version**: 1.0.0
|
||||
- **Target Framework**: .NET 9.0
|
||||
- **Authors**: David H Friedel Jr
|
||||
- **License**: MIT
|
||||
- **Output Path**: `C:\Users\logik\Dropbox\Nugets\ViewEngine.Client.1.0.0.nupkg`
|
||||
|
||||
### Dependencies
|
||||
- `Microsoft.Extensions.Http` (9.0.0)
|
||||
- `Microsoft.Extensions.Options` (9.0.0)
|
||||
- `Microsoft.Extensions.Options.ConfigurationExtensions` (9.0.0)
|
||||
|
||||
### Package Contents
|
||||
- Compiled DLL with XML documentation
|
||||
- README.md for NuGet.org package page
|
||||
- Full IntelliSense support
|
||||
- Source link support (future)
|
||||
|
||||
## Usage Patterns
|
||||
|
||||
### 1. Simple Usage (No DI)
|
||||
```csharp
|
||||
var client = new ViewEngineClient("ak_your-api-key-here");
|
||||
var page = await client.RetrieveAndWaitAsync(new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com"
|
||||
});
|
||||
```
|
||||
|
||||
### 2. Dependency Injection (Recommended)
|
||||
```csharp
|
||||
// Program.cs
|
||||
builder.Services.AddViewEngineClient(builder.Configuration);
|
||||
|
||||
// Controller
|
||||
public class MyController : ControllerBase
|
||||
{
|
||||
private readonly ViewEngineClient _client;
|
||||
public MyController(ViewEngineClient client) => _client = client;
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Options Pattern
|
||||
```csharp
|
||||
var options = new ViewEngineOptions
|
||||
{
|
||||
ApiKey = "ak_your-api-key-here",
|
||||
TimeoutSeconds = 120,
|
||||
MaxRetries = 3
|
||||
};
|
||||
var client = new ViewEngineClient(options);
|
||||
```
|
||||
|
||||
## Architecture Decisions
|
||||
|
||||
### 1. Multiple Constructors
|
||||
- Simple constructor with API key only
|
||||
- IOptions constructor for DI
|
||||
- HttpClient constructor for advanced DI scenarios
|
||||
- Flexible and supports various usage patterns
|
||||
|
||||
### 2. Retry Logic
|
||||
- Automatic retry for rate limits (429)
|
||||
- Automatic retry for server errors (500-599)
|
||||
- Automatic retry for network timeouts
|
||||
- Respects Retry-After headers
|
||||
- Exponential backoff with configurable base delay
|
||||
|
||||
### 3. Error Handling
|
||||
- Throws HttpRequestException for HTTP errors
|
||||
- Throws InvalidOperationException for failed retrievals
|
||||
- Throws OperationCanceledException for canceled requests
|
||||
- All errors include meaningful messages
|
||||
|
||||
### 4. Resource Management
|
||||
- Implements IDisposable
|
||||
- Properly disposes HttpClient when not injected
|
||||
- Does not dispose injected HttpClient (DI container manages it)
|
||||
|
||||
### 5. Type Safety
|
||||
- Strongly-typed DTOs for all requests/responses
|
||||
- Nullable reference types enabled
|
||||
- XML documentation for all public APIs
|
||||
- IntelliSense support
|
||||
|
||||
## Testing Recommendations
|
||||
|
||||
### Unit Testing
|
||||
```csharp
|
||||
// Mock HttpClient for unit tests
|
||||
var mockHandler = new MockHttpMessageHandler();
|
||||
var httpClient = new HttpClient(mockHandler);
|
||||
var options = Options.Create(new ViewEngineOptions { ApiKey = "test" });
|
||||
var client = new ViewEngineClient(httpClient, options);
|
||||
```
|
||||
|
||||
### Integration Testing
|
||||
```csharp
|
||||
// Use real API with test API key
|
||||
var client = new ViewEngineClient("ak_test_key");
|
||||
var page = await client.RetrieveAndWaitAsync(new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com"
|
||||
});
|
||||
Assert.NotNull(page.Title);
|
||||
```
|
||||
|
||||
## Build Information
|
||||
|
||||
- **Status**: ✅ Build succeeded (0 warnings, 0 errors)
|
||||
- **Package Created**: ✅ ViewEngine.Client.1.0.0.nupkg (30,151 bytes)
|
||||
- **Documentation**: ✅ XML documentation generated
|
||||
- **Target Framework**: .NET 9.0
|
||||
- **Language Version**: Latest
|
||||
|
||||
## Next Steps
|
||||
|
||||
### For Developers Using This Package
|
||||
1. Install via NuGet: `dotnet add package ViewEngine.Client`
|
||||
2. Get API key from https://www.viewengine.io
|
||||
3. Follow QUICKSTART.md for immediate use
|
||||
4. See Examples.md for advanced scenarios
|
||||
|
||||
### For Package Maintainers
|
||||
1. ✅ Initial release complete
|
||||
2. 🔄 Publish to NuGet.org (manual step)
|
||||
3. 🔄 Add source link support
|
||||
4. 🔄 Add unit tests project
|
||||
5. 🔄 Add integration tests
|
||||
6. 🔄 Set up CI/CD pipeline
|
||||
7. 🔄 Add code coverage reporting
|
||||
|
||||
### Future Enhancements
|
||||
- [ ] Add webhook support for job completion notifications
|
||||
- [ ] Add bulk retrieval helper methods
|
||||
- [ ] Add caching layer for frequently accessed pages
|
||||
- [ ] Add telemetry and diagnostics
|
||||
- [ ] Add support for .NET Standard 2.0 (broader compatibility)
|
||||
- [ ] Add rate limit awareness (track limits client-side)
|
||||
- [ ] Add health check endpoint support
|
||||
- [ ] Add billing/usage API support
|
||||
|
||||
## Documentation
|
||||
|
||||
- **README.md**: Main documentation with installation, configuration, and usage
|
||||
- **QUICKSTART.md**: Quick reference for common tasks
|
||||
- **Examples.md**: Comprehensive examples for all scenarios
|
||||
- **XML Docs**: IntelliSense documentation in code
|
||||
- **API Docs**: https://www.viewengine.io/docs
|
||||
|
||||
## Support
|
||||
|
||||
- GitHub: https://github.com/marketally/viewengine
|
||||
- Issues: https://github.com/marketally/viewengine/issues
|
||||
- Website: https://www.viewengine.io
|
||||
|
||||
## License
|
||||
|
||||
MIT License - Copyright © 2025 ViewEngine
|
||||
|
||||
---
|
||||
|
||||
**Package Created**: November 22, 2025
|
||||
**Build Status**: ✅ Ready for distribution
|
||||
**Quality**: Production-ready
|
||||
192
QUICKSTART.md
Normal file
192
QUICKSTART.md
Normal file
@ -0,0 +1,192 @@
|
||||
# ViewEngine.Client - Quick Start Guide
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
dotnet add package ViewEngine.Client
|
||||
```
|
||||
|
||||
## 30-Second Quick Start
|
||||
|
||||
```csharp
|
||||
using ViewEngine.Client;
|
||||
using ViewEngine.Client.Models;
|
||||
|
||||
var client = new ViewEngineClient("ak_your-api-key-here");
|
||||
|
||||
var pageData = await client.RetrieveAndWaitAsync(new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com"
|
||||
});
|
||||
|
||||
Console.WriteLine($"Title: {pageData.Title}");
|
||||
Console.WriteLine($"Content: {pageData.Body}");
|
||||
```
|
||||
|
||||
## Common Use Cases
|
||||
|
||||
### 1. Simple Page Retrieval
|
||||
```csharp
|
||||
var client = new ViewEngineClient("ak_your-api-key-here");
|
||||
var page = await client.RetrieveAndWaitAsync(new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com"
|
||||
});
|
||||
```
|
||||
|
||||
### 2. With ASP.NET Core DI
|
||||
|
||||
**appsettings.json:**
|
||||
```json
|
||||
{
|
||||
"ViewEngine": {
|
||||
"ApiKey": "ak_your-api-key-here"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Program.cs:**
|
||||
```csharp
|
||||
builder.Services.AddViewEngineClient(builder.Configuration);
|
||||
```
|
||||
|
||||
**Usage:**
|
||||
```csharp
|
||||
public class MyController : ControllerBase
|
||||
{
|
||||
private readonly ViewEngineClient _client;
|
||||
|
||||
public MyController(ViewEngineClient client) => _client = client;
|
||||
|
||||
[HttpGet]
|
||||
public async Task<IActionResult> GetPage(string url)
|
||||
{
|
||||
var page = await _client.RetrieveAndWaitAsync(new SubmitRetrievalRequest { Url = url });
|
||||
return Ok(page);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Custom Platform Selection
|
||||
```csharp
|
||||
var page = await client.RetrieveAndWaitAsync(new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com",
|
||||
PreferredPlatform = "Windows" // or "Android", "iOS"
|
||||
});
|
||||
```
|
||||
|
||||
### 4. High Priority Job
|
||||
```csharp
|
||||
var page = await client.RetrieveAndWaitAsync(new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com",
|
||||
Priority = 10 // 1-10, higher = faster (costs more)
|
||||
});
|
||||
```
|
||||
|
||||
### 5. Manual Polling
|
||||
```csharp
|
||||
var response = await client.SubmitRetrievalAsync(new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com"
|
||||
});
|
||||
|
||||
RetrievalStatusResponse status;
|
||||
do
|
||||
{
|
||||
await Task.Delay(2000);
|
||||
status = await client.GetRetrievalStatusAsync(response.RequestId);
|
||||
} while (status.Status != "complete" && status.Status != "failed");
|
||||
|
||||
if (status.Status == "complete")
|
||||
{
|
||||
var page = await client.GetPageContentAsync(response.RequestId);
|
||||
}
|
||||
```
|
||||
|
||||
## Key Features
|
||||
|
||||
| Feature | Method |
|
||||
|---------|--------|
|
||||
| Submit retrieval | `SubmitRetrievalAsync()` |
|
||||
| Check status | `GetRetrievalStatusAsync()` |
|
||||
| Get content | `GetPageContentAsync()` |
|
||||
| Submit & wait | `RetrieveAndWaitAsync()` |
|
||||
| Add client | `AddClientAsync()` |
|
||||
| List clients | `GetClientsAsync()` |
|
||||
| Client stats | `GetClientStatsAsync()` |
|
||||
|
||||
## Page Data Structure
|
||||
|
||||
```csharp
|
||||
public class PageData
|
||||
{
|
||||
string Title // Page title
|
||||
string Body // Full text content
|
||||
string MetaDescription // Meta description
|
||||
string Url // Final URL after redirects
|
||||
string FaviconUrl // Favicon URL
|
||||
string Thumbnail // Base64 PNG screenshot
|
||||
List<LinkInfo> Routes // Navigation links
|
||||
List<LinkInfo> BodyRoutes // Content links
|
||||
}
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
```csharp
|
||||
try
|
||||
{
|
||||
var page = await client.RetrieveAndWaitAsync(request);
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
// Network/HTTP errors
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
// Retrieval failed
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Request canceled
|
||||
}
|
||||
```
|
||||
|
||||
## Configuration Options
|
||||
|
||||
```csharp
|
||||
var options = new ViewEngineOptions
|
||||
{
|
||||
ApiKey = "ak_your-api-key-here",
|
||||
BaseUrl = "https://www.viewengine.io/api/v1",
|
||||
TimeoutSeconds = 120,
|
||||
MaxRetries = 3,
|
||||
DefaultPollingIntervalMs = 2000
|
||||
};
|
||||
|
||||
var client = new ViewEngineClient(options);
|
||||
```
|
||||
|
||||
## Request Options
|
||||
|
||||
```csharp
|
||||
var request = new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com",
|
||||
TimeoutSeconds = 60, // Max: 300
|
||||
ForceRefresh = false, // Bypass cache
|
||||
Priority = 5, // 1-10
|
||||
PreferredPlatform = "Windows", // Android, iOS, Windows
|
||||
ClientId = clientGuid, // Route to specific client
|
||||
CustomUserId = "custom-id" // Or use custom ID
|
||||
};
|
||||
```
|
||||
|
||||
## Need More Help?
|
||||
|
||||
- Full Documentation: `README.md`
|
||||
- Code Examples: `Examples.md`
|
||||
- API Docs: https://www.viewengine.io/docs
|
||||
- GitHub: https://github.com/marketally/viewengine
|
||||
300
README.md
Normal file
300
README.md
Normal file
@ -0,0 +1,300 @@
|
||||
# ViewEngine.Client
|
||||
|
||||
Official .NET client library for consuming the ViewEngine REST API. Retrieve web pages, extract content, and process web data with ViewEngine's distributed web scraping service.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
dotnet add package ViewEngine.Client
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```csharp
|
||||
using ViewEngine.Client;
|
||||
using ViewEngine.Client.Models;
|
||||
|
||||
// Create client with API key
|
||||
var client = new ViewEngineClient("ak_your-api-key-here");
|
||||
|
||||
// Submit a retrieval request
|
||||
var request = new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com",
|
||||
TimeoutSeconds = 60,
|
||||
Priority = 5,
|
||||
GenerateSummary = true // Optional: Generate AI summary
|
||||
};
|
||||
|
||||
// Submit and wait for completion
|
||||
var pageData = await client.RetrieveAndWaitAsync(request);
|
||||
|
||||
Console.WriteLine($"Title: {pageData.Title}");
|
||||
Console.WriteLine($"Summary: {pageData.Summary}"); // AI-generated summary if requested
|
||||
Console.WriteLine($"Body: {pageData.Body}");
|
||||
Console.WriteLine($"Links found: {pageData.Routes.Count}");
|
||||
```
|
||||
|
||||
### Advanced Usage with Polling
|
||||
|
||||
```csharp
|
||||
// Submit request
|
||||
var submitResponse = await client.SubmitRetrievalAsync(request);
|
||||
Console.WriteLine($"Request ID: {submitResponse.RequestId}");
|
||||
|
||||
// Poll for status
|
||||
RetrievalStatusResponse status;
|
||||
do
|
||||
{
|
||||
status = await client.GetRetrievalStatusAsync(submitResponse.RequestId);
|
||||
Console.WriteLine($"Status: {status.Status}");
|
||||
|
||||
if (status.Status == "complete")
|
||||
{
|
||||
// Download content
|
||||
var content = await client.GetPageContentAsync(submitResponse.RequestId);
|
||||
Console.WriteLine($"Retrieved: {content.Title}");
|
||||
break;
|
||||
}
|
||||
|
||||
await Task.Delay(2000); // Wait 2 seconds
|
||||
} while (status.Status == "queued" || status.Status == "processing");
|
||||
```
|
||||
|
||||
## Dependency Injection
|
||||
|
||||
### ASP.NET Core / Minimal API
|
||||
|
||||
**appsettings.json:**
|
||||
```json
|
||||
{
|
||||
"ViewEngine": {
|
||||
"ApiKey": "ak_your-api-key-here",
|
||||
"BaseUrl": "https://www.viewengine.io/api/v1",
|
||||
"TimeoutSeconds": 120,
|
||||
"MaxRetries": 3
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Program.cs:**
|
||||
```csharp
|
||||
using ViewEngine.Client.Extensions;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
// Add ViewEngine client
|
||||
builder.Services.AddViewEngineClient(builder.Configuration);
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
app.MapGet("/retrieve", async (ViewEngineClient client) =>
|
||||
{
|
||||
var request = new SubmitRetrievalRequest { Url = "https://example.com" };
|
||||
var pageData = await client.RetrieveAndWaitAsync(request);
|
||||
return Results.Ok(pageData);
|
||||
});
|
||||
|
||||
app.Run();
|
||||
```
|
||||
|
||||
### With Options Pattern
|
||||
|
||||
```csharp
|
||||
using ViewEngine.Client.Extensions;
|
||||
|
||||
builder.Services.AddViewEngineClient(options =>
|
||||
{
|
||||
options.ApiKey = "ak_your-api-key-here";
|
||||
options.BaseUrl = "https://www.viewengine.io/api/v1";
|
||||
options.TimeoutSeconds = 120;
|
||||
options.MaxRetries = 3;
|
||||
});
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
### Web Page Retrieval
|
||||
|
||||
```csharp
|
||||
var request = new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com",
|
||||
TimeoutSeconds = 60,
|
||||
ForceRefresh = true,
|
||||
Priority = 8,
|
||||
PreferredPlatform = "Windows", // Android, iOS, or Windows
|
||||
GenerateSummary = true // Generate AI summary of page content
|
||||
};
|
||||
|
||||
var pageData = await client.RetrieveAndWaitAsync(request);
|
||||
|
||||
// Access extracted data
|
||||
Console.WriteLine($"Title: {pageData.Title}");
|
||||
Console.WriteLine($"Summary: {pageData.Summary}"); // AI-generated summary
|
||||
Console.WriteLine($"Description: {pageData.MetaDescription}");
|
||||
Console.WriteLine($"Body Text: {pageData.Body}");
|
||||
Console.WriteLine($"Favicon: {pageData.FaviconUrl}");
|
||||
|
||||
// Navigation links
|
||||
foreach (var link in pageData.Routes)
|
||||
{
|
||||
Console.WriteLine($"{link.Text} -> {link.Url}");
|
||||
}
|
||||
|
||||
// Body links with ad detection
|
||||
foreach (var link in pageData.BodyRoutes)
|
||||
{
|
||||
if (link.IsPotentialAd)
|
||||
Console.WriteLine($"[AD] {link.Text} ({link.AdReason})");
|
||||
else
|
||||
Console.WriteLine($"{link.Text} -> {link.Url}");
|
||||
}
|
||||
```
|
||||
|
||||
### Client Management
|
||||
|
||||
```csharp
|
||||
// Add a client
|
||||
var addRequest = new AddClientRequest
|
||||
{
|
||||
Email = "user@example.com",
|
||||
Alias = "Production Feeder",
|
||||
CustomUserId = "prod-001",
|
||||
DailyMaximum = 1000
|
||||
};
|
||||
|
||||
var newClient = await client.AddClientAsync(addRequest);
|
||||
|
||||
// List all clients
|
||||
var clients = await client.GetClientsAsync();
|
||||
foreach (var c in clients)
|
||||
{
|
||||
Console.WriteLine($"{c.Alias}: {c.FeederOnline ? "Online" : "Offline"}");
|
||||
}
|
||||
|
||||
// Route job to specific client
|
||||
var routedRequest = new SubmitRetrievalRequest
|
||||
{
|
||||
Url = "https://example.com",
|
||||
ClientId = newClient.Id // or CustomUserId = "prod-001"
|
||||
};
|
||||
|
||||
// Get client stats
|
||||
var stats = await client.GetClientStatsAsync(newClient.Id);
|
||||
Console.WriteLine($"Total jobs: {stats.TotalJobsProcessed}");
|
||||
Console.WriteLine($"Success rate: {stats.SuccessRate}%");
|
||||
|
||||
// Suspend/Activate/Delete
|
||||
await client.SuspendClientAsync(newClient.Id);
|
||||
await client.ActivateClientAsync(newClient.Id);
|
||||
await client.DeleteClientAsync(newClient.Id);
|
||||
```
|
||||
|
||||
### Error Handling
|
||||
|
||||
```csharp
|
||||
try
|
||||
{
|
||||
var pageData = await client.RetrieveAndWaitAsync(request);
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
// HTTP errors (network, server errors)
|
||||
Console.WriteLine($"HTTP Error: {ex.Message}");
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
// Failed retrieval or invalid responses
|
||||
Console.WriteLine($"Operation Error: {ex.Message}");
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Request was canceled
|
||||
Console.WriteLine("Request canceled");
|
||||
}
|
||||
```
|
||||
|
||||
## Configuration Options
|
||||
|
||||
| Option | Default | Description |
|
||||
|--------|---------|-------------|
|
||||
| `ApiKey` | (required) | Your ViewEngine API key |
|
||||
| `BaseUrl` | `https://www.viewengine.io/api/v1` | API base URL |
|
||||
| `TimeoutSeconds` | `120` | HTTP request timeout |
|
||||
| `MaxRetries` | `3` | Maximum retry attempts |
|
||||
| `BaseDelayMs` | `1000` | Base delay for exponential backoff |
|
||||
| `DefaultPollingIntervalMs` | `2000` | Default polling interval for status checks |
|
||||
|
||||
## Request Options
|
||||
|
||||
### SubmitRetrievalRequest
|
||||
|
||||
| Property | Type | Default | Description |
|
||||
|----------|------|---------|-------------|
|
||||
| `Url` | `string` | (required) | URL to retrieve |
|
||||
| `TimeoutSeconds` | `int` | `60` | Max wait time (max: 300) |
|
||||
| `ForceRefresh` | `bool` | `false` | Bypass cache |
|
||||
| `RequiredQuorum` | `int?` | `null` | Feeders that must agree (1-10, Community mode) |
|
||||
| `Priority` | `int` | `5` | Job priority (1-10) |
|
||||
| `ClientId` | `Guid?` | `null` | Route to specific client |
|
||||
| `CustomUserId` | `string?` | `null` | Route using custom ID |
|
||||
| `PreferredPlatform` | `string?` | `null` | "Android", "iOS", or "Windows" |
|
||||
| `GenerateSummary` | `bool` | `false` | Generate AI summary of page content |
|
||||
|
||||
## Response Models
|
||||
|
||||
### PageData
|
||||
|
||||
```csharp
|
||||
public class PageData
|
||||
{
|
||||
public string Title { get; set; }
|
||||
public string Body { get; set; }
|
||||
public string? MetaDescription { get; set; }
|
||||
public string Url { get; set; }
|
||||
public string? FaviconUrl { get; set; }
|
||||
public string? Thumbnail { get; set; } // Base64 PNG
|
||||
public string? Summary { get; set; } // AI-generated summary (if requested)
|
||||
public List<LinkInfo> Routes { get; set; }
|
||||
public List<LinkInfo> BodyRoutes { get; set; }
|
||||
}
|
||||
```
|
||||
|
||||
### LinkInfo
|
||||
|
||||
```csharp
|
||||
public class LinkInfo
|
||||
{
|
||||
public string Url { get; set; }
|
||||
public string Text { get; set; }
|
||||
public int Rank { get; set; }
|
||||
public int Occurrences { get; set; }
|
||||
public bool IsPotentialAd { get; set; }
|
||||
public string? AdReason { get; set; }
|
||||
}
|
||||
```
|
||||
|
||||
## Retry & Rate Limiting
|
||||
|
||||
The client automatically handles:
|
||||
- **Rate limits (429)**: Respects `Retry-After` header with exponential backoff
|
||||
- **Server errors (500-5xx)**: Retries with exponential backoff
|
||||
- **Network timeouts**: Retries with exponential backoff
|
||||
- **Configurable retries**: Set `MaxRetries` in options
|
||||
|
||||
## API Documentation
|
||||
|
||||
For complete API documentation, visit: https://www.viewengine.io/docs
|
||||
|
||||
## Support
|
||||
|
||||
- Documentation: https://www.viewengine.io/docs
|
||||
- GitHub: https://github.com/marketally/viewengine
|
||||
- Issues: https://github.com/marketally/viewengine/issues
|
||||
|
||||
## License
|
||||
|
||||
MIT License - Copyright © 2025 ViewEngine
|
||||
93
ViewEngine.Client.csproj
Executable file
93
ViewEngine.Client.csproj
Executable file
@ -0,0 +1,93 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net9.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<LangVersion>latest</LangVersion>
|
||||
|
||||
<PackageOutputPath>C:\Users\logik\Dropbox\Nugets</PackageOutputPath>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
|
||||
<PackageId>ViewEngine.Client</PackageId>
|
||||
<Version>1.2.1</Version>
|
||||
<Authors>David H Friedel Jr</Authors>
|
||||
<Company>MarketAlly</Company>
|
||||
<Product>ViewEngine.Client</Product>
|
||||
|
||||
<Title>ViewEngine REST API Client for .NET</Title>
|
||||
<Description>
|
||||
Official .NET client library for consuming the ViewEngine REST API. Retrieve web pages, extract content, and process web data with ViewEngine's distributed web scraping service.
|
||||
|
||||
Key Features:
|
||||
- Submit web page retrieval requests via MCP API
|
||||
- Poll for job status and retrieve results
|
||||
- Download decrypted page content with full text extraction
|
||||
- Access screenshots, thumbnails, and artifacts
|
||||
- Manage clients and route jobs to specific feeders
|
||||
- Built-in retry logic with exponential backoff
|
||||
- Full support for API key authentication
|
||||
- Type-safe request/response models
|
||||
- Async/await support throughout
|
||||
- Dependency injection integration
|
||||
- .NET 9.0 compatible
|
||||
|
||||
Perfect for applications requiring web scraping, content extraction, web page monitoring, and automated browsing.
|
||||
</Description>
|
||||
|
||||
<Copyright>Copyright © 2025 MarketAlly</Copyright>
|
||||
<PackageIcon>icon.png</PackageIcon>
|
||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
||||
<PackageProjectUrl>https://www.viewengine.io</PackageProjectUrl>
|
||||
<RepositoryUrl>https://git.marketally.com/viewengine/ViewEngine.Client</RepositoryUrl>
|
||||
<RepositoryType>git</RepositoryType>
|
||||
|
||||
<PackageTags>viewengine dotnet rest-api api-client web-scraping webscraping webcrawler http-client content-extraction page-retrieval mcp distributed async dotnet9</PackageTags>
|
||||
|
||||
<PackageReleaseNotes>
|
||||
Version 1.0.0 (Initial Release):
|
||||
|
||||
🎉 NEW FEATURES:
|
||||
- ViewEngineClient - Main client for MCP API operations
|
||||
- Submit retrieval requests with customizable options (timeout, priority, platform preferences)
|
||||
- Poll for job status and completion
|
||||
- Download decrypted page content
|
||||
- Client management API support
|
||||
- Type-safe DTOs for all API operations
|
||||
- Built-in retry logic with exponential backoff
|
||||
- Rate limit handling
|
||||
- API key authentication
|
||||
- Full async/await support
|
||||
- Dependency injection ready
|
||||
|
||||
📦 ARCHITECTURE:
|
||||
- Clean separation of concerns
|
||||
- Strongly-typed request/response models
|
||||
- Configurable via IOptions pattern
|
||||
- HttpClient integration
|
||||
- Production-ready error handling
|
||||
</PackageReleaseNotes>
|
||||
|
||||
<GenerateDocumentationFile>true</GenerateDocumentationFile>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.10" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="9.0.10" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="9.0.10" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="icon.png">
|
||||
<Pack>true</Pack>
|
||||
<PackagePath>\</PackagePath>
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
<Visible>true</Visible>
|
||||
</None>
|
||||
<None Include="README.md" Pack="true" PackagePath="\" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
457
ViewEngineClient.cs
Normal file
457
ViewEngineClient.cs
Normal file
@ -0,0 +1,457 @@
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ViewEngine.Client.Configuration;
|
||||
using ViewEngine.Client.Models;
|
||||
|
||||
namespace ViewEngine.Client;
|
||||
|
||||
/// <summary>
|
||||
/// Client for interacting with the ViewEngine REST API
|
||||
/// </summary>
|
||||
public class ViewEngineClient : IDisposable
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly ViewEngineOptions _options;
|
||||
private readonly bool _disposeHttpClient;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new ViewEngineClient with the specified API key
|
||||
/// </summary>
|
||||
/// <param name="apiKey">Your ViewEngine API key</param>
|
||||
/// <param name="baseUrl">Optional base URL (defaults to https://www.viewengine.io)</param>
|
||||
public ViewEngineClient(string apiKey, string? baseUrl = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(apiKey))
|
||||
throw new ArgumentNullException(nameof(apiKey));
|
||||
|
||||
_options = new ViewEngineOptions
|
||||
{
|
||||
ApiKey = apiKey,
|
||||
BaseUrl = baseUrl ?? "https://www.viewengine.io"
|
||||
};
|
||||
|
||||
_httpClient = new HttpClient
|
||||
{
|
||||
BaseAddress = new Uri(_options.BaseUrl),
|
||||
Timeout = TimeSpan.FromSeconds(_options.TimeoutSeconds)
|
||||
};
|
||||
|
||||
ConfigureHttpClient();
|
||||
_disposeHttpClient = true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new ViewEngineClient with the specified options
|
||||
/// </summary>
|
||||
/// <param name="options">Configuration options</param>
|
||||
public ViewEngineClient(IOptions<ViewEngineOptions> options) : this(options.Value)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new ViewEngineClient with the specified options
|
||||
/// </summary>
|
||||
/// <param name="options">Configuration options</param>
|
||||
public ViewEngineClient(ViewEngineOptions options)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_options.ApiKey))
|
||||
throw new ArgumentException("API key is required", nameof(options));
|
||||
|
||||
_httpClient = new HttpClient
|
||||
{
|
||||
BaseAddress = new Uri(_options.BaseUrl),
|
||||
Timeout = TimeSpan.FromSeconds(_options.TimeoutSeconds)
|
||||
};
|
||||
|
||||
ConfigureHttpClient();
|
||||
_disposeHttpClient = true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new ViewEngineClient with a custom HttpClient (for dependency injection)
|
||||
/// </summary>
|
||||
/// <param name="httpClient">Configured HttpClient instance</param>
|
||||
/// <param name="options">Configuration options</param>
|
||||
public ViewEngineClient(HttpClient httpClient, IOptions<ViewEngineOptions> options)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_options.ApiKey))
|
||||
throw new ArgumentException("API key is required", nameof(options));
|
||||
|
||||
_httpClient.BaseAddress = new Uri(_options.BaseUrl);
|
||||
_httpClient.Timeout = TimeSpan.FromSeconds(_options.TimeoutSeconds);
|
||||
|
||||
ConfigureHttpClient();
|
||||
_disposeHttpClient = false; // Don't dispose injected HttpClient
|
||||
}
|
||||
|
||||
private void ConfigureHttpClient()
|
||||
{
|
||||
_httpClient.DefaultRequestHeaders.Clear();
|
||||
_httpClient.DefaultRequestHeaders.Add("X-API-Key", _options.ApiKey);
|
||||
_httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
|
||||
}
|
||||
|
||||
#region MCP Retrieval API
|
||||
|
||||
/// <summary>
|
||||
/// Submit a web page retrieval request
|
||||
/// </summary>
|
||||
/// <param name="request">Retrieval request parameters</param>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
/// <returns>Retrieval response with request ID</returns>
|
||||
public async Task<RetrievalResponse> SubmitRetrievalAsync(SubmitRetrievalRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (request == null)
|
||||
throw new ArgumentNullException(nameof(request));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Url))
|
||||
throw new ArgumentException("URL is required", nameof(request));
|
||||
|
||||
var response = await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
return await _httpClient.PostAsJsonAsync("/mcp/retrieve", request, cancellationToken);
|
||||
}, "SubmitRetrieval", cancellationToken);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<RetrievalResponse>(cancellationToken: cancellationToken);
|
||||
return result ?? throw new InvalidOperationException("Failed to deserialize retrieval response");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get the status of a retrieval request
|
||||
/// </summary>
|
||||
/// <param name="requestId">The request ID returned from SubmitRetrievalAsync</param>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
/// <returns>Current status of the retrieval request</returns>
|
||||
public async Task<RetrievalStatusResponse> GetRetrievalStatusAsync(Guid requestId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (requestId == Guid.Empty)
|
||||
throw new ArgumentException("Invalid request ID", nameof(requestId));
|
||||
|
||||
var response = await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
return await _httpClient.GetAsync($"/mcp/retrieve/{requestId}", cancellationToken);
|
||||
}, "GetRetrievalStatus", cancellationToken);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<RetrievalStatusResponse>(cancellationToken: cancellationToken);
|
||||
return result ?? throw new InvalidOperationException("Failed to deserialize status response");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Download the decrypted page content for a completed retrieval
|
||||
/// </summary>
|
||||
/// <param name="requestId">The request ID of the completed retrieval</param>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
/// <returns>Page content data</returns>
|
||||
public async Task<PageData> GetPageContentAsync(Guid requestId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (requestId == Guid.Empty)
|
||||
throw new ArgumentException("Invalid request ID", nameof(requestId));
|
||||
|
||||
var response = await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
return await _httpClient.GetAsync($"/mcp/retrieve/{requestId}/content", cancellationToken);
|
||||
}, "GetPageContent", cancellationToken);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<PageData>(cancellationToken: cancellationToken);
|
||||
return result ?? throw new InvalidOperationException("Failed to deserialize page content");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Submit a retrieval request and wait for completion
|
||||
/// </summary>
|
||||
/// <param name="request">Retrieval request parameters</param>
|
||||
/// <param name="pollingIntervalMs">How often to check status in milliseconds (default: 2000)</param>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
/// <returns>Page content data when complete</returns>
|
||||
public async Task<PageData> RetrieveAndWaitAsync(SubmitRetrievalRequest request, int? pollingIntervalMs = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Submit the request
|
||||
var submitResponse = await SubmitRetrievalAsync(request, cancellationToken);
|
||||
|
||||
// Poll for completion
|
||||
var interval = pollingIntervalMs ?? _options.DefaultPollingIntervalMs;
|
||||
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var status = await GetRetrievalStatusAsync(submitResponse.RequestId, cancellationToken);
|
||||
|
||||
if (status.Status.Equals("complete", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// Download and return the content
|
||||
return await GetPageContentAsync(submitResponse.RequestId, cancellationToken);
|
||||
}
|
||||
else if (status.Status.Equals("failed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"Retrieval failed: {status.Error ?? status.Message}");
|
||||
}
|
||||
else if (status.Status.Equals("canceled", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException("Retrieval was canceled");
|
||||
}
|
||||
|
||||
// Wait before polling again
|
||||
await Task.Delay(interval, cancellationToken);
|
||||
}
|
||||
|
||||
throw new OperationCanceledException("Operation was canceled", cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// List available MCP tools
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
/// <returns>List of available tools</returns>
|
||||
public async Task<object> ListToolsAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var response = await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
return await _httpClient.GetAsync("/mcp/tools", cancellationToken);
|
||||
}, "ListTools", cancellationToken);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<object>(cancellationToken: cancellationToken);
|
||||
return result ?? throw new InvalidOperationException("Failed to deserialize tools response");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Client Management API
|
||||
|
||||
/// <summary>
|
||||
/// Add a new client programmatically
|
||||
/// </summary>
|
||||
/// <param name="request">Client details</param>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
/// <returns>Created client information</returns>
|
||||
public async Task<ClientInfo> AddClientAsync(AddClientRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (request == null)
|
||||
throw new ArgumentNullException(nameof(request));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Email))
|
||||
throw new ArgumentException("Email is required", nameof(request));
|
||||
|
||||
var response = await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
return await _httpClient.PostAsJsonAsync("/v1/clients", request, cancellationToken);
|
||||
}, "AddClient", cancellationToken);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<ClientInfo>(cancellationToken: cancellationToken);
|
||||
return result ?? throw new InvalidOperationException("Failed to deserialize client response");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get a list of all clients
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
/// <returns>List of clients</returns>
|
||||
public async Task<List<ClientInfo>> GetClientsAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var response = await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
return await _httpClient.GetAsync("/v1/clients", cancellationToken);
|
||||
}, "GetClients", cancellationToken);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<List<ClientInfo>>(cancellationToken: cancellationToken);
|
||||
return result ?? new List<ClientInfo>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Update client settings
|
||||
/// </summary>
|
||||
/// <param name="clientId">Client ID</param>
|
||||
/// <param name="request">Updated settings</param>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
/// <returns>Updated client information</returns>
|
||||
public async Task<ClientInfo> UpdateClientAsync(Guid clientId, UpdateClientRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (clientId == Guid.Empty)
|
||||
throw new ArgumentException("Invalid client ID", nameof(clientId));
|
||||
|
||||
if (request == null)
|
||||
throw new ArgumentNullException(nameof(request));
|
||||
|
||||
var response = await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
return await _httpClient.PutAsJsonAsync($"/v1/clients/{clientId}", request, cancellationToken);
|
||||
}, "UpdateClient", cancellationToken);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<ClientInfo>(cancellationToken: cancellationToken);
|
||||
return result ?? throw new InvalidOperationException("Failed to deserialize client response");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Suspend a client
|
||||
/// </summary>
|
||||
/// <param name="clientId">Client ID</param>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
public async Task SuspendClientAsync(Guid clientId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (clientId == Guid.Empty)
|
||||
throw new ArgumentException("Invalid client ID", nameof(clientId));
|
||||
|
||||
var response = await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
return await _httpClient.PutAsync($"/v1/clients/{clientId}/suspend", null, cancellationToken);
|
||||
}, "SuspendClient", cancellationToken);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reactivate a suspended client
|
||||
/// </summary>
|
||||
/// <param name="clientId">Client ID</param>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
public async Task ActivateClientAsync(Guid clientId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (clientId == Guid.Empty)
|
||||
throw new ArgumentException("Invalid client ID", nameof(clientId));
|
||||
|
||||
var response = await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
return await _httpClient.PutAsync($"/v1/clients/{clientId}/activate", null, cancellationToken);
|
||||
}, "ActivateClient", cancellationToken);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove a client
|
||||
/// </summary>
|
||||
/// <param name="clientId">Client ID</param>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
public async Task DeleteClientAsync(Guid clientId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (clientId == Guid.Empty)
|
||||
throw new ArgumentException("Invalid client ID", nameof(clientId));
|
||||
|
||||
var response = await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
return await _httpClient.DeleteAsync($"/v1/clients/{clientId}", cancellationToken);
|
||||
}, "DeleteClient", cancellationToken);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get detailed statistics for a client
|
||||
/// </summary>
|
||||
/// <param name="clientId">Client ID</param>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
/// <returns>Client statistics</returns>
|
||||
public async Task<ClientStats> GetClientStatsAsync(Guid clientId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (clientId == Guid.Empty)
|
||||
throw new ArgumentException("Invalid client ID", nameof(clientId));
|
||||
|
||||
var response = await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
return await _httpClient.GetAsync($"/v1/clients/{clientId}/stats", cancellationToken);
|
||||
}, "GetClientStats", cancellationToken);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<ClientStats>(cancellationToken: cancellationToken);
|
||||
return result ?? throw new InvalidOperationException("Failed to deserialize stats response");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Retry Logic
|
||||
|
||||
private async Task<HttpResponseMessage> ExecuteWithRetryAsync(
|
||||
Func<Task<HttpResponseMessage>> action,
|
||||
string operationName,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
int attempt = 0;
|
||||
|
||||
while (attempt < _options.MaxRetries)
|
||||
{
|
||||
try
|
||||
{
|
||||
attempt++;
|
||||
var response = await action();
|
||||
|
||||
// Handle rate limiting (429) with exponential backoff
|
||||
if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
if (attempt >= _options.MaxRetries)
|
||||
{
|
||||
throw new HttpRequestException($"Rate limit exceeded for {operationName} after {attempt} attempts");
|
||||
}
|
||||
|
||||
// Check for Retry-After header
|
||||
var retryAfter = response.Headers.RetryAfter?.Delta?.TotalMilliseconds
|
||||
?? (_options.BaseDelayMs * Math.Pow(2, attempt - 1));
|
||||
|
||||
await Task.Delay((int)retryAfter, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Retry on server errors (500-599)
|
||||
if ((int)response.StatusCode >= 500 && (int)response.StatusCode < 600)
|
||||
{
|
||||
if (attempt >= _options.MaxRetries)
|
||||
{
|
||||
throw new HttpRequestException($"Server error for {operationName}: {response.StatusCode} after {attempt} attempts");
|
||||
}
|
||||
|
||||
var delay = _options.BaseDelayMs * Math.Pow(2, attempt - 1);
|
||||
await Task.Delay((int)delay, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
catch (TaskCanceledException) when (attempt < _options.MaxRetries && !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var delay = _options.BaseDelayMs * Math.Pow(2, attempt - 1);
|
||||
await Task.Delay((int)delay, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
catch (HttpRequestException) when (attempt < _options.MaxRetries)
|
||||
{
|
||||
var delay = _options.BaseDelayMs * Math.Pow(2, attempt - 1);
|
||||
await Task.Delay((int)delay, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Exception($"Failed after {_options.MaxRetries} attempts for {operationName}");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the ViewEngineClient and underlying HttpClient if not injected
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposeHttpClient)
|
||||
{
|
||||
_httpClient?.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user