viewengine/viewengine.client
2
0
Fork 0
Code Issues Packages Releases Wiki

Initial commit - ViewEngine.Client library

Browse Source
This commit is contained in:
David H. Friedel Jr.
2025-12-28 18:15:17 +00:00
commit 6571247025
12 changed files with 2352 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
.gitignore vendored Normal file
View File

@@ -0,0 +1,21 @@
# Build results
bin/
obj/
[Dd]ebug/
[Rr]elease/
# NuGet packages
*.nupkg
*.snupkg
# Visual Studio
.vs/
*.user
*.suo
# JetBrains Rider
.idea/
# OS files
.DS_Store
Thumbs.db

42
Configuration/ViewEngineOptions.cs Normal file
View File

@@ -0,0 +1,42 @@
namespace ViewEngine.Client.Configuration;
/// <summary>
/// Configuration options for the ViewEngine client
/// </summary>
public class ViewEngineOptions
{
/// <summary>
/// Configuration section name
/// </summary>
public const string SectionName = "ViewEngine";
/// <summary>
/// API key for authentication (required)
/// </summary>
public string ApiKey { get; set; } = string.Empty;
/// <summary>
/// Base URL for the ViewEngine API (default: https://www.viewengine.io)
/// </summary>
public string BaseUrl { get; set; } = "https://www.viewengine.io";
/// <summary>
/// HTTP timeout in seconds (default: 120)
/// </summary>
public int TimeoutSeconds { get; set; } = 120;
/// <summary>
/// Maximum number of retry attempts for failed requests (default: 3)
/// </summary>
public int MaxRetries { get; set; } = 3;
/// <summary>
/// Base delay in milliseconds for exponential backoff (default: 1000)
/// </summary>
public int BaseDelayMs { get; set; } = 1000;
/// <summary>
/// Default polling interval in milliseconds when checking job status (default: 2000)
/// </summary>
public int DefaultPollingIntervalMs { get; set; } = 2000;
}

464
Examples.md Normal file
View File

@@ -0,0 +1,464 @@
# ViewEngine.Client Usage Examples
## Table of Contents
- [Basic Usage](#basic-usage)
- [ASP.NET Core Integration](#aspnet-core-integration)
- [Console Application](#console-application)
- [Advanced Scenarios](#advanced-scenarios)
## Basic Usage
### Simple Web Page Retrieval
```csharp
using ViewEngine.Client;
using ViewEngine.Client.Models;
// Create client
var client = new ViewEngineClient("ak_your-api-key-here");
// Retrieve a page
var request = new SubmitRetrievalRequest
{
Url = "https://example.com"
};
var pageData = await client.RetrieveAndWaitAsync(request);
Console.WriteLine($"Title: {pageData.Title}");
Console.WriteLine($"Body: {pageData.Body}");
Console.WriteLine($"Found {pageData.Routes.Count} navigation links");
```
### Manual Polling
```csharp
// Submit request
var submitResponse = await client.SubmitRetrievalAsync(new SubmitRetrievalRequest
{
Url = "https://example.com",
TimeoutSeconds = 60,
Priority = 5
});
Console.WriteLine($"Request ID: {submitResponse.RequestId}");
Console.WriteLine($"Status: {submitResponse.Status}");
// Poll for completion
while (true)
{
var status = await client.GetRetrievalStatusAsync(submitResponse.RequestId);
Console.WriteLine($"Current status: {status.Status}");
if (status.Status == "complete")
{
var content = await client.GetPageContentAsync(submitResponse.RequestId);
Console.WriteLine($"Title: {content.Title}");
break;
}
else if (status.Status == "failed")
{
Console.WriteLine($"Failed: {status.Error}");
break;
}
await Task.Delay(2000); // Poll every 2 seconds
}
```
## ASP.NET Core Integration
### Minimal API Example
**Program.cs:**
```csharp
using ViewEngine.Client;
using ViewEngine.Client.Extensions;
using ViewEngine.Client.Models;
var builder = WebApplication.CreateBuilder(args);
// Add ViewEngine client from configuration
builder.Services.AddViewEngineClient(builder.Configuration);
var app = builder.Build();
// Simple endpoint
app.MapGet("/retrieve/{url}", async (string url, ViewEngineClient client) =>
{
var request = new SubmitRetrievalRequest { Url = url };
var pageData = await client.RetrieveAndWaitAsync(request);
return Results.Ok(new
{
pageData.Title,
pageData.Url,
LinkCount = pageData.Routes.Count + pageData.BodyRoutes.Count
});
});
// Endpoint with custom parameters
app.MapPost("/retrieve", async (SubmitRetrievalRequest request, ViewEngineClient client) =>
{
var pageData = await client.RetrieveAndWaitAsync(request);
return Results.Ok(pageData);
});
app.Run();
```
**appsettings.json:**
```json
{
"ViewEngine": {
"ApiKey": "ak_your-api-key-here",
"BaseUrl": "https://www.viewengine.io/api/v1",
"TimeoutSeconds": 120,
"MaxRetries": 3,
"DefaultPollingIntervalMs": 2000
}
}
```
### Controller Example
```csharp
using Microsoft.AspNetCore.Mvc;
using ViewEngine.Client;
using ViewEngine.Client.Models;
[ApiController]
[Route("api/[controller]")]
public class WebPagesController : ControllerBase
{
private readonly ViewEngineClient _client;
private readonly ILogger<WebPagesController> _logger;
public WebPagesController(ViewEngineClient client, ILogger<WebPagesController> logger)
{
_client = client;
_logger = logger;
}
[HttpPost("retrieve")]
public async Task<ActionResult<PageData>> RetrievePage([FromBody] SubmitRetrievalRequest request)
{
try
{
_logger.LogInformation("Retrieving page: {Url}", request.Url);
var pageData = await _client.RetrieveAndWaitAsync(request);
return Ok(pageData);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to retrieve page: {Url}", request.Url);
return StatusCode(500, new { error = ex.Message });
}
}
[HttpGet("status/{requestId}")]
public async Task<ActionResult<RetrievalStatusResponse>> GetStatus(Guid requestId)
{
var status = await _client.GetRetrievalStatusAsync(requestId);
return Ok(status);
}
}
```
## Console Application
### Full Featured Console App
```csharp
using System;
using System.Threading.Tasks;
using ViewEngine.Client;
using ViewEngine.Client.Models;
class Program
{
static async Task Main(string[] args)
{
// Get API key from environment variable or args
var apiKey = Environment.GetEnvironmentVariable("VIEWENGINE_API_KEY")
?? args.FirstOrDefault()
?? throw new Exception("API key required");
using var client = new ViewEngineClient(apiKey);
Console.WriteLine("ViewEngine Client Demo");
Console.WriteLine("======================");
while (true)
{
Console.Write("\nEnter URL to retrieve (or 'quit' to exit): ");
var url = Console.ReadLine();
if (string.IsNullOrWhiteSpace(url) || url.Equals("quit", StringComparison.OrdinalIgnoreCase))
break;
await RetrievePageAsync(client, url);
}
}
static async Task RetrievePageAsync(ViewEngineClient client, string url)
{
try
{
Console.WriteLine($"\nRetrieving: {url}");
var request = new SubmitRetrievalRequest
{
Url = url,
TimeoutSeconds = 60,
Priority = 5
};
var submitResponse = await client.SubmitRetrievalAsync(request);
Console.WriteLine($"Request ID: {submitResponse.RequestId}");
Console.WriteLine($"Status: {submitResponse.Status}");
// Poll for completion
RetrievalStatusResponse status;
do
{
await Task.Delay(2000);
status = await client.GetRetrievalStatusAsync(submitResponse.RequestId);
Console.WriteLine($"Status: {status.Status} - {status.Message}");
}
while (status.Status == "queued" || status.Status == "processing");
if (status.Status == "complete")
{
var pageData = await client.GetPageContentAsync(submitResponse.RequestId);
Console.WriteLine($"\n--- Page Data ---");
Console.WriteLine($"Title: {pageData.Title}");
Console.WriteLine($"URL: {pageData.Url}");
Console.WriteLine($"Meta Description: {pageData.MetaDescription}");
Console.WriteLine($"Body Length: {pageData.Body?.Length ?? 0} characters");
Console.WriteLine($"\n--- Navigation Links ({pageData.Routes.Count}) ---");
foreach (var link in pageData.Routes.Take(5))
{
Console.WriteLine($" [{link.Rank}] {link.Text} -> {link.Url}");
}
Console.WriteLine($"\n--- Body Links ({pageData.BodyRoutes.Count}) ---");
foreach (var link in pageData.BodyRoutes.Take(5))
{
var adMarker = link.IsPotentialAd ? "[AD] " : "";
Console.WriteLine($" {adMarker}[{link.Rank}] {link.Text} -> {link.Url}");
}
}
else
{
Console.WriteLine($"\nRetrieval failed: {status.Error}");
}
}
catch (Exception ex)
{
Console.WriteLine($"\nError: {ex.Message}");
}
}
}
```
## Advanced Scenarios
### Client Management
```csharp
// Add a new client
var addRequest = new AddClientRequest
{
Email = "feeder@example.com",
Alias = "Production Feeder #1",
CustomUserId = "prod-feeder-001",
DailyMaximum = 1000
};
var client = await viewEngineClient.AddClientAsync(addRequest);
Console.WriteLine($"Added client: {client.Id}");
// Route job to this specific client
var retrievalRequest = new SubmitRetrievalRequest
{
Url = "https://example.com",
ClientId = client.Id // Use the client's ID
};
// Or use custom user ID
var retrievalRequest2 = new SubmitRetrievalRequest
{
Url = "https://example.com",
CustomUserId = "prod-feeder-001"
};
// List all clients
var clients = await viewEngineClient.GetClientsAsync();
foreach (var c in clients)
{
Console.WriteLine($"{c.Alias}: {(c.FeederOnline ? "Online" : "Offline")}");
Console.WriteLine($" Jobs today: {c.JobsProcessedToday}/{c.DailyMaximum}");
}
// Get client statistics
var stats = await viewEngineClient.GetClientStatsAsync(client.Id);
Console.WriteLine($"Total jobs: {stats.TotalJobsProcessed}");
Console.WriteLine($"Success rate: {stats.SuccessRate:F2}%");
Console.WriteLine($"Avg processing time: {stats.AverageProcessingTimeMs:F0}ms");
// Update client settings
await viewEngineClient.UpdateClientAsync(client.Id, new UpdateClientRequest
{
DailyMaximum = 2000,
Alias = "Production Feeder #1 (Updated)"
});
// Suspend client
await viewEngineClient.SuspendClientAsync(client.Id);
// Reactivate
await viewEngineClient.ActivateClientAsync(client.Id);
// Delete client
await viewEngineClient.DeleteClientAsync(client.Id);
```
### Platform-Specific Retrieval
```csharp
// Request page to be rendered on specific platform
var request = new SubmitRetrievalRequest
{
Url = "https://example.com",
PreferredPlatform = "Windows" // or "Android", "iOS"
};
var pageData = await client.RetrieveAndWaitAsync(request);
```
### High Priority Jobs
```csharp
var request = new SubmitRetrievalRequest
{
Url = "https://example.com",
Priority = 10, // Highest priority (1-10)
TimeoutSeconds = 300 // Max 5 minutes
};
```
### Batch Processing
```csharp
var urls = new[]
{
"https://example.com",
"https://another-example.com",
"https://third-example.com"
};
// Submit all requests
var submitTasks = urls.Select(url => client.SubmitRetrievalAsync(new SubmitRetrievalRequest
{
Url = url,
Priority = 5
}));
var submitResponses = await Task.WhenAll(submitTasks);
// Wait for all to complete
var tasks = submitResponses.Select(async response =>
{
while (true)
{
var status = await client.GetRetrievalStatusAsync(response.RequestId);
if (status.Status == "complete")
{
return await client.GetPageContentAsync(response.RequestId);
}
else if (status.Status == "failed")
{
throw new Exception($"Failed to retrieve {status.Url}");
}
await Task.Delay(2000);
}
});
var results = await Task.WhenAll(tasks);
foreach (var pageData in results)
{
Console.WriteLine($"{pageData.Title} - {pageData.Url}");
}
```
### Error Handling
```csharp
try
{
var pageData = await client.RetrieveAndWaitAsync(request);
}
catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Unauthorized)
{
Console.WriteLine("Invalid API key");
}
catch (HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
{
Console.WriteLine("Rate limit exceeded");
}
catch (InvalidOperationException ex) when (ex.Message.Contains("failed"))
{
Console.WriteLine($"Retrieval failed: {ex.Message}");
}
catch (OperationCanceledException)
{
Console.WriteLine("Request was canceled");
}
catch (Exception ex)
{
Console.WriteLine($"Unexpected error: {ex.Message}");
}
```
### Custom Timeout and Retry Settings
```csharp
var options = new ViewEngineOptions
{
ApiKey = "ak_your-api-key-here",
TimeoutSeconds = 180, // 3 minutes
MaxRetries = 5,
BaseDelayMs = 500,
DefaultPollingIntervalMs = 1000 // Poll every second
};
using var client = new ViewEngineClient(options);
```
### Dependency Injection with Multiple Clients
```csharp
// Configure multiple named clients
builder.Services.AddHttpClient<ViewEngineClient>("Primary")
.ConfigureHttpClient((sp, client) =>
{
client.BaseAddress = new Uri("https://www.viewengine.io/api/v1");
client.DefaultRequestHeaders.Add("X-API-Key", "ak_primary_key");
});
builder.Services.AddHttpClient<ViewEngineClient>("Secondary")
.ConfigureHttpClient((sp, client) =>
{
client.BaseAddress = new Uri("https://www.viewengine.io/api/v1");
client.DefaultRequestHeaders.Add("X-API-Key", "ak_secondary_key");
});
```

67
Extensions/ServiceCollectionExtensions.cs Normal file
View File

@@ -0,0 +1,67 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using ViewEngine.Client.Configuration;
namespace ViewEngine.Client.Extensions;
/// <summary>
/// Extension methods for registering ViewEngine client services
/// </summary>
public static class ServiceCollectionExtensions
{
/// <summary>
/// Adds ViewEngine client services to the dependency injection container
/// </summary>
/// <param name="services">The service collection</param>
/// <param name="configuration">The configuration</param>
/// <returns>The service collection for chaining</returns>
public static IServiceCollection AddViewEngineClient(this IServiceCollection services, IConfiguration configuration)
{
// Bind configuration
services.Configure<ViewEngineOptions>(configuration.GetSection(ViewEngineOptions.SectionName));
// Register the client as a typed HttpClient
services.AddHttpClient<ViewEngineClient>((serviceProvider, httpClient) =>
{
var options = configuration.GetSection(ViewEngineOptions.SectionName).Get<ViewEngineOptions>()
?? throw new InvalidOperationException("ViewEngine configuration is missing");
if (string.IsNullOrWhiteSpace(options.ApiKey))
throw new InvalidOperationException("ViewEngine API key is required");
httpClient.BaseAddress = new Uri(options.BaseUrl);
httpClient.Timeout = TimeSpan.FromSeconds(options.TimeoutSeconds);
httpClient.DefaultRequestHeaders.Add("X-API-Key", options.ApiKey);
});
return services;
}
/// <summary>
/// Adds ViewEngine client services to the dependency injection container with custom configuration
/// </summary>
/// <param name="services">The service collection</param>
/// <param name="configureOptions">Action to configure options</param>
/// <returns>The service collection for chaining</returns>
public static IServiceCollection AddViewEngineClient(this IServiceCollection services, Action<ViewEngineOptions> configureOptions)
{
// Configure options
services.Configure(configureOptions);
// Register the client as a typed HttpClient
services.AddHttpClient<ViewEngineClient>((serviceProvider, httpClient) =>
{
var options = new ViewEngineOptions();
configureOptions(options);
if (string.IsNullOrWhiteSpace(options.ApiKey))
throw new InvalidOperationException("ViewEngine API key is required");
httpClient.BaseAddress = new Uri(options.BaseUrl);
httpClient.Timeout = TimeSpan.FromSeconds(options.TimeoutSeconds);
httpClient.DefaultRequestHeaders.Add("X-API-Key", options.ApiKey);
});
return services;
}
}

160
Models/ClientManagementModels.cs Normal file
View File

@@ -0,0 +1,160 @@
namespace ViewEngine.Client.Models;
/// <summary>
/// Request to add a client programmatically
/// </summary>
public class AddClientRequest
{
/// <summary>
/// Email address of the user to add as client
/// </summary>
public string Email { get; set; } = string.Empty;
/// <summary>
/// Friendly name for this client (e.g., "John's Feeder")
/// </summary>
public string? Alias { get; set; }
/// <summary>
/// Your custom identifier for this client (alternative to using clientId)
/// </summary>
public string? CustomUserId { get; set; }
/// <summary>
/// Max jobs per day for this client (0 = unlimited, default: 0)
/// </summary>
public int DailyMaximum { get; set; }
}
/// <summary>
/// Request to update client settings
/// </summary>
public class UpdateClientRequest
{
/// <summary>
/// Friendly name for this client
/// </summary>
public string? Alias { get; set; }
/// <summary>
/// Your custom identifier for this client
/// </summary>
public string? CustomUserId { get; set; }
/// <summary>
/// Max jobs per day for this client (0 = unlimited)
/// </summary>
public int? DailyMaximum { get; set; }
}
/// <summary>
/// Information about a client
/// </summary>
public class ClientInfo
{
/// <summary>
/// Client ID
/// </summary>
public Guid Id { get; set; }
/// <summary>
/// User ID of the client
/// </summary>
public Guid UserId { get; set; }
/// <summary>
/// Client's email address
/// </summary>
public string Email { get; set; } = string.Empty;
/// <summary>
/// Client's display name
/// </summary>
public string? DisplayName { get; set; }
/// <summary>
/// Friendly alias for the client
/// </summary>
public string? Alias { get; set; }
/// <summary>
/// Your custom user identifier
/// </summary>
public string? CustomUserId { get; set; }
/// <summary>
/// Client status: Active, Suspended, etc.
/// </summary>
public string Status { get; set; } = string.Empty;
/// <summary>
/// Maximum jobs per day
/// </summary>
public int DailyMaximum { get; set; }
/// <summary>
/// When the client was added
/// </summary>
public DateTime CreatedAt { get; set; }
/// <summary>
/// When the client was last active
/// </summary>
public DateTime? LastActiveAt { get; set; }
/// <summary>
/// Whether the client's feeder is currently online
/// </summary>
public bool FeederOnline { get; set; }
/// <summary>
/// Number of jobs processed today
/// </summary>
public int JobsProcessedToday { get; set; }
}
/// <summary>
/// Statistics for a client
/// </summary>
public class ClientStats
{
/// <summary>
/// Client ID
/// </summary>
public Guid ClientId { get; set; }
/// <summary>
/// Total jobs processed all time
/// </summary>
public int TotalJobsProcessed { get; set; }
/// <summary>
/// Jobs processed today
/// </summary>
public int JobsProcessedToday { get; set; }
/// <summary>
/// Jobs processed this month
/// </summary>
public int JobsProcessedThisMonth { get; set; }
/// <summary>
/// Average processing time in milliseconds
/// </summary>
public double AverageProcessingTimeMs { get; set; }
/// <summary>
/// Success rate (0-100)
/// </summary>
public double SuccessRate { get; set; }
/// <summary>
/// When the feeder was last online
/// </summary>
public DateTime? LastOnlineAt { get; set; }
/// <summary>
/// Current feeder status
/// </summary>
public bool IsOnline { get; set; }
}

312
Models/RetrievalModels.cs Normal file
View File

@@ -0,0 +1,312 @@
namespace ViewEngine.Client.Models;
/// <summary>
/// Request to submit a web page retrieval job
/// </summary>
public class SubmitRetrievalRequest
{
/// <summary>
/// The URL of the web page to retrieve (scheme optional, will default to https://)
/// </summary>
public string Url { get; set; } = string.Empty;
/// <summary>
/// Maximum time to wait in seconds (default: 60, max: 300)
/// </summary>
public int TimeoutSeconds { get; set; } = 60;
/// <summary>
/// Force fresh retrieval, bypassing cache (default: false)
/// </summary>
public bool ForceRefresh { get; set; }
/// <summary>
/// Number of feeders that must agree (Community mode only, 1-10)
/// </summary>
public int? RequiredQuorum { get; set; }
/// <summary>
/// Job priority (1=lowest, 10=highest, default: 5). Higher priority costs more.
/// </summary>
public int Priority { get; set; } = 5;
/// <summary>
/// Route job to specific client's feeder (requires Client Management setup)
/// </summary>
public Guid? ClientId { get; set; }
/// <summary>
/// Route job using your custom client identifier (alternative to clientId)
/// </summary>
public string? CustomUserId { get; set; }
/// <summary>
/// Restrict processing to a specific platform: "Android", "iOS", or "Windows" (Community mode only)
/// </summary>
public string? PreferredPlatform { get; set; }
/// <summary>
/// If true, generate an AI summary of the page content (default: false)
/// </summary>
public bool GenerateSummary { get; set; } = false;
/// <summary>
/// Target language code for translation (ISO 639-1, e.g., "en", "es", "de", "zh").
/// If specified and the page is in a different language, content will be translated.
/// Null = no translation, return content as-is.
/// </summary>
public string? TargetLanguage { get; set; }
}
/// <summary>
/// Response from submitting a retrieval request
/// </summary>
public class RetrievalResponse
{
/// <summary>
/// Unique identifier for this retrieval request
/// </summary>
public Guid RequestId { get; set; }
/// <summary>
/// Current status of the request
/// </summary>
public string Status { get; set; } = string.Empty;
/// <summary>
/// User-friendly message describing the status
/// </summary>
public string Message { get; set; } = string.Empty;
/// <summary>
/// Estimated wait time in seconds
/// </summary>
public int? EstimatedWaitTimeSeconds { get; set; }
}
/// <summary>
/// Status response for a retrieval request
/// </summary>
public class RetrievalStatusResponse
{
/// <summary>
/// The request ID
/// </summary>
public Guid RequestId { get; set; }
/// <summary>
/// The URL being retrieved
/// </summary>
public string Url { get; set; } = string.Empty;
/// <summary>
/// Current status: queued, processing, validating, complete, failed, canceled
/// </summary>
public string Status { get; set; } = string.Empty;
/// <summary>
/// User-friendly status message
/// </summary>
public string Message { get; set; } = string.Empty;
/// <summary>
/// Content information (available when status is complete)
/// </summary>
public ContentInfo? Content { get; set; }
/// <summary>
/// Error message if status is failed
/// </summary>
public string? Error { get; set; }
/// <summary>
/// When the request was created
/// </summary>
public DateTime CreatedAt { get; set; }
/// <summary>
/// When the request was completed
/// </summary>
public DateTime? CompletedAt { get; set; }
}
/// <summary>
/// Content information for a completed retrieval
/// </summary>
public class ContentInfo
{
/// <summary>
/// URL to download the decrypted page content
/// </summary>
public string PageDataUrl { get; set; } = string.Empty;
/// <summary>
/// Hash of the retrieved content
/// </summary>
public string? ContentHash { get; set; }
/// <summary>
/// Artifacts such as screenshots and thumbnails
/// </summary>
public ArtifactsInfo? Artifacts { get; set; }
/// <summary>
/// Performance and quality metrics
/// </summary>
public Dictionary<string, object>? Metrics { get; set; }
}
/// <summary>
/// Artifacts from page retrieval
/// </summary>
public class ArtifactsInfo
{
/// <summary>
/// URL to full page screenshot
/// </summary>
public string? Screenshot { get; set; }
/// <summary>
/// URL to page thumbnail
/// </summary>
public string? Thumbnail { get; set; }
}
/// <summary>
/// Page content data returned from the API
/// </summary>
public class PageData
{
/// <summary>
/// Page title
/// </summary>
public string Title { get; set; } = string.Empty;
/// <summary>
/// Full text content of the page
/// </summary>
public string Body { get; set; } = string.Empty;
/// <summary>
/// Meta description tag content
/// </summary>
public string? MetaDescription { get; set; }
/// <summary>
/// Final URL after any redirects
/// </summary>
public string Url { get; set; } = string.Empty;
/// <summary>
/// URL to the page's favicon
/// </summary>
public string? FaviconUrl { get; set; }
/// <summary>
/// Base64-encoded PNG screenshot of the page (typically 320x180)
/// </summary>
public string? Thumbnail { get; set; }
/// <summary>
/// Links found in navigation/header areas
/// </summary>
public List<LinkInfo> Routes { get; set; } = new();
/// <summary>
/// Links found in page body content
/// </summary>
public List<LinkInfo> BodyRoutes { get; set; } = new();
/// <summary>
/// Optional AI-generated summary of the page content
/// </summary>
public string? Summary { get; set; }
/// <summary>
/// HTTP status code from the page retrieval.
/// Null if status code is not available.
/// Common values: 200 (OK), 404 (Not Found), 500 (Server Error).
/// </summary>
public int? HttpStatusCode { get; set; }
/// <summary>
/// Returns true if the HTTP status code indicates success (2xx) or is not available.
/// </summary>
public bool IsSuccess => HttpStatusCode == null || (HttpStatusCode >= 200 && HttpStatusCode < 300);
/// <summary>
/// Returns true if the HTTP status code indicates a client error (4xx).
/// </summary>
public bool IsClientError => HttpStatusCode >= 400 && HttpStatusCode < 500;
/// <summary>
/// Returns true if the HTTP status code indicates a server error (5xx).
/// </summary>
public bool IsServerError => HttpStatusCode >= 500;
/// <summary>
/// Language of the page as declared in the HTML lang attribute (e.g., "en", "es", "zh-CN").
/// Null if not specified in the HTML.
/// </summary>
public string? HtmlLang { get; set; }
/// <summary>
/// Detected/confirmed language of the content (ISO 639-1 code, e.g., "en", "es", "zh").
/// May differ from HtmlLang if automatic language detection was performed.
/// </summary>
public string? DetectedLanguage { get; set; }
/// <summary>
/// If content was translated, this is the target language code.
/// Null if no translation was performed.
/// </summary>
public string? TranslatedTo { get; set; }
/// <summary>
/// Original body text before translation.
/// Only populated if translation occurred.
/// </summary>
public string? OriginalBody { get; set; }
/// <summary>
/// Original title before translation.
/// Only populated if translation occurred.
/// </summary>
public string? OriginalTitle { get; set; }
}
/// <summary>
/// Information about a link on the page
/// </summary>
public class LinkInfo
{
/// <summary>
/// The link URL
/// </summary>
public string Url { get; set; } = string.Empty;
/// <summary>
/// The anchor text (what the link says)
/// </summary>
public string Text { get; set; } = string.Empty;
/// <summary>
/// Importance ranking (higher = more important, typically 1-10)
/// </summary>
public int Rank { get; set; }
/// <summary>
/// How many times this link appears on the page
/// </summary>
public int Occurrences { get; set; }
/// <summary>
/// Whether this link is likely an advertisement
/// </summary>
public bool IsPotentialAd { get; set; }
/// <summary>
/// Explanation of why the link was flagged as an ad (null if not an ad)
/// </summary>
public string? AdReason { get; set; }
}

244
PROJECT_SUMMARY.md Normal file
View File

@@ -0,0 +1,244 @@
# ViewEngine.Client - Project Summary
## Overview
The **ViewEngine.Client** is a production-ready .NET 9 NuGet package that provides a clean, type-safe client library for consuming the ViewEngine REST API. It enables developers to easily retrieve web pages, extract content, and manage distributed web scraping operations.
## Project Structure
```
ViewEngine.Client/
├── Configuration/
│ └── ViewEngineOptions.cs # Configuration options for the client
├── Extensions/
│ └── ServiceCollectionExtensions.cs # Dependency injection helpers
├── Models/
│ ├── ClientManagementModels.cs # DTOs for client management API
│ └── RetrievalModels.cs # DTOs for retrieval API
├── ViewEngineClient.cs # Main client implementation
├── ViewEngine.Client.csproj # Project file with NuGet metadata
├── README.md # Comprehensive documentation
├── QUICKSTART.md # Quick reference guide
├── Examples.md # Detailed usage examples
└── PROJECT_SUMMARY.md # This file
```
## Key Features
### Core Functionality
- ✅ Submit web page retrieval requests via MCP API
- ✅ Poll for job status and retrieve results
- ✅ Download decrypted page content with full text extraction
- ✅ Access screenshots, thumbnails, and artifacts
- ✅ Manage clients and route jobs to specific feeders
- ✅ Built-in retry logic with exponential backoff
- ✅ Full support for API key authentication
- ✅ Type-safe request/response models
- ✅ Async/await support throughout
- ✅ Dependency injection integration
### Advanced Features
- ✅ Rate limit handling with retry-after support
- ✅ Exponential backoff for transient failures
- ✅ Server error retry logic
- ✅ Platform-specific retrieval (Android, iOS, Windows)
- ✅ Priority-based job scheduling
- ✅ Client routing with ClientId or CustomUserId
- ✅ Force refresh option to bypass cache
- ✅ Configurable timeouts and polling intervals
### Client Management
- ✅ Add clients programmatically
- ✅ List and filter clients
- ✅ Update client settings
- ✅ Suspend/activate clients
- ✅ Delete clients
- ✅ Get detailed client statistics
- ✅ Monitor feeder online status
- ✅ Track job processing metrics
## API Coverage
### MCP Retrieval API (/mcp/*)
- `POST /mcp/retrieve` - Submit retrieval request
- `GET /mcp/retrieve/{requestId}` - Get retrieval status
- `GET /mcp/retrieve/{requestId}/content` - Download page content
- `GET /mcp/tools` - List available tools
### Client Management API (/v1/clients/*)
- `POST /v1/clients` - Add client
- `GET /v1/clients` - List clients
- `PUT /v1/clients/{id}` - Update client
- `PUT /v1/clients/{id}/suspend` - Suspend client
- `PUT /v1/clients/{id}/activate` - Activate client
- `DELETE /v1/clients/{id}` - Delete client
- `GET /v1/clients/{id}/stats` - Get client stats
## NuGet Package Details
### Package Metadata
- **Package ID**: `ViewEngine.Client`
- **Version**: 1.0.0
- **Target Framework**: .NET 9.0
- **Authors**: David H Friedel Jr
- **License**: MIT
- **Output Path**: `C:\Users\logik\Dropbox\Nugets\ViewEngine.Client.1.0.0.nupkg`
### Dependencies
- `Microsoft.Extensions.Http` (9.0.0)
- `Microsoft.Extensions.Options` (9.0.0)
- `Microsoft.Extensions.Options.ConfigurationExtensions` (9.0.0)
### Package Contents
- Compiled DLL with XML documentation
- README.md for NuGet.org package page
- Full IntelliSense support
- Source link support (future)
## Usage Patterns
### 1. Simple Usage (No DI)
```csharp
var client = new ViewEngineClient("ak_your-api-key-here");
var page = await client.RetrieveAndWaitAsync(new SubmitRetrievalRequest
{
Url = "https://example.com"
});
```
### 2. Dependency Injection (Recommended)
```csharp
// Program.cs
builder.Services.AddViewEngineClient(builder.Configuration);
// Controller
public class MyController : ControllerBase
{
private readonly ViewEngineClient _client;
public MyController(ViewEngineClient client) => _client = client;
}
```
### 3. Options Pattern
```csharp
var options = new ViewEngineOptions
{
ApiKey = "ak_your-api-key-here",
TimeoutSeconds = 120,
MaxRetries = 3
};
var client = new ViewEngineClient(options);
```
## Architecture Decisions
### 1. Multiple Constructors
- Simple constructor with API key only
- IOptions constructor for DI
- HttpClient constructor for advanced DI scenarios
- Flexible and supports various usage patterns
### 2. Retry Logic
- Automatic retry for rate limits (429)
- Automatic retry for server errors (500-599)
- Automatic retry for network timeouts
- Respects Retry-After headers
- Exponential backoff with configurable base delay
### 3. Error Handling
- Throws HttpRequestException for HTTP errors
- Throws InvalidOperationException for failed retrievals
- Throws OperationCanceledException for canceled requests
- All errors include meaningful messages
### 4. Resource Management
- Implements IDisposable
- Properly disposes HttpClient when not injected
- Does not dispose injected HttpClient (DI container manages it)
### 5. Type Safety
- Strongly-typed DTOs for all requests/responses
- Nullable reference types enabled
- XML documentation for all public APIs
- IntelliSense support
## Testing Recommendations
### Unit Testing
```csharp
// Mock HttpClient for unit tests
var mockHandler = new MockHttpMessageHandler();
var httpClient = new HttpClient(mockHandler);
var options = Options.Create(new ViewEngineOptions { ApiKey = "test" });
var client = new ViewEngineClient(httpClient, options);
```
### Integration Testing
```csharp
// Use real API with test API key
var client = new ViewEngineClient("ak_test_key");
var page = await client.RetrieveAndWaitAsync(new SubmitRetrievalRequest
{
Url = "https://example.com"
});
Assert.NotNull(page.Title);
```
## Build Information
- **Status**: ✅ Build succeeded (0 warnings, 0 errors)
- **Package Created**: ✅ ViewEngine.Client.1.0.0.nupkg (30,151 bytes)
- **Documentation**: ✅ XML documentation generated
- **Target Framework**: .NET 9.0
- **Language Version**: Latest
## Next Steps
### For Developers Using This Package
1. Install via NuGet: `dotnet add package ViewEngine.Client`
2. Get API key from https://www.viewengine.io
3. Follow QUICKSTART.md for immediate use
4. See Examples.md for advanced scenarios
### For Package Maintainers
1. ✅ Initial release complete
2. 🔄 Publish to NuGet.org (manual step)
3. 🔄 Add source link support
4. 🔄 Add unit tests project
5. 🔄 Add integration tests
6. 🔄 Set up CI/CD pipeline
7. 🔄 Add code coverage reporting
### Future Enhancements
- [ ] Add webhook support for job completion notifications
- [ ] Add bulk retrieval helper methods
- [ ] Add caching layer for frequently accessed pages
- [ ] Add telemetry and diagnostics
- [ ] Add support for .NET Standard 2.0 (broader compatibility)
- [ ] Add rate limit awareness (track limits client-side)
- [ ] Add health check endpoint support
- [ ] Add billing/usage API support
## Documentation
- **README.md**: Main documentation with installation, configuration, and usage
- **QUICKSTART.md**: Quick reference for common tasks
- **Examples.md**: Comprehensive examples for all scenarios
- **XML Docs**: IntelliSense documentation in code
- **API Docs**: https://www.viewengine.io/docs
## Support
- GitHub: https://github.com/marketally/viewengine
- Issues: https://github.com/marketally/viewengine/issues
- Website: https://www.viewengine.io
## License
MIT License - Copyright © 2025 ViewEngine
---
**Package Created**: November 22, 2025
**Build Status**: ✅ Ready for distribution
**Quality**: Production-ready

192
QUICKSTART.md Normal file
View File

@@ -0,0 +1,192 @@
# ViewEngine.Client - Quick Start Guide
## Installation
```bash
dotnet add package ViewEngine.Client
```
## 30-Second Quick Start
```csharp
using ViewEngine.Client;
using ViewEngine.Client.Models;
var client = new ViewEngineClient("ak_your-api-key-here");
var pageData = await client.RetrieveAndWaitAsync(new SubmitRetrievalRequest
{
Url = "https://example.com"
});
Console.WriteLine($"Title: {pageData.Title}");
Console.WriteLine($"Content: {pageData.Body}");
```
## Common Use Cases
### 1. Simple Page Retrieval
```csharp
var client = new ViewEngineClient("ak_your-api-key-here");
var page = await client.RetrieveAndWaitAsync(new SubmitRetrievalRequest
{
Url = "https://example.com"
});
```
### 2. With ASP.NET Core DI
**appsettings.json:**
```json
{
"ViewEngine": {
"ApiKey": "ak_your-api-key-here"
}
}
```
**Program.cs:**
```csharp
builder.Services.AddViewEngineClient(builder.Configuration);
```
**Usage:**
```csharp
public class MyController : ControllerBase
{
private readonly ViewEngineClient _client;
public MyController(ViewEngineClient client) => _client = client;
[HttpGet]
public async Task<IActionResult> GetPage(string url)
{
var page = await _client.RetrieveAndWaitAsync(new SubmitRetrievalRequest { Url = url });
return Ok(page);
}
}
```
### 3. Custom Platform Selection
```csharp
var page = await client.RetrieveAndWaitAsync(new SubmitRetrievalRequest
{
Url = "https://example.com",
PreferredPlatform = "Windows" // or "Android", "iOS"
});
```
### 4. High Priority Job
```csharp
var page = await client.RetrieveAndWaitAsync(new SubmitRetrievalRequest
{
Url = "https://example.com",
Priority = 10 // 1-10, higher = faster (costs more)
});
```
### 5. Manual Polling
```csharp
var response = await client.SubmitRetrievalAsync(new SubmitRetrievalRequest
{
Url = "https://example.com"
});
RetrievalStatusResponse status;
do
{
await Task.Delay(2000);
status = await client.GetRetrievalStatusAsync(response.RequestId);
} while (status.Status != "complete" && status.Status != "failed");
if (status.Status == "complete")
{
var page = await client.GetPageContentAsync(response.RequestId);
}
```
## Key Features
| Feature | Method |
|---------|--------|
| Submit retrieval | `SubmitRetrievalAsync()` |
| Check status | `GetRetrievalStatusAsync()` |
| Get content | `GetPageContentAsync()` |
| Submit & wait | `RetrieveAndWaitAsync()` |
| Add client | `AddClientAsync()` |
| List clients | `GetClientsAsync()` |
| Client stats | `GetClientStatsAsync()` |
## Page Data Structure
```csharp
public class PageData
{
string Title // Page title
string Body // Full text content
string MetaDescription // Meta description
string Url // Final URL after redirects
string FaviconUrl // Favicon URL
string Thumbnail // Base64 PNG screenshot
List<LinkInfo> Routes // Navigation links
List<LinkInfo> BodyRoutes // Content links
}
```
## Error Handling
```csharp
try
{
var page = await client.RetrieveAndWaitAsync(request);
}
catch (HttpRequestException ex)
{
// Network/HTTP errors
}
catch (InvalidOperationException ex)
{
// Retrieval failed
}
catch (OperationCanceledException)
{
// Request canceled
}
```
## Configuration Options
```csharp
var options = new ViewEngineOptions
{
ApiKey = "ak_your-api-key-here",
BaseUrl = "https://www.viewengine.io/api/v1",
TimeoutSeconds = 120,
MaxRetries = 3,
DefaultPollingIntervalMs = 2000
};
var client = new ViewEngineClient(options);
```
## Request Options
```csharp
var request = new SubmitRetrievalRequest
{
Url = "https://example.com",
TimeoutSeconds = 60, // Max: 300
ForceRefresh = false, // Bypass cache
Priority = 5, // 1-10
PreferredPlatform = "Windows", // Android, iOS, Windows
ClientId = clientGuid, // Route to specific client
CustomUserId = "custom-id" // Or use custom ID
};
```
## Need More Help?
- Full Documentation: `README.md`
- Code Examples: `Examples.md`
- API Docs: https://www.viewengine.io/docs
- GitHub: https://github.com/marketally/viewengine

300
README.md Normal file
View File

@@ -0,0 +1,300 @@
# ViewEngine.Client
Official .NET client library for consuming the ViewEngine REST API. Retrieve web pages, extract content, and process web data with ViewEngine's distributed web scraping service.
## Installation
```bash
dotnet add package ViewEngine.Client
```
## Quick Start
### Basic Usage
```csharp
using ViewEngine.Client;
using ViewEngine.Client.Models;
// Create client with API key
var client = new ViewEngineClient("ak_your-api-key-here");
// Submit a retrieval request
var request = new SubmitRetrievalRequest
{
Url = "https://example.com",
TimeoutSeconds = 60,
Priority = 5,
GenerateSummary = true // Optional: Generate AI summary
};
// Submit and wait for completion
var pageData = await client.RetrieveAndWaitAsync(request);
Console.WriteLine($"Title: {pageData.Title}");
Console.WriteLine($"Summary: {pageData.Summary}"); // AI-generated summary if requested
Console.WriteLine($"Body: {pageData.Body}");
Console.WriteLine($"Links found: {pageData.Routes.Count}");
```
### Advanced Usage with Polling
```csharp
// Submit request
var submitResponse = await client.SubmitRetrievalAsync(request);
Console.WriteLine($"Request ID: {submitResponse.RequestId}");
// Poll for status
RetrievalStatusResponse status;
do
{
status = await client.GetRetrievalStatusAsync(submitResponse.RequestId);
Console.WriteLine($"Status: {status.Status}");
if (status.Status == "complete")
{
// Download content
var content = await client.GetPageContentAsync(submitResponse.RequestId);
Console.WriteLine($"Retrieved: {content.Title}");
break;
}
await Task.Delay(2000); // Wait 2 seconds
} while (status.Status == "queued" || status.Status == "processing");
```
## Dependency Injection
### ASP.NET Core / Minimal API
**appsettings.json:**
```json
{
"ViewEngine": {
"ApiKey": "ak_your-api-key-here",
"BaseUrl": "https://www.viewengine.io/api/v1",
"TimeoutSeconds": 120,
"MaxRetries": 3
}
}
```
**Program.cs:**
```csharp
using ViewEngine.Client.Extensions;
var builder = WebApplication.CreateBuilder(args);
// Add ViewEngine client
builder.Services.AddViewEngineClient(builder.Configuration);
var app = builder.Build();
app.MapGet("/retrieve", async (ViewEngineClient client) =>
{
var request = new SubmitRetrievalRequest { Url = "https://example.com" };
var pageData = await client.RetrieveAndWaitAsync(request);
return Results.Ok(pageData);
});
app.Run();
```
### With Options Pattern
```csharp
using ViewEngine.Client.Extensions;
builder.Services.AddViewEngineClient(options =>
{
options.ApiKey = "ak_your-api-key-here";
options.BaseUrl = "https://www.viewengine.io/api/v1";
options.TimeoutSeconds = 120;
options.MaxRetries = 3;
});
```
## Features
### Web Page Retrieval
```csharp
var request = new SubmitRetrievalRequest
{
Url = "https://example.com",
TimeoutSeconds = 60,
ForceRefresh = true,
Priority = 8,
PreferredPlatform = "Windows", // Android, iOS, or Windows
GenerateSummary = true // Generate AI summary of page content
};
var pageData = await client.RetrieveAndWaitAsync(request);
// Access extracted data
Console.WriteLine($"Title: {pageData.Title}");
Console.WriteLine($"Summary: {pageData.Summary}"); // AI-generated summary
Console.WriteLine($"Description: {pageData.MetaDescription}");
Console.WriteLine($"Body Text: {pageData.Body}");
Console.WriteLine($"Favicon: {pageData.FaviconUrl}");
// Navigation links
foreach (var link in pageData.Routes)
{
Console.WriteLine($"{link.Text} -> {link.Url}");
}
// Body links with ad detection
foreach (var link in pageData.BodyRoutes)
{
if (link.IsPotentialAd)
Console.WriteLine($"[AD] {link.Text} ({link.AdReason})");
else
Console.WriteLine($"{link.Text} -> {link.Url}");
}
```
### Client Management
```csharp
// Add a client
var addRequest = new AddClientRequest
{
Email = "user@example.com",
Alias = "Production Feeder",
CustomUserId = "prod-001",
DailyMaximum = 1000
};
var newClient = await client.AddClientAsync(addRequest);
// List all clients
var clients = await client.GetClientsAsync();
foreach (var c in clients)
{
Console.WriteLine($"{c.Alias}: {c.FeederOnline ? "Online" : "Offline"}");
}
// Route job to specific client
var routedRequest = new SubmitRetrievalRequest
{
Url = "https://example.com",
ClientId = newClient.Id // or CustomUserId = "prod-001"
};
// Get client stats
var stats = await client.GetClientStatsAsync(newClient.Id);
Console.WriteLine($"Total jobs: {stats.TotalJobsProcessed}");
Console.WriteLine($"Success rate: {stats.SuccessRate}%");
// Suspend/Activate/Delete
await client.SuspendClientAsync(newClient.Id);
await client.ActivateClientAsync(newClient.Id);
await client.DeleteClientAsync(newClient.Id);
```
### Error Handling
```csharp
try
{
var pageData = await client.RetrieveAndWaitAsync(request);
}
catch (HttpRequestException ex)
{
// HTTP errors (network, server errors)
Console.WriteLine($"HTTP Error: {ex.Message}");
}
catch (InvalidOperationException ex)
{
// Failed retrieval or invalid responses
Console.WriteLine($"Operation Error: {ex.Message}");
}
catch (OperationCanceledException)
{
// Request was canceled
Console.WriteLine("Request canceled");
}
```
## Configuration Options
| Option | Default | Description |
|--------|---------|-------------|
| `ApiKey` | (required) | Your ViewEngine API key |
| `BaseUrl` | `https://www.viewengine.io/api/v1` | API base URL |
| `TimeoutSeconds` | `120` | HTTP request timeout |
| `MaxRetries` | `3` | Maximum retry attempts |
| `BaseDelayMs` | `1000` | Base delay for exponential backoff |
| `DefaultPollingIntervalMs` | `2000` | Default polling interval for status checks |
## Request Options
### SubmitRetrievalRequest
| Property | Type | Default | Description |
|----------|------|---------|-------------|
| `Url` | `string` | (required) | URL to retrieve |
| `TimeoutSeconds` | `int` | `60` | Max wait time (max: 300) |
| `ForceRefresh` | `bool` | `false` | Bypass cache |
| `RequiredQuorum` | `int?` | `null` | Feeders that must agree (1-10, Community mode) |
| `Priority` | `int` | `5` | Job priority (1-10) |
| `ClientId` | `Guid?` | `null` | Route to specific client |
| `CustomUserId` | `string?` | `null` | Route using custom ID |
| `PreferredPlatform` | `string?` | `null` | "Android", "iOS", or "Windows" |
| `GenerateSummary` | `bool` | `false` | Generate AI summary of page content |
## Response Models
### PageData
```csharp
public class PageData
{
public string Title { get; set; }
public string Body { get; set; }
public string? MetaDescription { get; set; }
public string Url { get; set; }
public string? FaviconUrl { get; set; }
public string? Thumbnail { get; set; } // Base64 PNG
public string? Summary { get; set; } // AI-generated summary (if requested)
public List<LinkInfo> Routes { get; set; }
public List<LinkInfo> BodyRoutes { get; set; }
}
```
### LinkInfo
```csharp
public class LinkInfo
{
public string Url { get; set; }
public string Text { get; set; }
public int Rank { get; set; }
public int Occurrences { get; set; }
public bool IsPotentialAd { get; set; }
public string? AdReason { get; set; }
}
```
## Retry & Rate Limiting
The client automatically handles:
- **Rate limits (429)**: Respects `Retry-After` header with exponential backoff
- **Server errors (500-5xx)**: Retries with exponential backoff
- **Network timeouts**: Retries with exponential backoff
- **Configurable retries**: Set `MaxRetries` in options
## API Documentation
For complete API documentation, visit: https://www.viewengine.io/docs
## Support
- Documentation: https://www.viewengine.io/docs
- GitHub: https://github.com/marketally/viewengine
- Issues: https://github.com/marketally/viewengine/issues
## License
MIT License - Copyright © 2025 ViewEngine

93
ViewEngine.Client.csproj Executable file
View File

@@ -0,0 +1,93 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>latest</LangVersion>
<PackageOutputPath>C:\Users\logik\Dropbox\Nugets</PackageOutputPath>
</PropertyGroup>
<PropertyGroup>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<PackageId>ViewEngine.Client</PackageId>
<Version>1.2.1</Version>
<Authors>David H Friedel Jr</Authors>
<Company>MarketAlly</Company>
<Product>ViewEngine.Client</Product>
<Title>ViewEngine REST API Client for .NET</Title>
<Description>
Official .NET client library for consuming the ViewEngine REST API. Retrieve web pages, extract content, and process web data with ViewEngine's distributed web scraping service.
Key Features:
- Submit web page retrieval requests via MCP API
- Poll for job status and retrieve results
- Download decrypted page content with full text extraction
- Access screenshots, thumbnails, and artifacts
- Manage clients and route jobs to specific feeders
- Built-in retry logic with exponential backoff
- Full support for API key authentication
- Type-safe request/response models
- Async/await support throughout
- Dependency injection integration
- .NET 9.0 compatible
Perfect for applications requiring web scraping, content extraction, web page monitoring, and automated browsing.
</Description>
<Copyright>Copyright © 2025 MarketAlly</Copyright>
<PackageIcon>icon.png</PackageIcon>
<PackageReadmeFile>README.md</PackageReadmeFile>
<PackageLicenseExpression>MIT</PackageLicenseExpression>
<PackageProjectUrl>https://www.viewengine.io</PackageProjectUrl>
<RepositoryUrl>https://git.marketally.com/viewengine/ViewEngine.Client</RepositoryUrl>
<RepositoryType>git</RepositoryType>
<PackageTags>viewengine dotnet rest-api api-client web-scraping webscraping webcrawler http-client content-extraction page-retrieval mcp distributed async dotnet9</PackageTags>
<PackageReleaseNotes>
Version 1.0.0 (Initial Release):
🎉 NEW FEATURES:
- ViewEngineClient - Main client for MCP API operations
- Submit retrieval requests with customizable options (timeout, priority, platform preferences)
- Poll for job status and completion
- Download decrypted page content
- Client management API support
- Type-safe DTOs for all API operations
- Built-in retry logic with exponential backoff
- Rate limit handling
- API key authentication
- Full async/await support
- Dependency injection ready
📦 ARCHITECTURE:
- Clean separation of concerns
- Strongly-typed request/response models
- Configurable via IOptions pattern
- HttpClient integration
- Production-ready error handling
</PackageReleaseNotes>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.10" />
<PackageReference Include="Microsoft.Extensions.Options" Version="9.0.10" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="9.0.10" />
</ItemGroup>
<ItemGroup>
<None Include="icon.png">
<Pack>true</Pack>
<PackagePath>\</PackagePath>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<Visible>true</Visible>
</None>
<None Include="README.md" Pack="true" PackagePath="\" />
</ItemGroup>
</Project>

457
ViewEngineClient.cs Normal file
View File

@@ -0,0 +1,457 @@
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Text.Json;
using Microsoft.Extensions.Options;
using ViewEngine.Client.Configuration;
using ViewEngine.Client.Models;
namespace ViewEngine.Client;
/// <summary>
/// Client for interacting with the ViewEngine REST API
/// </summary>
public class ViewEngineClient : IDisposable
{
private readonly HttpClient _httpClient;
private readonly ViewEngineOptions _options;
private readonly bool _disposeHttpClient;
/// <summary>
/// Creates a new ViewEngineClient with the specified API key
/// </summary>
/// <param name="apiKey">Your ViewEngine API key</param>
/// <param name="baseUrl">Optional base URL (defaults to https://www.viewengine.io)</param>
public ViewEngineClient(string apiKey, string? baseUrl = null)
{
if (string.IsNullOrWhiteSpace(apiKey))
throw new ArgumentNullException(nameof(apiKey));
_options = new ViewEngineOptions
{
ApiKey = apiKey,
BaseUrl = baseUrl ?? "https://www.viewengine.io"
};
_httpClient = new HttpClient
{
BaseAddress = new Uri(_options.BaseUrl),
Timeout = TimeSpan.FromSeconds(_options.TimeoutSeconds)
};
ConfigureHttpClient();
_disposeHttpClient = true;
}
/// <summary>
/// Creates a new ViewEngineClient with the specified options
/// </summary>
/// <param name="options">Configuration options</param>
public ViewEngineClient(IOptions<ViewEngineOptions> options) : this(options.Value)
{
}
/// <summary>
/// Creates a new ViewEngineClient with the specified options
/// </summary>
/// <param name="options">Configuration options</param>
public ViewEngineClient(ViewEngineOptions options)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
if (string.IsNullOrWhiteSpace(_options.ApiKey))
throw new ArgumentException("API key is required", nameof(options));
_httpClient = new HttpClient
{
BaseAddress = new Uri(_options.BaseUrl),
Timeout = TimeSpan.FromSeconds(_options.TimeoutSeconds)
};
ConfigureHttpClient();
_disposeHttpClient = true;
}
/// <summary>
/// Creates a new ViewEngineClient with a custom HttpClient (for dependency injection)
/// </summary>
/// <param name="httpClient">Configured HttpClient instance</param>
/// <param name="options">Configuration options</param>
public ViewEngineClient(HttpClient httpClient, IOptions<ViewEngineOptions> options)
{
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
if (string.IsNullOrWhiteSpace(_options.ApiKey))
throw new ArgumentException("API key is required", nameof(options));
_httpClient.BaseAddress = new Uri(_options.BaseUrl);
_httpClient.Timeout = TimeSpan.FromSeconds(_options.TimeoutSeconds);
ConfigureHttpClient();
_disposeHttpClient = false; // Don't dispose injected HttpClient
}
private void ConfigureHttpClient()
{
_httpClient.DefaultRequestHeaders.Clear();
_httpClient.DefaultRequestHeaders.Add("X-API-Key", _options.ApiKey);
_httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
}
#region MCP Retrieval API
/// <summary>
/// Submit a web page retrieval request
/// </summary>
/// <param name="request">Retrieval request parameters</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>Retrieval response with request ID</returns>
public async Task<RetrievalResponse> SubmitRetrievalAsync(SubmitRetrievalRequest request, CancellationToken cancellationToken = default)
{
if (request == null)
throw new ArgumentNullException(nameof(request));
if (string.IsNullOrWhiteSpace(request.Url))
throw new ArgumentException("URL is required", nameof(request));
var response = await ExecuteWithRetryAsync(async () =>
{
return await _httpClient.PostAsJsonAsync("/mcp/retrieve", request, cancellationToken);
}, "SubmitRetrieval", cancellationToken);
response.EnsureSuccessStatusCode();
var result = await response.Content.ReadFromJsonAsync<RetrievalResponse>(cancellationToken: cancellationToken);
return result ?? throw new InvalidOperationException("Failed to deserialize retrieval response");
}
/// <summary>
/// Get the status of a retrieval request
/// </summary>
/// <param name="requestId">The request ID returned from SubmitRetrievalAsync</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>Current status of the retrieval request</returns>
public async Task<RetrievalStatusResponse> GetRetrievalStatusAsync(Guid requestId, CancellationToken cancellationToken = default)
{
if (requestId == Guid.Empty)
throw new ArgumentException("Invalid request ID", nameof(requestId));
var response = await ExecuteWithRetryAsync(async () =>
{
return await _httpClient.GetAsync($"/mcp/retrieve/{requestId}", cancellationToken);
}, "GetRetrievalStatus", cancellationToken);
response.EnsureSuccessStatusCode();
var result = await response.Content.ReadFromJsonAsync<RetrievalStatusResponse>(cancellationToken: cancellationToken);
return result ?? throw new InvalidOperationException("Failed to deserialize status response");
}
/// <summary>
/// Download the decrypted page content for a completed retrieval
/// </summary>
/// <param name="requestId">The request ID of the completed retrieval</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>Page content data</returns>
public async Task<PageData> GetPageContentAsync(Guid requestId, CancellationToken cancellationToken = default)
{
if (requestId == Guid.Empty)
throw new ArgumentException("Invalid request ID", nameof(requestId));
var response = await ExecuteWithRetryAsync(async () =>
{
return await _httpClient.GetAsync($"/mcp/retrieve/{requestId}/content", cancellationToken);
}, "GetPageContent", cancellationToken);
response.EnsureSuccessStatusCode();
var result = await response.Content.ReadFromJsonAsync<PageData>(cancellationToken: cancellationToken);
return result ?? throw new InvalidOperationException("Failed to deserialize page content");
}
/// <summary>
/// Submit a retrieval request and wait for completion
/// </summary>
/// <param name="request">Retrieval request parameters</param>
/// <param name="pollingIntervalMs">How often to check status in milliseconds (default: 2000)</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>Page content data when complete</returns>
public async Task<PageData> RetrieveAndWaitAsync(SubmitRetrievalRequest request, int? pollingIntervalMs = null, CancellationToken cancellationToken = default)
{
// Submit the request
var submitResponse = await SubmitRetrievalAsync(request, cancellationToken);
// Poll for completion
var interval = pollingIntervalMs ?? _options.DefaultPollingIntervalMs;
while (!cancellationToken.IsCancellationRequested)
{
var status = await GetRetrievalStatusAsync(submitResponse.RequestId, cancellationToken);
if (status.Status.Equals("complete", StringComparison.OrdinalIgnoreCase))
{
// Download and return the content
return await GetPageContentAsync(submitResponse.RequestId, cancellationToken);
}
else if (status.Status.Equals("failed", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException($"Retrieval failed: {status.Error ?? status.Message}");
}
else if (status.Status.Equals("canceled", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("Retrieval was canceled");
}
// Wait before polling again
await Task.Delay(interval, cancellationToken);
}
throw new OperationCanceledException("Operation was canceled", cancellationToken);
}
/// <summary>
/// List available MCP tools
/// </summary>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>List of available tools</returns>
public async Task<object> ListToolsAsync(CancellationToken cancellationToken = default)
{
var response = await ExecuteWithRetryAsync(async () =>
{
return await _httpClient.GetAsync("/mcp/tools", cancellationToken);
}, "ListTools", cancellationToken);
response.EnsureSuccessStatusCode();
var result = await response.Content.ReadFromJsonAsync<object>(cancellationToken: cancellationToken);
return result ?? throw new InvalidOperationException("Failed to deserialize tools response");
}
#endregion
#region Client Management API
/// <summary>
/// Add a new client programmatically
/// </summary>
/// <param name="request">Client details</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>Created client information</returns>
public async Task<ClientInfo> AddClientAsync(AddClientRequest request, CancellationToken cancellationToken = default)
{
if (request == null)
throw new ArgumentNullException(nameof(request));
if (string.IsNullOrWhiteSpace(request.Email))
throw new ArgumentException("Email is required", nameof(request));
var response = await ExecuteWithRetryAsync(async () =>
{
return await _httpClient.PostAsJsonAsync("/v1/clients", request, cancellationToken);
}, "AddClient", cancellationToken);
response.EnsureSuccessStatusCode();
var result = await response.Content.ReadFromJsonAsync<ClientInfo>(cancellationToken: cancellationToken);
return result ?? throw new InvalidOperationException("Failed to deserialize client response");
}
/// <summary>
/// Get a list of all clients
/// </summary>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>List of clients</returns>
public async Task<List<ClientInfo>> GetClientsAsync(CancellationToken cancellationToken = default)
{
var response = await ExecuteWithRetryAsync(async () =>
{
return await _httpClient.GetAsync("/v1/clients", cancellationToken);
}, "GetClients", cancellationToken);
response.EnsureSuccessStatusCode();
var result = await response.Content.ReadFromJsonAsync<List<ClientInfo>>(cancellationToken: cancellationToken);
return result ?? new List<ClientInfo>();
}
/// <summary>
/// Update client settings
/// </summary>
/// <param name="clientId">Client ID</param>
/// <param name="request">Updated settings</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>Updated client information</returns>
public async Task<ClientInfo> UpdateClientAsync(Guid clientId, UpdateClientRequest request, CancellationToken cancellationToken = default)
{
if (clientId == Guid.Empty)
throw new ArgumentException("Invalid client ID", nameof(clientId));
if (request == null)
throw new ArgumentNullException(nameof(request));
var response = await ExecuteWithRetryAsync(async () =>
{
return await _httpClient.PutAsJsonAsync($"/v1/clients/{clientId}", request, cancellationToken);
}, "UpdateClient", cancellationToken);
response.EnsureSuccessStatusCode();
var result = await response.Content.ReadFromJsonAsync<ClientInfo>(cancellationToken: cancellationToken);
return result ?? throw new InvalidOperationException("Failed to deserialize client response");
}
/// <summary>
/// Suspend a client
/// </summary>
/// <param name="clientId">Client ID</param>
/// <param name="cancellationToken">Cancellation token</param>
public async Task SuspendClientAsync(Guid clientId, CancellationToken cancellationToken = default)
{
if (clientId == Guid.Empty)
throw new ArgumentException("Invalid client ID", nameof(clientId));
var response = await ExecuteWithRetryAsync(async () =>
{
return await _httpClient.PutAsync($"/v1/clients/{clientId}/suspend", null, cancellationToken);
}, "SuspendClient", cancellationToken);
response.EnsureSuccessStatusCode();
}
/// <summary>
/// Reactivate a suspended client
/// </summary>
/// <param name="clientId">Client ID</param>
/// <param name="cancellationToken">Cancellation token</param>
public async Task ActivateClientAsync(Guid clientId, CancellationToken cancellationToken = default)
{
if (clientId == Guid.Empty)
throw new ArgumentException("Invalid client ID", nameof(clientId));
var response = await ExecuteWithRetryAsync(async () =>
{
return await _httpClient.PutAsync($"/v1/clients/{clientId}/activate", null, cancellationToken);
}, "ActivateClient", cancellationToken);
response.EnsureSuccessStatusCode();
}
/// <summary>
/// Remove a client
/// </summary>
/// <param name="clientId">Client ID</param>
/// <param name="cancellationToken">Cancellation token</param>
public async Task DeleteClientAsync(Guid clientId, CancellationToken cancellationToken = default)
{
if (clientId == Guid.Empty)
throw new ArgumentException("Invalid client ID", nameof(clientId));
var response = await ExecuteWithRetryAsync(async () =>
{
return await _httpClient.DeleteAsync($"/v1/clients/{clientId}", cancellationToken);
}, "DeleteClient", cancellationToken);
response.EnsureSuccessStatusCode();
}
/// <summary>
/// Get detailed statistics for a client
/// </summary>
/// <param name="clientId">Client ID</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>Client statistics</returns>
public async Task<ClientStats> GetClientStatsAsync(Guid clientId, CancellationToken cancellationToken = default)
{
if (clientId == Guid.Empty)
throw new ArgumentException("Invalid client ID", nameof(clientId));
var response = await ExecuteWithRetryAsync(async () =>
{
return await _httpClient.GetAsync($"/v1/clients/{clientId}/stats", cancellationToken);
}, "GetClientStats", cancellationToken);
response.EnsureSuccessStatusCode();
var result = await response.Content.ReadFromJsonAsync<ClientStats>(cancellationToken: cancellationToken);
return result ?? throw new InvalidOperationException("Failed to deserialize stats response");
}
#endregion
#region Retry Logic
private async Task<HttpResponseMessage> ExecuteWithRetryAsync(
Func<Task<HttpResponseMessage>> action,
string operationName,
CancellationToken cancellationToken = default)
{
int attempt = 0;
while (attempt < _options.MaxRetries)
{
try
{
attempt++;
var response = await action();
// Handle rate limiting (429) with exponential backoff
if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
{
if (attempt >= _options.MaxRetries)
{
throw new HttpRequestException($"Rate limit exceeded for {operationName} after {attempt} attempts");
}
// Check for Retry-After header
var retryAfter = response.Headers.RetryAfter?.Delta?.TotalMilliseconds
?? (_options.BaseDelayMs * Math.Pow(2, attempt - 1));
await Task.Delay((int)retryAfter, cancellationToken);
continue;
}
// Retry on server errors (500-599)
if ((int)response.StatusCode >= 500 && (int)response.StatusCode < 600)
{
if (attempt >= _options.MaxRetries)
{
throw new HttpRequestException($"Server error for {operationName}: {response.StatusCode} after {attempt} attempts");
}
var delay = _options.BaseDelayMs * Math.Pow(2, attempt - 1);
await Task.Delay((int)delay, cancellationToken);
continue;
}
return response;
}
catch (TaskCanceledException) when (attempt < _options.MaxRetries && !cancellationToken.IsCancellationRequested)
{
var delay = _options.BaseDelayMs * Math.Pow(2, attempt - 1);
await Task.Delay((int)delay, cancellationToken);
continue;
}
catch (HttpRequestException) when (attempt < _options.MaxRetries)
{
var delay = _options.BaseDelayMs * Math.Pow(2, attempt - 1);
await Task.Delay((int)delay, cancellationToken);
continue;
}
}
throw new Exception($"Failed after {_options.MaxRetries} attempts for {operationName}");
}
#endregion
/// <summary>
/// Disposes the ViewEngineClient and underlying HttpClient if not injected
/// </summary>
public void Dispose()
{
if (_disposeHttpClient)
{
_httpClient?.Dispose();
}
}
}

BIN
icon.png Executable file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.2 KiB