| | | 1 | | using System; |
| | | 2 | | using System.IO; |
| | | 3 | | using System.Net.Http; |
| | | 4 | | using System.Security.Cryptography; |
| | | 5 | | using System.Text; |
| | | 6 | | using System.Threading; |
| | | 7 | | using System.Threading.Tasks; |
| | | 8 | | using Microsoft.Extensions.Logging; |
| | | 9 | | using Microsoft.Extensions.Logging.Abstractions; |
| | | 10 | | |
| | | 11 | | namespace InferenceEngine.Core.ModelProviders |
| | | 12 | | { |
| | | 13 | | /// <summary> |
| | | 14 | | /// Model provider that downloads models from URLs with local caching. |
| | | 15 | | /// If the model already exists in the cache, returns the cached path immediately. |
| | | 16 | | /// Downloads are hardened (security assessment MEDIUM-1): model names are validated |
| | | 17 | | /// against path traversal, files are downloaded atomically via a temp file, an optional |
| | | 18 | | /// SHA-256 checksum is enforced, a size cap aborts runaway downloads, and redirects |
| | | 19 | | /// are capped. |
| | | 20 | | /// </summary> |
| | | 21 | | public class DownloadingModelProvider : IModelProvider |
| | | 22 | | { |
| | | 23 | | /// <summary>Default cap applied when <c>maxDownloadBytes</c> is not specified (2 GiB).</summary> |
| | | 24 | | public const long DefaultMaxDownloadBytes = 2L * 1024 * 1024 * 1024; |
| | | 25 | | |
| | | 26 | | private const int MaxAutomaticRedirections = 5; |
| | | 27 | | |
| | | 28 | | private readonly string _modelName; |
| | | 29 | | private readonly string[] _downloadUrls; |
| | | 30 | | private readonly string _cacheDirectory; |
| | | 31 | | private readonly string? _expectedSha256; |
| | | 32 | | private readonly long _maxDownloadBytes; |
| | | 33 | | private readonly ILogger _logger; |
| | | 34 | | private readonly HttpClient _http; |
| | 1 | 35 | | private static readonly HttpClient DefaultHttp = CreateHttpClient(); |
| | | 36 | | |
| | | 37 | | /// <summary> |
| | | 38 | | /// Creates a provider that downloads models from URLs with caching. |
| | | 39 | | /// </summary> |
| | | 40 | | /// <param name="modelName">Name of the model file (e.g., "yolov8n.onnx"). Must be a plain |
| | | 41 | | /// file name: directory separators, rooted paths, ".." segments, and invalid filename |
| | | 42 | | /// characters are rejected.</param> |
| | | 43 | | /// <param name="downloadUrls">One or more URLs to try downloading from (tried in order)</param> |
| | | 44 | | /// <param name="cacheDirectory">Directory to cache downloaded models. Defaults to local app data.</param> |
| | | 45 | | /// <param name="logger">Optional logger</param> |
| | | 46 | | /// <param name="sha256"> |
| | | 47 | | /// Optional expected SHA-256 of the model file as 64 lowercase hex characters. When set, the |
| | | 48 | | /// downloaded file is verified against it and a mismatch rejects (and deletes) the download. |
| | | 49 | | /// When null, the download is accepted but logged as unverified. |
| | | 50 | | /// </param> |
| | | 51 | | /// <param name="maxDownloadBytes"> |
| | | 52 | | /// Maximum allowed download size in bytes (validated against Content-Length and the running |
| | | 53 | | /// byte count). Must be positive; defaults to <see cref="DefaultMaxDownloadBytes"/> (2 GiB). |
| | | 54 | | /// </param> |
| | | 55 | | /// <param name="httpClient">Optional HTTP client to use (mainly for tests). Defaults to a |
| | | 56 | | /// shared client with automatic redirects capped at 5.</param> |
| | | 57 | | /// <exception cref="ArgumentNullException">A required argument is null.</exception> |
| | | 58 | | /// <exception cref="ArgumentException"> |
| | | 59 | | /// <paramref name="modelName"/> contains path traversal or invalid characters, no download |
| | | 60 | | /// URL is provided, <paramref name="sha256"/> is malformed, or |
| | | 61 | | /// <paramref name="maxDownloadBytes"/> is not positive. |
| | | 62 | | /// </exception> |
| | 46 | 63 | | public DownloadingModelProvider( |
| | 46 | 64 | | string modelName, |
| | 46 | 65 | | string[] downloadUrls, |
| | 46 | 66 | | string? cacheDirectory = null, |
| | 46 | 67 | | ILogger? logger = null, |
| | 46 | 68 | | string? sha256 = null, |
| | 46 | 69 | | long maxDownloadBytes = DefaultMaxDownloadBytes, |
| | 46 | 70 | | HttpClient? httpClient = null) |
| | | 71 | | { |
| | 46 | 72 | | _modelName = ValidateModelName(modelName); |
| | 35 | 73 | | _downloadUrls = downloadUrls ?? throw new ArgumentNullException(nameof(downloadUrls)); |
| | 34 | 74 | | _cacheDirectory = ValidateCacheDirectory(cacheDirectory ?? GetDefaultCacheDirectory()); |
| | 31 | 75 | | _expectedSha256 = ValidateSha256(sha256); |
| | 29 | 76 | | _maxDownloadBytes = maxDownloadBytes > 0 |
| | 29 | 77 | | ? maxDownloadBytes |
| | 29 | 78 | | : throw new ArgumentException("Maximum download size must be positive.", nameof(maxDownloadBytes)); |
| | 27 | 79 | | _logger = logger ?? NullLogger.Instance; |
| | 27 | 80 | | _http = httpClient ?? DefaultHttp; |
| | | 81 | | |
| | 27 | 82 | | if (_downloadUrls.Length == 0) |
| | | 83 | | { |
| | 1 | 84 | | throw new ArgumentException("At least one download URL must be provided", nameof(downloadUrls)); |
| | | 85 | | } |
| | 26 | 86 | | } |
| | | 87 | | |
| | | 88 | | /// <inheritdoc /> |
| | | 89 | | public async Task<string> GetModelPathAsync(CancellationToken cancellationToken = default) |
| | | 90 | | { |
| | 26 | 91 | | Directory.CreateDirectory(_cacheDirectory); |
| | 26 | 92 | | var modelPath = Path.Combine(_cacheDirectory, _modelName); |
| | 26 | 93 | | var tempPath = modelPath + ".downloading"; |
| | | 94 | | |
| | | 95 | | // Return cached model if available |
| | 26 | 96 | | if (File.Exists(modelPath)) |
| | | 97 | | { |
| | 6 | 98 | | _logger.LogDebug("Using cached model: {ModelPath}", modelPath); |
| | 6 | 99 | | return modelPath; |
| | | 100 | | } |
| | | 101 | | |
| | 20 | 102 | | _logger.LogInformation("Model not found in cache. Downloading {ModelName}...", _modelName); |
| | | 103 | | |
| | | 104 | | // Try each URL until one succeeds |
| | 20 | 105 | | Exception? lastError = null; |
| | 77 | 106 | | foreach (var url in _downloadUrls) |
| | | 107 | | { |
| | | 108 | | try |
| | | 109 | | { |
| | 23 | 110 | | _logger.LogDebug("Attempting download from: {Url}", url); |
| | 23 | 111 | | await DownloadModelAsync(url, tempPath, cancellationToken).ConfigureAwait(false); |
| | 9 | 112 | | File.Move(tempPath, modelPath, overwrite: true); |
| | 9 | 113 | | _logger.LogInformation("Successfully downloaded {ModelName} from {Url}", _modelName, url); |
| | 9 | 114 | | return modelPath; |
| | | 115 | | } |
| | 14 | 116 | | catch (Exception ex) |
| | | 117 | | { |
| | 14 | 118 | | DeleteFileBestEffort(tempPath); |
| | 14 | 119 | | lastError = ex; |
| | 14 | 120 | | _logger.LogWarning("Failed to download from {Url}: {Error}", url, ex.Message); |
| | | 121 | | // Try next URL |
| | 14 | 122 | | } |
| | 14 | 123 | | } |
| | | 124 | | |
| | 11 | 125 | | throw new InvalidOperationException( |
| | 11 | 126 | | $"Failed to download model '{_modelName}' from all sources. Last error: {lastError?.Message}", |
| | 11 | 127 | | lastError); |
| | 15 | 128 | | } |
| | | 129 | | |
| | | 130 | | private async Task DownloadModelAsync(string url, string tempPath, CancellationToken ct) |
| | | 131 | | { |
| | 23 | 132 | | using var response = await _http.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait( |
| | 17 | 133 | | response.EnsureSuccessStatusCode(); |
| | | 134 | | |
| | 14 | 135 | | if (response.Content.Headers.ContentLength is long declared && declared > _maxDownloadBytes) |
| | | 136 | | { |
| | 1 | 137 | | throw new InvalidOperationException( |
| | 1 | 138 | | $"Download from '{url}' declares {declared} bytes, which exceeds the maximum of {_maxDownloadBytes} |
| | | 139 | | } |
| | | 140 | | |
| | 13 | 141 | | await using var fs = new FileStream(tempPath, FileMode.Create, FileAccess.Write, FileShare.None); |
| | 13 | 142 | | await using var source = await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); |
| | | 143 | | |
| | 13 | 144 | | var buffer = new byte[81920]; |
| | 13 | 145 | | long total = 0; |
| | | 146 | | int read; |
| | 5238 | 147 | | while ((read = await source.ReadAsync(buffer.AsMemory(0, buffer.Length), ct).ConfigureAwait(false)) > 0) |
| | | 148 | | { |
| | 5226 | 149 | | total += read; |
| | 5226 | 150 | | if (total > _maxDownloadBytes) |
| | | 151 | | { |
| | 1 | 152 | | throw new InvalidOperationException( |
| | 1 | 153 | | $"Download from '{url}' exceeded the maximum of {_maxDownloadBytes} bytes."); |
| | | 154 | | } |
| | | 155 | | |
| | 5225 | 156 | | await fs.WriteAsync(buffer.AsMemory(0, read), ct).ConfigureAwait(false); |
| | | 157 | | } |
| | | 158 | | |
| | 10 | 159 | | if (_expectedSha256 is null) |
| | | 160 | | { |
| | 8 | 161 | | _logger.LogWarning( |
| | 8 | 162 | | "Model '{ModelName}' downloaded from '{Url}' without integrity verification: no sha256 is configured |
| | 8 | 163 | | _modelName, url); |
| | 8 | 164 | | return; |
| | | 165 | | } |
| | | 166 | | |
| | 2 | 167 | | fs.Close(); |
| | 2 | 168 | | var actual = await ComputeFileSha256Async(tempPath, ct).ConfigureAwait(false); |
| | 2 | 169 | | if (!string.Equals(actual, _expectedSha256, StringComparison.OrdinalIgnoreCase)) |
| | | 170 | | { |
| | 1 | 171 | | throw new InvalidOperationException( |
| | 1 | 172 | | $"SHA-256 mismatch for model '{_modelName}' downloaded from '{url}': " + |
| | 1 | 173 | | $"expected {_expectedSha256}, got {actual}."); |
| | | 174 | | } |
| | | 175 | | |
| | 1 | 176 | | _logger.LogInformation("SHA-256 verified for model '{ModelName}' downloaded from '{Url}'.", _modelName, url) |
| | 9 | 177 | | } |
| | | 178 | | |
| | | 179 | | private static async Task<string> ComputeFileSha256Async(string path, CancellationToken ct) |
| | | 180 | | { |
| | 2 | 181 | | await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); |
| | 2 | 182 | | using var sha = SHA256.Create(); |
| | 2 | 183 | | var hash = await sha.ComputeHashAsync(stream, ct).ConfigureAwait(false); |
| | 2 | 184 | | var sb = new StringBuilder(hash.Length * 2); |
| | 132 | 185 | | foreach (var b in hash) |
| | | 186 | | { |
| | 64 | 187 | | sb.Append(b.ToString("x2")); |
| | | 188 | | } |
| | | 189 | | |
| | 2 | 190 | | return sb.ToString(); |
| | 2 | 191 | | } |
| | | 192 | | |
| | | 193 | | private static HttpClient CreateHttpClient() |
| | | 194 | | { |
| | 1 | 195 | | var handler = new HttpClientHandler |
| | 1 | 196 | | { |
| | 1 | 197 | | MaxAutomaticRedirections = MaxAutomaticRedirections, |
| | 1 | 198 | | AllowAutoRedirect = true, |
| | 1 | 199 | | }; |
| | 1 | 200 | | return new HttpClient(handler); |
| | | 201 | | } |
| | | 202 | | |
| | | 203 | | private static string ValidateModelName(string? modelName) |
| | | 204 | | { |
| | 46 | 205 | | if (modelName is null) |
| | | 206 | | { |
| | 1 | 207 | | throw new ArgumentNullException(nameof(modelName)); |
| | | 208 | | } |
| | | 209 | | |
| | 45 | 210 | | if (modelName.Length == 0) |
| | | 211 | | { |
| | 1 | 212 | | throw new ArgumentException("Model name must not be empty.", nameof(modelName)); |
| | | 213 | | } |
| | | 214 | | |
| | 44 | 215 | | if (modelName.IndexOf('/') >= 0 || modelName.IndexOf('\\') >= 0) |
| | | 216 | | { |
| | 6 | 217 | | throw new ArgumentException( |
| | 6 | 218 | | $"Model name '{modelName}' must not contain directory separators.", nameof(modelName)); |
| | | 219 | | } |
| | | 220 | | |
| | 38 | 221 | | if (Path.IsPathRooted(modelName)) |
| | | 222 | | { |
| | 0 | 223 | | throw new ArgumentException( |
| | 0 | 224 | | $"Model name '{modelName}' must be a plain file name, not a rooted path.", nameof(modelName)); |
| | | 225 | | } |
| | | 226 | | |
| | 38 | 227 | | if (modelName == ".." || modelName.Contains("..", StringComparison.Ordinal)) |
| | | 228 | | { |
| | 2 | 229 | | throw new ArgumentException( |
| | 2 | 230 | | $"Model name '{modelName}' must not contain '..' segments.", nameof(modelName)); |
| | | 231 | | } |
| | | 232 | | |
| | 36 | 233 | | if (modelName.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) |
| | | 234 | | { |
| | 1 | 235 | | throw new ArgumentException( |
| | 1 | 236 | | $"Model name '{modelName}' contains invalid filename characters.", nameof(modelName)); |
| | | 237 | | } |
| | | 238 | | |
| | 35 | 239 | | return modelName; |
| | | 240 | | } |
| | | 241 | | |
| | | 242 | | private static string ValidateCacheDirectory(string cacheDirectory) |
| | | 243 | | { |
| | 34 | 244 | | if (string.IsNullOrWhiteSpace(cacheDirectory)) |
| | | 245 | | { |
| | 1 | 246 | | throw new ArgumentException("Cache directory must not be empty or whitespace.", nameof(cacheDirectory)); |
| | | 247 | | } |
| | | 248 | | |
| | 33 | 249 | | if (cacheDirectory == ".." || cacheDirectory.Contains("..", StringComparison.Ordinal)) |
| | | 250 | | { |
| | 1 | 251 | | throw new ArgumentException( |
| | 1 | 252 | | $"Cache directory '{cacheDirectory}' must not contain '..' segments.", nameof(cacheDirectory)); |
| | | 253 | | } |
| | | 254 | | |
| | 32 | 255 | | if (cacheDirectory.IndexOfAny(Path.GetInvalidPathChars()) >= 0) |
| | | 256 | | { |
| | 1 | 257 | | throw new ArgumentException( |
| | 1 | 258 | | $"Cache directory '{cacheDirectory}' contains invalid path characters.", nameof(cacheDirectory)); |
| | | 259 | | } |
| | | 260 | | |
| | 31 | 261 | | return cacheDirectory; |
| | | 262 | | } |
| | | 263 | | |
| | | 264 | | private static string? ValidateSha256(string? sha256) |
| | | 265 | | { |
| | 31 | 266 | | if (sha256 is null) |
| | | 267 | | { |
| | 27 | 268 | | return null; |
| | | 269 | | } |
| | | 270 | | |
| | 4 | 271 | | if (sha256.Length != 64) |
| | | 272 | | { |
| | 1 | 273 | | throw new ArgumentException("sha256 must be exactly 64 hex characters.", nameof(sha256)); |
| | | 274 | | } |
| | | 275 | | |
| | 263 | 276 | | foreach (var c in sha256) |
| | | 277 | | { |
| | 129 | 278 | | if (!Uri.IsHexDigit(c)) |
| | | 279 | | { |
| | 1 | 280 | | throw new ArgumentException("sha256 must contain only hex characters.", nameof(sha256)); |
| | | 281 | | } |
| | | 282 | | } |
| | | 283 | | |
| | 2 | 284 | | return sha256.ToLowerInvariant(); |
| | | 285 | | } |
| | | 286 | | |
| | | 287 | | private void DeleteFileBestEffort(string path) |
| | | 288 | | { |
| | | 289 | | try |
| | | 290 | | { |
| | 14 | 291 | | if (File.Exists(path)) |
| | | 292 | | { |
| | 4 | 293 | | File.Delete(path); |
| | | 294 | | } |
| | 13 | 295 | | } |
| | 1 | 296 | | catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) |
| | | 297 | | { |
| | 1 | 298 | | _logger.LogWarning(ex, "Failed to delete temporary download file '{Path}'.", path); |
| | 1 | 299 | | } |
| | 14 | 300 | | } |
| | | 301 | | |
| | | 302 | | private static string GetDefaultCacheDirectory() |
| | | 303 | | { |
| | 1 | 304 | | return Path.Combine( |
| | 1 | 305 | | Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), |
| | 1 | 306 | | "InferenceEngine", |
| | 1 | 307 | | "Models"); |
| | | 308 | | } |
| | | 309 | | } |
| | | 310 | | } |