| | | 1 | | using System.Collections.Immutable; |
| | | 2 | | using System.Diagnostics; |
| | | 3 | | using System.Runtime.InteropServices; |
| | | 4 | | using System.Threading; |
| | | 5 | | using Microsoft.ML.OnnxRuntime; |
| | | 6 | | using Microsoft.Extensions.Logging; |
| | | 7 | | using Microsoft.Extensions.Logging.Abstractions; |
| | | 8 | | using InferenceEngine.Core.Diagnostics; |
| | | 9 | | using InferenceEngine.Core.ModelProviders; |
| | | 10 | | |
| | | 11 | | namespace InferenceEngine.Core |
| | | 12 | | { |
| | | 13 | | /// <summary> |
| | | 14 | | /// The core abstract base class that handles ONNX Runtime session management, |
| | | 15 | | /// cross-platform hardware acceleration detection, and resource cleanup. |
| | | 16 | | /// Supports state passing between Pre and Post processing steps via a context object. |
| | | 17 | | /// </summary> |
| | | 18 | | public abstract class BaseInferenceEngine<TInput, TOutput> : IInferenceEngine<TInput, TOutput> |
| | | 19 | | { |
| | | 20 | | protected readonly InferenceEngineOptions _options; |
| | | 21 | | protected readonly ILogger _logger; |
| | | 22 | | protected InferenceSession? _session; |
| | | 23 | | protected RunOptions? _runOptions; |
| | | 24 | | |
| | | 25 | | private bool _isLoaded = false; |
| | 173 | 26 | | private readonly SemaphoreSlim _loadLock = new(1, 1); |
| | | 27 | | private IModelProvider? _modelProvider; |
| | 173 | 28 | | private List<ProviderDiagnosticCheck> _pendingChecks = new(); |
| | | 29 | | private int _nextOrder = 0; |
| | | 30 | | |
| | 126 | 31 | | public IReadOnlyDictionary<string, NodeMetadata>? InputMetadata { get; private set; } |
| | 96 | 32 | | public IReadOnlyDictionary<string, NodeMetadata>? OutputMetadata { get; private set; } |
| | 148 | 33 | | public string? PrimaryInputName { get; private set; } |
| | | 34 | | |
| | | 35 | | /// <summary> |
| | | 36 | | /// Returns <see langword="true"/> when the ONNX session was successfully initialized |
| | | 37 | | /// with the MIGraphX execution provider on AMD ROCm hardware. <see langword="false"/> on non-Linux-x64 platform |
| | | 38 | | /// when MIGraphX initialization failed, or when CPU fallback was used. |
| | | 39 | | /// </summary> |
| | 51 | 40 | | public bool IsRocmSessionActive { get; private set; } |
| | | 41 | | |
| | | 42 | | /// <summary> |
| | | 43 | | /// Snapshot of the engine's loaded state. Returns <see cref="InferenceEngineInfo.Empty"/> |
| | | 44 | | /// until <see cref="LoadAsync"/> completes. Thread-safe: reference assignment is atomic |
| | | 45 | | /// and <see cref="InferenceEngineInfo"/> is an immutable record. |
| | | 46 | | /// </summary> |
| | 457 | 47 | | public InferenceEngineInfo EngineInfo { get; private set; } = InferenceEngineInfo.Empty; |
| | | 48 | | |
| | | 49 | | /// <summary> |
| | | 50 | | /// Lifecycle + runtime snapshot (<see cref="EngineLoadState"/> + partial-or-full |
| | | 51 | | /// <see cref="InferenceEngineInfo"/> + load error). Transitions fire |
| | | 52 | | /// <see cref="InferenceEngineOptions.OnStatusChanged"/> when set. Out-of-process proxies |
| | | 53 | | /// mirror the worker's terminal states onto the same property. |
| | | 54 | | /// </summary> |
| | 714 | 55 | | public InferenceEngineStatus Status { get; private set; } = InferenceEngineStatus.NotLoaded; |
| | | 56 | | |
| | | 57 | | // --- Model Specification --- |
| | | 58 | | |
| | | 59 | | /// <summary> |
| | | 60 | | /// The name of the model file (e.g., "yolo11n.onnx"). |
| | | 61 | | /// Derived classes must implement this to specify which model they require. |
| | | 62 | | /// </summary> |
| | | 63 | | protected abstract string ModelName { get; } |
| | | 64 | | |
| | | 65 | | /// <summary> |
| | | 66 | | /// URLs where the model can be downloaded from, in priority order. |
| | | 67 | | /// Derived classes can override to provide custom model sources. |
| | | 68 | | /// If not overridden, the base engine will use the default ModelRegistry to find URLs. |
| | | 69 | | /// </summary> |
| | 3 | 70 | | protected virtual string[]? ModelDownloadUrls => null; |
| | | 71 | | |
| | | 72 | | /// <summary> |
| | | 73 | | /// Model registry for looking up download URLs by model name. |
| | | 74 | | /// Override to use a custom registry; defaults to the built-in registry. |
| | | 75 | | /// </summary> |
| | 3 | 76 | | protected virtual ModelRegistry ModelRegistry => _lazyRegistry.Value; |
| | | 77 | | |
| | | 78 | | private readonly Lazy<ModelRegistry> _lazyRegistry; |
| | | 79 | | |
| | | 80 | | /// <summary> |
| | | 81 | | /// Base directory for model cache. Defaults to LocalApplicationData. |
| | | 82 | | /// Derived classes can override to use a custom cache location. |
| | | 83 | | /// </summary> |
| | 4 | 84 | | protected virtual string ModelCacheDirectory => Path.Combine( |
| | 4 | 85 | | Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), |
| | 4 | 86 | | "InferenceEngine", |
| | 4 | 87 | | "Models"); |
| | | 88 | | |
| | 173 | 89 | | protected BaseInferenceEngine(InferenceEngineOptions options, ILogger? logger = null) |
| | | 90 | | { |
| | 173 | 91 | | _options = options ?? throw new ArgumentNullException(nameof(options)); |
| | 171 | 92 | | _logger = logger ?? NullLogger.Instance; |
| | 174 | 93 | | _lazyRegistry = new Lazy<ModelRegistry>(() => new ModelRegistry(logger: _logger), LazyThreadSafetyMode.Execu |
| | 171 | 94 | | } |
| | | 95 | | |
| | | 96 | | // --- Abstract Methods --- |
| | | 97 | | |
| | | 98 | | /// <summary> |
| | | 99 | | /// Takes raw input and returns ONNX tensors plus an optional context object. |
| | | 100 | | /// The context object is passed to PostProcessWithContext, allowing you to share state |
| | | 101 | | /// without overriding the main pipeline. |
| | | 102 | | /// </summary> |
| | | 103 | | protected abstract (List<NamedOnnxValue> InputTensors, object? Context) PreProcessWithContext(TInput input); |
| | | 104 | | |
| | | 105 | | /// <summary> |
| | | 106 | | /// Takes raw ONNX output and the context object from PreProcess, returning the final result. |
| | | 107 | | /// </summary> |
| | | 108 | | protected abstract TOutput PostProcessWithContext(IDisposableReadOnlyCollection<DisposableNamedOnnxValue> output |
| | | 109 | | |
| | | 110 | | // --- Public API --- |
| | | 111 | | |
| | | 112 | | /// <summary> |
| | | 113 | | /// Gets or creates the model provider. |
| | | 114 | | /// Uses options.ModelProvider if set, otherwise creates from ModelName/ModelDownloadUrls/ModelRegistry. |
| | | 115 | | /// </summary> |
| | | 116 | | private IModelProvider GetModelProvider() |
| | | 117 | | { |
| | 60 | 118 | | if (_options.ModelProvider is not null) |
| | | 119 | | { |
| | 53 | 120 | | return _options.ModelProvider; |
| | | 121 | | } |
| | | 122 | | |
| | 7 | 123 | | if (_modelProvider is null) |
| | | 124 | | { |
| | 7 | 125 | | var urls = ModelDownloadUrls; |
| | 7 | 126 | | string? expectedSha256 = null; |
| | | 127 | | |
| | 7 | 128 | | if (urls is null || urls.Length == 0) |
| | | 129 | | { |
| | 5 | 130 | | var entry = ModelRegistry.GetModelEntry(ModelName); |
| | 5 | 131 | | urls = entry?.Urls; |
| | | 132 | | |
| | 5 | 133 | | if (urls is null || urls.Length == 0) |
| | | 134 | | { |
| | 4 | 135 | | throw new InvalidOperationException( |
| | 4 | 136 | | $"Engine '{GetType().Name}' does not specify ModelDownloadUrls and model '{ModelName}' " + |
| | 4 | 137 | | $"was not found in the ModelRegistry. Either: " + |
| | 4 | 138 | | $"1) Override ModelDownloadUrls in your engine class, " + |
| | 4 | 139 | | $"2) Add '{ModelName}' to the model registry (models.json), or " + |
| | 4 | 140 | | $"3) Set {nameof(InferenceEngineOptions)}.{nameof(InferenceEngineOptions.ModelProvider)} exp |
| | | 141 | | } |
| | | 142 | | |
| | 1 | 143 | | expectedSha256 = entry!.Sha256; |
| | 1 | 144 | | _logger.LogInformation("Found model '{ModelName}' in registry with {Count} URL(s)", ModelName, urls. |
| | | 145 | | } |
| | | 146 | | |
| | 3 | 147 | | _modelProvider = new DownloadingModelProvider( |
| | 3 | 148 | | modelName: ModelName, |
| | 3 | 149 | | downloadUrls: urls, |
| | 3 | 150 | | cacheDirectory: ModelCacheDirectory, |
| | 3 | 151 | | logger: _logger, |
| | 3 | 152 | | sha256: expectedSha256, |
| | 3 | 153 | | maxDownloadBytes: _options.MaxModelDownloadBytes); |
| | | 154 | | } |
| | | 155 | | |
| | 3 | 156 | | return _modelProvider; |
| | | 157 | | } |
| | | 158 | | |
| | | 159 | | /// <summary> |
| | | 160 | | /// Initializes the inference session with the best available hardware provider. |
| | | 161 | | /// </summary> |
| | | 162 | | public virtual async Task LoadAsync() |
| | | 163 | | { |
| | | 164 | | // Serialize concurrent loads (H5): two concurrent LoadAsync calls must not both pass the |
| | | 165 | | // _isLoaded guard and double-construct the session / corrupt the check list. |
| | 67 | 166 | | await _loadLock.WaitAsync().ConfigureAwait(false); |
| | | 167 | | try |
| | | 168 | | { |
| | 70 | 169 | | if (_isLoaded) return; |
| | | 170 | | |
| | 60 | 171 | | SetStatus(new InferenceEngineStatus { State = EngineLoadState.Loading, Detail = "Resolving model…" }); |
| | | 172 | | try |
| | | 173 | | { |
| | | 174 | | |
| | 60 | 175 | | _logger.LogInformation("Initializing Inference Engine (engine={EngineType}, gpu={UseGpu}, deviceId={DeviceId |
| | 60 | 176 | | GetType().Name, _options.UseGpuAcceleration, _options.DeviceId); |
| | | 177 | | |
| | 60 | 178 | | var modelPath = await GetModelProvider().GetModelPathAsync(CancellationToken.None); |
| | 53 | 179 | | _logger.LogInformation("Model resolved to '{ModelPath}' (exists={Exists})", modelPath, File.Exists(modelPath |
| | | 180 | | |
| | 53 | 181 | | SetStatus(new InferenceEngineStatus { State = EngineLoadState.Loading, Detail = "Selecting execution provide |
| | | 182 | | |
| | 53 | 183 | | await Task.Run(() => |
| | 53 | 184 | | { |
| | 53 | 185 | | try |
| | 53 | 186 | | { |
| | 53 | 187 | | _pendingChecks = new List<ProviderDiagnosticCheck>(); |
| | 53 | 188 | | _nextOrder = 0; |
| | 53 | 189 | | |
| | 53 | 190 | | InferenceProvider provider = InferenceProvider.Unknown; |
| | 53 | 191 | | InferenceFallbackReason fallback; |
| | 53 | 192 | | int deviceId; |
| | 53 | 193 | | |
| | 53 | 194 | | SessionOptions? sessionOptions = null; |
| | 53 | 195 | | |
| | 53 | 196 | | try |
| | 53 | 197 | | { |
| | 53 | 198 | | sessionOptions = ConfiguredSessionOptions(out provider, out fallback, out deviceId); |
| | 53 | 199 | | |
| | 53 | 200 | | // Session construction is where a hardware EP compiles the model's |
| | 53 | 201 | | // kernels — including the multi-minute MIGraphX first-run compile. |
| | 53 | 202 | | SetStatus(new InferenceEngineStatus { State = EngineLoadState.Loading, Detail = "Constructing in |
| | 53 | 203 | | |
| | 53 | 204 | | try |
| | 53 | 205 | | { |
| | 53 | 206 | | _session = CreateInferenceSession(modelPath, sessionOptions); |
| | 48 | 207 | | if (fallback != InferenceFallbackReason.None) |
| | 53 | 208 | | { |
| | 39 | 209 | | AddCheck(ProviderDiagnosticCheck.Pass("cpu.fallback-options", "CreateCpuOnlySessionOptio |
| | 39 | 210 | | AddCheck(ProviderDiagnosticCheck.Pass("cpu.fallback-session", "CPU InferenceSession cons |
| | 53 | 211 | | } |
| | 53 | 212 | | else |
| | 53 | 213 | | { |
| | 9 | 214 | | AppendSessionCheck(provider, passed: true); |
| | 53 | 215 | | } |
| | 48 | 216 | | } |
| | 5 | 217 | | catch (Exception ex) |
| | 53 | 218 | | { |
| | 5 | 219 | | AppendSessionCheck(provider, passed: false, ex.Message); |
| | 5 | 220 | | throw; |
| | 53 | 221 | | } |
| | 48 | 222 | | } |
| | 5 | 223 | | catch (Exception ex) when ( |
| | 5 | 224 | | _options.UseGpuAcceleration && |
| | 5 | 225 | | provider != InferenceProvider.Cpu && |
| | 5 | 226 | | IsRecoverableNativeFailure(ex)) |
| | 53 | 227 | | { |
| | 3 | 228 | | _logger.LogWarning(ex, "Native session initialization failed. Falling back to CPU."); |
| | 3 | 229 | | AddCheck(ProviderDiagnosticCheck.Pass( |
| | 3 | 230 | | "session.native-recoverable", |
| | 3 | 231 | | "IsRecoverableNativeFailure detected, attempting CPU fallback", |
| | 3 | 232 | | "Fallback", |
| | 3 | 233 | | NextOrder())); |
| | 53 | 234 | | |
| | 3 | 235 | | sessionOptions?.Dispose(); |
| | 3 | 236 | | fallback = InferenceFallbackReason.NativeLibraryLoadFailed; |
| | 3 | 237 | | provider = InferenceProvider.Cpu; |
| | 3 | 238 | | deviceId = 0; |
| | 53 | 239 | | |
| | 3 | 240 | | sessionOptions = CreateCpuOnlySessionOptions(); |
| | 3 | 241 | | AppendFallbackChecks(sessionOptions, modelPath); |
| | 3 | 242 | | } |
| | 53 | 243 | | finally |
| | 53 | 244 | | { |
| | 53 | 245 | | sessionOptions?.Dispose(); |
| | 53 | 246 | | } |
| | 53 | 247 | | |
| | 51 | 248 | | _runOptions = new RunOptions(); |
| | 53 | 249 | | |
| | 51 | 250 | | InputMetadata = _session!.InputMetadata; |
| | 51 | 251 | | OutputMetadata = _session!.OutputMetadata; |
| | 51 | 252 | | PrimaryInputName = _session!.InputMetadata.Keys.FirstOrDefault(); |
| | 53 | 253 | | |
| | 51 | 254 | | IsRocmSessionActive = provider == InferenceProvider.MIGraphX; |
| | 53 | 255 | | |
| | 51 | 256 | | var info = new InferenceEngineInfo |
| | 51 | 257 | | { |
| | 51 | 258 | | Provider = provider, |
| | 51 | 259 | | Platform = HostPlatformDetector.Current(), |
| | 51 | 260 | | DeviceId = deviceId, |
| | 51 | 261 | | GpuAccelerationRequested = _options.UseGpuAcceleration, |
| | 51 | 262 | | ModelName = ModelName, |
| | 51 | 263 | | ModelPath = modelPath, |
| | 51 | 264 | | ModelSource = _options.ModelProvider is not null |
| | 51 | 265 | | ? Diagnostics.ModelSource.ExplicitPath |
| | 51 | 266 | | : Diagnostics.ModelSource.RegistryManaged, |
| | 51 | 267 | | IsFallback = fallback != InferenceFallbackReason.None, |
| | 51 | 268 | | FallbackReason = fallback, |
| | 51 | 269 | | ProviderOptions = ExtractProviderOptions(provider, deviceId), |
| | 51 | 270 | | SessionCreatedAt = DateTimeOffset.UtcNow, |
| | 51 | 271 | | Checks = _pendingChecks.ToArray(), |
| | 51 | 272 | | }; |
| | 53 | 273 | | |
| | 53 | 274 | | #if DEBUG |
| | 53 | 275 | | for (int i = 0; i < info.Checks.Count; i++) |
| | 53 | 276 | | Debug.Assert(info.Checks[i].Order == i, $"Check order mismatch at index {i}: expected {i}, got { |
| | 53 | 277 | | #endif |
| | 53 | 278 | | |
| | 51 | 279 | | EngineInfo = info; |
| | 51 | 280 | | SetStatus(new InferenceEngineStatus { State = EngineLoadState.Loaded, Info = info }); |
| | 53 | 281 | | |
| | 51 | 282 | | _logger.LogInformation( |
| | 51 | 283 | | "Model loaded with {Provider} provider (fallback={IsFallback}, reason={FallbackReason}). Primary |
| | 51 | 284 | | provider, info.IsFallback, fallback, PrimaryInputName); |
| | 51 | 285 | | _isLoaded = true; |
| | 53 | 286 | | |
| | 53 | 287 | | // M4: fire the session-initialized callback BEFORE warmup (it documents that it |
| | 53 | 288 | | // "fires once after the session is created"), so a long first-compile warmup does |
| | 53 | 289 | | // not delay it and a warmup problem cannot prevent it. |
| | 51 | 290 | | FireOnSessionInitialized(info); |
| | 53 | 291 | | |
| | 51 | 292 | | if (_options.WarmupOnLoad) |
| | 53 | 293 | | { |
| | 3 | 294 | | PerformWarmup(); |
| | 53 | 295 | | } |
| | 51 | 296 | | } |
| | 2 | 297 | | catch (Exception ex) |
| | 53 | 298 | | { |
| | 2 | 299 | | _logger.LogError(ex, "Failed to initialize Inference Engine."); |
| | 2 | 300 | | throw; |
| | 53 | 301 | | } |
| | 104 | 302 | | }); |
| | 51 | 303 | | } |
| | 9 | 304 | | catch (Exception ex) |
| | | 305 | | { |
| | | 306 | | // Surface the failure through the status object model: the partial |
| | | 307 | | // provider-selection trail (checks accumulated before the failure) explains |
| | | 308 | | // WHY, and the error string says WHAT threw. |
| | 9 | 309 | | SetStatus(new InferenceEngineStatus |
| | 9 | 310 | | { |
| | 9 | 311 | | State = EngineLoadState.Failed, |
| | 9 | 312 | | Info = new InferenceEngineInfo |
| | 9 | 313 | | { |
| | 9 | 314 | | Provider = InferenceProvider.Unknown, |
| | 9 | 315 | | Checks = _pendingChecks.ToArray(), |
| | 9 | 316 | | }, |
| | 9 | 317 | | LoadError = ex.Message, |
| | 9 | 318 | | }); |
| | 9 | 319 | | throw; |
| | | 320 | | } |
| | 51 | 321 | | } |
| | | 322 | | finally |
| | | 323 | | { |
| | 65 | 324 | | _loadLock.Release(); |
| | | 325 | | } |
| | 56 | 326 | | } |
| | | 327 | | |
| | | 328 | | public virtual async Task<TOutput> PredictAsync(TInput input) |
| | | 329 | | { |
| | 5 | 330 | | if (!_isLoaded) await LoadAsync(); |
| | | 331 | | |
| | 3 | 332 | | return await Task.Run(() => |
| | 3 | 333 | | { |
| | 3 | 334 | | var (inputTensors, context) = PreProcessWithContext(input); |
| | 3 | 335 | | |
| | 3 | 336 | | try |
| | 3 | 337 | | { |
| | 3 | 338 | | using var outputTensors = _session!.Run(inputTensors, _session.OutputNames, _runOptions); |
| | 3 | 339 | | return PostProcessWithContext(outputTensors, context); |
| | 3 | 340 | | } |
| | 3 | 341 | | finally |
| | 3 | 342 | | { |
| | 24 | 343 | | foreach(var t in inputTensors) |
| | 3 | 344 | | { |
| | 9 | 345 | | if (t is IDisposable disposable) |
| | 3 | 346 | | { |
| | 0 | 347 | | disposable.Dispose(); |
| | 3 | 348 | | } |
| | 3 | 349 | | } |
| | 3 | 350 | | } |
| | 6 | 351 | | }); |
| | 3 | 352 | | } |
| | | 353 | | |
| | | 354 | | // --- Internal/Protected Helpers --- |
| | | 355 | | |
| | | 356 | | // Test seams (T1/T2): virtualize the platform/probe/append/session-construction steps so the |
| | | 357 | | // most complex fallback branches can be exercised hermetically (on any CI host, with no GPU). |
| | | 358 | | // Defaults reproduce the original behavior exactly. |
| | | 359 | | |
| | | 360 | | /// <summary>Platform used to select the execution-provider branch. Override to force a branch in tests.</summar |
| | 2 | 361 | | protected virtual HostPlatform EffectivePlatform => HostPlatformDetector.Current(); |
| | | 362 | | |
| | | 363 | | /// <summary>True when the ROCm KFD device node is present. Override in tests.</summary> |
| | 2 | 364 | | protected virtual bool IsKfdPresent() => File.Exists("/dev/kfd"); |
| | | 365 | | |
| | | 366 | | /// <summary>Attempts to dlopen a native library; returns success and an error message on failure.</summary> |
| | | 367 | | protected virtual bool TryLoadNativeLibrary(string name, out string? error) |
| | | 368 | | { |
| | | 369 | | try |
| | | 370 | | { |
| | 1 | 371 | | bool loaded = NativeLibrary.TryLoad(name, typeof(InferenceSession).Assembly, |
| | 1 | 372 | | DllImportSearchPath.UseDllDirectoryForDependencies | DllImportSearchPath.System32, out var handle); |
| | 1 | 373 | | if (loaded && handle != IntPtr.Zero) NativeLibrary.Free(handle); |
| | 1 | 374 | | error = loaded ? null : "NativeLibrary.TryLoad returned false"; |
| | 1 | 375 | | return loaded; |
| | | 376 | | } |
| | 0 | 377 | | catch (Exception ex) |
| | | 378 | | { |
| | 0 | 379 | | error = ex.Message; |
| | 0 | 380 | | return false; |
| | | 381 | | } |
| | 1 | 382 | | } |
| | | 383 | | |
| | | 384 | | /// <summary> |
| | | 385 | | /// Appends the MIGraphX execution provider with the compiled-model (.mxr) cache |
| | | 386 | | /// directory resolved by <see cref="ResolveMIGraphXModelCacheDirectory"/>. The cache |
| | | 387 | | /// persists compiled MIGraphX programs across process restarts; without it the ORT |
| | | 388 | | /// provider compiles the full model on every session construction (an empty cache |
| | | 389 | | /// path explicitly means "always compile" in the EP). Override to inject a throw in tests. |
| | | 390 | | /// </summary> |
| | | 391 | | protected virtual void AppendMIGraphXProvider(SessionOptions options, int deviceId) |
| | | 392 | | { |
| | 1 | 393 | | var providerOptions = new Dictionary<string, string> |
| | 1 | 394 | | { |
| | 1 | 395 | | ["device_id"] = deviceId.ToString(), |
| | 1 | 396 | | }; |
| | | 397 | | |
| | 1 | 398 | | var cacheDir = ResolveMIGraphXModelCacheDirectory(); |
| | 1 | 399 | | if (cacheDir is not null) |
| | | 400 | | { |
| | | 401 | | try |
| | | 402 | | { |
| | 1 | 403 | | Directory.CreateDirectory(cacheDir); |
| | 0 | 404 | | providerOptions["migraphx_model_cache_dir"] = cacheDir; |
| | 0 | 405 | | _logger.LogInformation( |
| | 0 | 406 | | "MIGraphX compiled-model cache: {CacheDir} (first load compiles ~1-2 min; later loads reuse the |
| | 0 | 407 | | cacheDir); |
| | 0 | 408 | | } |
| | 1 | 409 | | catch (Exception ex) |
| | | 410 | | { |
| | | 411 | | // The cache is an optimization: a host that cannot host the cache directory |
| | | 412 | | // still gets GPU inference, just with a full recompile on every startup. |
| | 1 | 413 | | _logger.LogWarning(ex, |
| | 1 | 414 | | "Cannot create MIGraphX model cache directory {CacheDir}; continuing without .mxr caching (every |
| | 1 | 415 | | cacheDir); |
| | 1 | 416 | | } |
| | | 417 | | } |
| | | 418 | | |
| | 1 | 419 | | options.AppendExecutionProvider("MIGraphX", providerOptions); |
| | 0 | 420 | | } |
| | | 421 | | |
| | | 422 | | /// <summary> |
| | | 423 | | /// Resolves the MIGraphX compiled-model (.mxr) cache directory: an explicitly configured |
| | | 424 | | /// <see cref="InferenceEngineOptions.MIGraphXModelCacheDirectory"/> wins (empty string = |
| | | 425 | | /// disabled), otherwise a per-user default under the same root as |
| | | 426 | | /// <see cref="ModelCacheDirectory"/>. Returns <see langword="null"/> when caching is |
| | | 427 | | /// disabled or the host has no user-profile directory to anchor the default. |
| | | 428 | | /// Pure path computation — directory creation happens in |
| | | 429 | | /// <see cref="AppendMIGraphXProvider"/> only. |
| | | 430 | | /// </summary> |
| | | 431 | | protected virtual string? ResolveMIGraphXModelCacheDirectory() |
| | | 432 | | { |
| | 9 | 433 | | var configured = _options.MIGraphXModelCacheDirectory; |
| | 9 | 434 | | if (configured is not null) |
| | 5 | 435 | | return configured.Length == 0 ? null : configured; |
| | | 436 | | |
| | 4 | 437 | | var root = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); |
| | 4 | 438 | | return string.IsNullOrEmpty(root) |
| | 4 | 439 | | ? null |
| | 4 | 440 | | : Path.Combine(root, "InferenceEngine", "MIGraphXModelCache"); |
| | | 441 | | } |
| | | 442 | | |
| | | 443 | | /// <summary>Appends the DirectML execution provider. Override to inject a throw in tests.</summary> |
| | | 444 | | protected virtual void AppendDirectMlProvider(SessionOptions options, int deviceId) |
| | 1 | 445 | | => options.AppendExecutionProvider_DML(deviceId); |
| | | 446 | | |
| | | 447 | | /// <summary>Appends the CoreML execution provider. Override to inject a throw in tests.</summary> |
| | | 448 | | protected virtual void AppendCoreMlProvider(SessionOptions options, CoreMLFlags flags) |
| | 1 | 449 | | => options.AppendExecutionProvider_CoreML(flags); |
| | | 450 | | |
| | | 451 | | /// <summary>Constructs the ONNX session. Override to inject a recoverable native failure in tests.</summary> |
| | | 452 | | protected virtual InferenceSession CreateInferenceSession(string modelPath, SessionOptions options) |
| | 53 | 453 | | => new(modelPath, options); |
| | | 454 | | |
| | 369 | 455 | | private int NextOrder() => _nextOrder++; |
| | | 456 | | |
| | | 457 | | /// <summary> |
| | | 458 | | /// Single choke point for appending to the provider-selection check trail: keeps the list |
| | | 459 | | /// behavior identical while mirroring every check to the logger, so the forensic trail |
| | | 460 | | /// (which probe ran, what it found) lands in the log file even before EngineInfo reaches a |
| | | 461 | | /// consumer. Failed checks escalate to Warning; passes/skips log at Debug. |
| | | 462 | | /// </summary> |
| | | 463 | | private void AddCheck(ProviderDiagnosticCheck check) |
| | | 464 | | { |
| | 369 | 465 | | _pendingChecks.Add(check); |
| | 369 | 466 | | _logger.Log( |
| | 369 | 467 | | check.Result == Diagnostics.CheckResult.Failed ? LogLevel.Warning : LogLevel.Debug, |
| | 369 | 468 | | "check [{Order}] {Result} {Name} ({Category}): {Detail}", |
| | 369 | 469 | | check.Order, check.Result, check.Name, check.Category, check.Detail ?? check.Description); |
| | 369 | 470 | | } |
| | | 471 | | |
| | | 472 | | private SessionOptions ConfiguredSessionOptions( |
| | | 473 | | out InferenceProvider provider, |
| | | 474 | | out InferenceFallbackReason fallback, |
| | | 475 | | out int deviceId) |
| | | 476 | | { |
| | 54 | 477 | | if (!_options.UseGpuAcceleration) |
| | | 478 | | { |
| | 20 | 479 | | AddCheck(ProviderDiagnosticCheck.Fail( |
| | 20 | 480 | | "gpu.user-enabled", |
| | 20 | 481 | | "UseGpuAcceleration is enabled", |
| | 20 | 482 | | "Config", |
| | 20 | 483 | | NextOrder(), |
| | 20 | 484 | | "UseGpuAcceleration=false")); |
| | 20 | 485 | | AddCheck(ProviderDiagnosticCheck.Skip("rid.platform", "Host platform detection", "Platform", NextOrder() |
| | 20 | 486 | | AddCheck(ProviderDiagnosticCheck.Skip("provider.append", "Execution provider append", "Provider", NextOr |
| | 20 | 487 | | AddCheck(ProviderDiagnosticCheck.Skip("session.construct", "Session construction", "Session", NextOrder( |
| | | 488 | | |
| | 20 | 489 | | provider = InferenceProvider.Cpu; |
| | 20 | 490 | | fallback = InferenceFallbackReason.UserDisabledGpu; |
| | 20 | 491 | | deviceId = 0; |
| | 20 | 492 | | var options = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL }; |
| | 20 | 493 | | options.AppendExecutionProvider_CPU(); |
| | 20 | 494 | | return options; |
| | | 495 | | } |
| | | 496 | | |
| | 34 | 497 | | AddCheck(ProviderDiagnosticCheck.Pass("gpu.user-enabled", "UseGpuAcceleration is enabled", "Config", NextOrd |
| | | 498 | | |
| | 34 | 499 | | var platform = EffectivePlatform; |
| | 34 | 500 | | var (sessionOptions, prov, fb, dev) = platform switch |
| | 34 | 501 | | { |
| | 16 | 502 | | HostPlatform.LinuxX64 => TryRocmBranch(), |
| | 8 | 503 | | HostPlatform.WinX64 or HostPlatform.WinArm64 => TryDirectMlBranch(), |
| | 7 | 504 | | HostPlatform.OSX64 or HostPlatform.OSXArm64 => TryCoreMlBranch(), |
| | 3 | 505 | | _ => TryCpuBranch(InferenceFallbackReason.UnsupportedPlatform), |
| | 34 | 506 | | }; |
| | | 507 | | |
| | 34 | 508 | | provider = prov; |
| | 34 | 509 | | fallback = fb; |
| | 34 | 510 | | deviceId = dev; |
| | 34 | 511 | | return sessionOptions; |
| | | 512 | | } |
| | | 513 | | |
| | | 514 | | private (SessionOptions Options, InferenceProvider Provider, InferenceFallbackReason Fallback, int DeviceId) Try |
| | | 515 | | { |
| | 16 | 516 | | AddCheck(ProviderDiagnosticCheck.Pass("rid.platform", "Host is Linux x64", "Platform", NextOrder())); |
| | 16 | 517 | | var deviceId = _options.DeviceId; |
| | | 518 | | |
| | 16 | 519 | | if (!IsKfdPresent()) |
| | | 520 | | { |
| | 4 | 521 | | AddCheck(ProviderDiagnosticCheck.Fail("kfd.present", "/dev/kfd exists", "Provider", NextOrder(), "/dev/k |
| | 4 | 522 | | SkipRocmProviderChecks("precondition kfd.present failed"); |
| | 4 | 523 | | return BuildCpuFallback(InferenceFallbackReason.KfdDeviceMissing, deviceId); |
| | | 524 | | } |
| | 12 | 525 | | AddCheck(ProviderDiagnosticCheck.Pass("kfd.present", "/dev/kfd exists", "Provider", NextOrder())); |
| | | 526 | | |
| | 12 | 527 | | if (!TryLoadNativeLibrary("libhsa-runtime64.so", out var hsaError)) |
| | | 528 | | { |
| | 2 | 529 | | AddCheck(ProviderDiagnosticCheck.Fail("hsa.dlopen", "libhsa-runtime64.so is loadable", "Provider", NextO |
| | 2 | 530 | | SkipRocmProviderChecks("precondition hsa.dlopen failed", skipHsa: false); |
| | 2 | 531 | | return BuildCpuFallback(InferenceFallbackReason.HsaRuntimeUnloadable, deviceId); |
| | | 532 | | } |
| | 10 | 533 | | AddCheck(ProviderDiagnosticCheck.Pass("hsa.dlopen", "libhsa-runtime64.so is loadable", "Provider", NextOrder |
| | | 534 | | |
| | 10 | 535 | | if (!TryLoadNativeLibrary("libamdhip64.so.7", out var hipError)) |
| | | 536 | | { |
| | 2 | 537 | | AddCheck(ProviderDiagnosticCheck.Fail("hip.dlopen", "libamdhip64.so.7 is loadable", "Provider", NextOrde |
| | 2 | 538 | | hipError ?? "libamdhip64.so.7 not found. HIP runtime is required for MIGraphX GPU acceleration. " + |
| | 2 | 539 | | "Install: sudo apt install hip-runtime-amd (from AMD ROCm repository). " + |
| | 2 | 540 | | "See: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/")); |
| | 2 | 541 | | SkipRocmProviderChecks("precondition hip.dlopen failed", skipHsa: false, skipHip: false); |
| | 2 | 542 | | return BuildCpuFallback(InferenceFallbackReason.HipRuntimeMissing, deviceId); |
| | | 543 | | } |
| | 8 | 544 | | AddCheck(ProviderDiagnosticCheck.Pass("hip.dlopen", "libamdhip64.so.7 is loadable", "Provider", NextOrder()) |
| | | 545 | | |
| | 8 | 546 | | if (!TryLoadNativeLibrary("libmigraphx_c.so.3", out var mgxError)) |
| | | 547 | | { |
| | 2 | 548 | | AddCheck(ProviderDiagnosticCheck.Fail("migraphx.dlopen", "libmigraphx_c.so.3 is loadable", "Provider", N |
| | 2 | 549 | | mgxError ?? "libmigraphx_c.so.3 not found. MIGraphX runtime is required for GPU-accelerated inferenc |
| | 2 | 550 | | "Install: sudo apt install migraphx (from AMD ROCm repository). " + |
| | 2 | 551 | | "See: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/")); |
| | 2 | 552 | | SkipRocmProviderChecks("precondition migraphx.dlopen failed", skipHsa: false, skipHip: false, skipMigrap |
| | 2 | 553 | | return BuildCpuFallback(InferenceFallbackReason.MIGraphXRuntimeMissing, deviceId); |
| | | 554 | | } |
| | 6 | 555 | | AddCheck(ProviderDiagnosticCheck.Pass("migraphx.dlopen", "libmigraphx_c.so.3 is loadable", "Provider", NextO |
| | | 556 | | |
| | 6 | 557 | | var options = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL }; |
| | | 558 | | try |
| | | 559 | | { |
| | 6 | 560 | | AppendMIGraphXProvider(options, deviceId); |
| | 5 | 561 | | AddCheck(ProviderDiagnosticCheck.Pass("migraphx.append", "AppendExecutionProvider_MIGraphX", "Provider", |
| | 5 | 562 | | return (options, InferenceProvider.MIGraphX, InferenceFallbackReason.None, deviceId); |
| | | 563 | | } |
| | 1 | 564 | | catch (Exception ex) |
| | | 565 | | { |
| | 1 | 566 | | AddCheck(ProviderDiagnosticCheck.Fail("migraphx.append", "AppendExecutionProvider_MIGraphX", "Provider", |
| | 1 | 567 | | AddCheck(ProviderDiagnosticCheck.Skip("migraphx.session", "MIGraphX InferenceSession construction", "Ses |
| | 1 | 568 | | options.Dispose(); |
| | 1 | 569 | | return BuildCpuFallback(InferenceFallbackReason.ProviderAppendFailed, deviceId); |
| | | 570 | | } |
| | 6 | 571 | | } |
| | | 572 | | |
| | | 573 | | private void SkipRocmProviderChecks(string reason, |
| | | 574 | | bool skipHsa = true, bool skipHip = true, bool skipMigraphxDlopen = true, |
| | | 575 | | bool skipMigraphxAppend = true, bool skipMigraphxSession = true) |
| | | 576 | | { |
| | 10 | 577 | | if (skipHsa) |
| | 4 | 578 | | AddCheck(ProviderDiagnosticCheck.Skip("hsa.dlopen", "libhsa-runtime64.so is loadable", "Provider", NextO |
| | 10 | 579 | | if (skipHip) |
| | 6 | 580 | | AddCheck(ProviderDiagnosticCheck.Skip("hip.dlopen", "libamdhip64.so.7 is loadable", "Provider", NextOrde |
| | 10 | 581 | | if (skipMigraphxDlopen) |
| | 8 | 582 | | AddCheck(ProviderDiagnosticCheck.Skip("migraphx.dlopen", "libmigraphx_c.so.3 is loadable", "Provider", N |
| | 10 | 583 | | if (skipMigraphxAppend) |
| | 10 | 584 | | AddCheck(ProviderDiagnosticCheck.Skip("migraphx.append", "AppendExecutionProvider_MIGraphX succeeded", " |
| | 10 | 585 | | if (skipMigraphxSession) |
| | 10 | 586 | | AddCheck(ProviderDiagnosticCheck.Skip("migraphx.session", "MIGraphX InferenceSession construction", "Ses |
| | 10 | 587 | | } |
| | | 588 | | |
| | | 589 | | private (SessionOptions Options, InferenceProvider Provider, InferenceFallbackReason Fallback, int DeviceId) Try |
| | | 590 | | { |
| | 8 | 591 | | AddCheck(ProviderDiagnosticCheck.Pass("rid.platform", "Host is Windows", "Platform", NextOrder())); |
| | 8 | 592 | | var deviceId = _options.DeviceId; |
| | 8 | 593 | | var options = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL }; |
| | | 594 | | |
| | | 595 | | try |
| | | 596 | | { |
| | 8 | 597 | | AppendDirectMlProvider(options, deviceId); |
| | 4 | 598 | | AddCheck(ProviderDiagnosticCheck.Pass("dml.append", "AppendExecutionProvider_DML", "Provider", NextOrder |
| | 4 | 599 | | return (options, InferenceProvider.DirectML, InferenceFallbackReason.None, deviceId); |
| | | 600 | | } |
| | 4 | 601 | | catch (Exception ex) |
| | | 602 | | { |
| | 4 | 603 | | AddCheck(ProviderDiagnosticCheck.Fail("dml.append", "AppendExecutionProvider_DML", "Provider", NextOrder |
| | 4 | 604 | | AddCheck(ProviderDiagnosticCheck.Skip("dml.session", "DirectML InferenceSession construction", "Session" |
| | 4 | 605 | | options.Dispose(); |
| | 4 | 606 | | return BuildCpuFallback(InferenceFallbackReason.ProviderAppendFailed, deviceId); |
| | | 607 | | } |
| | 8 | 608 | | } |
| | | 609 | | |
| | | 610 | | private (SessionOptions Options, InferenceProvider Provider, InferenceFallbackReason Fallback, int DeviceId) Try |
| | | 611 | | { |
| | 7 | 612 | | AddCheck(ProviderDiagnosticCheck.Pass("rid.platform", "Host is macOS", "Platform", NextOrder())); |
| | 7 | 613 | | var deviceId = _options.DeviceId; |
| | 7 | 614 | | var options = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL }; |
| | | 615 | | |
| | | 616 | | try |
| | | 617 | | { |
| | 7 | 618 | | AppendCoreMlProvider(options, CoreMLFlags.COREML_FLAG_ENABLE_ON_SUBGRAPH); |
| | 3 | 619 | | AddCheck(ProviderDiagnosticCheck.Pass("coreml.append", "AppendExecutionProvider_CoreML", "Provider", Nex |
| | 3 | 620 | | return (options, InferenceProvider.CoreML, InferenceFallbackReason.None, deviceId); |
| | | 621 | | } |
| | 4 | 622 | | catch (Exception ex) |
| | | 623 | | { |
| | 4 | 624 | | AddCheck(ProviderDiagnosticCheck.Fail("coreml.append", "AppendExecutionProvider_CoreML", "Provider", Nex |
| | 4 | 625 | | AddCheck(ProviderDiagnosticCheck.Skip("coreml.session", "CoreML InferenceSession construction", "Session |
| | 4 | 626 | | options.Dispose(); |
| | 4 | 627 | | return BuildCpuFallback(InferenceFallbackReason.ProviderAppendFailed, deviceId); |
| | | 628 | | } |
| | 7 | 629 | | } |
| | | 630 | | |
| | | 631 | | private (SessionOptions Options, InferenceProvider Provider, InferenceFallbackReason Fallback, int DeviceId) Try |
| | | 632 | | { |
| | 3 | 633 | | AddCheck(ProviderDiagnosticCheck.Fail("rid.platform", "Host platform detection", "Platform", NextOrder(), $" |
| | 3 | 634 | | AddCheck(ProviderDiagnosticCheck.Skip("provider.append", "Execution provider append", "Provider", NextOrder( |
| | 3 | 635 | | AddCheck(ProviderDiagnosticCheck.Skip("session.construct", "Session construction", "Session", NextOrder(), " |
| | 3 | 636 | | var options = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL }; |
| | 3 | 637 | | options.AppendExecutionProvider_CPU(); |
| | 3 | 638 | | return (options, InferenceProvider.Cpu, reason, 0); |
| | | 639 | | } |
| | | 640 | | |
| | | 641 | | private (SessionOptions Options, InferenceProvider Provider, InferenceFallbackReason Fallback, int DeviceId) Bui |
| | | 642 | | { |
| | 19 | 643 | | var options = CreateCpuOnlySessionOptions(); |
| | 19 | 644 | | return (options, InferenceProvider.Cpu, reason, deviceId); |
| | | 645 | | } |
| | | 646 | | |
| | | 647 | | private void AppendSessionCheck(InferenceProvider provider, bool passed, string? detail = null) |
| | | 648 | | { |
| | 14 | 649 | | var name = provider switch |
| | 14 | 650 | | { |
| | 5 | 651 | | InferenceProvider.MIGraphX => "migraphx.session", |
| | 4 | 652 | | InferenceProvider.DirectML => "dml.session", |
| | 3 | 653 | | InferenceProvider.CoreML => "coreml.session", |
| | 2 | 654 | | _ => "cpu.session", |
| | 14 | 655 | | }; |
| | 14 | 656 | | var desc = $"{provider} InferenceSession construction"; |
| | 14 | 657 | | AddCheck(passed |
| | 14 | 658 | | ? ProviderDiagnosticCheck.Pass(name, desc, "Session", NextOrder(), detail) |
| | 14 | 659 | | : ProviderDiagnosticCheck.Fail(name, desc, "Session", NextOrder(), detail ?? "unknown failure")); |
| | 14 | 660 | | } |
| | | 661 | | |
| | | 662 | | private void AppendFallbackChecks(SessionOptions cpuOptions, string modelPath) |
| | | 663 | | { |
| | 3 | 664 | | AddCheck(ProviderDiagnosticCheck.Pass("cpu.fallback-options", "CreateCpuOnlySessionOptions() succeeded", "Fa |
| | | 665 | | try |
| | | 666 | | { |
| | 3 | 667 | | _session = CreateInferenceSession(modelPath, cpuOptions); |
| | 3 | 668 | | AddCheck(ProviderDiagnosticCheck.Pass("cpu.fallback-session", "CPU InferenceSession constructed", "Fallb |
| | 3 | 669 | | } |
| | 0 | 670 | | catch (Exception ex) |
| | | 671 | | { |
| | 0 | 672 | | AddCheck(ProviderDiagnosticCheck.Fail("cpu.fallback-session", "CPU InferenceSession construction", "Fall |
| | 0 | 673 | | throw; |
| | | 674 | | } |
| | 3 | 675 | | } |
| | | 676 | | |
| | | 677 | | private void FireOnSessionInitialized(InferenceEngineInfo info) |
| | | 678 | | { |
| | 187 | 679 | | if (_options.OnSessionInitialized is null) return; |
| | | 680 | | try |
| | | 681 | | { |
| | 3 | 682 | | _options.OnSessionInitialized(info); |
| | 2 | 683 | | } |
| | 1 | 684 | | catch (Exception ex) |
| | | 685 | | { |
| | 1 | 686 | | _logger.LogWarning(ex, "OnSessionInitialized callback threw; engine is still usable."); |
| | 1 | 687 | | } |
| | 3 | 688 | | } |
| | | 689 | | |
| | | 690 | | /// <summary> |
| | | 691 | | /// Publishes a status transition: updates <see cref="Status"/> then fires |
| | | 692 | | /// <see cref="InferenceEngineOptions.OnStatusChanged"/> (callback exceptions are |
| | | 693 | | /// swallowed and logged — a broken consumer callback must not kill the engine). |
| | | 694 | | /// Private protected: out-of-process proxies drive their own transitions with the |
| | | 695 | | /// worker-reported terminal states. |
| | | 696 | | /// </summary> |
| | | 697 | | private protected void SetStatus(InferenceEngineStatus status) |
| | | 698 | | { |
| | 307 | 699 | | Status = status; |
| | 596 | 700 | | if (_options.OnStatusChanged is null) return; |
| | | 701 | | try |
| | | 702 | | { |
| | 18 | 703 | | _options.OnStatusChanged(status); |
| | 16 | 704 | | } |
| | 2 | 705 | | catch (Exception ex) |
| | | 706 | | { |
| | 2 | 707 | | _logger.LogWarning(ex, "OnStatusChanged callback threw; engine is still usable."); |
| | 2 | 708 | | } |
| | 18 | 709 | | } |
| | | 710 | | |
| | | 711 | | /// <summary> |
| | | 712 | | /// Apply engine state that was loaded by an out-of-process worker (no in-process |
| | | 713 | | /// <see cref="InferenceSession"/>). Used by <c>OutOfProcessBaseInferenceEngine</c> to mirror |
| | | 714 | | /// the worker's <see cref="EngineInfo"/>/metadata so the diagnostics surface (provider, |
| | | 715 | | /// fallback reason, ordered checks) behaves identically to the in-process engine. Does not |
| | | 716 | | /// create a session; the proxy's <see cref="PredictAsync"/> marshals to the worker instead. |
| | | 717 | | /// </summary> |
| | | 718 | | protected void ApplyLoadedState( |
| | | 719 | | InferenceEngineInfo engineInfo, |
| | | 720 | | IReadOnlyDictionary<string, NodeMetadata>? inputMetadata, |
| | | 721 | | IReadOnlyDictionary<string, NodeMetadata>? outputMetadata, |
| | | 722 | | string? primaryInputName) |
| | | 723 | | { |
| | 44 | 724 | | EngineInfo = engineInfo; |
| | 44 | 725 | | InputMetadata = inputMetadata; |
| | 44 | 726 | | OutputMetadata = outputMetadata; |
| | 44 | 727 | | PrimaryInputName = primaryInputName; |
| | 44 | 728 | | _isLoaded = true; |
| | 44 | 729 | | FireOnSessionInitialized(engineInfo); |
| | 44 | 730 | | } |
| | | 731 | | |
| | | 732 | | /// <summary> |
| | | 733 | | /// Per-provider key/value options actually applied. At minimum surfaces <c>device_id</c> for |
| | | 734 | | /// hardware-accelerated providers (MIGraphX/DirectML/CoreML); MIGraphX additionally surfaces |
| | | 735 | | /// <c>migraphx_model_cache_dir</c> when the compiled-model cache is enabled. Empty for |
| | | 736 | | /// CPU/Unknown. |
| | | 737 | | /// </summary> |
| | | 738 | | private IReadOnlyDictionary<string, string> ExtractProviderOptions(InferenceProvider provider, int deviceId) |
| | | 739 | | { |
| | 54 | 740 | | if (provider == InferenceProvider.Cpu || provider == InferenceProvider.Unknown) |
| | 43 | 741 | | return ImmutableDictionary<string, string>.Empty; |
| | | 742 | | |
| | | 743 | | // Build the immutable tree directly instead of allocating a transient |
| | | 744 | | // Dictionary<string,string> + ToImmutableDictionary() on every session init. |
| | 11 | 745 | | var options = ImmutableDictionary<string, string>.Empty.Add("device_id", deviceId.ToString()); |
| | 11 | 746 | | if (provider == InferenceProvider.MIGraphX && ResolveMIGraphXModelCacheDirectory() is { } cacheDir) |
| | 4 | 747 | | options = options.Add("migraphx_model_cache_dir", cacheDir); |
| | 11 | 748 | | return options; |
| | | 749 | | } |
| | | 750 | | |
| | | 751 | | private static SessionOptions CreateCpuOnlySessionOptions() |
| | | 752 | | { |
| | 22 | 753 | | var options = new SessionOptions |
| | 22 | 754 | | { |
| | 22 | 755 | | GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL, |
| | 22 | 756 | | }; |
| | 22 | 757 | | options.AppendExecutionProvider_CPU(); |
| | 22 | 758 | | return options; |
| | | 759 | | } |
| | | 760 | | |
| | | 761 | | /// <summary> |
| | | 762 | | /// Determines whether an exception thrown during ONNX session initialization represents a |
| | | 763 | | /// recoverable native library loading failure. |
| | | 764 | | /// </summary> |
| | | 765 | | private static bool IsRecoverableNativeFailure(Exception ex) |
| | | 766 | | { |
| | 5 | 767 | | if (ex is OnnxRuntimeException or DllNotFoundException or BadImageFormatException) return true; |
| | 1 | 768 | | if (ex is TypeInitializationException tiex) |
| | 1 | 769 | | return tiex.InnerException is OnnxRuntimeException or DllNotFoundException or BadImageFormatException; |
| | 0 | 770 | | return false; |
| | | 771 | | } |
| | | 772 | | |
| | | 773 | | /// <summary> |
| | | 774 | | /// Generates dummy input for model warmup. |
| | | 775 | | /// Default implementation assumes float tensors. Override this for int64/string inputs. |
| | | 776 | | /// </summary> |
| | | 777 | | protected virtual List<NamedOnnxValue> GetWarmupInput() |
| | | 778 | | { |
| | 2 | 779 | | var inputs = new List<NamedOnnxValue>(); |
| | 3 | 780 | | if (InputMetadata == null) return inputs; |
| | | 781 | | |
| | 4 | 782 | | foreach (var inputMeta in InputMetadata) |
| | | 783 | | { |
| | 2 | 784 | | var shape = inputMeta.Value.Dimensions.Select(d => d == -1 ? 1 : d).ToArray(); |
| | 1 | 785 | | var tensorData = new Microsoft.ML.OnnxRuntime.Tensors.DenseTensor<float>(shape); |
| | 1 | 786 | | inputs.Add(NamedOnnxValue.CreateFromTensor(inputMeta.Key, tensorData)); |
| | | 787 | | } |
| | 1 | 788 | | return inputs; |
| | | 789 | | } |
| | | 790 | | |
| | | 791 | | private void PerformWarmup() |
| | | 792 | | { |
| | | 793 | | try |
| | | 794 | | { |
| | 3 | 795 | | _logger.LogInformation("Performing warmup inference..."); |
| | 3 | 796 | | var inputs = GetWarmupInput(); |
| | | 797 | | |
| | 2 | 798 | | if (inputs.Count > 0) |
| | | 799 | | { |
| | 2 | 800 | | using var results = _session!.Run(inputs); |
| | 1 | 801 | | _logger.LogInformation("Warmup complete."); |
| | | 802 | | } |
| | 1 | 803 | | } |
| | 2 | 804 | | catch (Exception ex) |
| | | 805 | | { |
| | 2 | 806 | | _logger.LogWarning($"Warmup failed (non-critical). Error: {ex.Message}"); |
| | 2 | 807 | | } |
| | 3 | 808 | | } |
| | | 809 | | |
| | | 810 | | public void Dispose() |
| | | 811 | | { |
| | 160 | 812 | | Dispose(true); |
| | 160 | 813 | | GC.SuppressFinalize(this); |
| | 160 | 814 | | } |
| | | 815 | | |
| | | 816 | | protected virtual void Dispose(bool disposing) |
| | | 817 | | { |
| | 168 | 818 | | if (disposing) |
| | | 819 | | { |
| | 168 | 820 | | _session?.Dispose(); |
| | 168 | 821 | | _runOptions?.Dispose(); |
| | 168 | 822 | | _session = null; |
| | 168 | 823 | | _isLoaded = false; |
| | | 824 | | // Silent reset (no OnStatusChanged fire): a disposed engine's prior state is not |
| | | 825 | | // a transition consumers need to react to. |
| | 168 | 826 | | Status = InferenceEngineStatus.NotLoaded; |
| | 168 | 827 | | _loadLock.Dispose(); |
| | | 828 | | } |
| | 168 | 829 | | } |
| | | 830 | | } |
| | | 831 | | } |