< Summary

Information
Class: InferenceEngine.Core.BaseInferenceEngine<T1, T2>
Assembly: InferenceEngine.Core
File(s): /home/runner/work/inference-engine-lib/inference-engine-lib/InferenceEngine.Core/BaseInferenceEngine.cs
Line coverage
96%
Covered lines: 439
Uncovered lines: 14
Coverable lines: 453
Total lines: 831
Line coverage: 96.9%
Branch coverage
87%
Covered branches: 123
Total branches: 141
Branch coverage: 87.2%
Method coverage

Feature is only available for sponsors

Upgrade to PRO version

Metrics

File(s)

/home/runner/work/inference-engine-lib/inference-engine-lib/InferenceEngine.Core/BaseInferenceEngine.cs

#LineLine coverage
 1using System.Collections.Immutable;
 2using System.Diagnostics;
 3using System.Runtime.InteropServices;
 4using System.Threading;
 5using Microsoft.ML.OnnxRuntime;
 6using Microsoft.Extensions.Logging;
 7using Microsoft.Extensions.Logging.Abstractions;
 8using InferenceEngine.Core.Diagnostics;
 9using InferenceEngine.Core.ModelProviders;
 10
 11namespace InferenceEngine.Core
 12{
 13    /// <summary>
 14    /// The core abstract base class that handles ONNX Runtime session management,
 15    /// cross-platform hardware acceleration detection, and resource cleanup.
 16    /// Supports state passing between Pre and Post processing steps via a context object.
 17    /// </summary>
 18    public abstract class BaseInferenceEngine<TInput, TOutput> : IInferenceEngine<TInput, TOutput>
 19    {
 20        protected readonly InferenceEngineOptions _options;
 21        protected readonly ILogger _logger;
 22        protected InferenceSession? _session;
 23        protected RunOptions? _runOptions;
 24
 25        private bool _isLoaded = false;
 17326        private readonly SemaphoreSlim _loadLock = new(1, 1);
 27        private IModelProvider? _modelProvider;
 17328        private List<ProviderDiagnosticCheck> _pendingChecks = new();
 29        private int _nextOrder = 0;
 30
 12631        public IReadOnlyDictionary<string, NodeMetadata>? InputMetadata { get; private set; }
 9632        public IReadOnlyDictionary<string, NodeMetadata>? OutputMetadata { get; private set; }
 14833        public string? PrimaryInputName { get; private set; }
 34
 35        /// <summary>
 36        /// Returns <see langword="true"/> when the ONNX session was successfully initialized
 37        /// with the MIGraphX execution provider on AMD ROCm hardware. <see langword="false"/> on non-Linux-x64 platform
 38        /// when MIGraphX initialization failed, or when CPU fallback was used.
 39        /// </summary>
 5140        public bool IsRocmSessionActive { get; private set; }
 41
 42        /// <summary>
 43        /// Snapshot of the engine's loaded state. Returns <see cref="InferenceEngineInfo.Empty"/>
 44        /// until <see cref="LoadAsync"/> completes. Thread-safe: reference assignment is atomic
 45        /// and <see cref="InferenceEngineInfo"/> is an immutable record.
 46        /// </summary>
 45747        public InferenceEngineInfo EngineInfo { get; private set; } = InferenceEngineInfo.Empty;
 48
 49        /// <summary>
 50        /// Lifecycle + runtime snapshot (<see cref="EngineLoadState"/> + partial-or-full
 51        /// <see cref="InferenceEngineInfo"/> + load error). Transitions fire
 52        /// <see cref="InferenceEngineOptions.OnStatusChanged"/> when set. Out-of-process proxies
 53        /// mirror the worker's terminal states onto the same property.
 54        /// </summary>
 71455        public InferenceEngineStatus Status { get; private set; } = InferenceEngineStatus.NotLoaded;
 56
 57        // --- Model Specification ---
 58
 59        /// <summary>
 60        /// The name of the model file (e.g., "yolo11n.onnx").
 61        /// Derived classes must implement this to specify which model they require.
 62        /// </summary>
 63        protected abstract string ModelName { get; }
 64
 65        /// <summary>
 66        /// URLs where the model can be downloaded from, in priority order.
 67        /// Derived classes can override to provide custom model sources.
 68        /// If not overridden, the base engine will use the default ModelRegistry to find URLs.
 69        /// </summary>
 370        protected virtual string[]? ModelDownloadUrls => null;
 71
 72        /// <summary>
 73        /// Model registry for looking up download URLs by model name.
 74        /// Override to use a custom registry; defaults to the built-in registry.
 75        /// </summary>
 376        protected virtual ModelRegistry ModelRegistry => _lazyRegistry.Value;
 77
 78        private readonly Lazy<ModelRegistry> _lazyRegistry;
 79
 80        /// <summary>
 81        /// Base directory for model cache. Defaults to LocalApplicationData.
 82        /// Derived classes can override to use a custom cache location.
 83        /// </summary>
 484        protected virtual string ModelCacheDirectory => Path.Combine(
 485            Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
 486            "InferenceEngine",
 487            "Models");
 88
 17389        protected BaseInferenceEngine(InferenceEngineOptions options, ILogger? logger = null)
 90        {
 17391            _options = options ?? throw new ArgumentNullException(nameof(options));
 17192            _logger = logger ?? NullLogger.Instance;
 17493            _lazyRegistry = new Lazy<ModelRegistry>(() => new ModelRegistry(logger: _logger), LazyThreadSafetyMode.Execu
 17194        }
 95
 96        // --- Abstract Methods ---
 97
 98        /// <summary>
 99        /// Takes raw input and returns ONNX tensors plus an optional context object.
 100        /// The context object is passed to PostProcessWithContext, allowing you to share state
 101        /// without overriding the main pipeline.
 102        /// </summary>
 103        protected abstract (List<NamedOnnxValue> InputTensors, object? Context) PreProcessWithContext(TInput input);
 104
 105        /// <summary>
 106        /// Takes raw ONNX output and the context object from PreProcess, returning the final result.
 107        /// </summary>
 108        protected abstract TOutput PostProcessWithContext(IDisposableReadOnlyCollection<DisposableNamedOnnxValue> output
 109
 110        // --- Public API ---
 111
 112        /// <summary>
 113        /// Gets or creates the model provider.
 114        /// Uses options.ModelProvider if set, otherwise creates from ModelName/ModelDownloadUrls/ModelRegistry.
 115        /// </summary>
 116        private IModelProvider GetModelProvider()
 117        {
 60118            if (_options.ModelProvider is not null)
 119            {
 53120                return _options.ModelProvider;
 121            }
 122
 7123            if (_modelProvider is null)
 124            {
 7125                var urls = ModelDownloadUrls;
 7126                string? expectedSha256 = null;
 127
 7128                if (urls is null || urls.Length == 0)
 129                {
 5130                    var entry = ModelRegistry.GetModelEntry(ModelName);
 5131                    urls = entry?.Urls;
 132
 5133                    if (urls is null || urls.Length == 0)
 134                    {
 4135                        throw new InvalidOperationException(
 4136                            $"Engine '{GetType().Name}' does not specify ModelDownloadUrls and model '{ModelName}' " +
 4137                            $"was not found in the ModelRegistry. Either: " +
 4138                            $"1) Override ModelDownloadUrls in your engine class, " +
 4139                            $"2) Add '{ModelName}' to the model registry (models.json), or " +
 4140                            $"3) Set {nameof(InferenceEngineOptions)}.{nameof(InferenceEngineOptions.ModelProvider)} exp
 141                    }
 142
 1143                    expectedSha256 = entry!.Sha256;
 1144                    _logger.LogInformation("Found model '{ModelName}' in registry with {Count} URL(s)", ModelName, urls.
 145                }
 146
 3147                _modelProvider = new DownloadingModelProvider(
 3148                    modelName: ModelName,
 3149                    downloadUrls: urls,
 3150                    cacheDirectory: ModelCacheDirectory,
 3151                    logger: _logger,
 3152                    sha256: expectedSha256,
 3153                    maxDownloadBytes: _options.MaxModelDownloadBytes);
 154            }
 155
 3156            return _modelProvider;
 157        }
 158
 159        /// <summary>
 160        /// Initializes the inference session with the best available hardware provider.
 161        /// </summary>
 162        public virtual async Task LoadAsync()
 163        {
 164            // Serialize concurrent loads (H5): two concurrent LoadAsync calls must not both pass the
 165            // _isLoaded guard and double-construct the session / corrupt the check list.
 67166            await _loadLock.WaitAsync().ConfigureAwait(false);
 167            try
 168            {
 70169                if (_isLoaded) return;
 170
 60171                SetStatus(new InferenceEngineStatus { State = EngineLoadState.Loading, Detail = "Resolving model…" });
 172                try
 173                {
 174
 60175            _logger.LogInformation("Initializing Inference Engine (engine={EngineType}, gpu={UseGpu}, deviceId={DeviceId
 60176                GetType().Name, _options.UseGpuAcceleration, _options.DeviceId);
 177
 60178            var modelPath = await GetModelProvider().GetModelPathAsync(CancellationToken.None);
 53179            _logger.LogInformation("Model resolved to '{ModelPath}' (exists={Exists})", modelPath, File.Exists(modelPath
 180
 53181            SetStatus(new InferenceEngineStatus { State = EngineLoadState.Loading, Detail = "Selecting execution provide
 182
 53183            await Task.Run(() =>
 53184            {
 53185                try
 53186                {
 53187                    _pendingChecks = new List<ProviderDiagnosticCheck>();
 53188                    _nextOrder = 0;
 53189
 53190                    InferenceProvider provider = InferenceProvider.Unknown;
 53191                    InferenceFallbackReason fallback;
 53192                    int deviceId;
 53193
 53194                    SessionOptions? sessionOptions = null;
 53195
 53196                    try
 53197                    {
 53198                        sessionOptions = ConfiguredSessionOptions(out provider, out fallback, out deviceId);
 53199
 53200                        // Session construction is where a hardware EP compiles the model's
 53201                        // kernels â€” including the multi-minute MIGraphX first-run compile.
 53202                        SetStatus(new InferenceEngineStatus { State = EngineLoadState.Loading, Detail = "Constructing in
 53203
 53204                        try
 53205                        {
 53206                            _session = CreateInferenceSession(modelPath, sessionOptions);
 48207                            if (fallback != InferenceFallbackReason.None)
 53208                            {
 39209                                AddCheck(ProviderDiagnosticCheck.Pass("cpu.fallback-options", "CreateCpuOnlySessionOptio
 39210                                AddCheck(ProviderDiagnosticCheck.Pass("cpu.fallback-session", "CPU InferenceSession cons
 53211                            }
 53212                            else
 53213                            {
 9214                                AppendSessionCheck(provider, passed: true);
 53215                            }
 48216                        }
 5217                        catch (Exception ex)
 53218                        {
 5219                            AppendSessionCheck(provider, passed: false, ex.Message);
 5220                            throw;
 53221                        }
 48222                    }
 5223                    catch (Exception ex) when (
 5224                        _options.UseGpuAcceleration &&
 5225                        provider != InferenceProvider.Cpu &&
 5226                        IsRecoverableNativeFailure(ex))
 53227                    {
 3228                        _logger.LogWarning(ex, "Native session initialization failed. Falling back to CPU.");
 3229                        AddCheck(ProviderDiagnosticCheck.Pass(
 3230                            "session.native-recoverable",
 3231                            "IsRecoverableNativeFailure detected, attempting CPU fallback",
 3232                            "Fallback",
 3233                            NextOrder()));
 53234
 3235                        sessionOptions?.Dispose();
 3236                        fallback = InferenceFallbackReason.NativeLibraryLoadFailed;
 3237                        provider = InferenceProvider.Cpu;
 3238                        deviceId = 0;
 53239
 3240                        sessionOptions = CreateCpuOnlySessionOptions();
 3241                        AppendFallbackChecks(sessionOptions, modelPath);
 3242                    }
 53243                    finally
 53244                    {
 53245                        sessionOptions?.Dispose();
 53246                    }
 53247
 51248                    _runOptions = new RunOptions();
 53249
 51250                    InputMetadata = _session!.InputMetadata;
 51251                    OutputMetadata = _session!.OutputMetadata;
 51252                    PrimaryInputName = _session!.InputMetadata.Keys.FirstOrDefault();
 53253
 51254                    IsRocmSessionActive = provider == InferenceProvider.MIGraphX;
 53255
 51256                    var info = new InferenceEngineInfo
 51257                    {
 51258                        Provider = provider,
 51259                        Platform = HostPlatformDetector.Current(),
 51260                        DeviceId = deviceId,
 51261                        GpuAccelerationRequested = _options.UseGpuAcceleration,
 51262                        ModelName = ModelName,
 51263                        ModelPath = modelPath,
 51264                        ModelSource = _options.ModelProvider is not null
 51265                            ? Diagnostics.ModelSource.ExplicitPath
 51266                            : Diagnostics.ModelSource.RegistryManaged,
 51267                        IsFallback = fallback != InferenceFallbackReason.None,
 51268                        FallbackReason = fallback,
 51269                        ProviderOptions = ExtractProviderOptions(provider, deviceId),
 51270                        SessionCreatedAt = DateTimeOffset.UtcNow,
 51271                        Checks = _pendingChecks.ToArray(),
 51272                    };
 53273
 53274#if DEBUG
 53275                    for (int i = 0; i < info.Checks.Count; i++)
 53276                        Debug.Assert(info.Checks[i].Order == i, $"Check order mismatch at index {i}: expected {i}, got {
 53277#endif
 53278
 51279                    EngineInfo = info;
 51280                    SetStatus(new InferenceEngineStatus { State = EngineLoadState.Loaded, Info = info });
 53281
 51282                    _logger.LogInformation(
 51283                        "Model loaded with {Provider} provider (fallback={IsFallback}, reason={FallbackReason}). Primary
 51284                        provider, info.IsFallback, fallback, PrimaryInputName);
 51285                    _isLoaded = true;
 53286
 53287                    // M4: fire the session-initialized callback BEFORE warmup (it documents that it
 53288                    // "fires once after the session is created"), so a long first-compile warmup does
 53289                    // not delay it and a warmup problem cannot prevent it.
 51290                    FireOnSessionInitialized(info);
 53291
 51292                    if (_options.WarmupOnLoad)
 53293                    {
 3294                        PerformWarmup();
 53295                    }
 51296                }
 2297                catch (Exception ex)
 53298                {
 2299                    _logger.LogError(ex, "Failed to initialize Inference Engine.");
 2300                    throw;
 53301                }
 104302            });
 51303                }
 9304                catch (Exception ex)
 305                {
 306                    // Surface the failure through the status object model: the partial
 307                    // provider-selection trail (checks accumulated before the failure) explains
 308                    // WHY, and the error string says WHAT threw.
 9309                    SetStatus(new InferenceEngineStatus
 9310                    {
 9311                        State = EngineLoadState.Failed,
 9312                        Info = new InferenceEngineInfo
 9313                        {
 9314                            Provider = InferenceProvider.Unknown,
 9315                            Checks = _pendingChecks.ToArray(),
 9316                        },
 9317                        LoadError = ex.Message,
 9318                    });
 9319                    throw;
 320                }
 51321            }
 322            finally
 323            {
 65324                _loadLock.Release();
 325            }
 56326        }
 327
 328        public virtual async Task<TOutput> PredictAsync(TInput input)
 329        {
 5330            if (!_isLoaded) await LoadAsync();
 331
 3332            return await Task.Run(() =>
 3333            {
 3334                var (inputTensors, context) = PreProcessWithContext(input);
 3335
 3336                try
 3337                {
 3338                    using var outputTensors = _session!.Run(inputTensors, _session.OutputNames, _runOptions);
 3339                    return PostProcessWithContext(outputTensors, context);
 3340                }
 3341                finally
 3342                {
 24343                    foreach(var t in inputTensors)
 3344                    {
 9345                        if (t is IDisposable disposable)
 3346                        {
 0347                            disposable.Dispose();
 3348                        }
 3349                    }
 3350                }
 6351            });
 3352        }
 353
 354        // --- Internal/Protected Helpers ---
 355
 356        // Test seams (T1/T2): virtualize the platform/probe/append/session-construction steps so the
 357        // most complex fallback branches can be exercised hermetically (on any CI host, with no GPU).
 358        // Defaults reproduce the original behavior exactly.
 359
 360        /// <summary>Platform used to select the execution-provider branch. Override to force a branch in tests.</summar
 2361        protected virtual HostPlatform EffectivePlatform => HostPlatformDetector.Current();
 362
 363        /// <summary>True when the ROCm KFD device node is present. Override in tests.</summary>
 2364        protected virtual bool IsKfdPresent() => File.Exists("/dev/kfd");
 365
 366        /// <summary>Attempts to dlopen a native library; returns success and an error message on failure.</summary>
 367        protected virtual bool TryLoadNativeLibrary(string name, out string? error)
 368        {
 369            try
 370            {
 1371                bool loaded = NativeLibrary.TryLoad(name, typeof(InferenceSession).Assembly,
 1372                    DllImportSearchPath.UseDllDirectoryForDependencies | DllImportSearchPath.System32, out var handle);
 1373                if (loaded && handle != IntPtr.Zero) NativeLibrary.Free(handle);
 1374                error = loaded ? null : "NativeLibrary.TryLoad returned false";
 1375                return loaded;
 376            }
 0377            catch (Exception ex)
 378            {
 0379                error = ex.Message;
 0380                return false;
 381            }
 1382        }
 383
 384        /// <summary>
 385        /// Appends the MIGraphX execution provider with the compiled-model (.mxr) cache
 386        /// directory resolved by <see cref="ResolveMIGraphXModelCacheDirectory"/>. The cache
 387        /// persists compiled MIGraphX programs across process restarts; without it the ORT
 388        /// provider compiles the full model on every session construction (an empty cache
 389        /// path explicitly means "always compile" in the EP). Override to inject a throw in tests.
 390        /// </summary>
 391        protected virtual void AppendMIGraphXProvider(SessionOptions options, int deviceId)
 392        {
 1393            var providerOptions = new Dictionary<string, string>
 1394            {
 1395                ["device_id"] = deviceId.ToString(),
 1396            };
 397
 1398            var cacheDir = ResolveMIGraphXModelCacheDirectory();
 1399            if (cacheDir is not null)
 400            {
 401                try
 402                {
 1403                    Directory.CreateDirectory(cacheDir);
 0404                    providerOptions["migraphx_model_cache_dir"] = cacheDir;
 0405                    _logger.LogInformation(
 0406                        "MIGraphX compiled-model cache: {CacheDir} (first load compiles ~1-2 min; later loads reuse the 
 0407                        cacheDir);
 0408                }
 1409                catch (Exception ex)
 410                {
 411                    // The cache is an optimization: a host that cannot host the cache directory
 412                    // still gets GPU inference, just with a full recompile on every startup.
 1413                    _logger.LogWarning(ex,
 1414                        "Cannot create MIGraphX model cache directory {CacheDir}; continuing without .mxr caching (every
 1415                        cacheDir);
 1416                }
 417            }
 418
 1419            options.AppendExecutionProvider("MIGraphX", providerOptions);
 0420        }
 421
 422        /// <summary>
 423        /// Resolves the MIGraphX compiled-model (.mxr) cache directory: an explicitly configured
 424        /// <see cref="InferenceEngineOptions.MIGraphXModelCacheDirectory"/> wins (empty string =
 425        /// disabled), otherwise a per-user default under the same root as
 426        /// <see cref="ModelCacheDirectory"/>. Returns <see langword="null"/> when caching is
 427        /// disabled or the host has no user-profile directory to anchor the default.
 428        /// Pure path computation â€” directory creation happens in
 429        /// <see cref="AppendMIGraphXProvider"/> only.
 430        /// </summary>
 431        protected virtual string? ResolveMIGraphXModelCacheDirectory()
 432        {
 9433            var configured = _options.MIGraphXModelCacheDirectory;
 9434            if (configured is not null)
 5435                return configured.Length == 0 ? null : configured;
 436
 4437            var root = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
 4438            return string.IsNullOrEmpty(root)
 4439                ? null
 4440                : Path.Combine(root, "InferenceEngine", "MIGraphXModelCache");
 441        }
 442
 443        /// <summary>Appends the DirectML execution provider. Override to inject a throw in tests.</summary>
 444        protected virtual void AppendDirectMlProvider(SessionOptions options, int deviceId)
 1445            => options.AppendExecutionProvider_DML(deviceId);
 446
 447        /// <summary>Appends the CoreML execution provider. Override to inject a throw in tests.</summary>
 448        protected virtual void AppendCoreMlProvider(SessionOptions options, CoreMLFlags flags)
 1449            => options.AppendExecutionProvider_CoreML(flags);
 450
 451        /// <summary>Constructs the ONNX session. Override to inject a recoverable native failure in tests.</summary>
 452        protected virtual InferenceSession CreateInferenceSession(string modelPath, SessionOptions options)
 53453            => new(modelPath, options);
 454
 369455        private int NextOrder() => _nextOrder++;
 456
 457        /// <summary>
 458        /// Single choke point for appending to the provider-selection check trail: keeps the list
 459        /// behavior identical while mirroring every check to the logger, so the forensic trail
 460        /// (which probe ran, what it found) lands in the log file even before EngineInfo reaches a
 461        /// consumer. Failed checks escalate to Warning; passes/skips log at Debug.
 462        /// </summary>
 463        private void AddCheck(ProviderDiagnosticCheck check)
 464        {
 369465            _pendingChecks.Add(check);
 369466            _logger.Log(
 369467                check.Result == Diagnostics.CheckResult.Failed ? LogLevel.Warning : LogLevel.Debug,
 369468                "check [{Order}] {Result} {Name} ({Category}): {Detail}",
 369469                check.Order, check.Result, check.Name, check.Category, check.Detail ?? check.Description);
 369470        }
 471
 472        private SessionOptions ConfiguredSessionOptions(
 473            out InferenceProvider provider,
 474            out InferenceFallbackReason fallback,
 475            out int deviceId)
 476        {
 54477            if (!_options.UseGpuAcceleration)
 478            {
 20479                AddCheck(ProviderDiagnosticCheck.Fail(
 20480                    "gpu.user-enabled",
 20481                    "UseGpuAcceleration is enabled",
 20482                    "Config",
 20483                    NextOrder(),
 20484                    "UseGpuAcceleration=false"));
 20485                AddCheck(ProviderDiagnosticCheck.Skip("rid.platform", "Host platform detection", "Platform", NextOrder()
 20486                AddCheck(ProviderDiagnosticCheck.Skip("provider.append", "Execution provider append", "Provider", NextOr
 20487                AddCheck(ProviderDiagnosticCheck.Skip("session.construct", "Session construction", "Session", NextOrder(
 488
 20489                provider = InferenceProvider.Cpu;
 20490                fallback = InferenceFallbackReason.UserDisabledGpu;
 20491                deviceId = 0;
 20492                var options = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL };
 20493                options.AppendExecutionProvider_CPU();
 20494                return options;
 495            }
 496
 34497            AddCheck(ProviderDiagnosticCheck.Pass("gpu.user-enabled", "UseGpuAcceleration is enabled", "Config", NextOrd
 498
 34499            var platform = EffectivePlatform;
 34500            var (sessionOptions, prov, fb, dev) = platform switch
 34501            {
 16502                HostPlatform.LinuxX64 => TryRocmBranch(),
 8503                HostPlatform.WinX64 or HostPlatform.WinArm64 => TryDirectMlBranch(),
 7504                HostPlatform.OSX64 or HostPlatform.OSXArm64 => TryCoreMlBranch(),
 3505                _ => TryCpuBranch(InferenceFallbackReason.UnsupportedPlatform),
 34506            };
 507
 34508            provider = prov;
 34509            fallback = fb;
 34510            deviceId = dev;
 34511            return sessionOptions;
 512        }
 513
 514        private (SessionOptions Options, InferenceProvider Provider, InferenceFallbackReason Fallback, int DeviceId) Try
 515        {
 16516            AddCheck(ProviderDiagnosticCheck.Pass("rid.platform", "Host is Linux x64", "Platform", NextOrder()));
 16517            var deviceId = _options.DeviceId;
 518
 16519            if (!IsKfdPresent())
 520            {
 4521                AddCheck(ProviderDiagnosticCheck.Fail("kfd.present", "/dev/kfd exists", "Provider", NextOrder(), "/dev/k
 4522                SkipRocmProviderChecks("precondition kfd.present failed");
 4523                return BuildCpuFallback(InferenceFallbackReason.KfdDeviceMissing, deviceId);
 524            }
 12525            AddCheck(ProviderDiagnosticCheck.Pass("kfd.present", "/dev/kfd exists", "Provider", NextOrder()));
 526
 12527            if (!TryLoadNativeLibrary("libhsa-runtime64.so", out var hsaError))
 528            {
 2529                AddCheck(ProviderDiagnosticCheck.Fail("hsa.dlopen", "libhsa-runtime64.so is loadable", "Provider", NextO
 2530                SkipRocmProviderChecks("precondition hsa.dlopen failed", skipHsa: false);
 2531                return BuildCpuFallback(InferenceFallbackReason.HsaRuntimeUnloadable, deviceId);
 532            }
 10533            AddCheck(ProviderDiagnosticCheck.Pass("hsa.dlopen", "libhsa-runtime64.so is loadable", "Provider", NextOrder
 534
 10535            if (!TryLoadNativeLibrary("libamdhip64.so.7", out var hipError))
 536            {
 2537                AddCheck(ProviderDiagnosticCheck.Fail("hip.dlopen", "libamdhip64.so.7 is loadable", "Provider", NextOrde
 2538                    hipError ?? "libamdhip64.so.7 not found. HIP runtime is required for MIGraphX GPU acceleration. " +
 2539                    "Install: sudo apt install hip-runtime-amd (from AMD ROCm repository). " +
 2540                    "See: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/"));
 2541                SkipRocmProviderChecks("precondition hip.dlopen failed", skipHsa: false, skipHip: false);
 2542                return BuildCpuFallback(InferenceFallbackReason.HipRuntimeMissing, deviceId);
 543            }
 8544            AddCheck(ProviderDiagnosticCheck.Pass("hip.dlopen", "libamdhip64.so.7 is loadable", "Provider", NextOrder())
 545
 8546            if (!TryLoadNativeLibrary("libmigraphx_c.so.3", out var mgxError))
 547            {
 2548                AddCheck(ProviderDiagnosticCheck.Fail("migraphx.dlopen", "libmigraphx_c.so.3 is loadable", "Provider", N
 2549                    mgxError ?? "libmigraphx_c.so.3 not found. MIGraphX runtime is required for GPU-accelerated inferenc
 2550                    "Install: sudo apt install migraphx (from AMD ROCm repository). " +
 2551                    "See: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/"));
 2552                SkipRocmProviderChecks("precondition migraphx.dlopen failed", skipHsa: false, skipHip: false, skipMigrap
 2553                return BuildCpuFallback(InferenceFallbackReason.MIGraphXRuntimeMissing, deviceId);
 554            }
 6555            AddCheck(ProviderDiagnosticCheck.Pass("migraphx.dlopen", "libmigraphx_c.so.3 is loadable", "Provider", NextO
 556
 6557            var options = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL };
 558            try
 559            {
 6560                AppendMIGraphXProvider(options, deviceId);
 5561                AddCheck(ProviderDiagnosticCheck.Pass("migraphx.append", "AppendExecutionProvider_MIGraphX", "Provider",
 5562                return (options, InferenceProvider.MIGraphX, InferenceFallbackReason.None, deviceId);
 563            }
 1564            catch (Exception ex)
 565            {
 1566                AddCheck(ProviderDiagnosticCheck.Fail("migraphx.append", "AppendExecutionProvider_MIGraphX", "Provider",
 1567                AddCheck(ProviderDiagnosticCheck.Skip("migraphx.session", "MIGraphX InferenceSession construction", "Ses
 1568                options.Dispose();
 1569                return BuildCpuFallback(InferenceFallbackReason.ProviderAppendFailed, deviceId);
 570            }
 6571        }
 572
 573        private void SkipRocmProviderChecks(string reason,
 574            bool skipHsa = true, bool skipHip = true, bool skipMigraphxDlopen = true,
 575            bool skipMigraphxAppend = true, bool skipMigraphxSession = true)
 576        {
 10577            if (skipHsa)
 4578                AddCheck(ProviderDiagnosticCheck.Skip("hsa.dlopen", "libhsa-runtime64.so is loadable", "Provider", NextO
 10579            if (skipHip)
 6580                AddCheck(ProviderDiagnosticCheck.Skip("hip.dlopen", "libamdhip64.so.7 is loadable", "Provider", NextOrde
 10581            if (skipMigraphxDlopen)
 8582                AddCheck(ProviderDiagnosticCheck.Skip("migraphx.dlopen", "libmigraphx_c.so.3 is loadable", "Provider", N
 10583            if (skipMigraphxAppend)
 10584                AddCheck(ProviderDiagnosticCheck.Skip("migraphx.append", "AppendExecutionProvider_MIGraphX succeeded", "
 10585            if (skipMigraphxSession)
 10586                AddCheck(ProviderDiagnosticCheck.Skip("migraphx.session", "MIGraphX InferenceSession construction", "Ses
 10587        }
 588
 589        private (SessionOptions Options, InferenceProvider Provider, InferenceFallbackReason Fallback, int DeviceId) Try
 590        {
 8591            AddCheck(ProviderDiagnosticCheck.Pass("rid.platform", "Host is Windows", "Platform", NextOrder()));
 8592            var deviceId = _options.DeviceId;
 8593            var options = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL };
 594
 595            try
 596            {
 8597                AppendDirectMlProvider(options, deviceId);
 4598                AddCheck(ProviderDiagnosticCheck.Pass("dml.append", "AppendExecutionProvider_DML", "Provider", NextOrder
 4599                return (options, InferenceProvider.DirectML, InferenceFallbackReason.None, deviceId);
 600            }
 4601            catch (Exception ex)
 602            {
 4603                AddCheck(ProviderDiagnosticCheck.Fail("dml.append", "AppendExecutionProvider_DML", "Provider", NextOrder
 4604                AddCheck(ProviderDiagnosticCheck.Skip("dml.session", "DirectML InferenceSession construction", "Session"
 4605                options.Dispose();
 4606                return BuildCpuFallback(InferenceFallbackReason.ProviderAppendFailed, deviceId);
 607            }
 8608        }
 609
 610        private (SessionOptions Options, InferenceProvider Provider, InferenceFallbackReason Fallback, int DeviceId) Try
 611        {
 7612            AddCheck(ProviderDiagnosticCheck.Pass("rid.platform", "Host is macOS", "Platform", NextOrder()));
 7613            var deviceId = _options.DeviceId;
 7614            var options = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL };
 615
 616            try
 617            {
 7618                AppendCoreMlProvider(options, CoreMLFlags.COREML_FLAG_ENABLE_ON_SUBGRAPH);
 3619                AddCheck(ProviderDiagnosticCheck.Pass("coreml.append", "AppendExecutionProvider_CoreML", "Provider", Nex
 3620                return (options, InferenceProvider.CoreML, InferenceFallbackReason.None, deviceId);
 621            }
 4622            catch (Exception ex)
 623            {
 4624                AddCheck(ProviderDiagnosticCheck.Fail("coreml.append", "AppendExecutionProvider_CoreML", "Provider", Nex
 4625                AddCheck(ProviderDiagnosticCheck.Skip("coreml.session", "CoreML InferenceSession construction", "Session
 4626                options.Dispose();
 4627                return BuildCpuFallback(InferenceFallbackReason.ProviderAppendFailed, deviceId);
 628            }
 7629        }
 630
 631        private (SessionOptions Options, InferenceProvider Provider, InferenceFallbackReason Fallback, int DeviceId) Try
 632        {
 3633            AddCheck(ProviderDiagnosticCheck.Fail("rid.platform", "Host platform detection", "Platform", NextOrder(), $"
 3634            AddCheck(ProviderDiagnosticCheck.Skip("provider.append", "Execution provider append", "Provider", NextOrder(
 3635            AddCheck(ProviderDiagnosticCheck.Skip("session.construct", "Session construction", "Session", NextOrder(), "
 3636            var options = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL };
 3637            options.AppendExecutionProvider_CPU();
 3638            return (options, InferenceProvider.Cpu, reason, 0);
 639        }
 640
 641        private (SessionOptions Options, InferenceProvider Provider, InferenceFallbackReason Fallback, int DeviceId) Bui
 642        {
 19643            var options = CreateCpuOnlySessionOptions();
 19644            return (options, InferenceProvider.Cpu, reason, deviceId);
 645        }
 646
 647        private void AppendSessionCheck(InferenceProvider provider, bool passed, string? detail = null)
 648        {
 14649            var name = provider switch
 14650            {
 5651                InferenceProvider.MIGraphX => "migraphx.session",
 4652                InferenceProvider.DirectML => "dml.session",
 3653                InferenceProvider.CoreML => "coreml.session",
 2654                _ => "cpu.session",
 14655            };
 14656            var desc = $"{provider} InferenceSession construction";
 14657            AddCheck(passed
 14658                ? ProviderDiagnosticCheck.Pass(name, desc, "Session", NextOrder(), detail)
 14659                : ProviderDiagnosticCheck.Fail(name, desc, "Session", NextOrder(), detail ?? "unknown failure"));
 14660        }
 661
 662        private void AppendFallbackChecks(SessionOptions cpuOptions, string modelPath)
 663        {
 3664            AddCheck(ProviderDiagnosticCheck.Pass("cpu.fallback-options", "CreateCpuOnlySessionOptions() succeeded", "Fa
 665            try
 666            {
 3667                _session = CreateInferenceSession(modelPath, cpuOptions);
 3668                AddCheck(ProviderDiagnosticCheck.Pass("cpu.fallback-session", "CPU InferenceSession constructed", "Fallb
 3669            }
 0670            catch (Exception ex)
 671            {
 0672                AddCheck(ProviderDiagnosticCheck.Fail("cpu.fallback-session", "CPU InferenceSession construction", "Fall
 0673                throw;
 674            }
 3675        }
 676
 677        private void FireOnSessionInitialized(InferenceEngineInfo info)
 678        {
 187679            if (_options.OnSessionInitialized is null) return;
 680            try
 681            {
 3682                _options.OnSessionInitialized(info);
 2683            }
 1684            catch (Exception ex)
 685            {
 1686                _logger.LogWarning(ex, "OnSessionInitialized callback threw; engine is still usable.");
 1687            }
 3688        }
 689
 690        /// <summary>
 691        /// Publishes a status transition: updates <see cref="Status"/> then fires
 692        /// <see cref="InferenceEngineOptions.OnStatusChanged"/> (callback exceptions are
 693        /// swallowed and logged â€” a broken consumer callback must not kill the engine).
 694        /// Private protected: out-of-process proxies drive their own transitions with the
 695        /// worker-reported terminal states.
 696        /// </summary>
 697        private protected void SetStatus(InferenceEngineStatus status)
 698        {
 307699            Status = status;
 596700            if (_options.OnStatusChanged is null) return;
 701            try
 702            {
 18703                _options.OnStatusChanged(status);
 16704            }
 2705            catch (Exception ex)
 706            {
 2707                _logger.LogWarning(ex, "OnStatusChanged callback threw; engine is still usable.");
 2708            }
 18709        }
 710
 711        /// <summary>
 712        /// Apply engine state that was loaded by an out-of-process worker (no in-process
 713        /// <see cref="InferenceSession"/>). Used by <c>OutOfProcessBaseInferenceEngine</c> to mirror
 714        /// the worker's <see cref="EngineInfo"/>/metadata so the diagnostics surface (provider,
 715        /// fallback reason, ordered checks) behaves identically to the in-process engine. Does not
 716        /// create a session; the proxy's <see cref="PredictAsync"/> marshals to the worker instead.
 717        /// </summary>
 718        protected void ApplyLoadedState(
 719            InferenceEngineInfo engineInfo,
 720            IReadOnlyDictionary<string, NodeMetadata>? inputMetadata,
 721            IReadOnlyDictionary<string, NodeMetadata>? outputMetadata,
 722            string? primaryInputName)
 723        {
 44724            EngineInfo = engineInfo;
 44725            InputMetadata = inputMetadata;
 44726            OutputMetadata = outputMetadata;
 44727            PrimaryInputName = primaryInputName;
 44728            _isLoaded = true;
 44729            FireOnSessionInitialized(engineInfo);
 44730        }
 731
 732        /// <summary>
 733        /// Per-provider key/value options actually applied. At minimum surfaces <c>device_id</c> for
 734        /// hardware-accelerated providers (MIGraphX/DirectML/CoreML); MIGraphX additionally surfaces
 735        /// <c>migraphx_model_cache_dir</c> when the compiled-model cache is enabled. Empty for
 736        /// CPU/Unknown.
 737        /// </summary>
 738        private IReadOnlyDictionary<string, string> ExtractProviderOptions(InferenceProvider provider, int deviceId)
 739        {
 54740            if (provider == InferenceProvider.Cpu || provider == InferenceProvider.Unknown)
 43741                return ImmutableDictionary<string, string>.Empty;
 742
 743            // Build the immutable tree directly instead of allocating a transient
 744            // Dictionary<string,string> + ToImmutableDictionary() on every session init.
 11745            var options = ImmutableDictionary<string, string>.Empty.Add("device_id", deviceId.ToString());
 11746            if (provider == InferenceProvider.MIGraphX && ResolveMIGraphXModelCacheDirectory() is { } cacheDir)
 4747                options = options.Add("migraphx_model_cache_dir", cacheDir);
 11748            return options;
 749        }
 750
 751        private static SessionOptions CreateCpuOnlySessionOptions()
 752        {
 22753            var options = new SessionOptions
 22754            {
 22755                GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL,
 22756            };
 22757            options.AppendExecutionProvider_CPU();
 22758            return options;
 759        }
 760
 761        /// <summary>
 762        /// Determines whether an exception thrown during ONNX session initialization represents a
 763        /// recoverable native library loading failure.
 764        /// </summary>
 765        private static bool IsRecoverableNativeFailure(Exception ex)
 766        {
 5767            if (ex is OnnxRuntimeException or DllNotFoundException or BadImageFormatException) return true;
 1768            if (ex is TypeInitializationException tiex)
 1769                return tiex.InnerException is OnnxRuntimeException or DllNotFoundException or BadImageFormatException;
 0770            return false;
 771        }
 772
 773        /// <summary>
 774        /// Generates dummy input for model warmup.
 775        /// Default implementation assumes float tensors. Override this for int64/string inputs.
 776        /// </summary>
 777        protected virtual List<NamedOnnxValue> GetWarmupInput()
 778        {
 2779            var inputs = new List<NamedOnnxValue>();
 3780            if (InputMetadata == null) return inputs;
 781
 4782            foreach (var inputMeta in InputMetadata)
 783            {
 2784                var shape = inputMeta.Value.Dimensions.Select(d => d == -1 ? 1 : d).ToArray();
 1785                var tensorData = new Microsoft.ML.OnnxRuntime.Tensors.DenseTensor<float>(shape);
 1786                inputs.Add(NamedOnnxValue.CreateFromTensor(inputMeta.Key, tensorData));
 787            }
 1788            return inputs;
 789        }
 790
 791        private void PerformWarmup()
 792        {
 793            try
 794            {
 3795                _logger.LogInformation("Performing warmup inference...");
 3796                var inputs = GetWarmupInput();
 797
 2798                if (inputs.Count > 0)
 799                {
 2800                    using var results = _session!.Run(inputs);
 1801                    _logger.LogInformation("Warmup complete.");
 802                }
 1803            }
 2804            catch (Exception ex)
 805            {
 2806                _logger.LogWarning($"Warmup failed (non-critical). Error: {ex.Message}");
 2807            }
 3808        }
 809
 810        public void Dispose()
 811        {
 160812            Dispose(true);
 160813            GC.SuppressFinalize(this);
 160814        }
 815
 816        protected virtual void Dispose(bool disposing)
 817        {
 168818            if (disposing)
 819            {
 168820                _session?.Dispose();
 168821                _runOptions?.Dispose();
 168822                _session = null;
 168823                _isLoaded = false;
 824                // Silent reset (no OnStatusChanged fire): a disposed engine's prior state is not
 825                // a transition consumers need to react to.
 168826                Status = InferenceEngineStatus.NotLoaded;
 168827                _loadLock.Dispose();
 828            }
 168829        }
 830    }
 831}

Methods/Properties

.ctor(InferenceEngine.Core.InferenceEngineOptions,Microsoft.Extensions.Logging.ILogger)
get_InputMetadata()
get_OutputMetadata()
get_PrimaryInputName()
get_IsRocmSessionActive()
get_EngineInfo()
get_Status()
get_ModelDownloadUrls()
get_ModelRegistry()
get_ModelCacheDirectory()
GetModelProvider()
LoadAsync()
PredictAsync()
get_EffectivePlatform()
IsKfdPresent()
TryLoadNativeLibrary(System.String,System.String&)
AppendMIGraphXProvider(Microsoft.ML.OnnxRuntime.SessionOptions,System.Int32)
ResolveMIGraphXModelCacheDirectory()
AppendDirectMlProvider(Microsoft.ML.OnnxRuntime.SessionOptions,System.Int32)
AppendCoreMlProvider(Microsoft.ML.OnnxRuntime.SessionOptions,Microsoft.ML.OnnxRuntime.CoreMLFlags)
CreateInferenceSession(System.String,Microsoft.ML.OnnxRuntime.SessionOptions)
NextOrder()
AddCheck(InferenceEngine.Core.Diagnostics.ProviderDiagnosticCheck)
ConfiguredSessionOptions(InferenceEngine.Core.Diagnostics.InferenceProvider&,InferenceEngine.Core.Diagnostics.InferenceFallbackReason&,System.Int32&)
TryRocmBranch()
SkipRocmProviderChecks(System.String,System.Boolean,System.Boolean,System.Boolean,System.Boolean,System.Boolean)
TryDirectMlBranch()
TryCoreMlBranch()
TryCpuBranch(InferenceEngine.Core.Diagnostics.InferenceFallbackReason)
BuildCpuFallback(InferenceEngine.Core.Diagnostics.InferenceFallbackReason,System.Int32)
AppendSessionCheck(InferenceEngine.Core.Diagnostics.InferenceProvider,System.Boolean,System.String)
AppendFallbackChecks(Microsoft.ML.OnnxRuntime.SessionOptions,System.String)
FireOnSessionInitialized(InferenceEngine.Core.Diagnostics.InferenceEngineInfo)
SetStatus(InferenceEngine.Core.Diagnostics.InferenceEngineStatus)
ApplyLoadedState(InferenceEngine.Core.Diagnostics.InferenceEngineInfo,System.Collections.Generic.IReadOnlyDictionary`2<System.String,Microsoft.ML.OnnxRuntime.NodeMetadata>,System.Collections.Generic.IReadOnlyDictionary`2<System.String,Microsoft.ML.OnnxRuntime.NodeMetadata>,System.String)
ExtractProviderOptions(InferenceEngine.Core.Diagnostics.InferenceProvider,System.Int32)
CreateCpuOnlySessionOptions()
IsRecoverableNativeFailure(System.Exception)
GetWarmupInput()
PerformWarmup()
Dispose()
Dispose(System.Boolean)