| | | 1 | | using InferenceEngine.Core.Diagnostics; |
| | | 2 | | using InferenceEngine.Core.ModelProviders; |
| | | 3 | | |
| | | 4 | | namespace InferenceEngine.Core |
| | | 5 | | { |
| | | 6 | | /// <summary> |
| | | 7 | | /// Configuration options for inference engine initialization. |
| | | 8 | | /// </summary> |
| | | 9 | | public class InferenceEngineOptions |
| | | 10 | | { |
| | | 11 | | /// <summary> |
| | | 12 | | /// Optional model provider. If not set, the engine will use its default provider |
| | | 13 | | /// (as defined by CreateDefaultModelProvider() in the specific engine class). |
| | | 14 | | /// </summary> |
| | 280 | 15 | | public IModelProvider? ModelProvider { get; set; } |
| | | 16 | | |
| | | 17 | | /// <summary> |
| | | 18 | | /// Convenience property for local model paths. |
| | | 19 | | /// Setting this creates a LocalModelProvider internally. |
| | | 20 | | /// </summary> |
| | | 21 | | public string? ModelPath |
| | | 22 | | { |
| | 27 | 23 | | get => ModelProvider is LocalModelProvider local ? local.ModelPath : null; |
| | 29 | 24 | | set => ModelProvider = value is not null ? new LocalModelProvider(value) : null; |
| | | 25 | | } |
| | | 26 | | |
| | | 27 | | /// <summary> |
| | | 28 | | /// If true, attempts to use DirectML (Windows) or CoreML (macOS). |
| | | 29 | | /// If false, forces CPU execution. |
| | | 30 | | /// </summary> |
| | 534 | 31 | | public bool UseGpuAcceleration { get; set; } = true; |
| | | 32 | | |
| | | 33 | | /// <summary> |
| | | 34 | | /// Specific GPU Device ID (default 0). Useful for multi-GPU setups. |
| | | 35 | | /// </summary> |
| | 129 | 36 | | public int DeviceId { get; set; } = 0; |
| | | 37 | | |
| | | 38 | | /// <summary> |
| | | 39 | | /// If true, runs a dummy inference during LoadAsync to force |
| | | 40 | | /// shader compilation/caching immediately. |
| | | 41 | | /// </summary> |
| | 415 | 42 | | public bool WarmupOnLoad { get; set; } = true; |
| | | 43 | | |
| | | 44 | | /// <summary> |
| | | 45 | | /// Directory for the MIGraphX compiled-model (.mxr) cache (Linux x64 only). When |
| | | 46 | | /// enabled, the MIGraphX execution provider persists each compiled program to disk and |
| | | 47 | | /// reloads it on later startups, cutting engine load from a ~1-2 minute kernel compile |
| | | 48 | | /// to a ~1-2 second cache load. Cache files are keyed by MIGraphX version, graph |
| | | 49 | | /// identity, GPU architecture, and input shapes, so model swaps, ROCm/MIGraphX |
| | | 50 | | /// upgrades, or a different GPU naturally miss the cache and recompile. |
| | | 51 | | /// <list type="bullet"> |
| | | 52 | | /// <item><c>null</c> (default): use <c>%LocalAppData%/InferenceEngine/MIGraphXModelCache</c>.</item> |
| | | 53 | | /// <item><c>""</c> (empty string): disable .mxr caching (every startup recompiles).</item> |
| | | 54 | | /// <item>non-empty: use that directory verbatim (created if missing).</item> |
| | | 55 | | /// </list> |
| | | 56 | | /// Note: quantization flags (fp16/int8) are not part of the cache key — delete the |
| | | 57 | | /// directory after changing quantization settings. |
| | | 58 | | /// </summary> |
| | 46 | 59 | | public string? MIGraphXModelCacheDirectory { get; set; } |
| | | 60 | | |
| | | 61 | | /// <summary> |
| | | 62 | | /// Optional callback invoked on every engine status transition (Loading → Loaded or |
| | | 63 | | /// Failed), on the LoadAsync thread pool thread. Receives the full |
| | | 64 | | /// <see cref="Diagnostics.InferenceEngineStatus"/> object model — on failure its |
| | | 65 | | /// <c>Info</c> carries the partial provider-selection trail and <c>LoadError</c> the |
| | | 66 | | /// error. Exceptions thrown by the callback are caught and logged as warnings; they do |
| | | 67 | | /// not abort the engine. |
| | | 68 | | /// <para> |
| | | 69 | | /// This is the recommended hook; <see cref="OnSessionInitialized"/> remains as the |
| | | 70 | | /// success-only equivalent. |
| | | 71 | | /// </para> |
| | | 72 | | /// </summary> |
| | 331 | 73 | | public Action<Diagnostics.InferenceEngineStatus>? OnStatusChanged { get; set; } |
| | | 74 | | |
| | | 75 | | /// <summary> |
| | | 76 | | /// Optional callback invoked once, on the LoadAsync thread pool thread, after the session |
| | | 77 | | /// is created (including fallback paths). Receives the fully-populated InferenceEngineInfo. |
| | | 78 | | /// Exceptions thrown by the callback are caught and logged as warnings; they do not |
| | | 79 | | /// abort the engine. |
| | | 80 | | /// <para> |
| | | 81 | | /// Invoked at most once per LoadAsync call. If LoadAsync is called again after the engine |
| | | 82 | | /// is already loaded, the callback does not fire. |
| | | 83 | | /// </para> |
| | | 84 | | /// <para> |
| | | 85 | | /// Runs on a Task.Run thread pool thread, not the caller's synchronization context. |
| | | 86 | | /// Consumers that touch UI state must marshal to the UI thread themselves. |
| | | 87 | | /// </para> |
| | | 88 | | /// </summary> |
| | 101 | 89 | | public Action<InferenceEngineInfo>? OnSessionInitialized { get; set; } |
| | | 90 | | |
| | | 91 | | /// <summary> |
| | | 92 | | /// Maximum size in bytes of a single model (or tokenizer asset) download. Downloads whose |
| | | 93 | | /// Content-Length or running byte count exceeds this are aborted and the partial file is |
| | | 94 | | /// deleted. Must be positive; the default is 2 GiB. This is a safety cap against |
| | | 95 | | /// compromised or misbehaving download sources, not a tuning knob. |
| | | 96 | | /// </summary> |
| | 210 | 97 | | public long MaxModelDownloadBytes { get; set; } = ModelProviders.DownloadingModelProvider.DefaultMaxDownloadByte |
| | | 98 | | } |
| | | 99 | | } |