| | | 1 | | using System; |
| | | 2 | | using System.Collections.Generic; |
| | | 3 | | using System.Diagnostics; |
| | | 4 | | using System.IO; |
| | | 5 | | using System.Linq; |
| | | 6 | | using System.Threading; |
| | | 7 | | using System.Threading.Tasks; |
| | | 8 | | using InferenceEngine.Core.Diagnostics; |
| | | 9 | | using InferenceEngine.Core.Worker; |
| | | 10 | | using Microsoft.Extensions.Logging; |
| | | 11 | | |
| | | 12 | | namespace InferenceEngine.Core; |
| | | 13 | | |
| | | 14 | | /// <summary> |
| | | 15 | | /// Host-side proxy that offers the same contract as <see cref="BaseInferenceEngine{TInput,TOutput}"/> |
| | | 16 | | /// but runs the ONNX session in a <b>child worker process</b> over a length-prefixed JSON pipe. |
| | | 17 | | /// The host never loads <c>libonnxruntime</c>/comgr; the worker never loads Mesa/GL — avoiding the |
| | | 18 | | /// Mesa <c>libLLVM-19</c> ↔ comgr <c>libLLVM-20</c> collision that aborts an in-process MIGraphX + |
| | | 19 | | /// GL-rendered UI. <see cref="BaseInferenceEngine{TInput,TOutput}"/>.EngineInfo is mirrored from the worker's Load resp |
| | | 20 | | /// diagnostics surface (provider, fallback reason, ordered checks) is identical to the in-process engine. |
| | | 21 | | /// </summary> |
| | | 22 | | /// <remarks> |
| | | 23 | | /// <para> |
| | | 24 | | /// This base overrides <see cref="LoadAsync"/> and <see cref="PredictAsync"/> to marshal to the |
| | | 25 | | /// worker; it never creates an in-process <see cref="Microsoft.ML.OnnxRuntime.InferenceSession"/>. |
| | | 26 | | /// The consumer supplies the worker executable path (its own app launched with worker args) and an |
| | | 27 | | /// <see cref="OutOfProcessEngineOptions"/>. |
| | | 28 | | /// </para> |
| | | 29 | | /// <para> |
| | | 30 | | /// <b>Congcurrency (H1):</b> all load/predict IPC is serialized by <see cref="_inFlight"/> so that |
| | | 31 | | /// concurrent calls never interleave write/read on the shared pipe (which would silently pair the |
| | | 32 | | /// wrong request with the wrong response). <see cref="Dispose"/> (M6) also takes that lock so it |
| | | 33 | | /// cannot tear down the pipe while a call is mid-flight. |
| | | 34 | | /// </para> |
| | | 35 | | /// </remarks> |
| | | 36 | | public abstract class OutOfProcessBaseInferenceEngine<TInput, TOutput> : BaseInferenceEngine<TInput, TOutput>, IAsyncDis |
| | | 37 | | { |
| | | 38 | | private readonly OutOfProcessEngineOptions _oopOptions; |
| | | 39 | | private readonly IWorkerSerializer<TInput, TOutput> _serializer; |
| | | 40 | | |
| | | 41 | | // Serializes all host↔worker IPC so concurrent PredictAsync calls cannot interleave their |
| | | 42 | | // write/read pairs on the shared stdin/stdout pipe (H1). Dispose acquires it too (M6). |
| | 24 | 43 | | private readonly SemaphoreSlim _inFlight = new(1, 1); |
| | | 44 | | |
| | | 45 | | // Serializes this proxy's status transitions with the stderr sniffing path |
| | | 46 | | // (HandleWorkerStderrLine): a compile marker racing a terminal transition must be dropped, |
| | | 47 | | // never regress Loaded/Failed back to Loading. |
| | 24 | 48 | | private readonly object _statusGate = new(); |
| | | 49 | | |
| | | 50 | | private Process? _process; |
| | | 51 | | private Stream? _toWorker; // host → worker stdin |
| | | 52 | | private Stream? _fromWorker; // worker stdout → host |
| | | 53 | | private bool _workerLoaded; |
| | | 54 | | private bool _disposed; |
| | | 55 | | |
| | | 56 | | protected OutOfProcessBaseInferenceEngine(OutOfProcessEngineOptions options, ILogger? logger = null) |
| | 24 | 57 | | : base(options.EngineOptions, logger) |
| | | 58 | | { |
| | 24 | 59 | | _oopOptions = options ?? throw new ArgumentNullException(nameof(options)); |
| | 24 | 60 | | _serializer = CreateSerializer(); |
| | 24 | 61 | | } |
| | | 62 | | |
| | | 63 | | /// <summary>Serializer used for the IPC payload. Override to customize; defaults to |
| | | 64 | | /// <see cref="SystemTextJsonWorkerSerializer{TInput,TOutput}"/>.</summary> |
| | | 65 | | protected virtual IWorkerSerializer<TInput, TOutput> CreateSerializer() |
| | 24 | 66 | | => new SystemTextJsonWorkerSerializer<TInput, TOutput>(); |
| | | 67 | | |
| | | 68 | | // The model lives in the worker; the host proxy owns no model. |
| | 1 | 69 | | protected override string ModelName => _oopOptions.EngineId; |
| | | 70 | | |
| | | 71 | | /// <summary> |
| | | 72 | | /// Spawn the worker, send <c>load</c>, and mirror the worker's <see cref="BaseInferenceEngine{TInput,TOutput}"/>.En |
| | | 73 | | /// proxy via <see cref="BaseInferenceEngine{TInput,TOutput}.ApplyLoadedState"/>. Does NOT build an |
| | | 74 | | /// in-process session. Idempotent and thread-safe. |
| | | 75 | | /// </summary> |
| | | 76 | | public override async Task LoadAsync() |
| | | 77 | | { |
| | 28 | 78 | | await _inFlight.WaitAsync().ConfigureAwait(false); |
| | | 79 | | try |
| | | 80 | | { |
| | 29 | 81 | | if (_workerLoaded) return; |
| | 25 | 82 | | TransitionStatus(new InferenceEngineStatus { State = EngineLoadState.Loading, Detail = "Starting engine work |
| | 25 | 83 | | await LoadLockedAsync().ConfigureAwait(false); |
| | 13 | 84 | | } |
| | | 85 | | finally |
| | | 86 | | { |
| | 27 | 87 | | _inFlight.Release(); |
| | | 88 | | } |
| | 15 | 89 | | } |
| | | 90 | | |
| | | 91 | | /// <summary> |
| | | 92 | | /// Pre-loads the worker without holding a caller-visible lock re-entrancy. Assumes the caller |
| | | 93 | | /// already holds <see cref="_inFlight"/>. Spawns the worker (once), sends <c>load</c>, and mirrors |
| | | 94 | | /// the worker's diagnostics onto this proxy. |
| | | 95 | | /// </summary> |
| | | 96 | | private async Task LoadLockedAsync() |
| | | 97 | | { |
| | 26 | 98 | | ThrowIfDisposed(); |
| | | 99 | | try |
| | | 100 | | { |
| | 26 | 101 | | await LoadLockedCoreAsync().ConfigureAwait(false); |
| | 14 | 102 | | } |
| | 12 | 103 | | catch (Exception ex) when (Status.State != EngineLoadState.Failed) |
| | | 104 | | { |
| | | 105 | | // Pipe-level failure (worker died, timeout, EOF): no worker EngineInfo exists, but |
| | | 106 | | // the failure still surfaces through the shared status object model. |
| | 7 | 107 | | TransitionStatus(new InferenceEngineStatus { State = EngineLoadState.Failed, LoadError = ex.Message }); |
| | 7 | 108 | | throw; |
| | | 109 | | } |
| | 14 | 110 | | } |
| | | 111 | | |
| | | 112 | | private async Task LoadLockedCoreAsync() |
| | | 113 | | { |
| | 26 | 114 | | var sw = Stopwatch.StartNew(); |
| | 26 | 115 | | StartWorker(); |
| | 26 | 116 | | _logger.LogInformation("oop {EngineId}: worker spawned in {Elapsed}ms (pid={Pid}); sending load", |
| | 26 | 117 | | _oopOptions.EngineId, sw.ElapsedMilliseconds, TryGetWorkerPid()); |
| | | 118 | | // From here the worker owns the timeline (provider selection, session construction, the |
| | | 119 | | // ~1-2 min first-run MIGraphX kernel compile); stderr sniffing refines the Detail while |
| | | 120 | | // we wait for the load frame. |
| | 26 | 121 | | TransitionStatus(new InferenceEngineStatus { State = EngineLoadState.Loading, Detail = "Waiting for engine to lo |
| | | 122 | | |
| | 26 | 123 | | var engineOpts = _oopOptions.EngineOptions; |
| | 26 | 124 | | await WriteAsync(new WorkerEnvelope |
| | 26 | 125 | | { |
| | 26 | 126 | | Kind = "load", |
| | 26 | 127 | | EngineId = _oopOptions.EngineId, |
| | 26 | 128 | | Options = new WorkerEngineOptions |
| | 26 | 129 | | { |
| | 26 | 130 | | ModelPath = engineOpts.ModelPath, |
| | 26 | 131 | | UseGpuAcceleration = engineOpts.UseGpuAcceleration, |
| | 26 | 132 | | DeviceId = engineOpts.DeviceId, |
| | 26 | 133 | | WarmupOnLoad = engineOpts.WarmupOnLoad, |
| | 26 | 134 | | MIGraphXModelCacheDirectory = engineOpts.MIGraphXModelCacheDirectory, |
| | 26 | 135 | | }, |
| | 26 | 136 | | }, _oopOptions.StartupTimeout).ConfigureAwait(false); |
| | | 137 | | |
| | 25 | 138 | | var loaded = await ReadAsync(_oopOptions.StartupTimeout).ConfigureAwait(false) |
| | 25 | 139 | | ?? throw new InvalidOperationException("Worker closed the pipe during load."); |
| | 19 | 140 | | _logger.LogInformation("oop {EngineId}: load response received after {Elapsed}ms (provider={Provider}, fallback= |
| | 19 | 141 | | _oopOptions.EngineId, sw.ElapsedMilliseconds, loaded.EngineInfo?.Provider, loaded.EngineInfo?.IsFallback, lo |
| | | 142 | | |
| | 19 | 143 | | if (loaded.Error is not null) |
| | | 144 | | { |
| | 1 | 145 | | _logger.LogError("oop {EngineId}: worker load error: {Error}", _oopOptions.EngineId, loaded.Error); |
| | | 146 | | // Keep the worker's partial EngineInfo (checks run before the failure) in the status. |
| | 1 | 147 | | TransitionStatus(new InferenceEngineStatus |
| | 1 | 148 | | { |
| | 1 | 149 | | State = EngineLoadState.Failed, |
| | 1 | 150 | | Info = loaded.EngineInfo, |
| | 1 | 151 | | LoadError = loaded.Error, |
| | 1 | 152 | | }); |
| | 1 | 153 | | throw new InvalidOperationException($"Worker failed to load the engine: {loaded.Error}"); |
| | | 154 | | } |
| | 18 | 155 | | if (loaded.EngineInfo is null) |
| | | 156 | | { |
| | 4 | 157 | | _logger.LogError("oop {EngineId}: worker load response had no EngineInfo", _oopOptions.EngineId); |
| | 4 | 158 | | TransitionStatus(new InferenceEngineStatus |
| | 4 | 159 | | { |
| | 4 | 160 | | State = EngineLoadState.Failed, |
| | 4 | 161 | | LoadError = "Worker load response had no EngineInfo.", |
| | 4 | 162 | | }); |
| | 4 | 163 | | throw new InvalidOperationException("Worker load response had no EngineInfo."); |
| | | 164 | | } |
| | | 165 | | |
| | | 166 | | // Mirror the worker's loaded diagnostics. Metadata (NodeMetadata) is ORT-specific and not |
| | | 167 | | // serialized in this revision; it stays null on the proxy (a follow-up can carry name/dims). |
| | 14 | 168 | | ApplyLoadedState(loaded.EngineInfo, inputMetadata: null, outputMetadata: null, primaryInputName: null); |
| | 14 | 169 | | _workerLoaded = true; |
| | 14 | 170 | | TransitionStatus(new InferenceEngineStatus { State = EngineLoadState.Loaded, Info = loaded.EngineInfo }); |
| | 14 | 171 | | } |
| | | 172 | | |
| | | 173 | | /// <summary>Marshal a <c>predict</c> to the worker and return its deserialized result.</summary> |
| | | 174 | | public override async Task<TOutput> PredictAsync(TInput input) |
| | | 175 | | { |
| | | 176 | | // Serialize the whole operation (auto-load + write + read) so two concurrent callers each |
| | | 177 | | // pair their own request with their own response (H1). A second caller simply waits. |
| | 10 | 178 | | await _inFlight.WaitAsync().ConfigureAwait(false); |
| | | 179 | | try |
| | | 180 | | { |
| | 8 | 181 | | if (!_workerLoaded) |
| | 1 | 182 | | await LoadLockedAsync().ConfigureAwait(false); |
| | 8 | 183 | | ThrowIfDisposed(); |
| | | 184 | | |
| | 8 | 185 | | await WriteAsync(new WorkerEnvelope { Kind = "predict", Payload = _serializer.SerializeInput(input) }, |
| | 8 | 186 | | _oopOptions.InferenceTimeout).ConfigureAwait(false); |
| | | 187 | | |
| | 8 | 188 | | var result = await ReadAsync(_oopOptions.InferenceTimeout).ConfigureAwait(false) |
| | 8 | 189 | | ?? throw new InvalidOperationException("Worker closed the pipe during predict."); |
| | | 190 | | |
| | 5 | 191 | | if (result.Error is not null) |
| | | 192 | | { |
| | 1 | 193 | | _logger.LogError("oop {EngineId}: worker predict failed: {Error}", _oopOptions.EngineId, result.Error); |
| | 1 | 194 | | throw new InvalidOperationException($"Worker predict failed: {result.Error}"); |
| | | 195 | | } |
| | 4 | 196 | | if (result.Payload is null) |
| | 1 | 197 | | throw new InvalidOperationException("Worker predict returned no payload."); |
| | | 198 | | |
| | 3 | 199 | | return _serializer.DeserializeOutput(result.Payload); |
| | | 200 | | } |
| | | 201 | | finally |
| | | 202 | | { |
| | 8 | 203 | | _inFlight.Release(); |
| | | 204 | | } |
| | 3 | 205 | | } |
| | | 206 | | |
| | | 207 | | // ---- plumbing ----------------------------------------------------------- |
| | | 208 | | |
| | | 209 | | /// <summary> |
| | | 210 | | /// Spawn the worker process and wire its stdin/stdout. Override in tests to bypass process |
| | | 211 | | /// creation and inject in-memory streams (call <see cref="SetWorkerStreamsForTesting"/>). |
| | | 212 | | /// </summary> |
| | | 213 | | protected virtual void StartWorker() |
| | | 214 | | { |
| | | 215 | | // Clean up any previously-started worker (e.g. a Load retry after a failure) BEFORE spawning |
| | | 216 | | // a new one, to avoid leaking worker processes / GPU memory. |
| | 5 | 217 | | if (_process is not null) |
| | | 218 | | { |
| | 5 | 219 | | try { if (!_process.HasExited) _process.Kill(); } |
| | 0 | 220 | | catch { /* best-effort */ } |
| | 2 | 221 | | _toWorker?.Dispose(); |
| | 2 | 222 | | _fromWorker?.Dispose(); |
| | 2 | 223 | | _process.Dispose(); |
| | 2 | 224 | | _process = null; |
| | 2 | 225 | | _toWorker = null; |
| | 2 | 226 | | _fromWorker = null; |
| | | 227 | | } |
| | | 228 | | |
| | 5 | 229 | | var psi = new ProcessStartInfo(_oopOptions.WorkerExecutablePath) |
| | 5 | 230 | | { |
| | 5 | 231 | | UseShellExecute = false, |
| | 5 | 232 | | RedirectStandardInput = true, |
| | 5 | 233 | | RedirectStandardOutput = true, |
| | 5 | 234 | | RedirectStandardError = true, |
| | 5 | 235 | | CreateNoWindow = true, |
| | 5 | 236 | | }; |
| | 22 | 237 | | foreach (var arg in _oopOptions.WorkerArguments) |
| | 6 | 238 | | psi.ArgumentList.Add(arg); |
| | 12 | 239 | | foreach (var kv in _oopOptions.WorkerEnvironment) |
| | 1 | 240 | | psi.Environment[kv.Key] = kv.Value; |
| | | 241 | | |
| | 5 | 242 | | _logger.LogInformation("oop {EngineId}: spawning worker exe={Exe} args={Args} envOverrides={Env}", |
| | 5 | 243 | | _oopOptions.EngineId, _oopOptions.WorkerExecutablePath, |
| | 5 | 244 | | string.Join(" ", _oopOptions.WorkerArguments), |
| | 5 | 245 | | _oopOptions.WorkerEnvironment.Count == 0 |
| | 5 | 246 | | ? "<none>" |
| | 6 | 247 | | : string.Join(",", _oopOptions.WorkerEnvironment.Select(kv => $"{kv.Key}={kv.Value}"))); |
| | | 248 | | |
| | 5 | 249 | | _process = new Process { StartInfo = psi, EnableRaisingEvents = true }; |
| | | 250 | | // Worker logs pass through to the host logger (stderr only; stdout is protocol-only). |
| | 5 | 251 | | _process.ErrorDataReceived += (_, e) => |
| | 5 | 252 | | { |
| | 7 | 253 | | if (!string.IsNullOrEmpty(e.Data)) |
| | 3 | 254 | | HandleWorkerStderrLine(e.Data); |
| | 12 | 255 | | }; |
| | | 256 | | // An unexpected worker death mid-session is the classic "engine silently stops working" |
| | | 257 | | // signature: capture the exit code forensically. Uses the sender (not the _process field) |
| | | 258 | | // because teardown may null the field before the event fires. |
| | 9 | 259 | | _process.Exited += (sender, _) => _logger.LogWarning( |
| | 9 | 260 | | "oop {EngineId}: worker process exited (code={ExitCode})", |
| | 9 | 261 | | _oopOptions.EngineId, TryGetExitCode(sender as Process)); |
| | | 262 | | |
| | 5 | 263 | | if (!_process.Start()) |
| | 0 | 264 | | throw new InvalidOperationException($"Failed to start worker process: {_oopOptions.WorkerExecutablePath}"); |
| | | 265 | | |
| | 5 | 266 | | _process.BeginErrorReadLine(); |
| | 5 | 267 | | _toWorker = _process.StandardInput.BaseStream; |
| | 5 | 268 | | _fromWorker = _process.StandardOutput.BaseStream; |
| | 5 | 269 | | } |
| | | 270 | | |
| | | 271 | | private static int TryGetExitCode(Process? process) |
| | | 272 | | { |
| | 4 | 273 | | try { return process is not null && process.HasExited ? process.ExitCode : -1; } |
| | 0 | 274 | | catch { return -1; } |
| | 4 | 275 | | } |
| | | 276 | | |
| | | 277 | | /// <summary>Publishes a proxy status transition under <see cref="_statusGate"/> so it is |
| | | 278 | | /// ordered against <see cref="HandleWorkerStderrLine"/> (which may fire from the stderr |
| | | 279 | | /// pump thread at any point during the load read).</summary> |
| | | 280 | | private void TransitionStatus(InferenceEngineStatus status) |
| | | 281 | | { |
| | 77 | 282 | | lock (_statusGate) |
| | 77 | 283 | | SetStatus(status); |
| | 77 | 284 | | } |
| | | 285 | | |
| | | 286 | | /// <summary> |
| | | 287 | | /// Single choke point for every worker stderr line: logs it, and while a load is in flight |
| | | 288 | | /// maps the native MIGraphX execution provider's compile markers onto the status object |
| | | 289 | | /// model (<see cref="InferenceEngineStatus.Detail"/>) so consumers can show what the slow |
| | | 290 | | /// first-run kernel compile is doing instead of a bare "loading". The markers are pinned |
| | | 291 | | /// against the ORT 1.24.1.36 ROCm native build (<c>libonnxruntime_providers_migraphx</c>: |
| | | 292 | | /// "Model Compile: Begin" / "Model Compile: Complete"); any other line is logged only. |
| | | 293 | | /// Virtual test seam — never spawns a process itself. |
| | | 294 | | /// </summary> |
| | | 295 | | protected virtual void HandleWorkerStderrLine(string line) |
| | | 296 | | { |
| | 8 | 297 | | _logger?.LogWarning("worker stderr: {Line}", line); |
| | | 298 | | |
| | | 299 | | // Guarded by _statusGate together with TransitionStatus: a marker that races a terminal |
| | | 300 | | // transition is dropped, so Loaded/Failed can never regress back to Loading. |
| | 8 | 301 | | lock (_statusGate) |
| | | 302 | | { |
| | 11 | 303 | | if (Status.State != EngineLoadState.Loading) return; |
| | 5 | 304 | | if (line.Contains("Model Compile: Begin", StringComparison.Ordinal)) |
| | 3 | 305 | | SetStatus(new InferenceEngineStatus |
| | 3 | 306 | | { |
| | 3 | 307 | | State = EngineLoadState.Loading, |
| | 3 | 308 | | Detail = "Compiling MIGraphX kernels (first run can take 1-2 minutes)", |
| | 3 | 309 | | }); |
| | 2 | 310 | | else if (line.Contains("Model Compile: Complete", StringComparison.Ordinal)) |
| | 1 | 311 | | SetStatus(new InferenceEngineStatus |
| | 1 | 312 | | { |
| | 1 | 313 | | State = EngineLoadState.Loading, |
| | 1 | 314 | | Detail = "Kernel compile complete", |
| | 1 | 315 | | }); |
| | 2 | 316 | | } |
| | 8 | 317 | | } |
| | | 318 | | |
| | | 319 | | private int? TryGetWorkerPid() |
| | | 320 | | { |
| | 26 | 321 | | try { return _process?.Id; } |
| | 0 | 322 | | catch { return null; } |
| | 26 | 323 | | } |
| | | 324 | | |
| | | 325 | | /// <summary>Test seam: wire the host↔worker streams directly without spawning a process.</summary> |
| | | 326 | | internal void SetWorkerStreamsForTesting(Stream toWorker, Stream fromWorker) |
| | | 327 | | { |
| | 20 | 328 | | _toWorker = toWorker; |
| | 20 | 329 | | _fromWorker = fromWorker; |
| | 20 | 330 | | } |
| | | 331 | | |
| | | 332 | | private async Task WriteAsync(WorkerEnvelope envelope, TimeSpan timeout) |
| | | 333 | | { |
| | 34 | 334 | | ThrowIfDisposed(); |
| | | 335 | | // Race the write against a timeout. Redirected stdin pipes do not reliably honor |
| | | 336 | | // cancellation, so on timeout we kill the worker (closing the read end of stdin) to unblock |
| | | 337 | | // the pending write and then surface a TimeoutException (H2). |
| | | 338 | | // Task.WaitAsync(TimeSpan) avoids the Task.Delay timer leak that Task.WhenAny would cause on |
| | | 339 | | // this per-prediction hot path. |
| | 34 | 340 | | var writeTask = WorkerFrame.WriteAsync(_toWorker!, envelope, CancellationToken.None); |
| | | 341 | | try |
| | | 342 | | { |
| | 34 | 343 | | await writeTask.WaitAsync(timeout).ConfigureAwait(false); |
| | 33 | 344 | | } |
| | 1 | 345 | | catch (TimeoutException) |
| | | 346 | | { |
| | 1 | 347 | | _logger.LogError("oop {EngineId}: pipe write timed out after {Timeout}; killing worker", |
| | 1 | 348 | | _oopOptions.EngineId, timeout); |
| | 1 | 349 | | KillWorker(); |
| | 1 | 350 | | throw new TimeoutException($"Worker did not accept a write within {timeout}."); |
| | | 351 | | } |
| | 33 | 352 | | } |
| | | 353 | | |
| | | 354 | | private async Task<WorkerEnvelope?> ReadAsync(TimeSpan timeout) |
| | | 355 | | { |
| | 33 | 356 | | ThrowIfDisposed(); |
| | | 357 | | // Redirected process stdout pipes do not promptly honor CancellationToken, so race the read |
| | | 358 | | // against a timeout; on expiry kill the worker and dispose the read stream to unblock the |
| | | 359 | | // pending read, then surface a TimeoutException (H2). |
| | | 360 | | // Task.WaitAsync(TimeSpan) avoids the Task.Delay timer leak that Task.WhenAny would cause on |
| | | 361 | | // this per-prediction hot path. |
| | 33 | 362 | | var readTask = WorkerFrame.ReadAsync(_fromWorker!, CancellationToken.None); |
| | | 363 | | try |
| | | 364 | | { |
| | 33 | 365 | | return await readTask.WaitAsync(timeout).ConfigureAwait(false); |
| | | 366 | | } |
| | 4 | 367 | | catch (TimeoutException) |
| | | 368 | | { |
| | 4 | 369 | | _logger.LogError("oop {EngineId}: pipe read timed out after {Timeout}; killing worker", |
| | 4 | 370 | | _oopOptions.EngineId, timeout); |
| | 4 | 371 | | KillWorker(); |
| | 4 | 372 | | throw new TimeoutException($"Worker did not respond within {timeout}."); |
| | | 373 | | } |
| | 28 | 374 | | } |
| | | 375 | | |
| | | 376 | | /// <summary> |
| | | 377 | | /// Forcefully terminates the worker process and disposes the host-side streams so any pending |
| | | 378 | | /// blocked read/write on the pipe unblocks. Best-effort; swallow teardown errors. |
| | | 379 | | /// </summary> |
| | | 380 | | private void KillWorker() |
| | | 381 | | { |
| | 11 | 382 | | try { if (_process is not null && !_process.HasExited) _process.Kill(); } |
| | 0 | 383 | | catch { /* best-effort */ } |
| | 10 | 384 | | try { _fromWorker?.Dispose(); } catch { /* best-effort */ } |
| | 10 | 385 | | try { _toWorker?.Dispose(); } catch { /* best-effort */ } |
| | 5 | 386 | | } |
| | | 387 | | |
| | | 388 | | private void ThrowIfDisposed() |
| | | 389 | | { |
| | 101 | 390 | | if (_disposed) throw new ObjectDisposedException(nameof(OutOfProcessBaseInferenceEngine<TInput, TOutput>)); |
| | 101 | 391 | | } |
| | | 392 | | |
| | | 393 | | /// <summary> |
| | | 394 | | /// Asynchronously tears down the worker: sends a graceful <c>shutdown</c> frame without |
| | | 395 | | /// blocking, then disposes the pipe/process and base session. Prefer |
| | | 396 | | /// <c>await using</c> over the synchronous <see cref="Dispose"/> — the synchronous path cannot |
| | | 397 | | /// (and must not) perform a sync-over-async write. |
| | | 398 | | /// </summary> |
| | | 399 | | public virtual async ValueTask DisposeAsync() |
| | | 400 | | { |
| | 8 | 401 | | if (!_disposed) |
| | | 402 | | { |
| | 7 | 403 | | await _inFlight.WaitAsync().ConfigureAwait(false); |
| | | 404 | | try |
| | | 405 | | { |
| | 7 | 406 | | if (!_disposed) |
| | | 407 | | { |
| | 7 | 408 | | _disposed = true; |
| | 7 | 409 | | await SendShutdownFrameAsync().ConfigureAwait(false); |
| | 7 | 410 | | TearDownPipeAndProcess(); |
| | | 411 | | } |
| | 7 | 412 | | } |
| | | 413 | | finally |
| | | 414 | | { |
| | 7 | 415 | | _inFlight.Release(); |
| | | 416 | | } |
| | 7 | 417 | | _inFlight.Dispose(); |
| | | 418 | | } |
| | | 419 | | |
| | 8 | 420 | | Dispose(disposing: true); |
| | 8 | 421 | | GC.SuppressFinalize(this); |
| | 8 | 422 | | } |
| | | 423 | | |
| | | 424 | | /// <summary>Best-effort asynchronous graceful-shutdown frame, only when a live pipe + process exist.</summary> |
| | | 425 | | private async ValueTask SendShutdownFrameAsync() |
| | | 426 | | { |
| | | 427 | | try |
| | | 428 | | { |
| | 7 | 429 | | if (_toWorker is not null && _process is not null && !_process.HasExited) |
| | | 430 | | { |
| | | 431 | | try |
| | | 432 | | { |
| | 1 | 433 | | await WorkerFrame.WriteAsync(_toWorker, new WorkerEnvelope { Kind = "shutdown" }, CancellationToken. |
| | 1 | 434 | | .ConfigureAwait(false); |
| | 1 | 435 | | } |
| | 0 | 436 | | catch { /* best-effort */ } |
| | | 437 | | } |
| | 7 | 438 | | } |
| | 0 | 439 | | catch { /* swallow teardown errors */ } |
| | 7 | 440 | | } |
| | | 441 | | |
| | | 442 | | /// <summary> |
| | | 443 | | /// Disposes the host side of the pipe and the worker process. Disposing stdin FIRST closes the |
| | | 444 | | /// worker's read end, which signals EOF and unblocks any pending stdout/stderr write so |
| | | 445 | | /// <see cref="Process.WaitForExit(int)"/> cannot deadlock; then the process is reaped/killed. |
| | | 446 | | /// </summary> |
| | | 447 | | private void TearDownPipeAndProcess() |
| | | 448 | | { |
| | 40 | 449 | | try { _toWorker?.Dispose(); } catch { /* best-effort */ } |
| | 40 | 450 | | try { _fromWorker?.Dispose(); } catch { /* best-effort */ } |
| | 20 | 451 | | _toWorker = null; |
| | 20 | 452 | | _fromWorker = null; |
| | | 453 | | |
| | | 454 | | try |
| | | 455 | | { |
| | 20 | 456 | | if (_process is not null && !_process.HasExited) |
| | | 457 | | { |
| | 1 | 458 | | if (!_process.WaitForExit(2000)) |
| | 0 | 459 | | try { _process.Kill(); } catch { /* best-effort */ } |
| | | 460 | | } |
| | 20 | 461 | | } |
| | 0 | 462 | | catch { /* swallow teardown errors */ } |
| | | 463 | | |
| | 20 | 464 | | _process?.Dispose(); |
| | 20 | 465 | | _process = null; |
| | 20 | 466 | | } |
| | | 467 | | |
| | | 468 | | protected override void Dispose(bool disposing) |
| | | 469 | | { |
| | 22 | 470 | | if (disposing && !_disposed) |
| | | 471 | | { |
| | | 472 | | // Take the in-flight lock (M6) so Dispose cannot tear down the pipe while a Load/Predict is |
| | | 473 | | // mid-write/read. In-flight IPC either completes or times out (H2) before releasing the lock. |
| | 13 | 474 | | _inFlight.Wait(); |
| | | 475 | | try |
| | | 476 | | { |
| | 13 | 477 | | if (!_disposed) |
| | | 478 | | { |
| | 13 | 479 | | _disposed = true; |
| | | 480 | | // Best-effort graceful shutdown via the synchronous WorkerFrame.Write (added on |
| | | 481 | | // main) — this avoids the sync-over-async .GetAwaiter().GetResult() that |
| | | 482 | | // originally lived here, while still sending an explicit shutdown frame (better |
| | | 483 | | // than relying on stdin EOF alone). Only sent when a live pipe + process exist; |
| | | 484 | | // the async DisposeAsync path sends the same frame without blocking. |
| | | 485 | | try |
| | | 486 | | { |
| | 13 | 487 | | if (_toWorker is not null && _process is not null && !_process.HasExited) |
| | 0 | 488 | | WorkerFrame.Write(_toWorker, new WorkerEnvelope { Kind = "shutdown" }); |
| | 13 | 489 | | } |
| | 0 | 490 | | catch { /* best-effort */ } |
| | 13 | 491 | | TearDownPipeAndProcess(); |
| | | 492 | | } |
| | 13 | 493 | | } |
| | | 494 | | finally |
| | | 495 | | { |
| | 13 | 496 | | _inFlight.Release(); |
| | 13 | 497 | | } |
| | | 498 | | |
| | 13 | 499 | | _inFlight.Dispose(); |
| | | 500 | | } |
| | 22 | 501 | | base.Dispose(disposing); |
| | 22 | 502 | | } |
| | | 503 | | |
| | | 504 | | // The one-shot pre/post hooks are unused: this engine marshals to the worker. |
| | | 505 | | protected sealed override (List<Microsoft.ML.OnnxRuntime.NamedOnnxValue> InputTensors, object? Context) |
| | | 506 | | PreProcessWithContext(TInput input) => |
| | 0 | 507 | | throw new NotSupportedException( |
| | 0 | 508 | | "Out-of-process engines marshal Load/Predict to a worker; PreProcessWithContext is not used."); |
| | | 509 | | protected sealed override TOutput PostProcessWithContext( |
| | | 510 | | Microsoft.ML.OnnxRuntime.IDisposableReadOnlyCollection<Microsoft.ML.OnnxRuntime.DisposableNamedOnnxValue> output |
| | | 511 | | object? context) => |
| | 0 | 512 | | throw new NotSupportedException( |
| | 0 | 513 | | "Out-of-process engines marshal Load/Predict to a worker; PostProcessWithContext is not used."); |
| | | 514 | | } |