| | | 1 | | using System; |
| | | 2 | | using System.Diagnostics; |
| | | 3 | | using System.IO; |
| | | 4 | | using System.Threading; |
| | | 5 | | using System.Threading.Tasks; |
| | | 6 | | using Microsoft.Extensions.Logging; |
| | | 7 | | using Microsoft.Extensions.Logging.Abstractions; |
| | | 8 | | |
| | | 9 | | namespace InferenceEngine.Core.Worker; |
| | | 10 | | |
| | | 11 | | /// <summary> |
| | | 12 | | /// The child-process message loop. The consumer's worker executable calls |
| | | 13 | | /// <see cref="RunAsync{TInput,TOutput}"/> with a factory that builds ITS engine (a normal |
| | | 14 | | /// <see cref="BaseInferenceEngine{TInput,TOutput}"/> — exactly like the headless |
| | | 15 | | /// <c>--diag inference</c> path). The loop reads length-prefixed JSON requests on stdin and writes |
| | | 16 | | /// responses on stdout; logging passes through on stderr. Because this process loads no |
| | | 17 | | /// Avalonia/GL, comgr's LLVM is the only LLVM in it → no Mesa/comgr collision. |
| | | 18 | | /// </summary> |
| | | 19 | | public static class InferenceWorkerHost |
| | | 20 | | { |
| | | 21 | | /// <param name="engineFactory">Builds the consumer's engine from the host-supplied options.</param> |
| | | 22 | | /// <param name="serializer">Optional serializer; defaults to <see cref="SystemTextJsonWorkerSerializer{TInput,TOutp |
| | | 23 | | /// <param name="input">Optional input stream (stdin). Tests inject a pipe; production uses console stdin.</param> |
| | | 24 | | /// <param name="output">Optional output stream (stdout). Tests inject a pipe; production uses console stdout.</para |
| | | 25 | | /// <param name="cancellationToken">Observed between requests.</param> |
| | | 26 | | /// <param name="logger">Optional logger for the worker loop itself (frame tracing, load |
| | | 27 | | /// outcomes, timings). stdout stays protocol-only; logger output goes to whatever sink the |
| | | 28 | | /// consumer configured (typically stderr, which the host proxy forwards).</param> |
| | | 29 | | public static async Task<int> RunAsync<TInput, TOutput>( |
| | | 30 | | Func<InferenceEngineOptions, BaseInferenceEngine<TInput, TOutput>> engineFactory, |
| | | 31 | | IWorkerSerializer<TInput, TOutput>? serializer = null, |
| | | 32 | | Stream? input = null, |
| | | 33 | | Stream? output = null, |
| | | 34 | | CancellationToken cancellationToken = default, |
| | | 35 | | ILogger? logger = null) |
| | | 36 | | { |
| | 11 | 37 | | var log = logger ?? NullLogger.Instance; |
| | 11 | 38 | | serializer ??= new SystemTextJsonWorkerSerializer<TInput, TOutput>(); |
| | | 39 | | |
| | | 40 | | // Redirect engine logging away from stdout (stdout is protocol-only); use stderr. |
| | 11 | 41 | | var stdin = input ?? Console.OpenStandardInput(); |
| | 11 | 42 | | var stdout = output ?? Console.OpenStandardOutput(); |
| | | 43 | | |
| | | 44 | | // In production (no injected streams), redirect Console.Out to Console.Error so any stray |
| | | 45 | | // Console.WriteLine from managed code or libraries cannot corrupt the stdout IPC channel. |
| | | 46 | | // (When streams are injected — i.e. in tests — leave the console untouched.) |
| | 11 | 47 | | if (output is null) |
| | | 48 | | { |
| | 1 | 49 | | Console.SetOut(Console.Error); |
| | | 50 | | } |
| | | 51 | | // Keep stdout free of any stray text writes — protocol frames only. |
| | | 52 | | |
| | 11 | 53 | | log.LogInformation("worker loop: started (pid={Pid}, {Mode})", |
| | 11 | 54 | | Environment.ProcessId, output is null ? "console stdio" : "injected streams"); |
| | | 55 | | |
| | 11 | 56 | | BaseInferenceEngine<TInput, TOutput>? engine = null; |
| | | 57 | | try |
| | | 58 | | { |
| | 23 | 59 | | while (!cancellationToken.IsCancellationRequested) |
| | | 60 | | { |
| | | 61 | | WorkerEnvelope? msg; |
| | | 62 | | try |
| | | 63 | | { |
| | 23 | 64 | | msg = await WorkerFrame.ReadAsync(stdin, cancellationToken).ConfigureAwait(false); |
| | 22 | 65 | | } |
| | 1 | 66 | | catch (OperationCanceledException) |
| | | 67 | | { |
| | 1 | 68 | | log.LogInformation("worker loop: cancelled; exiting"); |
| | 1 | 69 | | break; |
| | | 70 | | } |
| | 22 | 71 | | if (msg is null) // host closed stdin → exit |
| | | 72 | | { |
| | 3 | 73 | | log.LogInformation("worker loop: stdin closed by host; exiting"); |
| | 3 | 74 | | break; |
| | | 75 | | } |
| | | 76 | | |
| | 19 | 77 | | switch (msg.Kind) |
| | | 78 | | { |
| | | 79 | | case "load": |
| | 6 | 80 | | engine = await HandleLoadAsync(engineFactory, serializer, msg, stdout, log, cancellationToken).C |
| | 6 | 81 | | break; |
| | 5 | 82 | | case "predict" when engine is not null: |
| | 4 | 83 | | await HandlePredictAsync(engine, serializer, msg, stdout, log, cancellationToken).ConfigureAwait |
| | 4 | 84 | | break; |
| | | 85 | | case "shutdown": |
| | 7 | 86 | | log.LogInformation("worker loop: shutdown received; exiting 0"); |
| | 7 | 87 | | return 0; |
| | | 88 | | default: |
| | 2 | 89 | | log.LogWarning("worker loop: unknown or unexpected message kind '{Kind}'", msg.Kind); |
| | 2 | 90 | | await WorkerFrame.WriteAsync(stdout, |
| | 2 | 91 | | new WorkerEnvelope { Kind = "result", Error = $"Unknown or unexpected message kind: '{msg.Ki |
| | 2 | 92 | | cancellationToken).ConfigureAwait(false); |
| | | 93 | | break; |
| | | 94 | | } |
| | | 95 | | } |
| | 4 | 96 | | } |
| | | 97 | | finally |
| | | 98 | | { |
| | 11 | 99 | | (engine as IDisposable)?.Dispose(); |
| | | 100 | | } |
| | 4 | 101 | | return 0; |
| | 11 | 102 | | } |
| | | 103 | | |
| | | 104 | | private static async Task<BaseInferenceEngine<TInput, TOutput>?> HandleLoadAsync<TInput, TOutput>( |
| | | 105 | | Func<InferenceEngineOptions, BaseInferenceEngine<TInput, TOutput>> engineFactory, |
| | | 106 | | IWorkerSerializer<TInput, TOutput> serializer, |
| | | 107 | | WorkerEnvelope msg, |
| | | 108 | | Stream stdout, |
| | | 109 | | ILogger log, |
| | | 110 | | CancellationToken ct) |
| | | 111 | | { |
| | 6 | 112 | | var o = msg.Options ?? new WorkerEngineOptions(); |
| | 6 | 113 | | var engineOptions = new InferenceEngineOptions |
| | 6 | 114 | | { |
| | 6 | 115 | | ModelPath = o.ModelPath, |
| | 6 | 116 | | UseGpuAcceleration = o.UseGpuAcceleration, |
| | 6 | 117 | | DeviceId = o.DeviceId, |
| | 6 | 118 | | WarmupOnLoad = o.WarmupOnLoad, |
| | 6 | 119 | | MIGraphXModelCacheDirectory = o.MIGraphXModelCacheDirectory, |
| | 6 | 120 | | }; |
| | | 121 | | |
| | 6 | 122 | | var sw = Stopwatch.StartNew(); |
| | | 123 | | BaseInferenceEngine<TInput, TOutput> engine; |
| | | 124 | | try |
| | | 125 | | { |
| | 6 | 126 | | engine = engineFactory(engineOptions); |
| | 5 | 127 | | } |
| | | 128 | | catch (Exception ex) |
| | | 129 | | { |
| | 1 | 130 | | log.LogError(ex, "worker load: engine factory threw for engine '{EngineId}'", msg.EngineId); |
| | 1 | 131 | | await WorkerFrame.WriteAsync(stdout, new WorkerEnvelope { Kind = "loaded", Error = ex.ToString() }, ct).Conf |
| | 1 | 132 | | return null; |
| | | 133 | | } |
| | | 134 | | |
| | | 135 | | try |
| | | 136 | | { |
| | 5 | 137 | | await engine.LoadAsync().ConfigureAwait(false); // real EP selection happens here |
| | 4 | 138 | | log.LogInformation("worker load: engine '{EngineId}' loaded in {Elapsed}ms (provider={Provider}, fallback={I |
| | 4 | 139 | | msg.EngineId, sw.ElapsedMilliseconds, engine.EngineInfo.Provider, engine.EngineInfo.IsFallback, engine.E |
| | 4 | 140 | | await WorkerFrame.WriteAsync(stdout, new WorkerEnvelope { Kind = "loaded", EngineInfo = engine.EngineInfo }, |
| | 4 | 141 | | } |
| | 1 | 142 | | catch (Exception ex) |
| | | 143 | | { |
| | 1 | 144 | | log.LogError(ex, "worker load: engine '{EngineId}' LoadAsync failed after {Elapsed}ms", |
| | 1 | 145 | | msg.EngineId, sw.ElapsedMilliseconds); |
| | 1 | 146 | | await WorkerFrame.WriteAsync(stdout, |
| | 1 | 147 | | new WorkerEnvelope { Kind = "loaded", EngineInfo = engine.EngineInfo, Error = ex.ToString() }, |
| | 1 | 148 | | ct).ConfigureAwait(false); |
| | | 149 | | } |
| | 5 | 150 | | return engine; |
| | 6 | 151 | | } |
| | | 152 | | |
| | | 153 | | private static async Task HandlePredictAsync<TInput, TOutput>( |
| | | 154 | | BaseInferenceEngine<TInput, TOutput> engine, |
| | | 155 | | IWorkerSerializer<TInput, TOutput> serializer, |
| | | 156 | | WorkerEnvelope msg, |
| | | 157 | | Stream stdout, |
| | | 158 | | ILogger log, |
| | | 159 | | CancellationToken ct) |
| | | 160 | | { |
| | 4 | 161 | | var sw = Stopwatch.StartNew(); |
| | | 162 | | try |
| | | 163 | | { |
| | 4 | 164 | | var input = serializer.DeserializeInput(msg.Payload ?? string.Empty); |
| | 4 | 165 | | var output = await engine.PredictAsync(input).ConfigureAwait(false); |
| | 3 | 166 | | log.LogDebug("worker predict: ok in {Elapsed}ms", sw.ElapsedMilliseconds); |
| | 3 | 167 | | await WorkerFrame.WriteAsync(stdout, |
| | 3 | 168 | | new WorkerEnvelope { Kind = "result", Payload = serializer.SerializeOutput(output) }, |
| | 3 | 169 | | ct).ConfigureAwait(false); |
| | 3 | 170 | | } |
| | 1 | 171 | | catch (Exception ex) |
| | | 172 | | { |
| | 1 | 173 | | log.LogError(ex, "worker predict: failed after {Elapsed}ms", sw.ElapsedMilliseconds); |
| | 1 | 174 | | await WorkerFrame.WriteAsync(stdout, |
| | 1 | 175 | | new WorkerEnvelope { Kind = "result", Error = ex.ToString() }, |
| | 1 | 176 | | ct).ConfigureAwait(false); |
| | | 177 | | } |
| | 4 | 178 | | } |
| | | 179 | | } |