| | | 1 | | using System; |
| | | 2 | | using System.Buffers; |
| | | 3 | | using System.IO; |
| | | 4 | | using System.Text; |
| | | 5 | | using System.Text.Json; |
| | | 6 | | using System.Threading; |
| | | 7 | | using System.Threading.Tasks; |
| | | 8 | | using InferenceEngine.Core.Diagnostics; |
| | | 9 | | |
| | | 10 | | namespace InferenceEngine.Core.Worker; |
| | | 11 | | |
| | | 12 | | /// <summary>Serializable subset of <see cref="InferenceEngineOptions"/> sent over IPC. |
| | | 13 | | /// Delegates (<see cref="InferenceEngineOptions.OnSessionInitialized"/>) cannot cross the boundary |
| | | 14 | | /// and are intentionally omitted.</summary> |
| | | 15 | | public sealed class WorkerEngineOptions |
| | | 16 | | { |
| | 78 | 17 | | public string? ModelPath { get; set; } |
| | 119 | 18 | | public bool UseGpuAcceleration { get; set; } = true; |
| | 75 | 19 | | public int DeviceId { get; set; } = 0; |
| | 75 | 20 | | public bool WarmupOnLoad { get; set; } = false; |
| | | 21 | | |
| | | 22 | | /// <summary>MIGraphX compiled-model cache directory (null = worker-side default, |
| | | 23 | | /// empty string = disabled). Mirrors <see cref="InferenceEngineOptions.MIGraphXModelCacheDirectory"/>.</summary> |
| | 77 | 24 | | public string? MIGraphXModelCacheDirectory { get; set; } |
| | | 25 | | } |
| | | 26 | | |
| | | 27 | | /// <summary> |
| | | 28 | | /// Single envelope for all worker messages. The <see cref="Kind"/> discriminator selects the role |
| | | 29 | | /// of the other fields. Framed on the wire as <c><4-byte little-endian length><UTF-8 JSON></c>. |
| | | 30 | | /// </summary> |
| | | 31 | | public sealed class WorkerEnvelope |
| | | 32 | | { |
| | | 33 | | /// <summary>One of: <c>load</c>, <c>loaded</c>, <c>predict</c>, <c>result</c>, <c>shutdown</c>.</summary> |
| | | 34 | | public string Kind { get; set; } = ""; |
| | | 35 | | |
| | | 36 | | /// <summary>Consumer-defined engine id (load only).</summary> |
| | | 37 | | public string EngineId { get; set; } = ""; |
| | | 38 | | |
| | | 39 | | /// <summary>Serialized input (predict) / output (result) payload, crossed via |
| | | 40 | | /// <see cref="IWorkerSerializer{TInput,TOutput}"/>.</summary> |
| | | 41 | | public string? Payload { get; set; } |
| | | 42 | | |
| | | 43 | | /// <summary>Engine options (load only).</summary> |
| | | 44 | | public WorkerEngineOptions? Options { get; set; } |
| | | 45 | | |
| | | 46 | | /// <summary>Loaded engine diagnostics (loaded only).</summary> |
| | | 47 | | public InferenceEngineInfo? EngineInfo { get; set; } |
| | | 48 | | |
| | | 49 | | /// <summary>Non-null when the worker hit an error handling the request.</summary> |
| | | 50 | | public string? Error { get; set; } |
| | | 51 | | } |
| | | 52 | | |
| | | 53 | | /// <summary>Length-prefixed JSON framing over a duplex stream (stdin/stdout pipe).</summary> |
| | | 54 | | internal static class WorkerFrame |
| | | 55 | | { |
| | | 56 | | // 16 MiB cap: a corrupt/buggy length prefix is rejected before allocating, rather than forcing a |
| | | 57 | | // single huge contiguous (LOH) allocation. Any plausible IPC JSON envelope is far below this. |
| | | 58 | | private const int MaxFrameBytes = 16 * 1024 * 1024; // 16 MiB guard against corrupt length prefixes |
| | | 59 | | |
| | | 60 | | public static async Task<WorkerEnvelope?> ReadAsync(Stream stream, CancellationToken ct) |
| | | 61 | | { |
| | | 62 | | // Rent the 4-byte length prefix instead of allocating it; a clean EOF here (the worker is |
| | | 63 | | // gone before any frame bytes) is surfaced as a null return so the caller can shut down |
| | | 64 | | // gracefully instead of throwing. |
| | | 65 | | byte[]? prefix = null; |
| | | 66 | | int length; |
| | | 67 | | try |
| | | 68 | | { |
| | | 69 | | prefix = ArrayPool<byte>.Shared.Rent(4); |
| | | 70 | | if (!await ReadExactIntoAsync(stream, prefix.AsMemory(0, 4), ct).ConfigureAwait(false)) |
| | | 71 | | return null; // clean EOF |
| | | 72 | | length = BitConverter.ToInt32(prefix, 0); |
| | | 73 | | } |
| | | 74 | | finally |
| | | 75 | | { |
| | | 76 | | if (prefix is not null) |
| | | 77 | | ArrayPool<byte>.Shared.Return(prefix); |
| | | 78 | | } |
| | | 79 | | |
| | | 80 | | if (length <= 0) throw new InvalidDataException($"Invalid frame length: {length}."); |
| | | 81 | | if (length > MaxFrameBytes) throw new InvalidDataException($"Frame too large: {length} bytes."); |
| | | 82 | | |
| | | 83 | | // Rent the payload buffer and deserialize straight from the rented span, then return it. This |
| | | 84 | | // avoids allocating a fresh byte[length] on every IPC read; JsonSerializer.Deserialize copies |
| | | 85 | | // the strings out, so the buffer is safe to return before this method exits. |
| | | 86 | | byte[]? payload = null; |
| | | 87 | | try |
| | | 88 | | { |
| | | 89 | | payload = ArrayPool<byte>.Shared.Rent(length); |
| | | 90 | | if (!await ReadExactIntoAsync(stream, payload.AsMemory(0, length), ct).ConfigureAwait(false)) |
| | | 91 | | throw new EndOfStreamException("Stream closed mid-frame."); |
| | | 92 | | return JsonSerializer.Deserialize<WorkerEnvelope>(payload.AsSpan(0, length), WorkerJson.Options); |
| | | 93 | | } |
| | | 94 | | finally |
| | | 95 | | { |
| | | 96 | | if (payload is not null) |
| | | 97 | | ArrayPool<byte>.Shared.Return(payload); |
| | | 98 | | } |
| | | 99 | | } |
| | | 100 | | |
| | | 101 | | public static async Task WriteAsync(Stream stream, WorkerEnvelope envelope, CancellationToken ct) |
| | | 102 | | { |
| | | 103 | | var json = JsonSerializer.SerializeToUtf8Bytes(envelope, WorkerJson.Options); |
| | | 104 | | var prefix = BitConverter.GetBytes(json.Length); // 4-byte little-endian |
| | | 105 | | await stream.WriteAsync(prefix, ct).ConfigureAwait(false); |
| | | 106 | | await stream.WriteAsync(json, ct).ConfigureAwait(false); |
| | | 107 | | await stream.FlushAsync(ct).ConfigureAwait(false); |
| | | 108 | | } |
| | | 109 | | |
| | | 110 | | public static void Write(Stream stream, WorkerEnvelope envelope) |
| | | 111 | | { |
| | | 112 | | var json = JsonSerializer.SerializeToUtf8Bytes(envelope, WorkerJson.Options); |
| | | 113 | | var prefix = BitConverter.GetBytes(json.Length); // 4-byte little-endian |
| | | 114 | | stream.Write(prefix, 0, prefix.Length); |
| | | 115 | | stream.Write(json, 0, json.Length); |
| | | 116 | | stream.Flush(); |
| | | 117 | | } |
| | | 118 | | |
| | | 119 | | /// <summary> |
| | | 120 | | /// Reads exactly <c>destination.Length</c> bytes into <paramref name="destination"/>. Returns |
| | | 121 | | /// <see langword="false"/> on a clean EOF at the very start (zero bytes read) and throws |
| | | 122 | | /// <see cref="EndOfStreamException"/> if the stream ends partway through. Used by |
| | | 123 | | /// <see cref="ReadAsync"/> to distinguish graceful worker shutdown (null return) from a truncated |
| | | 124 | | /// frame (throw). Reads into a caller-supplied buffer so no per-frame allocation occurs. |
| | | 125 | | /// </summary> |
| | | 126 | | private static async Task<bool> ReadExactIntoAsync(Stream stream, Memory<byte> destination, CancellationToken ct) |
| | | 127 | | { |
| | | 128 | | int read = 0; |
| | | 129 | | int count = destination.Length; |
| | | 130 | | while (read < count) |
| | | 131 | | { |
| | | 132 | | var n = await stream.ReadAsync(destination.Slice(read, count - read), ct).ConfigureAwait(false); |
| | | 133 | | if (n == 0) |
| | | 134 | | return read == 0 ? false : throw new EndOfStreamException("Stream closed mid-frame."); |
| | | 135 | | read += n; |
| | | 136 | | } |
| | | 137 | | return true; |
| | | 138 | | } |
| | | 139 | | } |
| | | 140 | | |
| | | 141 | | internal static class WorkerJson |
| | | 142 | | { |
| | | 143 | | public static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web) { IncludeFields = true }; |
| | | 144 | | } |