| | | 1 | | using System; |
| | | 2 | | using System.Buffers; |
| | | 3 | | using System.Collections.Generic; |
| | | 4 | | using System.Runtime.CompilerServices; |
| | | 5 | | using System.Threading; |
| | | 6 | | using System.Threading.Tasks; |
| | | 7 | | using Microsoft.Extensions.Logging; |
| | | 8 | | using Microsoft.ML.OnnxRuntime; |
| | | 9 | | using Microsoft.ML.OnnxRuntime.Tensors; |
| | | 10 | | |
| | | 11 | | namespace InferenceEngine.Core; |
| | | 12 | | |
| | | 13 | | /// <summary> |
| | | 14 | | /// Reuses <see cref="BaseInferenceEngine{TInput, TOutput}"/> for session/acceleration/warmup/ |
| | | 15 | | /// model-download/diagnostics, and adds an autoregressive decode loop driven by engine-supplied |
| | | 16 | | /// hooks. One-shot consumers see no change: <see cref="PredictAsync"/> is routed through the |
| | | 17 | | /// streaming loop and reduced via <see cref="BuildFinalOutput"/>. |
| | | 18 | | /// </summary> |
| | | 19 | | /// <typeparam name="TInput">Raw engine input (e.g. a prompt).</typeparam> |
| | | 20 | | /// <typeparam name="TOutput">Fully-reduced one-shot result.</typeparam> |
| | | 21 | | /// <typeparam name="TStep">One streamed model-level step.</typeparam> |
| | | 22 | | public abstract class BaseStreamingInferenceEngine<TInput, TOutput, TStep> |
| | | 23 | | : BaseInferenceEngine<TInput, TOutput>, IStreamingInferenceEngine<TInput, TOutput, TStep> |
| | | 24 | | { |
| | | 25 | | // Single-in-flight-stream guard (§6.9): KV-cache state is per-run; a second concurrent |
| | | 26 | | // stream on the same instance must be rejected. |
| | | 27 | | private int _activeStreams; |
| | | 28 | | |
| | | 29 | | /// <summary>Creates the streaming engine, forwarding options + logger to the base.</summary> |
| | | 30 | | protected BaseStreamingInferenceEngine(InferenceEngineOptions options, ILogger? logger = null) |
| | 46 | 31 | | : base(options, logger) |
| | | 32 | | { |
| | 46 | 33 | | } |
| | | 34 | | |
| | | 35 | | /// <summary> |
| | | 36 | | /// The <see cref="StreamingOptions"/> for the stream currently in progress, or <see langword="null"/> |
| | | 37 | | /// outside a stream. Hooks (<see cref="BuildStepInputs"/>/<see cref="ProcessStepOutputs"/>) read this |
| | | 38 | | /// to honor <see cref="StreamingOptions.IncludePrefillInStream"/>, <see cref="StreamingOptions.EosTokenIds"/>, etc. |
| | | 39 | | /// </summary> |
| | 362 | 40 | | protected StreamingOptions? CurrentOptions { get; private set; } |
| | | 41 | | |
| | | 42 | | /// <summary>Default streaming options used when a caller passes <see langword="null"/>.</summary> |
| | 18 | 43 | | protected virtual StreamingOptions DefaultStreamingOptions => new(); |
| | | 44 | | |
| | | 45 | | /// <summary>One-shot behavior is preserved: run the stream to completion and reduce it.</summary> |
| | | 46 | | public override async Task<TOutput> PredictAsync(TInput input) |
| | | 47 | | { |
| | 2 | 48 | | var steps = new List<TStep>(); |
| | 22 | 49 | | await foreach (var step in PredictStreamingAsync(input).ConfigureAwait(false)) |
| | 9 | 50 | | steps.Add(step); |
| | 2 | 51 | | return BuildFinalOutput(steps); |
| | 2 | 52 | | } |
| | | 53 | | |
| | | 54 | | /// <inheritdoc/> |
| | | 55 | | public async IAsyncEnumerable<TStep> PredictStreamingAsync( |
| | | 56 | | TInput input, |
| | | 57 | | StreamingOptions? options = null, |
| | | 58 | | [EnumeratorCancellation] CancellationToken cancellationToken = default) |
| | | 59 | | { |
| | 28 | 60 | | await LoadAsync().ConfigureAwait(false); |
| | | 61 | | |
| | 28 | 62 | | if (Interlocked.CompareExchange(ref _activeStreams, 1, 0) != 0) |
| | 1 | 63 | | throw new InvalidOperationException( |
| | 1 | 64 | | "Only one in-flight stream is supported per engine instance (§6.9)."); |
| | | 65 | | |
| | 27 | 66 | | var opts = options ?? DefaultStreamingOptions; |
| | 27 | 67 | | CurrentOptions = opts; |
| | 27 | 68 | | object? state = null; |
| | | 69 | | |
| | | 70 | | try |
| | | 71 | | { |
| | 386 | 72 | | for (int step = 0; step < opts.MaxSteps; step++) |
| | | 73 | | { |
| | 191 | 74 | | cancellationToken.ThrowIfCancellationRequested(); |
| | | 75 | | |
| | 188 | 76 | | var (inputs, buildContext) = BuildStepInputs(input, state, step); |
| | | 77 | | |
| | | 78 | | IDisposableReadOnlyCollection<DisposableNamedOnnxValue> outputs; |
| | | 79 | | try |
| | | 80 | | { |
| | 188 | 81 | | outputs = RunStep(inputs); |
| | 188 | 82 | | } |
| | | 83 | | finally |
| | | 84 | | { |
| | 188 | 85 | | DisposeNamedOnnxValues(inputs); |
| | | 86 | | } |
| | | 87 | | |
| | | 88 | | TStep yielded; |
| | | 89 | | object? nextState; |
| | | 90 | | bool isEos; |
| | | 91 | | try |
| | | 92 | | { |
| | 188 | 93 | | (yielded, nextState, isEos) = ProcessStepOutputs(outputs, buildContext); |
| | 187 | 94 | | } |
| | | 95 | | finally |
| | | 96 | | { |
| | 188 | 97 | | outputs.Dispose(); |
| | | 98 | | } |
| | | 99 | | |
| | | 100 | | // Guard against an engine that reuses (mutates in place) the same state object |
| | | 101 | | // across steps: disposing the prior reference would also dispose the live state. |
| | 187 | 102 | | if (state is IDisposable priorState && !ReferenceEquals(state, nextState)) |
| | 6 | 103 | | priorState.Dispose(); |
| | 187 | 104 | | state = nextState; |
| | 187 | 105 | | yield return yielded; |
| | | 106 | | |
| | 204 | 107 | | if (isEos) yield break; |
| | | 108 | | } |
| | 2 | 109 | | } |
| | | 110 | | finally |
| | | 111 | | { |
| | | 112 | | // Reset per-run state and release the in-flight guard on every exit path |
| | | 113 | | // (success, EOS, MaxSteps, exception, and cancellation). |
| | 27 | 114 | | if (state is IDisposable finalState) |
| | 6 | 115 | | finalState.Dispose(); |
| | 27 | 116 | | state = null; |
| | 27 | 117 | | CurrentOptions = null; |
| | 27 | 118 | | Interlocked.Exchange(ref _activeStreams, 0); |
| | | 119 | | } |
| | 23 | 120 | | } |
| | | 121 | | |
| | | 122 | | // ---- Model-aware hooks: the engine knows its own I/O contract. ---------------- |
| | | 123 | | |
| | | 124 | | /// <summary> |
| | | 125 | | /// Build the ONNX inputs for a given step and return the carry-forward context. |
| | | 126 | | /// <paramref name="step"/> == 0 means prefill (full prompt); step > 0 means decode |
| | | 127 | | /// (feed the previous step's <c>present_key_values</c> back as <c>past_key_values</c>). |
| | | 128 | | /// </summary> |
| | | 129 | | protected abstract (List<NamedOnnxValue> Inputs, object? BuildContext) BuildStepInputs( |
| | | 130 | | TInput input, object? state, int step); |
| | | 131 | | |
| | | 132 | | /// <summary> |
| | | 133 | | /// Parse one forward pass's outputs into a streamed step, the next carry-forward state, and an |
| | | 134 | | /// end-of-sequence signal. The engine is responsible for token-level EOS detection against |
| | | 135 | | /// <see cref="StreamingOptions.EosTokenIds"/> (it holds the sampled token); the base enforces |
| | | 136 | | /// the returned <c>IsEos</c> flag and <see cref="StreamingOptions.MaxSteps"/>. |
| | | 137 | | /// </summary> |
| | | 138 | | protected abstract (TStep Step, object? NextState, bool IsEos) ProcessStepOutputs( |
| | | 139 | | IDisposableReadOnlyCollection<DisposableNamedOnnxValue> outputs, object? buildContext); |
| | | 140 | | |
| | | 141 | | /// <summary>Reduce the accumulated steps into the final one-shot <typeparamref name="TOutput"/>.</summary> |
| | | 142 | | protected abstract TOutput BuildFinalOutput(IReadOnlyList<TStep> steps); |
| | | 143 | | |
| | | 144 | | /// <summary> |
| | | 145 | | /// Samples one token id from a vocab-length logit row. Greedy argmax when |
| | | 146 | | /// <see cref="StreamingOptions.Sampling"/>'s temperature is disabled (0); otherwise |
| | | 147 | | /// temperature-scaled softmax sampling with optional top-k / top-p filtering. |
| | | 148 | | /// Engines call this from <see cref="ProcessStepOutputs"/> when they want the default sampler. |
| | | 149 | | /// </summary> |
| | | 150 | | protected virtual int SampleNextToken(ReadOnlySpan<float> logits) |
| | | 151 | | { |
| | 143 | 152 | | if (logits.IsEmpty) return -1; |
| | | 153 | | |
| | 141 | 154 | | var sampling = CurrentOptions?.Sampling; |
| | 141 | 155 | | if (sampling is null || sampling.Temperature <= 0f) |
| | 20 | 156 | | return Argmax(logits); |
| | | 157 | | |
| | | 158 | | // Temperature scaling. |
| | 121 | 159 | | float temperature = sampling.Temperature; |
| | 121 | 160 | | int vocab = logits.Length; |
| | | 161 | | |
| | | 162 | | // Rent vocab-sized buffers instead of allocating them on every step. For a typical |
| | | 163 | | // vocabulary (e.g. Phi-3's 32,006) a fresh double[32006] would land on the Large Object Heap |
| | | 164 | | // each token, causing severe GC pressure during generation. Rented arrays may be larger than |
| | | 165 | | // `vocab`, so every loop below is bounded by `vocab` and helpers take a vocab-length span. |
| | | 166 | | // Declare the rented arrays as nullable outside the try so a rent that throws cannot leak an |
| | | 167 | | // already-rented buffer; the finally block returns whichever buffers were successfully rented. |
| | 121 | 168 | | float[]? scaled = null; |
| | 121 | 169 | | double[]? probs = null; |
| | 121 | 170 | | int[]? order = null; |
| | | 171 | | try |
| | | 172 | | { |
| | 121 | 173 | | scaled = ArrayPool<float>.Shared.Rent(vocab); |
| | 121 | 174 | | probs = ArrayPool<double>.Shared.Rent(vocab); |
| | | 175 | | |
| | 968 | 176 | | for (int i = 0; i < vocab; i++) |
| | 363 | 177 | | scaled[i] = logits[i] / temperature; |
| | | 178 | | |
| | | 179 | | // Top-k filtering: keep only the k highest logits, mask the rest to -inf. |
| | 121 | 180 | | int topK = sampling.TopK; |
| | 121 | 181 | | if (topK > 0 && topK < vocab) |
| | | 182 | | { |
| | 31 | 183 | | float cutoff = NthLargest(scaled.AsSpan(0, vocab), topK); |
| | 248 | 184 | | for (int i = 0; i < vocab; i++) |
| | 93 | 185 | | if (scaled[i] < cutoff) |
| | 60 | 186 | | scaled[i] = float.NegativeInfinity; |
| | | 187 | | } |
| | | 188 | | |
| | | 189 | | // Softmax (numerically stable: subtract max of finite values). |
| | 121 | 190 | | float max = float.NegativeInfinity; |
| | 968 | 191 | | for (int i = 0; i < vocab; i++) |
| | 513 | 192 | | if (scaled[i] > max) max = scaled[i]; |
| | 121 | 193 | | if (float.IsNegativeInfinity(max)) |
| | 1 | 194 | | return Argmax(logits); // everything masked; fall back to greedy |
| | | 195 | | |
| | 120 | 196 | | double sum = 0d; |
| | 960 | 197 | | for (int i = 0; i < vocab; i++) |
| | | 198 | | { |
| | 360 | 199 | | double p = Math.Exp(scaled[i] - max); |
| | 360 | 200 | | probs[i] = p; |
| | 360 | 201 | | sum += p; |
| | | 202 | | } |
| | | 203 | | |
| | | 204 | | // Top-p (nucleus) filtering: keep the smallest set of highest-prob tokens whose |
| | | 205 | | // cumulative probability reaches p, sorted descending. |
| | 120 | 206 | | float topP = sampling.TopP; |
| | 120 | 207 | | if (topP > 0f && topP < 1f) |
| | | 208 | | { |
| | 30 | 209 | | order = ArrayPool<int>.Shared.Rent(vocab); |
| | 330 | 210 | | for (int i = 0; i < vocab; i++) order[i] = i; |
| | | 211 | | // Sort token indices by descending probability without per-step allocation: a struct |
| | | 212 | | // comparer is specialized by MemoryExtensions.Sort and never boxes, unlike |
| | | 213 | | // Comparer<int>.Create(lambda), which allocates a Comparer + delegate every token. |
| | 30 | 214 | | order.AsSpan(0, vocab).Sort(new DescendingProbabilityIndexComparer(probs)); |
| | 30 | 215 | | double cum = 0d; |
| | 30 | 216 | | bool reached = false; |
| | 240 | 217 | | for (int i = 0; i < vocab; i++) |
| | | 218 | | { |
| | 90 | 219 | | if (reached) |
| | | 220 | | { |
| | 60 | 221 | | probs[order[i]] = 0d; |
| | | 222 | | } |
| | | 223 | | else |
| | | 224 | | { |
| | 30 | 225 | | cum += probs[order[i]] / sum; |
| | 60 | 226 | | if (cum >= topP) reached = true; |
| | | 227 | | } |
| | | 228 | | } |
| | | 229 | | // Renormalize the kept set. |
| | 30 | 230 | | double keptSum = 0d; |
| | 330 | 231 | | for (int i = 0; i < vocab; i++) keptSum += probs[i]; |
| | 30 | 232 | | if (keptSum <= 0d) return Argmax(logits); |
| | 330 | 233 | | for (int i = 0; i < vocab; i++) probs[i] /= keptSum; |
| | | 234 | | } |
| | | 235 | | else |
| | | 236 | | { |
| | 990 | 237 | | for (int i = 0; i < vocab; i++) probs[i] /= sum; |
| | | 238 | | } |
| | | 239 | | |
| | | 240 | | // Sample from the resulting distribution. |
| | 120 | 241 | | double r = Random.Shared.NextDouble(); |
| | 120 | 242 | | double acc = 0d; |
| | 480 | 243 | | for (int i = 0; i < vocab; i++) |
| | | 244 | | { |
| | 240 | 245 | | acc += probs[i]; |
| | 360 | 246 | | if (r < acc) return i; |
| | | 247 | | } |
| | 0 | 248 | | return vocab - 1; |
| | | 249 | | } |
| | | 250 | | finally |
| | | 251 | | { |
| | 121 | 252 | | if (scaled is not null) |
| | 121 | 253 | | ArrayPool<float>.Shared.Return(scaled); |
| | 121 | 254 | | if (probs is not null) |
| | 121 | 255 | | ArrayPool<double>.Shared.Return(probs); |
| | 121 | 256 | | if (order is not null) |
| | 30 | 257 | | ArrayPool<int>.Shared.Return(order); |
| | 121 | 258 | | } |
| | 121 | 259 | | } |
| | | 260 | | |
| | | 261 | | private static int Argmax(ReadOnlySpan<float> logits) |
| | | 262 | | { |
| | 21 | 263 | | int best = 0; |
| | 21 | 264 | | float max = logits[0]; |
| | 192334 | 265 | | for (int i = 1; i < logits.Length; i++) |
| | | 266 | | { |
| | 96146 | 267 | | if (logits[i] > max) |
| | | 268 | | { |
| | 14 | 269 | | max = logits[i]; |
| | 14 | 270 | | best = i; |
| | | 271 | | } |
| | | 272 | | } |
| | 21 | 273 | | return best; |
| | | 274 | | } |
| | | 275 | | |
| | | 276 | | /// <summary>Returns the cutoff value of the k-th largest element (allocation-free partial sort). |
| | | 277 | | /// Maintains a sorted span of the k largest values seen; its minimum is the k-th largest. |
| | | 278 | | /// Operates on a vocab-length slice of a (possibly larger) rented array. Allocation-free (uses |
| | | 279 | | /// stackalloc for small k, an ArrayPool rental otherwise) so the hot sampling path never hits the |
| | | 280 | | /// managed heap — the previous PriorityQueue approach allocated on every token-generation step.</summary> |
| | | 281 | | internal static float NthLargest(ReadOnlySpan<float> values, int k) |
| | | 282 | | { |
| | 35 | 283 | | float[]? rented = null; |
| | | 284 | | // Slice the rented array to exactly k: ArrayPool.Rent may hand back a larger buffer (e.g. |
| | | 285 | | // 512 for k=300), and operating on the full buffer would track more than k elements and |
| | | 286 | | // yield the wrong (k+slack)-th-largest cutoff. |
| | 35 | 287 | | Span<float> temp = k <= 256 ? stackalloc float[k] : (rented = ArrayPool<float>.Shared.Rent(k)).AsSpan(0, k); |
| | | 288 | | try |
| | | 289 | | { |
| | 35 | 290 | | temp.Fill(float.NegativeInfinity); |
| | 2656 | 291 | | for (int i = 0; i < values.Length; i++) |
| | | 292 | | { |
| | 1293 | 293 | | float v = values[i]; |
| | 1293 | 294 | | if (v <= temp[0]) continue; |
| | | 295 | | |
| | | 296 | | // Insert v into the ascending-sorted top-k window, evicting the current minimum |
| | | 297 | | // (temp[0]). BinarySearch gives the insertion point; shift the sub-slice left by one |
| | | 298 | | // to drop the old minimum and place v at the now-vacated slot. |
| | 1263 | 299 | | int index = temp.BinarySearch(v); |
| | 2526 | 300 | | if (index < 0) index = ~index; |
| | 1263 | 301 | | if (index > 1) |
| | 900 | 302 | | temp.Slice(1, index - 1).CopyTo(temp.Slice(0, index - 1)); |
| | 1263 | 303 | | if (index > 0) |
| | 1260 | 304 | | temp[index - 1] = v; |
| | | 305 | | } |
| | 35 | 306 | | return temp[0]; |
| | | 307 | | } |
| | | 308 | | finally |
| | | 309 | | { |
| | 35 | 310 | | if (rented is not null) |
| | 2 | 311 | | ArrayPool<float>.Shared.Return(rented); |
| | 35 | 312 | | } |
| | 35 | 313 | | } |
| | | 314 | | |
| | | 315 | | /// <summary> |
| | | 316 | | /// Compares two array indices by the value they reference in descending order. A struct so that |
| | | 317 | | /// the span sort helper can specialize it without boxing or any heap allocation (a delegate-based |
| | | 318 | | /// comparer would allocate on every token-generation step). |
| | | 319 | | /// </summary> |
| | | 320 | | private readonly struct DescendingProbabilityIndexComparer : IComparer<int> |
| | | 321 | | { |
| | | 322 | | private readonly double[] _values; |
| | | 323 | | |
| | 30 | 324 | | public DescendingProbabilityIndexComparer(double[] values) => _values = values; |
| | | 325 | | |
| | 90 | 326 | | public int Compare(int a, int b) => _values[b].CompareTo(_values[a]); |
| | | 327 | | } |
| | | 328 | | |
| | | 329 | | /// <summary> |
| | | 330 | | /// Run one forward pass. Virtual so tests can substitute a fake session without a real model. |
| | | 331 | | /// </summary> |
| | | 332 | | protected virtual IDisposableReadOnlyCollection<DisposableNamedOnnxValue> RunStep( |
| | | 333 | | IReadOnlyCollection<NamedOnnxValue> inputs) |
| | | 334 | | { |
| | 8 | 335 | | var session = _session ?? throw new InvalidOperationException("Engine is not loaded."); |
| | 8 | 336 | | return session.Run(inputs, session.OutputNames, _runOptions); |
| | | 337 | | } |
| | | 338 | | |
| | | 339 | | /// <summary> |
| | | 340 | | /// Clone a <see cref="DenseTensor{T}"/> of <see cref="float"/> carried in a model output so it |
| | | 341 | | /// survives disposal of the output collection, renaming it from a <c>present_*</c> name to the |
| | | 342 | | /// matching <c>past_*</c> name for the next step. Engines call this from |
| | | 343 | | /// <see cref="ProcessStepOutputs"/> to carry the KV cache forward. Other dtypes require extension. |
| | | 344 | | /// </summary> |
| | | 345 | | protected static NamedOnnxValue? CloneAsPastKeyValue(DisposableNamedOnnxValue output, string pastName) |
| | | 346 | | { |
| | 9 | 347 | | if (output.Value is not DenseTensor<float> src) return null; |
| | 7 | 348 | | var copy = new DenseTensor<float>(src.Dimensions); |
| | 7 | 349 | | src.Buffer.Span.CopyTo(copy.Buffer.Span); |
| | 7 | 350 | | return NamedOnnxValue.CreateFromTensor(pastName, copy); |
| | | 351 | | } |
| | | 352 | | |
| | | 353 | | /// <summary> |
| | | 354 | | /// Convenience <see cref="IDisposable"/> holder for a list of carry-forward KV-cache tensors. |
| | | 355 | | /// Engines that use <see cref="CloneAsPastKeyValue"/> should wrap the cloned values in an |
| | | 356 | | /// instance of this type (or any other <see cref="IDisposable"/>) and return it from |
| | | 357 | | /// <see cref="ProcessStepOutputs"/> as the next state. The base loop disposes the previous |
| | | 358 | | /// state on step rollover and the final state on stream end, releasing the cloned tensors. |
| | | 359 | | /// </summary> |
| | | 360 | | protected sealed class KeyValueCacheState : IDisposable |
| | | 361 | | { |
| | | 362 | | private readonly List<NamedOnnxValue> _values; |
| | | 363 | | private bool _disposed; |
| | | 364 | | |
| | 3 | 365 | | public KeyValueCacheState(List<NamedOnnxValue> values) |
| | | 366 | | { |
| | 3 | 367 | | _values = values ?? throw new ArgumentNullException(nameof(values)); |
| | 2 | 368 | | } |
| | | 369 | | |
| | | 370 | | /// <summary>The carry-forward KV-cache tensors.</summary> |
| | 1 | 371 | | public IReadOnlyList<NamedOnnxValue> Values => _values; |
| | | 372 | | |
| | | 373 | | public void Dispose() |
| | | 374 | | { |
| | 4 | 375 | | if (_disposed) return; |
| | 2 | 376 | | _disposed = true; |
| | 2 | 377 | | DisposeNamedOnnxValues(_values); |
| | 2 | 378 | | } |
| | | 379 | | } |
| | | 380 | | |
| | | 381 | | /// <summary>Dispose any <see cref="NamedOnnxValue"/> items in <paramref name="values"/> that are disposable.</summa |
| | | 382 | | protected static void DisposeNamedOnnxValues(IReadOnlyCollection<NamedOnnxValue> values) |
| | | 383 | | { |
| | 494 | 384 | | foreach (var t in values) |
| | | 385 | | { |
| | 48 | 386 | | if (t is IDisposable disposable) |
| | 0 | 387 | | disposable.Dispose(); |
| | | 388 | | } |
| | 199 | 389 | | } |
| | | 390 | | |
| | | 391 | | // ---- One-shot pre/post hooks are unused on streaming engines (PredictAsync is |
| | | 392 | | // overridden to route through the stream). Provide stubs so concrete streaming |
| | | 393 | | // engines are not forced to implement dead methods. |
| | | 394 | | /// <summary>Not used by streaming engines; one-shot <see cref="PredictAsync"/> is routed through the stream.</summa |
| | | 395 | | protected sealed override (List<NamedOnnxValue> InputTensors, object? Context) PreProcessWithContext(TInput input) = |
| | 1 | 396 | | throw new NotSupportedException( |
| | 1 | 397 | | "Streaming engines route one-shot PredictAsync through PredictStreamingAsync; " + |
| | 1 | 398 | | "PreProcessWithContext is not used. Override BuildStepInputs instead."); |
| | | 399 | | |
| | | 400 | | /// <summary>Not used by streaming engines.</summary> |
| | | 401 | | protected sealed override TOutput PostProcessWithContext( |
| | | 402 | | IDisposableReadOnlyCollection<DisposableNamedOnnxValue> output, object? context) => |
| | 1 | 403 | | throw new NotSupportedException( |
| | 1 | 404 | | "Streaming engines route one-shot PredictAsync through PredictStreamingAsync; " + |
| | 1 | 405 | | "PostProcessWithContext is not used. Override ProcessStepOutputs instead."); |
| | | 406 | | } |