| | | 1 | | using System; |
| | | 2 | | using System.Collections.Generic; |
| | | 3 | | using Microsoft.Extensions.Logging; |
| | | 4 | | using Microsoft.ML.OnnxRuntime; |
| | | 5 | | using Microsoft.ML.OnnxRuntime.Tensors; |
| | | 6 | | using InferenceEngine.Core; |
| | | 7 | | |
| | | 8 | | namespace InferenceEngine.Examples |
| | | 9 | | { |
| | | 10 | | /// <summary> |
| | | 11 | | /// Reference skeleton for a streaming text-generation engine (e.g. an HF-optimum decoder |
| | | 12 | | /// export such as GPT-2 / OPT / Llama) built on |
| | | 13 | | /// <see cref="BaseStreamingInferenceEngine{TInput, TOutput, TStep}"/>. |
| | | 14 | | /// </summary> |
| | | 15 | | /// <remarks> |
| | | 16 | | /// <para> |
| | | 17 | | /// This is a <b>reference skeleton</b>: it demonstrates the prefill/decode tensor assembly, |
| | | 18 | | /// KV-cache carry-back, sampling, detokenization, and EOS-detection shape that a real |
| | | 19 | | /// streaming LLM engine requires, but it ships <b>no ONNX model and no tokenizer</b>. |
| | | 20 | | /// Tokenization is a consumer concern and is intentionally out of scope (see streaming spec §1). |
| | | 21 | | /// Provide a real HF-optimum ONNX export plus a tokenizer before calling |
| | | 22 | | /// <see cref="IInferenceEngine{TInput, TOutput}.LoadAsync"/>. |
| | | 23 | | /// </para> |
| | | 24 | | /// <para> |
| | | 25 | | /// Design reference: <c>plan_docs/inference-engine-streaming-upstream-spec.md</c>. |
| | | 26 | | /// </para> |
| | | 27 | | /// </remarks> |
| | | 28 | | public class StreamingTextGenerationExample |
| | | 29 | | : BaseStreamingInferenceEngine<string, string, string> |
| | | 30 | | { |
| | | 31 | | /// <summary> |
| | | 32 | | /// Creates the skeleton engine, forwarding options + logger to the base. |
| | | 33 | | /// </summary> |
| | | 34 | | /// <param name="options">Engine options (model path/provider, GPU, warmup).</param> |
| | | 35 | | /// <param name="logger">Optional logger.</param> |
| | | 36 | | public StreamingTextGenerationExample( |
| | | 37 | | InferenceEngineOptions options, |
| | | 38 | | ILogger<StreamingTextGenerationExample>? logger = null) |
| | 5 | 39 | | : base(options, logger) |
| | | 40 | | { |
| | 5 | 41 | | } |
| | | 42 | | |
| | | 43 | | /// <summary>Model file this engine expects. Not present in the default registry.</summary> |
| | 3 | 44 | | protected override string ModelName => "streaming-text-generation.onnx"; |
| | | 45 | | |
| | | 46 | | // Note: ModelDownloadUrls / ModelRegistry are not overridden — supply the model through |
| | | 47 | | // InferenceEngineOptions.ModelPath or InferenceEngineOptions.ModelProvider. |
| | | 48 | | |
| | | 49 | | /// <summary> |
| | | 50 | | /// Build the ONNX inputs for one forward pass and return the carry-forward context. |
| | | 51 | | /// <para> |
| | | 52 | | /// <paramref name="step"/> == 0 means prefill (encode the full prompt); |
| | | 53 | | /// step > 0 means decode (feed the previous step's sampled token plus the carried |
| | | 54 | | /// KV cache back as <c>past_key_values</c>). |
| | | 55 | | /// </para> |
| | | 56 | | /// </summary> |
| | | 57 | | protected override (List<NamedOnnxValue> Inputs, object? BuildContext) BuildStepInputs( |
| | | 58 | | string input, object? state, int step) |
| | | 59 | | { |
| | | 60 | | // Discover the model's I/O names from session metadata (populated by LoadAsync). |
| | | 61 | | // A typical HF-optimum decoder export exposes: input_ids, attention_mask, position_ids, |
| | | 62 | | // and one present_key_values.N.{key,value} pair per layer: |
| | | 63 | | // |
| | | 64 | | // string inputIdsName = PrimaryInputName ?? "input_ids"; |
| | | 65 | | // _ = InputMetadata; // map: input_ids, attention_mask, position_ids, past_key_values.* |
| | | 66 | | // _ = OutputMetadata; // map: logits, present_key_values.* |
| | | 67 | | |
| | 2 | 68 | | var inputs = new List<NamedOnnxValue>(); |
| | | 69 | | |
| | 2 | 70 | | if (step == 0) |
| | | 71 | | { |
| | | 72 | | // ---- Prefill: encode the full prompt. ---- |
| | | 73 | | // TODO: tokenize `input` into int64 ids with your tokenizer |
| | | 74 | | // (SentencePiece / HuggingFace tokenizers / Microsoft.ML.Tokenizers). |
| | | 75 | | // |
| | | 76 | | // long[] promptIds = Tokenizer.Encode(input); |
| | | 77 | | // var idsTensor = new DenseTensor<long>(promptIds, new[] { 1, promptIds.Length }); |
| | | 78 | | // inputs.Add(NamedOnnxValue.CreateFromTensor(inputIdsName, idsTensor)); |
| | | 79 | | // inputs.Add(NamedOnnxValue.CreateFromTensor("attention_mask", Ones(1, promptIds.Length))); |
| | | 80 | | // inputs.Add(NamedOnnxValue.CreateFromTensor("position_ids", Range(1, promptIds.Length))); |
| | | 81 | | // |
| | | 82 | | // Prefill feeds no past_key_values on step 0. |
| | | 83 | | } |
| | | 84 | | else |
| | | 85 | | { |
| | | 86 | | // ---- Decode: feed the single previously-sampled token + carried KV cache. ---- |
| | | 87 | | // var carry = (KvCacheState)state!; |
| | | 88 | | // |
| | | 89 | | // var idsTensor = new DenseTensor<long>(new[] { carry.LastToken }, new[] { 1, 1 }); |
| | | 90 | | // inputs.Add(NamedOnnxValue.CreateFromTensor(inputIdsName, idsTensor)); |
| | | 91 | | // |
| | | 92 | | // KV-cache carry-back: replay the previous step's present_* outputs as past_* inputs. |
| | | 93 | | // The carried tensors were cloned in ProcessStepOutputs via CloneAsPastKeyValue: |
| | | 94 | | // |
| | | 95 | | // foreach (var kv in carry.PastKeyValues) |
| | | 96 | | // inputs.Add(kv); |
| | | 97 | | } |
| | | 98 | | |
| | | 99 | | // No bundled model/tokenizer ships with this skeleton — guard so it never runs |
| | | 100 | | // half-configured. Supply a real model before calling LoadAsync. |
| | | 101 | | _ = inputs; |
| | 2 | 102 | | throw new NotSupportedException( |
| | 2 | 103 | | "Reference skeleton — provide a real HF-optimum ONNX export and a tokenizer. " + |
| | 2 | 104 | | "See plan_docs/inference-engine-streaming-upstream-spec.md."); |
| | | 105 | | } |
| | | 106 | | |
| | | 107 | | /// <summary> |
| | | 108 | | /// Decode one forward pass into the streamed text chunk, the next KV-cache state, and an |
| | | 109 | | /// end-of-sequence flag. The engine owns token-level EOS detection against |
| | | 110 | | /// <see cref="StreamingOptions.EosTokenIds"/>; the base enforces the returned |
| | | 111 | | /// <c>IsEos</c> flag and <see cref="StreamingOptions.MaxSteps"/>. |
| | | 112 | | /// </summary> |
| | | 113 | | protected override (string Step, object? NextState, bool IsEos) ProcessStepOutputs( |
| | | 114 | | IDisposableReadOnlyCollection<DisposableNamedOnnxValue> outputs, object? buildContext) |
| | | 115 | | { |
| | | 116 | | // TODO: locate the logits output (shape [batch, seq, vocab]) by name from OutputMetadata, |
| | | 117 | | // then read the next-token distribution from the last row: |
| | | 118 | | // |
| | | 119 | | // var logitsOut = outputs.First(o => o.Name == "logits"); |
| | | 120 | | // var logitsTensor = (DenseTensor<float>)logitsOut.Value; // [1, seqLen, vocab] |
| | | 121 | | // int vocabSize = logitsTensor.Dimensions[2]; |
| | | 122 | | // var vocabRow = logitsTensor.Buffer.Span.Slice(vocabSize); // last row |
| | | 123 | | // |
| | | 124 | | // Sample the next token id (greedy argmax via the base helper; override |
| | | 125 | | // SampleNextToken for top-k/top-p/temperature honoring CurrentOptions.Sampling): |
| | | 126 | | // |
| | | 127 | | // int nextToken = SampleNextToken(vocabRow); |
| | | 128 | | // |
| | | 129 | | // Detokenize the single id into the text chunk to yield (a consumer concern; spec §1): |
| | | 130 | | // |
| | | 131 | | // string chunk = Tokenizer.Decode(new[] { nextToken }); |
| | | 132 | | |
| | | 133 | | // KV-cache carry-back: clone each present_*.N output into the matching past_*.N input |
| | | 134 | | // so it survives disposal of this output collection, then stash it on the next state: |
| | | 135 | | // |
| | | 136 | | // var carried = new List<NamedOnnxValue>(); |
| | | 137 | | // foreach (var present in outputs.Where(o => o.Name.StartsWith("present", StringComparison.Ordinal))) |
| | | 138 | | // { |
| | | 139 | | // string pastName = present.Name.Replace("present", "past", StringComparison.Ordinal); |
| | | 140 | | // var past = CloneAsPastKeyValue(present, pastName); |
| | | 141 | | // if (past is not null) carried.Add(past); |
| | | 142 | | // } |
| | | 143 | | // object? nextState = new KvCacheState { LastToken = nextToken }; |
| | | 144 | | // |
| | | 145 | | // EOS detection (the engine holds the sampled token): |
| | | 146 | | // |
| | | 147 | | // bool isEos = CurrentOptions?.EosTokenIds.Contains(nextToken) ?? false; |
| | | 148 | | // return (chunk, nextState, isEos); |
| | | 149 | | |
| | 1 | 150 | | throw new NotSupportedException( |
| | 1 | 151 | | "Reference skeleton — provide a real model and tokenizer to decode steps. " + |
| | 1 | 152 | | "See plan_docs/inference-engine-streaming-upstream-spec.md."); |
| | | 153 | | } |
| | | 154 | | |
| | | 155 | | /// <summary> |
| | | 156 | | /// Reduce the accumulated streamed chunks into the final one-shot text. |
| | | 157 | | /// </summary> |
| | | 158 | | protected override string BuildFinalOutput(IReadOnlyList<string> steps) |
| | 1 | 159 | | => string.Concat(steps); |
| | | 160 | | |
| | | 161 | | /// <summary> |
| | | 162 | | /// Carry-forward state for the autoregressive decode loop: the last sampled token id and |
| | | 163 | | /// the cloned KV-cache tensors replayed as the next step's <c>past_key_values</c>. |
| | | 164 | | /// </summary> |
| | | 165 | | private sealed class KvCacheState |
| | | 166 | | { |
| | | 167 | | /// <summary>The token id sampled in the previous step.</summary> |
| | 0 | 168 | | public int LastToken { get; set; } |
| | | 169 | | |
| | | 170 | | /// <summary>Cloned present_* outputs renamed to past_* for the next forward pass.</summary> |
| | 0 | 171 | | public List<NamedOnnxValue> PastKeyValues { get; } = new(); |
| | | 172 | | } |
| | | 173 | | } |
| | | 174 | | } |