| | | 1 | | using System.Collections.Generic; |
| | | 2 | | |
| | | 3 | | namespace InferenceEngine.Core; |
| | | 4 | | |
| | | 5 | | /// <summary>Per-call streaming controls. Defaults are safe/greedy.</summary> |
| | | 6 | | public sealed class StreamingOptions |
| | | 7 | | { |
| | | 8 | | /// <summary>Maximum decode steps (prefill counts as step 0) before the loop stops.</summary> |
| | 354 | 9 | | public int MaxSteps { get; set; } = 256; |
| | | 10 | | |
| | | 11 | | /// <summary>Token ids that terminate generation when sampled.</summary> |
| | 90 | 12 | | public IReadOnlyList<int> EosTokenIds { get; set; } = System.Array.Empty<int>(); |
| | | 13 | | |
| | | 14 | | /// <summary>Sampling strategy. Honored by <c>BaseStreamingInferenceEngine.SampleNextToken</c>; |
| | | 15 | | /// temperature 0 (default) is greedy argmax; >0 enables softmax sampling with top-k/top-p.</summary> |
| | 180 | 16 | | public SamplingParams Sampling { get; set; } = new(); |
| | | 17 | | |
| | | 18 | | /// <summary>If true, also yield prompt tokens from the prefill pass. Default false.</summary> |
| | 3 | 19 | | public bool IncludePrefillInStream { get; set; } = false; |
| | | 20 | | |
| | | 21 | | /// <summary>Batch >=1 decoded steps per yield (null/1 = one step per yield). Default null.</summary> |
| | 3 | 22 | | public int? ChunkSize { get; set; } = null; |
| | | 23 | | } |
| | | 24 | | |
| | | 25 | | /// <summary>Sampling parameters. Disabled/identity values mean "off".</summary> |
| | | 26 | | public sealed class SamplingParams |
| | | 27 | | { |
| | | 28 | | /// <summary>0 => greedy (argmax).</summary> |
| | | 29 | | public float Temperature { get; set; } = 0f; |
| | | 30 | | |
| | | 31 | | /// <summary>0 => disabled.</summary> |
| | | 32 | | public int TopK { get; set; } = 0; |
| | | 33 | | |
| | | 34 | | /// <summary>1 => disabled.</summary> |
| | | 35 | | public float TopP { get; set; } = 1f; |
| | | 36 | | |
| | | 37 | | /// <summary>1 => disabled.</summary> |
| | | 38 | | public float RepetitionPenalty { get; set; } = 1f; |
| | | 39 | | } |