| | | 1 | | using System; |
| | | 2 | | using System.Collections.Generic; |
| | | 3 | | using System.Linq; |
| | | 4 | | using System.Threading; |
| | | 5 | | using System.Threading.Tasks; |
| | | 6 | | using Microsoft.Extensions.Logging; |
| | | 7 | | using Microsoft.ML.OnnxRuntime; |
| | | 8 | | using Microsoft.ML.OnnxRuntime.Tensors; |
| | | 9 | | using Microsoft.ML.Tokenizers; |
| | | 10 | | |
| | | 11 | | namespace InferenceEngine.Core.Text; |
| | | 12 | | |
| | | 13 | | /// <summary> |
| | | 14 | | /// One-shot text embedding engine backed by the all-MiniLM-L6-v2 ONNX model. Owns BERT |
| | | 15 | | /// tokenization end-to-end: callers pass plain strings and receive an L2-normalized embedding |
| | | 16 | | /// vector (<c>string in -> float[] out</c>). |
| | | 17 | | /// </summary> |
| | | 18 | | /// <remarks> |
| | | 19 | | /// <para> |
| | | 20 | | /// The embedding is produced by masked mean-pooling over the sequence dimension of the model's |
| | | 21 | | /// primary output (typically <c>last_hidden_state</c> / <c>sentence_embedding</c>), using the |
| | | 22 | | /// attention mask, followed by L2 normalization. The vector dimension is read from the model's |
| | | 23 | | /// output metadata and is never hard-coded. |
| | | 24 | | /// </para> |
| | | 25 | | /// <para> |
| | | 26 | | /// Tokenization uses an uncased BERT WordPiece tokenizer (<c>Microsoft.ML.Tokenizers</c>), the |
| | | 27 | | /// vocab of which is downloaded and cached via <see cref="TokenizerAssets.LoadMiniLmTokenizerAsync"/>. |
| | | 28 | | /// If the tokenizer asset is unavailable the engine still constructs (the embedding call will then |
| | | 29 | | /// throw a clear error), so the instance is always safe to create. |
| | | 30 | | /// </para> |
| | | 31 | | /// </remarks> |
| | | 32 | | public class TextEmbeddingEngine : BaseInferenceEngine<string, float[]> |
| | | 33 | | { |
| | | 34 | | /// <summary>MiniLM embedding model ONNX download URL (single self-contained file).</summary> |
| | | 35 | | internal const string MiniLmModelUrl = |
| | | 36 | | "https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/onnx/model.onnx"; |
| | | 37 | | |
| | | 38 | | private Tokenizer? _tokenizer; |
| | | 39 | | |
| | | 40 | | /// <summary> |
| | | 41 | | /// Test seam (used by <c>InferenceEngine.Tests</c>): injects a pre-built tokenizer without |
| | | 42 | | /// network access, mirroring the seam on <see cref="TextGenerationEngine"/>. Null (default) = |
| | | 43 | | /// the tokenizer is resolved lazily during <see cref="LoadAsync"/>. |
| | | 44 | | /// </summary> |
| | | 45 | | internal Tokenizer? TestTokenizer |
| | | 46 | | { |
| | 3 | 47 | | get => _tokenizer; |
| | 15 | 48 | | set => _tokenizer = value; |
| | | 49 | | } |
| | | 50 | | |
| | | 51 | | /// <summary> |
| | | 52 | | /// Creates the engine. The ONNX model and tokenizer are resolved lazily during |
| | | 53 | | /// <see cref="LoadAsync"/>. |
| | | 54 | | /// </summary> |
| | | 55 | | /// <param name="options">Engine options. GPU acceleration defaults on; pass <c>UseGpuAcceleration = false</c> for C |
| | | 56 | | /// <param name="logger">Optional typed logger.</param> |
| | | 57 | | public TextEmbeddingEngine(InferenceEngineOptions options, ILogger<TextEmbeddingEngine>? logger = null) |
| | 18 | 58 | | : base(options, logger) |
| | | 59 | | { |
| | 18 | 60 | | } |
| | | 61 | | |
| | | 62 | | /// <summary> |
| | | 63 | | /// Loads the MiniLM tokenizer (best-effort) and then initializes the ONNX session. A tokenizer |
| | | 64 | | /// load failure is logged as a warning and does not abort the session load; the first embedding |
| | | 65 | | /// call will surface a clear error if a tokenizer is genuinely required. |
| | | 66 | | /// </summary> |
| | | 67 | | public override async Task LoadAsync() |
| | | 68 | | { |
| | 3 | 69 | | if (_tokenizer == null) |
| | | 70 | | { |
| | | 71 | | try |
| | | 72 | | { |
| | 3 | 73 | | _tokenizer = await TokenizerAssets.LoadMiniLmTokenizerAsync(_options, _logger, CancellationToken.None) |
| | 3 | 74 | | .ConfigureAwait(false); |
| | 2 | 75 | | } |
| | 1 | 76 | | catch (Exception ex) |
| | | 77 | | { |
| | 1 | 78 | | _logger.LogWarning(ex, "Could not load the MiniLM tokenizer during LoadAsync."); |
| | 1 | 79 | | _tokenizer = null; |
| | 1 | 80 | | } |
| | | 81 | | } |
| | | 82 | | |
| | 3 | 83 | | await base.LoadAsync().ConfigureAwait(false); |
| | 3 | 84 | | } |
| | | 85 | | |
| | | 86 | | /// <summary>The model file name used for cache keys and registry lookups.</summary> |
| | 4 | 87 | | protected override string ModelName => "all-MiniLM-L6-v2.onnx"; |
| | | 88 | | |
| | | 89 | | /// <summary>Download URLs for the all-MiniLM-L6-v2 ONNX model.</summary> |
| | 1 | 90 | | protected override string[]? ModelDownloadUrls => new[] { MiniLmModelUrl }; |
| | | 91 | | |
| | | 92 | | /// <summary> |
| | | 93 | | /// Tokenizes the input, then assembles the model's declared integer inputs |
| | | 94 | | /// (<c>input_ids</c>, <c>attention_mask</c>, and <c>token_type_ids</c>/<c>position_ids</c> |
| | | 95 | | /// when the model declares them). Each tensor is typed to match its declared element type |
| | | 96 | | /// (<see cref="long"/> or <see cref="int"/>). |
| | | 97 | | /// </summary> |
| | | 98 | | /// <returns>The input tensors and the sequence length (carried as context for pooling).</returns> |
| | | 99 | | protected override (List<NamedOnnxValue> InputTensors, object? Context) PreProcessWithContext(string input) |
| | | 100 | | { |
| | 8 | 101 | | var tokenizer = _tokenizer ?? throw new InvalidOperationException( |
| | 8 | 102 | | "Tokenizer is not loaded. Ensure network access so the MiniLM vocab can be fetched."); |
| | | 103 | | |
| | 7 | 104 | | var ids = tokenizer.EncodeToIds(input ?? string.Empty); |
| | 7 | 105 | | if (ids.Count == 0) |
| | | 106 | | { |
| | 0 | 107 | | ids = new[] { 0 }; |
| | | 108 | | } |
| | | 109 | | |
| | 7 | 110 | | int seqLen = ids.Count; |
| | 7 | 111 | | var inputs = new List<NamedOnnxValue>(); |
| | 7 | 112 | | var metadata = InputMetadata ?? throw new InvalidOperationException("Engine is not loaded (no input metadata).") |
| | | 113 | | |
| | 48 | 114 | | foreach (var (name, nodeMeta) in metadata) |
| | | 115 | | { |
| | 50 | 116 | | var shape = nodeMeta.Dimensions.Select(d => d <= 0 ? 1 : d).ToArray(); |
| | | 117 | | // Force sequence dim to the actual token length for [1, seq] inputs. |
| | 33 | 118 | | if (shape.Length >= 2) shape[1] = seqLen; |
| | 18 | 119 | | if (shape.Length == 1) shape[0] = seqLen; |
| | 17 | 120 | | Type elementType = nodeMeta.ElementType ?? typeof(long); |
| | | 121 | | |
| | 17 | 122 | | if (name.Contains("input_ids", StringComparison.OrdinalIgnoreCase)) |
| | | 123 | | { |
| | 6 | 124 | | inputs.Add(CreateIntTensor(name, ids, shape, elementType)); |
| | | 125 | | } |
| | 11 | 126 | | else if (name.Contains("attention_mask", StringComparison.OrdinalIgnoreCase)) |
| | | 127 | | { |
| | 5 | 128 | | inputs.Add(CreateIntTensor(name, Enumerable.Repeat(1, seqLen), shape, elementType)); |
| | | 129 | | } |
| | 6 | 130 | | else if (name.Contains("token_type_ids", StringComparison.OrdinalIgnoreCase)) |
| | | 131 | | { |
| | 4 | 132 | | inputs.Add(CreateIntTensor(name, Enumerable.Repeat(0, seqLen), shape, elementType)); |
| | | 133 | | } |
| | 2 | 134 | | else if (name.Contains("position_ids", StringComparison.OrdinalIgnoreCase)) |
| | | 135 | | { |
| | 1 | 136 | | inputs.Add(CreateIntTensor(name, Enumerable.Range(0, seqLen), shape, elementType)); |
| | | 137 | | } |
| | | 138 | | else |
| | | 139 | | { |
| | | 140 | | // Unknown integer/other input: fill zeros with the matching declared type so the |
| | | 141 | | // forward pass has a valid binding. Fall back to int64 if not determinable. |
| | 1 | 142 | | inputs.Add(CreateIntTensor(name, Enumerable.Repeat(0, seqLen), shape, elementType)); |
| | | 143 | | } |
| | | 144 | | } |
| | | 145 | | |
| | 7 | 146 | | return (inputs, seqLen); |
| | | 147 | | } |
| | | 148 | | |
| | | 149 | | /// <summary> |
| | | 150 | | /// Selects the embedding output (preferring <c>sentence_embedding</c>/<c>pooler</c>/<c>pooled</c>), |
| | | 151 | | /// masked-mean-pools a 3D [1,seq,dim] tensor (or takes the row of a 2D [1,dim] tensor), and |
| | | 152 | | /// L2-normalizes the result. |
| | | 153 | | /// </summary> |
| | | 154 | | /// <param name="output">Raw ONNX outputs.</param> |
| | | 155 | | /// <param name="context">Sequence length carried from pre-processing.</param> |
| | | 156 | | /// <returns>An L2-normalized embedding vector.</returns> |
| | | 157 | | protected override float[] PostProcessWithContext( |
| | | 158 | | IDisposableReadOnlyCollection<DisposableNamedOnnxValue> output, |
| | | 159 | | object? context) |
| | | 160 | | { |
| | 11 | 161 | | int seqLen = context is int sl && sl > 0 ? sl : 1; |
| | 11 | 162 | | DenseTensor<float>? tensor = null; |
| | | 163 | | |
| | 40 | 164 | | foreach (var item in output) |
| | | 165 | | { |
| | 11 | 166 | | if (item.Value is not DenseTensor<float> f) continue; |
| | 11 | 167 | | string n = item.Name ?? string.Empty; |
| | 11 | 168 | | if (n.Contains("sentence_embedding", StringComparison.OrdinalIgnoreCase) || |
| | 11 | 169 | | n.Contains("pooled", StringComparison.OrdinalIgnoreCase) || |
| | 11 | 170 | | n.Contains("pooler", StringComparison.OrdinalIgnoreCase)) |
| | | 171 | | { |
| | 4 | 172 | | tensor = f; |
| | 4 | 173 | | break; |
| | | 174 | | } |
| | | 175 | | |
| | 7 | 176 | | tensor ??= f; |
| | | 177 | | } |
| | | 178 | | |
| | 11 | 179 | | if (tensor is null) |
| | | 180 | | { |
| | 1 | 181 | | throw new InvalidOperationException("No float output tensor was produced by the embedding model."); |
| | | 182 | | } |
| | | 183 | | |
| | | 184 | | int dim; |
| | 10 | 185 | | if (tensor.Dimensions.Length == 3) |
| | | 186 | | { |
| | 6 | 187 | | dim = tensor.Dimensions[2]; |
| | 6 | 188 | | var pooled = new float[dim]; |
| | 46 | 189 | | for (int s = 0; s < seqLen && s < tensor.Dimensions[1]; s++) |
| | | 190 | | { |
| | 9264 | 191 | | for (int d = 0; d < dim; d++) |
| | | 192 | | { |
| | 4615 | 193 | | pooled[d] += tensor[0, s, d]; |
| | | 194 | | } |
| | | 195 | | } |
| | | 196 | | // masked mean-pool over the valid sequence positions |
| | 6 | 197 | | float denom = Math.Max(1, Math.Min(seqLen, tensor.Dimensions[1])); |
| | 3480 | 198 | | for (int d = 0; d < dim; d++) pooled[d] /= denom; |
| | 6 | 199 | | return L2Normalize(pooled); |
| | | 200 | | } |
| | | 201 | | |
| | 4 | 202 | | if (tensor.Dimensions.Length == 2) |
| | | 203 | | { |
| | 2 | 204 | | dim = tensor.Dimensions[1]; |
| | 2 | 205 | | var row = new float[dim]; |
| | 19 | 206 | | for (int d = 0; d < dim; d++) row[d] = tensor[0, d]; |
| | 2 | 207 | | return L2Normalize(row); |
| | | 208 | | } |
| | | 209 | | |
| | 2 | 210 | | if (tensor.Dimensions.Length == 1) |
| | | 211 | | { |
| | 1 | 212 | | dim = tensor.Dimensions[0]; |
| | 1 | 213 | | var row = new float[dim]; |
| | 8 | 214 | | for (int d = 0; d < dim; d++) row[d] = tensor[d]; |
| | 1 | 215 | | return L2Normalize(row); |
| | | 216 | | } |
| | | 217 | | |
| | 1 | 218 | | throw new InvalidOperationException( |
| | 1 | 219 | | $"Unsupported embedding output rank {tensor.Dimensions.Length} (expected 1, 2, or 3)."); |
| | | 220 | | } |
| | | 221 | | |
| | | 222 | | /// <summary> |
| | | 223 | | /// Builds correctly-typed integer dummy tensors (int64/int32 per declared element type) so the |
| | | 224 | | /// base warmup pass does not fail on default float-zero tensors. |
| | | 225 | | /// </summary> |
| | | 226 | | protected override List<NamedOnnxValue> GetWarmupInput() |
| | | 227 | | { |
| | 2 | 228 | | var inputs = new List<NamedOnnxValue>(); |
| | 2 | 229 | | if (InputMetadata is null) return inputs; |
| | | 230 | | |
| | | 231 | | const int seq = 4; |
| | 10 | 232 | | foreach (var (name, nodeMeta) in InputMetadata) |
| | | 233 | | { |
| | 8 | 234 | | var shape = nodeMeta.Dimensions.Select(d => d <= 0 ? 1 : d).ToArray(); |
| | 5 | 235 | | if (shape.Length >= 2) shape[1] = seq; |
| | 4 | 236 | | if (shape.Length == 1) shape[0] = seq; |
| | 3 | 237 | | Type elementType = nodeMeta.ElementType ?? typeof(long); |
| | | 238 | | |
| | 3 | 239 | | if (elementType == typeof(int)) |
| | | 240 | | { |
| | 1 | 241 | | inputs.Add(NamedOnnxValue.CreateFromTensor(name, new DenseTensor<int>(shape))); |
| | | 242 | | } |
| | | 243 | | else |
| | | 244 | | { |
| | 2 | 245 | | inputs.Add(NamedOnnxValue.CreateFromTensor(name, new DenseTensor<long>(shape))); |
| | | 246 | | } |
| | | 247 | | } |
| | | 248 | | |
| | 2 | 249 | | return inputs; |
| | | 250 | | } |
| | | 251 | | |
| | | 252 | | /// <summary> |
| | | 253 | | /// Convenience helper that loads the engine and returns the embedding for a single string. |
| | | 254 | | /// </summary> |
| | | 255 | | public async Task<float[]> GenerateEmbeddingAsync(string text, CancellationToken ct = default) |
| | | 256 | | { |
| | 1 | 257 | | ct.ThrowIfCancellationRequested(); |
| | 0 | 258 | | await LoadAsync().ConfigureAwait(false); |
| | 0 | 259 | | ct.ThrowIfCancellationRequested(); |
| | 0 | 260 | | return await PredictAsync(text).ConfigureAwait(false); |
| | 0 | 261 | | } |
| | | 262 | | |
| | | 263 | | /// <summary> |
| | | 264 | | /// L2-normalizes a vector in place semantics: returns a new normalized vector. If the norm is |
| | | 265 | | /// zero the input is returned unchanged (no NaN). |
| | | 266 | | /// </summary> |
| | | 267 | | /// <param name="vector">The vector to normalize.</param> |
| | | 268 | | /// <returns>An L2-normalized copy of <paramref name="vector"/>.</returns> |
| | | 269 | | internal static float[] L2Normalize(ReadOnlySpan<float> vector) |
| | | 270 | | { |
| | 17 | 271 | | float sum = 0f; |
| | 3595 | 272 | | for (int i = 0; i < vector.Length; i++) sum += vector[i] * vector[i]; |
| | 17 | 273 | | double norm = Math.Sqrt(sum); |
| | 17 | 274 | | var result = new float[vector.Length]; |
| | 17 | 275 | | if (norm <= 0d) |
| | | 276 | | { |
| | 1 | 277 | | vector.CopyTo(result); |
| | 1 | 278 | | return result; |
| | | 279 | | } |
| | | 280 | | |
| | 16 | 281 | | float scale = (float)(1d / norm); |
| | 3584 | 282 | | for (int i = 0; i < vector.Length; i++) result[i] = vector[i] * scale; |
| | 16 | 283 | | return result; |
| | | 284 | | } |
| | | 285 | | |
| | | 286 | | /// <summary> |
| | | 287 | | /// Convenience overload that normalizes a writable vector's contents. |
| | | 288 | | /// </summary> |
| | 14 | 289 | | internal static float[] L2Normalize(float[] vector) => L2Normalize(vector.AsSpan()); |
| | | 290 | | |
| | | 291 | | /// <summary> |
| | | 292 | | /// Masked mean-pool of a 3D [1, seq, dim] tensor over the sequence axis, restricted to the first |
| | | 293 | | /// <paramref name="seqLen"/> positions. |
| | | 294 | | /// </summary> |
| | | 295 | | /// <param name="tensor">The [1, seq, dim] hidden-state tensor.</param> |
| | | 296 | | /// <param name="seqLen">Number of valid (masked) sequence positions.</param> |
| | | 297 | | /// <returns>The pooled [dim] vector.</returns> |
| | | 298 | | internal static float[] MeanPool(DenseTensor<float> tensor, int seqLen) |
| | | 299 | | { |
| | 5 | 300 | | if (tensor.Dimensions.Length != 3) |
| | | 301 | | { |
| | 0 | 302 | | throw new ArgumentException("MeanPool expects a rank-3 [1,seq,dim] tensor.", nameof(tensor)); |
| | | 303 | | } |
| | | 304 | | |
| | 5 | 305 | | int dim = tensor.Dimensions[2]; |
| | 5 | 306 | | int maxSeq = tensor.Dimensions[1]; |
| | 5 | 307 | | int valid = Math.Max(1, Math.Min(seqLen, maxSeq)); |
| | 5 | 308 | | var pooled = new float[dim]; |
| | 36 | 309 | | for (int s = 0; s < valid; s++) |
| | | 310 | | { |
| | 92 | 311 | | for (int d = 0; d < dim; d++) |
| | | 312 | | { |
| | 33 | 313 | | pooled[d] += tensor[0, s, d]; |
| | | 314 | | } |
| | | 315 | | } |
| | | 316 | | |
| | 46 | 317 | | for (int d = 0; d < dim; d++) pooled[d] /= valid; |
| | 5 | 318 | | return pooled; |
| | | 319 | | } |
| | | 320 | | |
| | | 321 | | /// <summary>Cosine similarity between two equal-length vectors (assumed non-degenerate).</summary> |
| | | 322 | | internal static float CosineSimilarity(ReadOnlySpan<float> a, ReadOnlySpan<float> b) |
| | | 323 | | { |
| | 6 | 324 | | if (a.Length != b.Length || a.Length == 0) return 0f; |
| | 18 | 325 | | double dot = 0, na = 0, nb = 0; |
| | 1572 | 326 | | for (int i = 0; i < a.Length; i++) |
| | | 327 | | { |
| | 780 | 328 | | dot += a[i] * b[i]; |
| | 780 | 329 | | na += a[i] * a[i]; |
| | 780 | 330 | | nb += b[i] * b[i]; |
| | | 331 | | } |
| | | 332 | | |
| | 6 | 333 | | double denom = Math.Sqrt(na) * Math.Sqrt(nb); |
| | 6 | 334 | | return denom <= 0d ? 0f : (float)(dot / denom); |
| | | 335 | | } |
| | | 336 | | |
| | | 337 | | /// <summary>Builds an integer-valued tensor matching <paramref name="elementType"/> (long or int).</summary> |
| | | 338 | | private static NamedOnnxValue CreateIntTensor(string name, IEnumerable<int> values, int[] shape, Type elementType) |
| | | 339 | | { |
| | 17 | 340 | | var list = values as int[] ?? values.ToArray(); |
| | 17 | 341 | | if (elementType == typeof(int)) |
| | | 342 | | { |
| | 2 | 343 | | var tensor = new DenseTensor<int>(shape); |
| | 2 | 344 | | var span = tensor.Buffer.Span; |
| | 2 | 345 | | int count = Math.Min(span.Length, list.Length); |
| | 8 | 346 | | for (int i = 0; i < count; i++) |
| | | 347 | | { |
| | 2 | 348 | | span[i] = list[i]; |
| | | 349 | | } |
| | | 350 | | |
| | 2 | 351 | | return NamedOnnxValue.CreateFromTensor(name, tensor); |
| | | 352 | | } |
| | | 353 | | |
| | 15 | 354 | | var longTensor = new DenseTensor<long>(shape); |
| | 15 | 355 | | var longSpan = longTensor.Buffer.Span; |
| | 15 | 356 | | int longCount = Math.Min(longSpan.Length, list.Length); |
| | 122 | 357 | | for (int i = 0; i < longCount; i++) |
| | | 358 | | { |
| | 46 | 359 | | longSpan[i] = list[i]; |
| | | 360 | | } |
| | | 361 | | |
| | 15 | 362 | | return NamedOnnxValue.CreateFromTensor(name, longTensor); |
| | | 363 | | } |
| | | 364 | | } |