| | | 1 | | using System; |
| | | 2 | | using System.IO; |
| | | 3 | | using System.Threading; |
| | | 4 | | using System.Threading.Tasks; |
| | | 5 | | using InferenceEngine.Core.ModelProviders; |
| | | 6 | | using Microsoft.Extensions.Logging; |
| | | 7 | | using Microsoft.ML.Tokenizers; |
| | | 8 | | |
| | | 9 | | namespace InferenceEngine.Core.Text; |
| | | 10 | | |
| | | 11 | | /// <summary> |
| | | 12 | | /// Resolves and constructs the <see cref="Tokenizer"/> instances used by the text engines. |
| | | 13 | | /// Downloads the vocab/model asset files once (delegating to <see cref="DownloadingModelProvider"/> |
| | | 14 | | /// for cache + retry), then builds the configured tokenizer over a read-only file stream. |
| | | 15 | | /// </summary> |
| | | 16 | | internal static class TokenizerAssets |
| | | 17 | | { |
| | | 18 | | /// <summary>MiniLM (all-MiniLM-L6-v2) BERT WordPiece vocabulary file.</summary> |
| | | 19 | | internal const string MiniLmVocabFileName = "vocab.txt"; |
| | | 20 | | |
| | | 21 | | /// <summary>MiniLM vocab download URL (uncased BERT WordPiece).</summary> |
| | | 22 | | internal const string MiniLmVocabUrl = |
| | | 23 | | "https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/vocab.txt"; |
| | | 24 | | |
| | | 25 | | /// <summary>Phi-3 SentencePiece tokenizer model file.</summary> |
| | | 26 | | internal const string Phi3TokenizerFileName = "tokenizer.model"; |
| | | 27 | | |
| | | 28 | | /// <summary>Phi-3 SentencePiece tokenizer model download URL.</summary> |
| | | 29 | | internal const string Phi3TokenizerUrl = |
| | | 30 | | "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-onnx/resolve/main/cpu_and_mobile/cpu-int4-rtn-block-32/ |
| | | 31 | | |
| | | 32 | | /// <summary> |
| | | 33 | | /// Loads the all-MiniLM-L6-v2 tokenizer (an uncased BERT WordPiece model) from a cached or |
| | | 34 | | /// freshly-downloaded <c>vocab.txt</c>. Returns the configured <see cref="BertTokenizer"/>. |
| | | 35 | | /// </summary> |
| | | 36 | | /// <param name="options">Engine options; the cache root is derived from the default model cache.</param> |
| | | 37 | | /// <param name="logger">Logger for download/failure diagnostics.</param> |
| | | 38 | | /// <param name="ct">Cancellation token.</param> |
| | | 39 | | /// <returns>A <see cref="Tokenizer"/> for all-MiniLM-L6-v2.</returns> |
| | | 40 | | /// <exception cref="InvalidOperationException">Thrown if the vocab asset cannot be fetched.</exception> |
| | | 41 | | public static async Task<Tokenizer> LoadMiniLmTokenizerAsync( |
| | | 42 | | InferenceEngineOptions options, |
| | | 43 | | ILogger logger, |
| | | 44 | | CancellationToken ct = default) |
| | | 45 | | { |
| | 4 | 46 | | var vocabPath = await DownloadAssetAsync( |
| | 4 | 47 | | MiniLmVocabFileName, |
| | 4 | 48 | | new[] { MiniLmVocabUrl }, |
| | 4 | 49 | | options, |
| | 4 | 50 | | logger, |
| | 4 | 51 | | ct).ConfigureAwait(false); |
| | | 52 | | |
| | 4 | 53 | | var bertOptions = new BertOptions { LowerCaseBeforeTokenization = true }; |
| | 4 | 54 | | await using var stream = new FileStream(vocabPath, FileMode.Open, FileAccess.Read, FileShare.Read); |
| | 4 | 55 | | var tokenizer = BertTokenizer.Create(stream, bertOptions); |
| | 2 | 56 | | logger.LogInformation("Loaded MiniLM (all-MiniLM-L6-v2) BERT tokenizer from {Path}", vocabPath); |
| | 2 | 57 | | return tokenizer; |
| | 2 | 58 | | } |
| | | 59 | | |
| | | 60 | | /// <summary> |
| | | 61 | | /// Loads the Phi-3 SentencePiece tokenizer from a cached or freshly-downloaded |
| | | 62 | | /// <c>tokenizer.model</c>. No BOS/EOS is auto-inserted; the engine drives EOS via |
| | | 63 | | /// <see cref="StreamingOptions.EosTokenIds"/>. |
| | | 64 | | /// </summary> |
| | | 65 | | /// <remarks> |
| | | 66 | | /// <para> |
| | | 67 | | /// In <c>Microsoft.ML.Tokenizers</c> 1.0.3 the <c>SentencePieceTokenizer</c> type has no public |
| | | 68 | | /// factory of its own. The public factory that reads a SentencePiece model and returns a |
| | | 69 | | /// <see cref="SentencePieceTokenizer"/>-derived instance is <see cref="LlamaTokenizer.Create"/>, |
| | | 70 | | /// whose signature matches the intended SentencePiece usage. Phi-3's tokenizer.model is a |
| | | 71 | | /// SentencePiece model and is read here via that factory. |
| | | 72 | | /// </para> |
| | | 73 | | /// </remarks> |
| | | 74 | | /// <param name="options">Engine options; the cache root is derived from the default model cache.</param> |
| | | 75 | | /// <param name="logger">Logger for download/failure diagnostics.</param> |
| | | 76 | | /// <param name="ct">Cancellation token.</param> |
| | | 77 | | /// <returns>A <see cref="Tokenizer"/> for Phi-3-mini-4k-instruct.</returns> |
| | | 78 | | /// <exception cref="InvalidOperationException">Thrown if the model asset cannot be fetched.</exception> |
| | | 79 | | public static async Task<Tokenizer> LoadPhi3TokenizerAsync( |
| | | 80 | | InferenceEngineOptions options, |
| | | 81 | | ILogger logger, |
| | | 82 | | CancellationToken ct = default) |
| | | 83 | | { |
| | 2 | 84 | | var modelPath = await DownloadAssetAsync( |
| | 2 | 85 | | Phi3TokenizerFileName, |
| | 2 | 86 | | new[] { Phi3TokenizerUrl }, |
| | 2 | 87 | | options, |
| | 2 | 88 | | logger, |
| | 2 | 89 | | ct).ConfigureAwait(false); |
| | | 90 | | |
| | 2 | 91 | | var specialTokens = new System.Collections.Generic.Dictionary<string, int> |
| | 2 | 92 | | { |
| | 2 | 93 | | ["<|endoftext|>"] = 32000, |
| | 2 | 94 | | ["<|im_start|>"] = 32001, |
| | 2 | 95 | | ["<|im_end|>"] = 32002, |
| | 2 | 96 | | }; |
| | | 97 | | |
| | 2 | 98 | | await using var stream = new FileStream(modelPath, FileMode.Open, FileAccess.Read, FileShare.Read); |
| | | 99 | | // LlamaTokenizer is the public SentencePiece-backed factory in Tokenizers 1.0.3 and returns |
| | | 100 | | // a SentencePieceTokenizer-derived Tokenizer. addBos/addEos=false: the engine owns EOS. |
| | 2 | 101 | | var tokenizer = LlamaTokenizer.Create( |
| | 2 | 102 | | stream, |
| | 2 | 103 | | addBeginOfSentence: false, |
| | 2 | 104 | | addEndOfSentence: false, |
| | 2 | 105 | | specialTokens); |
| | 1 | 106 | | logger.LogInformation("Loaded Phi-3 SentencePiece tokenizer from {Path}", modelPath); |
| | 1 | 107 | | return tokenizer; |
| | 1 | 108 | | } |
| | | 109 | | |
| | | 110 | | /// <summary> |
| | | 111 | | /// Test seam (used by <c>InferenceEngine.Tests</c>): overrides the tokenizers cache directory so |
| | | 112 | | /// tests can point the asset loader at an isolated temp directory (hermetic failure/corruption |
| | | 113 | | /// injection) instead of the shared user-level cache. Null (default) = production behavior. |
| | | 114 | | /// </summary> |
| | 34 | 115 | | internal static string? TokenizerCacheDirectoryOverride { get; set; } |
| | | 116 | | |
| | | 117 | | /// <summary> |
| | | 118 | | /// Downloads (or reuses a cached copy of) a single tokenizer asset file under the engine's |
| | | 119 | | /// tokenizers cache directory using <see cref="DownloadingModelProvider"/>. Internal so the |
| | | 120 | | /// test project can drive the download-failure wrapping directly. |
| | | 121 | | /// </summary> |
| | | 122 | | internal static async Task<string> DownloadAssetAsync( |
| | | 123 | | string fileName, |
| | | 124 | | string[] urls, |
| | | 125 | | InferenceEngineOptions options, |
| | | 126 | | ILogger logger, |
| | | 127 | | CancellationToken ct) |
| | | 128 | | { |
| | 10 | 129 | | _ = options ?? throw new ArgumentNullException(nameof(options)); |
| | 9 | 130 | | string cacheDir = GetTokenizersCacheDirectory(); |
| | 9 | 131 | | Directory.CreateDirectory(cacheDir); |
| | | 132 | | |
| | 9 | 133 | | var provider = new DownloadingModelProvider( |
| | 9 | 134 | | modelName: fileName, |
| | 9 | 135 | | downloadUrls: urls, |
| | 9 | 136 | | cacheDirectory: cacheDir, |
| | 9 | 137 | | logger: logger); |
| | | 138 | | |
| | | 139 | | try |
| | | 140 | | { |
| | 9 | 141 | | return await provider.GetModelPathAsync(ct).ConfigureAwait(false); |
| | | 142 | | } |
| | 2 | 143 | | catch (Exception ex) |
| | | 144 | | { |
| | 2 | 145 | | logger.LogError(ex, "Failed to obtain tokenizer asset '{File}' from {UrlCount} source(s)", fileName, urls.Le |
| | 2 | 146 | | throw new InvalidOperationException( |
| | 2 | 147 | | $"Tokenizer asset '{fileName}' could not be loaded or downloaded. " + |
| | 2 | 148 | | "Provide network access or place the file in the tokenizers cache directory.", ex); |
| | | 149 | | } |
| | 7 | 150 | | } |
| | | 151 | | |
| | | 152 | | /// <summary>Tokenizers cache directory, located alongside the default model cache.</summary> |
| | | 153 | | private static string GetTokenizersCacheDirectory() => |
| | 9 | 154 | | TokenizerCacheDirectoryOverride is { } overrideDir |
| | 9 | 155 | | ? overrideDir |
| | 9 | 156 | | : Path.Combine( |
| | 9 | 157 | | Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), |
| | 9 | 158 | | "InferenceEngine", |
| | 9 | 159 | | "Tokenizers"); |
| | | 160 | | } |