< Summary

Information
Class: InferenceEngine.Core.Text.TokenizerAssets
Assembly: InferenceEngine.Core
File(s): /home/runner/work/inference-engine-lib/inference-engine-lib/InferenceEngine.Core/Text/TokenizerAssets.cs
Line coverage
100%
Covered lines: 55
Uncovered lines: 0
Coverable lines: 55
Total lines: 160
Line coverage: 100%
Branch coverage
100%
Covered branches: 4
Total branches: 4
Branch coverage: 100%
Method coverage

Feature is only available for sponsors

Upgrade to PRO version

Metrics

MethodBranch coverage Crap Score Cyclomatic complexity Line coverage
LoadMiniLmTokenizerAsync()100%11100%
LoadPhi3TokenizerAsync()100%11100%
get_TokenizerCacheDirectoryOverride()100%11100%
DownloadAssetAsync()100%22100%
GetTokenizersCacheDirectory()100%22100%

File(s)

/home/runner/work/inference-engine-lib/inference-engine-lib/InferenceEngine.Core/Text/TokenizerAssets.cs

#LineLine coverage
 1using System;
 2using System.IO;
 3using System.Threading;
 4using System.Threading.Tasks;
 5using InferenceEngine.Core.ModelProviders;
 6using Microsoft.Extensions.Logging;
 7using Microsoft.ML.Tokenizers;
 8
 9namespace InferenceEngine.Core.Text;
 10
 11/// <summary>
 12/// Resolves and constructs the <see cref="Tokenizer"/> instances used by the text engines.
 13/// Downloads the vocab/model asset files once (delegating to <see cref="DownloadingModelProvider"/>
 14/// for cache + retry), then builds the configured tokenizer over a read-only file stream.
 15/// </summary>
 16internal static class TokenizerAssets
 17{
 18    /// <summary>MiniLM (all-MiniLM-L6-v2) BERT WordPiece vocabulary file.</summary>
 19    internal const string MiniLmVocabFileName = "vocab.txt";
 20
 21    /// <summary>MiniLM vocab download URL (uncased BERT WordPiece).</summary>
 22    internal const string MiniLmVocabUrl =
 23        "https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/vocab.txt";
 24
 25    /// <summary>Phi-3 SentencePiece tokenizer model file.</summary>
 26    internal const string Phi3TokenizerFileName = "tokenizer.model";
 27
 28    /// <summary>Phi-3 SentencePiece tokenizer model download URL.</summary>
 29    internal const string Phi3TokenizerUrl =
 30        "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-onnx/resolve/main/cpu_and_mobile/cpu-int4-rtn-block-32/
 31
 32    /// <summary>
 33    /// Loads the all-MiniLM-L6-v2 tokenizer (an uncased BERT WordPiece model) from a cached or
 34    /// freshly-downloaded <c>vocab.txt</c>. Returns the configured <see cref="BertTokenizer"/>.
 35    /// </summary>
 36    /// <param name="options">Engine options; the cache root is derived from the default model cache.</param>
 37    /// <param name="logger">Logger for download/failure diagnostics.</param>
 38    /// <param name="ct">Cancellation token.</param>
 39    /// <returns>A <see cref="Tokenizer"/> for all-MiniLM-L6-v2.</returns>
 40    /// <exception cref="InvalidOperationException">Thrown if the vocab asset cannot be fetched.</exception>
 41    public static async Task<Tokenizer> LoadMiniLmTokenizerAsync(
 42        InferenceEngineOptions options,
 43        ILogger logger,
 44        CancellationToken ct = default)
 45    {
 446        var vocabPath = await DownloadAssetAsync(
 447            MiniLmVocabFileName,
 448            new[] { MiniLmVocabUrl },
 449            options,
 450            logger,
 451            ct).ConfigureAwait(false);
 52
 453        var bertOptions = new BertOptions { LowerCaseBeforeTokenization = true };
 454        await using var stream = new FileStream(vocabPath, FileMode.Open, FileAccess.Read, FileShare.Read);
 455        var tokenizer = BertTokenizer.Create(stream, bertOptions);
 256        logger.LogInformation("Loaded MiniLM (all-MiniLM-L6-v2) BERT tokenizer from {Path}", vocabPath);
 257        return tokenizer;
 258    }
 59
 60    /// <summary>
 61    /// Loads the Phi-3 SentencePiece tokenizer from a cached or freshly-downloaded
 62    /// <c>tokenizer.model</c>. No BOS/EOS is auto-inserted; the engine drives EOS via
 63    /// <see cref="StreamingOptions.EosTokenIds"/>.
 64    /// </summary>
 65    /// <remarks>
 66    /// <para>
 67    /// In <c>Microsoft.ML.Tokenizers</c> 1.0.3 the <c>SentencePieceTokenizer</c> type has no public
 68    /// factory of its own. The public factory that reads a SentencePiece model and returns a
 69    /// <see cref="SentencePieceTokenizer"/>-derived instance is <see cref="LlamaTokenizer.Create"/>,
 70    /// whose signature matches the intended SentencePiece usage. Phi-3's tokenizer.model is a
 71    /// SentencePiece model and is read here via that factory.
 72    /// </para>
 73    /// </remarks>
 74    /// <param name="options">Engine options; the cache root is derived from the default model cache.</param>
 75    /// <param name="logger">Logger for download/failure diagnostics.</param>
 76    /// <param name="ct">Cancellation token.</param>
 77    /// <returns>A <see cref="Tokenizer"/> for Phi-3-mini-4k-instruct.</returns>
 78    /// <exception cref="InvalidOperationException">Thrown if the model asset cannot be fetched.</exception>
 79    public static async Task<Tokenizer> LoadPhi3TokenizerAsync(
 80        InferenceEngineOptions options,
 81        ILogger logger,
 82        CancellationToken ct = default)
 83    {
 284        var modelPath = await DownloadAssetAsync(
 285            Phi3TokenizerFileName,
 286            new[] { Phi3TokenizerUrl },
 287            options,
 288            logger,
 289            ct).ConfigureAwait(false);
 90
 291        var specialTokens = new System.Collections.Generic.Dictionary<string, int>
 292        {
 293            ["<|endoftext|>"] = 32000,
 294            ["<|im_start|>"] = 32001,
 295            ["<|im_end|>"] = 32002,
 296        };
 97
 298        await using var stream = new FileStream(modelPath, FileMode.Open, FileAccess.Read, FileShare.Read);
 99        // LlamaTokenizer is the public SentencePiece-backed factory in Tokenizers 1.0.3 and returns
 100        // a SentencePieceTokenizer-derived Tokenizer. addBos/addEos=false: the engine owns EOS.
 2101        var tokenizer = LlamaTokenizer.Create(
 2102            stream,
 2103            addBeginOfSentence: false,
 2104            addEndOfSentence: false,
 2105            specialTokens);
 1106        logger.LogInformation("Loaded Phi-3 SentencePiece tokenizer from {Path}", modelPath);
 1107        return tokenizer;
 1108    }
 109
 110    /// <summary>
 111    /// Test seam (used by <c>InferenceEngine.Tests</c>): overrides the tokenizers cache directory so
 112    /// tests can point the asset loader at an isolated temp directory (hermetic failure/corruption
 113    /// injection) instead of the shared user-level cache. Null (default) = production behavior.
 114    /// </summary>
 34115    internal static string? TokenizerCacheDirectoryOverride { get; set; }
 116
 117    /// <summary>
 118    /// Downloads (or reuses a cached copy of) a single tokenizer asset file under the engine's
 119    /// tokenizers cache directory using <see cref="DownloadingModelProvider"/>. Internal so the
 120    /// test project can drive the download-failure wrapping directly.
 121    /// </summary>
 122    internal static async Task<string> DownloadAssetAsync(
 123        string fileName,
 124        string[] urls,
 125        InferenceEngineOptions options,
 126        ILogger logger,
 127        CancellationToken ct)
 128    {
 10129        _ = options ?? throw new ArgumentNullException(nameof(options));
 9130        string cacheDir = GetTokenizersCacheDirectory();
 9131        Directory.CreateDirectory(cacheDir);
 132
 9133        var provider = new DownloadingModelProvider(
 9134            modelName: fileName,
 9135            downloadUrls: urls,
 9136            cacheDirectory: cacheDir,
 9137            logger: logger);
 138
 139        try
 140        {
 9141            return await provider.GetModelPathAsync(ct).ConfigureAwait(false);
 142        }
 2143        catch (Exception ex)
 144        {
 2145            logger.LogError(ex, "Failed to obtain tokenizer asset '{File}' from {UrlCount} source(s)", fileName, urls.Le
 2146            throw new InvalidOperationException(
 2147                $"Tokenizer asset '{fileName}' could not be loaded or downloaded. " +
 2148                "Provide network access or place the file in the tokenizers cache directory.", ex);
 149        }
 7150    }
 151
 152    /// <summary>Tokenizers cache directory, located alongside the default model cache.</summary>
 153    private static string GetTokenizersCacheDirectory() =>
 9154        TokenizerCacheDirectoryOverride is { } overrideDir
 9155            ? overrideDir
 9156            : Path.Combine(
 9157                Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
 9158                "InferenceEngine",
 9159                "Tokenizers");
 160}