| | | 1 | | using System; |
| | | 2 | | using System.Collections.Generic; |
| | | 3 | | using System.Linq; |
| | | 4 | | using System.Threading.Tasks; |
| | | 5 | | using Microsoft.Extensions.Logging; |
| | | 6 | | using Microsoft.ML.OnnxRuntime; |
| | | 7 | | using Microsoft.ML.OnnxRuntime.Tensors; |
| | | 8 | | using SkiaSharp; |
| | | 9 | | using InferenceEngine.Core; |
| | | 10 | | |
| | | 11 | | namespace InferenceEngine.Examples |
| | | 12 | | { |
| | | 13 | | /// <summary> |
| | | 14 | | /// Specialized Inference Engine for Face/Object Detection (YOLOv8 based). |
| | | 15 | | /// Adapts the generic BaseInferenceEngine to handle Computer Vision specific tasks |
| | | 16 | | /// like image resizing, aspect ratio scaling, and coordinate mapping. |
| | | 17 | | /// </summary> |
| | | 18 | | public class FaceDetectionEngine : BaseInferenceEngine<byte[], List<FaceDetectionEngine.PredictionResult>> |
| | | 19 | | { |
| | | 20 | | // Configuration |
| | | 21 | | private const float ConfidenceThreshold = 0.45f; |
| | | 22 | | private const float IouThreshold = 0.5f; |
| | | 23 | | |
| | | 24 | | // Diagnostics |
| | 0 | 25 | | private static readonly bool BboxDiagEnabled = |
| | 0 | 26 | | string.Equals(Environment.GetEnvironmentVariable("INFERENCEAPP_BBOX_DIAG"), "1", StringComparison.OrdinalIgn |
| | | 27 | | private const float CoordSpaceToleranceMargin = 1.15f; |
| | | 28 | | |
| | 6 | 29 | | private readonly string[] _labels = |
| | 6 | 30 | | [ |
| | 6 | 31 | | "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", |
| | 6 | 32 | | "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", |
| | 6 | 33 | | "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", |
| | 6 | 34 | | "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", |
| | 6 | 35 | | "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", |
| | 6 | 36 | | "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", |
| | 6 | 37 | | "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", |
| | 6 | 38 | | "cell phone", |
| | 6 | 39 | | "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", |
| | 6 | 40 | | "hair drier", "toothbrush" |
| | 6 | 41 | | ]; |
| | | 42 | | |
| | | 43 | | /// <summary> |
| | | 44 | | /// Name of the model file required by this engine. |
| | | 45 | | /// The yolo11n.onnx model is registered in the default ModelRegistry. |
| | | 46 | | /// </summary> |
| | 0 | 47 | | protected override string ModelName => "yolo11n.onnx"; |
| | | 48 | | |
| | | 49 | | // Note: ModelDownloadUrls is not overridden because yolo11n.onnx is in the default registry. |
| | | 50 | | // The engine will automatically find download URLs from models.json. |
| | | 51 | | // Override ModelDownloadUrls only for custom/unregistered models. |
| | | 52 | | |
| | | 53 | | public FaceDetectionEngine(InferenceEngineOptions options, ILogger<FaceDetectionEngine> logger) |
| | 6 | 54 | | : base(options, logger) |
| | | 55 | | { |
| | 6 | 56 | | } |
| | | 57 | | |
| | | 58 | | // --- Pipeline Implementation --- |
| | | 59 | | |
| | | 60 | | /// <summary> |
| | | 61 | | /// Preprocesses the image and returns the tensor + scaling context. |
| | | 62 | | /// </summary> |
| | | 63 | | protected override (List<NamedOnnxValue> InputTensors, object? Context) PreProcessWithContext(byte[] imageBytes) |
| | | 64 | | { |
| | | 65 | | // 1. Decode |
| | 0 | 66 | | using var sourceImage = SKBitmap.Decode(imageBytes) |
| | 0 | 67 | | ?? throw new ArgumentException("Failed to decode input image bytes", nameof(imageBytes)); |
| | | 68 | | |
| | | 69 | | // 2. Get Dimensions (Metadata is auto-populated by BaseInferenceEngine) |
| | | 70 | | // Dimensions are usually [-1, 3, Height, Width] or [-1, 3, -1, -1] for dynamic shapes |
| | 0 | 71 | | if (InputMetadata is null || PrimaryInputName is null || !InputMetadata.ContainsKey(PrimaryInputName)) |
| | | 72 | | { |
| | 0 | 73 | | throw new InvalidOperationException($"Engine is not loaded or model metadata is missing. Ensure {nameof( |
| | | 74 | | } |
| | 0 | 75 | | var dims = InputMetadata[PrimaryInputName].Dimensions; |
| | 0 | 76 | | int modelH = dims[2] > 0 ? dims[2] : 640; // Default to 640 for dynamic shapes |
| | 0 | 77 | | int modelW = dims[3] > 0 ? dims[3] : 640; |
| | | 78 | | |
| | | 79 | | // 3. Resize |
| | 0 | 80 | | using var resized = sourceImage.Resize(new SKImageInfo(modelW, modelH), SKSamplingOptions.Default) |
| | 0 | 81 | | ?? throw new Exception($"Failed to resize image to {modelW}x{modelH}"); |
| | | 82 | | |
| | | 83 | | // 4. Calculate Scale Factors (Context) |
| | 0 | 84 | | var context = new ImageScaleContext |
| | 0 | 85 | | { |
| | 0 | 86 | | ScaleX = (float)sourceImage.Width / modelW, |
| | 0 | 87 | | ScaleY = (float)sourceImage.Height / modelH, |
| | 0 | 88 | | SourceWidth = sourceImage.Width, |
| | 0 | 89 | | SourceHeight = sourceImage.Height |
| | 0 | 90 | | }; |
| | | 91 | | |
| | | 92 | | // 5. To Tensor (NCHW) |
| | 0 | 93 | | var tensor = new DenseTensor<float>([1, 3, modelH, modelW]); |
| | 0 | 94 | | using var converted = resized.Copy(SKColorType.Rgba8888); |
| | 0 | 95 | | var pixels = converted.Bytes; |
| | 0 | 96 | | var bytesPerPixel = converted.BytesPerPixel; |
| | | 97 | | |
| | 0 | 98 | | Parallel.For(0, modelH, y => |
| | 0 | 99 | | { |
| | 0 | 100 | | for (int x = 0; x < modelW; x++) |
| | 0 | 101 | | { |
| | 0 | 102 | | int offset = (y * modelW + x) * bytesPerPixel; |
| | 0 | 103 | | tensor[0, 0, y, x] = pixels[offset] / 255.0f; // R |
| | 0 | 104 | | tensor[0, 1, y, x] = pixels[offset + 1] / 255.0f; // G |
| | 0 | 105 | | tensor[0, 2, y, x] = pixels[offset + 2] / 255.0f; // B |
| | 0 | 106 | | } |
| | 0 | 107 | | }); |
| | | 108 | | |
| | 0 | 109 | | return (new List<NamedOnnxValue> { NamedOnnxValue.CreateFromTensor(PrimaryInputName, tensor) }, context); |
| | 0 | 110 | | } |
| | | 111 | | |
| | | 112 | | /// <summary> |
| | | 113 | | /// Postprocesses the output tensors using the scaling context to map boxes. |
| | | 114 | | /// </summary> |
| | | 115 | | protected override List<PredictionResult> PostProcessWithContext( |
| | | 116 | | IDisposableReadOnlyCollection<DisposableNamedOnnxValue> output, |
| | | 117 | | object? context) |
| | | 118 | | { |
| | 0 | 119 | | var scaleCtx = context as ImageScaleContext |
| | 0 | 120 | | ?? throw new InvalidOperationException("Invalid context type passed to PostProcess"); |
| | | 121 | | |
| | 0 | 122 | | var outputTensor = output.First(o => o.Name == "output0").AsTensor<float>(); |
| | 0 | 123 | | var predictions = new List<PredictionResult>(); |
| | | 124 | | |
| | 0 | 125 | | int channels = outputTensor.Dimensions[1]; |
| | 0 | 126 | | int anchors = outputTensor.Dimensions[2]; |
| | | 127 | | |
| | | 128 | | // 1. Determine Coordinate Space (Heuristic) |
| | 0 | 129 | | float maxCx = float.NegativeInfinity, maxCy = float.NegativeInfinity; |
| | 0 | 130 | | float maxW = float.NegativeInfinity, maxH = float.NegativeInfinity; |
| | | 131 | | |
| | 0 | 132 | | for (int i = 0; i < anchors; i++) |
| | | 133 | | { |
| | 0 | 134 | | if (outputTensor[0, 0, i] > maxCx) maxCx = outputTensor[0, 0, i]; |
| | 0 | 135 | | if (outputTensor[0, 1, i] > maxCy) maxCy = outputTensor[0, 1, i]; |
| | 0 | 136 | | if (outputTensor[0, 2, i] > maxW) maxW = outputTensor[0, 2, i]; |
| | 0 | 137 | | if (outputTensor[0, 3, i] > maxH) maxH = outputTensor[0, 3, i]; |
| | | 138 | | } |
| | | 139 | | |
| | | 140 | | // Using Dimensions from Metadata for accurate space inference |
| | | 141 | | // Apply same dynamic-shape fallback as PreProcessWithContext |
| | 0 | 142 | | var postDims = InputMetadata![PrimaryInputName!].Dimensions; |
| | 0 | 143 | | int modelH = postDims[2] > 0 ? postDims[2] : 640; |
| | 0 | 144 | | int modelW = postDims[3] > 0 ? postDims[3] : 640; |
| | | 145 | | |
| | 0 | 146 | | var coordSpace = InferBoxCoordSpace(maxCx, maxCy, maxW, maxH, scaleCtx.SourceWidth, scaleCtx.SourceHeight, m |
| | | 147 | | |
| | | 148 | | // 2. Filter & Decode |
| | 0 | 149 | | for (int i = 0; i < anchors; i++) |
| | | 150 | | { |
| | | 151 | | // Find max class score |
| | 0 | 152 | | float maxScore = 0; |
| | 0 | 153 | | int maxClassId = -1; |
| | | 154 | | |
| | 0 | 155 | | for (int c = 4; c < channels; c++) |
| | | 156 | | { |
| | 0 | 157 | | float score = outputTensor[0, c, i]; |
| | 0 | 158 | | if (score > maxScore) |
| | | 159 | | { |
| | 0 | 160 | | maxScore = score; |
| | 0 | 161 | | maxClassId = c - 4; |
| | | 162 | | } |
| | | 163 | | } |
| | | 164 | | |
| | 0 | 165 | | if (maxScore < ConfidenceThreshold) continue; |
| | | 166 | | |
| | | 167 | | // Extract Box |
| | 0 | 168 | | float cx = outputTensor[0, 0, i]; |
| | 0 | 169 | | float cy = outputTensor[0, 1, i]; |
| | 0 | 170 | | float w = outputTensor[0, 2, i]; |
| | 0 | 171 | | float h = outputTensor[0, 3, i]; |
| | | 172 | | |
| | | 173 | | // Center -> Top-Left |
| | 0 | 174 | | float x = cx - (w / 2); |
| | 0 | 175 | | float y = cy - (h / 2); |
| | | 176 | | |
| | | 177 | | // Decode |
| | 0 | 178 | | var (px, py, pw, ph) = DecodeToSourcePixels( |
| | 0 | 179 | | coordSpace, x, y, w, h, |
| | 0 | 180 | | scaleCtx.ScaleX, scaleCtx.ScaleY, |
| | 0 | 181 | | scaleCtx.SourceWidth, scaleCtx.SourceHeight); |
| | | 182 | | |
| | 0 | 183 | | predictions.Add(new PredictionResult |
| | 0 | 184 | | { |
| | 0 | 185 | | Label = maxClassId >= 0 && maxClassId < _labels.Length ? _labels[maxClassId] : "Unknown", |
| | 0 | 186 | | Confidence = maxScore, |
| | 0 | 187 | | X = px, Y = py, Width = pw, Height = ph |
| | 0 | 188 | | }); |
| | | 189 | | } |
| | | 190 | | |
| | 0 | 191 | | return ApplyNMS(predictions); |
| | | 192 | | } |
| | | 193 | | |
| | | 194 | | // --- DTOs & Helpers --- |
| | | 195 | | |
| | | 196 | | private class ImageScaleContext |
| | | 197 | | { |
| | 0 | 198 | | public float ScaleX { get; set; } |
| | 0 | 199 | | public float ScaleY { get; set; } |
| | 0 | 200 | | public int SourceWidth { get; set; } |
| | 0 | 201 | | public int SourceHeight { get; set; } |
| | | 202 | | } |
| | | 203 | | |
| | | 204 | | public class PredictionResult |
| | | 205 | | { |
| | 16 | 206 | | public string Label { get; set; } = string.Empty; |
| | 13 | 207 | | public float Confidence { get; set; } |
| | 23 | 208 | | public float X { get; set; } |
| | 23 | 209 | | public float Y { get; set; } |
| | 23 | 210 | | public float Width { get; set; } |
| | 23 | 211 | | public float Height { get; set; } |
| | | 212 | | } |
| | | 213 | | |
| | | 214 | | public enum BoxCoordSpace { Normalized01, ModelInputPixels, SourcePixels, Unknown } |
| | | 215 | | |
| | | 216 | | public BoxCoordSpace InferBoxCoordSpace( |
| | | 217 | | float maxCx, float maxCy, float maxW, float maxH, |
| | | 218 | | int srcW, int srcH, int modelW, int modelH) |
| | | 219 | | { |
| | 3 | 220 | | float maxAll = Math.Max(Math.Max(maxCx, maxCy), Math.Max(maxW, maxH)); |
| | | 221 | | |
| | 4 | 222 | | if (maxAll <= CoordSpaceToleranceMargin) return BoxCoordSpace.Normalized01; |
| | | 223 | | |
| | 2 | 224 | | float modelMax = Math.Max(modelW, modelH); |
| | 3 | 225 | | if (maxAll <= modelMax * CoordSpaceToleranceMargin) return BoxCoordSpace.ModelInputPixels; |
| | | 226 | | |
| | 1 | 227 | | float srcMax = Math.Max(srcW, srcH); |
| | 2 | 228 | | if (maxAll <= srcMax * CoordSpaceToleranceMargin) return BoxCoordSpace.SourcePixels; |
| | | 229 | | |
| | 0 | 230 | | return BoxCoordSpace.Unknown; |
| | | 231 | | } |
| | | 232 | | |
| | | 233 | | public static (float X, float Y, float W, float H) DecodeToSourcePixels( |
| | | 234 | | BoxCoordSpace space, float x, float y, float w, float h, float sX, float sY, int srcW, int srcH) |
| | | 235 | | { |
| | 4 | 236 | | return space switch |
| | 4 | 237 | | { |
| | 1 | 238 | | BoxCoordSpace.Normalized01 => (x * srcW, y * srcH, w * srcW, h * srcH), |
| | 1 | 239 | | BoxCoordSpace.ModelInputPixels => (x * sX, y * sY, w * sX, h * sY), |
| | 1 | 240 | | BoxCoordSpace.SourcePixels => (x, y, w, h), |
| | 1 | 241 | | _ => throw new InvalidOperationException($"Unknown coordinate space detected.") |
| | 4 | 242 | | }; |
| | | 243 | | } |
| | | 244 | | |
| | | 245 | | protected List<PredictionResult> ApplyNMS(List<PredictionResult> boxes) |
| | | 246 | | { |
| | 4 | 247 | | var sorted = boxes.OrderByDescending(b => b.Confidence).ToList(); |
| | 1 | 248 | | var selected = new List<PredictionResult>(); |
| | | 249 | | |
| | 3 | 250 | | while (sorted.Count > 0) |
| | | 251 | | { |
| | 2 | 252 | | var best = sorted[0]; |
| | 2 | 253 | | selected.Add(best); |
| | 2 | 254 | | sorted.RemoveAt(0); |
| | | 255 | | |
| | 8 | 256 | | for (int i = sorted.Count - 1; i >= 0; i--) |
| | | 257 | | { |
| | 3 | 258 | | if (CalculateIoU(best, sorted[i]) > IouThreshold) sorted.RemoveAt(i); |
| | | 259 | | } |
| | | 260 | | } |
| | 1 | 261 | | return selected; |
| | | 262 | | } |
| | | 263 | | |
| | | 264 | | protected float CalculateIoU(PredictionResult boxA, PredictionResult boxB) |
| | | 265 | | { |
| | 4 | 266 | | float xA = Math.Max(boxA.X, boxB.X); |
| | 4 | 267 | | float yA = Math.Max(boxA.Y, boxB.Y); |
| | 4 | 268 | | float xB = Math.Min(boxA.X + boxA.Width, boxB.X + boxB.Width); |
| | 4 | 269 | | float yB = Math.Min(boxA.Y + boxA.Height, boxB.Y + boxB.Height); |
| | | 270 | | |
| | 4 | 271 | | float interArea = Math.Max(0, xB - xA) * Math.Max(0, yB - yA); |
| | 4 | 272 | | float unionArea = (boxA.Width * boxA.Height) + (boxB.Width * boxB.Height) - interArea; |
| | | 273 | | |
| | 4 | 274 | | if (unionArea <= 0f) return 0f; |
| | | 275 | | |
| | 4 | 276 | | return interArea / unionArea; |
| | | 277 | | } |
| | | 278 | | } |
| | | 279 | | } |