Upload 2 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+palm_detection_lite.sentis filter=lfs diff=lfs merge=lfs -text

RunBlazePalm.cs

@@ -0,0 +1,348 @@
+using UnityEngine;
+using Unity.Sentis;
+using UnityEngine.Video;
+using UnityEngine.UI;
+using Lays = Unity.Sentis.Layers;
+
+/*
+ *                   Blaze Palm Inference
+ *                   ====================
+ *                   
+ * Basic inference script for blaze palm
+ * 
+ * Put this script on the Main Camera
+ * Put blazeface.sentis in the Assets/StreamingAssets folder
+ * Create a RawImage of in the scene
+ * Put a link to that image in previewUI
+ * Put a video in Assets/StreamingAssets folder and put the name of it int videoName
+ * Or put a test image in inputImage
+ * Set inputType to appropriate input
+ */
+
+
+public class RunBlazePalm : MonoBehaviour
+{
+    //Drag a link to a raw image here:
+    public RawImage previewUI = null;
+
+    // Put your bounding box sprite image here
+    public Sprite boundingBoxTexture;
+
+    // optional images for palm markers
+    public Sprite[] markerTextures;
+
+    public string videoName = "chatting.mp4";
+
+    // 
+    public Texture2D inputImage;
+
+    public InputType inputType = InputType.Video;
+
+    //Resolution of preview image/video
+    Vector2Int resolution = new Vector2Int(640, 640);
+    WebCamTexture webcam;
+    VideoPlayer video;
+
+    const BackendType backend = BackendType.GPUCompute;
+
+    RenderTexture targetTexture;
+    public enum InputType { Image, Video, Webcam };
+   
+    //Some adjustable parameters for the model
+    [SerializeField, Range(0, 1)] float iouThreshold = 0.5f;
+    [SerializeField, Range(0, 1)] float scoreThreshold = 0.5f;
+    const int maxOutputBoxes = 64;
+
+    IWorker worker;
+
+    //Holds image size
+    int size;
+
+    Ops ops;
+    ITensorAllocator allocator;
+
+    Model model;
+
+    //webcam device name:
+    const string deviceName = "";
+
+    bool closing = false;
+
+    const string regressorsOutput = "Identity";
+    const string classificatorsOutput = "Identity_1";
+
+    public struct BoundingBox
+    {
+        public float centerX;
+        public float centerY;
+        public float width;
+        public float height;
+    }
+
+    void Start()
+    {
+        allocator = new TensorCachingAllocator();
+
+        //(Note: if using a webcam on mobile get permissions here first)
+
+        targetTexture = new RenderTexture(resolution.x, resolution.y, 0);
+        previewUI.texture = targetTexture;
+
+        SetupInput();
+
+        SetupModel();
+
+        SetupEngine();
+    }
+
+    void SetupInput()
+    {
+        switch (inputType)
+        {
+            case InputType.Webcam:
+                {
+                    webcam = new WebCamTexture(deviceName, resolution.x, resolution.y);
+                    webcam.requestedFPS = 30;
+                    webcam.Play();
+                    break;
+                }
+            case InputType.Video:
+                {
+                    video = gameObject.AddComponent<VideoPlayer>();//new VideoPlayer();
+                    video.renderMode = VideoRenderMode.APIOnly;
+                    video.source = VideoSource.Url;
+                    video.url = Application.streamingAssetsPath + "/"+videoName;
+                    video.isLooping = true;
+                    video.Play();
+                    break;
+                }
+            default:
+                {
+                    Graphics.Blit(inputImage, targetTexture);
+                }
+                break;
+        }
+    }
+
+    void Update()
+    {
+        if (inputType == InputType.Webcam)
+        {
+            // Format video input
+            if (!webcam.didUpdateThisFrame) return;
+
+            var aspect1 = (float)webcam.width / webcam.height;
+            var aspect2 = (float)resolution.x / resolution.y;
+            var gap = aspect2 / aspect1;
+
+            var vflip = webcam.videoVerticallyMirrored;
+            var scale = new Vector2(gap, vflip ? -1 : 1);
+            var offset = new Vector2((1 - gap) / 2, vflip ? 1 : 0);
+
+            Graphics.Blit(webcam, targetTexture, scale, offset);
+        }
+        if (inputType == InputType.Video)
+        {
+            var aspect1 = (float)video.width / video.height;
+            var aspect2 = (float)resolution.x / resolution.y;
+            var gap = aspect2 / aspect1;
+
+            var vflip = false;
+            var scale = new Vector2(gap, vflip ? -1 : 1);
+            var offset = new Vector2((1 - gap) / 2, vflip ? 1 : 0);
+            Graphics.Blit(video.texture, targetTexture, scale, offset);
+        }
+        if (inputType == InputType.Image)
+        {
+            Graphics.Blit(inputImage, targetTexture);
+        }
+
+        if (Input.GetKeyDown(KeyCode.Escape))
+        {
+            closing = true;
+            Application.Quit();
+        }
+
+        if (Input.GetKeyDown(KeyCode.P))
+        {
+            previewUI.enabled = !previewUI.enabled;
+        }
+    }
+
+
+    void LateUpdate()
+    {
+        if (!closing)
+        {
+            RunInference(targetTexture);
+        }
+    }
+
+
+    void AddGrid(float[] offsets, int rows, int repeats, int cellWidth, ref int n)
+    {
+        for (int j = 0; j < repeats * rows * rows; j++)
+        {
+            offsets[n++] = cellWidth * ((j / repeats) % rows - (rows - 1) * 0.5f);
+            offsets[n++] = cellWidth * ((j / repeats / rows) - (rows - 1) * 0.5f);
+            n += 2;
+        }
+    }
+
+    float[] GetGridBoxCoords()
+    {
+        var offsets = new float[2016 * 4];
+        int n = 0;
+        AddGrid(offsets, 24, 2, 8, ref n);
+        AddGrid(offsets, 12, 6, 16, ref n);
+        return offsets;
+    }
+
+    void SetupModel()
+    {
+        float[] offsets = GetGridBoxCoords();
+
+        model = ModelLoader.Load(Application.streamingAssetsPath + "/palm_detection_lite.sentis");
+
+        //We need to add extra layers to the model in order to aggregate the box predicions:
+        size = model.inputs[0].shape.ToTensorShape()[2]; // Input tensor width (192)
+
+        model.AddConstant(new Lays.Constant("zero", new int[] { 0 }));
+        model.AddConstant(new Lays.Constant("two", new int[] { 2 }));
+        model.AddConstant(new Lays.Constant("four", new int[] { 4 }));
+
+        model.AddLayer(new Lays.Slice("boxes", regressorsOutput, "zero", "four", "two"));
+
+        model.AddLayer(new Lays.Transpose("scores", classificatorsOutput, new int[] { 0, 2, 1 }));
+
+        model.AddConstant(new Lays.Constant("offsets",
+            new TensorFloat(new TensorShape(1, offsets.Length/4, 4), offsets)
+         ));
+        model.AddLayer(new Lays.Add("boxCoords", "boxes", "offsets"));
+        model.AddOutput("boxCoords");
+
+        model.AddConstant(new Lays.Constant("maxOutputBoxes", new int[] { maxOutputBoxes }));
+        model.AddConstant(new Lays.Constant("iouThreshold", new float[] { iouThreshold }));
+        model.AddConstant(new Lays.Constant("scoreThreshold", new float[] { scoreThreshold }));
+
+        model.AddLayer(new Lays.NonMaxSuppression("NMS", "boxCoords", "scores",
+            "maxOutputBoxes", "iouThreshold", "scoreThreshold",
+            centerPointBox: Lays.CenterPointBox.Center
+        ));
+        model.AddOutput("NMS");
+    }
+    public void SetupEngine()
+    {
+        worker = WorkerFactory.CreateWorker(backend, model);
+        ops = WorkerFactory.CreateOps(backend, allocator);
+    }
+
+    void DrawPalms(TensorFloat index3, TensorFloat regressors, int NMAX, Vector2 scale)
+    {
+        for (int n = 0; n < NMAX; n++)
+        {
+            //Draw bounding box of face
+            var box = new BoundingBox
+            {
+                centerX = index3[0, n, 0] * scale.x,
+                centerY = index3[0, n, 1] * scale.y,
+                width = index3[0, n, 2] * scale.x,
+                height = index3[0, n, 3] * scale.y
+            };
+            DrawBox(box, boundingBoxTexture);
+            if (regressors == null) continue;
+            
+            //Draw markers starts of fingers
+            for (int j = 0; j < 7; j++)
+            {
+                var marker = new BoundingBox
+                {
+                    centerX = box.centerX + (regressors[0, n, 4 + j * 2] - regressors[0, n, 0]) * scale.x,
+                    centerY = box.centerY + (regressors[0, n, 4 + j * 2 + 1] - regressors[0, n, 1]) * scale.y,
+                    width = 4f * scale.x,
+                    height = 4f * scale.y,
+                };
+                DrawBox(marker, j < markerTextures.Length ? markerTextures[j] : boundingBoxTexture);
+            }
+        }
+    }
+
+    void RunInference(Texture source)
+    {
+        var transform = new TextureTransform();
+        transform.SetDimensions(size, size, 3);
+        transform.SetTensorLayout(0, 3, 1, 2);
+        using var image0 = TextureConverter.ToTensor(source, transform);
+
+        // Pre-process the image to make input in range (-1..1)
+        //using var image = ops.Mad(image0, 2f, -1f);
+
+        worker.Execute(image0);
+
+        using var boxCoords = worker.PeekOutput("boxCoords") as TensorFloat; //face coords
+        using var regressors = worker.PeekOutput(regressorsOutput) as TensorFloat; //contains markers
+
+        var NMS = worker.PeekOutput("NMS") as TensorInt;
+
+        using var boxCoords2 = boxCoords.ShallowReshape(boxCoords.shape.Unsqueeze(0)) as TensorFloat;
+        using var output = ops.GatherND(boxCoords2, NMS, 0);
+
+        using var regressors2 = regressors.ShallowReshape(regressors.shape.Unsqueeze(0)) as TensorFloat;
+        using var markersOutput = ops.GatherND(regressors2, NMS, 0);
+
+        output.MakeReadable();
+        markersOutput.MakeReadable();
+
+        ClearAnnotations();
+
+        Vector2 markerScale = previewUI.rectTransform.rect.size / size;
+
+        DrawPalms(output, markersOutput, output.shape[0], markerScale);
+
+    }
+
+
+    public void DrawBox(BoundingBox box, Sprite sprite)
+    {
+        var panel = new GameObject("ObjectBox");
+        panel.AddComponent<CanvasRenderer>();
+        panel.AddComponent<Image>();
+        panel.transform.SetParent(previewUI.transform, false);
+
+        var img = panel.GetComponent<Image>();
+        img.color = Color.white;
+        img.sprite = sprite;
+        img.type = Image.Type.Sliced;
+
+        panel.transform.localPosition = new Vector3(box.centerX, -box.centerY);
+        RectTransform rt = panel.GetComponent<RectTransform>();
+        rt.sizeDelta = new Vector2(box.width, box.height);
+    }
+    public void ClearAnnotations()
+    {
+        foreach (Transform child in previewUI.transform)
+        {
+            Destroy(child.gameObject);
+        }
+    }
+
+    void CleanUp()
+    {
+        closing = true;
+        ops?.Dispose();
+        allocator?.Dispose();
+        if (webcam) Destroy(webcam);
+        if (video) Destroy(video);
+        RenderTexture.active = null;
+        targetTexture.Release();
+        worker?.Dispose();
+        worker = null;
+    }
+
+    void OnDestroy()
+    {
+        CleanUp();
+    }
+
+}
+

palm_detection_lite.sentis

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d1746c58054f5419c994d8869e7945dd074d34bbedb27b518cb4d106447973b
+size 3953743