Cannot use for double player poselandmark

[chongyukwai]

Plugin Version or Commit ID

V0.16.1

Unity Version

2022.32f1

Your Host OS

Window 10

Target Platform

UnityEditor

Description

Code only for one person even numpeople is 2

Code to Reproduce the issue

using System.Collections;
using Mediapipe.Tasks.Vision.PoseLandmarker;
using UnityEngine;
using UnityEngine.Rendering;
using Mediapipe.Tasks.Components.Containers;
using System.Collections.Generic;
using UnityEngine.UI;

namespace Mediapipe.Unity.Sample.PoseLandmarkDetection
{
public class PoseLandmarkerRunner : VisionTaskApiRunner
{
[Header("UI References")]
[SerializeField] private PoseLandmarkerResultAnnotationController _poseLandmarkerResultAnnotationController;
[SerializeField] private Text[] leftCounterTexts = new Text[2];
[SerializeField] private Text[] rightCounterTexts = new Text[2];
[SerializeField] private Text[] leftStageTexts = new Text[2];
[SerializeField] private Text[] rightStageTexts = new Text[2];
[SerializeField] private Text[] calibrationTexts = new Text[2];

    [Header("Settings")]
    [SerializeField] private float calibrationDuration = 3f;
    [SerializeField] private float downAngleThreshold = 160f;
    [SerializeField] private float upAngleThreshold = 30f;
    [SerializeField] private float tPoseAngleThreshold = 30f;

    private Experimental.TextureFramePool _textureFramePool;
    private bool[] isCalibrating = new bool[2];
    private float[] calibrationStartTimes = new float[2];
    private float[] leftArmLengths = new float[2];
    private float[] rightArmLengths = new float[2];
    private bool[] isCalibrated = new bool[2];

    // Exercise tracking variables
    public  int[] leftCounters = new int[2];
    public int[] rightCounters = new int[2];
    public string[] leftStages = new string[2];
    public string[] rightStages = new string[2];

    // Thread-safe variables for calibration
    private bool[] _shouldStartCalibration = new bool[2];
    private bool[] _shouldCompleteCalibration = new bool[2];
    private float[] _calibrationTimes = new float[2];
    
    public readonly PoseLandmarkDetectionConfig config = new PoseLandmarkDetectionConfig();

    protected override IEnumerator Start()
    {
        // Initialize stages
        for (int i = 0; i < 2; i++)
        {
            leftStages[i] = "down";
            rightStages[i] = "down";
        }
        return base.Start();
    }

    protected override IEnumerator Run()
    {
        Debug.Log($"Initializing Pose Landmarker with config: {config}");

        yield return AssetLoader.PrepareAssetAsync(config.ModelPath);

        var options = config.GetPoseLandmarkerOptions(config.RunningMode == Tasks.Vision.Core.RunningMode.LIVE_STREAM ? OnPoseLandmarkDetectionOutput : null);
        taskApi = PoseLandmarker.CreateFromOptions(options, GpuManager.GpuResources);
        var imageSource = ImageSourceProvider.ImageSource;

        yield return imageSource.Play();

        if (!imageSource.isPrepared)
        {
            Logger.LogError(TAG, "Failed to start ImageSource, exiting...");
            yield break;
        }

        _textureFramePool = new Experimental.TextureFramePool(imageSource.textureWidth, imageSource.textureHeight, TextureFormat.RGBA32, 10);
        screen.Initialize(imageSource);

        SetupAnnotationController(_poseLandmarkerResultAnnotationController, imageSource);
        _poseLandmarkerResultAnnotationController.InitScreen(imageSource.textureWidth, imageSource.textureHeight);

        var transformationOptions = imageSource.GetTransformationOptions();
        var imageProcessingOptions = new Tasks.Vision.Core.ImageProcessingOptions(rotationDegrees: 0);

        AsyncGPUReadbackRequest req = default;
        var waitUntilReqDone = new WaitUntil(() => req.done);
        var waitForEndOfFrame = new WaitForEndOfFrame();
        var result = PoseLandmarkerResult.Alloc(options.numPoses, options.outputSegmentationMasks);

        var canUseGpuImage = SystemInfo.graphicsDeviceType == GraphicsDeviceType.OpenGLES3 && GpuManager.GpuResources != null;
        using var glContext = canUseGpuImage ? GpuManager.GetGlContext() : null;

        while (true)
        {
            if (isPaused)
            {
                yield return new WaitWhile(() => isPaused);
            }

            if (!_textureFramePool.TryGetTextureFrame(out var textureFrame))
            {
                yield return new WaitForEndOfFrame();
                continue;
            }

            Image image;
            switch (config.ImageReadMode)
            {
                case ImageReadMode.GPU:
                    textureFrame.ReadTextureOnGPU(imageSource.GetCurrentTexture(), transformationOptions.flipHorizontally, transformationOptions.flipVertically);
                    image = textureFrame.BuildGPUImage(glContext);
                    yield return waitForEndOfFrame;
                    break;
                case ImageReadMode.CPU:
                    yield return waitForEndOfFrame;
                    textureFrame.ReadTextureOnCPU(imageSource.GetCurrentTexture(), transformationOptions.flipHorizontally, transformationOptions.flipVertically);
                    image = textureFrame.BuildCPUImage();
                    textureFrame.Release();
                    break;
                default:
                    req = textureFrame.ReadTextureAsync(imageSource.GetCurrentTexture(), transformationOptions.flipHorizontally, transformationOptions.flipVertically);
                    yield return waitUntilReqDone;
                    image = textureFrame.BuildCPUImage();
                    textureFrame.Release();
                    break;
            }

            switch (taskApi.runningMode)
            {
                case Tasks.Vision.Core.RunningMode.IMAGE:
                    if (taskApi.TryDetect(image, imageProcessingOptions, ref result))
                    {
                        _poseLandmarkerResultAnnotationController.DrawNow(result);
                        ProcessPoseResult(result);
                    }
                    else
                    {
                        _poseLandmarkerResultAnnotationController.DrawNow(default);
                    }
                    DisposeAllMasks(result);
                    break;
                case Tasks.Vision.Core.RunningMode.VIDEO:
                    if (taskApi.TryDetectForVideo(image, GetCurrentTimestampMillisec(), imageProcessingOptions, ref result))
                    {
                        _poseLandmarkerResultAnnotationController.DrawNow(result);
                        ProcessPoseResult(result);
                    }
                    else
                    {
                        _poseLandmarkerResultAnnotationController.DrawNow(default);
                    }
                    DisposeAllMasks(result);
                    break;
                case Tasks.Vision.Core.RunningMode.LIVE_STREAM:
                    taskApi.DetectAsync(image, GetCurrentTimestampMillisec(), imageProcessingOptions);
                    break;
            }

            // Handle calibration timing on main thread
            for (int i = 0; i < 2; i++)
            {
                if (_shouldStartCalibration[i])
                {
                    calibrationStartTimes[i] = Time.time;
                    isCalibrating[i] = true;
                    _shouldStartCalibration[i] = false;
                    UpdateCalibrationText(i, "Hold T-Position for calibration...");
                }

                if (isCalibrating[i])
                {
                    _calibrationTimes[i] = Time.time - calibrationStartTimes[i];
                    float timeRemaining = calibrationDuration - _calibrationTimes[i];
                    UpdateCalibrationText(i, $"Hold T-Position... {timeRemaining.ToString("0.0")}s");
                }

                if (_shouldCompleteCalibration[i])
                {
                    isCalibrating[i] = false;
                    isCalibrated[i] = true;
                    _shouldCompleteCalibration[i] = false;
                    UpdateCalibrationText(i,
                        $"Player {i + 1} Calibrated!\nLeft: {leftArmLengths[i].ToString("0.00")}\nRight: {rightArmLengths[i].ToString("0.00")}");
                }
            }
        }
    }

    void OnPoseLandmarkDetectionOutput(PoseLandmarkerResult result, Image image, long timestamp)
    {
        _poseLandmarkerResultAnnotationController.DrawLater(result);
        ProcessPoseResult(result);
        DisposeAllMasks(result);
    }

    void ProcessPoseResult(PoseLandmarkerResult result)
    {
        if (result.poseLandmarks == null || result.poseLandmarks.Count == 0)
        {
            ResetAllUI();
            return;
        }

        int numPeople = Mathf.Min(result.poseLandmarks.Count, 2);

        for (int personIdx = 0; personIdx < numPeople; personIdx++)
        {
            var pose = result.poseLandmarks[personIdx];
            var landmarks = pose.landmarks;

            if (!isCalibrated[personIdx])
            {
                CheckForCalibration(personIdx, landmarks);
            }
            else
            {
                CheckForBicepCurl(personIdx, landmarks);
            }
        }

        // Reset UI for non-detected players
        for (int i = numPeople; i < 2; i++)
        {
            if (!isCalibrated[i])
            {
                UpdateCalibrationText(i, "Assume T-Position for calibration");
            }
            else
            {
                UpdateCurlUI(i, "No person detected");
            }
        }
    }

    void CheckForCalibration(int personIdx, System.Collections.Generic.List<Mediapipe.Tasks.Components.Containers.NormalizedLandmark> landmarks)
    {
        var leftShoulder = landmarks[PoseLandmarkIndices.LEFT_SHOULDER];
        var rightShoulder = landmarks[PoseLandmarkIndices.RIGHT_SHOULDER];
        var leftElbow = landmarks[PoseLandmarkIndices.LEFT_ELBOW];
        var rightElbow = landmarks[PoseLandmarkIndices.RIGHT_ELBOW];
        var leftWrist = landmarks[PoseLandmarkIndices.LEFT_WRIST];
        var rightWrist = landmarks[PoseLandmarkIndices.RIGHT_WRIST];

        float leftArmAngle = CalculateAngle(leftShoulder, leftElbow, leftWrist);
        float rightArmAngle = CalculateAngle(rightShoulder, rightElbow, rightWrist);

        bool isTPosition = (Mathf.Abs(leftArmAngle - 180f) < tPoseAngleThreshold) &&
                         (Mathf.Abs(rightArmAngle - 180f) < tPoseAngleThreshold);

        if (!isTPosition)
        {
            UpdateCalibrationText(personIdx, "Assume T-Position for calibration");
            isCalibrating[personIdx] = false;
            return;
        }

        if (!isCalibrating[personIdx] && !_shouldStartCalibration[personIdx])
        {
            _shouldStartCalibration[personIdx] = true;
            return;
        }

        if (isCalibrating[personIdx] && _calibrationTimes[personIdx] >= calibrationDuration)
        {
            leftArmLengths[personIdx] = CalculateDistance(leftShoulder, leftElbow) + CalculateDistance(leftElbow, leftWrist);
            rightArmLengths[personIdx] = CalculateDistance(rightShoulder, rightElbow) + CalculateDistance(rightElbow, rightWrist);
            _shouldCompleteCalibration[personIdx] = true;
        }
    }

    void CheckForBicepCurl(int personIdx, System.Collections.Generic.List<Mediapipe.Tasks.Components.Containers.NormalizedLandmark> landmarks)
    {
        var leftShoulder = landmarks[PoseLandmarkIndices.LEFT_SHOULDER];
        var leftElbow = landmarks[PoseLandmarkIndices.LEFT_ELBOW];
        var leftWrist = landmarks[PoseLandmarkIndices.LEFT_WRIST];
        float leftAngle = CalculateAngle(leftShoulder, leftElbow, leftWrist);

        var rightShoulder = landmarks[PoseLandmarkIndices.RIGHT_SHOULDER];
        var rightElbow = landmarks[PoseLandmarkIndices.RIGHT_ELBOW];
        var rightWrist = landmarks[PoseLandmarkIndices.RIGHT_WRIST];
        float rightAngle = CalculateAngle(rightShoulder, rightElbow, rightWrist);

        // Left arm logic
        if (leftAngle > downAngleThreshold) leftStages[personIdx] = "down";
        if (leftAngle < upAngleThreshold && leftStages[personIdx] == "down")
        {
            leftStages[personIdx] = "up";
            leftCounters[personIdx]++;
            Debug.Log($"Player {personIdx + 1} Left Counter: {leftCounters[personIdx]}");
        }

        // Right arm logic
        if (rightAngle > downAngleThreshold) rightStages[personIdx] = "down";
        if (rightAngle < upAngleThreshold && rightStages[personIdx] == "down")
        {
            rightStages[personIdx] = "up";
            rightCounters[personIdx]++;
            Debug.Log($"Player {personIdx + 1} Right Counter: {rightCounters[personIdx]}");
        }

        UpdateCurlUI(personIdx);
    }

    void ResetAllUI()
    {
        for (int i = 0; i < 2; i++)
        {
            if (!isCalibrated[i])
            {
                UpdateCalibrationText(i, "Assume T-Position for calibration");
            }
            else
            {
                UpdateCurlUI(i, "No person detected");
            }
        }
    }

    void UpdateCalibrationText(int personIdx, string message)
    {
        if (personIdx >= 0 && personIdx < calibrationTexts.Length && calibrationTexts[personIdx] != null)
        {
            MainThreadDispatcher.RunOnMainThread(() => {
                calibrationTexts[personIdx].text = message;
            });
        }
    }

    void UpdateCurlUI(int personIdx, string message = null)
    {
        if (personIdx < 0 || personIdx >= 2) return;

        MainThreadDispatcher.RunOnMainThread(() => {
            try
            {
                if (message != null)
                {
                    if (leftCounterTexts[personIdx] != null) leftCounterTexts[personIdx].text = message;
                    if (rightCounterTexts[personIdx] != null) rightCounterTexts[personIdx].text = message;
                    if (leftStageTexts[personIdx] != null) leftStageTexts[personIdx].text = message;
                    if (rightStageTexts[personIdx] != null) rightStageTexts[personIdx].text = message;
                }
                else
                {
                    if (leftCounterTexts[personIdx] != null)
                        leftCounterTexts[personIdx].text = $"P{personIdx + 1} Left: {leftCounters[personIdx]}";
                    if (rightCounterTexts[personIdx] != null)
                        rightCounterTexts[personIdx].text = $"P{personIdx + 1} Right: {rightCounters[personIdx]}";
                    if (leftStageTexts[personIdx] != null)
                        leftStageTexts[personIdx].text = $"P{personIdx + 1} Left: {leftStages[personIdx]}";
                    if (rightStageTexts[personIdx] != null)
                        rightStageTexts[personIdx].text = $"P{personIdx + 1} Right: {rightStages[personIdx]}";
                }
            }
            catch (System.Exception e)
            {
                Debug.LogError($"UI update failed for player {personIdx}: {e.Message}");
            }
        });
    }

    void DisposeAllMasks(PoseLandmarkerResult result)
    {
        if (result.segmentationMasks != null)
        {
            foreach (var mask in result.segmentationMasks)
            {
                mask.Dispose();
            }
        }
    }

    float CalculateAngle(Mediapipe.Tasks.Components.Containers.NormalizedLandmark a, Mediapipe.Tasks.Components.Containers.NormalizedLandmark b, Mediapipe.Tasks.Components.Containers.NormalizedLandmark c)
    {
        Vector2 aVec = new Vector2(a.x, a.y);
        Vector2 bVec = new Vector2(b.x, b.y);
        Vector2 cVec = new Vector2(c.x, c.y);

        Vector2 ba = aVec - bVec;
        Vector2 bc = cVec - bVec;

        float angleRad = Mathf.Atan2(bc.y, bc.x) - Mathf.Atan2(ba.y, ba.x);
        float angleDeg = Mathf.Abs(angleRad * Mathf.Rad2Deg);

        return angleDeg > 180.0f ? 360.0f - angleDeg : angleDeg;
    }

    float CalculateDistance(Mediapipe.Tasks.Components.Containers.NormalizedLandmark a, Mediapipe.Tasks.Components.Containers.NormalizedLandmark b)
    {
        return Mathf.Sqrt(Mathf.Pow(a.x - b.x, 2) + Mathf.Pow(a.y - b.y, 2));
    }
}

public static class PoseLandmarkIndices
{
    public const int NOSE = 0;
    public const int LEFT_EYE_INNER = 1;
    public const int LEFT_EYE = 2;
    public const int LEFT_EYE_OUTER = 3;
    public const int RIGHT_EYE_INNER = 4;
    public const int RIGHT_EYE = 5;
    public const int RIGHT_EYE_OUTER = 6;
    public const int LEFT_EAR = 7;
    public const int RIGHT_EAR = 8;
    public const int MOUTH_LEFT = 9;
    public const int MOUTH_RIGHT = 10;
    public const int LEFT_SHOULDER = 11;
    public const int RIGHT_SHOULDER = 12;
    public const int LEFT_ELBOW = 13;
    public const int RIGHT_ELBOW = 14;
    public const int LEFT_WRIST = 15;
    public const int RIGHT_WRIST = 16;
    public const int LEFT_PINKY = 17;
    public const int RIGHT_PINKY = 18;
    public const int LEFT_INDEX = 19;
    public const int RIGHT_INDEX = 20;
    public const int LEFT_THUMB = 21;
    public const int RIGHT_THUMB = 22;
    public const int LEFT_HIP = 23;
    public const int RIGHT_HIP = 24;
    public const int LEFT_KNEE = 25;
    public const int RIGHT_KNEE = 26;
    public const int LEFT_ANKLE = 27;
    public const int RIGHT_ANKLE = 28;
    public const int LEFT_HEEL = 29;
    public const int RIGHT_HEEL = 30;
    public const int LEFT_FOOT_INDEX = 31;
    public const int RIGHT_FOOT_INDEX = 32;
}

}

Additional Context

No response

[homuler]

Could you clarify the goal of this issue?

Code only for one person even numpeople is 2

What exactly do you want me to do?

[chongyukwai]

It isn't easy to show two people rigging on the screen, usually, only one dominates. Can you teach how to show two rigging in the screen as an example?

[homuler]

In my opinion, it might be a good idea to lower the values for minPoseDetectionConfidence or minPoseTrackingConfidence (see also https://ai.google.dev/edge/mediapipe/solutions/vision/pose_landmarker#configurations_options).

MediaPipeUnityPlugin/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Vision/PoseLandmarker/PoseLandmarkerOptions.cs

Lines 52 to 63 in 4fbe60f

	/// <summary>
	/// The minimum confidence score for the pose detection to be considered successful.
	/// </summary>
	public float minPoseDetectionConfidence { get; }
	/// <summary>
	/// The minimum confidence score of pose presence score in the pose landmark detection.
	/// </summary>
	public float minPosePresenceConfidence { get; }
	/// <summary>
	/// The minimum confidence score for the pose tracking to be considered successful.
	/// </summary>
	public float minTrackingConfidence { get; }

However, this plugin simply passes the MediaPipe output directly to Unity, so for methods to stabilize the output when numPoses >= 2, please ask in the official MediaPipe community.

Can you teach how to show two rigging in the screen as an example?

At least in my environment, it seems that with adjustments of those parameters, 2 people can be tracked reliably in the Pose Landmark Detection sample scene.

[chongyukwai]

It is still not showing the two rigs on the screen at the same time. Would you mind showing me the picture for reference?

[homuler]

I'm hesitant to upload images that include people, so unless it's absolutely necessary, please use any suitable images yourself to check with a sample scene (I was testing in IMAGE mode with all three confidence parameters set to 0.1. I do think higher values should be used in an actual environment).

Also, I can't answer questions about what specific values should be used for the parameters, so please ask in the official community.