Добавление 3D-точек в текущую 3D-позу для визуализации. ⇐ C#

[Anonymous]

У меня есть 3D-точки измерения, которые я использовал для расчета на Python и экспортировал их в json. Эти 3D-точки на самом деле получены из измерений SMPL Mesh. Как показано здесь.


Поэтому я пытаюсь сопоставить эти точки с вашей оценкой позы медиапайпа и визуализировать их.
Сначала я получаю точки медиапайпа из обратного вызова,
изображение
затем я выполняю разложение по сингулярным значениям между 3D-позой MediaPipe и точками измерения SMPL.
Но я не могу, по крайней мере, для первого испытания, получить визуализацию точек MediaPipe на Видео Правильно, как первый шаг для выравнивания.
Точки медиапайпа представляют собой красные сферы
[img]https: //i.sstatic.net/itAoWx0j.png[/img]

Весь код:

Код: Выделить всё

using MathNet.Numerics.LinearAlgebra;
using MathNet.Numerics.LinearAlgebra.Double;
using UnityEngine;
using System.Collections.Generic;
using System.Linq;
using Newtonsoft.Json.Linq;
using Accord.Math;
using Vector3 = Accord.Math.Vector3;

public class SMPLMediaPipeMapper : MonoBehaviour
{
public TextAsset smplDataJson;
private JObject smplData;
private Vector3[] smplJoints;
private Dictionary smplToMediaPipeMapping;
private Dictionary measurementPoints;
public Camera mainCamera;
public Vector3[] mediaPipeKeypoints;
public float mediaPipeToSMPLScale = 1f;
public GameObject smplSphere;
public GameObject mediaSphere;
public GameObject alignedSphere;

private bool measurementPointsInstantiated = false;

void Start()
{
mainCamera = Camera.main;
LoadSMPLData();
}

void LoadSMPLData()
{
smplData = JObject.Parse(smplDataJson.text);
var jointsValues = smplData["smpl_joints"].ToObject();
smplJoints = jointsValues.Select(j => new Vector3(j[0], j[1], j[2])).ToArray();
smplToMediaPipeMapping = smplData["smpl_to_mediapipe_mapping"].ToObject();
measurementPoints = smplData["measurement_points"]
.ToObject()
.ToDictionary(kvp => kvp.Key, kvp => new Vector3(-1.0f * kvp.Value[0], -1.0f * kvp.Value[1], -1.0f * kvp.Value[2]));
}

public static Vector3[] AlignPoints(Vector3[] mediaPipeKeypoints, Dictionary measurementPoints)
{
// Mapping the MediaPipe landmarks to measurement points
Dictionary mediaPipeToMeasurements = new Dictionary
{
{ 11, "LEFT_SHOULDER" },
{ 12, "RIGHT_SHOULDER" },
{ 15, "LEFT_WRIST" },
{ 16, "RIGHT_WRIST" },
{ 23, "LOW_LEFT_HIP" },
{ 24, "INSEAM_POINT" },  // Approximation for right hip
{ 25, "LEFT_THIGH" },    // Approximation for left knee
{ 27, "LEFT_ANKLE" },
{ 28, "RIGHT_ANKLE" },
{ 29, "LEFT_HEEL" },
{ 30, "RIGHT_HEEL" }
};

// Prepare lists for aligned points
List pointsA = new List();  // MediaPipe points
List pointsB = new List();  // Measurement points

// Align the measurement points to the corresponding MediaPipe landmarks
foreach (var map in mediaPipeToMeasurements)
{
int mediaPipeIndex = map.Key;
string measurementKey = map.Value;

// Make sure the measurement exists in the dictionary
if (measurementPoints.ContainsKey(measurementKey) && mediaPipeIndex < mediaPipeKeypoints.Length)
{
pointsA.Add(mediaPipeKeypoints[mediaPipeIndex]);        // MediaPipe point
pointsB.Add(measurementPoints[measurementKey]);  // Corresponding measurement point
}
}

// Convert the matched points to arrays
Vector3[] setA = pointsA.ToArray();
Vector3[] setB = pointsB.ToArray();

// Perform alignment using Procrustes or similar alignment method
return AlignPointsUsingProcrustes(setA, setB, mediaPipeKeypoints, measurementPoints);
}

public static Vector3[] AlignPointsUsingProcrustes(Vector3[] setA, Vector3[] setB, Vector3[] fullMediaPipeKeypoints, Dictionary  fullMeasurementPoints)
{
// Compute centroids for both sets
Vector3 centroidA = ComputeCentroid(setA);
Vector3 centroidB = ComputeCentroid(setB);

// Center the points by subtracting centroids
var centeredA = setA.Select(p => p - centroidA).ToArray();
var centeredB = setB.Select(p => p - centroidB).ToArray();

// Compute the rotation matrix using the centered points
var rotationMatrix = ComputeRotationMatrix(centeredA, centeredB);

// Apply the rotation to all measurement points (even those that were not mapped)
return fullMeasurementPoints.Values
.Select(p => ApplyRotation(p - centroidB, rotationMatrix) + centroidA)
.ToArray();
}

public void UpdateMediaPipeKeypoints(Vector3[] newKeypoints)
{
if (measurementPointsInstantiated)
return;

if (newKeypoints.Length != 33)
{
Debug.LogError("Expected 33 MediaPipe keypoints");
return;
}
mediaPipeKeypoints = newKeypoints;

Vector3[] alignedPoints = AlignPoints(mediaPipeKeypoints, measurementPoints);
measurementPointsInstantiated = true;

VisualizeMappedKeypoints(alignedPoints);
}

public static Vector3 ComputeCentroid(Vector3[] points)
{
Vector3 sum = new Vector3(0, 0, 0);
foreach (var point in points)
{
sum += point;
}
return sum / points.Length;
}

public static double[,] ComputeRotationMatrix(Vector3[] setA, Vector3[] setB)
{
// Compute the covariance matrix
double[,] covarianceMatrix = new double[3, 3];
for (int i = 0; i < setA.Length;  i++)
{
covarianceMatrix[0, 0] += setA[i].X * setB[i].X;
covarianceMatrix[0, 1] += setA[i].X * setB[i].Y;
covarianceMatrix[0, 2] += setA[i].X * setB[i].Z;
covarianceMatrix[1, 0] += setA[i].Y * setB[i].X;
covarianceMatrix[1, 1] += setA[i].Y * setB[i].Y;
covarianceMatrix[1, 2] += setA[i].Y * setB[i].Z;
covarianceMatrix[2, 0] += setA[i].Z * setB[i].X;
covarianceMatrix[2, 1] += setA[i].Z * setB[i].Y;
covarianceMatrix[2, 2] += setA[i].Z * setB[i].Z;
}

// Perform SVD on the covariance matrix
var svd = new Accord.Math.Decompositions.SingularValueDecomposition(covarianceMatrix);
var u = svd.LeftSingularVectors;
var vt = svd.RightSingularVectors;

// Compute the rotation matrix as U * Vt
var rotationMatrix = Accord.Math.Matrix.Dot(u, vt.Transpose());

return rotationMatrix;
}

public static Vector3 ApplyRotation(Vector3 point, double[,] rotationMatrix)
{
double X = rotationMatrix[0, 0] * point.X + rotationMatrix[0, 1] * point.Y + rotationMatrix[0, 2] * point.Z;
double Y = rotationMatrix[1, 0] * point.X + rotationMatrix[1, 1] * point.Y + rotationMatrix[1, 2] * point.Z;
double Z = rotationMatrix[2, 0] * point.X + rotationMatrix[2, 1] * point.Y + rotationMatrix[2, 2] * point.Z;
return new Vector3((float)X, (float)Y, (float)Z);
}

UnityEngine.Vector3 TransformPoint(UnityEngine.Matrix4x4 transformationMatrix, UnityEngine.Vector3 point, bool inverse = false)
{
return inverse
? transformationMatrix.inverse.MultiplyPoint3x4(point)
: transformationMatrix.MultiplyPoint3x4(point);
}

public void VisualizeMappedKeypoints(Vector3[] alignedPoints)
{
UnityMainThreadDispatcher.Enqueue(() =>
{
UnityEngine.Matrix4x4 transformationMatrix = UnityEngine.Matrix4x4.identity;

// Define MediaPipe keypoint names
string[] mediaPipeNames = new string[]
{
"Nose",
"Left Eye (Inner)",
"Left Eye",
"Left Eye (Outer)",
"Right Eye (Inner)",
"Right Eye",
"Right Eye (Outer)",
"Left Ear",
"Right Ear",
"Mouth (Left)",
"Mouth (Right)",
"Left Shoulder",
"Right Shoulder",
"Left Elbow",
"Right Elbow",
"Left Wrist",
"Right Wrist",
"Left Pinky",
"Right Pinky",
"Left Index",
"Right Index",
"Left Thumb",
"Right Thumb",
"Left Hip",
"Right Hip",
"Left Knee",
"Right Knee",
"Left Ankle",
"Right Ankle",
"Left Heel",
"Right Heel",
"Left Foot Index",
"Right Foot Index"
};

// Visualize MediaPipe keypoints
for (int i = 0; i < mediaPipeKeypoints.Length; i++)
{
if (mediaPipeKeypoints[i].Y < 0) continue;  // Skip invisible points
var point = mediaPipeKeypoints[i];
CreateSphereAtPoint(point, mediaSphere, mediaPipeNames[i]);
}
// Visualize aligned measurement points
foreach (var alignedPoint in alignedPoints)
{
CreateSphereAtPoint(alignedPoint, alignedSphere, "AlignedMeasurementPoint");
}
// Visualize measurement points
foreach (var measurement in measurementPoints)
{
CreateSphereAtPoint(measurement.Value, smplSphere, measurement.Key);
}
});
}

void CreateSphereAtPoint(Vector3 point, GameObject spherePrefab, string name)
{
var sphere = Instantiate(spherePrefab, new UnityEngine.Vector3(point.X, point.Y, point.Z), Quaternion.identity);
sphere.name = name;
}

}

Подробнее здесь: https://stackoverflow.com/questions/790 ... ualization