Координаты обнаружения лица iOS Vision не работают с Dockkit, в то время как правильный на предварительном слое ⇐ IOS

[Anonymous]

Я пытаюсь реализовать обнаружение лица, используя структуру Vision со следующей настройкой:

[*] Камера обратной в режиме портрета
[*] vnimagerequesthandler с .leftmirrored prevition
[*] avcupurevurevurevieh /> Dockkit для отслеживания лица < /li>
< /ul>
Текущая реализация: < /p>

Код: Выделить всё

class TestViewController: UIViewController {
private let captureSession = AVCaptureSession()
private let videoDataOutput = AVCaptureVideoDataOutput()

private lazy var previewLayer = AVCaptureVideoPreviewLayer(session: self.captureSession)

private var faceLayers: [CAShapeLayer] = []
private var dockAccessory: DockAccessory?
private var captureDevice: AVCaptureDevice?

override func viewDidLoad() {
super.viewDidLoad()

Task {
try await DockAccessoryManager.shared.setSystemTrackingEnabled(false)

for await accessory in try DockAccessoryManager.shared.accessoryStateChanges {
print(accessory.state)

DispatchQueue.main.async {
//self.connectedToDock = accessory.state == DockAccessory.State.docked
self.dockAccessory = accessory.accessory
}
}
}

setupCamera()
captureSession.startRunning()

func setupCamera() {
self.captureSession.sessionPreset = .hd1280x720
let deviceDiscoverySession = AVCaptureDevice.DiscoverySession(
deviceTypes: [
.builtInDualCamera,
.builtInTripleCamera
],
mediaType: .video,
position: .back
)
if let device = deviceDiscoverySession.devices.first {
if let deviceInput = try? AVCaptureDeviceInput(device: device) {
if captureSession.canAddInput(deviceInput) {
captureSession.addInput(deviceInput)

setupPreview()
}
}

self.captureDevice = device
}

func setupPreview() {
self.previewLayer.videoGravity = .resizeAspectFill
self.view.layer.addSublayer(self.previewLayer)
self.previewLayer.frame = self.view.frame

self.videoDataOutput.videoSettings = [
(kCVPixelBufferPixelFormatTypeKey as NSString) : NSNumber(value: kCVPixelFormatType_32BGRA)
] as [String : Any]

self.videoDataOutput.setSampleBufferDelegate(
self,
queue: DispatchQueue(label: "camera queue")
)
self.captureSession.addOutput(self.videoDataOutput)

let videoConnection = self.videoDataOutput.connection(with: .video)
videoConnection?.videoOrientation = .portrait
}
}
}

private var frameCounter = 0
private var lastTimestamp = Date()
}

extension TestViewController: AVCaptureVideoDataOutputSampleBufferDelegate {
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
return
}

let faceDetectionRequest = VNDetectFaceLandmarksRequest(completionHandler: { (request: VNRequest, error: Error?) in
DispatchQueue.main.async {
self.faceLayers.forEach({ drawing in drawing.removeFromSuperlayer() })

if let observations = request.results as? [VNFaceObservation] {
self.handleFaceDetectionObservations(
observations: observations,
imageBuffer,
sampleBuffer
)
}
}
})

let imageRequestHandler = VNImageRequestHandler(cvPixelBuffer: imageBuffer, orientation: .leftMirrored, options:  [:])

do {
try imageRequestHandler.perform([faceDetectionRequest])
} catch {
print(error.localizedDescription)
}
}

private func handleFaceDetectionObservations(
observations: [VNFaceObservation],
_ pixelBuffer: CVPixelBuffer,
_ sampleBuffer: CMSampleBuffer
) {
for observation in observations {
var faceRectConverted = self.previewLayer.layerRectConverted(fromMetadataOutputRect: observation.boundingBox)
let faceRectanglePath = CGPath(rect: faceRectConverted, transform: nil)

let faceLayer = CAShapeLayer()
faceLayer.path = faceRectanglePath
faceLayer.fillColor = UIColor.clear.cgColor
faceLayer.strokeColor = UIColor.yellow.cgColor

self.faceLayers.append(faceLayer)
self.view.layer.addSublayer(faceLayer)

if
let captureDevice = captureDevice,
let dockAccessory = dockAccessory
{
Task {
do {
try await trackWithDockKit(
observation.boundingBox,
dockAccessory,
pixelBuffer,
sampleBuffer
)
} catch {
print(error)
}
}
}
}
}

func trackWithDockKit(_ boundingBox: CGRect, _ dockAccessory: DockAccessory, _ pixelBuffer: CVPixelBuffer, _ cmSampelBuffer: CMSampleBuffer) async throws {
guard
let device = captureDevice
else {
fatalError("Kamera nicht verfügbar")
}

let size = CGSize(width: CVPixelBufferGetWidth(pixelBuffer), height: CVPixelBufferGetHeight(pixelBuffer))

let cameraInfo = DockAccessory.CameraInformation(
captureDevice: device.deviceType,
cameraPosition: device.position,
orientation: .corrected,
cameraIntrinsics: nil,
referenceDimensions: size
)

let observation = DockAccessory.Observation(
identifier: 0,
type: .object,
rect: boundingBox
)
let observations = [observation]

try await dockAccessory.track(observations, cameraInformation: cameraInfo)
}
}

Что работает:

Обнаружение лица работает правильно
Ограничивающие ящики для рисования на предварительном слое отлично работают, что работает. />
Отслеживание лиц Dockkit не совпадает с фактическими лицами < /li>
< /ul>
Примеры координат: < /strong>
Vision дает мне нормализованные координаты, такие как: (0,220, 0,051, 0,347, 0,618) < /code> < /p>

Что мне не хватает с точки зрения преобразования координат? Ограничительные ящики правильно отображаются на экране, используя Layerctconverted , но Dockkit, похоже, ожидает координат в другом формате. Как я могу правильно преобразовать координаты зрения для Dockkit?

Подробнее здесь: https://stackoverflow.com/questions/791 ... correct-on