Совместное использование снимков экрана отключает многоточечное соединение через WebRTC на устройствах iOS ⇐ IOS

[Anonymous]

Я создаю прототип приложения, которое демонстрирует возможности iOS по захвату экрана устройства, подключению к другому устройству и последующей потоковой передаче этого видео (в высоком разрешении и с низкой задержкой) по локальной сети Wi-Fi. Выбранная библиотека, с помощью которой я это реализую, — WebRTC (открыта для других предложений).
Проблема:
Я загружаю приложение на 2 устройства. Я транслирую экран одного устройства, в журналах появляется сообщение о необходимости подключения к другому устройству, которое я установил для приема, но после этого они, кажется, разрывают соединение с сообщением «[GCKSession] Не в подключенном состоянии, поэтому отказываюсь от участия для участника [ xxxxxxxxx] на канале [x]" на 5-м канале.
Я включил полный код, который вы можете запустить в Xcode ниже, вместе с консолью с подключающегося устройства во время последней сборки и удалил соединение.
Я зашел в тупик.
WebRTCManager.swift
import Foundation
import WebRTC
import ReplayKit
import MultipeerConnectivity

class WebRTCManager: NSObject, ObservableObject {
private var peerConnection: RTCPeerConnection?
@Published var localVideoTrack: RTCVideoTrack?
@Published var remoteVideoTrack: RTCVideoTrack?
private var peerConnectionFactory: RTCPeerConnectionFactory?
private var videoSource: RTCVideoSource?
private var videoCapturer: RTCVideoCapturer?

private var peerID: MCPeerID
private var session: MCSession
private var advertiser: MCNearbyServiceAdvertiser?
private var browser: MCNearbyServiceBrowser?

@Published var connectedPeers: [MCPeerID] = []
@Published var localSDP: String = ""
@Published var localICECandidates: [String] = []

@Published var isBroadcasting: Bool = false
@Published var remoteTrackAdded: Bool = false

private var isConnected = false

override init() {
RTCInitializeSSL()
peerConnectionFactory = RTCPeerConnectionFactory()

peerID = MCPeerID(displayName: UIDevice.current.name)
session = MCSession(peer: peerID, securityIdentity: nil, encryptionPreference: .none)

super.init()

session.delegate = self
}

func startBroadcasting() {
isBroadcasting = true
advertiser = MCNearbyServiceAdvertiser(peer: peerID, discoveryInfo: nil, serviceType: "screen-share")
advertiser?.delegate = self
advertiser?.startAdvertisingPeer()

setupPeerConnection()
setupVideoSource()
startScreenCapture()
}

func startReceiving() {
browser = MCNearbyServiceBrowser(peer: peerID, serviceType: "screen-share")
browser?.delegate = self
browser?.startBrowsingForPeers()

setupPeerConnection()
}

private func setupPeerConnection() {
let configuration = RTCConfiguration()
configuration.iceServers = [RTCIceServer(urlStrings: ["stun:stun.l.google.com:19302"])]
configuration.sdpSemantics = .unifiedPlan
let constraints = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil)
peerConnection = peerConnectionFactory?.peerConnection(with: configuration, constraints: constraints, delegate: self)

// Ensure peer connection state is checked before proceeding
guard let peerConnection = peerConnection else {
print("Failed to create peer connection")
return
}

peerConnection.delegate = self
}

private func setupVideoSource() {
videoSource = peerConnectionFactory?.videoSource()

#if targetEnvironment(simulator)
videoCapturer = RTCFileVideoCapturer(delegate: videoSource!)
#else
videoCapturer = RTCCameraVideoCapturer(delegate: videoSource!)
#endif

localVideoTrack = peerConnectionFactory?.videoTrack(with: videoSource!, trackId: "video0")

if let localVideoTrack = localVideoTrack {
peerConnection?.add(localVideoTrack, streamIds: ["stream0"])
print("Local video track added to peer connection")
} else {
print("Failed to create local video track")
}
}

private func startScreenCapture() {
let recorder = RPScreenRecorder.shared()
recorder.startCapture { [weak self] (sampleBuffer, type, error) in
guard let self = self else { return }

if let error = error {
print("Error starting screen capture: \(error)")
return
}

if type == .video {
self.processSampleBuffer(sampleBuffer, with: type)
}
} completionHandler: { error in
if let error = error {
print("Error in screen capture completion: \(error)")
} else {
print("Screen capture started successfully")
}
}
}

private func processSampleBuffer(_ sampleBuffer: CMSampleBuffer, with sampleBufferType: RPSampleBufferType) {
switch sampleBufferType {
case .video:
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }

let timestamp = NSDate().timeIntervalSince1970 * 1000
let videoFrame = RTCVideoFrame(buffer: RTCCVPixelBuffer(pixelBuffer: pixelBuffer),
rotation: ._0,
timeStampNs: Int64(timestamp))

self.videoSource?.capturer(self.videoCapturer!, didCapture: videoFrame)

default:
break
}
}

func stopBroadcasting() {
isBroadcasting = false
advertiser?.stopAdvertisingPeer()
advertiser = nil
stopScreenCapture()
closePeerConnection()
}

func stopReceiving() {
browser?.stopBrowsingForPeers()
browser = nil
closePeerConnection()
}

private func stopScreenCapture() {
RPScreenRecorder.shared().stopCapture { [weak self] error in
if let error = error {
print("Error stopping screen capture: \(error)")
} else {
print("Screen capture stopped successfully")
DispatchQueue.main.async {
self?.localVideoTrack = nil
}
}
}
}

private func closePeerConnection() {
guard let peerConnection = peerConnection else { return }

if isConnected {
// Properly handle disconnection if connected
peerConnection.close()
}
self.peerConnection = nil
DispatchQueue.main.async {
self.remoteVideoTrack = nil
self.localSDP = ""
self.localICECandidates.removeAll()
}
print("Peer connection closed")
}

private func createOffer() {
print("Creating offer")
let constraints = RTCMediaConstraints(mandatoryConstraints: [
"OfferToReceiveVideo": "true",
"OfferToReceiveAudio": "false"
], optionalConstraints: nil)
peerConnection?.offer(for: constraints) { [weak self] sdp, error in
guard let self = self, let sdp = sdp else {
print("Failed to create offer: \(error?.localizedDescription ?? "unknown error")")
return
}
self.peerConnection?.setLocalDescription(sdp) { error in
if let error = error {
print("Error setting local description: \(error)")
} else {
print("Local description (offer) set successfully")
self.sendSDP(sdp)
}
}
}
}

private func createAnswer() {
print("Creating answer")
let constraints = RTCMediaConstraints(mandatoryConstraints: [
"OfferToReceiveVideo": "true",
"OfferToReceiveAudio": "false"
], optionalConstraints: nil)
peerConnection?.answer(for: constraints) { [weak self] sdp, error in
guard let self = self, let sdp = sdp else {
print("Failed to create answer: \(error?.localizedDescription ?? "unknown error")")
return
}
self.peerConnection?.setLocalDescription(sdp) { error in
if let error = error {
print("Error setting local description: \(error)")
} else {
print("Local description (answer) set successfully")
self.sendSDP(sdp)
}
}
}
}

private func setRemoteDescription(_ sdp: RTCSessionDescription) {
peerConnection?.setRemoteDescription(sdp) { error in
if let error = error {
print("Error setting remote description: \(error)")
} else {
print("Remote description set successfully")
}
}
}

private func addIceCandidate(_ candidate: RTCIceCandidate) {
guard let peerConnection = peerConnection, isConnected else {
print("Cannot add ICE candidate, peer connection is not connected")
return
}
peerConnection.add(candidate)
print("ICE candidate added")
}

private func sendSDP(_ sdp: RTCSessionDescription) {
let dict: [String: Any] = ["type": sdp.type.rawValue, "sdp": sdp.sdp]
if let data = try? JSONSerialization.data(withJSONObject: dict, options: []) {
do {
try session.send(data, toPeers: session.connectedPeers, with: .reliable)
print("SDP sent to peers")
} catch {
print("Failed to send SDP: \(error)")
}
}
}
}

extension WebRTCManager: MCSessionDelegate, MCNearbyServiceAdvertiserDelegate, MCNearbyServiceBrowserDelegate {

func session(_ session: MCSession, peer peerID: MCPeerID, didChange state: MCSessionState) {
DispatchQueue.main.async {
self.connectedPeers = session.connectedPeers
switch state {
case .connected:
print("Peer connected: \(peerID.displayName)")
self.browser?.stopBrowsingForPeers()
self.advertiser?.stopAdvertisingPeer()
self.isConnected = true // Set isConnected to true here
if self.isBroadcasting {
self.createOffer()
}
case .connecting:
print("Peer connecting: \(peerID.displayName)")
case .notConnected:
print("Peer not connected: \(peerID.displayName)")
self.isConnected = false
@unknown default:
print("Unknown state: \(peerID.displayName)")
}
}
}

func session(_ session: MCSession, didReceive data: Data, fromPeer peerID: MCPeerID) {
let dict = try? JSONSerialization.jsonObject(with: data, options: []) as? [String: Any]
if let typeInt = dict?["type"] as? Int, let sdp = dict?["sdp"] as? String,
let type = RTCSdpType(rawValue: typeInt) {
let rtcSdp = RTCSessionDescription(type: type, sdp: sdp)

self.peerConnection?.setRemoteDescription(rtcSdp) { [weak self] error in
if let error = error {
print("Error setting remote description: \(error)")
} else {
print("Remote description set successfully")
if type == .offer {
self?.createAnswer()
}
}
}
} else if let sdp = dict?["candidate"] as? String,
let sdpMid = dict?["sdpMid"] as? String,
let sdpMLineIndexString = dict?["sdpMLineIndex"] as? String,
let sdpMLineIndex = Int32(sdpMLineIndexString) {
let candidate = RTCIceCandidate(sdp: sdp, sdpMLineIndex: sdpMLineIndex, sdpMid: sdpMid)
self.peerConnection?.add(candidate)
}
}

func session(_ session: MCSession, didReceive stream: InputStream, withName streamName: String, fromPeer peerID: MCPeerID) {}

func session(_ session: MCSession, didStartReceivingResourceWithName resourceName: String, fromPeer peerID: MCPeerID, with progress: Progress) {}

func session(_ session: MCSession, didFinishReceivingResourceWithName resourceName: String, fromPeer peerID: MCPeerID, at localURL: URL?, withError error: Error?) {}

func advertiser(_ advertiser: MCNearbyServiceAdvertiser, didReceiveInvitationFromPeer peerID: MCPeerID, withContext context: Data?, invitationHandler: @escaping (Bool, MCSession?) -> Void) {
invitationHandler(true, session)
}

func browser(_ browser: MCNearbyServiceBrowser, foundPeer peerID: MCPeerID, withDiscoveryInfo info: [String : String]?) {
browser.invitePeer(peerID, to: session, withContext: nil, timeout: 30)
}

func browser(_ browser: MCNearbyServiceBrowser, lostPeer peerID: MCPeerID) {}
}

extension WebRTCManager: RTCPeerConnectionDelegate {
func peerConnection(_ peerConnection: RTCPeerConnection, didChange stateChanged: RTCSignalingState) {
print("Signaling state changed: \(stateChanged.rawValue)")
}

func peerConnection(_ peerConnection: RTCPeerConnection, didAdd stream: RTCMediaStream) {
print("Stream added with ID: \(stream.streamId)")
if let videoTrack = stream.videoTracks.first {
print("Video track added: \(videoTrack.trackId)")
DispatchQueue.main.async {
self.remoteVideoTrack = videoTrack
self.remoteTrackAdded = true
self.objectWillChange.send()
print("Remote video track set")
}
} else {
print("No video tracks in the stream")
}
}

func peerConnection(_ peerConnection: RTCPeerConnection, didRemove stream: RTCMediaStream) {
print("Stream removed")
}

func peerConnectionShouldNegotiate(_ peerConnection: RTCPeerConnection) {
print("Negotiation needed")
}

func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceConnectionState) {
print("ICE connection state changed: \(newState.rawValue)")
switch newState {
case .checking, .connected, .completed:
print("ICE connected")
self.isConnected = true
case .failed, .disconnected, .closed:
print("ICE connection failed or closed")
self.isConnected = false
// Handle reconnection or cleanup if necessary
case .new:
print("New ICE connection")
case .count:
print("ICE count")
@unknown default:
print("Unknown ICE state")
}
}

func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceGatheringState) {
print("ICE gathering state changed: \(newState.rawValue)")
}

func peerConnection(_ peerConnection: RTCPeerConnection, didGenerate candidate: RTCIceCandidate) {
print("ICE candidate generated: \(candidate.sdp)")
DispatchQueue.main.async {
self.localICECandidates.append(candidate.sdp)
}

// Always send ICE candidates
let dict: [String: Any] = ["candidate": candidate.sdp, "sdpMid": candidate.sdpMid ?? "", "sdpMLineIndex": candidate.sdpMLineIndex]
if let data = try? JSONSerialization.data(withJSONObject: dict, options: []) {
do {
try session.send(data, toPeers: session.connectedPeers, with: .reliable)
print("ICE candidate sent to peers")
} catch {
print("Failed to send ICE candidate: \(error)")
}
}
}

func peerConnection(_ peerConnection: RTCPeerConnection, didRemove candidates: [RTCIceCandidate]) {
print("Removed ICE candidates")
}

func peerConnection(_ peerConnection: RTCPeerConnection, didOpen dataChannel: RTCDataChannel) {
print("Data channel opened")
}
}

extension Notification.Name {
static let remoteVideoTrackAdded = Notification.Name("remoteVideoTrackAdded")
}

RTCVideoView.swift
import SwiftUI
import WebRTC

struct RTCVideoView: UIViewRepresentable {
@ObservedObject var webRTCManager: WebRTCManager
var isLocal: Bool

func makeUIView(context: Context) -> RTCMTLVideoView {
let videoView = RTCMTLVideoView(frame: .zero)
videoView.videoContentMode = .scaleAspectFit
updateVideoTrack(videoView)
return videoView
}

func updateUIView(_ uiView: RTCMTLVideoView, context: Context) {
updateVideoTrack(uiView)
}

private func updateVideoTrack(_ uiView: RTCMTLVideoView) {
if isLocal {
if let localVideoTrack = webRTCManager.localVideoTrack {
localVideoTrack.add(uiView)
print("Local video track added to view")
} else {
print("Local video track is nil")
}
} else {
if let remoteVideoTrack = webRTCManager.remoteVideoTrack {
remoteVideoTrack.add(uiView)
print("Remote video track added to view")
} else {
print("Remote video track is nil")
}
}
}

func makeCoordinator() -> Coordinator {
Coordinator(self)
}

class Coordinator: NSObject {
var parent: RTCVideoView

init(_ parent: RTCVideoView) {
self.parent = parent
super.init()
NotificationCenter.default.addObserver(self, selector: #selector(remoteVideoTrackAdded), name: .remoteVideoTrackAdded, object: nil)
}

@objc func remoteVideoTrackAdded() {
DispatchQueue.main.async {
self.parent.webRTCManager.objectWillChange.send()
}
}
}
}


ContentView.swift
import SwiftUI
import WebRTC
import MultipeerConnectivity
import ReplayKit

struct ContentView: View {
@StateObject private var webRTCManager = WebRTCManager()
@State private var isBroadcasting = false
@State private var isReceiving = false

var body: some View {
VStack {
if isBroadcasting {
Text("Broadcasting")
.font(.headline)

// Local video preview
RTCVideoView(webRTCManager: webRTCManager, isLocal: true)
.frame(height: 200)
.background(Color.gray.opacity(0.3)) // Add a semi-transparent gray background
.cornerRadius(10)
Button("Stop Broadcasting") {
webRTCManager.stopBroadcasting()
isBroadcasting = false
}
.padding()
.background(Color.red)
.foregroundColor(.white)
.cornerRadius(10)

// ... (existing code for SDP and ICE candidates)
} else if isReceiving {
Text("Receiving")
.font(.headline)
RTCVideoView(webRTCManager: webRTCManager, isLocal: false)
.frame(height: 300)
.background(Color.gray.opacity(0.3)) // Add a semi-transparent gray background
.cornerRadius(10)
Button("Stop Receiving") {
webRTCManager.stopReceiving()
isReceiving = false
}
.padding()
.background(Color.red)
.foregroundColor(.white)
.cornerRadius(10)
} else {
Button("Start Broadcasting") {
webRTCManager.startBroadcasting()
isBroadcasting = true
}
.padding()
.background(Color.blue)
.foregroundColor(.white)
.cornerRadius(10)

Button("Start Receiving") {
webRTCManager.startReceiving()
isReceiving = true
}
.padding()
.background(Color.green)
.foregroundColor(.white)
.cornerRadius(10)
}

Text("Connected Peers: \(webRTCManager.connectedPeers.count)")
.font(.headline)
.padding()

if !webRTCManager.connectedPeers.isEmpty {
Text("Connected to:")
ForEach(webRTCManager.connectedPeers, id: \.self) { peer in
Text(peer.displayName)
}
}
}
.padding()
}
}

struct ContentView_Previews: PreviewProvider {
static var previews: some View {
ContentView()
}
}


Консоль подключения устройства:
2024-07-08 22:04:22.574897+0100 ScreenShare[2745:914162] Metal API Validation Enabled
Remote video track is nil
Remote video track is nil
2024-07-08 22:04:56.058516+0100 ScreenShare[2745:914299] [Client] Updating selectors after delegate removal failed with: Error Domain=NSCocoaErrorDomain Code=4099 "The connection to service named com.apple.commcenter.coretelephony.xpc was invalidated from this process." UserInfo={NSDebugDescription=The connection to service named com.apple.commcenter.coretelephony.xpc was invalidated from this process.}
2024-07-08 22:04:56.058657+0100 ScreenShare[2745:914299] [Client] Updating selectors after delegate addition failed with: Error Domain=NSCocoaErrorDomain Code=4099 "The connection to service named com.apple.commcenter.coretelephony.xpc was invalidated from this process." UserInfo={NSDebugDescription=The connection to service named com.apple.commcenter.coretelephony.xpc was invalidated from this process.}
Peer connecting: iPhone
Remote video track is nil
Peer connected: iPhone
Remote video track is nil
Signaling state changed: 3
Stream added with ID: stream0
Video track added: video0
Remote description set successfully
Creating answer
Remote video track set
Remote video track added to view
Signaling state changed: 0
Local description (answer) set successfully
SDP sent to peers
ICE gathering state changed: 1
ICE candidate generated: candidate:617392483 1 udp 2122260223 192.168.1.112 63716 typ host generation 0 ufrag umCW network-id 1 network-cost 10
ICE candidate sent to peers
ICE candidate generated: candidate:3503244297 1 udp 2122194687 169.254.104.95 64683 typ host generation 0 ufrag umCW network-id 2 network-cost 10
ICE candidate sent to peers
Remote video track added to view
Remote video track added to view
ICE candidate generated: candidate:1783584147 1 tcp 1518280447 192.168.1.112 51096 typ host tcptype passive generation 0 ufrag umCW network-id 1 network-cost 10
ICE candidate sent to peers
ICE candidate generated: candidate:2655828217 1 tcp 1518214911 169.254.104.95 51097 typ host tcptype passive generation 0 ufrag umCW network-id 2 network-cost 10
ICE candidate sent to peers
Remote video track added to view
ICE candidate generated: candidate:2776936407 1 udp 1686052607 2.28.217.67 63716 typ srflx raddr 192.168.1.112 rport 63716 generation 0 ufrag umCW network-id 1 network-cost 10
ICE candidate sent to peers
Remote video track added to view
Remote video track added to view
2024-07-08 22:05:06.151870+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [0].
2024-07-08 22:05:06.155864+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [1].
2024-07-08 22:05:06.158066+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [2].
2024-07-08 22:05:06.159428+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [3].
2024-07-08 22:05:06.160762+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [4].
2024-07-08 22:05:06.161831+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [5].
2024-07-08 22:05:06.162682+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [6].
ICE gathering state changed: 2

Я поигрался с настройками проекта и добавил все, что, по моему мнению, могло вызвать проблему — из-за его отсутствия в Info.plist — и добавил это в:


NSBonjourServices

_screen-share._tcp
_screen-share._udp

UIApplicationSceneManifest

UIApplicationSupportsMultipleScenes

UISceneConfigurations


UIBackgroundModes

audio
fetch
nearby-interaction
processing
voip





Подробнее здесь: https://stackoverflow.com/questions/787 ... across-ios