VR Meta Quest Streaming Vision к другому приложению в реальном времени. ⇐ C#

[Anonymous]

Всем добрый день. Я хочу создать 2 приложения, одно из которых будет запускать на VR Metaquest 2, а другой, например, на ПК. Я хочу сделать вывод гарнитуры виртуальной машины того, что пользователь видит в очках, может быть передана во второе приложение, которое работает на ПК. Я хочу сделать все соединение через Agora в версии Unity 6000.0.57f1 LTS с последней Agora SDK Video video v.4.5.1 для Unity. После последних тестов Unity не показывает никаких ошибок, но я все еще ничего не вижу в сыром изображении (оно чисто белое), где я отправляю его во втором приложении. Я работал над этим в течение 4 дней, и я не могу придумать ничего, где может быть проблема. В VR One у меня есть только установленная буровая установка, в которой у меня есть дубликат StreamCamera, которая превращает представление в rendertexture, и все это должно обрабатывать сценарий VrstreamManager, который находится на Streammanager Game Object. Во второй сцене есть только холст с Rawimage, где GameObject ReceiverManager имеет данный сценарий StreamReceiver. Агора отвечает за подключение видеопотока, как я упоминал. Здесь я прикрепляю отпечатки экранов и оба сценария. < /P>
using UnityEngine;
using UnityEngine.Rendering;
using Agora.Rtc;
using System;

public class VRStreamManager : MonoBehaviour
{
[Header("Agora Settings")]
[SerializeField] private string appID = "YOUR_AGORA_APP_ID";
[SerializeField] private string channelName = "vr-stream-channel";
[SerializeField] private string tempToken = "PASTE_YOUR_TOKEN_HERE";

[Header("Video Source")]
[SerializeField] private Camera streamingCamera;
[SerializeField] private RenderTexture vrRenderTexture;

private IRtcEngine rtcEngine;
private byte[] _buffer;
private bool _isEngineReady = false;
private bool _isQuitting = false;

#region Unity Lifecycle
void Start()
{
if (streamingCamera == null || vrRenderTexture == null)
{
Debug.LogError("Streaming Camera or VR RenderTexture is not assigned.");
return;
}
InitializeAgora();
JoinChannel();
}

private void OnEnable()
{
RenderPipelineManager.endCameraRendering += OnEndCameraRendering;
}

private void OnDisable()
{
RenderPipelineManager.endCameraRendering -= OnEndCameraRendering;
}

void OnApplicationQuit()
{
_isQuitting = true;
if (rtcEngine != null)
{
rtcEngine.LeaveChannel();
rtcEngine.Dispose();
rtcEngine = null;
}
}
#endregion

void OnEndCameraRendering(ScriptableRenderContext context, Camera camera)
{
if (camera != streamingCamera || !_isEngineReady || _isQuitting) return;
AsyncGPUReadback.Request(vrRenderTexture, 0, TextureFormat.RGBA32, OnCompleteReadback);
}

void OnCompleteReadback(AsyncGPUReadbackRequest request)
{
if (_isQuitting || rtcEngine == null) return;
if (request.hasError) { return; }

var rawTextureData = request.GetData();
if (_buffer == null || _buffer.Length != rawTextureData.Length)
{
_buffer = new byte[rawTextureData.Length];
}
rawTextureData.CopyTo(_buffer);

var externalVideoFrame = new ExternalVideoFrame
{
type = VIDEO_BUFFER_TYPE.VIDEO_BUFFER_RAW_DATA,
format = VIDEO_PIXEL_FORMAT.VIDEO_PIXEL_RGBA,
buffer = _buffer,
stride = vrRenderTexture.width,
height = vrRenderTexture.height,
timestamp = (long)(Time.realtimeSinceStartup * 1000)
};

rtcEngine.PushVideoFrame(externalVideoFrame);
}

#region Agora Setup
void InitializeAgora()
{
if (string.IsNullOrEmpty(appID) || appID == "YOUR_AGORA_APP_ID") { Debug.LogError("Agora App ID is not set."); return; }
rtcEngine = RtcEngine.CreateAgoraRtcEngine();
var context = new RtcEngineContext { appId = appID, channelProfile = CHANNEL_PROFILE_TYPE.CHANNEL_PROFILE_LIVE_BROADCASTING, audioScenario = AUDIO_SCENARIO_TYPE.AUDIO_SCENARIO_DEFAULT, areaCode = AREA_CODE.AREA_CODE_GLOB };
rtcEngine.Initialize(context);
rtcEngine.InitEventHandler(new UserRtcEventHandler(this));
rtcEngine.SetExternalVideoSource(true, false, EXTERNAL_VIDEO_SOURCE_TYPE.VIDEO_FRAME, new SenderOptions());
rtcEngine.EnableVideo();
}

void JoinChannel()
{
if (rtcEngine == null) return;
var options = new ChannelMediaOptions();

// --- FINAL, BULLETPROOF AUDIO CONTROL ---
options.publishMicrophoneTrack.SetValue(false); // Forcefully do not publish the mic

options.publishCameraTrack.SetValue(false);
options.publishCustomVideoTrack.SetValue(true);
options.clientRoleType.SetValue(CLIENT_ROLE_TYPE.CLIENT_ROLE_BROADCASTER);
rtcEngine.JoinChannel(tempToken, channelName, 0, options);
}

public void OnJoinChannelSuccess()
{
_isEngineReady = true;
Debug.Log($"VR SENDER: Successfully joined channel. Ready to send video.");
}

private class UserRtcEventHandler : IRtcEngineEventHandler
{
private readonly VRStreamManager _streamManager;
public UserRtcEventHandler(VRStreamManager manager) { _streamManager = manager; }
public override void OnJoinChannelSuccess(RtcConnection c, int e) { _streamManager.OnJoinChannelSuccess(); }
public override void OnError(int err, string msg) { Debug.LogError($"VR SENDER ERROR: Code: {err}, Message: {msg}"); }
}
#endregion
}
< /code>
using UnityEngine;
using UnityEngine.UI;
using Agora.Rtc;
using System.Collections.Generic;
using System;

public class StreamReceiver : MonoBehaviour
{
[Header("Agora Settings")]
[Tooltip("Get your App ID from the Agora console.")]
[SerializeField] private string appID = "YOUR_AGORA_APP_ID";
[Tooltip("The channel name must be identical on the sender and receiver.")]
[SerializeField] private string channelName = "vr-stream-channel";
[Tooltip("Paste the temporary token generated from the Agora Console here.")]
[SerializeField] private string tempToken = "PASTE_YOUR_TOKEN_HERE";

[Header("UI")]
[Tooltip("The RawImage component that will display the remote video.")]
[SerializeField] private RawImage remoteVideoSurface;

private IRtcEngine rtcEngine;
private uint remoteUid = 0;
private readonly Queue _mainThreadActions = new Queue();

void Start()
{
if (remoteVideoSurface == null)
{
Debug.LogError("Remote Video Surface (RawImage) is not assigned. Please assign it in the Inspector.");
return;
}
remoteVideoSurface.gameObject.SetActive(true);
remoteVideoSurface.color = Color.black;
remoteVideoSurface.texture = null; // Ensure texture is null at start
Debug.Log("StreamReceiver started. Initializing Agora...");
InitializeAgora();
JoinChannel();
}

void Update()
{
lock (_mainThreadActions)
{
while (_mainThreadActions.Count > 0)
{
_mainThreadActions.Dequeue().Invoke();
}
}
}

void InitializeAgora()
{
if (string.IsNullOrEmpty(appID) || appID == "YOUR_AGORA_APP_ID")
{
Debug.LogError("Agora App ID is not set.");
return;
}
rtcEngine = RtcEngine.CreateAgoraRtcEngine();
var context = new RtcEngineContext { appId = appID, channelProfile = CHANNEL_PROFILE_TYPE.CHANNEL_PROFILE_LIVE_BROADCASTING, audioScenario = AUDIO_SCENARIO_TYPE.AUDIO_SCENARIO_DEFAULT, areaCode = AREA_CODE.AREA_CODE_GLOB };
rtcEngine.Initialize(context);
rtcEngine.InitEventHandler(new UserRtcEventHandler(this));
rtcEngine.EnableVideo();
}

void JoinChannel()
{
if (rtcEngine == null) return;
rtcEngine.SetClientRole(CLIENT_ROLE_TYPE.CLIENT_ROLE_AUDIENCE);
var options = new ChannelMediaOptions();
options.autoSubscribeAudio.SetValue(true);
options.autoSubscribeVideo.SetValue(true);
options.clientRoleType.SetValue(CLIENT_ROLE_TYPE.CLIENT_ROLE_AUDIENCE);
rtcEngine.JoinChannel(tempToken, channelName, 0, options);
}

private void OnUserJoined(uint uid)
{
Debug.Log($"PC RECEIVER: Remote user joined with UID: {uid}.");
remoteUid = uid;

lock (_mainThreadActions)
{
_mainThreadActions.Enqueue(() =>
{
Debug.Log("Main thread: Preparing to create VideoSurface.");
if (remoteVideoSurface == null)
{
Debug.LogError("Main thread: RawImage is null. Cannot create VideoSurface.");
return;
}

var oldSurface = remoteVideoSurface.GetComponent();
if (oldSurface != null)
{
Debug.LogWarning("Main thread: Found and destroyed an old VideoSurface component.");
Destroy(oldSurface);
}

Debug.Log($"Main thread: Adding VideoSurface component for UID {uid}.");
var videoSurface = remoteVideoSurface.gameObject.AddComponent();
videoSurface.SetForUser(uid, channelName, VIDEO_SOURCE_TYPE.VIDEO_SOURCE_CUSTOM);
videoSurface.SetEnable(true);

remoteVideoSurface.color = Color.white;
Debug.Log("Main thread: VideoSurface created and enabled. RawImage is now white. You should see video.");
});
}
}

private void OnUserOffline(uint uid)
{
if (uid != remoteUid) return;
Debug.Log($"PC RECEIVER: Remote user left with UID: {uid}.");
lock (_mainThreadActions)
{
_mainThreadActions.Enqueue(() =>
{
if (remoteVideoSurface == null) return;
var videoSurface = remoteVideoSurface.GetComponent();
if (videoSurface != null)
{
Debug.Log("Main thread: Destroying VideoSurface for offline user.");
Destroy(videoSurface);
}
remoteVideoSurface.color = Color.black;
});
}
remoteUid = 0;
}

void OnApplicationQuit()
{
if (rtcEngine != null) { rtcEngine.LeaveChannel(); rtcEngine.Dispose(); rtcEngine = null; }
}

private class UserRtcEventHandler : IRtcEngineEventHandler
{
private readonly StreamReceiver _receiver;
public UserRtcEventHandler(StreamReceiver receiver) { _receiver = receiver; }
public override void OnJoinChannelSuccess(RtcConnection c, int e) { Debug.Log($"PC RECEIVER: Successfully joined channel '{c.channelId}'"); }
public override void OnUserJoined(RtcConnection c, uint uid, int e) { _receiver.OnUserJoined(uid); }
public override void OnUserOffline(RtcConnection c, uint uid, USER_OFFLINE_REASON_TYPE r) { _receiver.OnUserOffline(uid); }
public override void OnError(int err, string msg) { Debug.LogError($"PC RECEIVER ERROR: Code: {err}, Message: {msg}"); }
}
}
< /code>
Print screens :
PCScene VRScene

Подробнее здесь: https://stackoverflow.com/questions/797 ... ay-problem