Как реализовать кодирование видео с низкой задержкой и частотой 60 кадров в секунду в WebRTC на C# с помощью WebRTC? ⇐ C#

[Anonymous]

Недавно я пытался реализовать демонстрацию экрана с низкой задержкой со скоростью 60 кадров в секунду на C# с помощью библиотеки Sipsorcery. До сих пор я изо всех сил старался увеличить количество кадров, но получал только около 20 кадров в секунду.
Я пытался понять, дело в кодировке или в скриншоте. Я использовал профилировщик использования ЦП Visual Studio и увидел, что в основном это кодировщик. Тем не менее, я переделал функцию создания скриншотов и использовал SharpDX; Я заметил небольшое увеличение производительности, но не около 60 кадров в секунду. Это наводит меня на мысль, что проблема в кодировщике. Возможно, он мог бы использовать FFmpeg или какую-либо кодировку графического процессора. Заранее извините, если я указал неверную информацию, поскольку я относительно новичок в кодировании и декодировании видео.
Вот что я сделал на данный момент:

Код: Выделить всё

//VideoTestPatternSource.cs
using System;
using System.Collections.Generic;
using System.Drawing.Imaging;
using System.Drawing;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using Microsoft.Extensions.Logging;
using SIPSorceryMedia.Abstractions;
using SharpDX;
using SharpDX.Direct3D11;
using SharpDX.DXGI;
using Device = SharpDX.Direct3D11.Device;
using MapFlags = SharpDX.Direct3D11.MapFlags;
using System.Runtime.InteropServices;

namespace SIPSorcery.Media
{
public class VideoSource : IVideoSource, IDisposable
{
private const int VIDEO_SAMPLING_RATE = 90000;
private const int MAXIMUM_FRAMES_PER_SECOND = 20;
private const int DEFAULT_FRAMES_PER_SECOND = MAXIMUM_FRAMES_PER_SECOND; // Changed from 30 to 60
private const int MINIMUM_FRAMES_PER_SECOND = MAXIMUM_FRAMES_PER_SECOND - 5;
private const int TIMER_DISPOSE_WAIT_MILLISECONDS = 1000;
private const int VP8_SUGGESTED_FORMAT_ID = 96;
private const int H264_SUGGESTED_FORMAT_ID = 100;

public static readonly List SupportedFormats = new List
{
new VideoFormat(VideoCodecsEnum.VP8, VP8_SUGGESTED_FORMAT_ID, VIDEO_SAMPLING_RATE),
new VideoFormat(VideoCodecsEnum.H264, H264_SUGGESTED_FORMAT_ID, VIDEO_SAMPLING_RATE, "packetization-mode=1")
};

private int _frameSpacing;
private System.Threading.Timer _sendTestPatternTimer;
private bool _isStarted;
private bool _isPaused;
private bool _isClosed;
private bool _isMaxFrameRate;
private int _frameCount;
private SIPSorceryMedia.Abstractions.IVideoEncoder _videoEncoder;
private MediaFormatManager  _formatManager;

public event RawVideoSampleDelegate OnVideoSourceRawSample;

#pragma warning disable CS0067
public event RawVideoSampleFasterDelegate OnVideoSourceRawSampleFaster;
#pragma warning restore CS0067

public event EncodedSampleDelegate OnVideoSourceEncodedSample;

public event SourceErrorDelegate OnVideoSourceError;

private readonly int _screenWidth;
private readonly int _screenHeight;

private Factory1 _factory;
private Adapter1 _adapter;
private Device _device;
private Output1 _output1;
private OutputDuplication _duplicatedOutput;
private Texture2D _screenTexture;

public VideoSource(int width, int height, SIPSorceryMedia.Abstractions.IVideoEncoder encoder = null)
{
_screenWidth = width;
_screenHeight = height;
if (encoder != null)
{
_videoEncoder = encoder;
_formatManager = new MediaFormatManager(SupportedFormats);
}

try
{
InitializeDirectX();
_sendTestPatternTimer = new System.Threading.Timer(GeneratePattern, null, Timeout.Infinite, Timeout.Infinite);
_frameSpacing = 1000 / DEFAULT_FRAMES_PER_SECOND;
}
catch (Exception ex)
{
MessageBox.Show($"Failed to initialize: {ex.Message}");
}
}

private void InitializeDirectX()
{
_factory = new Factory1();
_adapter = _factory.GetAdapter1(0);
_device = new Device(_adapter);
var output = _adapter.GetOutput(0);
_output1 = output.QueryInterface();

var textureDesc = new Texture2DDescription
{
CpuAccessFlags = CpuAccessFlags.Read,
BindFlags = BindFlags.None,
Format = Format.B8G8R8A8_UNorm,
Width = _screenWidth,
Height = _screenHeight,
OptionFlags = ResourceOptionFlags.None,
MipLevels = 1,
ArraySize = 1,
SampleDescription = { Count = 1, Quality = 0 },
Usage = ResourceUsage.Staging
};

_screenTexture = new Texture2D(_device, textureDesc);

_duplicatedOutput = _output1.DuplicateOutput(_device);
}

public void RestrictFormats(Func filter) => _formatManager.RestrictFormats(filter);
public List GetVideoSourceFormats() => _formatManager.GetSourceFormats();
public void SetVideoSourceFormat(VideoFormat videoFormat) => _formatManager.SetSelectedFormat(videoFormat);
public List GetVideoSinkFormats() => _formatManager.GetSourceFormats();
public void SetVideoSinkFormat(VideoFormat videoFormat) => _formatManager.SetSelectedFormat(videoFormat);

public void ForceKeyFrame() => _videoEncoder?.ForceKeyFrame();
public bool HasEncodedVideoSubscribers() => OnVideoSourceEncodedSample != null;

public void ExternalVideoSourceRawSample(uint durationMilliseconds, int width, int height, byte[] sample, VideoPixelFormatsEnum pixelFormat) =>
throw new NotImplementedException("The test pattern video source does not offer any encoding services for external sources.");

public void ExternalVideoSourceRawSampleFaster(uint durationMilliseconds, RawImage rawImage) =>
throw new NotImplementedException("The test pattern video source does not offer any encoding services for external sources.");

public Task InitialiseVideoSourceDevice() =>
throw new NotImplementedException("The test pattern video source does not use a device.");
public bool IsVideoSourcePaused() => _isPaused;

public void SetFrameRate(int framesPerSecond)
{
if (framesPerSecond < MINIMUM_FRAMES_PER_SECOND || framesPerSecond >  MAXIMUM_FRAMES_PER_SECOND)
{
MessageBox.Show($"Frames per second not in the allowed range of {MINIMUM_FRAMES_PER_SECOND} to {MAXIMUM_FRAMES_PER_SECOND}, ignoring.");
}
else
{
_frameSpacing = 1000 / framesPerSecond;

if (_isStarted)
{
_sendTestPatternTimer?.Change(0, _frameSpacing);
}
}
}

public Task PauseVideo()
{
_isPaused = true;
_sendTestPatternTimer?.Change(Timeout.Infinite, Timeout.Infinite);
return Task.CompletedTask;
}

public Task ResumeVideo()
{
_isPaused = false;
_sendTestPatternTimer?.Change(0, _frameSpacing);
return Task.CompletedTask;
}

public Task StartVideo()
{
if (!_isStarted)
{
_isStarted = true;
if (_isMaxFrameRate)
{
GenerateMaxFrames();
}
else
{
try
{
_sendTestPatternTimer?.Change(0, _frameSpacing);
}
catch (Exception ex)
{
MessageBox.Show("The following error occured: " + ex);
}
}
}
return Task.CompletedTask;
}

public Task CloseVideo()
{
if (!_isClosed)
{
_isClosed = true;

ManualResetEventSlim mre = new ManualResetEventSlim();
_sendTestPatternTimer?.Dispose(mre.WaitHandle);
return Task.Run(() => mre.Wait(TIMER_DISPOSE_WAIT_MILLISECONDS));
}
return Task.CompletedTask;
}

private void GenerateMaxFrames()
{
DateTime lastGenerateTime = DateTime.Now;

while (!_isClosed && _isMaxFrameRate)
{
_frameSpacing = Convert.ToInt32(DateTime.Now.Subtract(lastGenerateTime).TotalMilliseconds);
GeneratePattern(null);
lastGenerateTime = DateTime.Now;
}
}

private void GeneratePattern(object state)
{
lock (_sendTestPatternTimer)
{
if (!_isClosed && (OnVideoSourceRawSample != null || OnVideoSourceEncodedSample != null))
{
_frameCount++;

var buffer = Snapshot(_screenWidth, _screenHeight);

if (OnVideoSourceRawSample != null)
{
OnVideoSourceRawSample?.Invoke((uint)_frameSpacing, _screenWidth, _screenHeight, buffer, VideoPixelFormatsEnum.Bgra);
}

if (_videoEncoder != null && OnVideoSourceEncodedSample != null && !_formatManager.SelectedFormat.IsEmpty())
{
var encodedBuffer = _videoEncoder.EncodeVideo(_screenWidth, _screenHeight, buffer, VideoPixelFormatsEnum.Bgra, _formatManager.SelectedFormat.Codec);

if (encodedBuffer != null)
{
uint fps = (_frameSpacing >  0) ? 1000 / (uint)_frameSpacing : MAXIMUM_FRAMES_PER_SECOND;
uint durationRtpTS = VIDEO_SAMPLING_RATE / fps;
OnVideoSourceEncodedSample.Invoke(durationRtpTS, encodedBuffer);
}
}

if (_frameCount == int.MaxValue)
{
_frameCount = 0;
}
}
}
}

private byte[] Snapshot(int width, int height)
{
SharpDX.DXGI.Resource screenResource;
OutputDuplicateFrameInformation duplicateFrameInformation;

try
{
_duplicatedOutput.AcquireNextFrame(10000, out duplicateFrameInformation, out screenResource);

using (var screenTexture2D = screenResource.QueryInterface())
_device.ImmediateContext.CopyResource(screenTexture2D, _screenTexture);

var mapSource = _device.ImmediateContext.MapSubresource(_screenTexture, 0, MapMode.Read, MapFlags.None);

int stride = width * 4; // 4 bytes per pixel (RGBA)
byte[] buffer = new byte[stride * height];
IntPtr sourcePtr = mapSource.DataPointer;
for (int y = 0; y < height; y++)
{
// Copy each row from source to buffer
Marshal.Copy(sourcePtr, buffer, y * stride, stride);
sourcePtr = IntPtr.Add(sourcePtr, mapSource.RowPitch);
}

_device.ImmediateContext.UnmapSubresource(_screenTexture, 0);

screenResource.Dispose();
_duplicatedOutput.ReleaseFrame();

// Correct pixel inversion (swap R and B channels)
for (int i = 0; i < buffer.Length; i += 4)
{
byte temp = buffer[i];     // Save R
buffer[i] = buffer[i + 2]; // Swap R (buffer[i]) with B (buffer[i+2])
buffer[i + 2] = temp;      // Swap B with R (saved R)
// buffer[i+1] (G) remains unchanged
// buffer[i+3] (A) remains unchanged
}

return buffer;
}
catch (SharpDXException e)
{
if (e.ResultCode.Code != SharpDX.DXGI.ResultCode.WaitTimeout.Result.Code)
{
throw;
}
return new byte[width * height * 4];
}
}

public void Dispose()
{
_isClosed = true;
_sendTestPatternTimer?.Dispose();
_videoEncoder?.Dispose();

_screenTexture?.Dispose();
_duplicatedOutput?.Dispose();
_output1?.Dispose();
_device?.Dispose();
_adapter?.Dispose();
_factory?.Dispose();
}
}
}

Для справки, вот что его загружает:

Код: Выделить всё

//Program.cs
using System;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using SIPSorcery.Media;
using SIPSorcery.Net;
using SIPSorceryMedia.Encoders;
using SIPSorceryMedia.FFmpeg;
using WebSocketSharp.Server;

namespace TESTPATTERNSERVER
{
class Program
{
private const int WEBSOCKET_PORT = 8081;

static void Main()
{
Console.WriteLine("WebRTC Get Started");

// Start web socket.
Console.WriteLine("Starting web socket server...");
var webSocketServer = new WebSocketServer(IPAddress.Any, WEBSOCKET_PORT);
webSocketServer.AddWebSocketService("/", (peer) => peer.CreatePeerConnection = () => CreatePeerConnection());
webSocketServer.Start();

Console.WriteLine($"Waiting for web socket connections on {webSocketServer.Address}:{webSocketServer.Port}...");

Console.WriteLine("Press any key exit.");
Console.ReadLine();
}

private static Task  CreatePeerConnection()
{
var pc = new RTCPeerConnection(null);

var testPatternSource = new VideoSource(1920, 1080, new VP8VideoEncoder());
var videoEndPoint = new SIPSorceryMedia.FFmpeg.FFmpegVideoEndPoint();

MediaStreamTrack videoTrack = new MediaStreamTrack(videoEndPoint.GetVideoSourceFormats(), MediaStreamStatusEnum.SendOnly);
pc.addTrack(videoTrack);

testPatternSource.OnVideoSourceEncodedSample += pc.SendVideo;
pc.OnVideoFormatsNegotiated += (formats) => testPatternSource.SetVideoSourceFormat(formats.First());

pc.onconnectionstatechange += async (state) =>
{
Console.WriteLine($"Peer connection state change to {state}.");

switch (state)
{
case RTCPeerConnectionState.connected:
await testPatternSource.StartVideo();
break;
case RTCPeerConnectionState.failed:
pc.Close("ice disconnection");
break;
case RTCPeerConnectionState.closed:
await testPatternSource.CloseVideo();
testPatternSource.Dispose();
break;
}
};

return Task.FromResult(pc);
}
}
}

Заранее спасибо!

Подробнее здесь: https://stackoverflow.com/questions/788 ... rp-with-we