Рисование изображения YUV на экране с аппаратным ускорением ⇐ Android

[Anonymous]

У нас есть прямой RTSP-поток с декодированными видеокадрами H.264 со скоростью 30 кадров в секунду.
Чтобы визуализировать их на экране, мы попытались использовать Surface, который мы передаем в наш MediaCodec.
Кроме того, нам необходимо предоставить декодированные кадры, чтобы пользователь нашего SDK мог сохранить кадр при необходимости.
Проблема в том, что как только вы предоставляете Surface для рисования в настройке MediaCodec метод, вы больше не получаете выходное изображение в onOutputBufferAvailable, а выходной буфер содержит специальный формат YUV для Surface, который вы не можете преобразовать, например, для сохранения на диске.
Если в методе configure мы передаем null вместо Surface, то мы получаем и выходное изображение, и выходной буфер в формате YUV.
Однако вы можете нарисовать только растровое изображение на Surface, поэтому его необходимо преобразовать в RGB. Поскольку у нас есть прямая трансляция со скоростью 30 кадров в секунду, это преобразование должно быть быстрым и аппаратно ускоренным. RenderScript устарел, поэтому мы попробовали представление SurfaceTexture с помощью OpenGL.
При этом подходе мы по-прежнему наблюдаем большую задержку, а также кажется слишком сложным просто нарисовать YUV-изображение на экране, одновременно отображая его как ByteBuffer или что-то подобное.
Есть ли у кого-нибудь совет, как эффективно решить эту проблему?
val decoder = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC)
val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height).apply {
setByteBuffer("csd-0", ByteBuffer.wrap(sps))
setByteBuffer("csd-1", ByteBuffer.wrap(pps))
}

decoder.setCallback(object : MediaCodec.Callback() {
override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {
freeVideoInputBuffers.trySend(index)
}

override fun onOutputBufferAvailable(
codec: MediaCodec,
index: Int,
info: MediaCodec.BufferInfo
) {
var bufferReleased = false
try {
val outputImage = codec.getOutputImage(index)
if (outputImage != null) {
val outputFormat = codec.getOutputFormat(index)

// Capture image properties BEFORE releasing buffer
val imageWidth = outputImage.width
val imageHeight = outputImage.height

// Create YUVFrame using buffer pool - copy data immediately
val planes = outputImage.planes
val yPlane = planes[0]
val uPlane = planes[1]
val vPlane = planes[2]

// Capture plane properties before release
val yRowStride = yPlane.rowStride
val uRowStride = uPlane.rowStride
val vRowStride = vPlane.rowStride
val yPixelStride = yPlane.pixelStride
val uPixelStride = uPlane.pixelStride
val vPixelStride = vPlane.pixelStride

// Acquire buffers from pool and copy synchronously
val yBuffer = bufferPool.acquire(yPlane.buffer.remaining())
val uBuffer = bufferPool.acquire(uPlane.buffer.remaining())
val vBuffer = bufferPool.acquire(vPlane.buffer.remaining())

yPlane.buffer.rewind()
yBuffer.put(yPlane.buffer)
yBuffer.rewind()

uPlane.buffer.rewind()
uBuffer.put(uPlane.buffer)
uBuffer.rewind()

vPlane.buffer.rewind()
vBuffer.put(vPlane.buffer)
vBuffer.rewind()

// Release MediaCodec buffer immediately after copy
codec.releaseOutputBuffer(index, false)
bufferReleased = true

// Create frame with copied data (using captured properties)
val yuvFrame = YUVFrame(
width = imageWidth,
height = imageHeight,
yPlane = yBuffer,
uPlane = uBuffer,
vPlane = vBuffer,
yRowStride = yRowStride,
uRowStride = uRowStride,
vRowStride = vRowStride,
yPixelStride = yPixelStride,
uPixelStride = uPixelStride,
vPixelStride = vPixelStride,
bufferPool = bufferPool,
mediaCodecCallback = null // Already released above
)

handleDecompressedFrame(
DecoderOutput(
yuvFrame = yuvFrame,
mediaFormat = outputFormat,
presentationTimestamp = info.presentationTimeUs,
width = width,
height = height
)
)
} else {
codec.releaseOutputBuffer(index, false)
bufferReleased = true
}
} catch (e: Exception) {
Logger.e(TAG, "Error processing output buffer", e)
if (!bufferReleased) {
try {
codec.releaseOutputBuffer(index, false)
} catch (releaseError: Exception) {
Logger.e(TAG, "Error releasing output buffer", releaseError)
}
}
}
}

override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) {
Logger.e(TAG, "MediaCodec error", e)
onDecodingError?.invoke()
}

override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) {
Logger.d(TAG, "Output format changed: $format")
}
})

decoder.configure(format, null, null, 0)
decoder.start()

Наш рендерер
internal class YuvGLRenderer(private val surfaceTexture: SurfaceTexture) {

companion object {
private const val TAG = "YuvGLRenderer"

// Vertex shader - simple pass-through
private const val VERTEX_SHADER = """
attribute vec4 aPosition;
attribute vec2 aTexCoord;
varying vec2 vTexCoord;

void main() {
gl_Position = aPosition;
vTexCoord = aTexCoord;
}
"""

// Fragment shader - YUV to RGB conversion using BT.601 color space
private const val FRAGMENT_SHADER = """
precision mediump float;

varying vec2 vTexCoord;
uniform sampler2D yTexture;
uniform sampler2D uTexture;
uniform sampler2D vTexture;

void main() {
float y = texture2D(yTexture, vTexCoord).r;
float u = texture2D(uTexture, vTexCoord).r - 0.5;
float v = texture2D(vTexture, vTexCoord).r - 0.5;

// BT.601 conversion
float r = y + 1.402 * v;
float g = y - 0.344136 * u - 0.714136 * v;
float b = y + 1.772 * u;

gl_FragColor = vec4(r, g, b, 1.0);
}
"""

// Full-screen quad vertices (2 triangles)
private val VERTICES = floatArrayOf(
-1.0f, -1.0f, // Bottom-left
1.0f, -1.0f, // Bottom-right
-1.0f, 1.0f, // Top-left
1.0f, 1.0f // Top-right
)

// Texture coordinates (flipped vertically for OpenGL)
private val TEX_COORDS = floatArrayOf(
0.0f, 1.0f, // Bottom-left
1.0f, 1.0f, // Bottom-right
0.0f, 0.0f, // Top-left
1.0f, 0.0f // Top-right
)
}

private var egl: EGL10? = null
private var eglDisplay: EGLDisplay? = null
private var eglContext: EGLContext? = null
private var eglSurface: EGLSurface? = null

private var program: Int = 0
private var yTextureId: Int = 0
private var uTextureId: Int = 0
private var vTextureId: Int = 0

private var aPositionHandle: Int = 0
private var aTexCoordHandle: Int = 0
private var yTextureHandle: Int = 0
private var uTextureHandle: Int = 0
private var vTextureHandle: Int = 0

private lateinit var vertexBuffer: FloatBuffer
private lateinit var texCoordBuffer: FloatBuffer

private var isInitialized = false
private var renderThread: HandlerThread? = null
private var renderHandler: Handler? = null
private val latestFrame = AtomicReference()

init {
setupBuffers()
}

private fun setupBuffers() {
vertexBuffer = ByteBuffer.allocateDirect(VERTICES.size * 4)
.order(ByteOrder.nativeOrder())
.asFloatBuffer()
.put(VERTICES)
.apply { position(0) }

texCoordBuffer = ByteBuffer.allocateDirect(TEX_COORDS.size * 4)
.order(ByteOrder.nativeOrder())
.asFloatBuffer()
.put(TEX_COORDS)
.apply { position(0) }
}

fun initialize(): Boolean {
try {
// Create dedicated rendering thread
val thread = HandlerThread("YuvGLRenderer").apply { start() }
renderThread = thread
renderHandler = Handler(thread.looper)

// Initialize EGL/GL on the render thread
val latch = CountDownLatch(1)
var success = false

renderHandler?.post {
try {
if (!initEGL()) {
Logger.e(TAG, "Failed to initialize EGL")
success = false
} else if (!initGL()) {
Logger.e(TAG, "Failed to initialize OpenGL")
success = false
} else {
isInitialized = true
success = true
Logger.i(TAG, "YuvGLRenderer initialized successfully")
}
} catch (e: Exception) {
Logger.e(TAG, "Error initializing renderer", e)
success = false
} finally {
latch.countDown()
}
}

latch.await()
return success
} catch (e: Exception) {
Logger.e(TAG, "Error initializing renderer", e)
return false
}
}

private fun initEGL(): Boolean {
egl = EGLContext.getEGL() as EGL10

eglDisplay = egl!!.eglGetDisplay(EGL10.EGL_DEFAULT_DISPLAY)
if (eglDisplay == EGL10.EGL_NO_DISPLAY) {
Logger.e(TAG, "eglGetDisplay failed")
return false
}

val version = IntArray(2)
if (!egl!!.eglInitialize(eglDisplay, version)) {
Logger.e(TAG, "eglInitialize failed")
return false
}

val configAttribs = intArrayOf(
EGL10.EGL_RENDERABLE_TYPE, 4, // EGL_OPENGL_ES2_BIT
EGL10.EGL_RED_SIZE, 8,
EGL10.EGL_GREEN_SIZE, 8,
EGL10.EGL_BLUE_SIZE, 8,
EGL10.EGL_ALPHA_SIZE, 8,
EGL10.EGL_DEPTH_SIZE, 0,
EGL10.EGL_STENCIL_SIZE, 0,
EGL10.EGL_NONE
)

val configs = arrayOfNulls(1)
val numConfigs = IntArray(1)
if (!egl!!.eglChooseConfig(eglDisplay, configAttribs, configs, 1, numConfigs)) {
Logger.e(TAG, "eglChooseConfig failed")
return false
}

val contextAttribs = intArrayOf(
0x3098, 2, // EGL_CONTEXT_CLIENT_VERSION
EGL10.EGL_NONE
)

eglContext = egl!!.eglCreateContext(
eglDisplay,
configs[0],
EGL10.EGL_NO_CONTEXT,
contextAttribs
)

if (eglContext == null || eglContext == EGL10.EGL_NO_CONTEXT) {
Logger.e(TAG, "eglCreateContext failed")
return false
}

eglSurface = egl!!.eglCreateWindowSurface(eglDisplay, configs[0], surfaceTexture, null)
if (eglSurface == null || eglSurface == EGL10.EGL_NO_SURFACE) {
Logger.e(TAG, "eglCreateWindowSurface failed")
return false
}

if (!egl!!.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)) {
Logger.e(TAG, "eglMakeCurrent failed")
return false
}

return true
}

private fun initGL(): Boolean {
// Compile shaders
val vertexShader = compileShader(GLES20.GL_VERTEX_SHADER, VERTEX_SHADER)
if (vertexShader == 0) return false

val fragmentShader = compileShader(GLES20.GL_FRAGMENT_SHADER, FRAGMENT_SHADER)
if (fragmentShader == 0) {
GLES20.glDeleteShader(vertexShader)
return false
}

// Create and link program
program = GLES20.glCreateProgram()
GLES20.glAttachShader(program, vertexShader)
GLES20.glAttachShader(program, fragmentShader)
GLES20.glLinkProgram(program)

val linkStatus = IntArray(1)
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0)
if (linkStatus[0] != GLES20.GL_TRUE) {
Logger.e(TAG, "Program link failed: ${GLES20.glGetProgramInfoLog(program)}")
GLES20.glDeleteShader(vertexShader)
GLES20.glDeleteShader(fragmentShader)
GLES20.glDeleteProgram(program)
return false
}

GLES20.glDeleteShader(vertexShader)
GLES20.glDeleteShader(fragmentShader)

// Get attribute and uniform locations
aPositionHandle = GLES20.glGetAttribLocation(program, "aPosition")
aTexCoordHandle = GLES20.glGetAttribLocation(program, "aTexCoord")
yTextureHandle = GLES20.glGetUniformLocation(program, "yTexture")
uTextureHandle = GLES20.glGetUniformLocation(program, "uTexture")
vTextureHandle = GLES20.glGetUniformLocation(program, "vTexture")

// Create textures for Y, U, V planes
val textures = IntArray(3)
GLES20.glGenTextures(3, textures, 0)
yTextureId = textures[0]
uTextureId = textures[1]
vTextureId = textures[2]

// Configure texture parameters
configureTexture(yTextureId)
configureTexture(uTextureId)
configureTexture(vTextureId)

return checkGLError("initGL")
}

private fun compileShader(type: Int, source: String): Int {
val shader = GLES20.glCreateShader(type)
GLES20.glShaderSource(shader, source)
GLES20.glCompileShader(shader)

val compileStatus = IntArray(1)
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compileStatus, 0)
if (compileStatus[0] != GLES20.GL_TRUE) {
Logger.e(TAG, "Shader compilation failed: ${GLES20.glGetShaderInfoLog(shader)}")
GLES20.glDeleteShader(shader)
return 0
}

return shader
}

private fun configureTexture(textureId: Int) {
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE)
}

fun renderFrame(yuvFrame: YUVFrame, viewWidth: Int, viewHeight: Int) {
if (!isInitialized) {
Logger.w(TAG, "Renderer not initialized")
yuvFrame.release()
return
}

// Store latest frame, releasing previous if exists
val oldFrame = latestFrame.getAndSet(yuvFrame)
oldFrame?.release()

// Post single render request (handler will process latest frame)
renderHandler?.removeCallbacksAndMessages(null)
renderHandler?.post {
val frame = latestFrame.getAndSet(null)
frame?.let {
try {
renderFrameInternal(it, viewWidth, viewHeight)
} catch (e: Exception) {
Logger.e(TAG, "Error rendering frame", e)
it.release()
}
}
}
}

private fun renderFrameInternal(yuvFrame: YUVFrame, viewWidth: Int, viewHeight: Int) {
try {
// Set viewport
GLES20.glViewport(0, 0, viewWidth, viewHeight)
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f)
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)

// Upload Y plane
uploadPlaneTexture(yTextureId, yuvFrame.yPlane, yuvFrame.width, yuvFrame.height,
yuvFrame.yRowStride, yuvFrame.yPixelStride)

// Upload U plane (half resolution)
val uvWidth = yuvFrame.width / 2
val uvHeight = yuvFrame.height / 2
uploadPlaneTexture(uTextureId, yuvFrame.uPlane, uvWidth, uvHeight,
yuvFrame.uRowStride, yuvFrame.uPixelStride)

// Upload V plane (half resolution)
uploadPlaneTexture(vTextureId, yuvFrame.vPlane, uvWidth, uvHeight,
yuvFrame.vRowStride, yuvFrame.vPixelStride)

// Draw the frame
GLES20.glUseProgram(program)

// Bind textures
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yTextureId)
GLES20.glUniform1i(yTextureHandle, 0)

GLES20.glActiveTexture(GLES20.GL_TEXTURE1)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, uTextureId)
GLES20.glUniform1i(uTextureHandle, 1)

GLES20.glActiveTexture(GLES20.GL_TEXTURE2)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, vTextureId)
GLES20.glUniform1i(vTextureHandle, 2)

// Set vertex attributes
GLES20.glEnableVertexAttribArray(aPositionHandle)
GLES20.glVertexAttribPointer(aPositionHandle, 2, GLES20.GL_FLOAT, false, 0, vertexBuffer)

GLES20.glEnableVertexAttribArray(aTexCoordHandle)
GLES20.glVertexAttribPointer(aTexCoordHandle, 2, GLES20.GL_FLOAT, false, 0, texCoordBuffer)

// Draw
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)

GLES20.glDisableVertexAttribArray(aPositionHandle)
GLES20.glDisableVertexAttribArray(aTexCoordHandle)

checkGLError("renderFrame")

// Swap buffers
egl?.eglSwapBuffers(eglDisplay, eglSurface)
} finally {
// Release frame buffers back to pool after GPU upload is complete
yuvFrame.release()
}
}

private fun uploadPlaneTexture(
textureId: Int,
planeBuffer: ByteBuffer,
width: Int,
height: Int,
rowStride: Int,
pixelStride: Int
) {
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId)

// Reset buffer position
planeBuffer.rewind()

if (pixelStride == 1 && rowStride == width) {
// Tightly packed - direct upload
GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D,
0,
GLES20.GL_LUMINANCE,
width,
height,
0,
GLES20.GL_LUMINANCE,
GLES20.GL_UNSIGNED_BYTE,
planeBuffer
)
} else {
// Need to handle stride/padding - copy to packed buffer
val packedBuffer = ByteBuffer.allocateDirect(width * height)
for (row in 0 until height) {
for (col in 0 until width) {
val index = row * rowStride + col * pixelStride
if (index < planeBuffer.limit()) {
packedBuffer.put(planeBuffer.get(index))
}
}
}
packedBuffer.rewind()

GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D,
0,
GLES20.GL_LUMINANCE,
width,
height,
0,
GLES20.GL_LUMINANCE,
GLES20.GL_UNSIGNED_BYTE,
packedBuffer
)
}
}

private fun checkGLError(op: String): Boolean {
val error = GLES20.glGetError()
if (error != GLES20.GL_NO_ERROR) {
Logger.e(TAG, "GL error after $op: ${GLUtils.getEGLErrorString(error)}")
return false
}
return true
}

fun release() {
try {
// Release any pending frame
latestFrame.getAndSet(null)?.release()

// Release GL resources on the render thread
val latch = CountDownLatch(1)
renderHandler?.post {
try {
if (program != 0) {
GLES20.glDeleteProgram(program)
program = 0
}

if (yTextureId != 0 || uTextureId != 0 || vTextureId != 0) {
GLES20.glDeleteTextures(3, intArrayOf(yTextureId, uTextureId, vTextureId), 0)
yTextureId = 0
uTextureId = 0
vTextureId = 0
}

egl?.let {
it.eglMakeCurrent(eglDisplay, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_CONTEXT)
it.eglDestroySurface(eglDisplay, eglSurface)
it.eglDestroyContext(eglDisplay, eglContext)
it.eglTerminate(eglDisplay)
}

eglSurface = null
eglContext = null
eglDisplay = null
egl = null

isInitialized = false
} finally {
latch.countDown()
}
}

latch.await()

// Quit the render thread
renderThread?.quitSafely()
renderThread?.join()
renderThread = null
renderHandler = null

Logger.i(TAG, "YuvGLRenderer released")
} catch (e: Exception) {
Logger.e(TAG, "Error releasing renderer", e)
}
}
}

Наше мнение
class RTSPPlayerView @JvmOverloads constructor(
context: Context,
attrs: AttributeSet? = null,
defStyleAttr: Int = 0
) : TextureView(context, attrs, defStyleAttr), TextureView.SurfaceTextureListener {

companion object {
private const val TAG = "RTSPPlayerView"
}

private var rtspClient: RTSPClient? = null
private var glRenderer: YuvGLRenderer? = null
private val viewScope = CoroutineScope(Dispatchers.Main + SupervisorJob())
private var audioFrameJob: Job? = null
private var videoFrameJob: Job? = null

init {
val identifier = System.identityHashCode(this)

surfaceTextureListener = this
isOpaque = false
}

override fun onDetachedFromWindow() {
super.onDetachedFromWindow()
stopAudioPlayback()
stopVideoRendering()
cleanup()
viewScope.cancel()
}

private fun cleanup() {
glRenderer?.release()
glRenderer = null
}

fun setRtspClient(client: RTSPClient) {
rtspClient = client

videoFrameJob?.cancel()
videoFrameJob = client.latestVideoFramePublisher
.onEach { videoFrame ->
renderFrame(videoFrame)
}
.launchIn(viewScope)
}

override fun onSurfaceTextureAvailable(surface: SurfaceTexture, width: Int, height: Int) {
Logger.d(TAG, "Surface texture available: width=$width, height=$height")

// Initialize OpenGL renderer
viewScope.launch(Dispatchers.Default) {
glRenderer?.release()
val renderer = YuvGLRenderer(surface)
if (renderer.initialize()) {
glRenderer = renderer
Logger.i(TAG, "OpenGL renderer initialized")
} else {
Logger.e(TAG, "Failed to initialize OpenGL renderer")
}
}
}

override fun onSurfaceTextureDestroyed(surface: SurfaceTexture): Boolean {
Logger.d(TAG, "Surface texture destroyed")
cleanup()
return true
}

private fun renderFrame(videoFrame: DecoderOutput?) {
videoFrame ?: return

try {
glRenderer?.renderFrame(videoFrame.yuvFrame, width, height)
} catch (e: Exception) {
Logger.e(TAG, "Error rendering frame", e)
}
}

private fun stopVideoRendering() {
videoFrameJob?.cancel()
videoFrameJob = null
Logger.i(TAG, "Video rendering stopped")
}
}


Подробнее здесь: https://stackoverflow.com/questions/798 ... ccelerated