Описание проблемы При запуске приложения:
- Предварительный просмотр камеры правильно отображает прямую трансляцию.
- Распознавание текста ML Kit обрабатывает изображение и идентифицирует текстовые блоки.
- Ограничительные рамки рисуются вокруг обнаруженных текстовых элементов.
- Однако эти ограничивающие рамки не совместить с фактическим текстом в предварительном просмотре камеры. Они выглядят смещенными или неправильно масштабированы.
Код: Выделить всё
package com.aviskaarlab.booksnap.ui.views.home
import android.Manifest
import android.content.pm.PackageManager
import android.os.Build
import android.os.Bundle
import android.util.Log
import androidx.appcompat.app.AppCompatActivity
import androidx.camera.core.AspectRatio
import androidx.camera.core.CameraSelector
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.Preview
import androidx.camera.core.resolutionselector.AspectRatioStrategy
import androidx.camera.core.resolutionselector.ResolutionSelector
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.camera.view.PreviewView
import androidx.core.content.ContextCompat
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
class MainActivity : AppCompatActivity() {
private lateinit var viewBinding: ActivityMainBinding
private lateinit var cameraExecutor: ExecutorService
private lateinit var textOverlay: TextOverlay
private lateinit var viewFinder: PreviewView
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
viewBinding = ActivityMainBinding.inflate(layoutInflater)
setContentView(viewBinding.root)
viewFinder = viewBinding.previewView
textOverlay = viewBinding.textOverlay
// Initialize camera and start preview
startCamera()
cameraExecutor = Executors.newSingleThreadExecutor()
}
private fun startCamera() {
val cameraProviderFuture = ProcessCameraProvider.getInstance(this)
cameraProviderFuture.addListener({
val resolutionSelector = ResolutionSelector.Builder()
.setAspectRatioStrategy(AspectRatioStrategy.RATIO_4_3_FALLBACK_AUTO_STRATEGY)
.build()
val rotation = viewFinder.display.rotation
val cameraProvider: ProcessCameraProvider = cameraProviderFuture.get()
// Preview
val preview = Preview.Builder()
.setResolutionSelector(resolutionSelector)
.setTargetRotation(rotation)
.build()
.also {
it.setSurfaceProvider(viewBinding.previewView.surfaceProvider)
}
// Image Analysis
val imageAnalysis = ImageAnalysis.Builder()
.setResolutionSelector(resolutionSelector)
.setTargetRotation(rotation)
.build()
.also {
it.setAnalyzer(
cameraExecutor,
BookSnapWordAnalyzer(
textOverlay,
viewBinding.previewView,
)
)
}
// Select back camera as default
val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
try {
cameraProvider.unbindAll()
cameraProvider.bindToLifecycle(
this, cameraSelector, preview, imageAnalysis
)
preview.setSurfaceProvider(viewFinder.surfaceProvider)
} catch (exc: Exception) {
Log.e(TAG, "Use case binding failed", exc)
}
}, ContextCompat.getMainExecutor(this))
}
override fun onDestroy() {
super.onDestroy()
cameraExecutor.shutdown()
}
companion object {
private const val TAG = "CameraXApp"
}
}
Код: Выделить всё
package com.aviskaarlab.booksnap.ui.views.home
import android.graphics.Matrix
import android.graphics.Rect
import android.graphics.RectF
import android.util.Log
import androidx.camera.core.ExperimentalGetImage
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.ImageProxy
import androidx.camera.view.PreviewView
import com.google.mlkit.vision.common.InputImage
import com.google.mlkit.vision.text.Text
import com.google.mlkit.vision.text.TextRecognition
import com.google.mlkit.vision.text.latin.TextRecognizerOptions
internal class BookSnapWordAnalyzer(
private val overlay: TextOverlay,
private val previewView: PreviewView,
) : ImageAnalysis.Analyzer {
companion object {
private const val TAG = "BookSnapWordAnalyzer"
private const val WORD_LENGTH = 4
}
private val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS)
private lateinit var visionText: Text
private lateinit var matrix: Matrix
private var rotationDegrees: Int = 0
@OptIn(ExperimentalGetImage::class)
override fun analyze(imageProxy: ImageProxy) {
val mediaImage = imageProxy.image ?: return
rotationDegrees = imageProxy.imageInfo.rotationDegrees
matrix = getCorrectionMatrix(imageProxy, previewView)
val image =
InputImage.fromMediaImage(mediaImage, imageProxy.imageInfo.rotationDegrees)
recognizer.process(image)
.addOnSuccessListener { visionText ->
this.visionText = visionText
val boxes = mutableListOf()
for (block in visionText.textBlocks) {
for (line in block.lines) {
for (element in line.elements) {
val elementText = element.text
val boundingBox = element.boundingBox
if (elementText.length >= WORD_LENGTH && boundingBox != null) {
boxes.add(
CustomRect(
adjustBoundingBox(boundingBox, imageProxy, previewView),
elementText
)
)
}
}
}
}
overlay.updateBoundingBoxes(boxes)
}
.addOnFailureListener { e ->
Log.e(TAG, "Text recognition failed", e)
}.addOnCompleteListener {
imageProxy.close()
}
}
private fun adjustBoundingBox(
rect: Rect,
imageProxy: ImageProxy,
previewView: PreviewView
): RectF {
val cropRect = imageProxy.cropRect
val imageWidth = cropRect.width()
val imageHeight = cropRect.height()
val previewWidth = previewView.width
val previewHeight = previewView.height
val scaleX = previewWidth.toFloat() / imageWidth
val scaleY = previewHeight.toFloat() / imageHeight
val verticalOffset = (previewHeight - imageHeight * scaleY) / 2
val horizontalOffset = (previewWidth - imageWidth * scaleX) / 2
val left = rect.left * scaleX + horizontalOffset
val top = rect.top * scaleY + verticalOffset
val right = rect.right * scaleX + horizontalOffset
val bottom = rect.bottom * scaleY + verticalOffset
return RectF(left, top, right, bottom)
}
private fun getCorrectionMatrix(
imageProxy: ImageProxy,
previewView: PreviewView,
): Matrix {
val cropRect = imageProxy.cropRect
val matrix = Matrix()
val source = floatArrayOf(
cropRect.left.toFloat(), cropRect.top.toFloat(),
cropRect.right.toFloat(), cropRect.top.toFloat(),
cropRect.right.toFloat(), cropRect.bottom.toFloat(),
cropRect.left.toFloat(), cropRect.bottom.toFloat()
)
val destination = floatArrayOf(
0f, 0f,
previewView.width.toFloat(), 0f,
previewView.width.toFloat(), previewView.height.toFloat(),
0f, previewView.height.toFloat()
)
matrix.setPolyToPoly(source, 0, destination, 0, 4)
return matrix
}
}
Код: Выделить всё
package com.aviskaarlab.booksnap.ui.views.home
import android.content.Context
import android.graphics.Canvas
import android.graphics.Color
import android.graphics.Paint
import android.graphics.RectF
import android.util.AttributeSet
import android.view.MotionEvent
import android.view.View
class TextOverlay @JvmOverloads constructor(
context: Context, attrs: AttributeSet? = null, defStyleAttr: Int = 0
) : View(context, attrs, defStyleAttr) {
private val paint = Paint().apply {
color = Color.RED
style = Paint.Style.STROKE
strokeWidth = 2.0f
}
private val boundingBoxes = mutableListOf()
private var clickListener: ((String) -> Unit)? = null
fun setOnRectangleClickListener(listener: (String) -> Unit) {
clickListener = listener
}
fun updateBoundingBoxes(newBoundingBoxes: List) {
boundingBoxes.clear()
boundingBoxes.addAll(newBoundingBoxes)
invalidate() // Redraw the view
}
override fun onDraw(canvas: Canvas) {
super.onDraw(canvas)
for (box in boundingBoxes) {
canvas.drawRect(box.rect, paint)
}
}
override fun onTouchEvent(event: MotionEvent): Boolean {
if (event.action == MotionEvent.ACTION_UP) {
val x = event.x
val y = event.y
for (box in boundingBoxes) {
if (box.rect.contains(x, y)) {
clickListener?.invoke("Clicked on rectangle ${box.text}")
return true
}
}
}
return true // Event handled
}
}
Прямоугольники, нарисованные вокруг распознанного текста, не совпадают с текстом в предварительном просмотре камеры. Как настроить координаты ограничительной рамки, чтобы они правильно накладывались на текст в прямом эфире с камеры? (см. прикрепленное изображение)
Подробнее здесь: https://stackoverflow.com/questions/786 ... ra-preview
