[camera_android_camerax] Fix NV21 buffer over-allocation and optimize per-frame conversion

packages/camera/camera_android_camerax/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.7.5
+
+* Fixes NV21 buffer over-allocation when the Y plane buffer is oversized.
+* Optimizes per-frame NV21 conversion by reusing a pre-allocated output buffer to reduce GC pressure.
+
 ## 0.7.4+3
 
 * Updates `ResolutionPreset.max` to prefer higher resolution over capture rate

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/CameraXLibrary.g.kt

@@ -5717,12 +5717,8 @@ abstract class PigeonApiImageProxy(
 abstract class PigeonApiImageProxyUtils(
     open val pigeonRegistrar: CameraXLibraryPigeonProxyApiRegistrar
 ) {
-  /** Returns a single buffer that is representative of three NV21-compatible [planes]. */
-  abstract fun getNv21Buffer(
-      imageWidth: Long,
-      imageHeight: Long,
-      planes: List<androidx.camera.core.ImageProxy.PlaneProxy>
-  ): ByteArray
+  /** Returns a single buffer that is representative of three NV21-compatible planes. */
+  abstract fun getNv21Buffer(imageProxy: androidx.camera.core.ImageProxy): ByteArray
 
   companion object {
     @Suppress("LocalVariableName")
@@ -5737,12 +5733,10 @@ abstract class PigeonApiImageProxyUtils(
         if (api != null) {
           channel.setMessageHandler { message, reply ->
             val args = message as List<Any?>
-            val imageWidthArg = args[0] as Long
-            val imageHeightArg = args[1] as Long
-            val planesArg = args[2] as List<androidx.camera.core.ImageProxy.PlaneProxy>
+            val imageProxyArg = args[0] as androidx.camera.core.ImageProxy
             val wrapped: List<Any?> =
                 try {
-                  listOf(api.getNv21Buffer(imageWidthArg, imageHeightArg, planesArg))
+                  listOf(api.getNv21Buffer(imageProxyArg))
                 } catch (exception: Throwable) {
                   CameraXLibraryPigeonUtils.wrapError(exception)
                 }

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyProxyApi.java

@@ -7,7 +7,8 @@
 import androidx.annotation.NonNull;
 import androidx.camera.core.ImageProxy;
 import androidx.camera.core.ImageProxy.PlaneProxy;
-import java.util.Arrays;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -38,7 +39,42 @@ public long height(ImageProxy pigeonInstance) {
   @NonNull
   @Override
   public List<PlaneProxy> getPlanes(ImageProxy pigeonInstance) {
-    return Arrays.asList(pigeonInstance.getPlanes());
+    PlaneProxy[] originalPlanes = pigeonInstance.getPlanes();
+    List<PlaneProxy> planes = new ArrayList<>();
+    int height = pigeonInstance.getHeight();
+
+    for (int i = 0; i < originalPlanes.length; i++) {
+      final PlaneProxy original = originalPlanes[i];
+      final int planeHeight = (i == 0) ? height : height / 2;
+
+      planes.add(
+          new PlaneProxy() {
+            @Override
+            public int getRowStride() {
+              return original.getRowStride();
+            }
+
+            @Override
+            public int getPixelStride() {
+              return original.getPixelStride();
+            }
+
+            @NonNull
+            @Override
+            public ByteBuffer getBuffer() {
+              ByteBuffer sourceBuffer = original.getBuffer();
+              // Create a duplicate so we don't modify the original's position/limit.
+              ByteBuffer slicedBuffer = sourceBuffer.duplicate();
+              // Limit the buffer to what's actually needed for this plane.
+              int maxValidBytes = planeHeight * original.getRowStride();
+              if (maxValidBytes < slicedBuffer.remaining()) {
+                slicedBuffer.limit(slicedBuffer.position() + maxValidBytes);
+              }
+              return slicedBuffer;
+            }
+          });
+    }
+    return planes;
   }
 
   @Override

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtils.java

@@ -15,9 +15,21 @@ public class ImageProxyUtils {
   /**
    * Converts list of {@link PlaneProxy}s in YUV_420_888 format (with VU planes in NV21 layout) to a
    * single NV21 {@code ByteBuffer}.
+   *
+   * <p>The result is written into {@code outBuffer}, which must have a capacity of at least {@code
+   * width * height * 3 / 2} bytes. Passing a pre-allocated, reusable buffer avoids per-frame heap
+   * allocations and reduces GC pressure in high-throughput camera pipelines.
+   *
+   * @param planes the YUV_420_888 planes (Y at index 0, U at 1, V at 2).
+   * @param width image width in pixels.
+   * @param height image height in pixels.
+   * @param outBuffer caller-owned buffer that receives the NV21 data; must be large enough.
+   * @return a {@link ByteBuffer} wrapping {@code outBuffer} with position 0 and limit set to the
+   *     number of bytes written ({@code width * height * 3 / 2}).
    */
   @NonNull
-  public static ByteBuffer planesToNV21(@NonNull List<PlaneProxy> planes, int width, int height) {
+  public static ByteBuffer planesToNV21(
+      @NonNull List<PlaneProxy> planes, int width, int height, @NonNull byte[] outBuffer) {
     if (planes.size() < 3) {
       throw new IllegalArgumentException(
           "The plane list must contain at least 3 planes (Y, U, V).");
@@ -36,33 +48,23 @@ public static ByteBuffer planesToNV21(@NonNull List<PlaneProxy> planes, int widt
     uBuffer.rewind();
     vBuffer.rewind();
 
-    // Allocate a byte array for the NV21 frame.
-    // NV21 = Y plane + interleaved VU plane.
-    // Y = width * height; VU = (width * height) / 2 (4:2:0 subsampling).
-    // If the Y plane includes padding, ySize may be larger than width*height,
-    // but only the valid Y bytes are copied, so output size remains correct.
-    int ySize = yBuffer.remaining();
-    byte[] nv21Bytes = new byte[ySize + (width * height / 2)];
+    int expectedYSize = width * height;
     int position = 0;
 
     int yRowStride = yPlane.getRowStride();
     if (yRowStride == width) {
       // If no padding, copy entire Y plane at once.
-      yBuffer.get(nv21Bytes, 0, ySize);
-      position = ySize;
+      yBuffer.get(outBuffer, 0, expectedYSize);
+      position = expectedYSize;
     } else {
-      // Copy row by row if padding exists.
-      byte[] row = new byte[width];
+      // Copy row by row, reading directly into outBuffer to avoid a temporary array.
       for (int rowIndex = 0; rowIndex < height; rowIndex++) {
-        yBuffer.get(row, 0, width);
-        System.arraycopy(row, 0, nv21Bytes, position, width);
+        yBuffer.get(outBuffer, position, width);
         position += width;
-        // Adjust buffer position to start of next row.
-        // After reading 'width' bytes, move ahead by (yRowStride - width)
-        // to skip any padding bytes at the end of the current row.
-        if (rowIndex < height - 1) {
-          yBuffer.position(yBuffer.position() - width + yRowStride);
-        }
+        // Skip padding bytes to advance to the start of the next row.
+        // On the last row this seeks past the limit, which is harmless because
+        // yBuffer is not read again after the loop.
+        yBuffer.position(yBuffer.position() - width + yRowStride);
       }
     }
 
@@ -85,13 +87,32 @@ public static ByteBuffer planesToNV21(@NonNull List<PlaneProxy> planes, int widt
       // Interleave V and U chroma data into the NV21 buffer.
       // In NV21, chroma bytes follow the Y plane in repeating VU pairs (VUVU...).
       for (int col = 0; col < width / 2; col++) {
-        int vIndex = col * vPixelStride;
-        int uIndex = col * uPixelStride;
-        nv21Bytes[position++] = vRowBuffer[vIndex];
-        nv21Bytes[position++] = uRowBuffer[uIndex];
+        outBuffer[position++] = vRowBuffer[col * vPixelStride];
+        outBuffer[position++] = uRowBuffer[col * uPixelStride];
       }
     }
 
-    return ByteBuffer.wrap(nv21Bytes);
+    return ByteBuffer.wrap(outBuffer, 0, expectedYSize + (expectedYSize / 2));
+  }
+
+  /**
+   * Converts list of {@link PlaneProxy}s in YUV_420_888 format (with VU planes in NV21 layout) to a
+   * single NV21 {@code ByteBuffer}.
+   *
+   * <p>Allocates a new output buffer sized {@code width * height * 3 / 2} bytes on every call. For
+   * hot paths such as per-frame camera callbacks, prefer {@link #planesToNV21(List, int, int,
+   * byte[])} with a pre-allocated, reusable buffer to reduce GC pressure.
+   *
+   * @param planes the YUV_420_888 planes (Y at index 0, U at 1, V at 2).
+   * @param width image width in pixels.
+   * @param height image height in pixels.
+   * @return a {@link ByteBuffer} containing the NV21 data with a capacity of {@code width * height
+   *     * 3 / 2} bytes.
+   */
+  @NonNull
+  public static ByteBuffer planesToNV21(@NonNull List<PlaneProxy> planes, int width, int height) {
+    int expectedYSize = width * height;
+    byte[] outBuffer = new byte[expectedYSize + (expectedYSize / 2)];
+    return planesToNV21(planes, width, height, outBuffer);
   }
 }

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtilsProxyApi.java

@@ -5,32 +5,29 @@
 package io.flutter.plugins.camerax;
 
 import androidx.annotation.NonNull;
-import androidx.camera.core.ImageProxy.PlaneProxy;
-import java.nio.ByteBuffer;
-import java.util.List;
+import androidx.camera.core.ImageProxy;
+import java.util.Arrays;
 
 /**
  * ProxyApi implementation for {@link ImageProxyUtils}. This class may handle instantiating native
  * object instances that are attached to a Dart instance or handle method calls on the associated
  * native class or an instance of that class.
  */
 public class ImageProxyUtilsProxyApi extends PigeonApiImageProxyUtils {
+
   ImageProxyUtilsProxyApi(@NonNull ProxyApiRegistrar pigeonRegistrar) {
     super(pigeonRegistrar);
   }
 
-  // List<? extends PlaneProxy> can be considered the same as List<PlaneProxy>.
-  @SuppressWarnings("unchecked")
   @NonNull
   @Override
-  public byte[] getNv21Buffer(
-      long imageWidth, long imageHeight, @NonNull List<? extends PlaneProxy> planes) {
-    final ByteBuffer nv21Buffer =
-        ImageProxyUtils.planesToNV21(
-            (List<PlaneProxy>) planes, (int) imageWidth, (int) imageHeight);
+  public byte[] getNv21Buffer(@NonNull ImageProxy imageProxy) {
+    final int width = imageProxy.getWidth();
+    final int height = imageProxy.getHeight();
+    final int expectedYSize = width * height;
+    final byte[] bytes = new byte[expectedYSize + (expectedYSize / 2)];
 
-    byte[] bytes = new byte[nv21Buffer.remaining()];
-    nv21Buffer.get(bytes, 0, bytes.length);
+    ImageProxyUtils.planesToNV21(Arrays.asList(imageProxy.getPlanes()), width, height, bytes);
 
     return bytes;
   }

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageProxyTest.java

@@ -11,7 +11,7 @@
 
 import androidx.camera.core.ImageProxy;
 import androidx.camera.core.ImageProxy.PlaneProxy;
-import java.util.Collections;
+import java.nio.ByteBuffer;
 import java.util.List;
 import org.junit.Test;
 
@@ -54,10 +54,45 @@ public void getPlanes_returnsExpectedPlanes() {
     final PigeonApiImageProxy api = new TestProxyApiRegistrar().getPigeonApiImageProxy();
 
     final ImageProxy instance = mock(ImageProxy.class);
-    final List<PlaneProxy> value = Collections.singletonList(mock(PlaneProxy.class));
-    when(instance.getPlanes()).thenReturn(value.toArray(new PlaneProxy[] {}));
+    when(instance.getHeight()).thenReturn(4);
 
-    assertEquals(value, api.getPlanes(instance));
+    final PlaneProxy mockPlane0 = mock(PlaneProxy.class);
+    final PlaneProxy mockPlane1 = mock(PlaneProxy.class);
+
+    when(mockPlane0.getRowStride()).thenReturn(10);
+    when(mockPlane0.getPixelStride()).thenReturn(1);
+    // Y plane buffer with some extra padding
+    ByteBuffer buffer0 = ByteBuffer.allocate(100);
+    when(mockPlane0.getBuffer()).thenReturn(buffer0);
+
+    when(mockPlane1.getRowStride()).thenReturn(5);
+    when(mockPlane1.getPixelStride()).thenReturn(2);
+    // U/V plane buffer
+    ByteBuffer buffer1 = ByteBuffer.allocate(50);
+    when(mockPlane1.getBuffer()).thenReturn(buffer1);
+
+    when(instance.getPlanes()).thenReturn(new PlaneProxy[] {mockPlane0, mockPlane1});
+
+    List<PlaneProxy> planes = api.getPlanes(instance);
+    assertEquals(2, planes.size());
+
+    // Verify Y plane wrapping
+    PlaneProxy plane0 = planes.get(0);
+    assertEquals(10, plane0.getRowStride());
+    assertEquals(1, plane0.getPixelStride());
+
+    ByteBuffer wrappedBuffer0 = plane0.getBuffer();
+    // expected size: planeHeight * rowStride = 4 * 10 = 40
+    assertEquals(40, wrappedBuffer0.remaining());
+
+    // Verify U/V plane wrapping
+    PlaneProxy plane1 = planes.get(1);
+    assertEquals(5, plane1.getRowStride());
+    assertEquals(2, plane1.getPixelStride());
+
+    ByteBuffer wrappedBuffer1 = plane1.getBuffer();
+    // expected size: (planeHeight / 2) * rowStride = 2 * 5 = 10
+    assertEquals(10, wrappedBuffer1.remaining());
   }
 
   @Test

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageProxyUtilsApiTest.java

@@ -5,15 +5,22 @@
 package io.flutter.plugins.camerax;
 
 import static org.junit.Assert.assertArrayEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.when;
 
+import androidx.camera.core.ImageProxy;
 import androidx.camera.core.ImageProxy.PlaneProxy;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
 import org.junit.Test;
 import org.mockito.MockedStatic;
-import org.mockito.Mockito;
+import org.mockito.stubbing.Answer;
 
 public class ImageProxyUtilsApiTest {
 
@@ -22,27 +29,41 @@ public void getNv21Buffer_returnsExpectedBytes() {
     final PigeonApiImageProxyUtils api = new TestProxyApiRegistrar().getPigeonApiImageProxyUtils();
 
     List<PlaneProxy> planes =
-        Arrays.asList(
-            Mockito.mock(PlaneProxy.class),
-            Mockito.mock(PlaneProxy.class),
-            Mockito.mock(PlaneProxy.class));
-    long width = 4;
-    long height = 2;
-    byte[] expectedBytes = new byte[] {1, 2, 3, 4, 5};
-    ByteBuffer mockBuffer = ByteBuffer.wrap(expectedBytes);
+        Arrays.asList(mock(PlaneProxy.class), mock(PlaneProxy.class), mock(PlaneProxy.class));
+    int width = 4;
+    int height = 2;
+    // Matches the size allocated by getNv21Buffer: width * height * 3 / 2.
+    final byte[] nv21Data = new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12};
 
     try (MockedStatic<ImageProxyUtils> mockedStatic = mockStatic(ImageProxyUtils.class)) {
+      // Simulate planesToNV21 filling the caller-owned outBuffer in-place.
       mockedStatic
           .when(
-              () ->
-                  ImageProxyUtils.planesToNV21(
-                      Mockito.anyList(), Mockito.anyInt(), Mockito.anyInt()))
-          .thenReturn(mockBuffer);
+              () -> ImageProxyUtils.planesToNV21(anyList(), anyInt(), anyInt(), any(byte[].class)))
+          .thenAnswer(
+              (Answer<ByteBuffer>)
+                  invocation -> {
+                    byte[] outBuffer = invocation.getArgument(3);
+                    System.arraycopy(nv21Data, 0, outBuffer, 0, outBuffer.length);
+                    return ByteBuffer.wrap(outBuffer);
+                  });
 
-      byte[] result = api.getNv21Buffer(width, height, planes);
+      ImageProxy imageProxy = mock(ImageProxy.class);
+      PlaneProxy[] planesArray = planes.toArray(new PlaneProxy[0]);
+      when(imageProxy.getPlanes()).thenReturn(planesArray);
+      when(imageProxy.getWidth()).thenReturn(width);
+      when(imageProxy.getHeight()).thenReturn(height);
 
-      assertArrayEquals(expectedBytes, result);
-      mockedStatic.verify(() -> ImageProxyUtils.planesToNV21(planes, (int) width, (int) height));
+      byte[] result = api.getNv21Buffer(imageProxy);
+
+      assertArrayEquals(nv21Data, result);
+      mockedStatic.verify(
+          () ->
+              ImageProxyUtils.planesToNV21(
+                  eq(Arrays.asList(planesArray)),
+                  eq(width),
+                  eq(height),
+                  any(byte[].class)));
     }
   }
 }