1 /* 2 * Copyright 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.graphics.ImageFormat; 20 import android.graphics.PixelFormat; 21 import android.graphics.Rect; 22 import android.hardware.camera2.utils.SurfaceUtils; 23 import android.hardware.HardwareBuffer; 24 import android.os.Handler; 25 import android.os.Looper; 26 import android.os.Message; 27 import android.util.Size; 28 import android.view.Surface; 29 30 import dalvik.system.VMRuntime; 31 32 import java.lang.ref.WeakReference; 33 import java.nio.ByteBuffer; 34 import java.nio.ByteOrder; 35 import java.nio.NioUtils; 36 import java.util.List; 37 import java.util.concurrent.CopyOnWriteArrayList; 38 39 /** 40 * <p> 41 * The ImageWriter class allows an application to produce Image data into a 42 * {@link android.view.Surface}, and have it be consumed by another component 43 * like {@link android.hardware.camera2.CameraDevice CameraDevice}. 44 * </p> 45 * <p> 46 * Several Android API classes can provide input {@link android.view.Surface 47 * Surface} objects for ImageWriter to produce data into, including 48 * {@link MediaCodec MediaCodec} (encoder), 49 * {@link android.hardware.camera2.CameraCaptureSession CameraCaptureSession} 50 * (reprocessing input), {@link ImageReader}, etc. 51 * </p> 52 * <p> 53 * The input Image data is encapsulated in {@link Image} objects. To produce 54 * Image data into a destination {@link android.view.Surface Surface}, the 55 * application can get an input Image via {@link #dequeueInputImage} then write 56 * Image data into it. Multiple such {@link Image} objects can be dequeued at 57 * the same time and queued back in any order, up to the number specified by the 58 * {@code maxImages} constructor parameter. 59 * </p> 60 * <p> 61 * If the application already has an Image from {@link ImageReader}, the 62 * application can directly queue this Image into the ImageWriter (via 63 * {@link #queueInputImage}), potentially with zero buffer copies. This 64 * even works if the image format of the ImageWriter is 65 * {@link ImageFormat#PRIVATE PRIVATE}, and prior to Android P is the only 66 * way to enqueue images into such an ImageWriter. Starting in Android P 67 * private images may also be accessed through their hardware buffers 68 * (when available) through the {@link Image#getHardwareBuffer()} method. 69 * Attempting to access the planes of a private image, will return an 70 * empty array. 71 * </p> 72 * <p> 73 * Once new input Images are queued into an ImageWriter, it's up to the 74 * downstream components (e.g. {@link ImageReader} or 75 * {@link android.hardware.camera2.CameraDevice}) to consume the Images. If the 76 * downstream components cannot consume the Images at least as fast as the 77 * ImageWriter production rate, the {@link #dequeueInputImage} call will 78 * eventually block and the application will have to drop input frames. 79 * </p> 80 * <p> 81 * If the consumer component that provided the input {@link android.view.Surface Surface} 82 * abandons the {@link android.view.Surface Surface}, {@link #queueInputImage queueing} 83 * or {@link #dequeueInputImage dequeueing} an {@link Image} will throw an 84 * {@link IllegalStateException}. 85 * </p> 86 */ 87 public class ImageWriter implements AutoCloseable { 88 private final Object mListenerLock = new Object(); 89 private OnImageReleasedListener mListener; 90 private ListenerHandler mListenerHandler; 91 private long mNativeContext; 92 93 // Field below is used by native code, do not access or modify. 94 private int mWriterFormat; 95 96 private final int mMaxImages; 97 // Keep track of the currently dequeued Image. This need to be thread safe as the images 98 // could be closed by different threads (e.g., application thread and GC thread). 99 private List<Image> mDequeuedImages = new CopyOnWriteArrayList<>(); 100 private int mEstimatedNativeAllocBytes; 101 102 /** 103 * <p> 104 * Create a new ImageWriter. 105 * </p> 106 * <p> 107 * The {@code maxImages} parameter determines the maximum number of 108 * {@link Image} objects that can be be dequeued from the 109 * {@code ImageWriter} simultaneously. Requesting more buffers will use up 110 * more memory, so it is important to use only the minimum number necessary. 111 * </p> 112 * <p> 113 * The input Image size and format depend on the Surface that is provided by 114 * the downstream consumer end-point. 115 * </p> 116 * 117 * @param surface The destination Surface this writer produces Image data 118 * into. 119 * @param maxImages The maximum number of Images the user will want to 120 * access simultaneously for producing Image data. This should be 121 * as small as possible to limit memory use. Once maxImages 122 * Images are dequeued by the user, one of them has to be queued 123 * back before a new Image can be dequeued for access via 124 * {@link #dequeueInputImage()}. 125 * @return a new ImageWriter instance. 126 */ newInstance(Surface surface, int maxImages)127 public static ImageWriter newInstance(Surface surface, int maxImages) { 128 return new ImageWriter(surface, maxImages, ImageFormat.UNKNOWN); 129 } 130 131 /** 132 * <p> 133 * Create a new ImageWriter with given number of max Images and format. 134 * </p> 135 * <p> 136 * The {@code maxImages} parameter determines the maximum number of 137 * {@link Image} objects that can be be dequeued from the 138 * {@code ImageWriter} simultaneously. Requesting more buffers will use up 139 * more memory, so it is important to use only the minimum number necessary. 140 * </p> 141 * <p> 142 * The format specifies the image format of this ImageWriter. The format 143 * from the {@code surface} will be overridden with this format. For example, 144 * if the surface is obtained from a {@link android.graphics.SurfaceTexture}, the default 145 * format may be {@link PixelFormat#RGBA_8888}. If the application creates an ImageWriter 146 * with this surface and {@link ImageFormat#PRIVATE}, this ImageWriter will be able to operate 147 * with {@link ImageFormat#PRIVATE} Images. 148 * </p> 149 * <p> 150 * Note that the consumer end-point may or may not be able to support Images with different 151 * format, for such case, the application should only use this method if the consumer is able 152 * to consume such images. 153 * </p> 154 * <p> 155 * The input Image size depends on the Surface that is provided by 156 * the downstream consumer end-point. 157 * </p> 158 * 159 * @param surface The destination Surface this writer produces Image data 160 * into. 161 * @param maxImages The maximum number of Images the user will want to 162 * access simultaneously for producing Image data. This should be 163 * as small as possible to limit memory use. Once maxImages 164 * Images are dequeued by the user, one of them has to be queued 165 * back before a new Image can be dequeued for access via 166 * {@link #dequeueInputImage()}. 167 * @param format The format of this ImageWriter. It can be any valid format specified by 168 * {@link ImageFormat} or {@link PixelFormat}. 169 * 170 * @return a new ImageWriter instance. 171 * @hide 172 */ newInstance(Surface surface, int maxImages, int format)173 public static ImageWriter newInstance(Surface surface, int maxImages, int format) { 174 if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) { 175 throw new IllegalArgumentException("Invalid format is specified: " + format); 176 } 177 return new ImageWriter(surface, maxImages, format); 178 } 179 180 /** 181 * @hide 182 */ ImageWriter(Surface surface, int maxImages, int format)183 protected ImageWriter(Surface surface, int maxImages, int format) { 184 if (surface == null || maxImages < 1) { 185 throw new IllegalArgumentException("Illegal input argument: surface " + surface 186 + ", maxImages: " + maxImages); 187 } 188 189 mMaxImages = maxImages; 190 191 if (format == ImageFormat.UNKNOWN) { 192 format = SurfaceUtils.getSurfaceFormat(surface); 193 } 194 // Note that the underlying BufferQueue is working in synchronous mode 195 // to avoid dropping any buffers. 196 mNativeContext = nativeInit(new WeakReference<>(this), surface, maxImages, format); 197 198 // Estimate the native buffer allocation size and register it so it gets accounted for 199 // during GC. Note that this doesn't include the buffers required by the buffer queue 200 // itself and the buffers requested by the producer. 201 // Only include memory for 1 buffer, since actually accounting for the memory used is 202 // complex, and 1 buffer is enough for the VM to treat the ImageWriter as being of some 203 // size. 204 Size surfSize = SurfaceUtils.getSurfaceSize(surface); 205 mEstimatedNativeAllocBytes = 206 ImageUtils.getEstimatedNativeAllocBytes(surfSize.getWidth(),surfSize.getHeight(), 207 format, /*buffer count*/ 1); 208 VMRuntime.getRuntime().registerNativeAllocation(mEstimatedNativeAllocBytes); 209 } 210 211 /** 212 * <p> 213 * Maximum number of Images that can be dequeued from the ImageWriter 214 * simultaneously (for example, with {@link #dequeueInputImage()}). 215 * </p> 216 * <p> 217 * An Image is considered dequeued after it's returned by 218 * {@link #dequeueInputImage()} from ImageWriter, and until the Image is 219 * sent back to ImageWriter via {@link #queueInputImage}, or 220 * {@link Image#close()}. 221 * </p> 222 * <p> 223 * Attempting to dequeue more than {@code maxImages} concurrently will 224 * result in the {@link #dequeueInputImage()} function throwing an 225 * {@link IllegalStateException}. 226 * </p> 227 * 228 * @return Maximum number of Images that can be dequeued from this 229 * ImageWriter. 230 * @see #dequeueInputImage 231 * @see #queueInputImage 232 * @see Image#close 233 */ getMaxImages()234 public int getMaxImages() { 235 return mMaxImages; 236 } 237 238 /** 239 * <p> 240 * Dequeue the next available input Image for the application to produce 241 * data into. 242 * </p> 243 * <p> 244 * This method requests a new input Image from ImageWriter. The application 245 * owns this Image after this call. Once the application fills the Image 246 * data, it is expected to return this Image back to ImageWriter for 247 * downstream consumer components (e.g. 248 * {@link android.hardware.camera2.CameraDevice}) to consume. The Image can 249 * be returned to ImageWriter via {@link #queueInputImage} or 250 * {@link Image#close()}. 251 * </p> 252 * <p> 253 * This call will block if all available input images have been queued by 254 * the application and the downstream consumer has not yet consumed any. 255 * When an Image is consumed by the downstream consumer and released, an 256 * {@link OnImageReleasedListener#onImageReleased} callback will be fired, 257 * which indicates that there is one input Image available. For non- 258 * {@link ImageFormat#PRIVATE PRIVATE} formats ( 259 * {@link ImageWriter#getFormat()} != {@link ImageFormat#PRIVATE}), it is 260 * recommended to dequeue the next Image only after this callback is fired, 261 * in the steady state. 262 * </p> 263 * <p> 264 * If the format of ImageWriter is {@link ImageFormat#PRIVATE PRIVATE} ( 265 * {@link ImageWriter#getFormat()} == {@link ImageFormat#PRIVATE}), the 266 * image buffer is accessible to the application only through the hardware 267 * buffer obtained through {@link Image#getHardwareBuffer()}. (On Android 268 * versions prior to P, dequeueing private buffers will cause an 269 * {@link IllegalStateException} to be thrown). Alternatively, 270 * the application can acquire images from some other component (e.g. an 271 * {@link ImageReader}), and queue them directly to this ImageWriter via the 272 * {@link ImageWriter#queueInputImage queueInputImage()} method. 273 * </p> 274 * 275 * @return The next available input Image from this ImageWriter. 276 * @throws IllegalStateException if {@code maxImages} Images are currently 277 * dequeued, or the input {@link android.view.Surface Surface} 278 * has been abandoned by the consumer component that provided 279 * the {@link android.view.Surface Surface}. Prior to Android 280 * P, throws if the ImageWriter format is 281 * {@link ImageFormat#PRIVATE PRIVATE}. 282 * @see #queueInputImage 283 * @see Image#close 284 */ dequeueInputImage()285 public Image dequeueInputImage() { 286 if (mDequeuedImages.size() >= mMaxImages) { 287 throw new IllegalStateException("Already dequeued max number of Images " + mMaxImages); 288 } 289 WriterSurfaceImage newImage = new WriterSurfaceImage(this); 290 nativeDequeueInputImage(mNativeContext, newImage); 291 mDequeuedImages.add(newImage); 292 newImage.mIsImageValid = true; 293 return newImage; 294 } 295 296 /** 297 * <p> 298 * Queue an input {@link Image} back to ImageWriter for the downstream 299 * consumer to access. 300 * </p> 301 * <p> 302 * The input {@link Image} could be from ImageReader (acquired via 303 * {@link ImageReader#acquireNextImage} or 304 * {@link ImageReader#acquireLatestImage}), or from this ImageWriter 305 * (acquired via {@link #dequeueInputImage}). In the former case, the Image 306 * data will be moved to this ImageWriter. Note that the Image properties 307 * (size, format, strides, etc.) must be the same as the properties of the 308 * images dequeued from this ImageWriter, or this method will throw an 309 * {@link IllegalArgumentException}. In the latter case, the application has 310 * filled the input image with data. This method then passes the filled 311 * buffer to the downstream consumer. In both cases, it's up to the caller 312 * to ensure that the Image timestamp (in nanoseconds) is correctly set, as 313 * the downstream component may want to use it to indicate the Image data 314 * capture time. 315 * </p> 316 * <p> 317 * After this method is called and the downstream consumer consumes and 318 * releases the Image, an {@link OnImageReleasedListener#onImageReleased} 319 * callback will fire. The application can use this callback to avoid 320 * sending Images faster than the downstream consumer processing rate in 321 * steady state. 322 * </p> 323 * <p> 324 * Passing in an Image from some other component (e.g. an 325 * {@link ImageReader}) requires a free input Image from this ImageWriter as 326 * the destination. In this case, this call will block, as 327 * {@link #dequeueInputImage} does, if there are no free Images available. 328 * To avoid blocking, the application should ensure that there is at least 329 * one free Image available in this ImageWriter before calling this method. 330 * </p> 331 * <p> 332 * After this call, the input Image is no longer valid for further access, 333 * as if the Image is {@link Image#close closed}. Attempting to access the 334 * {@link ByteBuffer ByteBuffers} returned by an earlier 335 * {@link Image.Plane#getBuffer Plane#getBuffer} call will result in an 336 * {@link IllegalStateException}. 337 * </p> 338 * 339 * @param image The Image to be queued back to ImageWriter for future 340 * consumption. 341 * @throws IllegalStateException if the image was already queued previously, 342 * or the image was aborted previously, or the input 343 * {@link android.view.Surface Surface} has been abandoned by the 344 * consumer component that provided the 345 * {@link android.view.Surface Surface}. 346 * @see #dequeueInputImage() 347 */ queueInputImage(Image image)348 public void queueInputImage(Image image) { 349 if (image == null) { 350 throw new IllegalArgumentException("image shouldn't be null"); 351 } 352 boolean ownedByMe = isImageOwnedByMe(image); 353 if (ownedByMe && !(((WriterSurfaceImage) image).mIsImageValid)) { 354 throw new IllegalStateException("Image from ImageWriter is invalid"); 355 } 356 357 // For images from other components, need to detach first, then attach. 358 if (!ownedByMe) { 359 if (!(image.getOwner() instanceof ImageReader)) { 360 throw new IllegalArgumentException("Only images from ImageReader can be queued to" 361 + " ImageWriter, other image source is not supported yet!"); 362 } 363 364 ImageReader prevOwner = (ImageReader) image.getOwner(); 365 366 prevOwner.detachImage(image); 367 attachAndQueueInputImage(image); 368 // This clears the native reference held by the original owner. 369 // When this Image is detached later by this ImageWriter, the 370 // native memory won't be leaked. 371 image.close(); 372 return; 373 } 374 375 Rect crop = image.getCropRect(); 376 nativeQueueInputImage(mNativeContext, image, image.getTimestamp(), crop.left, crop.top, 377 crop.right, crop.bottom, image.getTransform(), image.getScalingMode()); 378 379 /** 380 * Only remove and cleanup the Images that are owned by this 381 * ImageWriter. Images detached from other owners are only temporarily 382 * owned by this ImageWriter and will be detached immediately after they 383 * are released by downstream consumers, so there is no need to keep 384 * track of them in mDequeuedImages. 385 */ 386 if (ownedByMe) { 387 mDequeuedImages.remove(image); 388 // Do not call close here, as close is essentially cancel image. 389 WriterSurfaceImage wi = (WriterSurfaceImage) image; 390 wi.clearSurfacePlanes(); 391 wi.mIsImageValid = false; 392 } 393 } 394 395 /** 396 * Get the ImageWriter format. 397 * <p> 398 * This format may be different than the Image format returned by 399 * {@link Image#getFormat()}. However, if the ImageWriter format is 400 * {@link ImageFormat#PRIVATE PRIVATE}, calling {@link #dequeueInputImage()} 401 * will result in an {@link IllegalStateException}. 402 * </p> 403 * 404 * @return The ImageWriter format. 405 */ getFormat()406 public int getFormat() { 407 return mWriterFormat; 408 } 409 410 /** 411 * ImageWriter callback interface, used to to asynchronously notify the 412 * application of various ImageWriter events. 413 */ 414 public interface OnImageReleasedListener { 415 /** 416 * <p> 417 * Callback that is called when an input Image is released back to 418 * ImageWriter after the data consumption. 419 * </p> 420 * <p> 421 * The client can use this callback to be notified that an input Image 422 * has been consumed and released by the downstream consumer. More 423 * specifically, this callback will be fired for below cases: 424 * <li>The application dequeues an input Image via the 425 * {@link ImageWriter#dequeueInputImage dequeueInputImage()} method, 426 * uses it, and then queues it back to this ImageWriter via the 427 * {@link ImageWriter#queueInputImage queueInputImage()} method. After 428 * the downstream consumer uses and releases this image to this 429 * ImageWriter, this callback will be fired. This image will be 430 * available to be dequeued after this callback.</li> 431 * <li>The application obtains an Image from some other component (e.g. 432 * an {@link ImageReader}), uses it, and then queues it to this 433 * ImageWriter via {@link ImageWriter#queueInputImage queueInputImage()}. 434 * After the downstream consumer uses and releases this image to this 435 * ImageWriter, this callback will be fired.</li> 436 * </p> 437 * 438 * @param writer the ImageWriter the callback is associated with. 439 * @see ImageWriter 440 * @see Image 441 */ onImageReleased(ImageWriter writer)442 void onImageReleased(ImageWriter writer); 443 } 444 445 /** 446 * Register a listener to be invoked when an input Image is returned to the 447 * ImageWriter. 448 * 449 * @param listener The listener that will be run. 450 * @param handler The handler on which the listener should be invoked, or 451 * null if the listener should be invoked on the calling thread's 452 * looper. 453 * @throws IllegalArgumentException If no handler specified and the calling 454 * thread has no looper. 455 */ setOnImageReleasedListener(OnImageReleasedListener listener, Handler handler)456 public void setOnImageReleasedListener(OnImageReleasedListener listener, Handler handler) { 457 synchronized (mListenerLock) { 458 if (listener != null) { 459 Looper looper = handler != null ? handler.getLooper() : Looper.myLooper(); 460 if (looper == null) { 461 throw new IllegalArgumentException( 462 "handler is null but the current thread is not a looper"); 463 } 464 if (mListenerHandler == null || mListenerHandler.getLooper() != looper) { 465 mListenerHandler = new ListenerHandler(looper); 466 } 467 mListener = listener; 468 } else { 469 mListener = null; 470 mListenerHandler = null; 471 } 472 } 473 } 474 475 /** 476 * Free up all the resources associated with this ImageWriter. 477 * <p> 478 * After calling this method, this ImageWriter cannot be used. Calling any 479 * methods on this ImageWriter and Images previously provided by 480 * {@link #dequeueInputImage()} will result in an 481 * {@link IllegalStateException}, and attempting to write into 482 * {@link ByteBuffer ByteBuffers} returned by an earlier 483 * {@link Image.Plane#getBuffer Plane#getBuffer} call will have undefined 484 * behavior. 485 * </p> 486 */ 487 @Override close()488 public void close() { 489 setOnImageReleasedListener(null, null); 490 for (Image image : mDequeuedImages) { 491 image.close(); 492 } 493 mDequeuedImages.clear(); 494 nativeClose(mNativeContext); 495 mNativeContext = 0; 496 497 if (mEstimatedNativeAllocBytes > 0) { 498 VMRuntime.getRuntime().registerNativeFree(mEstimatedNativeAllocBytes); 499 mEstimatedNativeAllocBytes = 0; 500 } 501 } 502 503 @Override finalize()504 protected void finalize() throws Throwable { 505 try { 506 close(); 507 } finally { 508 super.finalize(); 509 } 510 } 511 512 /** 513 * <p> 514 * Attach and queue input Image to this ImageWriter. 515 * </p> 516 * <p> 517 * When the format of an Image is {@link ImageFormat#PRIVATE PRIVATE}, or 518 * the source Image is so large that copying its data is too expensive, this 519 * method can be used to migrate the source Image into ImageWriter without a 520 * data copy, and then queue it to this ImageWriter. The source Image must 521 * be detached from its previous owner already, or this call will throw an 522 * {@link IllegalStateException}. 523 * </p> 524 * <p> 525 * After this call, the ImageWriter takes ownership of this Image. This 526 * ownership will automatically be removed from this writer after the 527 * consumer releases this Image, that is, after 528 * {@link OnImageReleasedListener#onImageReleased}. The caller is responsible for 529 * closing this Image through {@link Image#close()} to free up the resources 530 * held by this Image. 531 * </p> 532 * 533 * @param image The source Image to be attached and queued into this 534 * ImageWriter for downstream consumer to use. 535 * @throws IllegalStateException if the Image is not detached from its 536 * previous owner, or the Image is already attached to this 537 * ImageWriter, or the source Image is invalid. 538 */ attachAndQueueInputImage(Image image)539 private void attachAndQueueInputImage(Image image) { 540 if (image == null) { 541 throw new IllegalArgumentException("image shouldn't be null"); 542 } 543 if (isImageOwnedByMe(image)) { 544 throw new IllegalArgumentException( 545 "Can not attach an image that is owned ImageWriter already"); 546 } 547 /** 548 * Throw ISE if the image is not attachable, which means that it is 549 * either owned by other entity now, or completely non-attachable (some 550 * stand-alone images are not backed by native gralloc buffer, thus not 551 * attachable). 552 */ 553 if (!image.isAttachable()) { 554 throw new IllegalStateException("Image was not detached from last owner, or image " 555 + " is not detachable"); 556 } 557 558 // TODO: what if attach failed, throw RTE or detach a slot then attach? 559 // need do some cleanup to make sure no orphaned 560 // buffer caused leak. 561 Rect crop = image.getCropRect(); 562 nativeAttachAndQueueImage(mNativeContext, image.getNativeContext(), image.getFormat(), 563 image.getTimestamp(), crop.left, crop.top, crop.right, crop.bottom, 564 image.getTransform(), image.getScalingMode()); 565 } 566 567 /** 568 * This custom handler runs asynchronously so callbacks don't get queued 569 * behind UI messages. 570 */ 571 private final class ListenerHandler extends Handler { ListenerHandler(Looper looper)572 public ListenerHandler(Looper looper) { 573 super(looper, null, true /* async */); 574 } 575 576 @Override handleMessage(Message msg)577 public void handleMessage(Message msg) { 578 OnImageReleasedListener listener; 579 synchronized (mListenerLock) { 580 listener = mListener; 581 } 582 if (listener != null) { 583 listener.onImageReleased(ImageWriter.this); 584 } 585 } 586 } 587 588 /** 589 * Called from Native code when an Event happens. This may be called from an 590 * arbitrary Binder thread, so access to the ImageWriter must be 591 * synchronized appropriately. 592 */ postEventFromNative(Object selfRef)593 private static void postEventFromNative(Object selfRef) { 594 @SuppressWarnings("unchecked") 595 WeakReference<ImageWriter> weakSelf = (WeakReference<ImageWriter>) selfRef; 596 final ImageWriter iw = weakSelf.get(); 597 if (iw == null) { 598 return; 599 } 600 601 final Handler handler; 602 synchronized (iw.mListenerLock) { 603 handler = iw.mListenerHandler; 604 } 605 if (handler != null) { 606 handler.sendEmptyMessage(0); 607 } 608 } 609 610 /** 611 * <p> 612 * Abort the Images that were dequeued from this ImageWriter, and return 613 * them to this writer for reuse. 614 * </p> 615 * <p> 616 * This method is used for the cases where the application dequeued the 617 * Image, may have filled the data, but does not want the downstream 618 * component to consume it. The Image will be returned to this ImageWriter 619 * for reuse after this call, and the ImageWriter will immediately have an 620 * Image available to be dequeued. This aborted Image will be invisible to 621 * the downstream consumer, as if nothing happened. 622 * </p> 623 * 624 * @param image The Image to be aborted. 625 * @see #dequeueInputImage() 626 * @see Image#close() 627 */ abortImage(Image image)628 private void abortImage(Image image) { 629 if (image == null) { 630 throw new IllegalArgumentException("image shouldn't be null"); 631 } 632 633 if (!mDequeuedImages.contains(image)) { 634 throw new IllegalStateException("It is illegal to abort some image that is not" 635 + " dequeued yet"); 636 } 637 638 WriterSurfaceImage wi = (WriterSurfaceImage) image; 639 if (!wi.mIsImageValid) { 640 return; 641 } 642 643 /** 644 * We only need abort Images that are owned and dequeued by ImageWriter. 645 * For attached Images, no need to abort, as there are only two cases: 646 * attached + queued successfully, and attach failed. Neither of the 647 * cases need abort. 648 */ 649 cancelImage(mNativeContext, image); 650 mDequeuedImages.remove(image); 651 wi.clearSurfacePlanes(); 652 wi.mIsImageValid = false; 653 } 654 isImageOwnedByMe(Image image)655 private boolean isImageOwnedByMe(Image image) { 656 if (!(image instanceof WriterSurfaceImage)) { 657 return false; 658 } 659 WriterSurfaceImage wi = (WriterSurfaceImage) image; 660 if (wi.getOwner() != this) { 661 return false; 662 } 663 664 return true; 665 } 666 667 private static class WriterSurfaceImage extends android.media.Image { 668 private ImageWriter mOwner; 669 // This field is used by native code, do not access or modify. 670 private long mNativeBuffer; 671 private int mNativeFenceFd = -1; 672 private SurfacePlane[] mPlanes; 673 private int mHeight = -1; 674 private int mWidth = -1; 675 private int mFormat = -1; 676 // When this default timestamp is used, timestamp for the input Image 677 // will be generated automatically when queueInputBuffer is called. 678 private final long DEFAULT_TIMESTAMP = Long.MIN_VALUE; 679 private long mTimestamp = DEFAULT_TIMESTAMP; 680 681 private int mTransform = 0; //Default no transform 682 private int mScalingMode = 0; //Default frozen scaling mode 683 WriterSurfaceImage(ImageWriter writer)684 public WriterSurfaceImage(ImageWriter writer) { 685 mOwner = writer; 686 } 687 688 @Override getFormat()689 public int getFormat() { 690 throwISEIfImageIsInvalid(); 691 692 if (mFormat == -1) { 693 mFormat = nativeGetFormat(); 694 } 695 return mFormat; 696 } 697 698 @Override getWidth()699 public int getWidth() { 700 throwISEIfImageIsInvalid(); 701 702 if (mWidth == -1) { 703 mWidth = nativeGetWidth(); 704 } 705 706 return mWidth; 707 } 708 709 @Override getHeight()710 public int getHeight() { 711 throwISEIfImageIsInvalid(); 712 713 if (mHeight == -1) { 714 mHeight = nativeGetHeight(); 715 } 716 717 return mHeight; 718 } 719 720 @Override getTransform()721 public int getTransform() { 722 throwISEIfImageIsInvalid(); 723 724 return mTransform; 725 } 726 727 @Override getScalingMode()728 public int getScalingMode() { 729 throwISEIfImageIsInvalid(); 730 731 return mScalingMode; 732 } 733 734 @Override getTimestamp()735 public long getTimestamp() { 736 throwISEIfImageIsInvalid(); 737 738 return mTimestamp; 739 } 740 741 @Override setTimestamp(long timestamp)742 public void setTimestamp(long timestamp) { 743 throwISEIfImageIsInvalid(); 744 745 mTimestamp = timestamp; 746 } 747 748 @Override getHardwareBuffer()749 public HardwareBuffer getHardwareBuffer() { 750 throwISEIfImageIsInvalid(); 751 752 return nativeGetHardwareBuffer(); 753 } 754 755 @Override getPlanes()756 public Plane[] getPlanes() { 757 throwISEIfImageIsInvalid(); 758 759 if (mPlanes == null) { 760 int numPlanes = ImageUtils.getNumPlanesForFormat(getFormat()); 761 mPlanes = nativeCreatePlanes(numPlanes, getOwner().getFormat()); 762 } 763 764 return mPlanes.clone(); 765 } 766 767 @Override isAttachable()768 boolean isAttachable() { 769 throwISEIfImageIsInvalid(); 770 // Don't allow Image to be detached from ImageWriter for now, as no 771 // detach API is exposed. 772 return false; 773 } 774 775 @Override getOwner()776 ImageWriter getOwner() { 777 throwISEIfImageIsInvalid(); 778 779 return mOwner; 780 } 781 782 @Override getNativeContext()783 long getNativeContext() { 784 throwISEIfImageIsInvalid(); 785 786 return mNativeBuffer; 787 } 788 789 @Override close()790 public void close() { 791 if (mIsImageValid) { 792 getOwner().abortImage(this); 793 } 794 } 795 796 @Override finalize()797 protected final void finalize() throws Throwable { 798 try { 799 close(); 800 } finally { 801 super.finalize(); 802 } 803 } 804 clearSurfacePlanes()805 private void clearSurfacePlanes() { 806 if (mIsImageValid && mPlanes != null) { 807 for (int i = 0; i < mPlanes.length; i++) { 808 if (mPlanes[i] != null) { 809 mPlanes[i].clearBuffer(); 810 mPlanes[i] = null; 811 } 812 } 813 } 814 } 815 816 private class SurfacePlane extends android.media.Image.Plane { 817 private ByteBuffer mBuffer; 818 final private int mPixelStride; 819 final private int mRowStride; 820 821 // SurfacePlane instance is created by native code when SurfaceImage#getPlanes() is 822 // called SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer)823 private SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer) { 824 mRowStride = rowStride; 825 mPixelStride = pixelStride; 826 mBuffer = buffer; 827 /** 828 * Set the byteBuffer order according to host endianness (native 829 * order), otherwise, the byteBuffer order defaults to 830 * ByteOrder.BIG_ENDIAN. 831 */ 832 mBuffer.order(ByteOrder.nativeOrder()); 833 } 834 835 @Override getRowStride()836 public int getRowStride() { 837 throwISEIfImageIsInvalid(); 838 return mRowStride; 839 } 840 841 @Override getPixelStride()842 public int getPixelStride() { 843 throwISEIfImageIsInvalid(); 844 return mPixelStride; 845 } 846 847 @Override getBuffer()848 public ByteBuffer getBuffer() { 849 throwISEIfImageIsInvalid(); 850 return mBuffer; 851 } 852 clearBuffer()853 private void clearBuffer() { 854 // Need null check first, as the getBuffer() may not be called 855 // before an Image is closed. 856 if (mBuffer == null) { 857 return; 858 } 859 860 if (mBuffer.isDirect()) { 861 NioUtils.freeDirectBuffer(mBuffer); 862 } 863 mBuffer = null; 864 } 865 866 } 867 868 // Create the SurfacePlane object and fill the information nativeCreatePlanes(int numPlanes, int writerFmt)869 private synchronized native SurfacePlane[] nativeCreatePlanes(int numPlanes, int writerFmt); 870 nativeGetWidth()871 private synchronized native int nativeGetWidth(); 872 nativeGetHeight()873 private synchronized native int nativeGetHeight(); 874 nativeGetFormat()875 private synchronized native int nativeGetFormat(); 876 nativeGetHardwareBuffer()877 private synchronized native HardwareBuffer nativeGetHardwareBuffer(); 878 } 879 880 // Native implemented ImageWriter methods. nativeInit(Object weakSelf, Surface surface, int maxImgs, int format)881 private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs, 882 int format); 883 nativeClose(long nativeCtx)884 private synchronized native void nativeClose(long nativeCtx); 885 nativeDequeueInputImage(long nativeCtx, Image wi)886 private synchronized native void nativeDequeueInputImage(long nativeCtx, Image wi); 887 nativeQueueInputImage(long nativeCtx, Image image, long timestampNs, int left, int top, int right, int bottom, int transform, int scalingMode)888 private synchronized native void nativeQueueInputImage(long nativeCtx, Image image, 889 long timestampNs, int left, int top, int right, int bottom, int transform, 890 int scalingMode); 891 nativeAttachAndQueueImage(long nativeCtx, long imageNativeBuffer, int imageFormat, long timestampNs, int left, int top, int right, int bottom, int transform, int scalingMode)892 private synchronized native int nativeAttachAndQueueImage(long nativeCtx, 893 long imageNativeBuffer, int imageFormat, long timestampNs, int left, 894 int top, int right, int bottom, int transform, int scalingMode); 895 cancelImage(long nativeCtx, Image image)896 private synchronized native void cancelImage(long nativeCtx, Image image); 897 898 /** 899 * We use a class initializer to allow the native code to cache some field 900 * offsets. 901 */ nativeClassInit()902 private static native void nativeClassInit(); 903 904 static { 905 System.loadLibrary("media_jni"); nativeClassInit()906 nativeClassInit(); 907 } 908 } 909