1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntDef; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.compat.annotation.UnsupportedAppUsage; 23 import android.graphics.ImageFormat; 24 import android.graphics.Rect; 25 import android.graphics.SurfaceTexture; 26 import android.hardware.HardwareBuffer; 27 import android.media.MediaCodecInfo.CodecCapabilities; 28 import android.os.Build; 29 import android.os.Bundle; 30 import android.os.Handler; 31 import android.os.IHwBinder; 32 import android.os.Looper; 33 import android.os.Message; 34 import android.os.PersistableBundle; 35 import android.view.Surface; 36 37 import java.io.IOException; 38 import java.lang.annotation.Retention; 39 import java.lang.annotation.RetentionPolicy; 40 import java.nio.ByteBuffer; 41 import java.nio.ByteOrder; 42 import java.nio.ReadOnlyBufferException; 43 import java.util.ArrayList; 44 import java.util.Arrays; 45 import java.util.Collections; 46 import java.util.HashMap; 47 import java.util.HashSet; 48 import java.util.Map; 49 import java.util.Objects; 50 import java.util.Set; 51 import java.util.concurrent.BlockingQueue; 52 import java.util.concurrent.LinkedBlockingQueue; 53 import java.util.concurrent.locks.Lock; 54 import java.util.concurrent.locks.ReentrantLock; 55 56 /** 57 MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components. 58 It is part of the Android low-level multimedia support infrastructure (normally used together 59 with {@link MediaExtractor}, {@link MediaSync}, {@link MediaMuxer}, {@link MediaCrypto}, 60 {@link MediaDrm}, {@link Image}, {@link Surface}, and {@link AudioTrack}.) 61 <p> 62 <center><object style="width: 540px; height: 205px;" type="image/svg+xml" 63 data="../../../images/media/mediacodec_buffers.svg"><img 64 src="../../../images/media/mediacodec_buffers.png" style="width: 540px; height: 205px" 65 alt="MediaCodec buffer flow diagram"></object></center> 66 <p> 67 In broad terms, a codec processes input data to generate output data. It processes data 68 asynchronously and uses a set of input and output buffers. At a simplistic level, you request 69 (or receive) an empty input buffer, fill it up with data and send it to the codec for 70 processing. The codec uses up the data and transforms it into one of its empty output buffers. 71 Finally, you request (or receive) a filled output buffer, consume its contents and release it 72 back to the codec. 73 74 <h3>Data Types</h3> 75 <p> 76 Codecs operate on three kinds of data: compressed data, raw audio data and raw video data. 77 All three kinds of data can be processed using {@link ByteBuffer ByteBuffers}, but you should use 78 a {@link Surface} for raw video data to improve codec performance. Surface uses native video 79 buffers without mapping or copying them to ByteBuffers; thus, it is much more efficient. 80 You normally cannot access the raw video data when using a Surface, but you can use the 81 {@link ImageReader} class to access unsecured decoded (raw) video frames. This may still be more 82 efficient than using ByteBuffers, as some native buffers may be mapped into {@linkplain 83 ByteBuffer#isDirect direct} ByteBuffers. When using ByteBuffer mode, you can access raw video 84 frames using the {@link Image} class and {@link #getInputImage getInput}/{@link #getOutputImage 85 OutputImage(int)}. 86 87 <h4>Compressed Buffers</h4> 88 <p> 89 Input buffers (for decoders) and output buffers (for encoders) contain compressed data according 90 to the {@linkplain MediaFormat#KEY_MIME format's type}. For video types this is normally a single 91 compressed video frame. For audio data this is normally a single access unit (an encoded audio 92 segment typically containing a few milliseconds of audio as dictated by the format type), but 93 this requirement is slightly relaxed in that a buffer may contain multiple encoded access units 94 of audio. In either case, buffers do not start or end on arbitrary byte boundaries, but rather on 95 frame/access unit boundaries unless they are flagged with {@link #BUFFER_FLAG_PARTIAL_FRAME}. 96 97 <h4>Raw Audio Buffers</h4> 98 <p> 99 Raw audio buffers contain entire frames of PCM audio data, which is one sample for each channel 100 in channel order. Each PCM audio sample is either a 16 bit signed integer or a float, 101 in native byte order. 102 Raw audio buffers in the float PCM encoding are only possible 103 if the MediaFormat's {@linkplain MediaFormat#KEY_PCM_ENCODING} 104 is set to {@linkplain AudioFormat#ENCODING_PCM_FLOAT} during MediaCodec 105 {@link #configure configure(…)} 106 and confirmed by {@link #getOutputFormat} for decoders 107 or {@link #getInputFormat} for encoders. 108 A sample method to check for float PCM in the MediaFormat is as follows: 109 110 <pre class=prettyprint> 111 static boolean isPcmFloat(MediaFormat format) { 112 return format.getInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT) 113 == AudioFormat.ENCODING_PCM_FLOAT; 114 }</pre> 115 116 In order to extract, in a short array, 117 one channel of a buffer containing 16 bit signed integer audio data, 118 the following code may be used: 119 120 <pre class=prettyprint> 121 // Assumes the buffer PCM encoding is 16 bit. 122 short[] getSamplesForChannel(MediaCodec codec, int bufferId, int channelIx) { 123 ByteBuffer outputBuffer = codec.getOutputBuffer(bufferId); 124 MediaFormat format = codec.getOutputFormat(bufferId); 125 ShortBuffer samples = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer(); 126 int numChannels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 127 if (channelIx < 0 || channelIx >= numChannels) { 128 return null; 129 } 130 short[] res = new short[samples.remaining() / numChannels]; 131 for (int i = 0; i < res.length; ++i) { 132 res[i] = samples.get(i * numChannels + channelIx); 133 } 134 return res; 135 }</pre> 136 137 <h4>Raw Video Buffers</h4> 138 <p> 139 In ByteBuffer mode video buffers are laid out according to their {@linkplain 140 MediaFormat#KEY_COLOR_FORMAT color format}. You can get the supported color formats as an array 141 from {@link #getCodecInfo}{@code .}{@link MediaCodecInfo#getCapabilitiesForType 142 getCapabilitiesForType(…)}{@code .}{@link CodecCapabilities#colorFormats colorFormats}. 143 Video codecs may support three kinds of color formats: 144 <ul> 145 <li><strong>native raw video format:</strong> This is marked by {@link 146 CodecCapabilities#COLOR_FormatSurface} and it can be used with an input or output Surface.</li> 147 <li><strong>flexible YUV buffers</strong> (such as {@link 148 CodecCapabilities#COLOR_FormatYUV420Flexible}): These can be used with an input/output Surface, 149 as well as in ByteBuffer mode, by using {@link #getInputImage getInput}/{@link #getOutputImage 150 OutputImage(int)}.</li> 151 <li><strong>other, specific formats:</strong> These are normally only supported in ByteBuffer 152 mode. Some color formats are vendor specific. Others are defined in {@link CodecCapabilities}. 153 For color formats that are equivalent to a flexible format, you can still use {@link 154 #getInputImage getInput}/{@link #getOutputImage OutputImage(int)}.</li> 155 </ul> 156 <p> 157 All video codecs support flexible YUV 4:2:0 buffers since {@link 158 android.os.Build.VERSION_CODES#LOLLIPOP_MR1}. 159 160 <h4>Accessing Raw Video ByteBuffers on Older Devices</h4> 161 <p> 162 Prior to {@link android.os.Build.VERSION_CODES#LOLLIPOP} and {@link Image} support, you need to 163 use the {@link MediaFormat#KEY_STRIDE} and {@link MediaFormat#KEY_SLICE_HEIGHT} output format 164 values to understand the layout of the raw output buffers. 165 <p class=note> 166 Note that on some devices the slice-height is advertised as 0. This could mean either that the 167 slice-height is the same as the frame height, or that the slice-height is the frame height 168 aligned to some value (usually a power of 2). Unfortunately, there is no standard and simple way 169 to tell the actual slice height in this case. Furthermore, the vertical stride of the {@code U} 170 plane in planar formats is also not specified or defined, though usually it is half of the slice 171 height. 172 <p> 173 The {@link MediaFormat#KEY_WIDTH} and {@link MediaFormat#KEY_HEIGHT} keys specify the size of the 174 video frames; however, for most encondings the video (picture) only occupies a portion of the 175 video frame. This is represented by the 'crop rectangle'. 176 <p> 177 You need to use the following keys to get the crop rectangle of raw output images from the 178 {@linkplain #getOutputFormat output format}. If these keys are not present, the video occupies the 179 entire video frame.The crop rectangle is understood in the context of the output frame 180 <em>before</em> applying any {@linkplain MediaFormat#KEY_ROTATION rotation}. 181 <table style="width: 0%"> 182 <thead> 183 <tr> 184 <th>Format Key</th> 185 <th>Type</th> 186 <th>Description</th> 187 </tr> 188 </thead> 189 <tbody> 190 <tr> 191 <td>{@code "crop-left"}</td> 192 <td>Integer</td> 193 <td>The left-coordinate (x) of the crop rectangle</td> 194 </tr><tr> 195 <td>{@code "crop-top"}</td> 196 <td>Integer</td> 197 <td>The top-coordinate (y) of the crop rectangle</td> 198 </tr><tr> 199 <td>{@code "crop-right"}</td> 200 <td>Integer</td> 201 <td>The right-coordinate (x) <strong>MINUS 1</strong> of the crop rectangle</td> 202 </tr><tr> 203 <td>{@code "crop-bottom"}</td> 204 <td>Integer</td> 205 <td>The bottom-coordinate (y) <strong>MINUS 1</strong> of the crop rectangle</td> 206 </tr><tr> 207 <td colspan=3> 208 The right and bottom coordinates can be understood as the coordinates of the right-most 209 valid column/bottom-most valid row of the cropped output image. 210 </td> 211 </tr> 212 </tbody> 213 </table> 214 <p> 215 The size of the video frame (before rotation) can be calculated as such: 216 <pre class=prettyprint> 217 MediaFormat format = decoder.getOutputFormat(…); 218 int width = format.getInteger(MediaFormat.KEY_WIDTH); 219 if (format.containsKey("crop-left") && format.containsKey("crop-right")) { 220 width = format.getInteger("crop-right") + 1 - format.getInteger("crop-left"); 221 } 222 int height = format.getInteger(MediaFormat.KEY_HEIGHT); 223 if (format.containsKey("crop-top") && format.containsKey("crop-bottom")) { 224 height = format.getInteger("crop-bottom") + 1 - format.getInteger("crop-top"); 225 } 226 </pre> 227 <p class=note> 228 Also note that the meaning of {@link BufferInfo#offset BufferInfo.offset} was not consistent across 229 devices. On some devices the offset pointed to the top-left pixel of the crop rectangle, while on 230 most devices it pointed to the top-left pixel of the entire frame. 231 232 <h3>States</h3> 233 <p> 234 During its life a codec conceptually exists in one of three states: Stopped, Executing or 235 Released. The Stopped collective state is actually the conglomeration of three states: 236 Uninitialized, Configured and Error, whereas the Executing state conceptually progresses through 237 three sub-states: Flushed, Running and End-of-Stream. 238 <p> 239 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 240 data="../../../images/media/mediacodec_states.svg"><img 241 src="../../../images/media/mediacodec_states.png" style="width: 519px; height: 356px" 242 alt="MediaCodec state diagram"></object></center> 243 <p> 244 When you create a codec using one of the factory methods, the codec is in the Uninitialized 245 state. First, you need to configure it via {@link #configure configure(…)}, which brings 246 it to the Configured state, then call {@link #start} to move it to the Executing state. In this 247 state you can process data through the buffer queue manipulation described above. 248 <p> 249 The Executing state has three sub-states: Flushed, Running and End-of-Stream. Immediately after 250 {@link #start} the codec is in the Flushed sub-state, where it holds all the buffers. As soon 251 as the first input buffer is dequeued, the codec moves to the Running sub-state, where it spends 252 most of its life. When you queue an input buffer with the {@linkplain #BUFFER_FLAG_END_OF_STREAM 253 end-of-stream marker}, the codec transitions to the End-of-Stream sub-state. In this state the 254 codec no longer accepts further input buffers, but still generates output buffers until the 255 end-of-stream is reached on the output. You can move back to the Flushed sub-state at any time 256 while in the Executing state using {@link #flush}. 257 <p> 258 Call {@link #stop} to return the codec to the Uninitialized state, whereupon it may be configured 259 again. When you are done using a codec, you must release it by calling {@link #release}. 260 <p> 261 On rare occasions the codec may encounter an error and move to the Error state. This is 262 communicated using an invalid return value from a queuing operation, or sometimes via an 263 exception. Call {@link #reset} to make the codec usable again. You can call it from any state to 264 move the codec back to the Uninitialized state. Otherwise, call {@link #release} to move to the 265 terminal Released state. 266 267 <h3>Creation</h3> 268 <p> 269 Use {@link MediaCodecList} to create a MediaCodec for a specific {@link MediaFormat}. When 270 decoding a file or a stream, you can get the desired format from {@link 271 MediaExtractor#getTrackFormat MediaExtractor.getTrackFormat}. Inject any specific features that 272 you want to add using {@link MediaFormat#setFeatureEnabled MediaFormat.setFeatureEnabled}, then 273 call {@link MediaCodecList#findDecoderForFormat MediaCodecList.findDecoderForFormat} to get the 274 name of a codec that can handle that specific media format. Finally, create the codec using 275 {@link #createByCodecName}. 276 <p class=note> 277 <strong>Note:</strong> On {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the format to 278 {@code MediaCodecList.findDecoder}/{@code EncoderForFormat} must not contain a {@linkplain 279 MediaFormat#KEY_FRAME_RATE frame rate}. Use 280 <code class=prettyprint>format.setString(MediaFormat.KEY_FRAME_RATE, null)</code> 281 to clear any existing frame rate setting in the format. 282 <p> 283 You can also create the preferred codec for a specific MIME type using {@link 284 #createDecoderByType createDecoder}/{@link #createEncoderByType EncoderByType(String)}. 285 This, however, cannot be used to inject features, and may create a codec that cannot handle the 286 specific desired media format. 287 288 <h4>Creating secure decoders</h4> 289 <p> 290 On versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and earlier, secure codecs might 291 not be listed in {@link MediaCodecList}, but may still be available on the system. Secure codecs 292 that exist can be instantiated by name only, by appending {@code ".secure"} to the name of a 293 regular codec (the name of all secure codecs must end in {@code ".secure"}.) {@link 294 #createByCodecName} will throw an {@code IOException} if the codec is not present on the system. 295 <p> 296 From {@link android.os.Build.VERSION_CODES#LOLLIPOP} onwards, you should use the {@link 297 CodecCapabilities#FEATURE_SecurePlayback} feature in the media format to create a secure decoder. 298 299 <h3>Initialization</h3> 300 <p> 301 After creating the codec, you can set a callback using {@link #setCallback setCallback} if you 302 want to process data asynchronously. Then, {@linkplain #configure configure} the codec using the 303 specific media format. This is when you can specify the output {@link Surface} for video 304 producers – codecs that generate raw video data (e.g. video decoders). This is also when 305 you can set the decryption parameters for secure codecs (see {@link MediaCrypto}). Finally, since 306 some codecs can operate in multiple modes, you must specify whether you want it to work as a 307 decoder or an encoder. 308 <p> 309 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you can query the resulting input and 310 output format in the Configured state. You can use this to verify the resulting configuration, 311 e.g. color formats, before starting the codec. 312 <p> 313 If you want to process raw input video buffers natively with a video consumer – a codec 314 that processes raw video input, such as a video encoder – create a destination Surface for 315 your input data using {@link #createInputSurface} after configuration. Alternately, set up the 316 codec to use a previously created {@linkplain #createPersistentInputSurface persistent input 317 surface} by calling {@link #setInputSurface}. 318 319 <h4 id=CSD><a name="CSD"></a>Codec-specific Data</h4> 320 <p> 321 Some formats, notably AAC audio and MPEG4, H.264 and H.265 video formats require the actual data 322 to be prefixed by a number of buffers containing setup data, or codec specific data. When 323 processing such compressed formats, this data must be submitted to the codec after {@link 324 #start} and before any frame data. Such data must be marked using the flag {@link 325 #BUFFER_FLAG_CODEC_CONFIG} in a call to {@link #queueInputBuffer queueInputBuffer}. 326 <p> 327 Codec-specific data can also be included in the format passed to {@link #configure configure} in 328 ByteBuffer entries with keys "csd-0", "csd-1", etc. These keys are always included in the track 329 {@link MediaFormat} obtained from the {@link MediaExtractor#getTrackFormat MediaExtractor}. 330 Codec-specific data in the format is automatically submitted to the codec upon {@link #start}; 331 you <strong>MUST NOT</strong> submit this data explicitly. If the format did not contain codec 332 specific data, you can choose to submit it using the specified number of buffers in the correct 333 order, according to the format requirements. In case of H.264 AVC, you can also concatenate all 334 codec-specific data and submit it as a single codec-config buffer. 335 <p> 336 Android uses the following codec-specific data buffers. These are also required to be set in 337 the track format for proper {@link MediaMuxer} track configuration. Each parameter set and the 338 codec-specific-data sections marked with (<sup>*</sup>) must start with a start code of 339 {@code "\x00\x00\x00\x01"}. 340 <p> 341 <style>td.NA { background: #ccc; } .mid > tr > td { vertical-align: middle; }</style> 342 <table> 343 <thead> 344 <th>Format</th> 345 <th>CSD buffer #0</th> 346 <th>CSD buffer #1</th> 347 <th>CSD buffer #2</th> 348 </thead> 349 <tbody class=mid> 350 <tr> 351 <td>AAC</td> 352 <td>Decoder-specific information from ESDS<sup>*</sup></td> 353 <td class=NA>Not Used</td> 354 <td class=NA>Not Used</td> 355 </tr> 356 <tr> 357 <td>VORBIS</td> 358 <td>Identification header</td> 359 <td>Setup header</td> 360 <td class=NA>Not Used</td> 361 </tr> 362 <tr> 363 <td>OPUS</td> 364 <td>Identification header</td> 365 <td>Pre-skip in nanosecs<br> 366 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)<br> 367 This overrides the pre-skip value in the identification header.</td> 368 <td>Seek Pre-roll in nanosecs<br> 369 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)</td> 370 </tr> 371 <tr> 372 <td>FLAC</td> 373 <td>"fLaC", the FLAC stream marker in ASCII,<br> 374 followed by the STREAMINFO block (the mandatory metadata block),<br> 375 optionally followed by any number of other metadata blocks</td> 376 <td class=NA>Not Used</td> 377 <td class=NA>Not Used</td> 378 </tr> 379 <tr> 380 <td>MPEG-4</td> 381 <td>Decoder-specific information from ESDS<sup>*</sup></td> 382 <td class=NA>Not Used</td> 383 <td class=NA>Not Used</td> 384 </tr> 385 <tr> 386 <td>H.264 AVC</td> 387 <td>SPS (Sequence Parameter Sets<sup>*</sup>)</td> 388 <td>PPS (Picture Parameter Sets<sup>*</sup>)</td> 389 <td class=NA>Not Used</td> 390 </tr> 391 <tr> 392 <td>H.265 HEVC</td> 393 <td>VPS (Video Parameter Sets<sup>*</sup>) +<br> 394 SPS (Sequence Parameter Sets<sup>*</sup>) +<br> 395 PPS (Picture Parameter Sets<sup>*</sup>)</td> 396 <td class=NA>Not Used</td> 397 <td class=NA>Not Used</td> 398 </tr> 399 <tr> 400 <td>VP9</td> 401 <td>VP9 <a href="http://wiki.webmproject.org/vp9-codecprivate">CodecPrivate</a> Data 402 (optional)</td> 403 <td class=NA>Not Used</td> 404 <td class=NA>Not Used</td> 405 </tr> 406 </tbody> 407 </table> 408 409 <p class=note> 410 <strong>Note:</strong> care must be taken if the codec is flushed immediately or shortly 411 after start, before any output buffer or output format change has been returned, as the codec 412 specific data may be lost during the flush. You must resubmit the data using buffers marked with 413 {@link #BUFFER_FLAG_CODEC_CONFIG} after such flush to ensure proper codec operation. 414 <p> 415 Encoders (or codecs that generate compressed data) will create and return the codec specific data 416 before any valid output buffer in output buffers marked with the {@linkplain 417 #BUFFER_FLAG_CODEC_CONFIG codec-config flag}. Buffers containing codec-specific-data have no 418 meaningful timestamps. 419 420 <h3>Data Processing</h3> 421 <p> 422 Each codec maintains a set of input and output buffers that are referred to by a buffer-ID in 423 API calls. After a successful call to {@link #start} the client "owns" neither input nor output 424 buffers. In synchronous mode, call {@link #dequeueInputBuffer dequeueInput}/{@link 425 #dequeueOutputBuffer OutputBuffer(…)} to obtain (get ownership of) an input or output 426 buffer from the codec. In asynchronous mode, you will automatically receive available buffers via 427 the {@link Callback#onInputBufferAvailable MediaCodec.Callback.onInput}/{@link 428 Callback#onOutputBufferAvailable OutputBufferAvailable(…)} callbacks. 429 <p> 430 Upon obtaining an input buffer, fill it with data and submit it to the codec using {@link 431 #queueInputBuffer queueInputBuffer} – or {@link #queueSecureInputBuffer 432 queueSecureInputBuffer} if using decryption. Do not submit multiple input buffers with the same 433 timestamp (unless it is <a href="#CSD">codec-specific data</a> marked as such). 434 <p> 435 The codec in turn will return a read-only output buffer via the {@link 436 Callback#onOutputBufferAvailable onOutputBufferAvailable} callback in asynchronous mode, or in 437 response to a {@link #dequeueOutputBuffer dequeueOutputBuffer} call in synchronous mode. After the 438 output buffer has been processed, call one of the {@link #releaseOutputBuffer 439 releaseOutputBuffer} methods to return the buffer to the codec. 440 <p> 441 While you are not required to resubmit/release buffers immediately to the codec, holding onto 442 input and/or output buffers may stall the codec, and this behavior is device dependent. 443 <strong>Specifically, it is possible that a codec may hold off on generating output buffers until 444 <em>all</em> outstanding buffers have been released/resubmitted.</strong> Therefore, try to 445 hold onto to available buffers as little as possible. 446 <p> 447 Depending on the API version, you can process data in three ways: 448 <table> 449 <thead> 450 <tr> 451 <th>Processing Mode</th> 452 <th>API version <= 20<br>Jelly Bean/KitKat</th> 453 <th>API version >= 21<br>Lollipop and later</th> 454 </tr> 455 </thead> 456 <tbody> 457 <tr> 458 <td>Synchronous API using buffer arrays</td> 459 <td>Supported</td> 460 <td>Deprecated</td> 461 </tr> 462 <tr> 463 <td>Synchronous API using buffers</td> 464 <td class=NA>Not Available</td> 465 <td>Supported</td> 466 </tr> 467 <tr> 468 <td>Asynchronous API using buffers</td> 469 <td class=NA>Not Available</td> 470 <td>Supported</td> 471 </tr> 472 </tbody> 473 </table> 474 475 <h4>Asynchronous Processing using Buffers</h4> 476 <p> 477 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the preferred method is to process data 478 asynchronously by setting a callback before calling {@link #configure configure}. Asynchronous 479 mode changes the state transitions slightly, because you must call {@link #start} after {@link 480 #flush} to transition the codec to the Running sub-state and start receiving input buffers. 481 Similarly, upon an initial call to {@code start} the codec will move directly to the Running 482 sub-state and start passing available input buffers via the callback. 483 <p> 484 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 485 data="../../../images/media/mediacodec_async_states.svg"><img 486 src="../../../images/media/mediacodec_async_states.png" style="width: 516px; height: 353px" 487 alt="MediaCodec state diagram for asynchronous operation"></object></center> 488 <p> 489 MediaCodec is typically used like this in asynchronous mode: 490 <pre class=prettyprint> 491 MediaCodec codec = MediaCodec.createByCodecName(name); 492 MediaFormat mOutputFormat; // member variable 493 codec.setCallback(new MediaCodec.Callback() { 494 {@literal @Override} 495 void onInputBufferAvailable(MediaCodec mc, int inputBufferId) { 496 ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); 497 // fill inputBuffer with valid data 498 … 499 codec.queueInputBuffer(inputBufferId, …); 500 } 501 502 {@literal @Override} 503 void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, …) { 504 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 505 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 506 // bufferFormat is equivalent to mOutputFormat 507 // outputBuffer is ready to be processed or rendered. 508 … 509 codec.releaseOutputBuffer(outputBufferId, …); 510 } 511 512 {@literal @Override} 513 void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { 514 // Subsequent data will conform to new format. 515 // Can ignore if using getOutputFormat(outputBufferId) 516 mOutputFormat = format; // option B 517 } 518 519 {@literal @Override} 520 void onError(…) { 521 … 522 } 523 }); 524 codec.configure(format, …); 525 mOutputFormat = codec.getOutputFormat(); // option B 526 codec.start(); 527 // wait for processing to complete 528 codec.stop(); 529 codec.release();</pre> 530 531 <h4>Synchronous Processing using Buffers</h4> 532 <p> 533 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you should retrieve input and output 534 buffers using {@link #getInputBuffer getInput}/{@link #getOutputBuffer OutputBuffer(int)} and/or 535 {@link #getInputImage getInput}/{@link #getOutputImage OutputImage(int)} even when using the 536 codec in synchronous mode. This allows certain optimizations by the framework, e.g. when 537 processing dynamic content. This optimization is disabled if you call {@link #getInputBuffers 538 getInput}/{@link #getOutputBuffers OutputBuffers()}. 539 540 <p class=note> 541 <strong>Note:</strong> do not mix the methods of using buffers and buffer arrays at the same 542 time. Specifically, only call {@code getInput}/{@code OutputBuffers} directly after {@link 543 #start} or after having dequeued an output buffer ID with the value of {@link 544 #INFO_OUTPUT_FORMAT_CHANGED}. 545 <p> 546 MediaCodec is typically used like this in synchronous mode: 547 <pre> 548 MediaCodec codec = MediaCodec.createByCodecName(name); 549 codec.configure(format, …); 550 MediaFormat outputFormat = codec.getOutputFormat(); // option B 551 codec.start(); 552 for (;;) { 553 int inputBufferId = codec.dequeueInputBuffer(timeoutUs); 554 if (inputBufferId >= 0) { 555 ByteBuffer inputBuffer = codec.getInputBuffer(…); 556 // fill inputBuffer with valid data 557 … 558 codec.queueInputBuffer(inputBufferId, …); 559 } 560 int outputBufferId = codec.dequeueOutputBuffer(…); 561 if (outputBufferId >= 0) { 562 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 563 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 564 // bufferFormat is identical to outputFormat 565 // outputBuffer is ready to be processed or rendered. 566 … 567 codec.releaseOutputBuffer(outputBufferId, …); 568 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 569 // Subsequent data will conform to new format. 570 // Can ignore if using getOutputFormat(outputBufferId) 571 outputFormat = codec.getOutputFormat(); // option B 572 } 573 } 574 codec.stop(); 575 codec.release();</pre> 576 577 <h4>Synchronous Processing using Buffer Arrays (deprecated)</h4> 578 <p> 579 In versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and before, the set of input and 580 output buffers are represented by the {@code ByteBuffer[]} arrays. After a successful call to 581 {@link #start}, retrieve the buffer arrays using {@link #getInputBuffers getInput}/{@link 582 #getOutputBuffers OutputBuffers()}. Use the buffer ID-s as indices into these arrays (when 583 non-negative), as demonstrated in the sample below. Note that there is no inherent correlation 584 between the size of the arrays and the number of input and output buffers used by the system, 585 although the array size provides an upper bound. 586 <pre> 587 MediaCodec codec = MediaCodec.createByCodecName(name); 588 codec.configure(format, …); 589 codec.start(); 590 ByteBuffer[] inputBuffers = codec.getInputBuffers(); 591 ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 592 for (;;) { 593 int inputBufferId = codec.dequeueInputBuffer(…); 594 if (inputBufferId >= 0) { 595 // fill inputBuffers[inputBufferId] with valid data 596 … 597 codec.queueInputBuffer(inputBufferId, …); 598 } 599 int outputBufferId = codec.dequeueOutputBuffer(…); 600 if (outputBufferId >= 0) { 601 // outputBuffers[outputBufferId] is ready to be processed or rendered. 602 … 603 codec.releaseOutputBuffer(outputBufferId, …); 604 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 605 outputBuffers = codec.getOutputBuffers(); 606 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 607 // Subsequent data will conform to new format. 608 MediaFormat format = codec.getOutputFormat(); 609 } 610 } 611 codec.stop(); 612 codec.release();</pre> 613 614 <h4>End-of-stream Handling</h4> 615 <p> 616 When you reach the end of the input data, you must signal it to the codec by specifying the 617 {@link #BUFFER_FLAG_END_OF_STREAM} flag in the call to {@link #queueInputBuffer 618 queueInputBuffer}. You can do this on the last valid input buffer, or by submitting an additional 619 empty input buffer with the end-of-stream flag set. If using an empty buffer, the timestamp will 620 be ignored. 621 <p> 622 The codec will continue to return output buffers until it eventually signals the end of the 623 output stream by specifying the same end-of-stream flag in the {@link BufferInfo} set in {@link 624 #dequeueOutputBuffer dequeueOutputBuffer} or returned via {@link Callback#onOutputBufferAvailable 625 onOutputBufferAvailable}. This can be set on the last valid output buffer, or on an empty buffer 626 after the last valid output buffer. The timestamp of such empty buffer should be ignored. 627 <p> 628 Do not submit additional input buffers after signaling the end of the input stream, unless the 629 codec has been flushed, or stopped and restarted. 630 631 <h4>Using an Output Surface</h4> 632 <p> 633 The data processing is nearly identical to the ByteBuffer mode when using an output {@link 634 Surface}; however, the output buffers will not be accessible, and are represented as {@code null} 635 values. E.g. {@link #getOutputBuffer getOutputBuffer}/{@link #getOutputImage Image(int)} will 636 return {@code null} and {@link #getOutputBuffers} will return an array containing only {@code 637 null}-s. 638 <p> 639 When using an output Surface, you can select whether or not to render each output buffer on the 640 surface. You have three choices: 641 <ul> 642 <li><strong>Do not render the buffer:</strong> Call {@link #releaseOutputBuffer(int, boolean) 643 releaseOutputBuffer(bufferId, false)}.</li> 644 <li><strong>Render the buffer with the default timestamp:</strong> Call {@link 645 #releaseOutputBuffer(int, boolean) releaseOutputBuffer(bufferId, true)}.</li> 646 <li><strong>Render the buffer with a specific timestamp:</strong> Call {@link 647 #releaseOutputBuffer(int, long) releaseOutputBuffer(bufferId, timestamp)}.</li> 648 </ul> 649 <p> 650 Since {@link android.os.Build.VERSION_CODES#M}, the default timestamp is the {@linkplain 651 BufferInfo#presentationTimeUs presentation timestamp} of the buffer (converted to nanoseconds). 652 It was not defined prior to that. 653 <p> 654 Also since {@link android.os.Build.VERSION_CODES#M}, you can change the output Surface 655 dynamically using {@link #setOutputSurface setOutputSurface}. 656 <p> 657 When rendering output to a Surface, the Surface may be configured to drop excessive frames (that 658 are not consumed by the Surface in a timely manner). Or it may be configured to not drop excessive 659 frames. In the latter mode if the Surface is not consuming output frames fast enough, it will 660 eventually block the decoder. Prior to {@link android.os.Build.VERSION_CODES#Q} the exact behavior 661 was undefined, with the exception that View surfaces (SurfaceView or TextureView) always dropped 662 excessive frames. Since {@link android.os.Build.VERSION_CODES#Q} the default behavior is to drop 663 excessive frames. Applications can opt out of this behavior for non-View surfaces (such as 664 ImageReader or SurfaceTexture) by targeting SDK {@link android.os.Build.VERSION_CODES#Q} and 665 setting the key {@code "allow-frame-drop"} to {@code 0} in their configure format. 666 667 <h4>Transformations When Rendering onto Surface</h4> 668 669 If the codec is configured into Surface mode, any crop rectangle, {@linkplain 670 MediaFormat#KEY_ROTATION rotation} and {@linkplain #setVideoScalingMode video scaling 671 mode} will be automatically applied with one exception: 672 <p class=note> 673 Prior to the {@link android.os.Build.VERSION_CODES#M} release, software decoders may not 674 have applied the rotation when being rendered onto a Surface. Unfortunately, there is no standard 675 and simple way to identify software decoders, or if they apply the rotation other than by trying 676 it out. 677 <p> 678 There are also some caveats. 679 <p class=note> 680 Note that the pixel aspect ratio is not considered when displaying the output onto the 681 Surface. This means that if you are using {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT} mode, you 682 must position the output Surface so that it has the proper final display aspect ratio. Conversely, 683 you can only use {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode for content with 684 square pixels (pixel aspect ratio or 1:1). 685 <p class=note> 686 Note also that as of {@link android.os.Build.VERSION_CODES#N} release, {@link 687 #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode may not work correctly for videos rotated 688 by 90 or 270 degrees. 689 <p class=note> 690 When setting the video scaling mode, note that it must be reset after each time the output 691 buffers change. Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, you can 692 do this after each time the output format changes. 693 694 <h4>Using an Input Surface</h4> 695 <p> 696 When using an input Surface, there are no accessible input buffers, as buffers are automatically 697 passed from the input surface to the codec. Calling {@link #dequeueInputBuffer 698 dequeueInputBuffer} will throw an {@code IllegalStateException}, and {@link #getInputBuffers} 699 returns a bogus {@code ByteBuffer[]} array that <strong>MUST NOT</strong> be written into. 700 <p> 701 Call {@link #signalEndOfInputStream} to signal end-of-stream. The input surface will stop 702 submitting data to the codec immediately after this call. 703 <p> 704 705 <h3>Seeking & Adaptive Playback Support</h3> 706 <p> 707 Video decoders (and in general codecs that consume compressed video data) behave differently 708 regarding seek and format change whether or not they support and are configured for adaptive 709 playback. You can check if a decoder supports {@linkplain 710 CodecCapabilities#FEATURE_AdaptivePlayback adaptive playback} via {@link 711 CodecCapabilities#isFeatureSupported CodecCapabilities.isFeatureSupported(String)}. Adaptive 712 playback support for video decoders is only activated if you configure the codec to decode onto a 713 {@link Surface}. 714 715 <h4 id=KeyFrames><a name="KeyFrames"></a>Stream Boundary and Key Frames</h4> 716 <p> 717 It is important that the input data after {@link #start} or {@link #flush} starts at a suitable 718 stream boundary: the first frame must a key frame. A <em>key frame</em> can be decoded 719 completely on its own (for most codecs this means an I-frame), and no frames that are to be 720 displayed after a key frame refer to frames before the key frame. 721 <p> 722 The following table summarizes suitable key frames for various video formats. 723 <table> 724 <thead> 725 <tr> 726 <th>Format</th> 727 <th>Suitable key frame</th> 728 </tr> 729 </thead> 730 <tbody class=mid> 731 <tr> 732 <td>VP9/VP8</td> 733 <td>a suitable intraframe where no subsequent frames refer to frames prior to this frame.<br> 734 <i>(There is no specific name for such key frame.)</i></td> 735 </tr> 736 <tr> 737 <td>H.265 HEVC</td> 738 <td>IDR or CRA</td> 739 </tr> 740 <tr> 741 <td>H.264 AVC</td> 742 <td>IDR</td> 743 </tr> 744 <tr> 745 <td>MPEG-4<br>H.263<br>MPEG-2</td> 746 <td>a suitable I-frame where no subsequent frames refer to frames prior to this frame.<br> 747 <i>(There is no specific name for such key frame.)</td> 748 </tr> 749 </tbody> 750 </table> 751 752 <h4>For decoders that do not support adaptive playback (including when not decoding onto a 753 Surface)</h4> 754 <p> 755 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 756 seek) you <strong>MUST</strong> flush the decoder. Since all output buffers are immediately 757 revoked at the point of the flush, you may want to first signal then wait for the end-of-stream 758 before you call {@code flush}. It is important that the input data after a flush starts at a 759 suitable stream boundary/key frame. 760 <p class=note> 761 <strong>Note:</strong> the format of the data submitted after a flush must not change; {@link 762 #flush} does not support format discontinuities; for that, a full {@link #stop} - {@link 763 #configure configure(…)} - {@link #start} cycle is necessary. 764 765 <p class=note> 766 <strong>Also note:</strong> if you flush the codec too soon after {@link #start} – 767 generally, before the first output buffer or output format change is received – you 768 will need to resubmit the codec-specific-data to the codec. See the <a 769 href="#CSD">codec-specific-data section</a> for more info. 770 771 <h4>For decoders that support and are configured for adaptive playback</h4> 772 <p> 773 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 774 seek) it is <em>not necessary</em> to flush the decoder; however, input data after the 775 discontinuity must start at a suitable stream boundary/key frame. 776 <p> 777 For some video formats - namely H.264, H.265, VP8 and VP9 - it is also possible to change the 778 picture size or configuration mid-stream. To do this you must package the entire new 779 codec-specific configuration data together with the key frame into a single buffer (including 780 any start codes), and submit it as a <strong>regular</strong> input buffer. 781 <p> 782 You will receive an {@link #INFO_OUTPUT_FORMAT_CHANGED} return value from {@link 783 #dequeueOutputBuffer dequeueOutputBuffer} or a {@link Callback#onOutputBufferAvailable 784 onOutputFormatChanged} callback just after the picture-size change takes place and before any 785 frames with the new size have been returned. 786 <p class=note> 787 <strong>Note:</strong> just as the case for codec-specific data, be careful when calling 788 {@link #flush} shortly after you have changed the picture size. If you have not received 789 confirmation of the picture size change, you will need to repeat the request for the new picture 790 size. 791 792 <h3>Error handling</h3> 793 <p> 794 The factory methods {@link #createByCodecName createByCodecName} and {@link #createDecoderByType 795 createDecoder}/{@link #createEncoderByType EncoderByType} throw {@code IOException} on failure 796 which you must catch or declare to pass up. MediaCodec methods throw {@code 797 IllegalStateException} when the method is called from a codec state that does not allow it; this 798 is typically due to incorrect application API usage. Methods involving secure buffers may throw 799 {@link CryptoException}, which has further error information obtainable from {@link 800 CryptoException#getErrorCode}. 801 <p> 802 Internal codec errors result in a {@link CodecException}, which may be due to media content 803 corruption, hardware failure, resource exhaustion, and so forth, even when the application is 804 correctly using the API. The recommended action when receiving a {@code CodecException} 805 can be determined by calling {@link CodecException#isRecoverable} and {@link 806 CodecException#isTransient}: 807 <ul> 808 <li><strong>recoverable errors:</strong> If {@code isRecoverable()} returns true, then call 809 {@link #stop}, {@link #configure configure(…)}, and {@link #start} to recover.</li> 810 <li><strong>transient errors:</strong> If {@code isTransient()} returns true, then resources are 811 temporarily unavailable and the method may be retried at a later time.</li> 812 <li><strong>fatal errors:</strong> If both {@code isRecoverable()} and {@code isTransient()} 813 return false, then the {@code CodecException} is fatal and the codec must be {@linkplain #reset 814 reset} or {@linkplain #release released}.</li> 815 </ul> 816 <p> 817 Both {@code isRecoverable()} and {@code isTransient()} do not return true at the same time. 818 819 <h2 id=History><a name="History"></a>Valid API Calls and API History</h2> 820 <p> 821 This sections summarizes the valid API calls in each state and the API history of the MediaCodec 822 class. For API version numbers, see {@link android.os.Build.VERSION_CODES}. 823 824 <style> 825 .api > tr > th, .api > tr > td { text-align: center; padding: 4px 4px; } 826 .api > tr > th { vertical-align: bottom; } 827 .api > tr > td { vertical-align: middle; } 828 .sml > tr > th, .sml > tr > td { text-align: center; padding: 2px 4px; } 829 .fn { text-align: left; } 830 .fn > code > a { font: 14px/19px Roboto Condensed, sans-serif; } 831 .deg45 { 832 white-space: nowrap; background: none; border: none; vertical-align: bottom; 833 width: 30px; height: 83px; 834 } 835 .deg45 > div { 836 transform: skew(-45deg, 0deg) translate(1px, -67px); 837 transform-origin: bottom left 0; 838 width: 30px; height: 20px; 839 } 840 .deg45 > div > div { border: 1px solid #ddd; background: #999; height: 90px; width: 42px; } 841 .deg45 > div > div > div { transform: skew(45deg, 0deg) translate(-55px, 55px) rotate(-45deg); } 842 </style> 843 844 <table align="right" style="width: 0%"> 845 <thead> 846 <tr><th>Symbol</th><th>Meaning</th></tr> 847 </thead> 848 <tbody class=sml> 849 <tr><td>●</td><td>Supported</td></tr> 850 <tr><td>⁕</td><td>Semantics changed</td></tr> 851 <tr><td>○</td><td>Experimental support</td></tr> 852 <tr><td>[ ]</td><td>Deprecated</td></tr> 853 <tr><td>⎋</td><td>Restricted to surface input mode</td></tr> 854 <tr><td>⎆</td><td>Restricted to surface output mode</td></tr> 855 <tr><td>▧</td><td>Restricted to ByteBuffer input mode</td></tr> 856 <tr><td>↩</td><td>Restricted to synchronous mode</td></tr> 857 <tr><td>⇄</td><td>Restricted to asynchronous mode</td></tr> 858 <tr><td>( )</td><td>Can be called, but shouldn't</td></tr> 859 </tbody> 860 </table> 861 862 <table style="width: 100%;"> 863 <thead class=api> 864 <tr> 865 <th class=deg45><div><div style="background:#4285f4"><div>Uninitialized</div></div></div></th> 866 <th class=deg45><div><div style="background:#f4b400"><div>Configured</div></div></div></th> 867 <th class=deg45><div><div style="background:#e67c73"><div>Flushed</div></div></div></th> 868 <th class=deg45><div><div style="background:#0f9d58"><div>Running</div></div></div></th> 869 <th class=deg45><div><div style="background:#f7cb4d"><div>End of Stream</div></div></div></th> 870 <th class=deg45><div><div style="background:#db4437"><div>Error</div></div></div></th> 871 <th class=deg45><div><div style="background:#666"><div>Released</div></div></div></th> 872 <th></th> 873 <th colspan="8">SDK Version</th> 874 </tr> 875 <tr> 876 <th colspan="7">State</th> 877 <th>Method</th> 878 <th>16</th> 879 <th>17</th> 880 <th>18</th> 881 <th>19</th> 882 <th>20</th> 883 <th>21</th> 884 <th>22</th> 885 <th>23</th> 886 </tr> 887 </thead> 888 <tbody class=api> 889 <tr> 890 <td></td> 891 <td></td> 892 <td></td> 893 <td></td> 894 <td></td> 895 <td></td> 896 <td></td> 897 <td class=fn>{@link #createByCodecName createByCodecName}</td> 898 <td>●</td> 899 <td>●</td> 900 <td>●</td> 901 <td>●</td> 902 <td>●</td> 903 <td>●</td> 904 <td>●</td> 905 <td>●</td> 906 </tr> 907 <tr> 908 <td></td> 909 <td></td> 910 <td></td> 911 <td></td> 912 <td></td> 913 <td></td> 914 <td></td> 915 <td class=fn>{@link #createDecoderByType createDecoderByType}</td> 916 <td>●</td> 917 <td>●</td> 918 <td>●</td> 919 <td>●</td> 920 <td>●</td> 921 <td>●</td> 922 <td>●</td> 923 <td>●</td> 924 </tr> 925 <tr> 926 <td></td> 927 <td></td> 928 <td></td> 929 <td></td> 930 <td></td> 931 <td></td> 932 <td></td> 933 <td class=fn>{@link #createEncoderByType createEncoderByType}</td> 934 <td>●</td> 935 <td>●</td> 936 <td>●</td> 937 <td>●</td> 938 <td>●</td> 939 <td>●</td> 940 <td>●</td> 941 <td>●</td> 942 </tr> 943 <tr> 944 <td></td> 945 <td></td> 946 <td></td> 947 <td></td> 948 <td></td> 949 <td></td> 950 <td></td> 951 <td class=fn>{@link #createPersistentInputSurface createPersistentInputSurface}</td> 952 <td></td> 953 <td></td> 954 <td></td> 955 <td></td> 956 <td></td> 957 <td></td> 958 <td></td> 959 <td>●</td> 960 </tr> 961 <tr> 962 <td>16+</td> 963 <td>-</td> 964 <td>-</td> 965 <td>-</td> 966 <td>-</td> 967 <td>-</td> 968 <td>-</td> 969 <td class=fn>{@link #configure configure}</td> 970 <td>●</td> 971 <td>●</td> 972 <td>●</td> 973 <td>●</td> 974 <td>●</td> 975 <td>⁕</td> 976 <td>●</td> 977 <td>●</td> 978 </tr> 979 <tr> 980 <td>-</td> 981 <td>18+</td> 982 <td>-</td> 983 <td>-</td> 984 <td>-</td> 985 <td>-</td> 986 <td>-</td> 987 <td class=fn>{@link #createInputSurface createInputSurface}</td> 988 <td></td> 989 <td></td> 990 <td>⎋</td> 991 <td>⎋</td> 992 <td>⎋</td> 993 <td>⎋</td> 994 <td>⎋</td> 995 <td>⎋</td> 996 </tr> 997 <tr> 998 <td>-</td> 999 <td>-</td> 1000 <td>16+</td> 1001 <td>16+</td> 1002 <td>(16+)</td> 1003 <td>-</td> 1004 <td>-</td> 1005 <td class=fn>{@link #dequeueInputBuffer dequeueInputBuffer}</td> 1006 <td>●</td> 1007 <td>●</td> 1008 <td>▧</td> 1009 <td>▧</td> 1010 <td>▧</td> 1011 <td>⁕▧↩</td> 1012 <td>▧↩</td> 1013 <td>▧↩</td> 1014 </tr> 1015 <tr> 1016 <td>-</td> 1017 <td>-</td> 1018 <td>16+</td> 1019 <td>16+</td> 1020 <td>16+</td> 1021 <td>-</td> 1022 <td>-</td> 1023 <td class=fn>{@link #dequeueOutputBuffer dequeueOutputBuffer}</td> 1024 <td>●</td> 1025 <td>●</td> 1026 <td>●</td> 1027 <td>●</td> 1028 <td>●</td> 1029 <td>⁕↩</td> 1030 <td>↩</td> 1031 <td>↩</td> 1032 </tr> 1033 <tr> 1034 <td>-</td> 1035 <td>-</td> 1036 <td>16+</td> 1037 <td>16+</td> 1038 <td>16+</td> 1039 <td>-</td> 1040 <td>-</td> 1041 <td class=fn>{@link #flush flush}</td> 1042 <td>●</td> 1043 <td>●</td> 1044 <td>●</td> 1045 <td>●</td> 1046 <td>●</td> 1047 <td>●</td> 1048 <td>●</td> 1049 <td>●</td> 1050 </tr> 1051 <tr> 1052 <td>18+</td> 1053 <td>18+</td> 1054 <td>18+</td> 1055 <td>18+</td> 1056 <td>18+</td> 1057 <td>18+</td> 1058 <td>-</td> 1059 <td class=fn>{@link #getCodecInfo getCodecInfo}</td> 1060 <td></td> 1061 <td></td> 1062 <td>●</td> 1063 <td>●</td> 1064 <td>●</td> 1065 <td>●</td> 1066 <td>●</td> 1067 <td>●</td> 1068 </tr> 1069 <tr> 1070 <td>-</td> 1071 <td>-</td> 1072 <td>(21+)</td> 1073 <td>21+</td> 1074 <td>(21+)</td> 1075 <td>-</td> 1076 <td>-</td> 1077 <td class=fn>{@link #getInputBuffer getInputBuffer}</td> 1078 <td></td> 1079 <td></td> 1080 <td></td> 1081 <td></td> 1082 <td></td> 1083 <td>●</td> 1084 <td>●</td> 1085 <td>●</td> 1086 </tr> 1087 <tr> 1088 <td>-</td> 1089 <td>-</td> 1090 <td>16+</td> 1091 <td>(16+)</td> 1092 <td>(16+)</td> 1093 <td>-</td> 1094 <td>-</td> 1095 <td class=fn>{@link #getInputBuffers getInputBuffers}</td> 1096 <td>●</td> 1097 <td>●</td> 1098 <td>●</td> 1099 <td>●</td> 1100 <td>●</td> 1101 <td>[⁕↩]</td> 1102 <td>[↩]</td> 1103 <td>[↩]</td> 1104 </tr> 1105 <tr> 1106 <td>-</td> 1107 <td>21+</td> 1108 <td>(21+)</td> 1109 <td>(21+)</td> 1110 <td>(21+)</td> 1111 <td>-</td> 1112 <td>-</td> 1113 <td class=fn>{@link #getInputFormat getInputFormat}</td> 1114 <td></td> 1115 <td></td> 1116 <td></td> 1117 <td></td> 1118 <td></td> 1119 <td>●</td> 1120 <td>●</td> 1121 <td>●</td> 1122 </tr> 1123 <tr> 1124 <td>-</td> 1125 <td>-</td> 1126 <td>(21+)</td> 1127 <td>21+</td> 1128 <td>(21+)</td> 1129 <td>-</td> 1130 <td>-</td> 1131 <td class=fn>{@link #getInputImage getInputImage}</td> 1132 <td></td> 1133 <td></td> 1134 <td></td> 1135 <td></td> 1136 <td></td> 1137 <td>○</td> 1138 <td>●</td> 1139 <td>●</td> 1140 </tr> 1141 <tr> 1142 <td>18+</td> 1143 <td>18+</td> 1144 <td>18+</td> 1145 <td>18+</td> 1146 <td>18+</td> 1147 <td>18+</td> 1148 <td>-</td> 1149 <td class=fn>{@link #getName getName}</td> 1150 <td></td> 1151 <td></td> 1152 <td>●</td> 1153 <td>●</td> 1154 <td>●</td> 1155 <td>●</td> 1156 <td>●</td> 1157 <td>●</td> 1158 </tr> 1159 <tr> 1160 <td>-</td> 1161 <td>-</td> 1162 <td>(21+)</td> 1163 <td>21+</td> 1164 <td>21+</td> 1165 <td>-</td> 1166 <td>-</td> 1167 <td class=fn>{@link #getOutputBuffer getOutputBuffer}</td> 1168 <td></td> 1169 <td></td> 1170 <td></td> 1171 <td></td> 1172 <td></td> 1173 <td>●</td> 1174 <td>●</td> 1175 <td>●</td> 1176 </tr> 1177 <tr> 1178 <td>-</td> 1179 <td>-</td> 1180 <td>16+</td> 1181 <td>16+</td> 1182 <td>16+</td> 1183 <td>-</td> 1184 <td>-</td> 1185 <td class=fn>{@link #getOutputBuffers getOutputBuffers}</td> 1186 <td>●</td> 1187 <td>●</td> 1188 <td>●</td> 1189 <td>●</td> 1190 <td>●</td> 1191 <td>[⁕↩]</td> 1192 <td>[↩]</td> 1193 <td>[↩]</td> 1194 </tr> 1195 <tr> 1196 <td>-</td> 1197 <td>21+</td> 1198 <td>16+</td> 1199 <td>16+</td> 1200 <td>16+</td> 1201 <td>-</td> 1202 <td>-</td> 1203 <td class=fn>{@link #getOutputFormat()}</td> 1204 <td>●</td> 1205 <td>●</td> 1206 <td>●</td> 1207 <td>●</td> 1208 <td>●</td> 1209 <td>●</td> 1210 <td>●</td> 1211 <td>●</td> 1212 </tr> 1213 <tr> 1214 <td>-</td> 1215 <td>-</td> 1216 <td>(21+)</td> 1217 <td>21+</td> 1218 <td>21+</td> 1219 <td>-</td> 1220 <td>-</td> 1221 <td class=fn>{@link #getOutputFormat(int)}</td> 1222 <td></td> 1223 <td></td> 1224 <td></td> 1225 <td></td> 1226 <td></td> 1227 <td>●</td> 1228 <td>●</td> 1229 <td>●</td> 1230 </tr> 1231 <tr> 1232 <td>-</td> 1233 <td>-</td> 1234 <td>(21+)</td> 1235 <td>21+</td> 1236 <td>21+</td> 1237 <td>-</td> 1238 <td>-</td> 1239 <td class=fn>{@link #getOutputImage getOutputImage}</td> 1240 <td></td> 1241 <td></td> 1242 <td></td> 1243 <td></td> 1244 <td></td> 1245 <td>○</td> 1246 <td>●</td> 1247 <td>●</td> 1248 </tr> 1249 <tr> 1250 <td>-</td> 1251 <td>-</td> 1252 <td>-</td> 1253 <td>16+</td> 1254 <td>(16+)</td> 1255 <td>-</td> 1256 <td>-</td> 1257 <td class=fn>{@link #queueInputBuffer queueInputBuffer}</td> 1258 <td>●</td> 1259 <td>●</td> 1260 <td>●</td> 1261 <td>●</td> 1262 <td>●</td> 1263 <td>⁕</td> 1264 <td>●</td> 1265 <td>●</td> 1266 </tr> 1267 <tr> 1268 <td>-</td> 1269 <td>-</td> 1270 <td>-</td> 1271 <td>16+</td> 1272 <td>(16+)</td> 1273 <td>-</td> 1274 <td>-</td> 1275 <td class=fn>{@link #queueSecureInputBuffer queueSecureInputBuffer}</td> 1276 <td>●</td> 1277 <td>●</td> 1278 <td>●</td> 1279 <td>●</td> 1280 <td>●</td> 1281 <td>⁕</td> 1282 <td>●</td> 1283 <td>●</td> 1284 </tr> 1285 <tr> 1286 <td>16+</td> 1287 <td>16+</td> 1288 <td>16+</td> 1289 <td>16+</td> 1290 <td>16+</td> 1291 <td>16+</td> 1292 <td>16+</td> 1293 <td class=fn>{@link #release release}</td> 1294 <td>●</td> 1295 <td>●</td> 1296 <td>●</td> 1297 <td>●</td> 1298 <td>●</td> 1299 <td>●</td> 1300 <td>●</td> 1301 <td>●</td> 1302 </tr> 1303 <tr> 1304 <td>-</td> 1305 <td>-</td> 1306 <td>-</td> 1307 <td>16+</td> 1308 <td>16+</td> 1309 <td>-</td> 1310 <td>-</td> 1311 <td class=fn>{@link #releaseOutputBuffer(int, boolean)}</td> 1312 <td>●</td> 1313 <td>●</td> 1314 <td>●</td> 1315 <td>●</td> 1316 <td>●</td> 1317 <td>⁕</td> 1318 <td>●</td> 1319 <td>⁕</td> 1320 </tr> 1321 <tr> 1322 <td>-</td> 1323 <td>-</td> 1324 <td>-</td> 1325 <td>21+</td> 1326 <td>21+</td> 1327 <td>-</td> 1328 <td>-</td> 1329 <td class=fn>{@link #releaseOutputBuffer(int, long)}</td> 1330 <td></td> 1331 <td></td> 1332 <td></td> 1333 <td></td> 1334 <td></td> 1335 <td>⎆</td> 1336 <td>⎆</td> 1337 <td>⎆</td> 1338 </tr> 1339 <tr> 1340 <td>21+</td> 1341 <td>21+</td> 1342 <td>21+</td> 1343 <td>21+</td> 1344 <td>21+</td> 1345 <td>21+</td> 1346 <td>-</td> 1347 <td class=fn>{@link #reset reset}</td> 1348 <td></td> 1349 <td></td> 1350 <td></td> 1351 <td></td> 1352 <td></td> 1353 <td>●</td> 1354 <td>●</td> 1355 <td>●</td> 1356 </tr> 1357 <tr> 1358 <td>21+</td> 1359 <td>-</td> 1360 <td>-</td> 1361 <td>-</td> 1362 <td>-</td> 1363 <td>-</td> 1364 <td>-</td> 1365 <td class=fn>{@link #setCallback(Callback) setCallback}</td> 1366 <td></td> 1367 <td></td> 1368 <td></td> 1369 <td></td> 1370 <td></td> 1371 <td>●</td> 1372 <td>●</td> 1373 <td>{@link #setCallback(Callback, Handler) ⁕}</td> 1374 </tr> 1375 <tr> 1376 <td>-</td> 1377 <td>23+</td> 1378 <td>-</td> 1379 <td>-</td> 1380 <td>-</td> 1381 <td>-</td> 1382 <td>-</td> 1383 <td class=fn>{@link #setInputSurface setInputSurface}</td> 1384 <td></td> 1385 <td></td> 1386 <td></td> 1387 <td></td> 1388 <td></td> 1389 <td></td> 1390 <td></td> 1391 <td>⎋</td> 1392 </tr> 1393 <tr> 1394 <td>23+</td> 1395 <td>23+</td> 1396 <td>23+</td> 1397 <td>23+</td> 1398 <td>23+</td> 1399 <td>(23+)</td> 1400 <td>(23+)</td> 1401 <td class=fn>{@link #setOnFrameRenderedListener setOnFrameRenderedListener}</td> 1402 <td></td> 1403 <td></td> 1404 <td></td> 1405 <td></td> 1406 <td></td> 1407 <td></td> 1408 <td></td> 1409 <td>○ ⎆</td> 1410 </tr> 1411 <tr> 1412 <td>-</td> 1413 <td>23+</td> 1414 <td>23+</td> 1415 <td>23+</td> 1416 <td>23+</td> 1417 <td>-</td> 1418 <td>-</td> 1419 <td class=fn>{@link #setOutputSurface setOutputSurface}</td> 1420 <td></td> 1421 <td></td> 1422 <td></td> 1423 <td></td> 1424 <td></td> 1425 <td></td> 1426 <td></td> 1427 <td>⎆</td> 1428 </tr> 1429 <tr> 1430 <td>19+</td> 1431 <td>19+</td> 1432 <td>19+</td> 1433 <td>19+</td> 1434 <td>19+</td> 1435 <td>(19+)</td> 1436 <td>-</td> 1437 <td class=fn>{@link #setParameters setParameters}</td> 1438 <td></td> 1439 <td></td> 1440 <td></td> 1441 <td>●</td> 1442 <td>●</td> 1443 <td>●</td> 1444 <td>●</td> 1445 <td>●</td> 1446 </tr> 1447 <tr> 1448 <td>-</td> 1449 <td>(16+)</td> 1450 <td>(16+)</td> 1451 <td>16+</td> 1452 <td>(16+)</td> 1453 <td>(16+)</td> 1454 <td>-</td> 1455 <td class=fn>{@link #setVideoScalingMode setVideoScalingMode}</td> 1456 <td>⎆</td> 1457 <td>⎆</td> 1458 <td>⎆</td> 1459 <td>⎆</td> 1460 <td>⎆</td> 1461 <td>⎆</td> 1462 <td>⎆</td> 1463 <td>⎆</td> 1464 </tr> 1465 <tr> 1466 <td>(29+)</td> 1467 <td>29+</td> 1468 <td>29+</td> 1469 <td>29+</td> 1470 <td>(29+)</td> 1471 <td>(29+)</td> 1472 <td>-</td> 1473 <td class=fn>{@link #setAudioPresentation setAudioPresentation}</td> 1474 <td></td> 1475 <td></td> 1476 <td></td> 1477 <td></td> 1478 <td></td> 1479 <td></td> 1480 <td></td> 1481 <td></td> 1482 </tr> 1483 <tr> 1484 <td>-</td> 1485 <td>-</td> 1486 <td>18+</td> 1487 <td>18+</td> 1488 <td>-</td> 1489 <td>-</td> 1490 <td>-</td> 1491 <td class=fn>{@link #signalEndOfInputStream signalEndOfInputStream}</td> 1492 <td></td> 1493 <td></td> 1494 <td>⎋</td> 1495 <td>⎋</td> 1496 <td>⎋</td> 1497 <td>⎋</td> 1498 <td>⎋</td> 1499 <td>⎋</td> 1500 </tr> 1501 <tr> 1502 <td>-</td> 1503 <td>16+</td> 1504 <td>21+(⇄)</td> 1505 <td>-</td> 1506 <td>-</td> 1507 <td>-</td> 1508 <td>-</td> 1509 <td class=fn>{@link #start start}</td> 1510 <td>●</td> 1511 <td>●</td> 1512 <td>●</td> 1513 <td>●</td> 1514 <td>●</td> 1515 <td>⁕</td> 1516 <td>●</td> 1517 <td>●</td> 1518 </tr> 1519 <tr> 1520 <td>-</td> 1521 <td>-</td> 1522 <td>16+</td> 1523 <td>16+</td> 1524 <td>16+</td> 1525 <td>-</td> 1526 <td>-</td> 1527 <td class=fn>{@link #stop stop}</td> 1528 <td>●</td> 1529 <td>●</td> 1530 <td>●</td> 1531 <td>●</td> 1532 <td>●</td> 1533 <td>●</td> 1534 <td>●</td> 1535 <td>●</td> 1536 </tr> 1537 </tbody> 1538 </table> 1539 */ 1540 final public class MediaCodec { 1541 /** 1542 * Per buffer metadata includes an offset and size specifying 1543 * the range of valid data in the associated codec (output) buffer. 1544 */ 1545 public final static class BufferInfo { 1546 /** 1547 * Update the buffer metadata information. 1548 * 1549 * @param newOffset the start-offset of the data in the buffer. 1550 * @param newSize the amount of data (in bytes) in the buffer. 1551 * @param newTimeUs the presentation timestamp in microseconds. 1552 * @param newFlags buffer flags associated with the buffer. This 1553 * should be a combination of {@link #BUFFER_FLAG_KEY_FRAME} and 1554 * {@link #BUFFER_FLAG_END_OF_STREAM}. 1555 */ set( int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags)1556 public void set( 1557 int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags) { 1558 offset = newOffset; 1559 size = newSize; 1560 presentationTimeUs = newTimeUs; 1561 flags = newFlags; 1562 } 1563 1564 /** 1565 * The start-offset of the data in the buffer. 1566 */ 1567 public int offset; 1568 1569 /** 1570 * The amount of data (in bytes) in the buffer. If this is {@code 0}, 1571 * the buffer has no data in it and can be discarded. The only 1572 * use of a 0-size buffer is to carry the end-of-stream marker. 1573 */ 1574 public int size; 1575 1576 /** 1577 * The presentation timestamp in microseconds for the buffer. 1578 * This is derived from the presentation timestamp passed in 1579 * with the corresponding input buffer. This should be ignored for 1580 * a 0-sized buffer. 1581 */ 1582 public long presentationTimeUs; 1583 1584 /** 1585 * Buffer flags associated with the buffer. A combination of 1586 * {@link #BUFFER_FLAG_KEY_FRAME} and {@link #BUFFER_FLAG_END_OF_STREAM}. 1587 * 1588 * <p>Encoded buffers that are key frames are marked with 1589 * {@link #BUFFER_FLAG_KEY_FRAME}. 1590 * 1591 * <p>The last output buffer corresponding to the input buffer 1592 * marked with {@link #BUFFER_FLAG_END_OF_STREAM} will also be marked 1593 * with {@link #BUFFER_FLAG_END_OF_STREAM}. In some cases this could 1594 * be an empty buffer, whose sole purpose is to carry the end-of-stream 1595 * marker. 1596 */ 1597 @BufferFlag 1598 public int flags; 1599 1600 /** @hide */ 1601 @NonNull dup()1602 public BufferInfo dup() { 1603 BufferInfo copy = new BufferInfo(); 1604 copy.set(offset, size, presentationTimeUs, flags); 1605 return copy; 1606 } 1607 }; 1608 1609 // The follow flag constants MUST stay in sync with their equivalents 1610 // in MediaCodec.h ! 1611 1612 /** 1613 * This indicates that the (encoded) buffer marked as such contains 1614 * the data for a key frame. 1615 * 1616 * @deprecated Use {@link #BUFFER_FLAG_KEY_FRAME} instead. 1617 */ 1618 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 1619 1620 /** 1621 * This indicates that the (encoded) buffer marked as such contains 1622 * the data for a key frame. 1623 */ 1624 public static final int BUFFER_FLAG_KEY_FRAME = 1; 1625 1626 /** 1627 * This indicated that the buffer marked as such contains codec 1628 * initialization / codec specific data instead of media data. 1629 */ 1630 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 1631 1632 /** 1633 * This signals the end of stream, i.e. no buffers will be available 1634 * after this, unless of course, {@link #flush} follows. 1635 */ 1636 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 1637 1638 /** 1639 * This indicates that the buffer only contains part of a frame, 1640 * and the decoder should batch the data until a buffer without 1641 * this flag appears before decoding the frame. 1642 */ 1643 public static final int BUFFER_FLAG_PARTIAL_FRAME = 8; 1644 1645 /** 1646 * This indicates that the buffer contains non-media data for the 1647 * muxer to process. 1648 * 1649 * All muxer data should start with a FOURCC header that determines the type of data. 1650 * 1651 * For example, when it contains Exif data sent to a MediaMuxer track of 1652 * {@link MediaFormat#MIMETYPE_IMAGE_ANDROID_HEIC} type, the data must start with 1653 * Exif header ("Exif\0\0"), followed by the TIFF header (See JEITA CP-3451C Section 4.5.2.) 1654 * 1655 * @hide 1656 */ 1657 public static final int BUFFER_FLAG_MUXER_DATA = 16; 1658 1659 /** @hide */ 1660 @IntDef( 1661 flag = true, 1662 value = { 1663 BUFFER_FLAG_SYNC_FRAME, 1664 BUFFER_FLAG_KEY_FRAME, 1665 BUFFER_FLAG_CODEC_CONFIG, 1666 BUFFER_FLAG_END_OF_STREAM, 1667 BUFFER_FLAG_PARTIAL_FRAME, 1668 BUFFER_FLAG_MUXER_DATA, 1669 }) 1670 @Retention(RetentionPolicy.SOURCE) 1671 public @interface BufferFlag {} 1672 1673 private EventHandler mEventHandler; 1674 private EventHandler mOnFrameRenderedHandler; 1675 private EventHandler mCallbackHandler; 1676 private Callback mCallback; 1677 private OnFrameRenderedListener mOnFrameRenderedListener; 1678 private final Object mListenerLock = new Object(); 1679 private MediaCodecInfo mCodecInfo; 1680 private final Object mCodecInfoLock = new Object(); 1681 private MediaCrypto mCrypto; 1682 1683 private static final int EVENT_CALLBACK = 1; 1684 private static final int EVENT_SET_CALLBACK = 2; 1685 private static final int EVENT_FRAME_RENDERED = 3; 1686 1687 private static final int CB_INPUT_AVAILABLE = 1; 1688 private static final int CB_OUTPUT_AVAILABLE = 2; 1689 private static final int CB_ERROR = 3; 1690 private static final int CB_OUTPUT_FORMAT_CHANGE = 4; 1691 1692 private class EventHandler extends Handler { 1693 private MediaCodec mCodec; 1694 EventHandler(@onNull MediaCodec codec, @NonNull Looper looper)1695 public EventHandler(@NonNull MediaCodec codec, @NonNull Looper looper) { 1696 super(looper); 1697 mCodec = codec; 1698 } 1699 1700 @Override handleMessage(@onNull Message msg)1701 public void handleMessage(@NonNull Message msg) { 1702 switch (msg.what) { 1703 case EVENT_CALLBACK: 1704 { 1705 handleCallback(msg); 1706 break; 1707 } 1708 case EVENT_SET_CALLBACK: 1709 { 1710 mCallback = (MediaCodec.Callback) msg.obj; 1711 break; 1712 } 1713 case EVENT_FRAME_RENDERED: 1714 Map<String, Object> map = (Map<String, Object>)msg.obj; 1715 for (int i = 0; ; ++i) { 1716 Object mediaTimeUs = map.get(i + "-media-time-us"); 1717 Object systemNano = map.get(i + "-system-nano"); 1718 OnFrameRenderedListener onFrameRenderedListener; 1719 synchronized (mListenerLock) { 1720 onFrameRenderedListener = mOnFrameRenderedListener; 1721 } 1722 if (mediaTimeUs == null || systemNano == null 1723 || onFrameRenderedListener == null) { 1724 break; 1725 } 1726 onFrameRenderedListener.onFrameRendered( 1727 mCodec, (long)mediaTimeUs, (long)systemNano); 1728 } 1729 break; 1730 default: 1731 { 1732 break; 1733 } 1734 } 1735 } 1736 handleCallback(@onNull Message msg)1737 private void handleCallback(@NonNull Message msg) { 1738 if (mCallback == null) { 1739 return; 1740 } 1741 1742 switch (msg.arg1) { 1743 case CB_INPUT_AVAILABLE: 1744 { 1745 int index = msg.arg2; 1746 synchronized(mBufferLock) { 1747 switch (mBufferMode) { 1748 case BUFFER_MODE_LEGACY: 1749 validateInputByteBuffer(mCachedInputBuffers, index); 1750 break; 1751 case BUFFER_MODE_BLOCK: 1752 while (mQueueRequests.size() <= index) { 1753 mQueueRequests.add(null); 1754 } 1755 QueueRequest request = mQueueRequests.get(index); 1756 if (request == null) { 1757 request = new QueueRequest(mCodec, index); 1758 mQueueRequests.set(index, request); 1759 } 1760 request.setAccessible(true); 1761 break; 1762 default: 1763 throw new IllegalStateException( 1764 "Unrecognized buffer mode: " + mBufferMode); 1765 } 1766 } 1767 mCallback.onInputBufferAvailable(mCodec, index); 1768 break; 1769 } 1770 1771 case CB_OUTPUT_AVAILABLE: 1772 { 1773 int index = msg.arg2; 1774 BufferInfo info = (MediaCodec.BufferInfo) msg.obj; 1775 synchronized(mBufferLock) { 1776 switch (mBufferMode) { 1777 case BUFFER_MODE_LEGACY: 1778 validateOutputByteBuffer(mCachedOutputBuffers, index, info); 1779 break; 1780 case BUFFER_MODE_BLOCK: 1781 while (mOutputFrames.size() <= index) { 1782 mOutputFrames.add(null); 1783 } 1784 OutputFrame frame = mOutputFrames.get(index); 1785 if (frame == null) { 1786 frame = new OutputFrame(index); 1787 mOutputFrames.set(index, frame); 1788 } 1789 frame.setBufferInfo(info); 1790 frame.setAccessible(true); 1791 break; 1792 default: 1793 throw new IllegalStateException( 1794 "Unrecognized buffer mode: " + mBufferMode); 1795 } 1796 } 1797 mCallback.onOutputBufferAvailable( 1798 mCodec, index, info); 1799 break; 1800 } 1801 1802 case CB_ERROR: 1803 { 1804 mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj); 1805 break; 1806 } 1807 1808 case CB_OUTPUT_FORMAT_CHANGE: 1809 { 1810 mCallback.onOutputFormatChanged(mCodec, 1811 new MediaFormat((Map<String, Object>) msg.obj)); 1812 break; 1813 } 1814 1815 default: 1816 { 1817 break; 1818 } 1819 } 1820 } 1821 } 1822 1823 private boolean mHasSurface = false; 1824 1825 /** 1826 * Instantiate the preferred decoder supporting input data of the given mime type. 1827 * 1828 * The following is a partial list of defined mime types and their semantics: 1829 * <ul> 1830 * <li>"video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm) 1831 * <li>"video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm) 1832 * <li>"video/avc" - H.264/AVC video 1833 * <li>"video/hevc" - H.265/HEVC video 1834 * <li>"video/mp4v-es" - MPEG4 video 1835 * <li>"video/3gpp" - H.263 video 1836 * <li>"audio/3gpp" - AMR narrowband audio 1837 * <li>"audio/amr-wb" - AMR wideband audio 1838 * <li>"audio/mpeg" - MPEG1/2 audio layer III 1839 * <li>"audio/mp4a-latm" - AAC audio (note, this is raw AAC packets, not packaged in LATM!) 1840 * <li>"audio/vorbis" - vorbis audio 1841 * <li>"audio/g711-alaw" - G.711 alaw audio 1842 * <li>"audio/g711-mlaw" - G.711 ulaw audio 1843 * </ul> 1844 * 1845 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findDecoderForFormat} 1846 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1847 * given format. 1848 * 1849 * @param type The mime type of the input data. 1850 * @throws IOException if the codec cannot be created. 1851 * @throws IllegalArgumentException if type is not a valid mime type. 1852 * @throws NullPointerException if type is null. 1853 */ 1854 @NonNull createDecoderByType(@onNull String type)1855 public static MediaCodec createDecoderByType(@NonNull String type) 1856 throws IOException { 1857 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 1858 } 1859 1860 /** 1861 * Instantiate the preferred encoder supporting output data of the given mime type. 1862 * 1863 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findEncoderForFormat} 1864 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1865 * given format. 1866 * 1867 * @param type The desired mime type of the output data. 1868 * @throws IOException if the codec cannot be created. 1869 * @throws IllegalArgumentException if type is not a valid mime type. 1870 * @throws NullPointerException if type is null. 1871 */ 1872 @NonNull createEncoderByType(@onNull String type)1873 public static MediaCodec createEncoderByType(@NonNull String type) 1874 throws IOException { 1875 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 1876 } 1877 1878 /** 1879 * If you know the exact name of the component you want to instantiate 1880 * use this method to instantiate it. Use with caution. 1881 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 1882 * @param name The name of the codec to be instantiated. 1883 * @throws IOException if the codec cannot be created. 1884 * @throws IllegalArgumentException if name is not valid. 1885 * @throws NullPointerException if name is null. 1886 */ 1887 @NonNull createByCodecName(@onNull String name)1888 public static MediaCodec createByCodecName(@NonNull String name) 1889 throws IOException { 1890 return new MediaCodec( 1891 name, false /* nameIsType */, false /* unused */); 1892 } 1893 MediaCodec( @onNull String name, boolean nameIsType, boolean encoder)1894 private MediaCodec( 1895 @NonNull String name, boolean nameIsType, boolean encoder) { 1896 Looper looper; 1897 if ((looper = Looper.myLooper()) != null) { 1898 mEventHandler = new EventHandler(this, looper); 1899 } else if ((looper = Looper.getMainLooper()) != null) { 1900 mEventHandler = new EventHandler(this, looper); 1901 } else { 1902 mEventHandler = null; 1903 } 1904 mCallbackHandler = mEventHandler; 1905 mOnFrameRenderedHandler = mEventHandler; 1906 1907 mBufferLock = new Object(); 1908 1909 // save name used at creation 1910 mNameAtCreation = nameIsType ? null : name; 1911 1912 native_setup(name, nameIsType, encoder); 1913 } 1914 1915 private String mNameAtCreation; 1916 1917 @Override finalize()1918 protected void finalize() { 1919 native_finalize(); 1920 mCrypto = null; 1921 } 1922 1923 /** 1924 * Returns the codec to its initial (Uninitialized) state. 1925 * 1926 * Call this if an {@link MediaCodec.CodecException#isRecoverable unrecoverable} 1927 * error has occured to reset the codec to its initial state after creation. 1928 * 1929 * @throws CodecException if an unrecoverable error has occured and the codec 1930 * could not be reset. 1931 * @throws IllegalStateException if in the Released state. 1932 */ reset()1933 public final void reset() { 1934 freeAllTrackedBuffers(); // free buffers first 1935 native_reset(); 1936 mCrypto = null; 1937 } 1938 native_reset()1939 private native final void native_reset(); 1940 1941 /** 1942 * Free up resources used by the codec instance. 1943 * 1944 * Make sure you call this when you're done to free up any opened 1945 * component instance instead of relying on the garbage collector 1946 * to do this for you at some point in the future. 1947 */ release()1948 public final void release() { 1949 freeAllTrackedBuffers(); // free buffers first 1950 native_release(); 1951 mCrypto = null; 1952 } 1953 native_release()1954 private native final void native_release(); 1955 1956 /** 1957 * If this codec is to be used as an encoder, pass this flag. 1958 */ 1959 public static final int CONFIGURE_FLAG_ENCODE = 1; 1960 1961 /** 1962 * If this codec is to be used with {@link LinearBlock} and/or {@link 1963 * HardwareBuffer}, pass this flag. 1964 * <p> 1965 * When this flag is set, the following APIs throw {@link IncompatibleWithBlockModelException}. 1966 * <ul> 1967 * <li>{@link #getInputBuffer} 1968 * <li>{@link #getInputImage} 1969 * <li>{@link #getInputBuffers} 1970 * <li>{@link #getOutputBuffer} 1971 * <li>{@link #getOutputImage} 1972 * <li>{@link #getOutputBuffers} 1973 * <li>{@link #queueInputBuffer} 1974 * <li>{@link #queueSecureInputBuffer} 1975 * <li>{@link #dequeueInputBuffer} 1976 * <li>{@link #dequeueOutputBuffer} 1977 * </ul> 1978 */ 1979 public static final int CONFIGURE_FLAG_USE_BLOCK_MODEL = 2; 1980 1981 /** @hide */ 1982 @IntDef( 1983 flag = true, 1984 value = { 1985 CONFIGURE_FLAG_ENCODE, 1986 CONFIGURE_FLAG_USE_BLOCK_MODEL, 1987 }) 1988 @Retention(RetentionPolicy.SOURCE) 1989 public @interface ConfigureFlag {} 1990 1991 /** 1992 * Thrown when the codec is configured for block model and an incompatible API is called. 1993 */ 1994 public class IncompatibleWithBlockModelException extends RuntimeException { IncompatibleWithBlockModelException()1995 IncompatibleWithBlockModelException() { } 1996 IncompatibleWithBlockModelException(String message)1997 IncompatibleWithBlockModelException(String message) { 1998 super(message); 1999 } 2000 IncompatibleWithBlockModelException(String message, Throwable cause)2001 IncompatibleWithBlockModelException(String message, Throwable cause) { 2002 super(message, cause); 2003 } 2004 IncompatibleWithBlockModelException(Throwable cause)2005 IncompatibleWithBlockModelException(Throwable cause) { 2006 super(cause); 2007 } 2008 } 2009 2010 /** 2011 * Configures a component. 2012 * 2013 * @param format The format of the input data (decoder) or the desired 2014 * format of the output data (encoder). Passing {@code null} 2015 * as {@code format} is equivalent to passing an 2016 * {@link MediaFormat#MediaFormat an empty mediaformat}. 2017 * @param surface Specify a surface on which to render the output of this 2018 * decoder. Pass {@code null} as {@code surface} if the 2019 * codec does not generate raw video output (e.g. not a video 2020 * decoder) and/or if you want to configure the codec for 2021 * {@link ByteBuffer} output. 2022 * @param crypto Specify a crypto object to facilitate secure decryption 2023 * of the media data. Pass {@code null} as {@code crypto} for 2024 * non-secure codecs. 2025 * Please note that {@link MediaCodec} does NOT take ownership 2026 * of the {@link MediaCrypto} object; it is the application's 2027 * responsibility to properly cleanup the {@link MediaCrypto} object 2028 * when not in use. 2029 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 2030 * component as an encoder. 2031 * @throws IllegalArgumentException if the surface has been released (or is invalid), 2032 * or the format is unacceptable (e.g. missing a mandatory key), 2033 * or the flags are not set properly 2034 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 2035 * @throws IllegalStateException if not in the Uninitialized state. 2036 * @throws CryptoException upon DRM error. 2037 * @throws CodecException upon codec error. 2038 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @ConfigureFlag int flags)2039 public void configure( 2040 @Nullable MediaFormat format, 2041 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2042 @ConfigureFlag int flags) { 2043 configure(format, surface, crypto, null, flags); 2044 } 2045 2046 /** 2047 * Configure a component to be used with a descrambler. 2048 * @param format The format of the input data (decoder) or the desired 2049 * format of the output data (encoder). Passing {@code null} 2050 * as {@code format} is equivalent to passing an 2051 * {@link MediaFormat#MediaFormat an empty mediaformat}. 2052 * @param surface Specify a surface on which to render the output of this 2053 * decoder. Pass {@code null} as {@code surface} if the 2054 * codec does not generate raw video output (e.g. not a video 2055 * decoder) and/or if you want to configure the codec for 2056 * {@link ByteBuffer} output. 2057 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 2058 * component as an encoder. 2059 * @param descrambler Specify a descrambler object to facilitate secure 2060 * descrambling of the media data, or null for non-secure codecs. 2061 * @throws IllegalArgumentException if the surface has been released (or is invalid), 2062 * or the format is unacceptable (e.g. missing a mandatory key), 2063 * or the flags are not set properly 2064 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 2065 * @throws IllegalStateException if not in the Uninitialized state. 2066 * @throws CryptoException upon DRM error. 2067 * @throws CodecException upon codec error. 2068 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler)2069 public void configure( 2070 @Nullable MediaFormat format, @Nullable Surface surface, 2071 @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler) { 2072 configure(format, surface, null, 2073 descrambler != null ? descrambler.getBinder() : null, flags); 2074 } 2075 2076 private static final int BUFFER_MODE_INVALID = -1; 2077 private static final int BUFFER_MODE_LEGACY = 0; 2078 private static final int BUFFER_MODE_BLOCK = 1; 2079 private int mBufferMode = BUFFER_MODE_INVALID; 2080 configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2081 private void configure( 2082 @Nullable MediaFormat format, @Nullable Surface surface, 2083 @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, 2084 @ConfigureFlag int flags) { 2085 if (crypto != null && descramblerBinder != null) { 2086 throw new IllegalArgumentException("Can't use crypto and descrambler together!"); 2087 } 2088 2089 String[] keys = null; 2090 Object[] values = null; 2091 2092 if (format != null) { 2093 Map<String, Object> formatMap = format.getMap(); 2094 keys = new String[formatMap.size()]; 2095 values = new Object[formatMap.size()]; 2096 2097 int i = 0; 2098 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 2099 if (entry.getKey().equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 2100 int sessionId = 0; 2101 try { 2102 sessionId = (Integer)entry.getValue(); 2103 } 2104 catch (Exception e) { 2105 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 2106 } 2107 keys[i] = "audio-hw-sync"; 2108 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 2109 } else { 2110 keys[i] = entry.getKey(); 2111 values[i] = entry.getValue(); 2112 } 2113 ++i; 2114 } 2115 } 2116 2117 mHasSurface = surface != null; 2118 mCrypto = crypto; 2119 synchronized (mBufferLock) { 2120 if ((flags & CONFIGURE_FLAG_USE_BLOCK_MODEL) != 0) { 2121 mBufferMode = BUFFER_MODE_BLOCK; 2122 } else { 2123 mBufferMode = BUFFER_MODE_LEGACY; 2124 } 2125 } 2126 2127 native_configure(keys, values, surface, crypto, descramblerBinder, flags); 2128 } 2129 2130 /** 2131 * Dynamically sets the output surface of a codec. 2132 * <p> 2133 * This can only be used if the codec was configured with an output surface. The 2134 * new output surface should have a compatible usage type to the original output surface. 2135 * E.g. codecs may not support switching from a SurfaceTexture (GPU readable) output 2136 * to ImageReader (software readable) output. 2137 * @param surface the output surface to use. It must not be {@code null}. 2138 * @throws IllegalStateException if the codec does not support setting the output 2139 * surface in the current state. 2140 * @throws IllegalArgumentException if the new surface is not of a suitable type for the codec. 2141 */ setOutputSurface(@onNull Surface surface)2142 public void setOutputSurface(@NonNull Surface surface) { 2143 if (!mHasSurface) { 2144 throw new IllegalStateException("codec was not configured for an output surface"); 2145 } 2146 native_setSurface(surface); 2147 } 2148 native_setSurface(@onNull Surface surface)2149 private native void native_setSurface(@NonNull Surface surface); 2150 2151 /** 2152 * Create a persistent input surface that can be used with codecs that normally have an input 2153 * surface, such as video encoders. A persistent input can be reused by subsequent 2154 * {@link MediaCodec} or {@link MediaRecorder} instances, but can only be used by at 2155 * most one codec or recorder instance concurrently. 2156 * <p> 2157 * The application is responsible for calling release() on the Surface when done. 2158 * 2159 * @return an input surface that can be used with {@link #setInputSurface}. 2160 */ 2161 @NonNull createPersistentInputSurface()2162 public static Surface createPersistentInputSurface() { 2163 return native_createPersistentInputSurface(); 2164 } 2165 2166 static class PersistentSurface extends Surface { 2167 @SuppressWarnings("unused") PersistentSurface()2168 PersistentSurface() {} // used by native 2169 2170 @Override release()2171 public void release() { 2172 native_releasePersistentInputSurface(this); 2173 super.release(); 2174 } 2175 2176 private long mPersistentObject; 2177 }; 2178 2179 /** 2180 * Configures the codec (e.g. encoder) to use a persistent input surface in place of input 2181 * buffers. This may only be called after {@link #configure} and before {@link #start}, in 2182 * lieu of {@link #createInputSurface}. 2183 * @param surface a persistent input surface created by {@link #createPersistentInputSurface} 2184 * @throws IllegalStateException if not in the Configured state or does not require an input 2185 * surface. 2186 * @throws IllegalArgumentException if the surface was not created by 2187 * {@link #createPersistentInputSurface}. 2188 */ setInputSurface(@onNull Surface surface)2189 public void setInputSurface(@NonNull Surface surface) { 2190 if (!(surface instanceof PersistentSurface)) { 2191 throw new IllegalArgumentException("not a PersistentSurface"); 2192 } 2193 native_setInputSurface(surface); 2194 } 2195 2196 @NonNull native_createPersistentInputSurface()2197 private static native final PersistentSurface native_createPersistentInputSurface(); native_releasePersistentInputSurface(@onNull Surface surface)2198 private static native final void native_releasePersistentInputSurface(@NonNull Surface surface); native_setInputSurface(@onNull Surface surface)2199 private native final void native_setInputSurface(@NonNull Surface surface); 2200 native_setCallback(@ullable Callback cb)2201 private native final void native_setCallback(@Nullable Callback cb); 2202 native_configure( @ullable String[] keys, @Nullable Object[] values, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2203 private native final void native_configure( 2204 @Nullable String[] keys, @Nullable Object[] values, 2205 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2206 @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags); 2207 2208 /** 2209 * Requests a Surface to use as the input to an encoder, in place of input buffers. This 2210 * may only be called after {@link #configure} and before {@link #start}. 2211 * <p> 2212 * The application is responsible for calling release() on the Surface when 2213 * done. 2214 * <p> 2215 * The Surface must be rendered with a hardware-accelerated API, such as OpenGL ES. 2216 * {@link android.view.Surface#lockCanvas(android.graphics.Rect)} may fail or produce 2217 * unexpected results. 2218 * @throws IllegalStateException if not in the Configured state. 2219 */ 2220 @NonNull createInputSurface()2221 public native final Surface createInputSurface(); 2222 2223 /** 2224 * After successfully configuring the component, call {@code start}. 2225 * <p> 2226 * Call {@code start} also if the codec is configured in asynchronous mode, 2227 * and it has just been flushed, to resume requesting input buffers. 2228 * @throws IllegalStateException if not in the Configured state 2229 * or just after {@link #flush} for a codec that is configured 2230 * in asynchronous mode. 2231 * @throws MediaCodec.CodecException upon codec error. Note that some codec errors 2232 * for start may be attributed to future method calls. 2233 */ start()2234 public final void start() { 2235 native_start(); 2236 synchronized(mBufferLock) { 2237 cacheBuffers(true /* input */); 2238 cacheBuffers(false /* input */); 2239 } 2240 } native_start()2241 private native final void native_start(); 2242 2243 /** 2244 * Finish the decode/encode session, note that the codec instance 2245 * remains active and ready to be {@link #start}ed again. 2246 * To ensure that it is available to other client call {@link #release} 2247 * and don't just rely on garbage collection to eventually do this for you. 2248 * @throws IllegalStateException if in the Released state. 2249 */ stop()2250 public final void stop() { 2251 native_stop(); 2252 freeAllTrackedBuffers(); 2253 2254 synchronized (mListenerLock) { 2255 if (mCallbackHandler != null) { 2256 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 2257 mCallbackHandler.removeMessages(EVENT_CALLBACK); 2258 } 2259 if (mOnFrameRenderedHandler != null) { 2260 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 2261 } 2262 } 2263 } 2264 native_stop()2265 private native final void native_stop(); 2266 2267 /** 2268 * Flush both input and output ports of the component. 2269 * <p> 2270 * Upon return, all indices previously returned in calls to {@link #dequeueInputBuffer 2271 * dequeueInputBuffer} and {@link #dequeueOutputBuffer dequeueOutputBuffer} — or obtained 2272 * via {@link Callback#onInputBufferAvailable onInputBufferAvailable} or 2273 * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} callbacks — become 2274 * invalid, and all buffers are owned by the codec. 2275 * <p> 2276 * If the codec is configured in asynchronous mode, call {@link #start} 2277 * after {@code flush} has returned to resume codec operations. The codec 2278 * will not request input buffers until this has happened. 2279 * <strong>Note, however, that there may still be outstanding {@code onOutputBufferAvailable} 2280 * callbacks that were not handled prior to calling {@code flush}. 2281 * The indices returned via these callbacks also become invalid upon calling {@code flush} and 2282 * should be discarded.</strong> 2283 * <p> 2284 * If the codec is configured in synchronous mode, codec will resume 2285 * automatically if it is configured with an input surface. Otherwise, it 2286 * will resume when {@link #dequeueInputBuffer dequeueInputBuffer} is called. 2287 * 2288 * @throws IllegalStateException if not in the Executing state. 2289 * @throws MediaCodec.CodecException upon codec error. 2290 */ flush()2291 public final void flush() { 2292 synchronized(mBufferLock) { 2293 invalidateByteBuffers(mCachedInputBuffers); 2294 invalidateByteBuffers(mCachedOutputBuffers); 2295 mDequeuedInputBuffers.clear(); 2296 mDequeuedOutputBuffers.clear(); 2297 } 2298 native_flush(); 2299 } 2300 native_flush()2301 private native final void native_flush(); 2302 2303 /** 2304 * Thrown when an internal codec error occurs. 2305 */ 2306 public final static class CodecException extends IllegalStateException { 2307 @UnsupportedAppUsage CodecException(int errorCode, int actionCode, @Nullable String detailMessage)2308 CodecException(int errorCode, int actionCode, @Nullable String detailMessage) { 2309 super(detailMessage); 2310 mErrorCode = errorCode; 2311 mActionCode = actionCode; 2312 2313 // TODO get this from codec 2314 final String sign = errorCode < 0 ? "neg_" : ""; 2315 mDiagnosticInfo = 2316 "android.media.MediaCodec.error_" + sign + Math.abs(errorCode); 2317 } 2318 2319 /** 2320 * Returns true if the codec exception is a transient issue, 2321 * perhaps due to resource constraints, and that the method 2322 * (or encoding/decoding) may be retried at a later time. 2323 */ 2324 public boolean isTransient() { 2325 return mActionCode == ACTION_TRANSIENT; 2326 } 2327 2328 /** 2329 * Returns true if the codec cannot proceed further, 2330 * but can be recovered by stopping, configuring, 2331 * and starting again. 2332 */ 2333 public boolean isRecoverable() { 2334 return mActionCode == ACTION_RECOVERABLE; 2335 } 2336 2337 /** 2338 * Retrieve the error code associated with a CodecException 2339 */ 2340 public int getErrorCode() { 2341 return mErrorCode; 2342 } 2343 2344 /** 2345 * Retrieve a developer-readable diagnostic information string 2346 * associated with the exception. Do not show this to end-users, 2347 * since this string will not be localized or generally 2348 * comprehensible to end-users. 2349 */ 2350 public @NonNull String getDiagnosticInfo() { 2351 return mDiagnosticInfo; 2352 } 2353 2354 /** 2355 * This indicates required resource was not able to be allocated. 2356 */ 2357 public static final int ERROR_INSUFFICIENT_RESOURCE = 1100; 2358 2359 /** 2360 * This indicates the resource manager reclaimed the media resource used by the codec. 2361 * <p> 2362 * With this exception, the codec must be released, as it has moved to terminal state. 2363 */ 2364 public static final int ERROR_RECLAIMED = 1101; 2365 2366 /** @hide */ 2367 @IntDef({ 2368 ERROR_INSUFFICIENT_RESOURCE, 2369 ERROR_RECLAIMED, 2370 }) 2371 @Retention(RetentionPolicy.SOURCE) 2372 public @interface ReasonCode {} 2373 2374 /* Must be in sync with android_media_MediaCodec.cpp */ 2375 private final static int ACTION_TRANSIENT = 1; 2376 private final static int ACTION_RECOVERABLE = 2; 2377 2378 private final String mDiagnosticInfo; 2379 private final int mErrorCode; 2380 private final int mActionCode; 2381 } 2382 2383 /** 2384 * Thrown when a crypto error occurs while queueing a secure input buffer. 2385 */ 2386 public final static class CryptoException extends RuntimeException { 2387 public CryptoException(int errorCode, @Nullable String detailMessage) { 2388 super(detailMessage); 2389 mErrorCode = errorCode; 2390 } 2391 2392 /** 2393 * This indicates that the requested key was not found when trying to 2394 * perform a decrypt operation. The operation can be retried after adding 2395 * the correct decryption key. 2396 */ 2397 public static final int ERROR_NO_KEY = 1; 2398 2399 /** 2400 * This indicates that the key used for decryption is no longer 2401 * valid due to license term expiration. The operation can be retried 2402 * after updating the expired keys. 2403 */ 2404 public static final int ERROR_KEY_EXPIRED = 2; 2405 2406 /** 2407 * This indicates that a required crypto resource was not able to be 2408 * allocated while attempting the requested operation. The operation 2409 * can be retried if the app is able to release resources. 2410 */ 2411 public static final int ERROR_RESOURCE_BUSY = 3; 2412 2413 /** 2414 * This indicates that the output protection levels supported by the 2415 * device are not sufficient to meet the requirements set by the 2416 * content owner in the license policy. 2417 */ 2418 public static final int ERROR_INSUFFICIENT_OUTPUT_PROTECTION = 4; 2419 2420 /** 2421 * This indicates that decryption was attempted on a session that is 2422 * not opened, which could be due to a failure to open the session, 2423 * closing the session prematurely, or the session being reclaimed 2424 * by the resource manager. 2425 */ 2426 public static final int ERROR_SESSION_NOT_OPENED = 5; 2427 2428 /** 2429 * This indicates that an operation was attempted that could not be 2430 * supported by the crypto system of the device in its current 2431 * configuration. It may occur when the license policy requires 2432 * device security features that aren't supported by the device, 2433 * or due to an internal error in the crypto system that prevents 2434 * the specified security policy from being met. 2435 */ 2436 public static final int ERROR_UNSUPPORTED_OPERATION = 6; 2437 2438 /** 2439 * This indicates that the security level of the device is not 2440 * sufficient to meet the requirements set by the content owner 2441 * in the license policy. 2442 */ 2443 public static final int ERROR_INSUFFICIENT_SECURITY = 7; 2444 2445 /** 2446 * This indicates that the video frame being decrypted exceeds 2447 * the size of the device's protected output buffers. When 2448 * encountering this error the app should try playing content 2449 * of a lower resolution. 2450 */ 2451 public static final int ERROR_FRAME_TOO_LARGE = 8; 2452 2453 /** 2454 * This error indicates that session state has been 2455 * invalidated. It can occur on devices that are not capable 2456 * of retaining crypto session state across device 2457 * suspend/resume. The session must be closed and a new 2458 * session opened to resume operation. 2459 */ 2460 public static final int ERROR_LOST_STATE = 9; 2461 2462 /** @hide */ 2463 @IntDef({ 2464 ERROR_NO_KEY, 2465 ERROR_KEY_EXPIRED, 2466 ERROR_RESOURCE_BUSY, 2467 ERROR_INSUFFICIENT_OUTPUT_PROTECTION, 2468 ERROR_SESSION_NOT_OPENED, 2469 ERROR_UNSUPPORTED_OPERATION, 2470 ERROR_INSUFFICIENT_SECURITY, 2471 ERROR_FRAME_TOO_LARGE, 2472 ERROR_LOST_STATE 2473 }) 2474 @Retention(RetentionPolicy.SOURCE) 2475 public @interface CryptoErrorCode {} 2476 2477 /** 2478 * Retrieve the error code associated with a CryptoException 2479 */ 2480 @CryptoErrorCode 2481 public int getErrorCode() { 2482 return mErrorCode; 2483 } 2484 2485 private int mErrorCode; 2486 } 2487 2488 /** 2489 * After filling a range of the input buffer at the specified index 2490 * submit it to the component. Once an input buffer is queued to 2491 * the codec, it MUST NOT be used until it is later retrieved by 2492 * {@link #getInputBuffer} in response to a {@link #dequeueInputBuffer} 2493 * return value or a {@link Callback#onInputBufferAvailable} 2494 * callback. 2495 * <p> 2496 * Many decoders require the actual compressed data stream to be 2497 * preceded by "codec specific data", i.e. setup data used to initialize 2498 * the codec such as PPS/SPS in the case of AVC video or code tables 2499 * in the case of vorbis audio. 2500 * The class {@link android.media.MediaExtractor} provides codec 2501 * specific data as part of 2502 * the returned track format in entries named "csd-0", "csd-1" ... 2503 * <p> 2504 * These buffers can be submitted directly after {@link #start} or 2505 * {@link #flush} by specifying the flag {@link 2506 * #BUFFER_FLAG_CODEC_CONFIG}. However, if you configure the 2507 * codec with a {@link MediaFormat} containing these keys, they 2508 * will be automatically submitted by MediaCodec directly after 2509 * start. Therefore, the use of {@link 2510 * #BUFFER_FLAG_CODEC_CONFIG} flag is discouraged and is 2511 * recommended only for advanced users. 2512 * <p> 2513 * To indicate that this is the final piece of input data (or rather that 2514 * no more input data follows unless the decoder is subsequently flushed) 2515 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 2516 * <p class=note> 2517 * <strong>Note:</strong> Prior to {@link android.os.Build.VERSION_CODES#M}, 2518 * {@code presentationTimeUs} was not propagated to the frame timestamp of (rendered) 2519 * Surface output buffers, and the resulting frame timestamp was undefined. 2520 * Use {@link #releaseOutputBuffer(int, long)} to ensure a specific frame timestamp is set. 2521 * Similarly, since frame timestamps can be used by the destination surface for rendering 2522 * synchronization, <strong>care must be taken to normalize presentationTimeUs so as to not be 2523 * mistaken for a system time. (See {@linkplain #releaseOutputBuffer(int, long) 2524 * SurfaceView specifics}).</strong> 2525 * 2526 * @param index The index of a client-owned input buffer previously returned 2527 * in a call to {@link #dequeueInputBuffer}. 2528 * @param offset The byte offset into the input buffer at which the data starts. 2529 * @param size The number of bytes of valid input data. 2530 * @param presentationTimeUs The presentation timestamp in microseconds for this 2531 * buffer. This is normally the media time at which this 2532 * buffer should be presented (rendered). When using an output 2533 * surface, this will be propagated as the {@link 2534 * SurfaceTexture#getTimestamp timestamp} for the frame (after 2535 * conversion to nanoseconds). 2536 * @param flags A bitmask of flags 2537 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2538 * While not prohibited, most codecs do not use the 2539 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2540 * @throws IllegalStateException if not in the Executing state. 2541 * @throws MediaCodec.CodecException upon codec error. 2542 * @throws CryptoException if a crypto object has been specified in 2543 * {@link #configure} 2544 */ 2545 public final void queueInputBuffer( 2546 int index, 2547 int offset, int size, long presentationTimeUs, int flags) 2548 throws CryptoException { 2549 synchronized(mBufferLock) { 2550 if (mBufferMode == BUFFER_MODE_BLOCK) { 2551 throw new IncompatibleWithBlockModelException("queueInputBuffer() " 2552 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 2553 + "Please use getQueueRequest() to queue buffers"); 2554 } 2555 invalidateByteBuffer(mCachedInputBuffers, index); 2556 mDequeuedInputBuffers.remove(index); 2557 } 2558 try { 2559 native_queueInputBuffer( 2560 index, offset, size, presentationTimeUs, flags); 2561 } catch (CryptoException | IllegalStateException e) { 2562 revalidateByteBuffer(mCachedInputBuffers, index); 2563 throw e; 2564 } 2565 } 2566 2567 private native final void native_queueInputBuffer( 2568 int index, 2569 int offset, int size, long presentationTimeUs, int flags) 2570 throws CryptoException; 2571 2572 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 2573 public static final int CRYPTO_MODE_AES_CTR = 1; 2574 public static final int CRYPTO_MODE_AES_CBC = 2; 2575 2576 /** 2577 * Metadata describing the structure of an encrypted input sample. 2578 * <p> 2579 * A buffer's data is considered to be partitioned into "subSamples". Each subSample starts with 2580 * a run of plain, unencrypted bytes followed by a run of encrypted bytes. Either of these runs 2581 * may be empty. If pattern encryption applies, each of the encrypted runs is encrypted only 2582 * partly, according to a repeating pattern of "encrypt" and "skip" blocks. 2583 * {@link #numBytesOfClearData} can be null to indicate that all data is encrypted, and 2584 * {@link #numBytesOfEncryptedData} can be null to indicate that all data is clear. At least one 2585 * of {@link #numBytesOfClearData} and {@link #numBytesOfEncryptedData} must be non-null. 2586 * <p> 2587 * This information encapsulates per-sample metadata as outlined in ISO/IEC FDIS 23001-7:2016 2588 * "Common encryption in ISO base media file format files". 2589 * <p> 2590 * <h3>ISO-CENC Schemes</h3> 2591 * ISO/IEC FDIS 23001-7:2016 defines four possible schemes by which media may be encrypted, 2592 * corresponding to each possible combination of an AES mode with the presence or absence of 2593 * patterned encryption. 2594 * 2595 * <table style="width: 0%"> 2596 * <thead> 2597 * <tr> 2598 * <th> </th> 2599 * <th>AES-CTR</th> 2600 * <th>AES-CBC</th> 2601 * </tr> 2602 * </thead> 2603 * <tbody> 2604 * <tr> 2605 * <th>Without Patterns</th> 2606 * <td>cenc</td> 2607 * <td>cbc1</td> 2608 * </tr><tr> 2609 * <th>With Patterns</th> 2610 * <td>cens</td> 2611 * <td>cbcs</td> 2612 * </tr> 2613 * </tbody> 2614 * </table> 2615 * 2616 * For {@code CryptoInfo}, the scheme is selected implicitly by the combination of the 2617 * {@link #mode} field and the value set with {@link #setPattern}. For the pattern, setting the 2618 * pattern to all zeroes (that is, both {@code blocksToEncrypt} and {@code blocksToSkip} are 2619 * zero) is interpreted as turning patterns off completely. A scheme that does not use patterns 2620 * will be selected, either cenc or cbc1. Setting the pattern to any nonzero value will choose 2621 * one of the pattern-supporting schemes, cens or cbcs. The default pattern if 2622 * {@link #setPattern} is never called is all zeroes. 2623 * <p> 2624 * <h4>HLS SAMPLE-AES Audio</h4> 2625 * HLS SAMPLE-AES audio is encrypted in a manner compatible with the cbcs scheme, except that it 2626 * does not use patterned encryption. However, if {@link #setPattern} is used to set the pattern 2627 * to all zeroes, this will be interpreted as selecting the cbc1 scheme. The cbc1 scheme cannot 2628 * successfully decrypt HLS SAMPLE-AES audio because of differences in how the IVs are handled. 2629 * For this reason, it is recommended that a pattern of {@code 1} encrypted block and {@code 0} 2630 * skip blocks be used with HLS SAMPLE-AES audio. This will trigger decryption to use cbcs mode 2631 * while still decrypting every block. 2632 */ 2633 public final static class CryptoInfo { 2634 /** 2635 * The number of subSamples that make up the buffer's contents. 2636 */ 2637 public int numSubSamples; 2638 /** 2639 * The number of leading unencrypted bytes in each subSample. If null, all bytes are treated 2640 * as encrypted and {@link #numBytesOfEncryptedData} must be specified. 2641 */ 2642 public int[] numBytesOfClearData; 2643 /** 2644 * The number of trailing encrypted bytes in each subSample. If null, all bytes are treated 2645 * as clear and {@link #numBytesOfClearData} must be specified. 2646 */ 2647 public int[] numBytesOfEncryptedData; 2648 /** 2649 * A 16-byte key id 2650 */ 2651 public byte[] key; 2652 /** 2653 * A 16-byte initialization vector 2654 */ 2655 public byte[] iv; 2656 /** 2657 * The type of encryption that has been applied, 2658 * see {@link #CRYPTO_MODE_UNENCRYPTED}, {@link #CRYPTO_MODE_AES_CTR} 2659 * and {@link #CRYPTO_MODE_AES_CBC} 2660 */ 2661 public int mode; 2662 2663 /** 2664 * Metadata describing an encryption pattern for the protected bytes in a subsample. An 2665 * encryption pattern consists of a repeating sequence of crypto blocks comprised of a 2666 * number of encrypted blocks followed by a number of unencrypted, or skipped, blocks. 2667 */ 2668 public final static class Pattern { 2669 /** 2670 * Number of blocks to be encrypted in the pattern. If both this and 2671 * {@link #mSkipBlocks} are zero, pattern encryption is inoperative. 2672 */ 2673 private int mEncryptBlocks; 2674 2675 /** 2676 * Number of blocks to be skipped (left clear) in the pattern. If both this and 2677 * {@link #mEncryptBlocks} are zero, pattern encryption is inoperative. 2678 */ 2679 private int mSkipBlocks; 2680 2681 /** 2682 * Construct a sample encryption pattern given the number of blocks to encrypt and skip 2683 * in the pattern. If both parameters are zero, pattern encryption is inoperative. 2684 */ 2685 public Pattern(int blocksToEncrypt, int blocksToSkip) { 2686 set(blocksToEncrypt, blocksToSkip); 2687 } 2688 2689 /** 2690 * Set the number of blocks to encrypt and skip in a sample encryption pattern. If both 2691 * parameters are zero, pattern encryption is inoperative. 2692 */ 2693 public void set(int blocksToEncrypt, int blocksToSkip) { 2694 mEncryptBlocks = blocksToEncrypt; 2695 mSkipBlocks = blocksToSkip; 2696 } 2697 2698 /** 2699 * Return the number of blocks to skip in a sample encryption pattern. 2700 */ 2701 public int getSkipBlocks() { 2702 return mSkipBlocks; 2703 } 2704 2705 /** 2706 * Return the number of blocks to encrypt in a sample encryption pattern. 2707 */ 2708 public int getEncryptBlocks() { 2709 return mEncryptBlocks; 2710 } 2711 }; 2712 2713 private final Pattern zeroPattern = new Pattern(0, 0); 2714 2715 /** 2716 * The pattern applicable to the protected data in each subsample. 2717 */ 2718 private Pattern pattern; 2719 2720 /** 2721 * Set the subsample count, clear/encrypted sizes, key, IV and mode fields of 2722 * a {@link MediaCodec.CryptoInfo} instance. 2723 */ 2724 public void set( 2725 int newNumSubSamples, 2726 @NonNull int[] newNumBytesOfClearData, 2727 @NonNull int[] newNumBytesOfEncryptedData, 2728 @NonNull byte[] newKey, 2729 @NonNull byte[] newIV, 2730 int newMode) { 2731 numSubSamples = newNumSubSamples; 2732 numBytesOfClearData = newNumBytesOfClearData; 2733 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 2734 key = newKey; 2735 iv = newIV; 2736 mode = newMode; 2737 pattern = zeroPattern; 2738 } 2739 2740 /** 2741 * Set the encryption pattern on a {@link MediaCodec.CryptoInfo} instance. 2742 * See {@link MediaCodec.CryptoInfo.Pattern}. 2743 */ 2744 public void setPattern(Pattern newPattern) { 2745 if (newPattern == null) { 2746 newPattern = zeroPattern; 2747 } 2748 pattern = newPattern; 2749 } 2750 2751 private void setPattern(int blocksToEncrypt, int blocksToSkip) { 2752 pattern = new Pattern(blocksToEncrypt, blocksToSkip); 2753 } 2754 2755 @Override 2756 public String toString() { 2757 StringBuilder builder = new StringBuilder(); 2758 builder.append(numSubSamples + " subsamples, key ["); 2759 String hexdigits = "0123456789abcdef"; 2760 for (int i = 0; i < key.length; i++) { 2761 builder.append(hexdigits.charAt((key[i] & 0xf0) >> 4)); 2762 builder.append(hexdigits.charAt(key[i] & 0x0f)); 2763 } 2764 builder.append("], iv ["); 2765 for (int i = 0; i < iv.length; i++) { 2766 builder.append(hexdigits.charAt((iv[i] & 0xf0) >> 4)); 2767 builder.append(hexdigits.charAt(iv[i] & 0x0f)); 2768 } 2769 builder.append("], clear "); Arrays.toString(numBytesOfClearData)2770 builder.append(Arrays.toString(numBytesOfClearData)); 2771 builder.append(", encrypted "); Arrays.toString(numBytesOfEncryptedData)2772 builder.append(Arrays.toString(numBytesOfEncryptedData)); 2773 builder.append(", pattern (encrypt: "); builder.append(pattern.mEncryptBlocks)2774 builder.append(pattern.mEncryptBlocks); 2775 builder.append(", skip: "); builder.append(pattern.mSkipBlocks)2776 builder.append(pattern.mSkipBlocks); 2777 builder.append(")"); 2778 return builder.toString(); 2779 } 2780 }; 2781 2782 /** 2783 * Similar to {@link #queueInputBuffer queueInputBuffer} but submits a buffer that is 2784 * potentially encrypted. 2785 * <strong>Check out further notes at {@link #queueInputBuffer queueInputBuffer}.</strong> 2786 * 2787 * @param index The index of a client-owned input buffer previously returned 2788 * in a call to {@link #dequeueInputBuffer}. 2789 * @param offset The byte offset into the input buffer at which the data starts. 2790 * @param info Metadata required to facilitate decryption, the object can be 2791 * reused immediately after this call returns. 2792 * @param presentationTimeUs The presentation timestamp in microseconds for this 2793 * buffer. This is normally the media time at which this 2794 * buffer should be presented (rendered). 2795 * @param flags A bitmask of flags 2796 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2797 * While not prohibited, most codecs do not use the 2798 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2799 * @throws IllegalStateException if not in the Executing state. 2800 * @throws MediaCodec.CodecException upon codec error. 2801 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 2802 * An error code associated with the exception helps identify the 2803 * reason for the failure. 2804 */ queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2805 public final void queueSecureInputBuffer( 2806 int index, 2807 int offset, 2808 @NonNull CryptoInfo info, 2809 long presentationTimeUs, 2810 int flags) throws CryptoException { 2811 synchronized(mBufferLock) { 2812 if (mBufferMode == BUFFER_MODE_BLOCK) { 2813 throw new IncompatibleWithBlockModelException("queueSecureInputBuffer() " 2814 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 2815 + "Please use getQueueRequest() to queue buffers"); 2816 } 2817 invalidateByteBuffer(mCachedInputBuffers, index); 2818 mDequeuedInputBuffers.remove(index); 2819 } 2820 try { 2821 native_queueSecureInputBuffer( 2822 index, offset, info, presentationTimeUs, flags); 2823 } catch (CryptoException | IllegalStateException e) { 2824 revalidateByteBuffer(mCachedInputBuffers, index); 2825 throw e; 2826 } 2827 } 2828 native_queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2829 private native final void native_queueSecureInputBuffer( 2830 int index, 2831 int offset, 2832 @NonNull CryptoInfo info, 2833 long presentationTimeUs, 2834 int flags) throws CryptoException; 2835 2836 /** 2837 * Returns the index of an input buffer to be filled with valid data 2838 * or -1 if no such buffer is currently available. 2839 * This method will return immediately if timeoutUs == 0, wait indefinitely 2840 * for the availability of an input buffer if timeoutUs < 0 or wait up 2841 * to "timeoutUs" microseconds if timeoutUs > 0. 2842 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 2843 * @throws IllegalStateException if not in the Executing state, 2844 * or codec is configured in asynchronous mode. 2845 * @throws MediaCodec.CodecException upon codec error. 2846 */ dequeueInputBuffer(long timeoutUs)2847 public final int dequeueInputBuffer(long timeoutUs) { 2848 synchronized (mBufferLock) { 2849 if (mBufferMode == BUFFER_MODE_BLOCK) { 2850 throw new IncompatibleWithBlockModelException("dequeueInputBuffer() " 2851 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 2852 + "Please use MediaCodec.Callback objectes to get input buffer slots."); 2853 } 2854 } 2855 int res = native_dequeueInputBuffer(timeoutUs); 2856 if (res >= 0) { 2857 synchronized(mBufferLock) { 2858 validateInputByteBuffer(mCachedInputBuffers, res); 2859 } 2860 } 2861 return res; 2862 } 2863 native_dequeueInputBuffer(long timeoutUs)2864 private native final int native_dequeueInputBuffer(long timeoutUs); 2865 2866 /** 2867 * Section of memory that represents a linear block. Applications may 2868 * acquire a block via {@link LinearBlock#obtain} and queue all or part 2869 * of the block as an input buffer to a codec, or get a block allocated by 2870 * codec as an output buffer from {@link OutputFrame}. 2871 * 2872 * {@see QueueRequest#setLinearBlock} 2873 * {@see QueueRequest#setEncryptedLinearBlock} 2874 * {@see OutputFrame#getLinearBlock} 2875 */ 2876 public static final class LinearBlock { 2877 // No public constructors. LinearBlock()2878 private LinearBlock() {} 2879 2880 /** 2881 * Returns true if the buffer is mappable. 2882 * @throws IllegalStateException if invalid 2883 */ isMappable()2884 public boolean isMappable() { 2885 synchronized (mLock) { 2886 if (!mValid) { 2887 throw new IllegalStateException("The linear block is invalid"); 2888 } 2889 return mMappable; 2890 } 2891 } 2892 2893 /** 2894 * Map the memory and return the mapped region. 2895 * <p> 2896 * The returned memory region becomes inaccessible after 2897 * {@link #recycle}, or the buffer is queued to the codecs and not 2898 * returned to the client yet. 2899 * 2900 * @return mapped memory region as {@link ByteBuffer} object 2901 * @throws IllegalStateException if not mappable or invalid 2902 */ map()2903 public @NonNull ByteBuffer map() { 2904 synchronized (mLock) { 2905 if (!mValid) { 2906 throw new IllegalStateException("The linear block is invalid"); 2907 } 2908 if (!mMappable) { 2909 throw new IllegalStateException("The linear block is not mappable"); 2910 } 2911 if (mMapped == null) { 2912 mMapped = native_map(); 2913 } 2914 return mMapped; 2915 } 2916 } 2917 native_map()2918 private native ByteBuffer native_map(); 2919 2920 /** 2921 * Mark this block as ready to be recycled by the framework once it is 2922 * no longer in use. All operations to this object after 2923 * this call will cause exceptions, as well as attempt to access the 2924 * previously mapped memory region. Caller should clear all references 2925 * to this object after this call. 2926 * <p> 2927 * To avoid excessive memory consumption, it is recommended that callers 2928 * recycle buffers as soon as they no longer need the buffers 2929 * 2930 * @throws IllegalStateException if invalid 2931 */ recycle()2932 public void recycle() { 2933 synchronized (mLock) { 2934 if (!mValid) { 2935 throw new IllegalStateException("The linear block is invalid"); 2936 } 2937 if (mMapped != null) { 2938 mMapped.setAccessible(false); 2939 mMapped = null; 2940 } 2941 native_recycle(); 2942 mValid = false; 2943 mNativeContext = 0; 2944 } 2945 sPool.offer(this); 2946 } 2947 native_recycle()2948 private native void native_recycle(); 2949 native_obtain(int capacity, String[] codecNames)2950 private native void native_obtain(int capacity, String[] codecNames); 2951 2952 @Override finalize()2953 protected void finalize() { 2954 native_recycle(); 2955 } 2956 2957 /** 2958 * Returns true if it is possible to allocate a linear block that can be 2959 * passed to all listed codecs as input buffers without copying the 2960 * content. 2961 * <p> 2962 * Note that even if this function returns true, {@link #obtain} may 2963 * still throw due to invalid arguments or allocation failure. 2964 * 2965 * @param codecNames list of codecs that the client wants to use a 2966 * linear block without copying. Null entries are 2967 * ignored. 2968 */ isCodecCopyFreeCompatible(@onNull String[] codecNames)2969 public static boolean isCodecCopyFreeCompatible(@NonNull String[] codecNames) { 2970 return native_checkCompatible(codecNames); 2971 } 2972 native_checkCompatible(@onNull String[] codecNames)2973 private static native boolean native_checkCompatible(@NonNull String[] codecNames); 2974 2975 /** 2976 * Obtain a linear block object no smaller than {@code capacity}. 2977 * If {@link #isCodecCopyFreeCompatible} with the same 2978 * {@code codecNames} returned true, the returned 2979 * {@link LinearBlock} object can be queued to the listed codecs without 2980 * copying. The returned {@link LinearBlock} object is always 2981 * read/write mappable. 2982 * 2983 * @param capacity requested capacity of the linear block in bytes 2984 * @param codecNames list of codecs that the client wants to use this 2985 * linear block without copying. Null entries are 2986 * ignored. 2987 * @return a linear block object. 2988 * @throws IllegalArgumentException if the capacity is invalid or 2989 * codecNames contains invalid name 2990 * @throws IOException if an error occurred while allocating a buffer 2991 */ obtain( int capacity, @NonNull String[] codecNames)2992 public static @Nullable LinearBlock obtain( 2993 int capacity, @NonNull String[] codecNames) { 2994 LinearBlock buffer = sPool.poll(); 2995 if (buffer == null) { 2996 buffer = new LinearBlock(); 2997 } 2998 synchronized (buffer.mLock) { 2999 buffer.native_obtain(capacity, codecNames); 3000 } 3001 return buffer; 3002 } 3003 3004 // Called from native setInternalStateLocked(long context, boolean isMappable)3005 private void setInternalStateLocked(long context, boolean isMappable) { 3006 mNativeContext = context; 3007 mMappable = isMappable; 3008 mValid = (context != 0); 3009 } 3010 3011 private static final BlockingQueue<LinearBlock> sPool = 3012 new LinkedBlockingQueue<>(); 3013 3014 private final Object mLock = new Object(); 3015 private boolean mValid = false; 3016 private boolean mMappable = false; 3017 private ByteBuffer mMapped = null; 3018 private long mNativeContext = 0; 3019 } 3020 3021 /** 3022 * Map a {@link HardwareBuffer} object into {@link Image}, so that the content of the buffer is 3023 * accessible. Depending on the usage and pixel format of the hardware buffer, it may not be 3024 * mappable; this method returns null in that case. 3025 * 3026 * @param hardwareBuffer {@link HardwareBuffer} to map. 3027 * @return Mapped {@link Image} object, or null if the buffer is not mappable. 3028 */ mapHardwareBuffer(@onNull HardwareBuffer hardwareBuffer)3029 public static @Nullable Image mapHardwareBuffer(@NonNull HardwareBuffer hardwareBuffer) { 3030 return native_mapHardwareBuffer(hardwareBuffer); 3031 } 3032 native_mapHardwareBuffer( @onNull HardwareBuffer hardwareBuffer)3033 private static native @Nullable Image native_mapHardwareBuffer( 3034 @NonNull HardwareBuffer hardwareBuffer); 3035 native_closeMediaImage(long context)3036 private static native void native_closeMediaImage(long context); 3037 3038 /** 3039 * Builder-like class for queue requests. Use this class to prepare a 3040 * queue request and send it. 3041 */ 3042 public final class QueueRequest { 3043 // No public constructor QueueRequest(@onNull MediaCodec codec, int index)3044 private QueueRequest(@NonNull MediaCodec codec, int index) { 3045 mCodec = codec; 3046 mIndex = index; 3047 } 3048 3049 /** 3050 * Set a linear block to this queue request. Exactly one buffer must be 3051 * set for a queue request before calling {@link #queue}. It is possible 3052 * to use the same {@link LinearBlock} object for multiple queue 3053 * requests. The behavior is undefined if the range of the buffer 3054 * overlaps for multiple requests, or the application writes into the 3055 * region being processed by the codec. 3056 * 3057 * @param block The linear block object 3058 * @param offset The byte offset into the input buffer at which the data starts. 3059 * @param size The number of bytes of valid input data. 3060 * @return this object 3061 * @throws IllegalStateException if a buffer is already set 3062 */ setLinearBlock( @onNull LinearBlock block, int offset, int size)3063 public @NonNull QueueRequest setLinearBlock( 3064 @NonNull LinearBlock block, 3065 int offset, 3066 int size) { 3067 if (!isAccessible()) { 3068 throw new IllegalStateException("The request is stale"); 3069 } 3070 if (mLinearBlock != null || mHardwareBuffer != null) { 3071 throw new IllegalStateException("Cannot set block twice"); 3072 } 3073 mLinearBlock = block; 3074 mOffset = offset; 3075 mSize = size; 3076 mCryptoInfo = null; 3077 return this; 3078 } 3079 3080 /** 3081 * Set an encrypted linear block to this queue request. Exactly one buffer must be 3082 * set for a queue request before calling {@link #queue}. It is possible 3083 * to use the same {@link LinearBlock} object for multiple queue 3084 * requests. The behavior is undefined if the range of the buffer 3085 * overlaps for multiple requests, or the application writes into the 3086 * region being processed by the codec. 3087 * 3088 * @param block The linear block object 3089 * @param offset The byte offset into the input buffer at which the data starts. 3090 * @param size The number of bytes of valid input data. 3091 * @param cryptoInfo Metadata describing the structure of the encrypted input sample. 3092 * @return this object 3093 * @throws IllegalStateException if a buffer is already set 3094 */ setEncryptedLinearBlock( @onNull LinearBlock block, int offset, int size, @NonNull MediaCodec.CryptoInfo cryptoInfo)3095 public @NonNull QueueRequest setEncryptedLinearBlock( 3096 @NonNull LinearBlock block, 3097 int offset, 3098 int size, 3099 @NonNull MediaCodec.CryptoInfo cryptoInfo) { 3100 Objects.requireNonNull(cryptoInfo); 3101 if (!isAccessible()) { 3102 throw new IllegalStateException("The request is stale"); 3103 } 3104 if (mLinearBlock != null || mHardwareBuffer != null) { 3105 throw new IllegalStateException("Cannot set block twice"); 3106 } 3107 mLinearBlock = block; 3108 mOffset = offset; 3109 mSize = size; 3110 mCryptoInfo = cryptoInfo; 3111 return this; 3112 } 3113 3114 /** 3115 * Set a harware graphic buffer to this queue request. Exactly one buffer must 3116 * be set for a queue request before calling {@link #queue}. 3117 * <p> 3118 * Note: buffers should have format {@link HardwareBuffer#YCBCR_420_888}, 3119 * a single layer, and an appropriate usage ({@link HardwareBuffer#USAGE_CPU_READ_OFTEN} 3120 * for software codecs and {@link HardwareBuffer#USAGE_VIDEO_ENCODE} for hardware) 3121 * for codecs to recognize. Codecs may throw exception if the buffer is not recognizable. 3122 * 3123 * @param buffer The hardware graphic buffer object 3124 * @return this object 3125 * @throws IllegalStateException if a buffer is already set 3126 */ setHardwareBuffer( @onNull HardwareBuffer buffer)3127 public @NonNull QueueRequest setHardwareBuffer( 3128 @NonNull HardwareBuffer buffer) { 3129 if (!isAccessible()) { 3130 throw new IllegalStateException("The request is stale"); 3131 } 3132 if (mLinearBlock != null || mHardwareBuffer != null) { 3133 throw new IllegalStateException("Cannot set block twice"); 3134 } 3135 mHardwareBuffer = buffer; 3136 return this; 3137 } 3138 3139 /** 3140 * Set timestamp to this queue request. 3141 * 3142 * @param presentationTimeUs The presentation timestamp in microseconds for this 3143 * buffer. This is normally the media time at which this 3144 * buffer should be presented (rendered). When using an output 3145 * surface, this will be propagated as the {@link 3146 * SurfaceTexture#getTimestamp timestamp} for the frame (after 3147 * conversion to nanoseconds). 3148 * @return this object 3149 */ setPresentationTimeUs(long presentationTimeUs)3150 public @NonNull QueueRequest setPresentationTimeUs(long presentationTimeUs) { 3151 if (!isAccessible()) { 3152 throw new IllegalStateException("The request is stale"); 3153 } 3154 mPresentationTimeUs = presentationTimeUs; 3155 return this; 3156 } 3157 3158 /** 3159 * Set flags to this queue request. 3160 * 3161 * @param flags A bitmask of flags 3162 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 3163 * While not prohibited, most codecs do not use the 3164 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 3165 * @return this object 3166 */ setFlags(@ufferFlag int flags)3167 public @NonNull QueueRequest setFlags(@BufferFlag int flags) { 3168 if (!isAccessible()) { 3169 throw new IllegalStateException("The request is stale"); 3170 } 3171 mFlags = flags; 3172 return this; 3173 } 3174 3175 /** 3176 * Add an integer parameter. 3177 * See {@link MediaFormat} for an exhaustive list of supported keys with 3178 * values of type int, that can also be set with {@link MediaFormat#setInteger}. 3179 * 3180 * If there was {@link MediaCodec#setParameters} 3181 * call with the same key which is not processed by the codec yet, the 3182 * value set from this method will override the unprocessed value. 3183 * 3184 * @return this object 3185 */ setIntegerParameter( @onNull String key, int value)3186 public @NonNull QueueRequest setIntegerParameter( 3187 @NonNull String key, int value) { 3188 if (!isAccessible()) { 3189 throw new IllegalStateException("The request is stale"); 3190 } 3191 mTuningKeys.add(key); 3192 mTuningValues.add(Integer.valueOf(value)); 3193 return this; 3194 } 3195 3196 /** 3197 * Add a long parameter. 3198 * See {@link MediaFormat} for an exhaustive list of supported keys with 3199 * values of type long, that can also be set with {@link MediaFormat#setLong}. 3200 * 3201 * If there was {@link MediaCodec#setParameters} 3202 * call with the same key which is not processed by the codec yet, the 3203 * value set from this method will override the unprocessed value. 3204 * 3205 * @return this object 3206 */ setLongParameter( @onNull String key, long value)3207 public @NonNull QueueRequest setLongParameter( 3208 @NonNull String key, long value) { 3209 if (!isAccessible()) { 3210 throw new IllegalStateException("The request is stale"); 3211 } 3212 mTuningKeys.add(key); 3213 mTuningValues.add(Long.valueOf(value)); 3214 return this; 3215 } 3216 3217 /** 3218 * Add a float parameter. 3219 * See {@link MediaFormat} for an exhaustive list of supported keys with 3220 * values of type float, that can also be set with {@link MediaFormat#setFloat}. 3221 * 3222 * If there was {@link MediaCodec#setParameters} 3223 * call with the same key which is not processed by the codec yet, the 3224 * value set from this method will override the unprocessed value. 3225 * 3226 * @return this object 3227 */ setFloatParameter( @onNull String key, float value)3228 public @NonNull QueueRequest setFloatParameter( 3229 @NonNull String key, float value) { 3230 if (!isAccessible()) { 3231 throw new IllegalStateException("The request is stale"); 3232 } 3233 mTuningKeys.add(key); 3234 mTuningValues.add(Float.valueOf(value)); 3235 return this; 3236 } 3237 3238 /** 3239 * Add a {@link ByteBuffer} parameter. 3240 * See {@link MediaFormat} for an exhaustive list of supported keys with 3241 * values of byte buffer, that can also be set with {@link MediaFormat#setByteBuffer}. 3242 * 3243 * If there was {@link MediaCodec#setParameters} 3244 * call with the same key which is not processed by the codec yet, the 3245 * value set from this method will override the unprocessed value. 3246 * 3247 * @return this object 3248 */ setByteBufferParameter( @onNull String key, @NonNull ByteBuffer value)3249 public @NonNull QueueRequest setByteBufferParameter( 3250 @NonNull String key, @NonNull ByteBuffer value) { 3251 if (!isAccessible()) { 3252 throw new IllegalStateException("The request is stale"); 3253 } 3254 mTuningKeys.add(key); 3255 mTuningValues.add(value); 3256 return this; 3257 } 3258 3259 /** 3260 * Add a string parameter. 3261 * See {@link MediaFormat} for an exhaustive list of supported keys with 3262 * values of type string, that can also be set with {@link MediaFormat#setString}. 3263 * 3264 * If there was {@link MediaCodec#setParameters} 3265 * call with the same key which is not processed by the codec yet, the 3266 * value set from this method will override the unprocessed value. 3267 * 3268 * @return this object 3269 */ setStringParameter( @onNull String key, @NonNull String value)3270 public @NonNull QueueRequest setStringParameter( 3271 @NonNull String key, @NonNull String value) { 3272 if (!isAccessible()) { 3273 throw new IllegalStateException("The request is stale"); 3274 } 3275 mTuningKeys.add(key); 3276 mTuningValues.add(value); 3277 return this; 3278 } 3279 3280 /** 3281 * Finish building a queue request and queue the buffers with tunings. 3282 */ queue()3283 public void queue() { 3284 if (!isAccessible()) { 3285 throw new IllegalStateException("The request is stale"); 3286 } 3287 if (mLinearBlock == null && mHardwareBuffer == null) { 3288 throw new IllegalStateException("No block is set"); 3289 } 3290 setAccessible(false); 3291 if (mLinearBlock != null) { 3292 mCodec.native_queueLinearBlock( 3293 mIndex, mLinearBlock, mOffset, mSize, mCryptoInfo, 3294 mPresentationTimeUs, mFlags, 3295 mTuningKeys, mTuningValues); 3296 } else if (mHardwareBuffer != null) { 3297 mCodec.native_queueHardwareBuffer( 3298 mIndex, mHardwareBuffer, mPresentationTimeUs, mFlags, 3299 mTuningKeys, mTuningValues); 3300 } 3301 clear(); 3302 } 3303 clear()3304 @NonNull QueueRequest clear() { 3305 mLinearBlock = null; 3306 mOffset = 0; 3307 mSize = 0; 3308 mCryptoInfo = null; 3309 mHardwareBuffer = null; 3310 mPresentationTimeUs = 0; 3311 mFlags = 0; 3312 mTuningKeys.clear(); 3313 mTuningValues.clear(); 3314 return this; 3315 } 3316 isAccessible()3317 boolean isAccessible() { 3318 return mAccessible; 3319 } 3320 setAccessible(boolean accessible)3321 @NonNull QueueRequest setAccessible(boolean accessible) { 3322 mAccessible = accessible; 3323 return this; 3324 } 3325 3326 private final MediaCodec mCodec; 3327 private final int mIndex; 3328 private LinearBlock mLinearBlock = null; 3329 private int mOffset = 0; 3330 private int mSize = 0; 3331 private MediaCodec.CryptoInfo mCryptoInfo = null; 3332 private HardwareBuffer mHardwareBuffer = null; 3333 private long mPresentationTimeUs = 0; 3334 private @BufferFlag int mFlags = 0; 3335 private final ArrayList<String> mTuningKeys = new ArrayList<>(); 3336 private final ArrayList<Object> mTuningValues = new ArrayList<>(); 3337 3338 private boolean mAccessible = false; 3339 } 3340 native_queueLinearBlock( int index, @NonNull LinearBlock block, int offset, int size, @Nullable CryptoInfo cryptoInfo, long presentationTimeUs, int flags, @NonNull ArrayList<String> keys, @NonNull ArrayList<Object> values)3341 private native void native_queueLinearBlock( 3342 int index, 3343 @NonNull LinearBlock block, 3344 int offset, 3345 int size, 3346 @Nullable CryptoInfo cryptoInfo, 3347 long presentationTimeUs, 3348 int flags, 3349 @NonNull ArrayList<String> keys, 3350 @NonNull ArrayList<Object> values); 3351 native_queueHardwareBuffer( int index, @NonNull HardwareBuffer buffer, long presentationTimeUs, int flags, @NonNull ArrayList<String> keys, @NonNull ArrayList<Object> values)3352 private native void native_queueHardwareBuffer( 3353 int index, 3354 @NonNull HardwareBuffer buffer, 3355 long presentationTimeUs, 3356 int flags, 3357 @NonNull ArrayList<String> keys, 3358 @NonNull ArrayList<Object> values); 3359 3360 private final ArrayList<QueueRequest> mQueueRequests = new ArrayList<>(); 3361 3362 /** 3363 * Return a {@link QueueRequest} object for an input slot index. 3364 * 3365 * @param index input slot index from 3366 * {@link Callback#onInputBufferAvailable} 3367 * @return queue request object 3368 * @throws IllegalStateException if not using block model 3369 * @throws IllegalArgumentException if the input slot is not available or 3370 * the index is out of range 3371 */ getQueueRequest(int index)3372 public @NonNull QueueRequest getQueueRequest(int index) { 3373 synchronized (mBufferLock) { 3374 if (mBufferMode != BUFFER_MODE_BLOCK) { 3375 throw new IllegalStateException("The codec is not configured for block model"); 3376 } 3377 if (index < 0 || index >= mQueueRequests.size()) { 3378 throw new IndexOutOfBoundsException("Expected range of index: [0," 3379 + (mQueueRequests.size() - 1) + "]; actual: " + index); 3380 } 3381 QueueRequest request = mQueueRequests.get(index); 3382 if (request == null) { 3383 throw new IllegalArgumentException("Unavailable index: " + index); 3384 } 3385 if (!request.isAccessible()) { 3386 throw new IllegalArgumentException( 3387 "The request is stale at index " + index); 3388 } 3389 return request.clear(); 3390 } 3391 } 3392 3393 /** 3394 * If a non-negative timeout had been specified in the call 3395 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 3396 */ 3397 public static final int INFO_TRY_AGAIN_LATER = -1; 3398 3399 /** 3400 * The output format has changed, subsequent data will follow the new 3401 * format. {@link #getOutputFormat()} returns the new format. Note, that 3402 * you can also use the new {@link #getOutputFormat(int)} method to 3403 * get the format for a specific output buffer. This frees you from 3404 * having to track output format changes. 3405 */ 3406 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 3407 3408 /** 3409 * The output buffers have changed, the client must refer to the new 3410 * set of output buffers returned by {@link #getOutputBuffers} from 3411 * this point on. 3412 * 3413 * <p>Additionally, this event signals that the video scaling mode 3414 * may have been reset to the default.</p> 3415 * 3416 * @deprecated This return value can be ignored as {@link 3417 * #getOutputBuffers} has been deprecated. Client should 3418 * request a current buffer using on of the get-buffer or 3419 * get-image methods each time one has been dequeued. 3420 */ 3421 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 3422 3423 /** @hide */ 3424 @IntDef({ 3425 INFO_TRY_AGAIN_LATER, 3426 INFO_OUTPUT_FORMAT_CHANGED, 3427 INFO_OUTPUT_BUFFERS_CHANGED, 3428 }) 3429 @Retention(RetentionPolicy.SOURCE) 3430 public @interface OutputBufferInfo {} 3431 3432 /** 3433 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 3434 * Returns the index of an output buffer that has been successfully 3435 * decoded or one of the INFO_* constants. 3436 * @param info Will be filled with buffer meta data. 3437 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 3438 * @throws IllegalStateException if not in the Executing state, 3439 * or codec is configured in asynchronous mode. 3440 * @throws MediaCodec.CodecException upon codec error. 3441 */ 3442 @OutputBufferInfo dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)3443 public final int dequeueOutputBuffer( 3444 @NonNull BufferInfo info, long timeoutUs) { 3445 synchronized (mBufferLock) { 3446 if (mBufferMode == BUFFER_MODE_BLOCK) { 3447 throw new IncompatibleWithBlockModelException("dequeueOutputBuffer() " 3448 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3449 + "Please use MediaCodec.Callback objects to get output buffer slots."); 3450 } 3451 } 3452 int res = native_dequeueOutputBuffer(info, timeoutUs); 3453 synchronized (mBufferLock) { 3454 if (res == INFO_OUTPUT_BUFFERS_CHANGED) { 3455 cacheBuffers(false /* input */); 3456 } else if (res >= 0) { 3457 validateOutputByteBuffer(mCachedOutputBuffers, res, info); 3458 if (mHasSurface) { 3459 mDequeuedOutputInfos.put(res, info.dup()); 3460 } 3461 } 3462 } 3463 return res; 3464 } 3465 native_dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)3466 private native final int native_dequeueOutputBuffer( 3467 @NonNull BufferInfo info, long timeoutUs); 3468 3469 /** 3470 * If you are done with a buffer, use this call to return the buffer to the codec 3471 * or to render it on the output surface. If you configured the codec with an 3472 * output surface, setting {@code render} to {@code true} will first send the buffer 3473 * to that output surface. The surface will release the buffer back to the codec once 3474 * it is no longer used/displayed. 3475 * 3476 * Once an output buffer is released to the codec, it MUST NOT 3477 * be used until it is later retrieved by {@link #getOutputBuffer} in response 3478 * to a {@link #dequeueOutputBuffer} return value or a 3479 * {@link Callback#onOutputBufferAvailable} callback. 3480 * 3481 * @param index The index of a client-owned output buffer previously returned 3482 * from a call to {@link #dequeueOutputBuffer}. 3483 * @param render If a valid surface was specified when configuring the codec, 3484 * passing true renders this output buffer to the surface. 3485 * @throws IllegalStateException if not in the Executing state. 3486 * @throws MediaCodec.CodecException upon codec error. 3487 */ releaseOutputBuffer(int index, boolean render)3488 public final void releaseOutputBuffer(int index, boolean render) { 3489 releaseOutputBufferInternal(index, render, false /* updatePTS */, 0 /* dummy */); 3490 } 3491 3492 /** 3493 * If you are done with a buffer, use this call to update its surface timestamp 3494 * and return it to the codec to render it on the output surface. If you 3495 * have not specified an output surface when configuring this video codec, 3496 * this call will simply return the buffer to the codec.<p> 3497 * 3498 * The timestamp may have special meaning depending on the destination surface. 3499 * 3500 * <table> 3501 * <tr><th>SurfaceView specifics</th></tr> 3502 * <tr><td> 3503 * If you render your buffer on a {@link android.view.SurfaceView}, 3504 * you can use the timestamp to render the buffer at a specific time (at the 3505 * VSYNC at or after the buffer timestamp). For this to work, the timestamp 3506 * needs to be <i>reasonably close</i> to the current {@link System#nanoTime}. 3507 * Currently, this is set as within one (1) second. A few notes: 3508 * 3509 * <ul> 3510 * <li>the buffer will not be returned to the codec until the timestamp 3511 * has passed and the buffer is no longer used by the {@link android.view.Surface}. 3512 * <li>buffers are processed sequentially, so you may block subsequent buffers to 3513 * be displayed on the {@link android.view.Surface}. This is important if you 3514 * want to react to user action, e.g. stop the video or seek. 3515 * <li>if multiple buffers are sent to the {@link android.view.Surface} to be 3516 * rendered at the same VSYNC, the last one will be shown, and the other ones 3517 * will be dropped. 3518 * <li>if the timestamp is <em>not</em> "reasonably close" to the current system 3519 * time, the {@link android.view.Surface} will ignore the timestamp, and 3520 * display the buffer at the earliest feasible time. In this mode it will not 3521 * drop frames. 3522 * <li>for best performance and quality, call this method when you are about 3523 * two VSYNCs' time before the desired render time. For 60Hz displays, this is 3524 * about 33 msec. 3525 * </ul> 3526 * </td></tr> 3527 * </table> 3528 * 3529 * Once an output buffer is released to the codec, it MUST NOT 3530 * be used until it is later retrieved by {@link #getOutputBuffer} in response 3531 * to a {@link #dequeueOutputBuffer} return value or a 3532 * {@link Callback#onOutputBufferAvailable} callback. 3533 * 3534 * @param index The index of a client-owned output buffer previously returned 3535 * from a call to {@link #dequeueOutputBuffer}. 3536 * @param renderTimestampNs The timestamp to associate with this buffer when 3537 * it is sent to the Surface. 3538 * @throws IllegalStateException if not in the Executing state. 3539 * @throws MediaCodec.CodecException upon codec error. 3540 */ releaseOutputBuffer(int index, long renderTimestampNs)3541 public final void releaseOutputBuffer(int index, long renderTimestampNs) { 3542 releaseOutputBufferInternal( 3543 index, true /* render */, true /* updatePTS */, renderTimestampNs); 3544 } 3545 releaseOutputBufferInternal( int index, boolean render, boolean updatePts, long renderTimestampNs)3546 private void releaseOutputBufferInternal( 3547 int index, boolean render, boolean updatePts, long renderTimestampNs) { 3548 BufferInfo info = null; 3549 synchronized(mBufferLock) { 3550 switch (mBufferMode) { 3551 case BUFFER_MODE_LEGACY: 3552 invalidateByteBuffer(mCachedOutputBuffers, index); 3553 mDequeuedOutputBuffers.remove(index); 3554 if (mHasSurface) { 3555 info = mDequeuedOutputInfos.remove(index); 3556 } 3557 break; 3558 case BUFFER_MODE_BLOCK: 3559 OutputFrame frame = mOutputFrames.get(index); 3560 frame.setAccessible(false); 3561 frame.clear(); 3562 break; 3563 default: 3564 throw new IllegalStateException( 3565 "Unrecognized buffer mode: " + mBufferMode); 3566 } 3567 } 3568 releaseOutputBuffer( 3569 index, render, updatePts, renderTimestampNs); 3570 } 3571 3572 @UnsupportedAppUsage releaseOutputBuffer( int index, boolean render, boolean updatePTS, long timeNs)3573 private native final void releaseOutputBuffer( 3574 int index, boolean render, boolean updatePTS, long timeNs); 3575 3576 /** 3577 * Signals end-of-stream on input. Equivalent to submitting an empty buffer with 3578 * {@link #BUFFER_FLAG_END_OF_STREAM} set. This may only be used with 3579 * encoders receiving input from a Surface created by {@link #createInputSurface}. 3580 * @throws IllegalStateException if not in the Executing state. 3581 * @throws MediaCodec.CodecException upon codec error. 3582 */ signalEndOfInputStream()3583 public native final void signalEndOfInputStream(); 3584 3585 /** 3586 * Call this after dequeueOutputBuffer signals a format change by returning 3587 * {@link #INFO_OUTPUT_FORMAT_CHANGED}. 3588 * You can also call this after {@link #configure} returns 3589 * successfully to get the output format initially configured 3590 * for the codec. Do this to determine what optional 3591 * configuration parameters were supported by the codec. 3592 * 3593 * @throws IllegalStateException if not in the Executing or 3594 * Configured state. 3595 * @throws MediaCodec.CodecException upon codec error. 3596 */ 3597 @NonNull getOutputFormat()3598 public final MediaFormat getOutputFormat() { 3599 return new MediaFormat(getFormatNative(false /* input */)); 3600 } 3601 3602 /** 3603 * Call this after {@link #configure} returns successfully to 3604 * get the input format accepted by the codec. Do this to 3605 * determine what optional configuration parameters were 3606 * supported by the codec. 3607 * 3608 * @throws IllegalStateException if not in the Executing or 3609 * Configured state. 3610 * @throws MediaCodec.CodecException upon codec error. 3611 */ 3612 @NonNull getInputFormat()3613 public final MediaFormat getInputFormat() { 3614 return new MediaFormat(getFormatNative(true /* input */)); 3615 } 3616 3617 /** 3618 * Returns the output format for a specific output buffer. 3619 * 3620 * @param index The index of a client-owned input buffer previously 3621 * returned from a call to {@link #dequeueInputBuffer}. 3622 * 3623 * @return the format for the output buffer, or null if the index 3624 * is not a dequeued output buffer. 3625 */ 3626 @NonNull getOutputFormat(int index)3627 public final MediaFormat getOutputFormat(int index) { 3628 return new MediaFormat(getOutputFormatNative(index)); 3629 } 3630 3631 @NonNull getFormatNative(boolean input)3632 private native final Map<String, Object> getFormatNative(boolean input); 3633 3634 @NonNull getOutputFormatNative(int index)3635 private native final Map<String, Object> getOutputFormatNative(int index); 3636 3637 // used to track dequeued buffers 3638 private static class BufferMap { 3639 // various returned representations of the codec buffer 3640 private static class CodecBuffer { 3641 private Image mImage; 3642 private ByteBuffer mByteBuffer; 3643 free()3644 public void free() { 3645 if (mByteBuffer != null) { 3646 // all of our ByteBuffers are direct 3647 java.nio.NioUtils.freeDirectBuffer(mByteBuffer); 3648 mByteBuffer = null; 3649 } 3650 if (mImage != null) { 3651 mImage.close(); 3652 mImage = null; 3653 } 3654 } 3655 setImage(@ullable Image image)3656 public void setImage(@Nullable Image image) { 3657 free(); 3658 mImage = image; 3659 } 3660 setByteBuffer(@ullable ByteBuffer buffer)3661 public void setByteBuffer(@Nullable ByteBuffer buffer) { 3662 free(); 3663 mByteBuffer = buffer; 3664 } 3665 } 3666 3667 private final Map<Integer, CodecBuffer> mMap = 3668 new HashMap<Integer, CodecBuffer>(); 3669 remove(int index)3670 public void remove(int index) { 3671 CodecBuffer buffer = mMap.get(index); 3672 if (buffer != null) { 3673 buffer.free(); 3674 mMap.remove(index); 3675 } 3676 } 3677 put(int index, @Nullable ByteBuffer newBuffer)3678 public void put(int index, @Nullable ByteBuffer newBuffer) { 3679 CodecBuffer buffer = mMap.get(index); 3680 if (buffer == null) { // likely 3681 buffer = new CodecBuffer(); 3682 mMap.put(index, buffer); 3683 } 3684 buffer.setByteBuffer(newBuffer); 3685 } 3686 put(int index, @Nullable Image newImage)3687 public void put(int index, @Nullable Image newImage) { 3688 CodecBuffer buffer = mMap.get(index); 3689 if (buffer == null) { // likely 3690 buffer = new CodecBuffer(); 3691 mMap.put(index, buffer); 3692 } 3693 buffer.setImage(newImage); 3694 } 3695 clear()3696 public void clear() { 3697 for (CodecBuffer buffer: mMap.values()) { 3698 buffer.free(); 3699 } 3700 mMap.clear(); 3701 } 3702 } 3703 3704 private ByteBuffer[] mCachedInputBuffers; 3705 private ByteBuffer[] mCachedOutputBuffers; 3706 private final BufferMap mDequeuedInputBuffers = new BufferMap(); 3707 private final BufferMap mDequeuedOutputBuffers = new BufferMap(); 3708 private final Map<Integer, BufferInfo> mDequeuedOutputInfos = 3709 new HashMap<Integer, BufferInfo>(); 3710 final private Object mBufferLock; 3711 invalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)3712 private final void invalidateByteBuffer( 3713 @Nullable ByteBuffer[] buffers, int index) { 3714 if (buffers != null && index >= 0 && index < buffers.length) { 3715 ByteBuffer buffer = buffers[index]; 3716 if (buffer != null) { 3717 buffer.setAccessible(false); 3718 } 3719 } 3720 } 3721 validateInputByteBuffer( @ullable ByteBuffer[] buffers, int index)3722 private final void validateInputByteBuffer( 3723 @Nullable ByteBuffer[] buffers, int index) { 3724 if (buffers != null && index >= 0 && index < buffers.length) { 3725 ByteBuffer buffer = buffers[index]; 3726 if (buffer != null) { 3727 buffer.setAccessible(true); 3728 buffer.clear(); 3729 } 3730 } 3731 } 3732 revalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)3733 private final void revalidateByteBuffer( 3734 @Nullable ByteBuffer[] buffers, int index) { 3735 synchronized(mBufferLock) { 3736 if (buffers != null && index >= 0 && index < buffers.length) { 3737 ByteBuffer buffer = buffers[index]; 3738 if (buffer != null) { 3739 buffer.setAccessible(true); 3740 } 3741 } 3742 } 3743 } 3744 validateOutputByteBuffer( @ullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info)3745 private final void validateOutputByteBuffer( 3746 @Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) { 3747 if (buffers != null && index >= 0 && index < buffers.length) { 3748 ByteBuffer buffer = buffers[index]; 3749 if (buffer != null) { 3750 buffer.setAccessible(true); 3751 buffer.limit(info.offset + info.size).position(info.offset); 3752 } 3753 } 3754 } 3755 invalidateByteBuffers(@ullable ByteBuffer[] buffers)3756 private final void invalidateByteBuffers(@Nullable ByteBuffer[] buffers) { 3757 if (buffers != null) { 3758 for (ByteBuffer buffer: buffers) { 3759 if (buffer != null) { 3760 buffer.setAccessible(false); 3761 } 3762 } 3763 } 3764 } 3765 freeByteBuffer(@ullable ByteBuffer buffer)3766 private final void freeByteBuffer(@Nullable ByteBuffer buffer) { 3767 if (buffer != null /* && buffer.isDirect() */) { 3768 // all of our ByteBuffers are direct 3769 java.nio.NioUtils.freeDirectBuffer(buffer); 3770 } 3771 } 3772 freeByteBuffers(@ullable ByteBuffer[] buffers)3773 private final void freeByteBuffers(@Nullable ByteBuffer[] buffers) { 3774 if (buffers != null) { 3775 for (ByteBuffer buffer: buffers) { 3776 freeByteBuffer(buffer); 3777 } 3778 } 3779 } 3780 freeAllTrackedBuffers()3781 private final void freeAllTrackedBuffers() { 3782 synchronized(mBufferLock) { 3783 freeByteBuffers(mCachedInputBuffers); 3784 freeByteBuffers(mCachedOutputBuffers); 3785 mCachedInputBuffers = null; 3786 mCachedOutputBuffers = null; 3787 mDequeuedInputBuffers.clear(); 3788 mDequeuedOutputBuffers.clear(); 3789 mQueueRequests.clear(); 3790 mOutputFrames.clear(); 3791 } 3792 } 3793 cacheBuffers(boolean input)3794 private final void cacheBuffers(boolean input) { 3795 ByteBuffer[] buffers = null; 3796 try { 3797 buffers = getBuffers(input); 3798 invalidateByteBuffers(buffers); 3799 } catch (IllegalStateException e) { 3800 // we don't get buffers in async mode 3801 } 3802 if (input) { 3803 mCachedInputBuffers = buffers; 3804 } else { 3805 mCachedOutputBuffers = buffers; 3806 } 3807 } 3808 3809 /** 3810 * Retrieve the set of input buffers. Call this after start() 3811 * returns. After calling this method, any ByteBuffers 3812 * previously returned by an earlier call to this method MUST no 3813 * longer be used. 3814 * 3815 * @deprecated Use the new {@link #getInputBuffer} method instead 3816 * each time an input buffer is dequeued. 3817 * 3818 * <b>Note:</b> As of API 21, dequeued input buffers are 3819 * automatically {@link java.nio.Buffer#clear cleared}. 3820 * 3821 * <em>Do not use this method if using an input surface.</em> 3822 * 3823 * @throws IllegalStateException if not in the Executing state, 3824 * or codec is configured in asynchronous mode. 3825 * @throws MediaCodec.CodecException upon codec error. 3826 */ 3827 @NonNull getInputBuffers()3828 public ByteBuffer[] getInputBuffers() { 3829 synchronized (mBufferLock) { 3830 if (mBufferMode == BUFFER_MODE_BLOCK) { 3831 throw new IncompatibleWithBlockModelException("getInputBuffers() " 3832 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3833 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 3834 + "objects and attach to QueueRequest objects."); 3835 } 3836 if (mCachedInputBuffers == null) { 3837 throw new IllegalStateException(); 3838 } 3839 // FIXME: check codec status 3840 return mCachedInputBuffers; 3841 } 3842 } 3843 3844 /** 3845 * Retrieve the set of output buffers. Call this after start() 3846 * returns and whenever dequeueOutputBuffer signals an output 3847 * buffer change by returning {@link 3848 * #INFO_OUTPUT_BUFFERS_CHANGED}. After calling this method, any 3849 * ByteBuffers previously returned by an earlier call to this 3850 * method MUST no longer be used. 3851 * 3852 * @deprecated Use the new {@link #getOutputBuffer} method instead 3853 * each time an output buffer is dequeued. This method is not 3854 * supported if codec is configured in asynchronous mode. 3855 * 3856 * <b>Note:</b> As of API 21, the position and limit of output 3857 * buffers that are dequeued will be set to the valid data 3858 * range. 3859 * 3860 * <em>Do not use this method if using an output surface.</em> 3861 * 3862 * @throws IllegalStateException if not in the Executing state, 3863 * or codec is configured in asynchronous mode. 3864 * @throws MediaCodec.CodecException upon codec error. 3865 */ 3866 @NonNull getOutputBuffers()3867 public ByteBuffer[] getOutputBuffers() { 3868 synchronized (mBufferLock) { 3869 if (mBufferMode == BUFFER_MODE_BLOCK) { 3870 throw new IncompatibleWithBlockModelException("getOutputBuffers() " 3871 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3872 + "Please use getOutputFrame to get output frames."); 3873 } 3874 if (mCachedOutputBuffers == null) { 3875 throw new IllegalStateException(); 3876 } 3877 // FIXME: check codec status 3878 return mCachedOutputBuffers; 3879 } 3880 } 3881 3882 /** 3883 * Returns a {@link java.nio.Buffer#clear cleared}, writable ByteBuffer 3884 * object for a dequeued input buffer index to contain the input data. 3885 * 3886 * After calling this method any ByteBuffer or Image object 3887 * previously returned for the same input index MUST no longer 3888 * be used. 3889 * 3890 * @param index The index of a client-owned input buffer previously 3891 * returned from a call to {@link #dequeueInputBuffer}, 3892 * or received via an onInputBufferAvailable callback. 3893 * 3894 * @return the input buffer, or null if the index is not a dequeued 3895 * input buffer, or if the codec is configured for surface input. 3896 * 3897 * @throws IllegalStateException if not in the Executing state. 3898 * @throws MediaCodec.CodecException upon codec error. 3899 */ 3900 @Nullable getInputBuffer(int index)3901 public ByteBuffer getInputBuffer(int index) { 3902 synchronized (mBufferLock) { 3903 if (mBufferMode == BUFFER_MODE_BLOCK) { 3904 throw new IncompatibleWithBlockModelException("getInputBuffer() " 3905 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3906 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 3907 + "objects and attach to QueueRequest objects."); 3908 } 3909 } 3910 ByteBuffer newBuffer = getBuffer(true /* input */, index); 3911 synchronized (mBufferLock) { 3912 invalidateByteBuffer(mCachedInputBuffers, index); 3913 mDequeuedInputBuffers.put(index, newBuffer); 3914 } 3915 return newBuffer; 3916 } 3917 3918 /** 3919 * Returns a writable Image object for a dequeued input buffer 3920 * index to contain the raw input video frame. 3921 * 3922 * After calling this method any ByteBuffer or Image object 3923 * previously returned for the same input index MUST no longer 3924 * be used. 3925 * 3926 * @param index The index of a client-owned input buffer previously 3927 * returned from a call to {@link #dequeueInputBuffer}, 3928 * or received via an onInputBufferAvailable callback. 3929 * 3930 * @return the input image, or null if the index is not a 3931 * dequeued input buffer, or not a ByteBuffer that contains a 3932 * raw image. 3933 * 3934 * @throws IllegalStateException if not in the Executing state. 3935 * @throws MediaCodec.CodecException upon codec error. 3936 */ 3937 @Nullable getInputImage(int index)3938 public Image getInputImage(int index) { 3939 synchronized (mBufferLock) { 3940 if (mBufferMode == BUFFER_MODE_BLOCK) { 3941 throw new IncompatibleWithBlockModelException("getInputImage() " 3942 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3943 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 3944 + "objects and attach to QueueRequest objects."); 3945 } 3946 } 3947 Image newImage = getImage(true /* input */, index); 3948 synchronized (mBufferLock) { 3949 invalidateByteBuffer(mCachedInputBuffers, index); 3950 mDequeuedInputBuffers.put(index, newImage); 3951 } 3952 return newImage; 3953 } 3954 3955 /** 3956 * Returns a read-only ByteBuffer for a dequeued output buffer 3957 * index. The position and limit of the returned buffer are set 3958 * to the valid output data. 3959 * 3960 * After calling this method, any ByteBuffer or Image object 3961 * previously returned for the same output index MUST no longer 3962 * be used. 3963 * 3964 * @param index The index of a client-owned output buffer previously 3965 * returned from a call to {@link #dequeueOutputBuffer}, 3966 * or received via an onOutputBufferAvailable callback. 3967 * 3968 * @return the output buffer, or null if the index is not a dequeued 3969 * output buffer, or the codec is configured with an output surface. 3970 * 3971 * @throws IllegalStateException if not in the Executing state. 3972 * @throws MediaCodec.CodecException upon codec error. 3973 */ 3974 @Nullable getOutputBuffer(int index)3975 public ByteBuffer getOutputBuffer(int index) { 3976 synchronized (mBufferLock) { 3977 if (mBufferMode == BUFFER_MODE_BLOCK) { 3978 throw new IncompatibleWithBlockModelException("getOutputBuffer() " 3979 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3980 + "Please use getOutputFrame() to get output frames."); 3981 } 3982 } 3983 ByteBuffer newBuffer = getBuffer(false /* input */, index); 3984 synchronized (mBufferLock) { 3985 invalidateByteBuffer(mCachedOutputBuffers, index); 3986 mDequeuedOutputBuffers.put(index, newBuffer); 3987 } 3988 return newBuffer; 3989 } 3990 3991 /** 3992 * Returns a read-only Image object for a dequeued output buffer 3993 * index that contains the raw video frame. 3994 * 3995 * After calling this method, any ByteBuffer or Image object previously 3996 * returned for the same output index MUST no longer be used. 3997 * 3998 * @param index The index of a client-owned output buffer previously 3999 * returned from a call to {@link #dequeueOutputBuffer}, 4000 * or received via an onOutputBufferAvailable callback. 4001 * 4002 * @return the output image, or null if the index is not a 4003 * dequeued output buffer, not a raw video frame, or if the codec 4004 * was configured with an output surface. 4005 * 4006 * @throws IllegalStateException if not in the Executing state. 4007 * @throws MediaCodec.CodecException upon codec error. 4008 */ 4009 @Nullable getOutputImage(int index)4010 public Image getOutputImage(int index) { 4011 synchronized (mBufferLock) { 4012 if (mBufferMode == BUFFER_MODE_BLOCK) { 4013 throw new IncompatibleWithBlockModelException("getOutputImage() " 4014 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4015 + "Please use getOutputFrame() to get output frames."); 4016 } 4017 } 4018 Image newImage = getImage(false /* input */, index); 4019 synchronized (mBufferLock) { 4020 invalidateByteBuffer(mCachedOutputBuffers, index); 4021 mDequeuedOutputBuffers.put(index, newImage); 4022 } 4023 return newImage; 4024 } 4025 4026 /** 4027 * A single output frame and its associated metadata. 4028 */ 4029 public static final class OutputFrame { 4030 // No public constructor OutputFrame(int index)4031 OutputFrame(int index) { 4032 mIndex = index; 4033 } 4034 4035 /** 4036 * Returns the output linear block, or null if this frame is empty. 4037 * 4038 * @throws IllegalStateException if this output frame is not linear. 4039 */ getLinearBlock()4040 public @Nullable LinearBlock getLinearBlock() { 4041 if (mHardwareBuffer != null) { 4042 throw new IllegalStateException("This output frame is not linear"); 4043 } 4044 return mLinearBlock; 4045 } 4046 4047 /** 4048 * Returns the output hardware graphic buffer, or null if this frame is empty. 4049 * 4050 * @throws IllegalStateException if this output frame is not graphic. 4051 */ getHardwareBuffer()4052 public @Nullable HardwareBuffer getHardwareBuffer() { 4053 if (mLinearBlock != null) { 4054 throw new IllegalStateException("This output frame is not graphic"); 4055 } 4056 return mHardwareBuffer; 4057 } 4058 4059 /** 4060 * Returns the presentation timestamp in microseconds. 4061 */ getPresentationTimeUs()4062 public long getPresentationTimeUs() { 4063 return mPresentationTimeUs; 4064 } 4065 4066 /** 4067 * Returns the buffer flags. 4068 */ getFlags()4069 public @BufferFlag int getFlags() { 4070 return mFlags; 4071 } 4072 4073 /** 4074 * Returns a read-only {@link MediaFormat} for this frame. The returned 4075 * object is valid only until the client calls {@link MediaCodec#releaseOutputBuffer}. 4076 */ getFormat()4077 public @NonNull MediaFormat getFormat() { 4078 return mFormat; 4079 } 4080 4081 /** 4082 * Returns an unmodifiable set of the names of entries that has changed from 4083 * the previous frame. The entries may have been removed/changed/added. 4084 * Client can find out what the change is by querying {@link MediaFormat} 4085 * object returned from {@link #getFormat}. 4086 */ getChangedKeys()4087 public @NonNull Set<String> getChangedKeys() { 4088 if (mKeySet.isEmpty() && !mChangedKeys.isEmpty()) { 4089 mKeySet.addAll(mChangedKeys); 4090 } 4091 return Collections.unmodifiableSet(mKeySet); 4092 } 4093 clear()4094 void clear() { 4095 mLinearBlock = null; 4096 mHardwareBuffer = null; 4097 mFormat = null; 4098 mChangedKeys.clear(); 4099 mKeySet.clear(); 4100 mLoaded = false; 4101 } 4102 isAccessible()4103 boolean isAccessible() { 4104 return mAccessible; 4105 } 4106 setAccessible(boolean accessible)4107 void setAccessible(boolean accessible) { 4108 mAccessible = accessible; 4109 } 4110 setBufferInfo(MediaCodec.BufferInfo info)4111 void setBufferInfo(MediaCodec.BufferInfo info) { 4112 mPresentationTimeUs = info.presentationTimeUs; 4113 mFlags = info.flags; 4114 } 4115 isLoaded()4116 boolean isLoaded() { 4117 return mLoaded; 4118 } 4119 setLoaded(boolean loaded)4120 void setLoaded(boolean loaded) { 4121 mLoaded = loaded; 4122 } 4123 4124 private final int mIndex; 4125 private LinearBlock mLinearBlock = null; 4126 private HardwareBuffer mHardwareBuffer = null; 4127 private long mPresentationTimeUs = 0; 4128 private @BufferFlag int mFlags = 0; 4129 private MediaFormat mFormat = null; 4130 private final ArrayList<String> mChangedKeys = new ArrayList<>(); 4131 private final Set<String> mKeySet = new HashSet<>(); 4132 private boolean mAccessible = false; 4133 private boolean mLoaded = false; 4134 } 4135 4136 private final ArrayList<OutputFrame> mOutputFrames = new ArrayList<>(); 4137 4138 /** 4139 * Returns an {@link OutputFrame} object. 4140 * 4141 * @param index output buffer index from 4142 * {@link Callback#onOutputBufferAvailable} 4143 * @return {@link OutputFrame} object describing the output buffer 4144 * @throws IllegalStateException if not using block model 4145 * @throws IllegalArgumentException if the output buffer is not available or 4146 * the index is out of range 4147 */ getOutputFrame(int index)4148 public @NonNull OutputFrame getOutputFrame(int index) { 4149 synchronized (mBufferLock) { 4150 if (mBufferMode != BUFFER_MODE_BLOCK) { 4151 throw new IllegalStateException("The codec is not configured for block model"); 4152 } 4153 if (index < 0 || index >= mOutputFrames.size()) { 4154 throw new IndexOutOfBoundsException("Expected range of index: [0," 4155 + (mQueueRequests.size() - 1) + "]; actual: " + index); 4156 } 4157 OutputFrame frame = mOutputFrames.get(index); 4158 if (frame == null) { 4159 throw new IllegalArgumentException("Unavailable index: " + index); 4160 } 4161 if (!frame.isAccessible()) { 4162 throw new IllegalArgumentException( 4163 "The output frame is stale at index " + index); 4164 } 4165 if (!frame.isLoaded()) { 4166 native_getOutputFrame(frame, index); 4167 frame.setLoaded(true); 4168 } 4169 return frame; 4170 } 4171 } 4172 native_getOutputFrame(OutputFrame frame, int index)4173 private native void native_getOutputFrame(OutputFrame frame, int index); 4174 4175 /** 4176 * The content is scaled to the surface dimensions 4177 */ 4178 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 4179 4180 /** 4181 * The content is scaled, maintaining its aspect ratio, the whole 4182 * surface area is used, content may be cropped. 4183 * <p class=note> 4184 * This mode is only suitable for content with 1:1 pixel aspect ratio as you cannot 4185 * configure the pixel aspect ratio for a {@link Surface}. 4186 * <p class=note> 4187 * As of {@link android.os.Build.VERSION_CODES#N} release, this mode may not work if 4188 * the video is {@linkplain MediaFormat#KEY_ROTATION rotated} by 90 or 270 degrees. 4189 */ 4190 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 4191 4192 /** @hide */ 4193 @IntDef({ 4194 VIDEO_SCALING_MODE_SCALE_TO_FIT, 4195 VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING, 4196 }) 4197 @Retention(RetentionPolicy.SOURCE) 4198 public @interface VideoScalingMode {} 4199 4200 /** 4201 * If a surface has been specified in a previous call to {@link #configure} 4202 * specifies the scaling mode to use. The default is "scale to fit". 4203 * <p class=note> 4204 * The scaling mode may be reset to the <strong>default</strong> each time an 4205 * {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is received from the codec; therefore, the client 4206 * must call this method after every buffer change event (and before the first output buffer is 4207 * released for rendering) to ensure consistent scaling mode. 4208 * <p class=note> 4209 * Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, this can also be done 4210 * after each {@link #INFO_OUTPUT_FORMAT_CHANGED} event. 4211 * 4212 * @throws IllegalArgumentException if mode is not recognized. 4213 * @throws IllegalStateException if in the Released state. 4214 */ setVideoScalingMode(@ideoScalingMode int mode)4215 public native final void setVideoScalingMode(@VideoScalingMode int mode); 4216 4217 /** 4218 * Sets the audio presentation. 4219 * @param presentation see {@link AudioPresentation}. In particular, id should be set. 4220 */ setAudioPresentation(@onNull AudioPresentation presentation)4221 public void setAudioPresentation(@NonNull AudioPresentation presentation) { 4222 if (presentation == null) { 4223 throw new NullPointerException("audio presentation is null"); 4224 } 4225 native_setAudioPresentation(presentation.getPresentationId(), presentation.getProgramId()); 4226 } 4227 native_setAudioPresentation(int presentationId, int programId)4228 private native void native_setAudioPresentation(int presentationId, int programId); 4229 4230 /** 4231 * Retrieve the codec name. 4232 * 4233 * If the codec was created by createDecoderByType or createEncoderByType, what component is 4234 * chosen is not known beforehand. This method returns the name of the codec that was 4235 * selected by the platform. 4236 * 4237 * <strong>Note:</strong> Implementations may provide multiple aliases (codec 4238 * names) for the same underlying codec, any of which can be used to instantiate the same 4239 * underlying codec in {@link MediaCodec#createByCodecName}. This method returns the 4240 * name used to create the codec in this case. 4241 * 4242 * @throws IllegalStateException if in the Released state. 4243 */ 4244 @NonNull getName()4245 public final String getName() { 4246 // get canonical name to handle exception 4247 String canonicalName = getCanonicalName(); 4248 return mNameAtCreation != null ? mNameAtCreation : canonicalName; 4249 } 4250 4251 /** 4252 * Retrieve the underlying codec name. 4253 * 4254 * This method is similar to {@link #getName}, except that it returns the underlying component 4255 * name even if an alias was used to create this MediaCodec object by name, 4256 * 4257 * @throws IllegalStateException if in the Released state. 4258 */ 4259 @NonNull getCanonicalName()4260 public native final String getCanonicalName(); 4261 4262 /** 4263 * Return Metrics data about the current codec instance. 4264 * 4265 * @return a {@link PersistableBundle} containing the set of attributes and values 4266 * available for the media being handled by this instance of MediaCodec 4267 * The attributes are descibed in {@link MetricsConstants}. 4268 * 4269 * Additional vendor-specific fields may also be present in 4270 * the return value. 4271 */ getMetrics()4272 public PersistableBundle getMetrics() { 4273 PersistableBundle bundle = native_getMetrics(); 4274 return bundle; 4275 } 4276 native_getMetrics()4277 private native PersistableBundle native_getMetrics(); 4278 4279 /** 4280 * Change a video encoder's target bitrate on the fly. The value is an 4281 * Integer object containing the new bitrate in bps. 4282 * 4283 * @see #setParameters(Bundle) 4284 */ 4285 public static final String PARAMETER_KEY_VIDEO_BITRATE = "video-bitrate"; 4286 4287 /** 4288 * Temporarily suspend/resume encoding of input data. While suspended 4289 * input data is effectively discarded instead of being fed into the 4290 * encoder. This parameter really only makes sense to use with an encoder 4291 * in "surface-input" mode, as the client code has no control over the 4292 * input-side of the encoder in that case. 4293 * The value is an Integer object containing the value 1 to suspend 4294 * or the value 0 to resume. 4295 * 4296 * @see #setParameters(Bundle) 4297 */ 4298 public static final String PARAMETER_KEY_SUSPEND = "drop-input-frames"; 4299 4300 /** 4301 * When {@link #PARAMETER_KEY_SUSPEND} is present, the client can also 4302 * optionally use this key to specify the timestamp (in micro-second) 4303 * at which the suspend/resume operation takes effect. 4304 * 4305 * Note that the specified timestamp must be greater than or equal to the 4306 * timestamp of any previously queued suspend/resume operations. 4307 * 4308 * The value is a long int, indicating the timestamp to suspend/resume. 4309 * 4310 * @see #setParameters(Bundle) 4311 */ 4312 public static final String PARAMETER_KEY_SUSPEND_TIME = "drop-start-time-us"; 4313 4314 /** 4315 * Specify an offset (in micro-second) to be added on top of the timestamps 4316 * onward. A typical use case is to apply an adjust to the timestamps after 4317 * a period of pause by the user. 4318 * 4319 * This parameter can only be used on an encoder in "surface-input" mode. 4320 * 4321 * The value is a long int, indicating the timestamp offset to be applied. 4322 * 4323 * @see #setParameters(Bundle) 4324 */ 4325 public static final String PARAMETER_KEY_OFFSET_TIME = "time-offset-us"; 4326 4327 /** 4328 * Request that the encoder produce a sync frame "soon". 4329 * Provide an Integer with the value 0. 4330 * 4331 * @see #setParameters(Bundle) 4332 */ 4333 public static final String PARAMETER_KEY_REQUEST_SYNC_FRAME = "request-sync"; 4334 4335 /** 4336 * Set the HDR10+ metadata on the next queued input frame. 4337 * 4338 * Provide a byte array of data that's conforming to the 4339 * user_data_registered_itu_t_t35() syntax of SEI message for ST 2094-40. 4340 *<p> 4341 * For decoders: 4342 *<p> 4343 * When a decoder is configured for one of the HDR10+ profiles that uses 4344 * out-of-band metadata (such as {@link 4345 * MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus} or {@link 4346 * MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), this 4347 * parameter sets the HDR10+ metadata on the next input buffer queued 4348 * to the decoder. A decoder supporting these profiles must propagate 4349 * the metadata to the format of the output buffer corresponding to this 4350 * particular input buffer (under key {@link MediaFormat#KEY_HDR10_PLUS_INFO}). 4351 * The metadata should be applied to that output buffer and the buffers 4352 * following it (in display order), until the next output buffer (in 4353 * display order) upon which an HDR10+ metadata is set. 4354 *<p> 4355 * This parameter shouldn't be set if the decoder is not configured for 4356 * an HDR10+ profile that uses out-of-band metadata. In particular, 4357 * it shouldn't be set for HDR10+ profiles that uses in-band metadata 4358 * where the metadata is embedded in the input buffers, for example 4359 * {@link MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}. 4360 *<p> 4361 * For encoders: 4362 *<p> 4363 * When an encoder is configured for one of the HDR10+ profiles and the 4364 * operates in byte buffer input mode (instead of surface input mode), 4365 * this parameter sets the HDR10+ metadata on the next input buffer queued 4366 * to the encoder. For the HDR10+ profiles that uses out-of-band metadata 4367 * (such as {@link MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus}, 4368 * or {@link MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), 4369 * the metadata must be propagated to the format of the output buffer 4370 * corresponding to this particular input buffer (under key {@link 4371 * MediaFormat#KEY_HDR10_PLUS_INFO}). For the HDR10+ profiles that uses 4372 * in-band metadata (such as {@link 4373 * MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}), the 4374 * metadata info must be embedded in the corresponding output buffer itself. 4375 *<p> 4376 * This parameter shouldn't be set if the encoder is not configured for 4377 * an HDR10+ profile, or if it's operating in surface input mode. 4378 *<p> 4379 * 4380 * @see MediaFormat#KEY_HDR10_PLUS_INFO 4381 */ 4382 public static final String PARAMETER_KEY_HDR10_PLUS_INFO = MediaFormat.KEY_HDR10_PLUS_INFO; 4383 4384 /** 4385 * Enable/disable low latency decoding mode. 4386 * When enabled, the decoder doesn't hold input and output data more than 4387 * required by the codec standards. 4388 * The value is an Integer object containing the value 1 to enable 4389 * or the value 0 to disable. 4390 * 4391 * @see #setParameters(Bundle) 4392 * @see MediaFormat#KEY_LOW_LATENCY 4393 */ 4394 public static final String PARAMETER_KEY_LOW_LATENCY = 4395 MediaFormat.KEY_LOW_LATENCY; 4396 4397 /** 4398 * Communicate additional parameter changes to the component instance. 4399 * <b>Note:</b> Some of these parameter changes may silently fail to apply. 4400 * 4401 * @param params The bundle of parameters to set. 4402 * @throws IllegalStateException if in the Released state. 4403 */ setParameters(@ullable Bundle params)4404 public final void setParameters(@Nullable Bundle params) { 4405 if (params == null) { 4406 return; 4407 } 4408 4409 String[] keys = new String[params.size()]; 4410 Object[] values = new Object[params.size()]; 4411 4412 int i = 0; 4413 for (final String key: params.keySet()) { 4414 keys[i] = key; 4415 Object value = params.get(key); 4416 4417 // Bundle's byte array is a byte[], JNI layer only takes ByteBuffer 4418 if (value instanceof byte[]) { 4419 values[i] = ByteBuffer.wrap((byte[])value); 4420 } else { 4421 values[i] = value; 4422 } 4423 ++i; 4424 } 4425 4426 setParameters(keys, values); 4427 } 4428 4429 /** 4430 * Sets an asynchronous callback for actionable MediaCodec events. 4431 * 4432 * If the client intends to use the component in asynchronous mode, 4433 * a valid callback should be provided before {@link #configure} is called. 4434 * 4435 * When asynchronous callback is enabled, the client should not call 4436 * {@link #getInputBuffers}, {@link #getOutputBuffers}, 4437 * {@link #dequeueInputBuffer(long)} or {@link #dequeueOutputBuffer(BufferInfo, long)}. 4438 * <p> 4439 * Also, {@link #flush} behaves differently in asynchronous mode. After calling 4440 * {@code flush}, you must call {@link #start} to "resume" receiving input buffers, 4441 * even if an input surface was created. 4442 * 4443 * @param cb The callback that will run. Use {@code null} to clear a previously 4444 * set callback (before {@link #configure configure} is called and run 4445 * in synchronous mode). 4446 * @param handler Callbacks will happen on the handler's thread. If {@code null}, 4447 * callbacks are done on the default thread (the caller's thread or the 4448 * main thread.) 4449 */ setCallback(@ullable Callback cb, @Nullable Handler handler)4450 public void setCallback(@Nullable /* MediaCodec. */ Callback cb, @Nullable Handler handler) { 4451 if (cb != null) { 4452 synchronized (mListenerLock) { 4453 EventHandler newHandler = getEventHandlerOn(handler, mCallbackHandler); 4454 // NOTE: there are no callbacks on the handler at this time, but check anyways 4455 // even if we were to extend this to be callable dynamically, it must 4456 // be called when codec is flushed, so no messages are pending. 4457 if (newHandler != mCallbackHandler) { 4458 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 4459 mCallbackHandler.removeMessages(EVENT_CALLBACK); 4460 mCallbackHandler = newHandler; 4461 } 4462 } 4463 } else if (mCallbackHandler != null) { 4464 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 4465 mCallbackHandler.removeMessages(EVENT_CALLBACK); 4466 } 4467 4468 if (mCallbackHandler != null) { 4469 // set java callback on main handler 4470 Message msg = mCallbackHandler.obtainMessage(EVENT_SET_CALLBACK, 0, 0, cb); 4471 mCallbackHandler.sendMessage(msg); 4472 4473 // set native handler here, don't post to handler because 4474 // it may cause the callback to be delayed and set in a wrong state. 4475 // Note that native codec may start sending events to the callback 4476 // handler after this returns. 4477 native_setCallback(cb); 4478 } 4479 } 4480 4481 /** 4482 * Sets an asynchronous callback for actionable MediaCodec events on the default 4483 * looper. 4484 * <p> 4485 * Same as {@link #setCallback(Callback, Handler)} with handler set to null. 4486 * @param cb The callback that will run. Use {@code null} to clear a previously 4487 * set callback (before {@link #configure configure} is called and run 4488 * in synchronous mode). 4489 * @see #setCallback(Callback, Handler) 4490 */ setCallback(@ullable Callback cb)4491 public void setCallback(@Nullable /* MediaCodec. */ Callback cb) { 4492 setCallback(cb, null /* handler */); 4493 } 4494 4495 /** 4496 * Listener to be called when an output frame has rendered on the output surface 4497 * 4498 * @see MediaCodec#setOnFrameRenderedListener 4499 */ 4500 public interface OnFrameRenderedListener { 4501 4502 /** 4503 * Called when an output frame has rendered on the output surface. 4504 * <p> 4505 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 4506 * render timing samples, and can be significantly delayed and batched. Some frames may have 4507 * been rendered even if there was no callback generated. 4508 * 4509 * @param codec the MediaCodec instance 4510 * @param presentationTimeUs the presentation time (media time) of the frame rendered. 4511 * This is usually the same as specified in {@link #queueInputBuffer}; however, 4512 * some codecs may alter the media time by applying some time-based transformation, 4513 * such as frame rate conversion. In that case, presentation time corresponds 4514 * to the actual output frame rendered. 4515 * @param nanoTime The system time when the frame was rendered. 4516 * 4517 * @see System#nanoTime 4518 */ onFrameRendered( @onNull MediaCodec codec, long presentationTimeUs, long nanoTime)4519 public void onFrameRendered( 4520 @NonNull MediaCodec codec, long presentationTimeUs, long nanoTime); 4521 } 4522 4523 /** 4524 * Registers a callback to be invoked when an output frame is rendered on the output surface. 4525 * <p> 4526 * This method can be called in any codec state, but will only have an effect in the 4527 * Executing state for codecs that render buffers to the output surface. 4528 * <p> 4529 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 4530 * render timing samples, and can be significantly delayed and batched. Some frames may have 4531 * been rendered even if there was no callback generated. 4532 * 4533 * @param listener the callback that will be run 4534 * @param handler the callback will be run on the handler's thread. If {@code null}, 4535 * the callback will be run on the default thread, which is the looper 4536 * from which the codec was created, or a new thread if there was none. 4537 */ setOnFrameRenderedListener( @ullable OnFrameRenderedListener listener, @Nullable Handler handler)4538 public void setOnFrameRenderedListener( 4539 @Nullable OnFrameRenderedListener listener, @Nullable Handler handler) { 4540 synchronized (mListenerLock) { 4541 mOnFrameRenderedListener = listener; 4542 if (listener != null) { 4543 EventHandler newHandler = getEventHandlerOn(handler, mOnFrameRenderedHandler); 4544 if (newHandler != mOnFrameRenderedHandler) { 4545 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 4546 } 4547 mOnFrameRenderedHandler = newHandler; 4548 } else if (mOnFrameRenderedHandler != null) { 4549 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 4550 } 4551 native_enableOnFrameRenderedListener(listener != null); 4552 } 4553 } 4554 native_enableOnFrameRenderedListener(boolean enable)4555 private native void native_enableOnFrameRenderedListener(boolean enable); 4556 getEventHandlerOn( @ullable Handler handler, @NonNull EventHandler lastHandler)4557 private EventHandler getEventHandlerOn( 4558 @Nullable Handler handler, @NonNull EventHandler lastHandler) { 4559 if (handler == null) { 4560 return mEventHandler; 4561 } else { 4562 Looper looper = handler.getLooper(); 4563 if (lastHandler.getLooper() == looper) { 4564 return lastHandler; 4565 } else { 4566 return new EventHandler(this, looper); 4567 } 4568 } 4569 } 4570 4571 /** 4572 * MediaCodec callback interface. Used to notify the user asynchronously 4573 * of various MediaCodec events. 4574 */ 4575 public static abstract class Callback { 4576 /** 4577 * Called when an input buffer becomes available. 4578 * 4579 * @param codec The MediaCodec object. 4580 * @param index The index of the available input buffer. 4581 */ onInputBufferAvailable(@onNull MediaCodec codec, int index)4582 public abstract void onInputBufferAvailable(@NonNull MediaCodec codec, int index); 4583 4584 /** 4585 * Called when an output buffer becomes available. 4586 * 4587 * @param codec The MediaCodec object. 4588 * @param index The index of the available output buffer. 4589 * @param info Info regarding the available output buffer {@link MediaCodec.BufferInfo}. 4590 */ onOutputBufferAvailable( @onNull MediaCodec codec, int index, @NonNull BufferInfo info)4591 public abstract void onOutputBufferAvailable( 4592 @NonNull MediaCodec codec, int index, @NonNull BufferInfo info); 4593 4594 /** 4595 * Called when the MediaCodec encountered an error 4596 * 4597 * @param codec The MediaCodec object. 4598 * @param e The {@link MediaCodec.CodecException} object describing the error. 4599 */ onError(@onNull MediaCodec codec, @NonNull CodecException e)4600 public abstract void onError(@NonNull MediaCodec codec, @NonNull CodecException e); 4601 4602 /** 4603 * Called when the output format has changed 4604 * 4605 * @param codec The MediaCodec object. 4606 * @param format The new output format. 4607 */ onOutputFormatChanged( @onNull MediaCodec codec, @NonNull MediaFormat format)4608 public abstract void onOutputFormatChanged( 4609 @NonNull MediaCodec codec, @NonNull MediaFormat format); 4610 } 4611 postEventFromNative( int what, int arg1, int arg2, @Nullable Object obj)4612 private void postEventFromNative( 4613 int what, int arg1, int arg2, @Nullable Object obj) { 4614 synchronized (mListenerLock) { 4615 EventHandler handler = mEventHandler; 4616 if (what == EVENT_CALLBACK) { 4617 handler = mCallbackHandler; 4618 } else if (what == EVENT_FRAME_RENDERED) { 4619 handler = mOnFrameRenderedHandler; 4620 } 4621 if (handler != null) { 4622 Message msg = handler.obtainMessage(what, arg1, arg2, obj); 4623 handler.sendMessage(msg); 4624 } 4625 } 4626 } 4627 4628 @UnsupportedAppUsage setParameters(@onNull String[] keys, @NonNull Object[] values)4629 private native final void setParameters(@NonNull String[] keys, @NonNull Object[] values); 4630 4631 /** 4632 * Get the codec info. If the codec was created by createDecoderByType 4633 * or createEncoderByType, what component is chosen is not known beforehand, 4634 * and thus the caller does not have the MediaCodecInfo. 4635 * @throws IllegalStateException if in the Released state. 4636 */ 4637 @NonNull getCodecInfo()4638 public MediaCodecInfo getCodecInfo() { 4639 // Get the codec name first. If the codec is already released, 4640 // IllegalStateException will be thrown here. 4641 String name = getName(); 4642 synchronized (mCodecInfoLock) { 4643 if (mCodecInfo == null) { 4644 // Get the codec info for this codec itself first. Only initialize 4645 // the full codec list if this somehow fails because it can be slow. 4646 mCodecInfo = getOwnCodecInfo(); 4647 if (mCodecInfo == null) { 4648 mCodecInfo = MediaCodecList.getInfoFor(name); 4649 } 4650 } 4651 return mCodecInfo; 4652 } 4653 } 4654 4655 @NonNull getOwnCodecInfo()4656 private native final MediaCodecInfo getOwnCodecInfo(); 4657 4658 @NonNull 4659 @UnsupportedAppUsage getBuffers(boolean input)4660 private native final ByteBuffer[] getBuffers(boolean input); 4661 4662 @Nullable getBuffer(boolean input, int index)4663 private native final ByteBuffer getBuffer(boolean input, int index); 4664 4665 @Nullable getImage(boolean input, int index)4666 private native final Image getImage(boolean input, int index); 4667 native_init()4668 private static native final void native_init(); 4669 native_setup( @onNull String name, boolean nameIsType, boolean encoder)4670 private native final void native_setup( 4671 @NonNull String name, boolean nameIsType, boolean encoder); 4672 native_finalize()4673 private native final void native_finalize(); 4674 4675 static { 4676 System.loadLibrary("media_jni"); native_init()4677 native_init(); 4678 } 4679 4680 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023) 4681 private long mNativeContext = 0; 4682 private final Lock mNativeContextLock = new ReentrantLock(); 4683 lockAndGetContext()4684 private final long lockAndGetContext() { 4685 mNativeContextLock.lock(); 4686 return mNativeContext; 4687 } 4688 setAndUnlockContext(long context)4689 private final void setAndUnlockContext(long context) { 4690 mNativeContext = context; 4691 mNativeContextLock.unlock(); 4692 } 4693 4694 /** @hide */ 4695 public static class MediaImage extends Image { 4696 private final boolean mIsReadOnly; 4697 private final int mWidth; 4698 private final int mHeight; 4699 private final int mFormat; 4700 private long mTimestamp; 4701 private final Plane[] mPlanes; 4702 private final ByteBuffer mBuffer; 4703 private final ByteBuffer mInfo; 4704 private final int mXOffset; 4705 private final int mYOffset; 4706 private final long mBufferContext; 4707 4708 private final static int TYPE_YUV = 1; 4709 4710 private final int mTransform = 0; //Default no transform 4711 private final int mScalingMode = 0; //Default frozen scaling mode 4712 4713 @Override getFormat()4714 public int getFormat() { 4715 throwISEIfImageIsInvalid(); 4716 return mFormat; 4717 } 4718 4719 @Override getHeight()4720 public int getHeight() { 4721 throwISEIfImageIsInvalid(); 4722 return mHeight; 4723 } 4724 4725 @Override getWidth()4726 public int getWidth() { 4727 throwISEIfImageIsInvalid(); 4728 return mWidth; 4729 } 4730 4731 @Override getTransform()4732 public int getTransform() { 4733 throwISEIfImageIsInvalid(); 4734 return mTransform; 4735 } 4736 4737 @Override getScalingMode()4738 public int getScalingMode() { 4739 throwISEIfImageIsInvalid(); 4740 return mScalingMode; 4741 } 4742 4743 @Override getTimestamp()4744 public long getTimestamp() { 4745 throwISEIfImageIsInvalid(); 4746 return mTimestamp; 4747 } 4748 4749 @Override 4750 @NonNull getPlanes()4751 public Plane[] getPlanes() { 4752 throwISEIfImageIsInvalid(); 4753 return Arrays.copyOf(mPlanes, mPlanes.length); 4754 } 4755 4756 @Override close()4757 public void close() { 4758 if (mIsImageValid) { 4759 if (mBuffer != null) { 4760 java.nio.NioUtils.freeDirectBuffer(mBuffer); 4761 } 4762 if (mBufferContext != 0) { 4763 native_closeMediaImage(mBufferContext); 4764 } 4765 mIsImageValid = false; 4766 } 4767 } 4768 4769 /** 4770 * Set the crop rectangle associated with this frame. 4771 * <p> 4772 * The crop rectangle specifies the region of valid pixels in the image, 4773 * using coordinates in the largest-resolution plane. 4774 */ 4775 @Override setCropRect(@ullable Rect cropRect)4776 public void setCropRect(@Nullable Rect cropRect) { 4777 if (mIsReadOnly) { 4778 throw new ReadOnlyBufferException(); 4779 } 4780 super.setCropRect(cropRect); 4781 } 4782 MediaImage( @onNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect)4783 public MediaImage( 4784 @NonNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, 4785 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) { 4786 mFormat = ImageFormat.YUV_420_888; 4787 mTimestamp = timestamp; 4788 mIsImageValid = true; 4789 mIsReadOnly = buffer.isReadOnly(); 4790 mBuffer = buffer.duplicate(); 4791 4792 // save offsets and info 4793 mXOffset = xOffset; 4794 mYOffset = yOffset; 4795 mInfo = info; 4796 4797 mBufferContext = 0; 4798 4799 // read media-info. See MediaImage2 4800 if (info.remaining() == 104) { 4801 int type = info.getInt(); 4802 if (type != TYPE_YUV) { 4803 throw new UnsupportedOperationException("unsupported type: " + type); 4804 } 4805 int numPlanes = info.getInt(); 4806 if (numPlanes != 3) { 4807 throw new RuntimeException("unexpected number of planes: " + numPlanes); 4808 } 4809 mWidth = info.getInt(); 4810 mHeight = info.getInt(); 4811 if (mWidth < 1 || mHeight < 1) { 4812 throw new UnsupportedOperationException( 4813 "unsupported size: " + mWidth + "x" + mHeight); 4814 } 4815 int bitDepth = info.getInt(); 4816 if (bitDepth != 8) { 4817 throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth); 4818 } 4819 int bitDepthAllocated = info.getInt(); 4820 if (bitDepthAllocated != 8) { 4821 throw new UnsupportedOperationException( 4822 "unsupported allocated bit depth: " + bitDepthAllocated); 4823 } 4824 mPlanes = new MediaPlane[numPlanes]; 4825 for (int ix = 0; ix < numPlanes; ix++) { 4826 int planeOffset = info.getInt(); 4827 int colInc = info.getInt(); 4828 int rowInc = info.getInt(); 4829 int horiz = info.getInt(); 4830 int vert = info.getInt(); 4831 if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) { 4832 throw new UnsupportedOperationException("unexpected subsampling: " 4833 + horiz + "x" + vert + " on plane " + ix); 4834 } 4835 if (colInc < 1 || rowInc < 1) { 4836 throw new UnsupportedOperationException("unexpected strides: " 4837 + colInc + " pixel, " + rowInc + " row on plane " + ix); 4838 } 4839 buffer.clear(); 4840 buffer.position(mBuffer.position() + planeOffset 4841 + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc); 4842 buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8) 4843 + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc); 4844 mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc); 4845 } 4846 } else { 4847 throw new UnsupportedOperationException( 4848 "unsupported info length: " + info.remaining()); 4849 } 4850 4851 if (cropRect == null) { 4852 cropRect = new Rect(0, 0, mWidth, mHeight); 4853 } 4854 cropRect.offset(-xOffset, -yOffset); 4855 super.setCropRect(cropRect); 4856 } 4857 MediaImage( @onNull ByteBuffer[] buffers, int[] rowStrides, int[] pixelStrides, int width, int height, int format, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect, long context)4858 public MediaImage( 4859 @NonNull ByteBuffer[] buffers, int[] rowStrides, int[] pixelStrides, 4860 int width, int height, int format, boolean readOnly, 4861 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect, long context) { 4862 if (buffers.length != rowStrides.length || buffers.length != pixelStrides.length) { 4863 throw new IllegalArgumentException( 4864 "buffers, rowStrides and pixelStrides should have the same length"); 4865 } 4866 mWidth = width; 4867 mHeight = height; 4868 mFormat = format; 4869 mTimestamp = timestamp; 4870 mIsImageValid = true; 4871 mIsReadOnly = readOnly; 4872 mBuffer = null; 4873 mInfo = null; 4874 mPlanes = new MediaPlane[buffers.length]; 4875 for (int i = 0; i < buffers.length; ++i) { 4876 mPlanes[i] = new MediaPlane(buffers[i], rowStrides[i], pixelStrides[i]); 4877 } 4878 4879 // save offsets and info 4880 mXOffset = xOffset; 4881 mYOffset = yOffset; 4882 4883 if (cropRect == null) { 4884 cropRect = new Rect(0, 0, mWidth, mHeight); 4885 } 4886 cropRect.offset(-xOffset, -yOffset); 4887 super.setCropRect(cropRect); 4888 4889 mBufferContext = context; 4890 } 4891 4892 private class MediaPlane extends Plane { MediaPlane(@onNull ByteBuffer buffer, int rowInc, int colInc)4893 public MediaPlane(@NonNull ByteBuffer buffer, int rowInc, int colInc) { 4894 mData = buffer; 4895 mRowInc = rowInc; 4896 mColInc = colInc; 4897 } 4898 4899 @Override getRowStride()4900 public int getRowStride() { 4901 throwISEIfImageIsInvalid(); 4902 return mRowInc; 4903 } 4904 4905 @Override getPixelStride()4906 public int getPixelStride() { 4907 throwISEIfImageIsInvalid(); 4908 return mColInc; 4909 } 4910 4911 @Override 4912 @NonNull getBuffer()4913 public ByteBuffer getBuffer() { 4914 throwISEIfImageIsInvalid(); 4915 return mData; 4916 } 4917 4918 private final int mRowInc; 4919 private final int mColInc; 4920 private final ByteBuffer mData; 4921 } 4922 } 4923 4924 public final static class MetricsConstants 4925 { MetricsConstants()4926 private MetricsConstants() {} 4927 4928 /** 4929 * Key to extract the codec being used 4930 * from the {@link MediaCodec#getMetrics} return value. 4931 * The value is a String. 4932 */ 4933 public static final String CODEC = "android.media.mediacodec.codec"; 4934 4935 /** 4936 * Key to extract the MIME type 4937 * from the {@link MediaCodec#getMetrics} return value. 4938 * The value is a String. 4939 */ 4940 public static final String MIME_TYPE = "android.media.mediacodec.mime"; 4941 4942 /** 4943 * Key to extract what the codec mode 4944 * from the {@link MediaCodec#getMetrics} return value. 4945 * The value is a String. Values will be one of the constants 4946 * {@link #MODE_AUDIO} or {@link #MODE_VIDEO}. 4947 */ 4948 public static final String MODE = "android.media.mediacodec.mode"; 4949 4950 /** 4951 * The value returned for the key {@link #MODE} when the 4952 * codec is a audio codec. 4953 */ 4954 public static final String MODE_AUDIO = "audio"; 4955 4956 /** 4957 * The value returned for the key {@link #MODE} when the 4958 * codec is a video codec. 4959 */ 4960 public static final String MODE_VIDEO = "video"; 4961 4962 /** 4963 * Key to extract the flag indicating whether the codec is running 4964 * as an encoder or decoder from the {@link MediaCodec#getMetrics} return value. 4965 * The value is an integer. 4966 * A 0 indicates decoder; 1 indicates encoder. 4967 */ 4968 public static final String ENCODER = "android.media.mediacodec.encoder"; 4969 4970 /** 4971 * Key to extract the flag indicating whether the codec is running 4972 * in secure (DRM) mode from the {@link MediaCodec#getMetrics} return value. 4973 * The value is an integer. 4974 */ 4975 public static final String SECURE = "android.media.mediacodec.secure"; 4976 4977 /** 4978 * Key to extract the width (in pixels) of the video track 4979 * from the {@link MediaCodec#getMetrics} return value. 4980 * The value is an integer. 4981 */ 4982 public static final String WIDTH = "android.media.mediacodec.width"; 4983 4984 /** 4985 * Key to extract the height (in pixels) of the video track 4986 * from the {@link MediaCodec#getMetrics} return value. 4987 * The value is an integer. 4988 */ 4989 public static final String HEIGHT = "android.media.mediacodec.height"; 4990 4991 /** 4992 * Key to extract the rotation (in degrees) to properly orient the video 4993 * from the {@link MediaCodec#getMetrics} return. 4994 * The value is a integer. 4995 */ 4996 public static final String ROTATION = "android.media.mediacodec.rotation"; 4997 4998 } 4999 } 5000
