1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntDef; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.graphics.ImageFormat; 23 import android.graphics.Rect; 24 import android.graphics.SurfaceTexture; 25 import android.media.MediaCodecInfo.CodecCapabilities; 26 import android.os.Bundle; 27 import android.os.Handler; 28 import android.os.Looper; 29 import android.os.Message; 30 import android.view.Surface; 31 32 import java.io.IOException; 33 import java.lang.annotation.Retention; 34 import java.lang.annotation.RetentionPolicy; 35 import java.nio.ByteBuffer; 36 import java.nio.ByteOrder; 37 import java.nio.ReadOnlyBufferException; 38 import java.util.Arrays; 39 import java.util.HashMap; 40 import java.util.Map; 41 42 /** 43 MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components. 44 It is part of the Android low-level multimedia support infrastructure (normally used together 45 with {@link MediaExtractor}, {@link MediaSync}, {@link MediaMuxer}, {@link MediaCrypto}, 46 {@link MediaDrm}, {@link Image}, {@link Surface}, and {@link AudioTrack}.) 47 <p> 48 <center><object style="width: 540px; height: 205px;" type="image/svg+xml" 49 data="../../../images/media/mediacodec_buffers.svg"><img 50 src="../../../images/media/mediacodec_buffers.png" style="width: 540px; height: 205px" 51 alt="MediaCodec buffer flow diagram"></object></center> 52 <p> 53 In broad terms, a codec processes input data to generate output data. It processes data 54 asynchronously and uses a set of input and output buffers. At a simplistic level, you request 55 (or receive) an empty input buffer, fill it up with data and send it to the codec for 56 processing. The codec uses up the data and transforms it into one of its empty output buffers. 57 Finally, you request (or receive) a filled output buffer, consume its contents and release it 58 back to the codec. 59 60 <h3>Data Types</h3> 61 <p> 62 Codecs operate on three kinds of data: compressed data, raw audio data and raw video data. 63 All three kinds of data can be processed using {@link ByteBuffer ByteBuffers}, but you should use 64 a {@link Surface} for raw video data to improve codec performance. Surface uses native video 65 buffers without mapping or copying them to ByteBuffers; thus, it is much more efficient. 66 You normally cannot access the raw video data when using a Surface, but you can use the 67 {@link ImageReader} class to access unsecured decoded (raw) video frames. This may still be more 68 efficient than using ByteBuffers, as some native buffers may be mapped into {@linkplain 69 ByteBuffer#isDirect direct} ByteBuffers. When using ByteBuffer mode, you can access raw video 70 frames using the {@link Image} class and {@link #getInputImage getInput}/{@link #getOutputImage 71 OutputImage(int)}. 72 73 <h4>Compressed Buffers</h4> 74 <p> 75 Input buffers (for decoders) and output buffers (for encoders) contain compressed data according 76 to the {@linkplain MediaFormat#KEY_MIME format's type}. For video types this is a single 77 compressed video frame. For audio data this is normally a single access unit (an encoded audio 78 segment typically containing a few milliseconds of audio as dictated by the format type), but 79 this requirement is slightly relaxed in that a buffer may contain multiple encoded access units 80 of audio. In either case, buffers do not start or end on arbitrary byte boundaries, but rather on 81 frame/access unit boundaries. 82 83 <h4>Raw Audio Buffers</h4> 84 <p> 85 Raw audio buffers contain entire frames of PCM audio data, which is one sample for each channel 86 in channel order. Each sample is a {@linkplain AudioFormat#ENCODING_PCM_16BIT 16-bit signed 87 integer in native byte order}. 88 89 <pre class=prettyprint> 90 short[] getSamplesForChannel(MediaCodec codec, int bufferId, int channelIx) { 91 ByteBuffer outputBuffer = codec.getOutputBuffer(bufferId); 92 MediaFormat format = codec.getOutputFormat(bufferId); 93 ShortBuffer samples = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer(); 94 int numChannels = formet.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 95 if (channelIx < 0 || channelIx >= numChannels) { 96 return null; 97 } 98 short[] res = new short[samples.remaining() / numChannels]; 99 for (int i = 0; i < res.length; ++i) { 100 res[i] = samples.get(i * numChannels + channelIx); 101 } 102 return res; 103 }</pre> 104 105 <h4>Raw Video Buffers</h4> 106 <p> 107 In ByteBuffer mode video buffers are laid out according to their {@linkplain 108 MediaFormat#KEY_COLOR_FORMAT color format}. You can get the supported color formats as an array 109 from {@link #getCodecInfo}{@code .}{@link MediaCodecInfo#getCapabilitiesForType 110 getCapabilitiesForType(…)}{@code .}{@link CodecCapabilities#colorFormats colorFormats}. 111 Video codecs may support three kinds of color formats: 112 <ul> 113 <li><strong>native raw video format:</strong> This is marked by {@link 114 CodecCapabilities#COLOR_FormatSurface} and it can be used with an input or output Surface.</li> 115 <li><strong>flexible YUV buffers</strong> (such as {@link 116 CodecCapabilities#COLOR_FormatYUV420Flexible}): These can be used with an input/output Surface, 117 as well as in ByteBuffer mode, by using {@link #getInputImage getInput}/{@link #getOutputImage 118 OutputImage(int)}.</li> 119 <li><strong>other, specific formats:</strong> These are normally only supported in ByteBuffer 120 mode. Some color formats are vendor specific. Others are defined in {@link CodecCapabilities}. 121 For color formats that are equivalent to a flexible format, you can still use {@link 122 #getInputImage getInput}/{@link #getOutputImage OutputImage(int)}.</li> 123 </ul> 124 <p> 125 All video codecs support flexible YUV 4:2:0 buffers since {@link 126 android.os.Build.VERSION_CODES#LOLLIPOP_MR1}. 127 128 <h4>Accessing Raw Video ByteBuffers on Older Devices</h4> 129 <p> 130 Prior to {@link android.os.Build.VERSION_CODES#LOLLIPOP} and {@link Image} support, you need to 131 use the {@link MediaFormat#KEY_STRIDE} and {@link MediaFormat#KEY_SLICE_HEIGHT} output format 132 values to understand the layout of the raw output buffers. 133 <p class=note> 134 Note that on some devices the slice-height is advertised as 0. This could mean either that the 135 slice-height is the same as the frame height, or that the slice-height is the frame height 136 aligned to some value (usually a power of 2). Unfortunately, there is no way to tell the actual 137 slice height in this case. Furthermore, the vertical stride of the {@code U} plane in planar 138 formats is also not specified or defined, though usually it is half of the slice height. 139 <p> 140 The {@link MediaFormat#KEY_WIDTH} and {@link MediaFormat#KEY_HEIGHT} keys specify the size of the 141 video frames; however, for most encondings the video (picture) only occupies a portion of the 142 video frame. This is represented by the 'crop rectangle'. 143 <p> 144 You need to use the following keys to get the crop rectangle of raw output images from the 145 {@linkplain #getOutputFormat output format}. If these keys are not present, the video occupies the 146 entire video frame.The crop rectangle is understood in the context of the output frame 147 <em>before</em> applying any {@linkplain MediaFormat#KEY_ROTATION rotation}. 148 <table style="width: 0%"> 149 <thead> 150 <tr> 151 <th>Format Key</th> 152 <th>Type</th> 153 <th>Description</th> 154 </tr> 155 </thead> 156 <tbody> 157 <tr> 158 <td>{@code "crop-left"}</td> 159 <td>Integer</td> 160 <td>The left-coordinate (x) of the crop rectangle</td> 161 </tr><tr> 162 <td>{@code "crop-top"}</td> 163 <td>Integer</td> 164 <td>The top-coordinate (y) of the crop rectangle</td> 165 </tr><tr> 166 <td>{@code "crop-right"}</td> 167 <td>Integer</td> 168 <td>The right-coordinate (x) <strong>MINUS 1</strong> of the crop rectangle</td> 169 </tr><tr> 170 <td>{@code "crop-bottom"}</td> 171 <td>Integer</td> 172 <td>The bottom-coordinate (y) <strong>MINUS 1</strong> of the crop rectangle</td> 173 </tr><tr> 174 <td colspan=3> 175 The right and bottom coordinates can be understood as the coordinates of the right-most 176 valid column/bottom-most valid row of the cropped output image. 177 </td> 178 </tr> 179 </tbody> 180 </table> 181 <p> 182 The size of the video frame (before rotation) can be calculated as such: 183 <pre class=prettyprint> 184 MediaFormat format = decoder.getOutputFormat(…); 185 int width = format.getInteger(MediaFormat.KEY_WIDTH); 186 if (format.containsKey("crop-left") && format.containsKey("crop-right")) { 187 width = format.getInteger("crop-right") + 1 - format.getInteger("crop-left"); 188 } 189 int height = format.getInteger(MediaFormat.KEY_HEIGHT); 190 if (format.containsKey("crop-top") && format.containsKey("crop-bottom")) { 191 height = format.getInteger("crop-bottom") + 1 - format.getInteger("crop-top"); 192 } 193 </pre> 194 <p class=note> 195 Also note that the meaning of {@link BufferInfo#offset BufferInfo.offset} was not consistent across 196 devices. On some devices the offset pointed to the top-left pixel of the crop rectangle, while on 197 most devices it pointed to the top-left pixel of the entire frame. 198 199 <h3>States</h3> 200 <p> 201 During its life a codec conceptually exists in one of three states: Stopped, Executing or 202 Released. The Stopped collective state is actually the conglomeration of three states: 203 Uninitialized, Configured and Error, whereas the Executing state conceptually progresses through 204 three sub-states: Flushed, Running and End-of-Stream. 205 <p> 206 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 207 data="../../../images/media/mediacodec_states.svg"><img 208 src="../../../images/media/mediacodec_states.png" style="width: 519px; height: 356px" 209 alt="MediaCodec state diagram"></object></center> 210 <p> 211 When you create a codec using one of the factory methods, the codec is in the Uninitialized 212 state. First, you need to configure it via {@link #configure configure(…)}, which brings 213 it to the Configured state, then call {@link #start} to move it to the Executing state. In this 214 state you can process data through the buffer queue manipulation described above. 215 <p> 216 The Executing state has three sub-states: Flushed, Running and End-of-Stream. Immediately after 217 {@link #start} the codec is in the Flushed sub-state, where it holds all the buffers. As soon 218 as the first input buffer is dequeued, the codec moves to the Running sub-state, where it spends 219 most of its life. When you queue an input buffer with the {@linkplain #BUFFER_FLAG_END_OF_STREAM 220 end-of-stream marker}, the codec transitions to the End-of-Stream sub-state. In this state the 221 codec no longer accepts further input buffers, but still generates output buffers until the 222 end-of-stream is reached on the output. You can move back to the Flushed sub-state at any time 223 while in the Executing state using {@link #flush}. 224 <p> 225 Call {@link #stop} to return the codec to the Uninitialized state, whereupon it may be configured 226 again. When you are done using a codec, you must release it by calling {@link #release}. 227 <p> 228 On rare occasions the codec may encounter an error and move to the Error state. This is 229 communicated using an invalid return value from a queuing operation, or sometimes via an 230 exception. Call {@link #reset} to make the codec usable again. You can call it from any state to 231 move the codec back to the Uninitialized state. Otherwise, call {@link #release} to move to the 232 terminal Released state. 233 234 <h3>Creation</h3> 235 <p> 236 Use {@link MediaCodecList} to create a MediaCodec for a specific {@link MediaFormat}. When 237 decoding a file or a stream, you can get the desired format from {@link 238 MediaExtractor#getTrackFormat MediaExtractor.getTrackFormat}. Inject any specific features that 239 you want to add using {@link MediaFormat#setFeatureEnabled MediaFormat.setFeatureEnabled}, then 240 call {@link MediaCodecList#findDecoderForFormat MediaCodecList.findDecoderForFormat} to get the 241 name of a codec that can handle that specific media format. Finally, create the codec using 242 {@link #createByCodecName}. 243 <p class=note> 244 <strong>Note:</strong> On {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the format to 245 {@code MediaCodecList.findDecoder}/{@code EncoderForFormat} must not contain a {@linkplain 246 MediaFormat#KEY_FRAME_RATE frame rate}. Use 247 <code class=prettyprint>format.setString(MediaFormat.KEY_FRAME_RATE, null)</code> 248 to clear any existing frame rate setting in the format. 249 <p> 250 You can also create the preferred codec for a specific MIME type using {@link 251 #createDecoderByType createDecoder}/{@link #createEncoderByType EncoderByType(String)}. 252 This, however, cannot be used to inject features, and may create a codec that cannot handle the 253 specific desired media format. 254 255 <h4>Creating secure decoders</h4> 256 <p> 257 On versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and earlier, secure codecs might 258 not be listed in {@link MediaCodecList}, but may still be available on the system. Secure codecs 259 that exist can be instantiated by name only, by appending {@code ".secure"} to the name of a 260 regular codec (the name of all secure codecs must end in {@code ".secure"}.) {@link 261 #createByCodecName} will throw an {@code IOException} if the codec is not present on the system. 262 <p> 263 From {@link android.os.Build.VERSION_CODES#LOLLIPOP} onwards, you should use the {@link 264 CodecCapabilities#FEATURE_SecurePlayback} feature in the media format to create a secure decoder. 265 266 <h3>Initialization</h3> 267 <p> 268 After creating the codec, you can set a callback using {@link #setCallback setCallback} if you 269 want to process data asynchronously. Then, {@linkplain #configure configure} the codec using the 270 specific media format. This is when you can specify the output {@link Surface} for video 271 producers – codecs that generate raw video data (e.g. video decoders). This is also when 272 you can set the decryption parameters for secure codecs (see {@link MediaCrypto}). Finally, since 273 some codecs can operate in multiple modes, you must specify whether you want it to work as a 274 decoder or an encoder. 275 <p> 276 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you can query the resulting input and 277 output format in the Configured state. You can use this to verify the resulting configuration, 278 e.g. color formats, before starting the codec. 279 <p> 280 If you want to process raw input video buffers natively with a video consumer – a codec 281 that processes raw video input, such as a video encoder – create a destination Surface for 282 your input data using {@link #createInputSurface} after configuration. Alternately, set up the 283 codec to use a previously created {@linkplain #createPersistentInputSurface persistent input 284 surface} by calling {@link #setInputSurface}. 285 286 <h4 id=CSD><a name="CSD"></a>Codec-specific Data</h4> 287 <p> 288 Some formats, notably AAC audio and MPEG4, H.264 and H.265 video formats require the actual data 289 to be prefixed by a number of buffers containing setup data, or codec specific data. When 290 processing such compressed formats, this data must be submitted to the codec after {@link 291 #start} and before any frame data. Such data must be marked using the flag {@link 292 #BUFFER_FLAG_CODEC_CONFIG} in a call to {@link #queueInputBuffer queueInputBuffer}. 293 <p> 294 Codec-specific data can also be included in the format passed to {@link #configure configure} in 295 ByteBuffer entries with keys "csd-0", "csd-1", etc. These keys are always included in the track 296 {@link MediaFormat} obtained from the {@link MediaExtractor#getTrackFormat MediaExtractor}. 297 Codec-specific data in the format is automatically submitted to the codec upon {@link #start}; 298 you <strong>MUST NOT</strong> submit this data explicitly. If the format did not contain codec 299 specific data, you can choose to submit it using the specified number of buffers in the correct 300 order, according to the format requirements. In case of H.264 AVC, you can also concatenate all 301 codec-specific data and submit it as a single codec-config buffer. 302 <p> 303 Android uses the following codec-specific data buffers. These are also required to be set in 304 the track format for proper {@link MediaMuxer} track configuration. Each parameter set and the 305 codec-specific-data sections marked with (<sup>*</sup>) must start with a start code of 306 {@code "\x00\x00\x00\x01"}. 307 <p> 308 <style>td.NA { background: #ccc; } .mid > tr > td { vertical-align: middle; }</style> 309 <table> 310 <thead> 311 <th>Format</th> 312 <th>CSD buffer #0</th> 313 <th>CSD buffer #1</th> 314 <th>CSD buffer #2</th> 315 </thead> 316 <tbody class=mid> 317 <tr> 318 <td>AAC</td> 319 <td>Decoder-specific information from ESDS<sup>*</sup></td> 320 <td class=NA>Not Used</td> 321 <td class=NA>Not Used</td> 322 </tr> 323 <tr> 324 <td>VORBIS</td> 325 <td>Identification header</td> 326 <td>Setup header</td> 327 <td class=NA>Not Used</td> 328 </tr> 329 <tr> 330 <td>OPUS</td> 331 <td>Identification header</td> 332 <td>Pre-skip in nanosecs<br> 333 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)<br> 334 This overrides the pre-skip value in the identification header.</td> 335 <td>Seek Pre-roll in nanosecs<br> 336 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)</td> 337 </tr> 338 <tr> 339 <td>MPEG-4</td> 340 <td>Decoder-specific information from ESDS<sup>*</sup></td> 341 <td class=NA>Not Used</td> 342 <td class=NA>Not Used</td> 343 </tr> 344 <tr> 345 <td>H.264 AVC</td> 346 <td>SPS (Sequence Parameter Sets<sup>*</sup>)</td> 347 <td>PPS (Picture Parameter Sets<sup>*</sup>)</td> 348 <td class=NA>Not Used</td> 349 </tr> 350 <tr> 351 <td>H.265 HEVC</td> 352 <td>VPS (Video Parameter Sets<sup>*</sup>) +<br> 353 SPS (Sequence Parameter Sets<sup>*</sup>) +<br> 354 PPS (Picture Parameter Sets<sup>*</sup>)</td> 355 <td class=NA>Not Used</td> 356 <td class=NA>Not Used</td> 357 </tr> 358 <tr> 359 <td>VP9</td> 360 <td>VP9 <a href="http://wiki.webmproject.org/vp9-codecprivate">CodecPrivate</a> Data 361 (optional)</td> 362 <td class=NA>Not Used</td> 363 <td class=NA>Not Used</td> 364 </tr> 365 </tbody> 366 </table> 367 368 <p class=note> 369 <strong>Note:</strong> care must be taken if the codec is flushed immediately or shortly 370 after start, before any output buffer or output format change has been returned, as the codec 371 specific data may be lost during the flush. You must resubmit the data using buffers marked with 372 {@link #BUFFER_FLAG_CODEC_CONFIG} after such flush to ensure proper codec operation. 373 <p> 374 Encoders (or codecs that generate compressed data) will create and return the codec specific data 375 before any valid output buffer in output buffers marked with the {@linkplain 376 #BUFFER_FLAG_CODEC_CONFIG codec-config flag}. Buffers containing codec-specific-data have no 377 meaningful timestamps. 378 379 <h3>Data Processing</h3> 380 <p> 381 Each codec maintains a set of input and output buffers that are referred to by a buffer-ID in 382 API calls. After a successful call to {@link #start} the client "owns" neither input nor output 383 buffers. In synchronous mode, call {@link #dequeueInputBuffer dequeueInput}/{@link 384 #dequeueOutputBuffer OutputBuffer(…)} to obtain (get ownership of) an input or output 385 buffer from the codec. In asynchronous mode, you will automatically receive available buffers via 386 the {@link Callback#onInputBufferAvailable MediaCodec.Callback.onInput}/{@link 387 Callback#onOutputBufferAvailable OutputBufferAvailable(…)} callbacks. 388 <p> 389 Upon obtaining an input buffer, fill it with data and submit it to the codec using {@link 390 #queueInputBuffer queueInputBuffer} – or {@link #queueSecureInputBuffer 391 queueSecureInputBuffer} if using decryption. Do not submit multiple input buffers with the same 392 timestamp (unless it is <a href="#CSD">codec-specific data</a> marked as such). 393 <p> 394 The codec in turn will return a read-only output buffer via the {@link 395 Callback#onOutputBufferAvailable onOutputBufferAvailable} callback in asynchronous mode, or in 396 response to a {@link #dequeueOutputBuffer dequeuOutputBuffer} call in synchronous mode. After the 397 output buffer has been processed, call one of the {@link #releaseOutputBuffer 398 releaseOutputBuffer} methods to return the buffer to the codec. 399 <p> 400 While you are not required to resubmit/release buffers immediately to the codec, holding onto 401 input and/or output buffers may stall the codec, and this behavior is device dependent. 402 <strong>Specifically, it is possible that a codec may hold off on generating output buffers until 403 <em>all</em> outstanding buffers have been released/resubmitted.</strong> Therefore, try to 404 hold onto to available buffers as little as possible. 405 <p> 406 Depending on the API version, you can process data in three ways: 407 <table> 408 <thead> 409 <tr> 410 <th>Processing Mode</th> 411 <th>API version <= 20<br>Jelly Bean/KitKat</th> 412 <th>API version >= 21<br>Lollipop and later</th> 413 </tr> 414 </thead> 415 <tbody> 416 <tr> 417 <td>Synchronous API using buffer arrays</td> 418 <td>Supported</td> 419 <td>Deprecated</td> 420 </tr> 421 <tr> 422 <td>Synchronous API using buffers</td> 423 <td class=NA>Not Available</td> 424 <td>Supported</td> 425 </tr> 426 <tr> 427 <td>Asynchronous API using buffers</td> 428 <td class=NA>Not Available</td> 429 <td>Supported</td> 430 </tr> 431 </tbody> 432 </table> 433 434 <h4>Asynchronous Processing using Buffers</h4> 435 <p> 436 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the preferred method is to process data 437 asynchronously by setting a callback before calling {@link #configure configure}. Asynchronous 438 mode changes the state transitions slightly, because you must call {@link #start} after {@link 439 #flush} to transition the codec to the Running sub-state and start receiving input buffers. 440 Similarly, upon an initial call to {@code start} the codec will move directly to the Running 441 sub-state and start passing available input buffers via the callback. 442 <p> 443 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 444 data="../../../images/media/mediacodec_async_states.svg"><img 445 src="../../../images/media/mediacodec_async_states.png" style="width: 516px; height: 353px" 446 alt="MediaCodec state diagram for asynchronous operation"></object></center> 447 <p> 448 MediaCodec is typically used like this in asynchronous mode: 449 <pre class=prettyprint> 450 MediaCodec codec = MediaCodec.createByCodecName(name); 451 MediaFormat mOutputFormat; // member variable 452 codec.setCallback(new MediaCodec.Callback() { 453 {@literal @Override} 454 void onInputBufferAvailable(MediaCodec mc, int inputBufferId) { 455 ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); 456 // fill inputBuffer with valid data 457 … 458 codec.queueInputBuffer(inputBufferId, …); 459 } 460 461 {@literal @Override} 462 void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, …) { 463 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 464 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 465 // bufferFormat is equivalent to mOutputFormat 466 // outputBuffer is ready to be processed or rendered. 467 … 468 codec.releaseOutputBuffer(outputBufferId, …); 469 } 470 471 {@literal @Override} 472 void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { 473 // Subsequent data will conform to new format. 474 // Can ignore if using getOutputFormat(outputBufferId) 475 mOutputFormat = format; // option B 476 } 477 478 {@literal @Override} 479 void onError(…) { 480 … 481 } 482 }); 483 codec.configure(format, …); 484 mOutputFormat = codec.getOutputFormat(); // option B 485 codec.start(); 486 // wait for processing to complete 487 codec.stop(); 488 codec.release();</pre> 489 490 <h4>Synchronous Processing using Buffers</h4> 491 <p> 492 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you should retrieve input and output 493 buffers using {@link #getInputBuffer getInput}/{@link #getOutputBuffer OutputBuffer(int)} and/or 494 {@link #getInputImage getInput}/{@link #getOutputImage OutputImage(int)} even when using the 495 codec in synchronous mode. This allows certain optimizations by the framework, e.g. when 496 processing dynamic content. This optimization is disabled if you call {@link #getInputBuffers 497 getInput}/{@link #getOutputBuffers OutputBuffers()}. 498 499 <p class=note> 500 <strong>Note:</strong> do not mix the methods of using buffers and buffer arrays at the same 501 time. Specifically, only call {@code getInput}/{@code OutputBuffers} directly after {@link 502 #start} or after having dequeued an output buffer ID with the value of {@link 503 #INFO_OUTPUT_FORMAT_CHANGED}. 504 <p> 505 MediaCodec is typically used like this in synchronous mode: 506 <pre> 507 MediaCodec codec = MediaCodec.createByCodecName(name); 508 codec.configure(format, …); 509 MediaFormat outputFormat = codec.getOutputFormat(); // option B 510 codec.start(); 511 for (;;) { 512 int inputBufferId = codec.dequeueInputBuffer(timeoutUs); 513 if (inputBufferId >= 0) { 514 ByteBuffer inputBuffer = codec.getInputBuffer(…); 515 // fill inputBuffer with valid data 516 … 517 codec.queueInputBuffer(inputBufferId, …); 518 } 519 int outputBufferId = codec.dequeueOutputBuffer(…); 520 if (outputBufferId >= 0) { 521 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 522 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 523 // bufferFormat is identical to outputFormat 524 // outputBuffer is ready to be processed or rendered. 525 … 526 codec.releaseOutputBuffer(outputBufferId, …); 527 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 528 // Subsequent data will conform to new format. 529 // Can ignore if using getOutputFormat(outputBufferId) 530 outputFormat = codec.getOutputFormat(); // option B 531 } 532 } 533 codec.stop(); 534 codec.release();</pre> 535 536 <h4>Synchronous Processing using Buffer Arrays (deprecated)</h4> 537 <p> 538 In versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and before, the set of input and 539 output buffers are represented by the {@code ByteBuffer[]} arrays. After a successful call to 540 {@link #start}, retrieve the buffer arrays using {@link #getInputBuffers getInput}/{@link 541 #getOutputBuffers OutputBuffers()}. Use the buffer ID-s as indices into these arrays (when 542 non-negative), as demonstrated in the sample below. Note that there is no inherent correlation 543 between the size of the arrays and the number of input and output buffers used by the system, 544 although the array size provides an upper bound. 545 <pre> 546 MediaCodec codec = MediaCodec.createByCodecName(name); 547 codec.configure(format, …); 548 codec.start(); 549 ByteBuffer[] inputBuffers = codec.getInputBuffers(); 550 ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 551 for (;;) { 552 int inputBufferId = codec.dequeueInputBuffer(…); 553 if (inputBufferId >= 0) { 554 // fill inputBuffers[inputBufferId] with valid data 555 … 556 codec.queueInputBuffer(inputBufferId, …); 557 } 558 int outputBufferId = codec.dequeueOutputBuffer(…); 559 if (outputBufferId >= 0) { 560 // outputBuffers[outputBufferId] is ready to be processed or rendered. 561 … 562 codec.releaseOutputBuffer(outputBufferId, …); 563 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 564 outputBuffers = codec.getOutputBuffers(); 565 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 566 // Subsequent data will conform to new format. 567 MediaFormat format = codec.getOutputFormat(); 568 } 569 } 570 codec.stop(); 571 codec.release();</pre> 572 573 <h4>End-of-stream Handling</h4> 574 <p> 575 When you reach the end of the input data, you must signal it to the codec by specifying the 576 {@link #BUFFER_FLAG_END_OF_STREAM} flag in the call to {@link #queueInputBuffer 577 queueInputBuffer}. You can do this on the last valid input buffer, or by submitting an additional 578 empty input buffer with the end-of-stream flag set. If using an empty buffer, the timestamp will 579 be ignored. 580 <p> 581 The codec will continue to return output buffers until it eventually signals the end of the 582 output stream by specifying the same end-of-stream flag in the {@link BufferInfo} set in {@link 583 #dequeueOutputBuffer dequeueOutputBuffer} or returned via {@link Callback#onOutputBufferAvailable 584 onOutputBufferAvailable}. This can be set on the last valid output buffer, or on an empty buffer 585 after the last valid output buffer. The timestamp of such empty buffer should be ignored. 586 <p> 587 Do not submit additional input buffers after signaling the end of the input stream, unless the 588 codec has been flushed, or stopped and restarted. 589 590 <h4>Using an Output Surface</h4> 591 <p> 592 The data processing is nearly identical to the ByteBuffer mode when using an output {@link 593 Surface}; however, the output buffers will not be accessible, and are represented as {@code null} 594 values. E.g. {@link #getOutputBuffer getOutputBuffer}/{@link #getOutputImage Image(int)} will 595 return {@code null} and {@link #getOutputBuffers} will return an array containing only {@code 596 null}-s. 597 <p> 598 When using an output Surface, you can select whether or not to render each output buffer on the 599 surface. You have three choices: 600 <ul> 601 <li><strong>Do not render the buffer:</strong> Call {@link #releaseOutputBuffer(int, boolean) 602 releaseOutputBuffer(bufferId, false)}.</li> 603 <li><strong>Render the buffer with the default timestamp:</strong> Call {@link 604 #releaseOutputBuffer(int, boolean) releaseOutputBuffer(bufferId, true)}.</li> 605 <li><strong>Render the buffer with a specific timestamp:</strong> Call {@link 606 #releaseOutputBuffer(int, long) releaseOutputBuffer(bufferId, timestamp)}.</li> 607 </ul> 608 <p> 609 Since {@link android.os.Build.VERSION_CODES#M}, the default timestamp is the {@linkplain 610 BufferInfo#presentationTimeUs presentation timestamp} of the buffer (converted to nanoseconds). 611 It was not defined prior to that. 612 <p> 613 Also since {@link android.os.Build.VERSION_CODES#M}, you can change the output Surface 614 dynamically using {@link #setOutputSurface setOutputSurface}. 615 616 <h4>Transformations When Rendering onto Surface</h4> 617 618 If the codec is configured into Surface mode, any crop rectangle, {@linkplain 619 MediaFormat#KEY_ROTATION rotation} and {@linkplain #setVideoScalingMode video scaling 620 mode} will be automatically applied with one exception: 621 <p class=note> 622 Prior to the {@link android.os.Build.VERSION_CODES#M} release, software decoders may not 623 have applied the rotation when being rendered onto a Surface. Unfortunately, there is no way to 624 identify software decoders, or if they apply the rotation other than by trying it out. 625 <p> 626 There are also some caveats. 627 <p class=note> 628 Note that the pixel aspect ratio is not considered when displaying the output onto the 629 Surface. This means that if you are using {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT} mode, you 630 must position the output Surface so that it has the proper final display aspect ratio. Conversely, 631 you can only use {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode for content with 632 square pixels (pixel aspect ratio or 1:1). 633 <p class=note> 634 Note also that as of {@link android.os.Build.VERSION_CODES#N} release, {@link 635 #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode may not work correctly for videos rotated 636 by 90 or 270 degrees. 637 <p class=note> 638 When setting the video scaling mode, note that it must be reset after each time the output 639 buffers change. Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, you can 640 do this after each time the output format changes. 641 642 <h4>Using an Input Surface</h4> 643 <p> 644 When using an input Surface, there are no accessible input buffers, as buffers are automatically 645 passed from the input surface to the codec. Calling {@link #dequeueInputBuffer 646 dequeueInputBuffer} will throw an {@code IllegalStateException}, and {@link #getInputBuffers} 647 returns a bogus {@code ByteBuffer[]} array that <strong>MUST NOT</strong> be written into. 648 <p> 649 Call {@link #signalEndOfInputStream} to signal end-of-stream. The input surface will stop 650 submitting data to the codec immediately after this call. 651 <p> 652 653 <h3>Seeking & Adaptive Playback Support</h3> 654 <p> 655 Video decoders (and in general codecs that consume compressed video data) behave differently 656 regarding seek and format change whether or not they support and are configured for adaptive 657 playback. You can check if a decoder supports {@linkplain 658 CodecCapabilities#FEATURE_AdaptivePlayback adaptive playback} via {@link 659 CodecCapabilities#isFeatureSupported CodecCapabilities.isFeatureSupported(String)}. Adaptive 660 playback support for video decoders is only activated if you configure the codec to decode onto a 661 {@link Surface}. 662 663 <h4 id=KeyFrames><a name="KeyFrames"></a>Stream Boundary and Key Frames</h4> 664 <p> 665 It is important that the input data after {@link #start} or {@link #flush} starts at a suitable 666 stream boundary: the first frame must a key frame. A <em>key frame</em> can be decoded 667 completely on its own (for most codecs this means an I-frame), and no frames that are to be 668 displayed after a key frame refer to frames before the key frame. 669 <p> 670 The following table summarizes suitable key frames for various video formats. 671 <table> 672 <thead> 673 <tr> 674 <th>Format</th> 675 <th>Suitable key frame</th> 676 </tr> 677 </thead> 678 <tbody class=mid> 679 <tr> 680 <td>VP9/VP8</td> 681 <td>a suitable intraframe where no subsequent frames refer to frames prior to this frame.<br> 682 <i>(There is no specific name for such key frame.)</i></td> 683 </tr> 684 <tr> 685 <td>H.265 HEVC</td> 686 <td>IDR or CRA</td> 687 </tr> 688 <tr> 689 <td>H.264 AVC</td> 690 <td>IDR</td> 691 </tr> 692 <tr> 693 <td>MPEG-4<br>H.263<br>MPEG-2</td> 694 <td>a suitable I-frame where no subsequent frames refer to frames prior to this frame.<br> 695 <i>(There is no specific name for such key frame.)</td> 696 </tr> 697 </tbody> 698 </table> 699 700 <h4>For decoders that do not support adaptive playback (including when not decoding onto a 701 Surface)</h4> 702 <p> 703 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 704 seek) you <strong>MUST</strong> flush the decoder. Since all output buffers are immediately 705 revoked at the point of the flush, you may want to first signal then wait for the end-of-stream 706 before you call {@code flush}. It is important that the input data after a flush starts at a 707 suitable stream boundary/key frame. 708 <p class=note> 709 <strong>Note:</strong> the format of the data submitted after a flush must not change; {@link 710 #flush} does not support format discontinuities; for that, a full {@link #stop} - {@link 711 #configure configure(…)} - {@link #start} cycle is necessary. 712 713 <p class=note> 714 <strong>Also note:</strong> if you flush the codec too soon after {@link #start} – 715 generally, before the first output buffer or output format change is received – you 716 will need to resubmit the codec-specific-data to the codec. See the <a 717 href="#CSD">codec-specific-data section</a> for more info. 718 719 <h4>For decoders that support and are configured for adaptive playback</h4> 720 <p> 721 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 722 seek) it is <em>not necessary</em> to flush the decoder; however, input data after the 723 discontinuity must start at a suitable stream boundary/key frame. 724 <p> 725 For some video formats - namely H.264, H.265, VP8 and VP9 - it is also possible to change the 726 picture size or configuration mid-stream. To do this you must package the entire new 727 codec-specific configuration data together with the key frame into a single buffer (including 728 any start codes), and submit it as a <strong>regular</strong> input buffer. 729 <p> 730 You will receive an {@link #INFO_OUTPUT_FORMAT_CHANGED} return value from {@link 731 #dequeueOutputBuffer dequeueOutputBuffer} or a {@link Callback#onOutputBufferAvailable 732 onOutputFormatChanged} callback just after the picture-size change takes place and before any 733 frames with the new size have been returned. 734 <p class=note> 735 <strong>Note:</strong> just as the case for codec-specific data, be careful when calling 736 {@link #flush} shortly after you have changed the picture size. If you have not received 737 confirmation of the picture size change, you will need to repeat the request for the new picture 738 size. 739 740 <h3>Error handling</h3> 741 <p> 742 The factory methods {@link #createByCodecName createByCodecName} and {@link #createDecoderByType 743 createDecoder}/{@link #createEncoderByType EncoderByType} throw {@code IOException} on failure 744 which you must catch or declare to pass up. MediaCodec methods throw {@code 745 IllegalStateException} when the method is called from a codec state that does not allow it; this 746 is typically due to incorrect application API usage. Methods involving secure buffers may throw 747 {@link CryptoException}, which has further error information obtainable from {@link 748 CryptoException#getErrorCode}. 749 <p> 750 Internal codec errors result in a {@link CodecException}, which may be due to media content 751 corruption, hardware failure, resource exhaustion, and so forth, even when the application is 752 correctly using the API. The recommended action when receiving a {@code CodecException} 753 can be determined by calling {@link CodecException#isRecoverable} and {@link 754 CodecException#isTransient}: 755 <ul> 756 <li><strong>recoverable errors:</strong> If {@code isRecoverable()} returns true, then call 757 {@link #stop}, {@link #configure configure(…)}, and {@link #start} to recover.</li> 758 <li><strong>transient errors:</strong> If {@code isTransient()} returns true, then resources are 759 temporarily unavailable and the method may be retried at a later time.</li> 760 <li><strong>fatal errors:</strong> If both {@code isRecoverable()} and {@code isTransient()} 761 return false, then the {@code CodecException} is fatal and the codec must be {@linkplain #reset 762 reset} or {@linkplain #release released}.</li> 763 </ul> 764 <p> 765 Both {@code isRecoverable()} and {@code isTransient()} do not return true at the same time. 766 767 <h2 id=History><a name="History"></a>Valid API Calls and API History</h2> 768 <p> 769 This sections summarizes the valid API calls in each state and the API history of the MediaCodec 770 class. For API version numbers, see {@link android.os.Build.VERSION_CODES}. 771 772 <style> 773 .api > tr > th, .api > tr > td { text-align: center; padding: 4px 4px; } 774 .api > tr > th { vertical-align: bottom; } 775 .api > tr > td { vertical-align: middle; } 776 .sml > tr > th, .sml > tr > td { text-align: center; padding: 2px 4px; } 777 .fn { text-align: left; } 778 .fn > code > a { font: 14px/19px Roboto Condensed, sans-serif; } 779 .deg45 { 780 white-space: nowrap; background: none; border: none; vertical-align: bottom; 781 width: 30px; height: 83px; 782 } 783 .deg45 > div { 784 transform: skew(-45deg, 0deg) translate(1px, -67px); 785 transform-origin: bottom left 0; 786 width: 30px; height: 20px; 787 } 788 .deg45 > div > div { border: 1px solid #ddd; background: #999; height: 90px; width: 42px; } 789 .deg45 > div > div > div { transform: skew(45deg, 0deg) translate(-55px, 55px) rotate(-45deg); } 790 </style> 791 792 <table align="right" style="width: 0%"> 793 <thead> 794 <tr><th>Symbol</th><th>Meaning</th></tr> 795 </thead> 796 <tbody class=sml> 797 <tr><td>●</td><td>Supported</td></tr> 798 <tr><td>⁕</td><td>Semantics changed</td></tr> 799 <tr><td>○</td><td>Experimental support</td></tr> 800 <tr><td>[ ]</td><td>Deprecated</td></tr> 801 <tr><td>⎋</td><td>Restricted to surface input mode</td></tr> 802 <tr><td>⎆</td><td>Restricted to surface output mode</td></tr> 803 <tr><td>▧</td><td>Restricted to ByteBuffer input mode</td></tr> 804 <tr><td>↩</td><td>Restricted to synchronous mode</td></tr> 805 <tr><td>⇄</td><td>Restricted to asynchronous mode</td></tr> 806 <tr><td>( )</td><td>Can be called, but shouldn't</td></tr> 807 </tbody> 808 </table> 809 810 <table style="width: 100%;"> 811 <thead class=api> 812 <tr> 813 <th class=deg45><div><div style="background:#4285f4"><div>Uninitialized</div></div></div></th> 814 <th class=deg45><div><div style="background:#f4b400"><div>Configured</div></div></div></th> 815 <th class=deg45><div><div style="background:#e67c73"><div>Flushed</div></div></div></th> 816 <th class=deg45><div><div style="background:#0f9d58"><div>Running</div></div></div></th> 817 <th class=deg45><div><div style="background:#f7cb4d"><div>End of Stream</div></div></div></th> 818 <th class=deg45><div><div style="background:#db4437"><div>Error</div></div></div></th> 819 <th class=deg45><div><div style="background:#666"><div>Released</div></div></div></th> 820 <th></th> 821 <th colspan="8">SDK Version</th> 822 </tr> 823 <tr> 824 <th colspan="7">State</th> 825 <th>Method</th> 826 <th>16</th> 827 <th>17</th> 828 <th>18</th> 829 <th>19</th> 830 <th>20</th> 831 <th>21</th> 832 <th>22</th> 833 <th>23</th> 834 </tr> 835 </thead> 836 <tbody class=api> 837 <tr> 838 <td></td> 839 <td></td> 840 <td></td> 841 <td></td> 842 <td></td> 843 <td></td> 844 <td></td> 845 <td class=fn>{@link #createByCodecName createByCodecName}</td> 846 <td>●</td> 847 <td>●</td> 848 <td>●</td> 849 <td>●</td> 850 <td>●</td> 851 <td>●</td> 852 <td>●</td> 853 <td>●</td> 854 </tr> 855 <tr> 856 <td></td> 857 <td></td> 858 <td></td> 859 <td></td> 860 <td></td> 861 <td></td> 862 <td></td> 863 <td class=fn>{@link #createDecoderByType createDecoderByType}</td> 864 <td>●</td> 865 <td>●</td> 866 <td>●</td> 867 <td>●</td> 868 <td>●</td> 869 <td>●</td> 870 <td>●</td> 871 <td>●</td> 872 </tr> 873 <tr> 874 <td></td> 875 <td></td> 876 <td></td> 877 <td></td> 878 <td></td> 879 <td></td> 880 <td></td> 881 <td class=fn>{@link #createEncoderByType createEncoderByType}</td> 882 <td>●</td> 883 <td>●</td> 884 <td>●</td> 885 <td>●</td> 886 <td>●</td> 887 <td>●</td> 888 <td>●</td> 889 <td>●</td> 890 </tr> 891 <tr> 892 <td></td> 893 <td></td> 894 <td></td> 895 <td></td> 896 <td></td> 897 <td></td> 898 <td></td> 899 <td class=fn>{@link #createPersistentInputSurface createPersistentInputSurface}</td> 900 <td></td> 901 <td></td> 902 <td></td> 903 <td></td> 904 <td></td> 905 <td></td> 906 <td></td> 907 <td>●</td> 908 </tr> 909 <tr> 910 <td>16+</td> 911 <td>-</td> 912 <td>-</td> 913 <td>-</td> 914 <td>-</td> 915 <td>-</td> 916 <td>-</td> 917 <td class=fn>{@link #configure configure}</td> 918 <td>●</td> 919 <td>●</td> 920 <td>●</td> 921 <td>●</td> 922 <td>●</td> 923 <td>⁕</td> 924 <td>●</td> 925 <td>●</td> 926 </tr> 927 <tr> 928 <td>-</td> 929 <td>18+</td> 930 <td>-</td> 931 <td>-</td> 932 <td>-</td> 933 <td>-</td> 934 <td>-</td> 935 <td class=fn>{@link #createInputSurface createInputSurface}</td> 936 <td></td> 937 <td></td> 938 <td>⎋</td> 939 <td>⎋</td> 940 <td>⎋</td> 941 <td>⎋</td> 942 <td>⎋</td> 943 <td>⎋</td> 944 </tr> 945 <tr> 946 <td>-</td> 947 <td>-</td> 948 <td>16+</td> 949 <td>16+</td> 950 <td>(16+)</td> 951 <td>-</td> 952 <td>-</td> 953 <td class=fn>{@link #dequeueInputBuffer dequeueInputBuffer}</td> 954 <td>●</td> 955 <td>●</td> 956 <td>▧</td> 957 <td>▧</td> 958 <td>▧</td> 959 <td>⁕▧↩</td> 960 <td>▧↩</td> 961 <td>▧↩</td> 962 </tr> 963 <tr> 964 <td>-</td> 965 <td>-</td> 966 <td>16+</td> 967 <td>16+</td> 968 <td>16+</td> 969 <td>-</td> 970 <td>-</td> 971 <td class=fn>{@link #dequeueOutputBuffer dequeueOutputBuffer}</td> 972 <td>●</td> 973 <td>●</td> 974 <td>●</td> 975 <td>●</td> 976 <td>●</td> 977 <td>⁕↩</td> 978 <td>↩</td> 979 <td>↩</td> 980 </tr> 981 <tr> 982 <td>-</td> 983 <td>-</td> 984 <td>16+</td> 985 <td>16+</td> 986 <td>16+</td> 987 <td>-</td> 988 <td>-</td> 989 <td class=fn>{@link #flush flush}</td> 990 <td>●</td> 991 <td>●</td> 992 <td>●</td> 993 <td>●</td> 994 <td>●</td> 995 <td>●</td> 996 <td>●</td> 997 <td>●</td> 998 </tr> 999 <tr> 1000 <td>18+</td> 1001 <td>18+</td> 1002 <td>18+</td> 1003 <td>18+</td> 1004 <td>18+</td> 1005 <td>18+</td> 1006 <td>-</td> 1007 <td class=fn>{@link #getCodecInfo getCodecInfo}</td> 1008 <td></td> 1009 <td></td> 1010 <td>●</td> 1011 <td>●</td> 1012 <td>●</td> 1013 <td>●</td> 1014 <td>●</td> 1015 <td>●</td> 1016 </tr> 1017 <tr> 1018 <td>-</td> 1019 <td>-</td> 1020 <td>(21+)</td> 1021 <td>21+</td> 1022 <td>(21+)</td> 1023 <td>-</td> 1024 <td>-</td> 1025 <td class=fn>{@link #getInputBuffer getInputBuffer}</td> 1026 <td></td> 1027 <td></td> 1028 <td></td> 1029 <td></td> 1030 <td></td> 1031 <td>●</td> 1032 <td>●</td> 1033 <td>●</td> 1034 </tr> 1035 <tr> 1036 <td>-</td> 1037 <td>-</td> 1038 <td>16+</td> 1039 <td>(16+)</td> 1040 <td>(16+)</td> 1041 <td>-</td> 1042 <td>-</td> 1043 <td class=fn>{@link #getInputBuffers getInputBuffers}</td> 1044 <td>●</td> 1045 <td>●</td> 1046 <td>●</td> 1047 <td>●</td> 1048 <td>●</td> 1049 <td>[⁕↩]</td> 1050 <td>[↩]</td> 1051 <td>[↩]</td> 1052 </tr> 1053 <tr> 1054 <td>-</td> 1055 <td>21+</td> 1056 <td>(21+)</td> 1057 <td>(21+)</td> 1058 <td>(21+)</td> 1059 <td>-</td> 1060 <td>-</td> 1061 <td class=fn>{@link #getInputFormat getInputFormat}</td> 1062 <td></td> 1063 <td></td> 1064 <td></td> 1065 <td></td> 1066 <td></td> 1067 <td>●</td> 1068 <td>●</td> 1069 <td>●</td> 1070 </tr> 1071 <tr> 1072 <td>-</td> 1073 <td>-</td> 1074 <td>(21+)</td> 1075 <td>21+</td> 1076 <td>(21+)</td> 1077 <td>-</td> 1078 <td>-</td> 1079 <td class=fn>{@link #getInputImage getInputImage}</td> 1080 <td></td> 1081 <td></td> 1082 <td></td> 1083 <td></td> 1084 <td></td> 1085 <td>○</td> 1086 <td>●</td> 1087 <td>●</td> 1088 </tr> 1089 <tr> 1090 <td>18+</td> 1091 <td>18+</td> 1092 <td>18+</td> 1093 <td>18+</td> 1094 <td>18+</td> 1095 <td>18+</td> 1096 <td>-</td> 1097 <td class=fn>{@link #getName getName}</td> 1098 <td></td> 1099 <td></td> 1100 <td>●</td> 1101 <td>●</td> 1102 <td>●</td> 1103 <td>●</td> 1104 <td>●</td> 1105 <td>●</td> 1106 </tr> 1107 <tr> 1108 <td>-</td> 1109 <td>-</td> 1110 <td>(21+)</td> 1111 <td>21+</td> 1112 <td>21+</td> 1113 <td>-</td> 1114 <td>-</td> 1115 <td class=fn>{@link #getOutputBuffer getOutputBuffer}</td> 1116 <td></td> 1117 <td></td> 1118 <td></td> 1119 <td></td> 1120 <td></td> 1121 <td>●</td> 1122 <td>●</td> 1123 <td>●</td> 1124 </tr> 1125 <tr> 1126 <td>-</td> 1127 <td>-</td> 1128 <td>16+</td> 1129 <td>16+</td> 1130 <td>16+</td> 1131 <td>-</td> 1132 <td>-</td> 1133 <td class=fn>{@link #getOutputBuffers getOutputBuffers}</td> 1134 <td>●</td> 1135 <td>●</td> 1136 <td>●</td> 1137 <td>●</td> 1138 <td>●</td> 1139 <td>[⁕↩]</td> 1140 <td>[↩]</td> 1141 <td>[↩]</td> 1142 </tr> 1143 <tr> 1144 <td>-</td> 1145 <td>21+</td> 1146 <td>16+</td> 1147 <td>16+</td> 1148 <td>16+</td> 1149 <td>-</td> 1150 <td>-</td> 1151 <td class=fn>{@link #getOutputFormat()}</td> 1152 <td>●</td> 1153 <td>●</td> 1154 <td>●</td> 1155 <td>●</td> 1156 <td>●</td> 1157 <td>●</td> 1158 <td>●</td> 1159 <td>●</td> 1160 </tr> 1161 <tr> 1162 <td>-</td> 1163 <td>-</td> 1164 <td>(21+)</td> 1165 <td>21+</td> 1166 <td>21+</td> 1167 <td>-</td> 1168 <td>-</td> 1169 <td class=fn>{@link #getOutputFormat(int)}</td> 1170 <td></td> 1171 <td></td> 1172 <td></td> 1173 <td></td> 1174 <td></td> 1175 <td>●</td> 1176 <td>●</td> 1177 <td>●</td> 1178 </tr> 1179 <tr> 1180 <td>-</td> 1181 <td>-</td> 1182 <td>(21+)</td> 1183 <td>21+</td> 1184 <td>21+</td> 1185 <td>-</td> 1186 <td>-</td> 1187 <td class=fn>{@link #getOutputImage getOutputImage}</td> 1188 <td></td> 1189 <td></td> 1190 <td></td> 1191 <td></td> 1192 <td></td> 1193 <td>○</td> 1194 <td>●</td> 1195 <td>●</td> 1196 </tr> 1197 <tr> 1198 <td>-</td> 1199 <td>-</td> 1200 <td>-</td> 1201 <td>16+</td> 1202 <td>(16+)</td> 1203 <td>-</td> 1204 <td>-</td> 1205 <td class=fn>{@link #queueInputBuffer queueInputBuffer}</td> 1206 <td>●</td> 1207 <td>●</td> 1208 <td>●</td> 1209 <td>●</td> 1210 <td>●</td> 1211 <td>⁕</td> 1212 <td>●</td> 1213 <td>●</td> 1214 </tr> 1215 <tr> 1216 <td>-</td> 1217 <td>-</td> 1218 <td>-</td> 1219 <td>16+</td> 1220 <td>(16+)</td> 1221 <td>-</td> 1222 <td>-</td> 1223 <td class=fn>{@link #queueSecureInputBuffer queueSecureInputBuffer}</td> 1224 <td>●</td> 1225 <td>●</td> 1226 <td>●</td> 1227 <td>●</td> 1228 <td>●</td> 1229 <td>⁕</td> 1230 <td>●</td> 1231 <td>●</td> 1232 </tr> 1233 <tr> 1234 <td>16+</td> 1235 <td>16+</td> 1236 <td>16+</td> 1237 <td>16+</td> 1238 <td>16+</td> 1239 <td>16+</td> 1240 <td>16+</td> 1241 <td class=fn>{@link #release release}</td> 1242 <td>●</td> 1243 <td>●</td> 1244 <td>●</td> 1245 <td>●</td> 1246 <td>●</td> 1247 <td>●</td> 1248 <td>●</td> 1249 <td>●</td> 1250 </tr> 1251 <tr> 1252 <td>-</td> 1253 <td>-</td> 1254 <td>-</td> 1255 <td>16+</td> 1256 <td>16+</td> 1257 <td>-</td> 1258 <td>-</td> 1259 <td class=fn>{@link #releaseOutputBuffer(int, boolean)}</td> 1260 <td>●</td> 1261 <td>●</td> 1262 <td>●</td> 1263 <td>●</td> 1264 <td>●</td> 1265 <td>⁕</td> 1266 <td>●</td> 1267 <td>⁕</td> 1268 </tr> 1269 <tr> 1270 <td>-</td> 1271 <td>-</td> 1272 <td>-</td> 1273 <td>21+</td> 1274 <td>21+</td> 1275 <td>-</td> 1276 <td>-</td> 1277 <td class=fn>{@link #releaseOutputBuffer(int, long)}</td> 1278 <td></td> 1279 <td></td> 1280 <td></td> 1281 <td></td> 1282 <td></td> 1283 <td>⎆</td> 1284 <td>⎆</td> 1285 <td>⎆</td> 1286 </tr> 1287 <tr> 1288 <td>21+</td> 1289 <td>21+</td> 1290 <td>21+</td> 1291 <td>21+</td> 1292 <td>21+</td> 1293 <td>21+</td> 1294 <td>-</td> 1295 <td class=fn>{@link #reset reset}</td> 1296 <td></td> 1297 <td></td> 1298 <td></td> 1299 <td></td> 1300 <td></td> 1301 <td>●</td> 1302 <td>●</td> 1303 <td>●</td> 1304 </tr> 1305 <tr> 1306 <td>21+</td> 1307 <td>-</td> 1308 <td>-</td> 1309 <td>-</td> 1310 <td>-</td> 1311 <td>-</td> 1312 <td>-</td> 1313 <td class=fn>{@link #setCallback(Callback) setCallback}</td> 1314 <td></td> 1315 <td></td> 1316 <td></td> 1317 <td></td> 1318 <td></td> 1319 <td>●</td> 1320 <td>●</td> 1321 <td>{@link #setCallback(Callback, Handler) ⁕}</td> 1322 </tr> 1323 <tr> 1324 <td>-</td> 1325 <td>23+</td> 1326 <td>-</td> 1327 <td>-</td> 1328 <td>-</td> 1329 <td>-</td> 1330 <td>-</td> 1331 <td class=fn>{@link #setInputSurface setInputSurface}</td> 1332 <td></td> 1333 <td></td> 1334 <td></td> 1335 <td></td> 1336 <td></td> 1337 <td></td> 1338 <td></td> 1339 <td>⎋</td> 1340 </tr> 1341 <tr> 1342 <td>23+</td> 1343 <td>23+</td> 1344 <td>23+</td> 1345 <td>23+</td> 1346 <td>23+</td> 1347 <td>(23+)</td> 1348 <td>(23+)</td> 1349 <td class=fn>{@link #setOnFrameRenderedListener setOnFrameRenderedListener}</td> 1350 <td></td> 1351 <td></td> 1352 <td></td> 1353 <td></td> 1354 <td></td> 1355 <td></td> 1356 <td></td> 1357 <td>○ ⎆</td> 1358 </tr> 1359 <tr> 1360 <td>-</td> 1361 <td>23+</td> 1362 <td>23+</td> 1363 <td>23+</td> 1364 <td>23+</td> 1365 <td>-</td> 1366 <td>-</td> 1367 <td class=fn>{@link #setOutputSurface setOutputSurface}</td> 1368 <td></td> 1369 <td></td> 1370 <td></td> 1371 <td></td> 1372 <td></td> 1373 <td></td> 1374 <td></td> 1375 <td>⎆</td> 1376 </tr> 1377 <tr> 1378 <td>19+</td> 1379 <td>19+</td> 1380 <td>19+</td> 1381 <td>19+</td> 1382 <td>19+</td> 1383 <td>(19+)</td> 1384 <td>-</td> 1385 <td class=fn>{@link #setParameters setParameters}</td> 1386 <td></td> 1387 <td></td> 1388 <td></td> 1389 <td>●</td> 1390 <td>●</td> 1391 <td>●</td> 1392 <td>●</td> 1393 <td>●</td> 1394 </tr> 1395 <tr> 1396 <td>-</td> 1397 <td>(16+)</td> 1398 <td>(16+)</td> 1399 <td>16+</td> 1400 <td>(16+)</td> 1401 <td>(16+)</td> 1402 <td>-</td> 1403 <td class=fn>{@link #setVideoScalingMode setVideoScalingMode}</td> 1404 <td>⎆</td> 1405 <td>⎆</td> 1406 <td>⎆</td> 1407 <td>⎆</td> 1408 <td>⎆</td> 1409 <td>⎆</td> 1410 <td>⎆</td> 1411 <td>⎆</td> 1412 </tr> 1413 <tr> 1414 <td>-</td> 1415 <td>-</td> 1416 <td>18+</td> 1417 <td>18+</td> 1418 <td>-</td> 1419 <td>-</td> 1420 <td>-</td> 1421 <td class=fn>{@link #signalEndOfInputStream signalEndOfInputStream}</td> 1422 <td></td> 1423 <td></td> 1424 <td>⎋</td> 1425 <td>⎋</td> 1426 <td>⎋</td> 1427 <td>⎋</td> 1428 <td>⎋</td> 1429 <td>⎋</td> 1430 </tr> 1431 <tr> 1432 <td>-</td> 1433 <td>16+</td> 1434 <td>21+(⇄)</td> 1435 <td>-</td> 1436 <td>-</td> 1437 <td>-</td> 1438 <td>-</td> 1439 <td class=fn>{@link #start start}</td> 1440 <td>●</td> 1441 <td>●</td> 1442 <td>●</td> 1443 <td>●</td> 1444 <td>●</td> 1445 <td>⁕</td> 1446 <td>●</td> 1447 <td>●</td> 1448 </tr> 1449 <tr> 1450 <td>-</td> 1451 <td>-</td> 1452 <td>16+</td> 1453 <td>16+</td> 1454 <td>16+</td> 1455 <td>-</td> 1456 <td>-</td> 1457 <td class=fn>{@link #stop stop}</td> 1458 <td>●</td> 1459 <td>●</td> 1460 <td>●</td> 1461 <td>●</td> 1462 <td>●</td> 1463 <td>●</td> 1464 <td>●</td> 1465 <td>●</td> 1466 </tr> 1467 </tbody> 1468 </table> 1469 */ 1470 final public class MediaCodec { 1471 /** 1472 * Per buffer metadata includes an offset and size specifying 1473 * the range of valid data in the associated codec (output) buffer. 1474 */ 1475 public final static class BufferInfo { 1476 /** 1477 * Update the buffer metadata information. 1478 * 1479 * @param newOffset the start-offset of the data in the buffer. 1480 * @param newSize the amount of data (in bytes) in the buffer. 1481 * @param newTimeUs the presentation timestamp in microseconds. 1482 * @param newFlags buffer flags associated with the buffer. This 1483 * should be a combination of {@link #BUFFER_FLAG_KEY_FRAME} and 1484 * {@link #BUFFER_FLAG_END_OF_STREAM}. 1485 */ set( int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags)1486 public void set( 1487 int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags) { 1488 offset = newOffset; 1489 size = newSize; 1490 presentationTimeUs = newTimeUs; 1491 flags = newFlags; 1492 } 1493 1494 /** 1495 * The start-offset of the data in the buffer. 1496 */ 1497 public int offset; 1498 1499 /** 1500 * The amount of data (in bytes) in the buffer. If this is {@code 0}, 1501 * the buffer has no data in it and can be discarded. The only 1502 * use of a 0-size buffer is to carry the end-of-stream marker. 1503 */ 1504 public int size; 1505 1506 /** 1507 * The presentation timestamp in microseconds for the buffer. 1508 * This is derived from the presentation timestamp passed in 1509 * with the corresponding input buffer. This should be ignored for 1510 * a 0-sized buffer. 1511 */ 1512 public long presentationTimeUs; 1513 1514 /** 1515 * Buffer flags associated with the buffer. A combination of 1516 * {@link #BUFFER_FLAG_KEY_FRAME} and {@link #BUFFER_FLAG_END_OF_STREAM}. 1517 * 1518 * <p>Encoded buffers that are key frames are marked with 1519 * {@link #BUFFER_FLAG_KEY_FRAME}. 1520 * 1521 * <p>The last output buffer corresponding to the input buffer 1522 * marked with {@link #BUFFER_FLAG_END_OF_STREAM} will also be marked 1523 * with {@link #BUFFER_FLAG_END_OF_STREAM}. In some cases this could 1524 * be an empty buffer, whose sole purpose is to carry the end-of-stream 1525 * marker. 1526 */ 1527 @BufferFlag 1528 public int flags; 1529 1530 /** @hide */ 1531 @NonNull dup()1532 public BufferInfo dup() { 1533 BufferInfo copy = new BufferInfo(); 1534 copy.set(offset, size, presentationTimeUs, flags); 1535 return copy; 1536 } 1537 }; 1538 1539 // The follow flag constants MUST stay in sync with their equivalents 1540 // in MediaCodec.h ! 1541 1542 /** 1543 * This indicates that the (encoded) buffer marked as such contains 1544 * the data for a key frame. 1545 * 1546 * @deprecated Use {@link #BUFFER_FLAG_KEY_FRAME} instead. 1547 */ 1548 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 1549 1550 /** 1551 * This indicates that the (encoded) buffer marked as such contains 1552 * the data for a key frame. 1553 */ 1554 public static final int BUFFER_FLAG_KEY_FRAME = 1; 1555 1556 /** 1557 * This indicated that the buffer marked as such contains codec 1558 * initialization / codec specific data instead of media data. 1559 */ 1560 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 1561 1562 /** 1563 * This signals the end of stream, i.e. no buffers will be available 1564 * after this, unless of course, {@link #flush} follows. 1565 */ 1566 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 1567 1568 /** @hide */ 1569 @IntDef( 1570 flag = true, 1571 value = { 1572 BUFFER_FLAG_SYNC_FRAME, 1573 BUFFER_FLAG_KEY_FRAME, 1574 BUFFER_FLAG_CODEC_CONFIG, 1575 BUFFER_FLAG_END_OF_STREAM, 1576 }) 1577 @Retention(RetentionPolicy.SOURCE) 1578 public @interface BufferFlag {} 1579 1580 private EventHandler mEventHandler; 1581 private EventHandler mOnFrameRenderedHandler; 1582 private EventHandler mCallbackHandler; 1583 private Callback mCallback; 1584 private OnFrameRenderedListener mOnFrameRenderedListener; 1585 private Object mListenerLock = new Object(); 1586 1587 private static final int EVENT_CALLBACK = 1; 1588 private static final int EVENT_SET_CALLBACK = 2; 1589 private static final int EVENT_FRAME_RENDERED = 3; 1590 1591 private static final int CB_INPUT_AVAILABLE = 1; 1592 private static final int CB_OUTPUT_AVAILABLE = 2; 1593 private static final int CB_ERROR = 3; 1594 private static final int CB_OUTPUT_FORMAT_CHANGE = 4; 1595 1596 private class EventHandler extends Handler { 1597 private MediaCodec mCodec; 1598 EventHandler(@onNull MediaCodec codec, @NonNull Looper looper)1599 public EventHandler(@NonNull MediaCodec codec, @NonNull Looper looper) { 1600 super(looper); 1601 mCodec = codec; 1602 } 1603 1604 @Override handleMessage(@onNull Message msg)1605 public void handleMessage(@NonNull Message msg) { 1606 switch (msg.what) { 1607 case EVENT_CALLBACK: 1608 { 1609 handleCallback(msg); 1610 break; 1611 } 1612 case EVENT_SET_CALLBACK: 1613 { 1614 mCallback = (MediaCodec.Callback) msg.obj; 1615 break; 1616 } 1617 case EVENT_FRAME_RENDERED: 1618 synchronized (mListenerLock) { 1619 Map<String, Object> map = (Map<String, Object>)msg.obj; 1620 for (int i = 0; ; ++i) { 1621 Object mediaTimeUs = map.get(i + "-media-time-us"); 1622 Object systemNano = map.get(i + "-system-nano"); 1623 if (mediaTimeUs == null || systemNano == null 1624 || mOnFrameRenderedListener == null) { 1625 break; 1626 } 1627 mOnFrameRenderedListener.onFrameRendered( 1628 mCodec, (long)mediaTimeUs, (long)systemNano); 1629 } 1630 break; 1631 } 1632 default: 1633 { 1634 break; 1635 } 1636 } 1637 } 1638 handleCallback(@onNull Message msg)1639 private void handleCallback(@NonNull Message msg) { 1640 if (mCallback == null) { 1641 return; 1642 } 1643 1644 switch (msg.arg1) { 1645 case CB_INPUT_AVAILABLE: 1646 { 1647 int index = msg.arg2; 1648 synchronized(mBufferLock) { 1649 validateInputByteBuffer(mCachedInputBuffers, index); 1650 } 1651 mCallback.onInputBufferAvailable(mCodec, index); 1652 break; 1653 } 1654 1655 case CB_OUTPUT_AVAILABLE: 1656 { 1657 int index = msg.arg2; 1658 BufferInfo info = (MediaCodec.BufferInfo) msg.obj; 1659 synchronized(mBufferLock) { 1660 validateOutputByteBuffer(mCachedOutputBuffers, index, info); 1661 } 1662 mCallback.onOutputBufferAvailable( 1663 mCodec, index, info); 1664 break; 1665 } 1666 1667 case CB_ERROR: 1668 { 1669 mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj); 1670 break; 1671 } 1672 1673 case CB_OUTPUT_FORMAT_CHANGE: 1674 { 1675 mCallback.onOutputFormatChanged(mCodec, 1676 new MediaFormat((Map<String, Object>) msg.obj)); 1677 break; 1678 } 1679 1680 default: 1681 { 1682 break; 1683 } 1684 } 1685 } 1686 } 1687 1688 private boolean mHasSurface = false; 1689 1690 /** 1691 * Instantiate the preferred decoder supporting input data of the given mime type. 1692 * 1693 * The following is a partial list of defined mime types and their semantics: 1694 * <ul> 1695 * <li>"video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm) 1696 * <li>"video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm) 1697 * <li>"video/avc" - H.264/AVC video 1698 * <li>"video/hevc" - H.265/HEVC video 1699 * <li>"video/mp4v-es" - MPEG4 video 1700 * <li>"video/3gpp" - H.263 video 1701 * <li>"audio/3gpp" - AMR narrowband audio 1702 * <li>"audio/amr-wb" - AMR wideband audio 1703 * <li>"audio/mpeg" - MPEG1/2 audio layer III 1704 * <li>"audio/mp4a-latm" - AAC audio (note, this is raw AAC packets, not packaged in LATM!) 1705 * <li>"audio/vorbis" - vorbis audio 1706 * <li>"audio/g711-alaw" - G.711 alaw audio 1707 * <li>"audio/g711-mlaw" - G.711 ulaw audio 1708 * </ul> 1709 * 1710 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findDecoderForFormat} 1711 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1712 * given format. 1713 * 1714 * @param type The mime type of the input data. 1715 * @throws IOException if the codec cannot be created. 1716 * @throws IllegalArgumentException if type is not a valid mime type. 1717 * @throws NullPointerException if type is null. 1718 */ 1719 @NonNull createDecoderByType(@onNull String type)1720 public static MediaCodec createDecoderByType(@NonNull String type) 1721 throws IOException { 1722 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 1723 } 1724 1725 /** 1726 * Instantiate the preferred encoder supporting output data of the given mime type. 1727 * 1728 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findEncoderForFormat} 1729 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1730 * given format. 1731 * 1732 * @param type The desired mime type of the output data. 1733 * @throws IOException if the codec cannot be created. 1734 * @throws IllegalArgumentException if type is not a valid mime type. 1735 * @throws NullPointerException if type is null. 1736 */ 1737 @NonNull createEncoderByType(@onNull String type)1738 public static MediaCodec createEncoderByType(@NonNull String type) 1739 throws IOException { 1740 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 1741 } 1742 1743 /** 1744 * If you know the exact name of the component you want to instantiate 1745 * use this method to instantiate it. Use with caution. 1746 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 1747 * @param name The name of the codec to be instantiated. 1748 * @throws IOException if the codec cannot be created. 1749 * @throws IllegalArgumentException if name is not valid. 1750 * @throws NullPointerException if name is null. 1751 */ 1752 @NonNull createByCodecName(@onNull String name)1753 public static MediaCodec createByCodecName(@NonNull String name) 1754 throws IOException { 1755 return new MediaCodec( 1756 name, false /* nameIsType */, false /* unused */); 1757 } 1758 MediaCodec( @onNull String name, boolean nameIsType, boolean encoder)1759 private MediaCodec( 1760 @NonNull String name, boolean nameIsType, boolean encoder) { 1761 Looper looper; 1762 if ((looper = Looper.myLooper()) != null) { 1763 mEventHandler = new EventHandler(this, looper); 1764 } else if ((looper = Looper.getMainLooper()) != null) { 1765 mEventHandler = new EventHandler(this, looper); 1766 } else { 1767 mEventHandler = null; 1768 } 1769 mCallbackHandler = mEventHandler; 1770 mOnFrameRenderedHandler = mEventHandler; 1771 1772 mBufferLock = new Object(); 1773 1774 native_setup(name, nameIsType, encoder); 1775 } 1776 1777 @Override finalize()1778 protected void finalize() { 1779 native_finalize(); 1780 } 1781 1782 /** 1783 * Returns the codec to its initial (Uninitialized) state. 1784 * 1785 * Call this if an {@link MediaCodec.CodecException#isRecoverable unrecoverable} 1786 * error has occured to reset the codec to its initial state after creation. 1787 * 1788 * @throws CodecException if an unrecoverable error has occured and the codec 1789 * could not be reset. 1790 * @throws IllegalStateException if in the Released state. 1791 */ reset()1792 public final void reset() { 1793 freeAllTrackedBuffers(); // free buffers first 1794 native_reset(); 1795 } 1796 native_reset()1797 private native final void native_reset(); 1798 1799 /** 1800 * Free up resources used by the codec instance. 1801 * 1802 * Make sure you call this when you're done to free up any opened 1803 * component instance instead of relying on the garbage collector 1804 * to do this for you at some point in the future. 1805 */ release()1806 public final void release() { 1807 freeAllTrackedBuffers(); // free buffers first 1808 native_release(); 1809 } 1810 native_release()1811 private native final void native_release(); 1812 1813 /** 1814 * If this codec is to be used as an encoder, pass this flag. 1815 */ 1816 public static final int CONFIGURE_FLAG_ENCODE = 1; 1817 1818 /** @hide */ 1819 @IntDef(flag = true, value = { CONFIGURE_FLAG_ENCODE }) 1820 @Retention(RetentionPolicy.SOURCE) 1821 public @interface ConfigureFlag {} 1822 1823 /** 1824 * Configures a component. 1825 * 1826 * @param format The format of the input data (decoder) or the desired 1827 * format of the output data (encoder). Passing {@code null} 1828 * as {@code format} is equivalent to passing an 1829 * {@link MediaFormat#MediaFormat an empty mediaformat}. 1830 * @param surface Specify a surface on which to render the output of this 1831 * decoder. Pass {@code null} as {@code surface} if the 1832 * codec does not generate raw video output (e.g. not a video 1833 * decoder) and/or if you want to configure the codec for 1834 * {@link ByteBuffer} output. 1835 * @param crypto Specify a crypto object to facilitate secure decryption 1836 * of the media data. Pass {@code null} as {@code crypto} for 1837 * non-secure codecs. 1838 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 1839 * component as an encoder. 1840 * @throws IllegalArgumentException if the surface has been released (or is invalid), 1841 * or the format is unacceptable (e.g. missing a mandatory key), 1842 * or the flags are not set properly 1843 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 1844 * @throws IllegalStateException if not in the Uninitialized state. 1845 * @throws CryptoException upon DRM error. 1846 * @throws CodecException upon codec error. 1847 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @ConfigureFlag int flags)1848 public void configure( 1849 @Nullable MediaFormat format, 1850 @Nullable Surface surface, @Nullable MediaCrypto crypto, 1851 @ConfigureFlag int flags) { 1852 String[] keys = null; 1853 Object[] values = null; 1854 1855 if (format != null) { 1856 Map<String, Object> formatMap = format.getMap(); 1857 keys = new String[formatMap.size()]; 1858 values = new Object[formatMap.size()]; 1859 1860 int i = 0; 1861 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 1862 if (entry.getKey().equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 1863 int sessionId = 0; 1864 try { 1865 sessionId = (Integer)entry.getValue(); 1866 } 1867 catch (Exception e) { 1868 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 1869 } 1870 keys[i] = "audio-hw-sync"; 1871 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 1872 } else { 1873 keys[i] = entry.getKey(); 1874 values[i] = entry.getValue(); 1875 } 1876 ++i; 1877 } 1878 } 1879 1880 mHasSurface = surface != null; 1881 1882 native_configure(keys, values, surface, crypto, flags); 1883 } 1884 1885 /** 1886 * Dynamically sets the output surface of a codec. 1887 * <p> 1888 * This can only be used if the codec was configured with an output surface. The 1889 * new output surface should have a compatible usage type to the original output surface. 1890 * E.g. codecs may not support switching from a SurfaceTexture (GPU readable) output 1891 * to ImageReader (software readable) output. 1892 * @param surface the output surface to use. It must not be {@code null}. 1893 * @throws IllegalStateException if the codec does not support setting the output 1894 * surface in the current state. 1895 * @throws IllegalArgumentException if the new surface is not of a suitable type for the codec. 1896 */ setOutputSurface(@onNull Surface surface)1897 public void setOutputSurface(@NonNull Surface surface) { 1898 if (!mHasSurface) { 1899 throw new IllegalStateException("codec was not configured for an output surface"); 1900 } 1901 native_setSurface(surface); 1902 } 1903 native_setSurface(@onNull Surface surface)1904 private native void native_setSurface(@NonNull Surface surface); 1905 1906 /** 1907 * Create a persistent input surface that can be used with codecs that normally have an input 1908 * surface, such as video encoders. A persistent input can be reused by subsequent 1909 * {@link MediaCodec} or {@link MediaRecorder} instances, but can only be used by at 1910 * most one codec or recorder instance concurrently. 1911 * <p> 1912 * The application is responsible for calling release() on the Surface when done. 1913 * 1914 * @return an input surface that can be used with {@link #setInputSurface}. 1915 */ 1916 @NonNull createPersistentInputSurface()1917 public static Surface createPersistentInputSurface() { 1918 return native_createPersistentInputSurface(); 1919 } 1920 1921 static class PersistentSurface extends Surface { 1922 @SuppressWarnings("unused") PersistentSurface()1923 PersistentSurface() {} // used by native 1924 1925 @Override release()1926 public void release() { 1927 native_releasePersistentInputSurface(this); 1928 super.release(); 1929 } 1930 1931 private long mPersistentObject; 1932 }; 1933 1934 /** 1935 * Configures the codec (e.g. encoder) to use a persistent input surface in place of input 1936 * buffers. This may only be called after {@link #configure} and before {@link #start}, in 1937 * lieu of {@link #createInputSurface}. 1938 * @param surface a persistent input surface created by {@link #createPersistentInputSurface} 1939 * @throws IllegalStateException if not in the Configured state or does not require an input 1940 * surface. 1941 * @throws IllegalArgumentException if the surface was not created by 1942 * {@link #createPersistentInputSurface}. 1943 */ setInputSurface(@onNull Surface surface)1944 public void setInputSurface(@NonNull Surface surface) { 1945 if (!(surface instanceof PersistentSurface)) { 1946 throw new IllegalArgumentException("not a PersistentSurface"); 1947 } 1948 native_setInputSurface(surface); 1949 } 1950 1951 @NonNull native_createPersistentInputSurface()1952 private static native final PersistentSurface native_createPersistentInputSurface(); native_releasePersistentInputSurface(@onNull Surface surface)1953 private static native final void native_releasePersistentInputSurface(@NonNull Surface surface); native_setInputSurface(@onNull Surface surface)1954 private native final void native_setInputSurface(@NonNull Surface surface); 1955 native_setCallback(@ullable Callback cb)1956 private native final void native_setCallback(@Nullable Callback cb); 1957 native_configure( @ullable String[] keys, @Nullable Object[] values, @Nullable Surface surface, @Nullable MediaCrypto crypto, @ConfigureFlag int flags)1958 private native final void native_configure( 1959 @Nullable String[] keys, @Nullable Object[] values, 1960 @Nullable Surface surface, @Nullable MediaCrypto crypto, @ConfigureFlag int flags); 1961 1962 /** 1963 * Requests a Surface to use as the input to an encoder, in place of input buffers. This 1964 * may only be called after {@link #configure} and before {@link #start}. 1965 * <p> 1966 * The application is responsible for calling release() on the Surface when 1967 * done. 1968 * <p> 1969 * The Surface must be rendered with a hardware-accelerated API, such as OpenGL ES. 1970 * {@link android.view.Surface#lockCanvas(android.graphics.Rect)} may fail or produce 1971 * unexpected results. 1972 * @throws IllegalStateException if not in the Configured state. 1973 */ 1974 @NonNull createInputSurface()1975 public native final Surface createInputSurface(); 1976 1977 /** 1978 * After successfully configuring the component, call {@code start}. 1979 * <p> 1980 * Call {@code start} also if the codec is configured in asynchronous mode, 1981 * and it has just been flushed, to resume requesting input buffers. 1982 * @throws IllegalStateException if not in the Configured state 1983 * or just after {@link #flush} for a codec that is configured 1984 * in asynchronous mode. 1985 * @throws MediaCodec.CodecException upon codec error. Note that some codec errors 1986 * for start may be attributed to future method calls. 1987 */ start()1988 public final void start() { 1989 native_start(); 1990 synchronized(mBufferLock) { 1991 cacheBuffers(true /* input */); 1992 cacheBuffers(false /* input */); 1993 } 1994 } native_start()1995 private native final void native_start(); 1996 1997 /** 1998 * Finish the decode/encode session, note that the codec instance 1999 * remains active and ready to be {@link #start}ed again. 2000 * To ensure that it is available to other client call {@link #release} 2001 * and don't just rely on garbage collection to eventually do this for you. 2002 * @throws IllegalStateException if in the Released state. 2003 */ stop()2004 public final void stop() { 2005 native_stop(); 2006 freeAllTrackedBuffers(); 2007 2008 synchronized (mListenerLock) { 2009 if (mCallbackHandler != null) { 2010 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 2011 mCallbackHandler.removeMessages(EVENT_CALLBACK); 2012 } 2013 if (mOnFrameRenderedHandler != null) { 2014 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 2015 } 2016 } 2017 } 2018 native_stop()2019 private native final void native_stop(); 2020 2021 /** 2022 * Flush both input and output ports of the component. 2023 * <p> 2024 * Upon return, all indices previously returned in calls to {@link #dequeueInputBuffer 2025 * dequeueInputBuffer} and {@link #dequeueOutputBuffer dequeueOutputBuffer} — or obtained 2026 * via {@link Callback#onInputBufferAvailable onInputBufferAvailable} or 2027 * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} callbacks — become 2028 * invalid, and all buffers are owned by the codec. 2029 * <p> 2030 * If the codec is configured in asynchronous mode, call {@link #start} 2031 * after {@code flush} has returned to resume codec operations. The codec 2032 * will not request input buffers until this has happened. 2033 * <strong>Note, however, that there may still be outstanding {@code onOutputBufferAvailable} 2034 * callbacks that were not handled prior to calling {@code flush}. 2035 * The indices returned via these callbacks also become invalid upon calling {@code flush} and 2036 * should be discarded.</strong> 2037 * <p> 2038 * If the codec is configured in synchronous mode, codec will resume 2039 * automatically if it is configured with an input surface. Otherwise, it 2040 * will resume when {@link #dequeueInputBuffer dequeueInputBuffer} is called. 2041 * 2042 * @throws IllegalStateException if not in the Executing state. 2043 * @throws MediaCodec.CodecException upon codec error. 2044 */ flush()2045 public final void flush() { 2046 synchronized(mBufferLock) { 2047 invalidateByteBuffers(mCachedInputBuffers); 2048 invalidateByteBuffers(mCachedOutputBuffers); 2049 mDequeuedInputBuffers.clear(); 2050 mDequeuedOutputBuffers.clear(); 2051 } 2052 native_flush(); 2053 } 2054 native_flush()2055 private native final void native_flush(); 2056 2057 /** 2058 * Thrown when an internal codec error occurs. 2059 */ 2060 public final static class CodecException extends IllegalStateException { CodecException(int errorCode, int actionCode, @Nullable String detailMessage)2061 CodecException(int errorCode, int actionCode, @Nullable String detailMessage) { 2062 super(detailMessage); 2063 mErrorCode = errorCode; 2064 mActionCode = actionCode; 2065 2066 // TODO get this from codec 2067 final String sign = errorCode < 0 ? "neg_" : ""; 2068 mDiagnosticInfo = 2069 "android.media.MediaCodec.error_" + sign + Math.abs(errorCode); 2070 } 2071 2072 /** 2073 * Returns true if the codec exception is a transient issue, 2074 * perhaps due to resource constraints, and that the method 2075 * (or encoding/decoding) may be retried at a later time. 2076 */ 2077 public boolean isTransient() { 2078 return mActionCode == ACTION_TRANSIENT; 2079 } 2080 2081 /** 2082 * Returns true if the codec cannot proceed further, 2083 * but can be recovered by stopping, configuring, 2084 * and starting again. 2085 */ 2086 public boolean isRecoverable() { 2087 return mActionCode == ACTION_RECOVERABLE; 2088 } 2089 2090 /** 2091 * Retrieve the error code associated with a CodecException 2092 */ 2093 public int getErrorCode() { 2094 return mErrorCode; 2095 } 2096 2097 /** 2098 * Retrieve a developer-readable diagnostic information string 2099 * associated with the exception. Do not show this to end-users, 2100 * since this string will not be localized or generally 2101 * comprehensible to end-users. 2102 */ 2103 public @NonNull String getDiagnosticInfo() { 2104 return mDiagnosticInfo; 2105 } 2106 2107 /** 2108 * This indicates required resource was not able to be allocated. 2109 */ 2110 public static final int ERROR_INSUFFICIENT_RESOURCE = 1100; 2111 2112 /** 2113 * This indicates the resource manager reclaimed the media resource used by the codec. 2114 * <p> 2115 * With this exception, the codec must be released, as it has moved to terminal state. 2116 */ 2117 public static final int ERROR_RECLAIMED = 1101; 2118 2119 /** @hide */ 2120 @IntDef({ 2121 ERROR_INSUFFICIENT_RESOURCE, 2122 ERROR_RECLAIMED, 2123 }) 2124 @Retention(RetentionPolicy.SOURCE) 2125 public @interface ReasonCode {} 2126 2127 /* Must be in sync with android_media_MediaCodec.cpp */ 2128 private final static int ACTION_TRANSIENT = 1; 2129 private final static int ACTION_RECOVERABLE = 2; 2130 2131 private final String mDiagnosticInfo; 2132 private final int mErrorCode; 2133 private final int mActionCode; 2134 } 2135 2136 /** 2137 * Thrown when a crypto error occurs while queueing a secure input buffer. 2138 */ 2139 public final static class CryptoException extends RuntimeException { 2140 public CryptoException(int errorCode, @Nullable String detailMessage) { 2141 super(detailMessage); 2142 mErrorCode = errorCode; 2143 } 2144 2145 /** 2146 * This indicates that the requested key was not found when trying to 2147 * perform a decrypt operation. The operation can be retried after adding 2148 * the correct decryption key. 2149 */ 2150 public static final int ERROR_NO_KEY = 1; 2151 2152 /** 2153 * This indicates that the key used for decryption is no longer 2154 * valid due to license term expiration. The operation can be retried 2155 * after updating the expired keys. 2156 */ 2157 public static final int ERROR_KEY_EXPIRED = 2; 2158 2159 /** 2160 * This indicates that a required crypto resource was not able to be 2161 * allocated while attempting the requested operation. The operation 2162 * can be retried if the app is able to release resources. 2163 */ 2164 public static final int ERROR_RESOURCE_BUSY = 3; 2165 2166 /** 2167 * This indicates that the output protection levels supported by the 2168 * device are not sufficient to meet the requirements set by the 2169 * content owner in the license policy. 2170 */ 2171 public static final int ERROR_INSUFFICIENT_OUTPUT_PROTECTION = 4; 2172 2173 /** 2174 * This indicates that decryption was attempted on a session that is 2175 * not opened, which could be due to a failure to open the session, 2176 * closing the session prematurely, or the session being reclaimed 2177 * by the resource manager. 2178 */ 2179 public static final int ERROR_SESSION_NOT_OPENED = 5; 2180 2181 /** 2182 * This indicates that an operation was attempted that could not be 2183 * supported by the crypto system of the device in its current 2184 * configuration. It may occur when the license policy requires 2185 * device security features that aren't supported by the device, 2186 * or due to an internal error in the crypto system that prevents 2187 * the specified security policy from being met. 2188 */ 2189 public static final int ERROR_UNSUPPORTED_OPERATION = 6; 2190 2191 /** @hide */ 2192 @IntDef({ 2193 ERROR_NO_KEY, 2194 ERROR_KEY_EXPIRED, 2195 ERROR_RESOURCE_BUSY, 2196 ERROR_INSUFFICIENT_OUTPUT_PROTECTION, 2197 ERROR_SESSION_NOT_OPENED, 2198 ERROR_UNSUPPORTED_OPERATION 2199 }) 2200 @Retention(RetentionPolicy.SOURCE) 2201 public @interface CryptoErrorCode {} 2202 2203 /** 2204 * Retrieve the error code associated with a CryptoException 2205 */ 2206 @CryptoErrorCode 2207 public int getErrorCode() { 2208 return mErrorCode; 2209 } 2210 2211 private int mErrorCode; 2212 } 2213 2214 /** 2215 * After filling a range of the input buffer at the specified index 2216 * submit it to the component. Once an input buffer is queued to 2217 * the codec, it MUST NOT be used until it is later retrieved by 2218 * {@link #getInputBuffer} in response to a {@link #dequeueInputBuffer} 2219 * return value or a {@link Callback#onInputBufferAvailable} 2220 * callback. 2221 * <p> 2222 * Many decoders require the actual compressed data stream to be 2223 * preceded by "codec specific data", i.e. setup data used to initialize 2224 * the codec such as PPS/SPS in the case of AVC video or code tables 2225 * in the case of vorbis audio. 2226 * The class {@link android.media.MediaExtractor} provides codec 2227 * specific data as part of 2228 * the returned track format in entries named "csd-0", "csd-1" ... 2229 * <p> 2230 * These buffers can be submitted directly after {@link #start} or 2231 * {@link #flush} by specifying the flag {@link 2232 * #BUFFER_FLAG_CODEC_CONFIG}. However, if you configure the 2233 * codec with a {@link MediaFormat} containing these keys, they 2234 * will be automatically submitted by MediaCodec directly after 2235 * start. Therefore, the use of {@link 2236 * #BUFFER_FLAG_CODEC_CONFIG} flag is discouraged and is 2237 * recommended only for advanced users. 2238 * <p> 2239 * To indicate that this is the final piece of input data (or rather that 2240 * no more input data follows unless the decoder is subsequently flushed) 2241 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 2242 * <p class=note> 2243 * <strong>Note:</strong> Prior to {@link android.os.Build.VERSION_CODES#M}, 2244 * {@code presentationTimeUs} was not propagated to the frame timestamp of (rendered) 2245 * Surface output buffers, and the resulting frame timestamp was undefined. 2246 * Use {@link #releaseOutputBuffer(int, long)} to ensure a specific frame timestamp is set. 2247 * Similarly, since frame timestamps can be used by the destination surface for rendering 2248 * synchronization, <strong>care must be taken to normalize presentationTimeUs so as to not be 2249 * mistaken for a system time. (See {@linkplain #releaseOutputBuffer(int, long) 2250 * SurfaceView specifics}).</strong> 2251 * 2252 * @param index The index of a client-owned input buffer previously returned 2253 * in a call to {@link #dequeueInputBuffer}. 2254 * @param offset The byte offset into the input buffer at which the data starts. 2255 * @param size The number of bytes of valid input data. 2256 * @param presentationTimeUs The presentation timestamp in microseconds for this 2257 * buffer. This is normally the media time at which this 2258 * buffer should be presented (rendered). When using an output 2259 * surface, this will be propagated as the {@link 2260 * SurfaceTexture#getTimestamp timestamp} for the frame (after 2261 * conversion to nanoseconds). 2262 * @param flags A bitmask of flags 2263 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2264 * While not prohibited, most codecs do not use the 2265 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2266 * @throws IllegalStateException if not in the Executing state. 2267 * @throws MediaCodec.CodecException upon codec error. 2268 * @throws CryptoException if a crypto object has been specified in 2269 * {@link #configure} 2270 */ 2271 public final void queueInputBuffer( 2272 int index, 2273 int offset, int size, long presentationTimeUs, int flags) 2274 throws CryptoException { 2275 synchronized(mBufferLock) { 2276 invalidateByteBuffer(mCachedInputBuffers, index); 2277 mDequeuedInputBuffers.remove(index); 2278 } 2279 try { 2280 native_queueInputBuffer( 2281 index, offset, size, presentationTimeUs, flags); 2282 } catch (CryptoException | IllegalStateException e) { 2283 revalidateByteBuffer(mCachedInputBuffers, index); 2284 throw e; 2285 } 2286 } 2287 2288 private native final void native_queueInputBuffer( 2289 int index, 2290 int offset, int size, long presentationTimeUs, int flags) 2291 throws CryptoException; 2292 2293 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 2294 public static final int CRYPTO_MODE_AES_CTR = 1; 2295 public static final int CRYPTO_MODE_AES_CBC = 2; 2296 2297 /** 2298 * Metadata describing the structure of a (at least partially) encrypted 2299 * input sample. 2300 * A buffer's data is considered to be partitioned into "subSamples", 2301 * each subSample starts with a (potentially empty) run of plain, 2302 * unencrypted bytes followed by a (also potentially empty) run of 2303 * encrypted bytes. If pattern encryption applies, each of the latter runs 2304 * is encrypted only partly, according to a repeating pattern of "encrypt" 2305 * and "skip" blocks. numBytesOfClearData can be null to indicate that all 2306 * data is encrypted. This information encapsulates per-sample metadata as 2307 * outlined in ISO/IEC FDIS 23001-7:2011 "Common encryption in ISO base 2308 * media file format files". 2309 */ 2310 public final static class CryptoInfo { 2311 /** 2312 * The number of subSamples that make up the buffer's contents. 2313 */ 2314 public int numSubSamples; 2315 /** 2316 * The number of leading unencrypted bytes in each subSample. 2317 */ 2318 public int[] numBytesOfClearData; 2319 /** 2320 * The number of trailing encrypted bytes in each subSample. 2321 */ 2322 public int[] numBytesOfEncryptedData; 2323 /** 2324 * A 16-byte key id 2325 */ 2326 public byte[] key; 2327 /** 2328 * A 16-byte initialization vector 2329 */ 2330 public byte[] iv; 2331 /** 2332 * The type of encryption that has been applied, 2333 * see {@link #CRYPTO_MODE_UNENCRYPTED}, {@link #CRYPTO_MODE_AES_CTR} 2334 * and {@link #CRYPTO_MODE_AES_CBC} 2335 */ 2336 public int mode; 2337 2338 /** 2339 * Metadata describing an encryption pattern for the protected bytes in 2340 * a subsample. An encryption pattern consists of a repeating sequence 2341 * of crypto blocks comprised of a number of encrypted blocks followed 2342 * by a number of unencrypted, or skipped, blocks. 2343 */ 2344 public final static class Pattern { 2345 /** 2346 * Number of blocks to be encrypted in the pattern. If zero, pattern 2347 * encryption is inoperative. 2348 */ 2349 private int mEncryptBlocks; 2350 2351 /** 2352 * Number of blocks to be skipped (left clear) in the pattern. If zero, 2353 * pattern encryption is inoperative. 2354 */ 2355 private int mSkipBlocks; 2356 2357 /** 2358 * Construct a sample encryption pattern given the number of blocks to 2359 * encrypt and skip in the pattern. 2360 */ 2361 public Pattern(int blocksToEncrypt, int blocksToSkip) { 2362 set(blocksToEncrypt, blocksToSkip); 2363 } 2364 2365 /** 2366 * Set the number of blocks to encrypt and skip in a sample encryption 2367 * pattern. 2368 */ 2369 public void set(int blocksToEncrypt, int blocksToSkip) { 2370 mEncryptBlocks = blocksToEncrypt; 2371 mSkipBlocks = blocksToSkip; 2372 } 2373 2374 /** 2375 * Return the number of blocks to skip in a sample encryption pattern. 2376 */ 2377 public int getSkipBlocks() { 2378 return mSkipBlocks; 2379 } 2380 2381 /** 2382 * Return the number of blocks to encrypt in a sample encryption pattern. 2383 */ 2384 public int getEncryptBlocks() { 2385 return mEncryptBlocks; 2386 } 2387 }; 2388 2389 /** 2390 * The pattern applicable to the protected data in each subsample. 2391 */ 2392 private Pattern pattern; 2393 2394 /** 2395 * Set the subsample count, clear/encrypted sizes, key, IV and mode fields of 2396 * a {@link MediaCodec.CryptoInfo} instance. 2397 */ 2398 public void set( 2399 int newNumSubSamples, 2400 @NonNull int[] newNumBytesOfClearData, 2401 @NonNull int[] newNumBytesOfEncryptedData, 2402 @NonNull byte[] newKey, 2403 @NonNull byte[] newIV, 2404 int newMode) { 2405 numSubSamples = newNumSubSamples; 2406 numBytesOfClearData = newNumBytesOfClearData; 2407 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 2408 key = newKey; 2409 iv = newIV; 2410 mode = newMode; 2411 pattern = new Pattern(0, 0); 2412 } 2413 2414 /** 2415 * Set the encryption pattern on a {@link MediaCodec.CryptoInfo} instance. 2416 * See {@link MediaCodec.CryptoInfo.Pattern}. 2417 */ 2418 public void setPattern(Pattern newPattern) { 2419 pattern = newPattern; 2420 } 2421 2422 @Override 2423 public String toString() { 2424 StringBuilder builder = new StringBuilder(); 2425 builder.append(numSubSamples + " subsamples, key ["); 2426 String hexdigits = "0123456789abcdef"; 2427 for (int i = 0; i < key.length; i++) { 2428 builder.append(hexdigits.charAt((key[i] & 0xf0) >> 4)); 2429 builder.append(hexdigits.charAt(key[i] & 0x0f)); 2430 } 2431 builder.append("], iv ["); 2432 for (int i = 0; i < key.length; i++) { 2433 builder.append(hexdigits.charAt((iv[i] & 0xf0) >> 4)); 2434 builder.append(hexdigits.charAt(iv[i] & 0x0f)); 2435 } 2436 builder.append("], clear "); Arrays.toString(numBytesOfClearData)2437 builder.append(Arrays.toString(numBytesOfClearData)); 2438 builder.append(", encrypted "); Arrays.toString(numBytesOfEncryptedData)2439 builder.append(Arrays.toString(numBytesOfEncryptedData)); 2440 return builder.toString(); 2441 } 2442 }; 2443 2444 /** 2445 * Similar to {@link #queueInputBuffer queueInputBuffer} but submits a buffer that is 2446 * potentially encrypted. 2447 * <strong>Check out further notes at {@link #queueInputBuffer queueInputBuffer}.</strong> 2448 * 2449 * @param index The index of a client-owned input buffer previously returned 2450 * in a call to {@link #dequeueInputBuffer}. 2451 * @param offset The byte offset into the input buffer at which the data starts. 2452 * @param info Metadata required to facilitate decryption, the object can be 2453 * reused immediately after this call returns. 2454 * @param presentationTimeUs The presentation timestamp in microseconds for this 2455 * buffer. This is normally the media time at which this 2456 * buffer should be presented (rendered). 2457 * @param flags A bitmask of flags 2458 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2459 * While not prohibited, most codecs do not use the 2460 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2461 * @throws IllegalStateException if not in the Executing state. 2462 * @throws MediaCodec.CodecException upon codec error. 2463 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 2464 * An error code associated with the exception helps identify the 2465 * reason for the failure. 2466 */ queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2467 public final void queueSecureInputBuffer( 2468 int index, 2469 int offset, 2470 @NonNull CryptoInfo info, 2471 long presentationTimeUs, 2472 int flags) throws CryptoException { 2473 synchronized(mBufferLock) { 2474 invalidateByteBuffer(mCachedInputBuffers, index); 2475 mDequeuedInputBuffers.remove(index); 2476 } 2477 try { 2478 native_queueSecureInputBuffer( 2479 index, offset, info, presentationTimeUs, flags); 2480 } catch (CryptoException | IllegalStateException e) { 2481 revalidateByteBuffer(mCachedInputBuffers, index); 2482 throw e; 2483 } 2484 } 2485 native_queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2486 private native final void native_queueSecureInputBuffer( 2487 int index, 2488 int offset, 2489 @NonNull CryptoInfo info, 2490 long presentationTimeUs, 2491 int flags) throws CryptoException; 2492 2493 /** 2494 * Returns the index of an input buffer to be filled with valid data 2495 * or -1 if no such buffer is currently available. 2496 * This method will return immediately if timeoutUs == 0, wait indefinitely 2497 * for the availability of an input buffer if timeoutUs < 0 or wait up 2498 * to "timeoutUs" microseconds if timeoutUs > 0. 2499 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 2500 * @throws IllegalStateException if not in the Executing state, 2501 * or codec is configured in asynchronous mode. 2502 * @throws MediaCodec.CodecException upon codec error. 2503 */ dequeueInputBuffer(long timeoutUs)2504 public final int dequeueInputBuffer(long timeoutUs) { 2505 int res = native_dequeueInputBuffer(timeoutUs); 2506 if (res >= 0) { 2507 synchronized(mBufferLock) { 2508 validateInputByteBuffer(mCachedInputBuffers, res); 2509 } 2510 } 2511 return res; 2512 } 2513 native_dequeueInputBuffer(long timeoutUs)2514 private native final int native_dequeueInputBuffer(long timeoutUs); 2515 2516 /** 2517 * If a non-negative timeout had been specified in the call 2518 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 2519 */ 2520 public static final int INFO_TRY_AGAIN_LATER = -1; 2521 2522 /** 2523 * The output format has changed, subsequent data will follow the new 2524 * format. {@link #getOutputFormat()} returns the new format. Note, that 2525 * you can also use the new {@link #getOutputFormat(int)} method to 2526 * get the format for a specific output buffer. This frees you from 2527 * having to track output format changes. 2528 */ 2529 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 2530 2531 /** 2532 * The output buffers have changed, the client must refer to the new 2533 * set of output buffers returned by {@link #getOutputBuffers} from 2534 * this point on. 2535 * 2536 * <p>Additionally, this event signals that the video scaling mode 2537 * may have been reset to the default.</p> 2538 * 2539 * @deprecated This return value can be ignored as {@link 2540 * #getOutputBuffers} has been deprecated. Client should 2541 * request a current buffer using on of the get-buffer or 2542 * get-image methods each time one has been dequeued. 2543 */ 2544 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 2545 2546 /** @hide */ 2547 @IntDef({ 2548 INFO_TRY_AGAIN_LATER, 2549 INFO_OUTPUT_FORMAT_CHANGED, 2550 INFO_OUTPUT_BUFFERS_CHANGED, 2551 }) 2552 @Retention(RetentionPolicy.SOURCE) 2553 public @interface OutputBufferInfo {} 2554 2555 /** 2556 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 2557 * Returns the index of an output buffer that has been successfully 2558 * decoded or one of the INFO_* constants. 2559 * @param info Will be filled with buffer meta data. 2560 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 2561 * @throws IllegalStateException if not in the Executing state, 2562 * or codec is configured in asynchronous mode. 2563 * @throws MediaCodec.CodecException upon codec error. 2564 */ 2565 @OutputBufferInfo dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)2566 public final int dequeueOutputBuffer( 2567 @NonNull BufferInfo info, long timeoutUs) { 2568 int res = native_dequeueOutputBuffer(info, timeoutUs); 2569 synchronized(mBufferLock) { 2570 if (res == INFO_OUTPUT_BUFFERS_CHANGED) { 2571 cacheBuffers(false /* input */); 2572 } else if (res >= 0) { 2573 validateOutputByteBuffer(mCachedOutputBuffers, res, info); 2574 if (mHasSurface) { 2575 mDequeuedOutputInfos.put(res, info.dup()); 2576 } 2577 } 2578 } 2579 return res; 2580 } 2581 native_dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)2582 private native final int native_dequeueOutputBuffer( 2583 @NonNull BufferInfo info, long timeoutUs); 2584 2585 /** 2586 * If you are done with a buffer, use this call to return the buffer to the codec 2587 * or to render it on the output surface. If you configured the codec with an 2588 * output surface, setting {@code render} to {@code true} will first send the buffer 2589 * to that output surface. The surface will release the buffer back to the codec once 2590 * it is no longer used/displayed. 2591 * 2592 * Once an output buffer is released to the codec, it MUST NOT 2593 * be used until it is later retrieved by {@link #getOutputBuffer} in response 2594 * to a {@link #dequeueOutputBuffer} return value or a 2595 * {@link Callback#onOutputBufferAvailable} callback. 2596 * 2597 * @param index The index of a client-owned output buffer previously returned 2598 * from a call to {@link #dequeueOutputBuffer}. 2599 * @param render If a valid surface was specified when configuring the codec, 2600 * passing true renders this output buffer to the surface. 2601 * @throws IllegalStateException if not in the Executing state. 2602 * @throws MediaCodec.CodecException upon codec error. 2603 */ releaseOutputBuffer(int index, boolean render)2604 public final void releaseOutputBuffer(int index, boolean render) { 2605 BufferInfo info = null; 2606 synchronized(mBufferLock) { 2607 invalidateByteBuffer(mCachedOutputBuffers, index); 2608 mDequeuedOutputBuffers.remove(index); 2609 if (mHasSurface) { 2610 info = mDequeuedOutputInfos.remove(index); 2611 } 2612 } 2613 releaseOutputBuffer(index, render, false /* updatePTS */, 0 /* dummy */); 2614 } 2615 2616 /** 2617 * If you are done with a buffer, use this call to update its surface timestamp 2618 * and return it to the codec to render it on the output surface. If you 2619 * have not specified an output surface when configuring this video codec, 2620 * this call will simply return the buffer to the codec.<p> 2621 * 2622 * The timestamp may have special meaning depending on the destination surface. 2623 * 2624 * <table> 2625 * <tr><th>SurfaceView specifics</th></tr> 2626 * <tr><td> 2627 * If you render your buffer on a {@link android.view.SurfaceView}, 2628 * you can use the timestamp to render the buffer at a specific time (at the 2629 * VSYNC at or after the buffer timestamp). For this to work, the timestamp 2630 * needs to be <i>reasonably close</i> to the current {@link System#nanoTime}. 2631 * Currently, this is set as within one (1) second. A few notes: 2632 * 2633 * <ul> 2634 * <li>the buffer will not be returned to the codec until the timestamp 2635 * has passed and the buffer is no longer used by the {@link android.view.Surface}. 2636 * <li>buffers are processed sequentially, so you may block subsequent buffers to 2637 * be displayed on the {@link android.view.Surface}. This is important if you 2638 * want to react to user action, e.g. stop the video or seek. 2639 * <li>if multiple buffers are sent to the {@link android.view.Surface} to be 2640 * rendered at the same VSYNC, the last one will be shown, and the other ones 2641 * will be dropped. 2642 * <li>if the timestamp is <em>not</em> "reasonably close" to the current system 2643 * time, the {@link android.view.Surface} will ignore the timestamp, and 2644 * display the buffer at the earliest feasible time. In this mode it will not 2645 * drop frames. 2646 * <li>for best performance and quality, call this method when you are about 2647 * two VSYNCs' time before the desired render time. For 60Hz displays, this is 2648 * about 33 msec. 2649 * </ul> 2650 * </td></tr> 2651 * </table> 2652 * 2653 * Once an output buffer is released to the codec, it MUST NOT 2654 * be used until it is later retrieved by {@link #getOutputBuffer} in response 2655 * to a {@link #dequeueOutputBuffer} return value or a 2656 * {@link Callback#onOutputBufferAvailable} callback. 2657 * 2658 * @param index The index of a client-owned output buffer previously returned 2659 * from a call to {@link #dequeueOutputBuffer}. 2660 * @param renderTimestampNs The timestamp to associate with this buffer when 2661 * it is sent to the Surface. 2662 * @throws IllegalStateException if not in the Executing state. 2663 * @throws MediaCodec.CodecException upon codec error. 2664 */ releaseOutputBuffer(int index, long renderTimestampNs)2665 public final void releaseOutputBuffer(int index, long renderTimestampNs) { 2666 BufferInfo info = null; 2667 synchronized(mBufferLock) { 2668 invalidateByteBuffer(mCachedOutputBuffers, index); 2669 mDequeuedOutputBuffers.remove(index); 2670 if (mHasSurface) { 2671 info = mDequeuedOutputInfos.remove(index); 2672 } 2673 } 2674 releaseOutputBuffer( 2675 index, true /* render */, true /* updatePTS */, renderTimestampNs); 2676 } 2677 releaseOutputBuffer( int index, boolean render, boolean updatePTS, long timeNs)2678 private native final void releaseOutputBuffer( 2679 int index, boolean render, boolean updatePTS, long timeNs); 2680 2681 /** 2682 * Signals end-of-stream on input. Equivalent to submitting an empty buffer with 2683 * {@link #BUFFER_FLAG_END_OF_STREAM} set. This may only be used with 2684 * encoders receiving input from a Surface created by {@link #createInputSurface}. 2685 * @throws IllegalStateException if not in the Executing state. 2686 * @throws MediaCodec.CodecException upon codec error. 2687 */ signalEndOfInputStream()2688 public native final void signalEndOfInputStream(); 2689 2690 /** 2691 * Call this after dequeueOutputBuffer signals a format change by returning 2692 * {@link #INFO_OUTPUT_FORMAT_CHANGED}. 2693 * You can also call this after {@link #configure} returns 2694 * successfully to get the output format initially configured 2695 * for the codec. Do this to determine what optional 2696 * configuration parameters were supported by the codec. 2697 * 2698 * @throws IllegalStateException if not in the Executing or 2699 * Configured state. 2700 * @throws MediaCodec.CodecException upon codec error. 2701 */ 2702 @NonNull getOutputFormat()2703 public final MediaFormat getOutputFormat() { 2704 return new MediaFormat(getFormatNative(false /* input */)); 2705 } 2706 2707 /** 2708 * Call this after {@link #configure} returns successfully to 2709 * get the input format accepted by the codec. Do this to 2710 * determine what optional configuration parameters were 2711 * supported by the codec. 2712 * 2713 * @throws IllegalStateException if not in the Executing or 2714 * Configured state. 2715 * @throws MediaCodec.CodecException upon codec error. 2716 */ 2717 @NonNull getInputFormat()2718 public final MediaFormat getInputFormat() { 2719 return new MediaFormat(getFormatNative(true /* input */)); 2720 } 2721 2722 /** 2723 * Returns the output format for a specific output buffer. 2724 * 2725 * @param index The index of a client-owned input buffer previously 2726 * returned from a call to {@link #dequeueInputBuffer}. 2727 * 2728 * @return the format for the output buffer, or null if the index 2729 * is not a dequeued output buffer. 2730 */ 2731 @NonNull getOutputFormat(int index)2732 public final MediaFormat getOutputFormat(int index) { 2733 return new MediaFormat(getOutputFormatNative(index)); 2734 } 2735 2736 @NonNull getFormatNative(boolean input)2737 private native final Map<String, Object> getFormatNative(boolean input); 2738 2739 @NonNull getOutputFormatNative(int index)2740 private native final Map<String, Object> getOutputFormatNative(int index); 2741 2742 // used to track dequeued buffers 2743 private static class BufferMap { 2744 // various returned representations of the codec buffer 2745 private static class CodecBuffer { 2746 private Image mImage; 2747 private ByteBuffer mByteBuffer; 2748 free()2749 public void free() { 2750 if (mByteBuffer != null) { 2751 // all of our ByteBuffers are direct 2752 java.nio.NioUtils.freeDirectBuffer(mByteBuffer); 2753 mByteBuffer = null; 2754 } 2755 if (mImage != null) { 2756 mImage.close(); 2757 mImage = null; 2758 } 2759 } 2760 setImage(@ullable Image image)2761 public void setImage(@Nullable Image image) { 2762 free(); 2763 mImage = image; 2764 } 2765 setByteBuffer(@ullable ByteBuffer buffer)2766 public void setByteBuffer(@Nullable ByteBuffer buffer) { 2767 free(); 2768 mByteBuffer = buffer; 2769 } 2770 } 2771 2772 private final Map<Integer, CodecBuffer> mMap = 2773 new HashMap<Integer, CodecBuffer>(); 2774 remove(int index)2775 public void remove(int index) { 2776 CodecBuffer buffer = mMap.get(index); 2777 if (buffer != null) { 2778 buffer.free(); 2779 mMap.remove(index); 2780 } 2781 } 2782 put(int index, @Nullable ByteBuffer newBuffer)2783 public void put(int index, @Nullable ByteBuffer newBuffer) { 2784 CodecBuffer buffer = mMap.get(index); 2785 if (buffer == null) { // likely 2786 buffer = new CodecBuffer(); 2787 mMap.put(index, buffer); 2788 } 2789 buffer.setByteBuffer(newBuffer); 2790 } 2791 put(int index, @Nullable Image newImage)2792 public void put(int index, @Nullable Image newImage) { 2793 CodecBuffer buffer = mMap.get(index); 2794 if (buffer == null) { // likely 2795 buffer = new CodecBuffer(); 2796 mMap.put(index, buffer); 2797 } 2798 buffer.setImage(newImage); 2799 } 2800 clear()2801 public void clear() { 2802 for (CodecBuffer buffer: mMap.values()) { 2803 buffer.free(); 2804 } 2805 mMap.clear(); 2806 } 2807 } 2808 2809 private ByteBuffer[] mCachedInputBuffers; 2810 private ByteBuffer[] mCachedOutputBuffers; 2811 private final BufferMap mDequeuedInputBuffers = new BufferMap(); 2812 private final BufferMap mDequeuedOutputBuffers = new BufferMap(); 2813 private final Map<Integer, BufferInfo> mDequeuedOutputInfos = 2814 new HashMap<Integer, BufferInfo>(); 2815 final private Object mBufferLock; 2816 invalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)2817 private final void invalidateByteBuffer( 2818 @Nullable ByteBuffer[] buffers, int index) { 2819 if (buffers != null && index >= 0 && index < buffers.length) { 2820 ByteBuffer buffer = buffers[index]; 2821 if (buffer != null) { 2822 buffer.setAccessible(false); 2823 } 2824 } 2825 } 2826 validateInputByteBuffer( @ullable ByteBuffer[] buffers, int index)2827 private final void validateInputByteBuffer( 2828 @Nullable ByteBuffer[] buffers, int index) { 2829 if (buffers != null && index >= 0 && index < buffers.length) { 2830 ByteBuffer buffer = buffers[index]; 2831 if (buffer != null) { 2832 buffer.setAccessible(true); 2833 buffer.clear(); 2834 } 2835 } 2836 } 2837 revalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)2838 private final void revalidateByteBuffer( 2839 @Nullable ByteBuffer[] buffers, int index) { 2840 synchronized(mBufferLock) { 2841 if (buffers != null && index >= 0 && index < buffers.length) { 2842 ByteBuffer buffer = buffers[index]; 2843 if (buffer != null) { 2844 buffer.setAccessible(true); 2845 } 2846 } 2847 } 2848 } 2849 validateOutputByteBuffer( @ullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info)2850 private final void validateOutputByteBuffer( 2851 @Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) { 2852 if (buffers != null && index >= 0 && index < buffers.length) { 2853 ByteBuffer buffer = buffers[index]; 2854 if (buffer != null) { 2855 buffer.setAccessible(true); 2856 buffer.limit(info.offset + info.size).position(info.offset); 2857 } 2858 } 2859 } 2860 invalidateByteBuffers(@ullable ByteBuffer[] buffers)2861 private final void invalidateByteBuffers(@Nullable ByteBuffer[] buffers) { 2862 if (buffers != null) { 2863 for (ByteBuffer buffer: buffers) { 2864 if (buffer != null) { 2865 buffer.setAccessible(false); 2866 } 2867 } 2868 } 2869 } 2870 freeByteBuffer(@ullable ByteBuffer buffer)2871 private final void freeByteBuffer(@Nullable ByteBuffer buffer) { 2872 if (buffer != null /* && buffer.isDirect() */) { 2873 // all of our ByteBuffers are direct 2874 java.nio.NioUtils.freeDirectBuffer(buffer); 2875 } 2876 } 2877 freeByteBuffers(@ullable ByteBuffer[] buffers)2878 private final void freeByteBuffers(@Nullable ByteBuffer[] buffers) { 2879 if (buffers != null) { 2880 for (ByteBuffer buffer: buffers) { 2881 freeByteBuffer(buffer); 2882 } 2883 } 2884 } 2885 freeAllTrackedBuffers()2886 private final void freeAllTrackedBuffers() { 2887 synchronized(mBufferLock) { 2888 freeByteBuffers(mCachedInputBuffers); 2889 freeByteBuffers(mCachedOutputBuffers); 2890 mCachedInputBuffers = null; 2891 mCachedOutputBuffers = null; 2892 mDequeuedInputBuffers.clear(); 2893 mDequeuedOutputBuffers.clear(); 2894 } 2895 } 2896 cacheBuffers(boolean input)2897 private final void cacheBuffers(boolean input) { 2898 ByteBuffer[] buffers = null; 2899 try { 2900 buffers = getBuffers(input); 2901 invalidateByteBuffers(buffers); 2902 } catch (IllegalStateException e) { 2903 // we don't get buffers in async mode 2904 } 2905 if (input) { 2906 mCachedInputBuffers = buffers; 2907 } else { 2908 mCachedOutputBuffers = buffers; 2909 } 2910 } 2911 2912 /** 2913 * Retrieve the set of input buffers. Call this after start() 2914 * returns. After calling this method, any ByteBuffers 2915 * previously returned by an earlier call to this method MUST no 2916 * longer be used. 2917 * 2918 * @deprecated Use the new {@link #getInputBuffer} method instead 2919 * each time an input buffer is dequeued. 2920 * 2921 * <b>Note:</b> As of API 21, dequeued input buffers are 2922 * automatically {@link java.nio.Buffer#clear cleared}. 2923 * 2924 * <em>Do not use this method if using an input surface.</em> 2925 * 2926 * @throws IllegalStateException if not in the Executing state, 2927 * or codec is configured in asynchronous mode. 2928 * @throws MediaCodec.CodecException upon codec error. 2929 */ 2930 @NonNull getInputBuffers()2931 public ByteBuffer[] getInputBuffers() { 2932 if (mCachedInputBuffers == null) { 2933 throw new IllegalStateException(); 2934 } 2935 // FIXME: check codec status 2936 return mCachedInputBuffers; 2937 } 2938 2939 /** 2940 * Retrieve the set of output buffers. Call this after start() 2941 * returns and whenever dequeueOutputBuffer signals an output 2942 * buffer change by returning {@link 2943 * #INFO_OUTPUT_BUFFERS_CHANGED}. After calling this method, any 2944 * ByteBuffers previously returned by an earlier call to this 2945 * method MUST no longer be used. 2946 * 2947 * @deprecated Use the new {@link #getOutputBuffer} method instead 2948 * each time an output buffer is dequeued. This method is not 2949 * supported if codec is configured in asynchronous mode. 2950 * 2951 * <b>Note:</b> As of API 21, the position and limit of output 2952 * buffers that are dequeued will be set to the valid data 2953 * range. 2954 * 2955 * <em>Do not use this method if using an output surface.</em> 2956 * 2957 * @throws IllegalStateException if not in the Executing state, 2958 * or codec is configured in asynchronous mode. 2959 * @throws MediaCodec.CodecException upon codec error. 2960 */ 2961 @NonNull getOutputBuffers()2962 public ByteBuffer[] getOutputBuffers() { 2963 if (mCachedOutputBuffers == null) { 2964 throw new IllegalStateException(); 2965 } 2966 // FIXME: check codec status 2967 return mCachedOutputBuffers; 2968 } 2969 2970 /** 2971 * Returns a {@link java.nio.Buffer#clear cleared}, writable ByteBuffer 2972 * object for a dequeued input buffer index to contain the input data. 2973 * 2974 * After calling this method any ByteBuffer or Image object 2975 * previously returned for the same input index MUST no longer 2976 * be used. 2977 * 2978 * @param index The index of a client-owned input buffer previously 2979 * returned from a call to {@link #dequeueInputBuffer}, 2980 * or received via an onInputBufferAvailable callback. 2981 * 2982 * @return the input buffer, or null if the index is not a dequeued 2983 * input buffer, or if the codec is configured for surface input. 2984 * 2985 * @throws IllegalStateException if not in the Executing state. 2986 * @throws MediaCodec.CodecException upon codec error. 2987 */ 2988 @Nullable getInputBuffer(int index)2989 public ByteBuffer getInputBuffer(int index) { 2990 ByteBuffer newBuffer = getBuffer(true /* input */, index); 2991 synchronized(mBufferLock) { 2992 invalidateByteBuffer(mCachedInputBuffers, index); 2993 mDequeuedInputBuffers.put(index, newBuffer); 2994 } 2995 return newBuffer; 2996 } 2997 2998 /** 2999 * Returns a writable Image object for a dequeued input buffer 3000 * index to contain the raw input video frame. 3001 * 3002 * After calling this method any ByteBuffer or Image object 3003 * previously returned for the same input index MUST no longer 3004 * be used. 3005 * 3006 * @param index The index of a client-owned input buffer previously 3007 * returned from a call to {@link #dequeueInputBuffer}, 3008 * or received via an onInputBufferAvailable callback. 3009 * 3010 * @return the input image, or null if the index is not a 3011 * dequeued input buffer, or not a ByteBuffer that contains a 3012 * raw image. 3013 * 3014 * @throws IllegalStateException if not in the Executing state. 3015 * @throws MediaCodec.CodecException upon codec error. 3016 */ 3017 @Nullable getInputImage(int index)3018 public Image getInputImage(int index) { 3019 Image newImage = getImage(true /* input */, index); 3020 synchronized(mBufferLock) { 3021 invalidateByteBuffer(mCachedInputBuffers, index); 3022 mDequeuedInputBuffers.put(index, newImage); 3023 } 3024 return newImage; 3025 } 3026 3027 /** 3028 * Returns a read-only ByteBuffer for a dequeued output buffer 3029 * index. The position and limit of the returned buffer are set 3030 * to the valid output data. 3031 * 3032 * After calling this method, any ByteBuffer or Image object 3033 * previously returned for the same output index MUST no longer 3034 * be used. 3035 * 3036 * @param index The index of a client-owned output buffer previously 3037 * returned from a call to {@link #dequeueOutputBuffer}, 3038 * or received via an onOutputBufferAvailable callback. 3039 * 3040 * @return the output buffer, or null if the index is not a dequeued 3041 * output buffer, or the codec is configured with an output surface. 3042 * 3043 * @throws IllegalStateException if not in the Executing state. 3044 * @throws MediaCodec.CodecException upon codec error. 3045 */ 3046 @Nullable getOutputBuffer(int index)3047 public ByteBuffer getOutputBuffer(int index) { 3048 ByteBuffer newBuffer = getBuffer(false /* input */, index); 3049 synchronized(mBufferLock) { 3050 invalidateByteBuffer(mCachedOutputBuffers, index); 3051 mDequeuedOutputBuffers.put(index, newBuffer); 3052 } 3053 return newBuffer; 3054 } 3055 3056 /** 3057 * Returns a read-only Image object for a dequeued output buffer 3058 * index that contains the raw video frame. 3059 * 3060 * After calling this method, any ByteBuffer or Image object previously 3061 * returned for the same output index MUST no longer be used. 3062 * 3063 * @param index The index of a client-owned output buffer previously 3064 * returned from a call to {@link #dequeueOutputBuffer}, 3065 * or received via an onOutputBufferAvailable callback. 3066 * 3067 * @return the output image, or null if the index is not a 3068 * dequeued output buffer, not a raw video frame, or if the codec 3069 * was configured with an output surface. 3070 * 3071 * @throws IllegalStateException if not in the Executing state. 3072 * @throws MediaCodec.CodecException upon codec error. 3073 */ 3074 @Nullable getOutputImage(int index)3075 public Image getOutputImage(int index) { 3076 Image newImage = getImage(false /* input */, index); 3077 synchronized(mBufferLock) { 3078 invalidateByteBuffer(mCachedOutputBuffers, index); 3079 mDequeuedOutputBuffers.put(index, newImage); 3080 } 3081 return newImage; 3082 } 3083 3084 /** 3085 * The content is scaled to the surface dimensions 3086 */ 3087 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 3088 3089 /** 3090 * The content is scaled, maintaining its aspect ratio, the whole 3091 * surface area is used, content may be cropped. 3092 * <p class=note> 3093 * This mode is only suitable for content with 1:1 pixel aspect ratio as you cannot 3094 * configure the pixel aspect ratio for a {@link Surface}. 3095 * <p class=note> 3096 * As of {@link android.os.Build.VERSION_CODES#N} release, this mode may not work if 3097 * the video is {@linkplain MediaFormat#KEY_ROTATION rotated} by 90 or 270 degrees. 3098 */ 3099 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 3100 3101 /** @hide */ 3102 @IntDef({ 3103 VIDEO_SCALING_MODE_SCALE_TO_FIT, 3104 VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING, 3105 }) 3106 @Retention(RetentionPolicy.SOURCE) 3107 public @interface VideoScalingMode {} 3108 3109 /** 3110 * If a surface has been specified in a previous call to {@link #configure} 3111 * specifies the scaling mode to use. The default is "scale to fit". 3112 * <p class=note> 3113 * The scaling mode may be reset to the <strong>default</strong> each time an 3114 * {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is received from the codec; therefore, the client 3115 * must call this method after every buffer change event (and before the first output buffer is 3116 * released for rendering) to ensure consistent scaling mode. 3117 * <p class=note> 3118 * Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, this can also be done 3119 * after each {@link #INFO_OUTPUT_FORMAT_CHANGED} event. 3120 * 3121 * @throws IllegalArgumentException if mode is not recognized. 3122 * @throws IllegalStateException if in the Released state. 3123 */ setVideoScalingMode(@ideoScalingMode int mode)3124 public native final void setVideoScalingMode(@VideoScalingMode int mode); 3125 3126 /** 3127 * Get the component name. If the codec was created by createDecoderByType 3128 * or createEncoderByType, what component is chosen is not known beforehand. 3129 * @throws IllegalStateException if in the Released state. 3130 */ 3131 @NonNull getName()3132 public native final String getName(); 3133 3134 /** 3135 * Change a video encoder's target bitrate on the fly. The value is an 3136 * Integer object containing the new bitrate in bps. 3137 */ 3138 public static final String PARAMETER_KEY_VIDEO_BITRATE = "video-bitrate"; 3139 3140 /** 3141 * Temporarily suspend/resume encoding of input data. While suspended 3142 * input data is effectively discarded instead of being fed into the 3143 * encoder. This parameter really only makes sense to use with an encoder 3144 * in "surface-input" mode, as the client code has no control over the 3145 * input-side of the encoder in that case. 3146 * The value is an Integer object containing the value 1 to suspend 3147 * or the value 0 to resume. 3148 */ 3149 public static final String PARAMETER_KEY_SUSPEND = "drop-input-frames"; 3150 3151 /** 3152 * Request that the encoder produce a sync frame "soon". 3153 * Provide an Integer with the value 0. 3154 */ 3155 public static final String PARAMETER_KEY_REQUEST_SYNC_FRAME = "request-sync"; 3156 3157 /** 3158 * Communicate additional parameter changes to the component instance. 3159 * <b>Note:</b> Some of these parameter changes may silently fail to apply. 3160 * 3161 * @param params The bundle of parameters to set. 3162 * @throws IllegalStateException if in the Released state. 3163 */ setParameters(@ullable Bundle params)3164 public final void setParameters(@Nullable Bundle params) { 3165 if (params == null) { 3166 return; 3167 } 3168 3169 String[] keys = new String[params.size()]; 3170 Object[] values = new Object[params.size()]; 3171 3172 int i = 0; 3173 for (final String key: params.keySet()) { 3174 keys[i] = key; 3175 values[i] = params.get(key); 3176 ++i; 3177 } 3178 3179 setParameters(keys, values); 3180 } 3181 3182 /** 3183 * Sets an asynchronous callback for actionable MediaCodec events. 3184 * 3185 * If the client intends to use the component in asynchronous mode, 3186 * a valid callback should be provided before {@link #configure} is called. 3187 * 3188 * When asynchronous callback is enabled, the client should not call 3189 * {@link #getInputBuffers}, {@link #getOutputBuffers}, 3190 * {@link #dequeueInputBuffer(long)} or {@link #dequeueOutputBuffer(BufferInfo, long)}. 3191 * <p> 3192 * Also, {@link #flush} behaves differently in asynchronous mode. After calling 3193 * {@code flush}, you must call {@link #start} to "resume" receiving input buffers, 3194 * even if an input surface was created. 3195 * 3196 * @param cb The callback that will run. Use {@code null} to clear a previously 3197 * set callback (before {@link #configure configure} is called and run 3198 * in synchronous mode). 3199 * @param handler Callbacks will happen on the handler's thread. If {@code null}, 3200 * callbacks are done on the default thread (the caller's thread or the 3201 * main thread.) 3202 */ setCallback(@ullable Callback cb, @Nullable Handler handler)3203 public void setCallback(@Nullable /* MediaCodec. */ Callback cb, @Nullable Handler handler) { 3204 if (cb != null) { 3205 synchronized (mListenerLock) { 3206 EventHandler newHandler = getEventHandlerOn(handler, mCallbackHandler); 3207 // NOTE: there are no callbacks on the handler at this time, but check anyways 3208 // even if we were to extend this to be callable dynamically, it must 3209 // be called when codec is flushed, so no messages are pending. 3210 if (newHandler != mCallbackHandler) { 3211 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 3212 mCallbackHandler.removeMessages(EVENT_CALLBACK); 3213 mCallbackHandler = newHandler; 3214 } 3215 } 3216 } else if (mCallbackHandler != null) { 3217 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 3218 mCallbackHandler.removeMessages(EVENT_CALLBACK); 3219 } 3220 3221 if (mCallbackHandler != null) { 3222 // set java callback on main handler 3223 Message msg = mCallbackHandler.obtainMessage(EVENT_SET_CALLBACK, 0, 0, cb); 3224 mCallbackHandler.sendMessage(msg); 3225 3226 // set native handler here, don't post to handler because 3227 // it may cause the callback to be delayed and set in a wrong state. 3228 // Note that native codec may start sending events to the callback 3229 // handler after this returns. 3230 native_setCallback(cb); 3231 } 3232 } 3233 3234 /** 3235 * Sets an asynchronous callback for actionable MediaCodec events on the default 3236 * looper. 3237 * <p> 3238 * Same as {@link #setCallback(Callback, Handler)} with handler set to null. 3239 * @param cb The callback that will run. Use {@code null} to clear a previously 3240 * set callback (before {@link #configure configure} is called and run 3241 * in synchronous mode). 3242 * @see #setCallback(Callback, Handler) 3243 */ setCallback(@ullable Callback cb)3244 public void setCallback(@Nullable /* MediaCodec. */ Callback cb) { 3245 setCallback(cb, null /* handler */); 3246 } 3247 3248 /** 3249 * Listener to be called when an output frame has rendered on the output surface 3250 * 3251 * @see MediaCodec#setOnFrameRenderedListener 3252 */ 3253 public interface OnFrameRenderedListener { 3254 3255 /** 3256 * Called when an output frame has rendered on the output surface. 3257 * <p> 3258 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 3259 * render timing samples, and can be significantly delayed and batched. Some frames may have 3260 * been rendered even if there was no callback generated. 3261 * 3262 * @param codec the MediaCodec instance 3263 * @param presentationTimeUs the presentation time (media time) of the frame rendered. 3264 * This is usually the same as specified in {@link #queueInputBuffer}; however, 3265 * some codecs may alter the media time by applying some time-based transformation, 3266 * such as frame rate conversion. In that case, presentation time corresponds 3267 * to the actual output frame rendered. 3268 * @param nanoTime The system time when the frame was rendered. 3269 * 3270 * @see System#nanoTime 3271 */ onFrameRendered( @onNull MediaCodec codec, long presentationTimeUs, long nanoTime)3272 public void onFrameRendered( 3273 @NonNull MediaCodec codec, long presentationTimeUs, long nanoTime); 3274 } 3275 3276 /** 3277 * Registers a callback to be invoked when an output frame is rendered on the output surface. 3278 * <p> 3279 * This method can be called in any codec state, but will only have an effect in the 3280 * Executing state for codecs that render buffers to the output surface. 3281 * <p> 3282 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 3283 * render timing samples, and can be significantly delayed and batched. Some frames may have 3284 * been rendered even if there was no callback generated. 3285 * 3286 * @param listener the callback that will be run 3287 * @param handler the callback will be run on the handler's thread. If {@code null}, 3288 * the callback will be run on the default thread, which is the looper 3289 * from which the codec was created, or a new thread if there was none. 3290 */ setOnFrameRenderedListener( @ullable OnFrameRenderedListener listener, @Nullable Handler handler)3291 public void setOnFrameRenderedListener( 3292 @Nullable OnFrameRenderedListener listener, @Nullable Handler handler) { 3293 synchronized (mListenerLock) { 3294 mOnFrameRenderedListener = listener; 3295 if (listener != null) { 3296 EventHandler newHandler = getEventHandlerOn(handler, mOnFrameRenderedHandler); 3297 if (newHandler != mOnFrameRenderedHandler) { 3298 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 3299 } 3300 mOnFrameRenderedHandler = newHandler; 3301 } else if (mOnFrameRenderedHandler != null) { 3302 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 3303 } 3304 native_enableOnFrameRenderedListener(listener != null); 3305 } 3306 } 3307 native_enableOnFrameRenderedListener(boolean enable)3308 private native void native_enableOnFrameRenderedListener(boolean enable); 3309 getEventHandlerOn( @ullable Handler handler, @NonNull EventHandler lastHandler)3310 private EventHandler getEventHandlerOn( 3311 @Nullable Handler handler, @NonNull EventHandler lastHandler) { 3312 if (handler == null) { 3313 return mEventHandler; 3314 } else { 3315 Looper looper = handler.getLooper(); 3316 if (lastHandler.getLooper() == looper) { 3317 return lastHandler; 3318 } else { 3319 return new EventHandler(this, looper); 3320 } 3321 } 3322 } 3323 3324 /** 3325 * MediaCodec callback interface. Used to notify the user asynchronously 3326 * of various MediaCodec events. 3327 */ 3328 public static abstract class Callback { 3329 /** 3330 * Called when an input buffer becomes available. 3331 * 3332 * @param codec The MediaCodec object. 3333 * @param index The index of the available input buffer. 3334 */ onInputBufferAvailable(@onNull MediaCodec codec, int index)3335 public abstract void onInputBufferAvailable(@NonNull MediaCodec codec, int index); 3336 3337 /** 3338 * Called when an output buffer becomes available. 3339 * 3340 * @param codec The MediaCodec object. 3341 * @param index The index of the available output buffer. 3342 * @param info Info regarding the available output buffer {@link MediaCodec.BufferInfo}. 3343 */ onOutputBufferAvailable( @onNull MediaCodec codec, int index, @NonNull BufferInfo info)3344 public abstract void onOutputBufferAvailable( 3345 @NonNull MediaCodec codec, int index, @NonNull BufferInfo info); 3346 3347 /** 3348 * Called when the MediaCodec encountered an error 3349 * 3350 * @param codec The MediaCodec object. 3351 * @param e The {@link MediaCodec.CodecException} object describing the error. 3352 */ onError(@onNull MediaCodec codec, @NonNull CodecException e)3353 public abstract void onError(@NonNull MediaCodec codec, @NonNull CodecException e); 3354 3355 /** 3356 * Called when the output format has changed 3357 * 3358 * @param codec The MediaCodec object. 3359 * @param format The new output format. 3360 */ onOutputFormatChanged( @onNull MediaCodec codec, @NonNull MediaFormat format)3361 public abstract void onOutputFormatChanged( 3362 @NonNull MediaCodec codec, @NonNull MediaFormat format); 3363 } 3364 postEventFromNative( int what, int arg1, int arg2, @Nullable Object obj)3365 private void postEventFromNative( 3366 int what, int arg1, int arg2, @Nullable Object obj) { 3367 synchronized (mListenerLock) { 3368 EventHandler handler = mEventHandler; 3369 if (what == EVENT_CALLBACK) { 3370 handler = mCallbackHandler; 3371 } else if (what == EVENT_FRAME_RENDERED) { 3372 handler = mOnFrameRenderedHandler; 3373 } 3374 if (handler != null) { 3375 Message msg = handler.obtainMessage(what, arg1, arg2, obj); 3376 handler.sendMessage(msg); 3377 } 3378 } 3379 } 3380 setParameters(@onNull String[] keys, @NonNull Object[] values)3381 private native final void setParameters(@NonNull String[] keys, @NonNull Object[] values); 3382 3383 /** 3384 * Get the codec info. If the codec was created by createDecoderByType 3385 * or createEncoderByType, what component is chosen is not known beforehand, 3386 * and thus the caller does not have the MediaCodecInfo. 3387 * @throws IllegalStateException if in the Released state. 3388 */ 3389 @NonNull getCodecInfo()3390 public MediaCodecInfo getCodecInfo() { 3391 return MediaCodecList.getInfoFor(getName()); 3392 } 3393 3394 @NonNull getBuffers(boolean input)3395 private native final ByteBuffer[] getBuffers(boolean input); 3396 3397 @Nullable getBuffer(boolean input, int index)3398 private native final ByteBuffer getBuffer(boolean input, int index); 3399 3400 @Nullable getImage(boolean input, int index)3401 private native final Image getImage(boolean input, int index); 3402 native_init()3403 private static native final void native_init(); 3404 native_setup( @onNull String name, boolean nameIsType, boolean encoder)3405 private native final void native_setup( 3406 @NonNull String name, boolean nameIsType, boolean encoder); 3407 native_finalize()3408 private native final void native_finalize(); 3409 3410 static { 3411 System.loadLibrary("media_jni"); native_init()3412 native_init(); 3413 } 3414 3415 private long mNativeContext; 3416 3417 /** @hide */ 3418 public static class MediaImage extends Image { 3419 private final boolean mIsReadOnly; 3420 private final int mWidth; 3421 private final int mHeight; 3422 private final int mFormat; 3423 private long mTimestamp; 3424 private final Plane[] mPlanes; 3425 private final ByteBuffer mBuffer; 3426 private final ByteBuffer mInfo; 3427 private final int mXOffset; 3428 private final int mYOffset; 3429 3430 private final static int TYPE_YUV = 1; 3431 3432 @Override getFormat()3433 public int getFormat() { 3434 throwISEIfImageIsInvalid(); 3435 return mFormat; 3436 } 3437 3438 @Override getHeight()3439 public int getHeight() { 3440 throwISEIfImageIsInvalid(); 3441 return mHeight; 3442 } 3443 3444 @Override getWidth()3445 public int getWidth() { 3446 throwISEIfImageIsInvalid(); 3447 return mWidth; 3448 } 3449 3450 @Override getTimestamp()3451 public long getTimestamp() { 3452 throwISEIfImageIsInvalid(); 3453 return mTimestamp; 3454 } 3455 3456 @Override 3457 @NonNull getPlanes()3458 public Plane[] getPlanes() { 3459 throwISEIfImageIsInvalid(); 3460 return Arrays.copyOf(mPlanes, mPlanes.length); 3461 } 3462 3463 @Override close()3464 public void close() { 3465 if (mIsImageValid) { 3466 java.nio.NioUtils.freeDirectBuffer(mBuffer); 3467 mIsImageValid = false; 3468 } 3469 } 3470 3471 /** 3472 * Set the crop rectangle associated with this frame. 3473 * <p> 3474 * The crop rectangle specifies the region of valid pixels in the image, 3475 * using coordinates in the largest-resolution plane. 3476 */ 3477 @Override setCropRect(@ullable Rect cropRect)3478 public void setCropRect(@Nullable Rect cropRect) { 3479 if (mIsReadOnly) { 3480 throw new ReadOnlyBufferException(); 3481 } 3482 super.setCropRect(cropRect); 3483 } 3484 3485 MediaImage( @onNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect)3486 public MediaImage( 3487 @NonNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, 3488 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) { 3489 mFormat = ImageFormat.YUV_420_888; 3490 mTimestamp = timestamp; 3491 mIsImageValid = true; 3492 mIsReadOnly = buffer.isReadOnly(); 3493 mBuffer = buffer.duplicate(); 3494 3495 // save offsets and info 3496 mXOffset = xOffset; 3497 mYOffset = yOffset; 3498 mInfo = info; 3499 3500 // read media-info. See MediaImage2 3501 if (info.remaining() == 104) { 3502 int type = info.getInt(); 3503 if (type != TYPE_YUV) { 3504 throw new UnsupportedOperationException("unsupported type: " + type); 3505 } 3506 int numPlanes = info.getInt(); 3507 if (numPlanes != 3) { 3508 throw new RuntimeException("unexpected number of planes: " + numPlanes); 3509 } 3510 mWidth = info.getInt(); 3511 mHeight = info.getInt(); 3512 if (mWidth < 1 || mHeight < 1) { 3513 throw new UnsupportedOperationException( 3514 "unsupported size: " + mWidth + "x" + mHeight); 3515 } 3516 int bitDepth = info.getInt(); 3517 if (bitDepth != 8) { 3518 throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth); 3519 } 3520 int bitDepthAllocated = info.getInt(); 3521 if (bitDepthAllocated != 8) { 3522 throw new UnsupportedOperationException( 3523 "unsupported allocated bit depth: " + bitDepthAllocated); 3524 } 3525 mPlanes = new MediaPlane[numPlanes]; 3526 for (int ix = 0; ix < numPlanes; ix++) { 3527 int planeOffset = info.getInt(); 3528 int colInc = info.getInt(); 3529 int rowInc = info.getInt(); 3530 int horiz = info.getInt(); 3531 int vert = info.getInt(); 3532 if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) { 3533 throw new UnsupportedOperationException("unexpected subsampling: " 3534 + horiz + "x" + vert + " on plane " + ix); 3535 } 3536 if (colInc < 1 || rowInc < 1) { 3537 throw new UnsupportedOperationException("unexpected strides: " 3538 + colInc + " pixel, " + rowInc + " row on plane " + ix); 3539 } 3540 3541 buffer.clear(); 3542 buffer.position(mBuffer.position() + planeOffset 3543 + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc); 3544 buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8) 3545 + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc); 3546 mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc); 3547 } 3548 } else { 3549 throw new UnsupportedOperationException( 3550 "unsupported info length: " + info.remaining()); 3551 } 3552 3553 if (cropRect == null) { 3554 cropRect = new Rect(0, 0, mWidth, mHeight); 3555 } 3556 cropRect.offset(-xOffset, -yOffset); 3557 super.setCropRect(cropRect); 3558 } 3559 3560 private class MediaPlane extends Plane { MediaPlane(@onNull ByteBuffer buffer, int rowInc, int colInc)3561 public MediaPlane(@NonNull ByteBuffer buffer, int rowInc, int colInc) { 3562 mData = buffer; 3563 mRowInc = rowInc; 3564 mColInc = colInc; 3565 } 3566 3567 @Override getRowStride()3568 public int getRowStride() { 3569 throwISEIfImageIsInvalid(); 3570 return mRowInc; 3571 } 3572 3573 @Override getPixelStride()3574 public int getPixelStride() { 3575 throwISEIfImageIsInvalid(); 3576 return mColInc; 3577 } 3578 3579 @Override 3580 @NonNull getBuffer()3581 public ByteBuffer getBuffer() { 3582 throwISEIfImageIsInvalid(); 3583 return mData; 3584 } 3585 3586 private final int mRowInc; 3587 private final int mColInc; 3588 private final ByteBuffer mData; 3589 } 3590 } 3591 } 3592
