1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntDef; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.graphics.ImageFormat; 23 import android.graphics.Rect; 24 import android.graphics.SurfaceTexture; 25 import android.media.MediaCodecInfo.CodecCapabilities; 26 import android.os.Bundle; 27 import android.os.Handler; 28 import android.os.IBinder; 29 import android.os.IHwBinder; 30 import android.os.Looper; 31 import android.os.Message; 32 import android.os.PersistableBundle; 33 import android.view.Surface; 34 35 import java.io.IOException; 36 import java.lang.annotation.Retention; 37 import java.lang.annotation.RetentionPolicy; 38 import java.nio.ByteBuffer; 39 import java.nio.ByteOrder; 40 import java.nio.ReadOnlyBufferException; 41 import java.util.Arrays; 42 import java.util.HashMap; 43 import java.util.Map; 44 45 /** 46 MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components. 47 It is part of the Android low-level multimedia support infrastructure (normally used together 48 with {@link MediaExtractor}, {@link MediaSync}, {@link MediaMuxer}, {@link MediaCrypto}, 49 {@link MediaDrm}, {@link Image}, {@link Surface}, and {@link AudioTrack}.) 50 <p> 51 <center><object style="width: 540px; height: 205px;" type="image/svg+xml" 52 data="../../../images/media/mediacodec_buffers.svg"><img 53 src="../../../images/media/mediacodec_buffers.png" style="width: 540px; height: 205px" 54 alt="MediaCodec buffer flow diagram"></object></center> 55 <p> 56 In broad terms, a codec processes input data to generate output data. It processes data 57 asynchronously and uses a set of input and output buffers. At a simplistic level, you request 58 (or receive) an empty input buffer, fill it up with data and send it to the codec for 59 processing. The codec uses up the data and transforms it into one of its empty output buffers. 60 Finally, you request (or receive) a filled output buffer, consume its contents and release it 61 back to the codec. 62 63 <h3>Data Types</h3> 64 <p> 65 Codecs operate on three kinds of data: compressed data, raw audio data and raw video data. 66 All three kinds of data can be processed using {@link ByteBuffer ByteBuffers}, but you should use 67 a {@link Surface} for raw video data to improve codec performance. Surface uses native video 68 buffers without mapping or copying them to ByteBuffers; thus, it is much more efficient. 69 You normally cannot access the raw video data when using a Surface, but you can use the 70 {@link ImageReader} class to access unsecured decoded (raw) video frames. This may still be more 71 efficient than using ByteBuffers, as some native buffers may be mapped into {@linkplain 72 ByteBuffer#isDirect direct} ByteBuffers. When using ByteBuffer mode, you can access raw video 73 frames using the {@link Image} class and {@link #getInputImage getInput}/{@link #getOutputImage 74 OutputImage(int)}. 75 76 <h4>Compressed Buffers</h4> 77 <p> 78 Input buffers (for decoders) and output buffers (for encoders) contain compressed data according 79 to the {@linkplain MediaFormat#KEY_MIME format's type}. For video types this is normally a single 80 compressed video frame. For audio data this is normally a single access unit (an encoded audio 81 segment typically containing a few milliseconds of audio as dictated by the format type), but 82 this requirement is slightly relaxed in that a buffer may contain multiple encoded access units 83 of audio. In either case, buffers do not start or end on arbitrary byte boundaries, but rather on 84 frame/access unit boundaries unless they are flagged with {@link #BUFFER_FLAG_PARTIAL_FRAME}. 85 86 <h4>Raw Audio Buffers</h4> 87 <p> 88 Raw audio buffers contain entire frames of PCM audio data, which is one sample for each channel 89 in channel order. Each sample is a {@linkplain AudioFormat#ENCODING_PCM_16BIT 16-bit signed 90 integer in native byte order}. 91 92 <pre class=prettyprint> 93 short[] getSamplesForChannel(MediaCodec codec, int bufferId, int channelIx) { 94 ByteBuffer outputBuffer = codec.getOutputBuffer(bufferId); 95 MediaFormat format = codec.getOutputFormat(bufferId); 96 ShortBuffer samples = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer(); 97 int numChannels = formet.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 98 if (channelIx < 0 || channelIx >= numChannels) { 99 return null; 100 } 101 short[] res = new short[samples.remaining() / numChannels]; 102 for (int i = 0; i < res.length; ++i) { 103 res[i] = samples.get(i * numChannels + channelIx); 104 } 105 return res; 106 }</pre> 107 108 <h4>Raw Video Buffers</h4> 109 <p> 110 In ByteBuffer mode video buffers are laid out according to their {@linkplain 111 MediaFormat#KEY_COLOR_FORMAT color format}. You can get the supported color formats as an array 112 from {@link #getCodecInfo}{@code .}{@link MediaCodecInfo#getCapabilitiesForType 113 getCapabilitiesForType(…)}{@code .}{@link CodecCapabilities#colorFormats colorFormats}. 114 Video codecs may support three kinds of color formats: 115 <ul> 116 <li><strong>native raw video format:</strong> This is marked by {@link 117 CodecCapabilities#COLOR_FormatSurface} and it can be used with an input or output Surface.</li> 118 <li><strong>flexible YUV buffers</strong> (such as {@link 119 CodecCapabilities#COLOR_FormatYUV420Flexible}): These can be used with an input/output Surface, 120 as well as in ByteBuffer mode, by using {@link #getInputImage getInput}/{@link #getOutputImage 121 OutputImage(int)}.</li> 122 <li><strong>other, specific formats:</strong> These are normally only supported in ByteBuffer 123 mode. Some color formats are vendor specific. Others are defined in {@link CodecCapabilities}. 124 For color formats that are equivalent to a flexible format, you can still use {@link 125 #getInputImage getInput}/{@link #getOutputImage OutputImage(int)}.</li> 126 </ul> 127 <p> 128 All video codecs support flexible YUV 4:2:0 buffers since {@link 129 android.os.Build.VERSION_CODES#LOLLIPOP_MR1}. 130 131 <h4>Accessing Raw Video ByteBuffers on Older Devices</h4> 132 <p> 133 Prior to {@link android.os.Build.VERSION_CODES#LOLLIPOP} and {@link Image} support, you need to 134 use the {@link MediaFormat#KEY_STRIDE} and {@link MediaFormat#KEY_SLICE_HEIGHT} output format 135 values to understand the layout of the raw output buffers. 136 <p class=note> 137 Note that on some devices the slice-height is advertised as 0. This could mean either that the 138 slice-height is the same as the frame height, or that the slice-height is the frame height 139 aligned to some value (usually a power of 2). Unfortunately, there is no standard and simple way 140 to tell the actual slice height in this case. Furthermore, the vertical stride of the {@code U} 141 plane in planar formats is also not specified or defined, though usually it is half of the slice 142 height. 143 <p> 144 The {@link MediaFormat#KEY_WIDTH} and {@link MediaFormat#KEY_HEIGHT} keys specify the size of the 145 video frames; however, for most encondings the video (picture) only occupies a portion of the 146 video frame. This is represented by the 'crop rectangle'. 147 <p> 148 You need to use the following keys to get the crop rectangle of raw output images from the 149 {@linkplain #getOutputFormat output format}. If these keys are not present, the video occupies the 150 entire video frame.The crop rectangle is understood in the context of the output frame 151 <em>before</em> applying any {@linkplain MediaFormat#KEY_ROTATION rotation}. 152 <table style="width: 0%"> 153 <thead> 154 <tr> 155 <th>Format Key</th> 156 <th>Type</th> 157 <th>Description</th> 158 </tr> 159 </thead> 160 <tbody> 161 <tr> 162 <td>{@code "crop-left"}</td> 163 <td>Integer</td> 164 <td>The left-coordinate (x) of the crop rectangle</td> 165 </tr><tr> 166 <td>{@code "crop-top"}</td> 167 <td>Integer</td> 168 <td>The top-coordinate (y) of the crop rectangle</td> 169 </tr><tr> 170 <td>{@code "crop-right"}</td> 171 <td>Integer</td> 172 <td>The right-coordinate (x) <strong>MINUS 1</strong> of the crop rectangle</td> 173 </tr><tr> 174 <td>{@code "crop-bottom"}</td> 175 <td>Integer</td> 176 <td>The bottom-coordinate (y) <strong>MINUS 1</strong> of the crop rectangle</td> 177 </tr><tr> 178 <td colspan=3> 179 The right and bottom coordinates can be understood as the coordinates of the right-most 180 valid column/bottom-most valid row of the cropped output image. 181 </td> 182 </tr> 183 </tbody> 184 </table> 185 <p> 186 The size of the video frame (before rotation) can be calculated as such: 187 <pre class=prettyprint> 188 MediaFormat format = decoder.getOutputFormat(…); 189 int width = format.getInteger(MediaFormat.KEY_WIDTH); 190 if (format.containsKey("crop-left") && format.containsKey("crop-right")) { 191 width = format.getInteger("crop-right") + 1 - format.getInteger("crop-left"); 192 } 193 int height = format.getInteger(MediaFormat.KEY_HEIGHT); 194 if (format.containsKey("crop-top") && format.containsKey("crop-bottom")) { 195 height = format.getInteger("crop-bottom") + 1 - format.getInteger("crop-top"); 196 } 197 </pre> 198 <p class=note> 199 Also note that the meaning of {@link BufferInfo#offset BufferInfo.offset} was not consistent across 200 devices. On some devices the offset pointed to the top-left pixel of the crop rectangle, while on 201 most devices it pointed to the top-left pixel of the entire frame. 202 203 <h3>States</h3> 204 <p> 205 During its life a codec conceptually exists in one of three states: Stopped, Executing or 206 Released. The Stopped collective state is actually the conglomeration of three states: 207 Uninitialized, Configured and Error, whereas the Executing state conceptually progresses through 208 three sub-states: Flushed, Running and End-of-Stream. 209 <p> 210 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 211 data="../../../images/media/mediacodec_states.svg"><img 212 src="../../../images/media/mediacodec_states.png" style="width: 519px; height: 356px" 213 alt="MediaCodec state diagram"></object></center> 214 <p> 215 When you create a codec using one of the factory methods, the codec is in the Uninitialized 216 state. First, you need to configure it via {@link #configure configure(…)}, which brings 217 it to the Configured state, then call {@link #start} to move it to the Executing state. In this 218 state you can process data through the buffer queue manipulation described above. 219 <p> 220 The Executing state has three sub-states: Flushed, Running and End-of-Stream. Immediately after 221 {@link #start} the codec is in the Flushed sub-state, where it holds all the buffers. As soon 222 as the first input buffer is dequeued, the codec moves to the Running sub-state, where it spends 223 most of its life. When you queue an input buffer with the {@linkplain #BUFFER_FLAG_END_OF_STREAM 224 end-of-stream marker}, the codec transitions to the End-of-Stream sub-state. In this state the 225 codec no longer accepts further input buffers, but still generates output buffers until the 226 end-of-stream is reached on the output. You can move back to the Flushed sub-state at any time 227 while in the Executing state using {@link #flush}. 228 <p> 229 Call {@link #stop} to return the codec to the Uninitialized state, whereupon it may be configured 230 again. When you are done using a codec, you must release it by calling {@link #release}. 231 <p> 232 On rare occasions the codec may encounter an error and move to the Error state. This is 233 communicated using an invalid return value from a queuing operation, or sometimes via an 234 exception. Call {@link #reset} to make the codec usable again. You can call it from any state to 235 move the codec back to the Uninitialized state. Otherwise, call {@link #release} to move to the 236 terminal Released state. 237 238 <h3>Creation</h3> 239 <p> 240 Use {@link MediaCodecList} to create a MediaCodec for a specific {@link MediaFormat}. When 241 decoding a file or a stream, you can get the desired format from {@link 242 MediaExtractor#getTrackFormat MediaExtractor.getTrackFormat}. Inject any specific features that 243 you want to add using {@link MediaFormat#setFeatureEnabled MediaFormat.setFeatureEnabled}, then 244 call {@link MediaCodecList#findDecoderForFormat MediaCodecList.findDecoderForFormat} to get the 245 name of a codec that can handle that specific media format. Finally, create the codec using 246 {@link #createByCodecName}. 247 <p class=note> 248 <strong>Note:</strong> On {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the format to 249 {@code MediaCodecList.findDecoder}/{@code EncoderForFormat} must not contain a {@linkplain 250 MediaFormat#KEY_FRAME_RATE frame rate}. Use 251 <code class=prettyprint>format.setString(MediaFormat.KEY_FRAME_RATE, null)</code> 252 to clear any existing frame rate setting in the format. 253 <p> 254 You can also create the preferred codec for a specific MIME type using {@link 255 #createDecoderByType createDecoder}/{@link #createEncoderByType EncoderByType(String)}. 256 This, however, cannot be used to inject features, and may create a codec that cannot handle the 257 specific desired media format. 258 259 <h4>Creating secure decoders</h4> 260 <p> 261 On versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and earlier, secure codecs might 262 not be listed in {@link MediaCodecList}, but may still be available on the system. Secure codecs 263 that exist can be instantiated by name only, by appending {@code ".secure"} to the name of a 264 regular codec (the name of all secure codecs must end in {@code ".secure"}.) {@link 265 #createByCodecName} will throw an {@code IOException} if the codec is not present on the system. 266 <p> 267 From {@link android.os.Build.VERSION_CODES#LOLLIPOP} onwards, you should use the {@link 268 CodecCapabilities#FEATURE_SecurePlayback} feature in the media format to create a secure decoder. 269 270 <h3>Initialization</h3> 271 <p> 272 After creating the codec, you can set a callback using {@link #setCallback setCallback} if you 273 want to process data asynchronously. Then, {@linkplain #configure configure} the codec using the 274 specific media format. This is when you can specify the output {@link Surface} for video 275 producers – codecs that generate raw video data (e.g. video decoders). This is also when 276 you can set the decryption parameters for secure codecs (see {@link MediaCrypto}). Finally, since 277 some codecs can operate in multiple modes, you must specify whether you want it to work as a 278 decoder or an encoder. 279 <p> 280 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you can query the resulting input and 281 output format in the Configured state. You can use this to verify the resulting configuration, 282 e.g. color formats, before starting the codec. 283 <p> 284 If you want to process raw input video buffers natively with a video consumer – a codec 285 that processes raw video input, such as a video encoder – create a destination Surface for 286 your input data using {@link #createInputSurface} after configuration. Alternately, set up the 287 codec to use a previously created {@linkplain #createPersistentInputSurface persistent input 288 surface} by calling {@link #setInputSurface}. 289 290 <h4 id=CSD><a name="CSD"></a>Codec-specific Data</h4> 291 <p> 292 Some formats, notably AAC audio and MPEG4, H.264 and H.265 video formats require the actual data 293 to be prefixed by a number of buffers containing setup data, or codec specific data. When 294 processing such compressed formats, this data must be submitted to the codec after {@link 295 #start} and before any frame data. Such data must be marked using the flag {@link 296 #BUFFER_FLAG_CODEC_CONFIG} in a call to {@link #queueInputBuffer queueInputBuffer}. 297 <p> 298 Codec-specific data can also be included in the format passed to {@link #configure configure} in 299 ByteBuffer entries with keys "csd-0", "csd-1", etc. These keys are always included in the track 300 {@link MediaFormat} obtained from the {@link MediaExtractor#getTrackFormat MediaExtractor}. 301 Codec-specific data in the format is automatically submitted to the codec upon {@link #start}; 302 you <strong>MUST NOT</strong> submit this data explicitly. If the format did not contain codec 303 specific data, you can choose to submit it using the specified number of buffers in the correct 304 order, according to the format requirements. In case of H.264 AVC, you can also concatenate all 305 codec-specific data and submit it as a single codec-config buffer. 306 <p> 307 Android uses the following codec-specific data buffers. These are also required to be set in 308 the track format for proper {@link MediaMuxer} track configuration. Each parameter set and the 309 codec-specific-data sections marked with (<sup>*</sup>) must start with a start code of 310 {@code "\x00\x00\x00\x01"}. 311 <p> 312 <style>td.NA { background: #ccc; } .mid > tr > td { vertical-align: middle; }</style> 313 <table> 314 <thead> 315 <th>Format</th> 316 <th>CSD buffer #0</th> 317 <th>CSD buffer #1</th> 318 <th>CSD buffer #2</th> 319 </thead> 320 <tbody class=mid> 321 <tr> 322 <td>AAC</td> 323 <td>Decoder-specific information from ESDS<sup>*</sup></td> 324 <td class=NA>Not Used</td> 325 <td class=NA>Not Used</td> 326 </tr> 327 <tr> 328 <td>VORBIS</td> 329 <td>Identification header</td> 330 <td>Setup header</td> 331 <td class=NA>Not Used</td> 332 </tr> 333 <tr> 334 <td>OPUS</td> 335 <td>Identification header</td> 336 <td>Pre-skip in nanosecs<br> 337 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)<br> 338 This overrides the pre-skip value in the identification header.</td> 339 <td>Seek Pre-roll in nanosecs<br> 340 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)</td> 341 </tr> 342 <tr> 343 <td>FLAC</td> 344 <td>mandatory metadata block (called the STREAMINFO block),<br> 345 optionally followed by any number of other metadata blocks</td> 346 <td class=NA>Not Used</td> 347 <td class=NA>Not Used</td> 348 </tr> 349 <tr> 350 <td>MPEG-4</td> 351 <td>Decoder-specific information from ESDS<sup>*</sup></td> 352 <td class=NA>Not Used</td> 353 <td class=NA>Not Used</td> 354 </tr> 355 <tr> 356 <td>H.264 AVC</td> 357 <td>SPS (Sequence Parameter Sets<sup>*</sup>)</td> 358 <td>PPS (Picture Parameter Sets<sup>*</sup>)</td> 359 <td class=NA>Not Used</td> 360 </tr> 361 <tr> 362 <td>H.265 HEVC</td> 363 <td>VPS (Video Parameter Sets<sup>*</sup>) +<br> 364 SPS (Sequence Parameter Sets<sup>*</sup>) +<br> 365 PPS (Picture Parameter Sets<sup>*</sup>)</td> 366 <td class=NA>Not Used</td> 367 <td class=NA>Not Used</td> 368 </tr> 369 <tr> 370 <td>VP9</td> 371 <td>VP9 <a href="http://wiki.webmproject.org/vp9-codecprivate">CodecPrivate</a> Data 372 (optional)</td> 373 <td class=NA>Not Used</td> 374 <td class=NA>Not Used</td> 375 </tr> 376 </tbody> 377 </table> 378 379 <p class=note> 380 <strong>Note:</strong> care must be taken if the codec is flushed immediately or shortly 381 after start, before any output buffer or output format change has been returned, as the codec 382 specific data may be lost during the flush. You must resubmit the data using buffers marked with 383 {@link #BUFFER_FLAG_CODEC_CONFIG} after such flush to ensure proper codec operation. 384 <p> 385 Encoders (or codecs that generate compressed data) will create and return the codec specific data 386 before any valid output buffer in output buffers marked with the {@linkplain 387 #BUFFER_FLAG_CODEC_CONFIG codec-config flag}. Buffers containing codec-specific-data have no 388 meaningful timestamps. 389 390 <h3>Data Processing</h3> 391 <p> 392 Each codec maintains a set of input and output buffers that are referred to by a buffer-ID in 393 API calls. After a successful call to {@link #start} the client "owns" neither input nor output 394 buffers. In synchronous mode, call {@link #dequeueInputBuffer dequeueInput}/{@link 395 #dequeueOutputBuffer OutputBuffer(…)} to obtain (get ownership of) an input or output 396 buffer from the codec. In asynchronous mode, you will automatically receive available buffers via 397 the {@link Callback#onInputBufferAvailable MediaCodec.Callback.onInput}/{@link 398 Callback#onOutputBufferAvailable OutputBufferAvailable(…)} callbacks. 399 <p> 400 Upon obtaining an input buffer, fill it with data and submit it to the codec using {@link 401 #queueInputBuffer queueInputBuffer} – or {@link #queueSecureInputBuffer 402 queueSecureInputBuffer} if using decryption. Do not submit multiple input buffers with the same 403 timestamp (unless it is <a href="#CSD">codec-specific data</a> marked as such). 404 <p> 405 The codec in turn will return a read-only output buffer via the {@link 406 Callback#onOutputBufferAvailable onOutputBufferAvailable} callback in asynchronous mode, or in 407 response to a {@link #dequeueOutputBuffer dequeuOutputBuffer} call in synchronous mode. After the 408 output buffer has been processed, call one of the {@link #releaseOutputBuffer 409 releaseOutputBuffer} methods to return the buffer to the codec. 410 <p> 411 While you are not required to resubmit/release buffers immediately to the codec, holding onto 412 input and/or output buffers may stall the codec, and this behavior is device dependent. 413 <strong>Specifically, it is possible that a codec may hold off on generating output buffers until 414 <em>all</em> outstanding buffers have been released/resubmitted.</strong> Therefore, try to 415 hold onto to available buffers as little as possible. 416 <p> 417 Depending on the API version, you can process data in three ways: 418 <table> 419 <thead> 420 <tr> 421 <th>Processing Mode</th> 422 <th>API version <= 20<br>Jelly Bean/KitKat</th> 423 <th>API version >= 21<br>Lollipop and later</th> 424 </tr> 425 </thead> 426 <tbody> 427 <tr> 428 <td>Synchronous API using buffer arrays</td> 429 <td>Supported</td> 430 <td>Deprecated</td> 431 </tr> 432 <tr> 433 <td>Synchronous API using buffers</td> 434 <td class=NA>Not Available</td> 435 <td>Supported</td> 436 </tr> 437 <tr> 438 <td>Asynchronous API using buffers</td> 439 <td class=NA>Not Available</td> 440 <td>Supported</td> 441 </tr> 442 </tbody> 443 </table> 444 445 <h4>Asynchronous Processing using Buffers</h4> 446 <p> 447 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the preferred method is to process data 448 asynchronously by setting a callback before calling {@link #configure configure}. Asynchronous 449 mode changes the state transitions slightly, because you must call {@link #start} after {@link 450 #flush} to transition the codec to the Running sub-state and start receiving input buffers. 451 Similarly, upon an initial call to {@code start} the codec will move directly to the Running 452 sub-state and start passing available input buffers via the callback. 453 <p> 454 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 455 data="../../../images/media/mediacodec_async_states.svg"><img 456 src="../../../images/media/mediacodec_async_states.png" style="width: 516px; height: 353px" 457 alt="MediaCodec state diagram for asynchronous operation"></object></center> 458 <p> 459 MediaCodec is typically used like this in asynchronous mode: 460 <pre class=prettyprint> 461 MediaCodec codec = MediaCodec.createByCodecName(name); 462 MediaFormat mOutputFormat; // member variable 463 codec.setCallback(new MediaCodec.Callback() { 464 {@literal @Override} 465 void onInputBufferAvailable(MediaCodec mc, int inputBufferId) { 466 ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); 467 // fill inputBuffer with valid data 468 … 469 codec.queueInputBuffer(inputBufferId, …); 470 } 471 472 {@literal @Override} 473 void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, …) { 474 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 475 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 476 // bufferFormat is equivalent to mOutputFormat 477 // outputBuffer is ready to be processed or rendered. 478 … 479 codec.releaseOutputBuffer(outputBufferId, …); 480 } 481 482 {@literal @Override} 483 void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { 484 // Subsequent data will conform to new format. 485 // Can ignore if using getOutputFormat(outputBufferId) 486 mOutputFormat = format; // option B 487 } 488 489 {@literal @Override} 490 void onError(…) { 491 … 492 } 493 }); 494 codec.configure(format, …); 495 mOutputFormat = codec.getOutputFormat(); // option B 496 codec.start(); 497 // wait for processing to complete 498 codec.stop(); 499 codec.release();</pre> 500 501 <h4>Synchronous Processing using Buffers</h4> 502 <p> 503 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you should retrieve input and output 504 buffers using {@link #getInputBuffer getInput}/{@link #getOutputBuffer OutputBuffer(int)} and/or 505 {@link #getInputImage getInput}/{@link #getOutputImage OutputImage(int)} even when using the 506 codec in synchronous mode. This allows certain optimizations by the framework, e.g. when 507 processing dynamic content. This optimization is disabled if you call {@link #getInputBuffers 508 getInput}/{@link #getOutputBuffers OutputBuffers()}. 509 510 <p class=note> 511 <strong>Note:</strong> do not mix the methods of using buffers and buffer arrays at the same 512 time. Specifically, only call {@code getInput}/{@code OutputBuffers} directly after {@link 513 #start} or after having dequeued an output buffer ID with the value of {@link 514 #INFO_OUTPUT_FORMAT_CHANGED}. 515 <p> 516 MediaCodec is typically used like this in synchronous mode: 517 <pre> 518 MediaCodec codec = MediaCodec.createByCodecName(name); 519 codec.configure(format, …); 520 MediaFormat outputFormat = codec.getOutputFormat(); // option B 521 codec.start(); 522 for (;;) { 523 int inputBufferId = codec.dequeueInputBuffer(timeoutUs); 524 if (inputBufferId >= 0) { 525 ByteBuffer inputBuffer = codec.getInputBuffer(…); 526 // fill inputBuffer with valid data 527 … 528 codec.queueInputBuffer(inputBufferId, …); 529 } 530 int outputBufferId = codec.dequeueOutputBuffer(…); 531 if (outputBufferId >= 0) { 532 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 533 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 534 // bufferFormat is identical to outputFormat 535 // outputBuffer is ready to be processed or rendered. 536 … 537 codec.releaseOutputBuffer(outputBufferId, …); 538 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 539 // Subsequent data will conform to new format. 540 // Can ignore if using getOutputFormat(outputBufferId) 541 outputFormat = codec.getOutputFormat(); // option B 542 } 543 } 544 codec.stop(); 545 codec.release();</pre> 546 547 <h4>Synchronous Processing using Buffer Arrays (deprecated)</h4> 548 <p> 549 In versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and before, the set of input and 550 output buffers are represented by the {@code ByteBuffer[]} arrays. After a successful call to 551 {@link #start}, retrieve the buffer arrays using {@link #getInputBuffers getInput}/{@link 552 #getOutputBuffers OutputBuffers()}. Use the buffer ID-s as indices into these arrays (when 553 non-negative), as demonstrated in the sample below. Note that there is no inherent correlation 554 between the size of the arrays and the number of input and output buffers used by the system, 555 although the array size provides an upper bound. 556 <pre> 557 MediaCodec codec = MediaCodec.createByCodecName(name); 558 codec.configure(format, …); 559 codec.start(); 560 ByteBuffer[] inputBuffers = codec.getInputBuffers(); 561 ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 562 for (;;) { 563 int inputBufferId = codec.dequeueInputBuffer(…); 564 if (inputBufferId >= 0) { 565 // fill inputBuffers[inputBufferId] with valid data 566 … 567 codec.queueInputBuffer(inputBufferId, …); 568 } 569 int outputBufferId = codec.dequeueOutputBuffer(…); 570 if (outputBufferId >= 0) { 571 // outputBuffers[outputBufferId] is ready to be processed or rendered. 572 … 573 codec.releaseOutputBuffer(outputBufferId, …); 574 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 575 outputBuffers = codec.getOutputBuffers(); 576 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 577 // Subsequent data will conform to new format. 578 MediaFormat format = codec.getOutputFormat(); 579 } 580 } 581 codec.stop(); 582 codec.release();</pre> 583 584 <h4>End-of-stream Handling</h4> 585 <p> 586 When you reach the end of the input data, you must signal it to the codec by specifying the 587 {@link #BUFFER_FLAG_END_OF_STREAM} flag in the call to {@link #queueInputBuffer 588 queueInputBuffer}. You can do this on the last valid input buffer, or by submitting an additional 589 empty input buffer with the end-of-stream flag set. If using an empty buffer, the timestamp will 590 be ignored. 591 <p> 592 The codec will continue to return output buffers until it eventually signals the end of the 593 output stream by specifying the same end-of-stream flag in the {@link BufferInfo} set in {@link 594 #dequeueOutputBuffer dequeueOutputBuffer} or returned via {@link Callback#onOutputBufferAvailable 595 onOutputBufferAvailable}. This can be set on the last valid output buffer, or on an empty buffer 596 after the last valid output buffer. The timestamp of such empty buffer should be ignored. 597 <p> 598 Do not submit additional input buffers after signaling the end of the input stream, unless the 599 codec has been flushed, or stopped and restarted. 600 601 <h4>Using an Output Surface</h4> 602 <p> 603 The data processing is nearly identical to the ByteBuffer mode when using an output {@link 604 Surface}; however, the output buffers will not be accessible, and are represented as {@code null} 605 values. E.g. {@link #getOutputBuffer getOutputBuffer}/{@link #getOutputImage Image(int)} will 606 return {@code null} and {@link #getOutputBuffers} will return an array containing only {@code 607 null}-s. 608 <p> 609 When using an output Surface, you can select whether or not to render each output buffer on the 610 surface. You have three choices: 611 <ul> 612 <li><strong>Do not render the buffer:</strong> Call {@link #releaseOutputBuffer(int, boolean) 613 releaseOutputBuffer(bufferId, false)}.</li> 614 <li><strong>Render the buffer with the default timestamp:</strong> Call {@link 615 #releaseOutputBuffer(int, boolean) releaseOutputBuffer(bufferId, true)}.</li> 616 <li><strong>Render the buffer with a specific timestamp:</strong> Call {@link 617 #releaseOutputBuffer(int, long) releaseOutputBuffer(bufferId, timestamp)}.</li> 618 </ul> 619 <p> 620 Since {@link android.os.Build.VERSION_CODES#M}, the default timestamp is the {@linkplain 621 BufferInfo#presentationTimeUs presentation timestamp} of the buffer (converted to nanoseconds). 622 It was not defined prior to that. 623 <p> 624 Also since {@link android.os.Build.VERSION_CODES#M}, you can change the output Surface 625 dynamically using {@link #setOutputSurface setOutputSurface}. 626 627 <h4>Transformations When Rendering onto Surface</h4> 628 629 If the codec is configured into Surface mode, any crop rectangle, {@linkplain 630 MediaFormat#KEY_ROTATION rotation} and {@linkplain #setVideoScalingMode video scaling 631 mode} will be automatically applied with one exception: 632 <p class=note> 633 Prior to the {@link android.os.Build.VERSION_CODES#M} release, software decoders may not 634 have applied the rotation when being rendered onto a Surface. Unfortunately, there is no standard 635 and simple way to identify software decoders, or if they apply the rotation other than by trying 636 it out. 637 <p> 638 There are also some caveats. 639 <p class=note> 640 Note that the pixel aspect ratio is not considered when displaying the output onto the 641 Surface. This means that if you are using {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT} mode, you 642 must position the output Surface so that it has the proper final display aspect ratio. Conversely, 643 you can only use {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode for content with 644 square pixels (pixel aspect ratio or 1:1). 645 <p class=note> 646 Note also that as of {@link android.os.Build.VERSION_CODES#N} release, {@link 647 #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode may not work correctly for videos rotated 648 by 90 or 270 degrees. 649 <p class=note> 650 When setting the video scaling mode, note that it must be reset after each time the output 651 buffers change. Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, you can 652 do this after each time the output format changes. 653 654 <h4>Using an Input Surface</h4> 655 <p> 656 When using an input Surface, there are no accessible input buffers, as buffers are automatically 657 passed from the input surface to the codec. Calling {@link #dequeueInputBuffer 658 dequeueInputBuffer} will throw an {@code IllegalStateException}, and {@link #getInputBuffers} 659 returns a bogus {@code ByteBuffer[]} array that <strong>MUST NOT</strong> be written into. 660 <p> 661 Call {@link #signalEndOfInputStream} to signal end-of-stream. The input surface will stop 662 submitting data to the codec immediately after this call. 663 <p> 664 665 <h3>Seeking & Adaptive Playback Support</h3> 666 <p> 667 Video decoders (and in general codecs that consume compressed video data) behave differently 668 regarding seek and format change whether or not they support and are configured for adaptive 669 playback. You can check if a decoder supports {@linkplain 670 CodecCapabilities#FEATURE_AdaptivePlayback adaptive playback} via {@link 671 CodecCapabilities#isFeatureSupported CodecCapabilities.isFeatureSupported(String)}. Adaptive 672 playback support for video decoders is only activated if you configure the codec to decode onto a 673 {@link Surface}. 674 675 <h4 id=KeyFrames><a name="KeyFrames"></a>Stream Boundary and Key Frames</h4> 676 <p> 677 It is important that the input data after {@link #start} or {@link #flush} starts at a suitable 678 stream boundary: the first frame must a key frame. A <em>key frame</em> can be decoded 679 completely on its own (for most codecs this means an I-frame), and no frames that are to be 680 displayed after a key frame refer to frames before the key frame. 681 <p> 682 The following table summarizes suitable key frames for various video formats. 683 <table> 684 <thead> 685 <tr> 686 <th>Format</th> 687 <th>Suitable key frame</th> 688 </tr> 689 </thead> 690 <tbody class=mid> 691 <tr> 692 <td>VP9/VP8</td> 693 <td>a suitable intraframe where no subsequent frames refer to frames prior to this frame.<br> 694 <i>(There is no specific name for such key frame.)</i></td> 695 </tr> 696 <tr> 697 <td>H.265 HEVC</td> 698 <td>IDR or CRA</td> 699 </tr> 700 <tr> 701 <td>H.264 AVC</td> 702 <td>IDR</td> 703 </tr> 704 <tr> 705 <td>MPEG-4<br>H.263<br>MPEG-2</td> 706 <td>a suitable I-frame where no subsequent frames refer to frames prior to this frame.<br> 707 <i>(There is no specific name for such key frame.)</td> 708 </tr> 709 </tbody> 710 </table> 711 712 <h4>For decoders that do not support adaptive playback (including when not decoding onto a 713 Surface)</h4> 714 <p> 715 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 716 seek) you <strong>MUST</strong> flush the decoder. Since all output buffers are immediately 717 revoked at the point of the flush, you may want to first signal then wait for the end-of-stream 718 before you call {@code flush}. It is important that the input data after a flush starts at a 719 suitable stream boundary/key frame. 720 <p class=note> 721 <strong>Note:</strong> the format of the data submitted after a flush must not change; {@link 722 #flush} does not support format discontinuities; for that, a full {@link #stop} - {@link 723 #configure configure(…)} - {@link #start} cycle is necessary. 724 725 <p class=note> 726 <strong>Also note:</strong> if you flush the codec too soon after {@link #start} – 727 generally, before the first output buffer or output format change is received – you 728 will need to resubmit the codec-specific-data to the codec. See the <a 729 href="#CSD">codec-specific-data section</a> for more info. 730 731 <h4>For decoders that support and are configured for adaptive playback</h4> 732 <p> 733 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 734 seek) it is <em>not necessary</em> to flush the decoder; however, input data after the 735 discontinuity must start at a suitable stream boundary/key frame. 736 <p> 737 For some video formats - namely H.264, H.265, VP8 and VP9 - it is also possible to change the 738 picture size or configuration mid-stream. To do this you must package the entire new 739 codec-specific configuration data together with the key frame into a single buffer (including 740 any start codes), and submit it as a <strong>regular</strong> input buffer. 741 <p> 742 You will receive an {@link #INFO_OUTPUT_FORMAT_CHANGED} return value from {@link 743 #dequeueOutputBuffer dequeueOutputBuffer} or a {@link Callback#onOutputBufferAvailable 744 onOutputFormatChanged} callback just after the picture-size change takes place and before any 745 frames with the new size have been returned. 746 <p class=note> 747 <strong>Note:</strong> just as the case for codec-specific data, be careful when calling 748 {@link #flush} shortly after you have changed the picture size. If you have not received 749 confirmation of the picture size change, you will need to repeat the request for the new picture 750 size. 751 752 <h3>Error handling</h3> 753 <p> 754 The factory methods {@link #createByCodecName createByCodecName} and {@link #createDecoderByType 755 createDecoder}/{@link #createEncoderByType EncoderByType} throw {@code IOException} on failure 756 which you must catch or declare to pass up. MediaCodec methods throw {@code 757 IllegalStateException} when the method is called from a codec state that does not allow it; this 758 is typically due to incorrect application API usage. Methods involving secure buffers may throw 759 {@link CryptoException}, which has further error information obtainable from {@link 760 CryptoException#getErrorCode}. 761 <p> 762 Internal codec errors result in a {@link CodecException}, which may be due to media content 763 corruption, hardware failure, resource exhaustion, and so forth, even when the application is 764 correctly using the API. The recommended action when receiving a {@code CodecException} 765 can be determined by calling {@link CodecException#isRecoverable} and {@link 766 CodecException#isTransient}: 767 <ul> 768 <li><strong>recoverable errors:</strong> If {@code isRecoverable()} returns true, then call 769 {@link #stop}, {@link #configure configure(…)}, and {@link #start} to recover.</li> 770 <li><strong>transient errors:</strong> If {@code isTransient()} returns true, then resources are 771 temporarily unavailable and the method may be retried at a later time.</li> 772 <li><strong>fatal errors:</strong> If both {@code isRecoverable()} and {@code isTransient()} 773 return false, then the {@code CodecException} is fatal and the codec must be {@linkplain #reset 774 reset} or {@linkplain #release released}.</li> 775 </ul> 776 <p> 777 Both {@code isRecoverable()} and {@code isTransient()} do not return true at the same time. 778 779 <h2 id=History><a name="History"></a>Valid API Calls and API History</h2> 780 <p> 781 This sections summarizes the valid API calls in each state and the API history of the MediaCodec 782 class. For API version numbers, see {@link android.os.Build.VERSION_CODES}. 783 784 <style> 785 .api > tr > th, .api > tr > td { text-align: center; padding: 4px 4px; } 786 .api > tr > th { vertical-align: bottom; } 787 .api > tr > td { vertical-align: middle; } 788 .sml > tr > th, .sml > tr > td { text-align: center; padding: 2px 4px; } 789 .fn { text-align: left; } 790 .fn > code > a { font: 14px/19px Roboto Condensed, sans-serif; } 791 .deg45 { 792 white-space: nowrap; background: none; border: none; vertical-align: bottom; 793 width: 30px; height: 83px; 794 } 795 .deg45 > div { 796 transform: skew(-45deg, 0deg) translate(1px, -67px); 797 transform-origin: bottom left 0; 798 width: 30px; height: 20px; 799 } 800 .deg45 > div > div { border: 1px solid #ddd; background: #999; height: 90px; width: 42px; } 801 .deg45 > div > div > div { transform: skew(45deg, 0deg) translate(-55px, 55px) rotate(-45deg); } 802 </style> 803 804 <table align="right" style="width: 0%"> 805 <thead> 806 <tr><th>Symbol</th><th>Meaning</th></tr> 807 </thead> 808 <tbody class=sml> 809 <tr><td>●</td><td>Supported</td></tr> 810 <tr><td>⁕</td><td>Semantics changed</td></tr> 811 <tr><td>○</td><td>Experimental support</td></tr> 812 <tr><td>[ ]</td><td>Deprecated</td></tr> 813 <tr><td>⎋</td><td>Restricted to surface input mode</td></tr> 814 <tr><td>⎆</td><td>Restricted to surface output mode</td></tr> 815 <tr><td>▧</td><td>Restricted to ByteBuffer input mode</td></tr> 816 <tr><td>↩</td><td>Restricted to synchronous mode</td></tr> 817 <tr><td>⇄</td><td>Restricted to asynchronous mode</td></tr> 818 <tr><td>( )</td><td>Can be called, but shouldn't</td></tr> 819 </tbody> 820 </table> 821 822 <table style="width: 100%;"> 823 <thead class=api> 824 <tr> 825 <th class=deg45><div><div style="background:#4285f4"><div>Uninitialized</div></div></div></th> 826 <th class=deg45><div><div style="background:#f4b400"><div>Configured</div></div></div></th> 827 <th class=deg45><div><div style="background:#e67c73"><div>Flushed</div></div></div></th> 828 <th class=deg45><div><div style="background:#0f9d58"><div>Running</div></div></div></th> 829 <th class=deg45><div><div style="background:#f7cb4d"><div>End of Stream</div></div></div></th> 830 <th class=deg45><div><div style="background:#db4437"><div>Error</div></div></div></th> 831 <th class=deg45><div><div style="background:#666"><div>Released</div></div></div></th> 832 <th></th> 833 <th colspan="8">SDK Version</th> 834 </tr> 835 <tr> 836 <th colspan="7">State</th> 837 <th>Method</th> 838 <th>16</th> 839 <th>17</th> 840 <th>18</th> 841 <th>19</th> 842 <th>20</th> 843 <th>21</th> 844 <th>22</th> 845 <th>23</th> 846 </tr> 847 </thead> 848 <tbody class=api> 849 <tr> 850 <td></td> 851 <td></td> 852 <td></td> 853 <td></td> 854 <td></td> 855 <td></td> 856 <td></td> 857 <td class=fn>{@link #createByCodecName createByCodecName}</td> 858 <td>●</td> 859 <td>●</td> 860 <td>●</td> 861 <td>●</td> 862 <td>●</td> 863 <td>●</td> 864 <td>●</td> 865 <td>●</td> 866 </tr> 867 <tr> 868 <td></td> 869 <td></td> 870 <td></td> 871 <td></td> 872 <td></td> 873 <td></td> 874 <td></td> 875 <td class=fn>{@link #createDecoderByType createDecoderByType}</td> 876 <td>●</td> 877 <td>●</td> 878 <td>●</td> 879 <td>●</td> 880 <td>●</td> 881 <td>●</td> 882 <td>●</td> 883 <td>●</td> 884 </tr> 885 <tr> 886 <td></td> 887 <td></td> 888 <td></td> 889 <td></td> 890 <td></td> 891 <td></td> 892 <td></td> 893 <td class=fn>{@link #createEncoderByType createEncoderByType}</td> 894 <td>●</td> 895 <td>●</td> 896 <td>●</td> 897 <td>●</td> 898 <td>●</td> 899 <td>●</td> 900 <td>●</td> 901 <td>●</td> 902 </tr> 903 <tr> 904 <td></td> 905 <td></td> 906 <td></td> 907 <td></td> 908 <td></td> 909 <td></td> 910 <td></td> 911 <td class=fn>{@link #createPersistentInputSurface createPersistentInputSurface}</td> 912 <td></td> 913 <td></td> 914 <td></td> 915 <td></td> 916 <td></td> 917 <td></td> 918 <td></td> 919 <td>●</td> 920 </tr> 921 <tr> 922 <td>16+</td> 923 <td>-</td> 924 <td>-</td> 925 <td>-</td> 926 <td>-</td> 927 <td>-</td> 928 <td>-</td> 929 <td class=fn>{@link #configure configure}</td> 930 <td>●</td> 931 <td>●</td> 932 <td>●</td> 933 <td>●</td> 934 <td>●</td> 935 <td>⁕</td> 936 <td>●</td> 937 <td>●</td> 938 </tr> 939 <tr> 940 <td>-</td> 941 <td>18+</td> 942 <td>-</td> 943 <td>-</td> 944 <td>-</td> 945 <td>-</td> 946 <td>-</td> 947 <td class=fn>{@link #createInputSurface createInputSurface}</td> 948 <td></td> 949 <td></td> 950 <td>⎋</td> 951 <td>⎋</td> 952 <td>⎋</td> 953 <td>⎋</td> 954 <td>⎋</td> 955 <td>⎋</td> 956 </tr> 957 <tr> 958 <td>-</td> 959 <td>-</td> 960 <td>16+</td> 961 <td>16+</td> 962 <td>(16+)</td> 963 <td>-</td> 964 <td>-</td> 965 <td class=fn>{@link #dequeueInputBuffer dequeueInputBuffer}</td> 966 <td>●</td> 967 <td>●</td> 968 <td>▧</td> 969 <td>▧</td> 970 <td>▧</td> 971 <td>⁕▧↩</td> 972 <td>▧↩</td> 973 <td>▧↩</td> 974 </tr> 975 <tr> 976 <td>-</td> 977 <td>-</td> 978 <td>16+</td> 979 <td>16+</td> 980 <td>16+</td> 981 <td>-</td> 982 <td>-</td> 983 <td class=fn>{@link #dequeueOutputBuffer dequeueOutputBuffer}</td> 984 <td>●</td> 985 <td>●</td> 986 <td>●</td> 987 <td>●</td> 988 <td>●</td> 989 <td>⁕↩</td> 990 <td>↩</td> 991 <td>↩</td> 992 </tr> 993 <tr> 994 <td>-</td> 995 <td>-</td> 996 <td>16+</td> 997 <td>16+</td> 998 <td>16+</td> 999 <td>-</td> 1000 <td>-</td> 1001 <td class=fn>{@link #flush flush}</td> 1002 <td>●</td> 1003 <td>●</td> 1004 <td>●</td> 1005 <td>●</td> 1006 <td>●</td> 1007 <td>●</td> 1008 <td>●</td> 1009 <td>●</td> 1010 </tr> 1011 <tr> 1012 <td>18+</td> 1013 <td>18+</td> 1014 <td>18+</td> 1015 <td>18+</td> 1016 <td>18+</td> 1017 <td>18+</td> 1018 <td>-</td> 1019 <td class=fn>{@link #getCodecInfo getCodecInfo}</td> 1020 <td></td> 1021 <td></td> 1022 <td>●</td> 1023 <td>●</td> 1024 <td>●</td> 1025 <td>●</td> 1026 <td>●</td> 1027 <td>●</td> 1028 </tr> 1029 <tr> 1030 <td>-</td> 1031 <td>-</td> 1032 <td>(21+)</td> 1033 <td>21+</td> 1034 <td>(21+)</td> 1035 <td>-</td> 1036 <td>-</td> 1037 <td class=fn>{@link #getInputBuffer getInputBuffer}</td> 1038 <td></td> 1039 <td></td> 1040 <td></td> 1041 <td></td> 1042 <td></td> 1043 <td>●</td> 1044 <td>●</td> 1045 <td>●</td> 1046 </tr> 1047 <tr> 1048 <td>-</td> 1049 <td>-</td> 1050 <td>16+</td> 1051 <td>(16+)</td> 1052 <td>(16+)</td> 1053 <td>-</td> 1054 <td>-</td> 1055 <td class=fn>{@link #getInputBuffers getInputBuffers}</td> 1056 <td>●</td> 1057 <td>●</td> 1058 <td>●</td> 1059 <td>●</td> 1060 <td>●</td> 1061 <td>[⁕↩]</td> 1062 <td>[↩]</td> 1063 <td>[↩]</td> 1064 </tr> 1065 <tr> 1066 <td>-</td> 1067 <td>21+</td> 1068 <td>(21+)</td> 1069 <td>(21+)</td> 1070 <td>(21+)</td> 1071 <td>-</td> 1072 <td>-</td> 1073 <td class=fn>{@link #getInputFormat getInputFormat}</td> 1074 <td></td> 1075 <td></td> 1076 <td></td> 1077 <td></td> 1078 <td></td> 1079 <td>●</td> 1080 <td>●</td> 1081 <td>●</td> 1082 </tr> 1083 <tr> 1084 <td>-</td> 1085 <td>-</td> 1086 <td>(21+)</td> 1087 <td>21+</td> 1088 <td>(21+)</td> 1089 <td>-</td> 1090 <td>-</td> 1091 <td class=fn>{@link #getInputImage getInputImage}</td> 1092 <td></td> 1093 <td></td> 1094 <td></td> 1095 <td></td> 1096 <td></td> 1097 <td>○</td> 1098 <td>●</td> 1099 <td>●</td> 1100 </tr> 1101 <tr> 1102 <td>18+</td> 1103 <td>18+</td> 1104 <td>18+</td> 1105 <td>18+</td> 1106 <td>18+</td> 1107 <td>18+</td> 1108 <td>-</td> 1109 <td class=fn>{@link #getName getName}</td> 1110 <td></td> 1111 <td></td> 1112 <td>●</td> 1113 <td>●</td> 1114 <td>●</td> 1115 <td>●</td> 1116 <td>●</td> 1117 <td>●</td> 1118 </tr> 1119 <tr> 1120 <td>-</td> 1121 <td>-</td> 1122 <td>(21+)</td> 1123 <td>21+</td> 1124 <td>21+</td> 1125 <td>-</td> 1126 <td>-</td> 1127 <td class=fn>{@link #getOutputBuffer getOutputBuffer}</td> 1128 <td></td> 1129 <td></td> 1130 <td></td> 1131 <td></td> 1132 <td></td> 1133 <td>●</td> 1134 <td>●</td> 1135 <td>●</td> 1136 </tr> 1137 <tr> 1138 <td>-</td> 1139 <td>-</td> 1140 <td>16+</td> 1141 <td>16+</td> 1142 <td>16+</td> 1143 <td>-</td> 1144 <td>-</td> 1145 <td class=fn>{@link #getOutputBuffers getOutputBuffers}</td> 1146 <td>●</td> 1147 <td>●</td> 1148 <td>●</td> 1149 <td>●</td> 1150 <td>●</td> 1151 <td>[⁕↩]</td> 1152 <td>[↩]</td> 1153 <td>[↩]</td> 1154 </tr> 1155 <tr> 1156 <td>-</td> 1157 <td>21+</td> 1158 <td>16+</td> 1159 <td>16+</td> 1160 <td>16+</td> 1161 <td>-</td> 1162 <td>-</td> 1163 <td class=fn>{@link #getOutputFormat()}</td> 1164 <td>●</td> 1165 <td>●</td> 1166 <td>●</td> 1167 <td>●</td> 1168 <td>●</td> 1169 <td>●</td> 1170 <td>●</td> 1171 <td>●</td> 1172 </tr> 1173 <tr> 1174 <td>-</td> 1175 <td>-</td> 1176 <td>(21+)</td> 1177 <td>21+</td> 1178 <td>21+</td> 1179 <td>-</td> 1180 <td>-</td> 1181 <td class=fn>{@link #getOutputFormat(int)}</td> 1182 <td></td> 1183 <td></td> 1184 <td></td> 1185 <td></td> 1186 <td></td> 1187 <td>●</td> 1188 <td>●</td> 1189 <td>●</td> 1190 </tr> 1191 <tr> 1192 <td>-</td> 1193 <td>-</td> 1194 <td>(21+)</td> 1195 <td>21+</td> 1196 <td>21+</td> 1197 <td>-</td> 1198 <td>-</td> 1199 <td class=fn>{@link #getOutputImage getOutputImage}</td> 1200 <td></td> 1201 <td></td> 1202 <td></td> 1203 <td></td> 1204 <td></td> 1205 <td>○</td> 1206 <td>●</td> 1207 <td>●</td> 1208 </tr> 1209 <tr> 1210 <td>-</td> 1211 <td>-</td> 1212 <td>-</td> 1213 <td>16+</td> 1214 <td>(16+)</td> 1215 <td>-</td> 1216 <td>-</td> 1217 <td class=fn>{@link #queueInputBuffer queueInputBuffer}</td> 1218 <td>●</td> 1219 <td>●</td> 1220 <td>●</td> 1221 <td>●</td> 1222 <td>●</td> 1223 <td>⁕</td> 1224 <td>●</td> 1225 <td>●</td> 1226 </tr> 1227 <tr> 1228 <td>-</td> 1229 <td>-</td> 1230 <td>-</td> 1231 <td>16+</td> 1232 <td>(16+)</td> 1233 <td>-</td> 1234 <td>-</td> 1235 <td class=fn>{@link #queueSecureInputBuffer queueSecureInputBuffer}</td> 1236 <td>●</td> 1237 <td>●</td> 1238 <td>●</td> 1239 <td>●</td> 1240 <td>●</td> 1241 <td>⁕</td> 1242 <td>●</td> 1243 <td>●</td> 1244 </tr> 1245 <tr> 1246 <td>16+</td> 1247 <td>16+</td> 1248 <td>16+</td> 1249 <td>16+</td> 1250 <td>16+</td> 1251 <td>16+</td> 1252 <td>16+</td> 1253 <td class=fn>{@link #release release}</td> 1254 <td>●</td> 1255 <td>●</td> 1256 <td>●</td> 1257 <td>●</td> 1258 <td>●</td> 1259 <td>●</td> 1260 <td>●</td> 1261 <td>●</td> 1262 </tr> 1263 <tr> 1264 <td>-</td> 1265 <td>-</td> 1266 <td>-</td> 1267 <td>16+</td> 1268 <td>16+</td> 1269 <td>-</td> 1270 <td>-</td> 1271 <td class=fn>{@link #releaseOutputBuffer(int, boolean)}</td> 1272 <td>●</td> 1273 <td>●</td> 1274 <td>●</td> 1275 <td>●</td> 1276 <td>●</td> 1277 <td>⁕</td> 1278 <td>●</td> 1279 <td>⁕</td> 1280 </tr> 1281 <tr> 1282 <td>-</td> 1283 <td>-</td> 1284 <td>-</td> 1285 <td>21+</td> 1286 <td>21+</td> 1287 <td>-</td> 1288 <td>-</td> 1289 <td class=fn>{@link #releaseOutputBuffer(int, long)}</td> 1290 <td></td> 1291 <td></td> 1292 <td></td> 1293 <td></td> 1294 <td></td> 1295 <td>⎆</td> 1296 <td>⎆</td> 1297 <td>⎆</td> 1298 </tr> 1299 <tr> 1300 <td>21+</td> 1301 <td>21+</td> 1302 <td>21+</td> 1303 <td>21+</td> 1304 <td>21+</td> 1305 <td>21+</td> 1306 <td>-</td> 1307 <td class=fn>{@link #reset reset}</td> 1308 <td></td> 1309 <td></td> 1310 <td></td> 1311 <td></td> 1312 <td></td> 1313 <td>●</td> 1314 <td>●</td> 1315 <td>●</td> 1316 </tr> 1317 <tr> 1318 <td>21+</td> 1319 <td>-</td> 1320 <td>-</td> 1321 <td>-</td> 1322 <td>-</td> 1323 <td>-</td> 1324 <td>-</td> 1325 <td class=fn>{@link #setCallback(Callback) setCallback}</td> 1326 <td></td> 1327 <td></td> 1328 <td></td> 1329 <td></td> 1330 <td></td> 1331 <td>●</td> 1332 <td>●</td> 1333 <td>{@link #setCallback(Callback, Handler) ⁕}</td> 1334 </tr> 1335 <tr> 1336 <td>-</td> 1337 <td>23+</td> 1338 <td>-</td> 1339 <td>-</td> 1340 <td>-</td> 1341 <td>-</td> 1342 <td>-</td> 1343 <td class=fn>{@link #setInputSurface setInputSurface}</td> 1344 <td></td> 1345 <td></td> 1346 <td></td> 1347 <td></td> 1348 <td></td> 1349 <td></td> 1350 <td></td> 1351 <td>⎋</td> 1352 </tr> 1353 <tr> 1354 <td>23+</td> 1355 <td>23+</td> 1356 <td>23+</td> 1357 <td>23+</td> 1358 <td>23+</td> 1359 <td>(23+)</td> 1360 <td>(23+)</td> 1361 <td class=fn>{@link #setOnFrameRenderedListener setOnFrameRenderedListener}</td> 1362 <td></td> 1363 <td></td> 1364 <td></td> 1365 <td></td> 1366 <td></td> 1367 <td></td> 1368 <td></td> 1369 <td>○ ⎆</td> 1370 </tr> 1371 <tr> 1372 <td>-</td> 1373 <td>23+</td> 1374 <td>23+</td> 1375 <td>23+</td> 1376 <td>23+</td> 1377 <td>-</td> 1378 <td>-</td> 1379 <td class=fn>{@link #setOutputSurface setOutputSurface}</td> 1380 <td></td> 1381 <td></td> 1382 <td></td> 1383 <td></td> 1384 <td></td> 1385 <td></td> 1386 <td></td> 1387 <td>⎆</td> 1388 </tr> 1389 <tr> 1390 <td>19+</td> 1391 <td>19+</td> 1392 <td>19+</td> 1393 <td>19+</td> 1394 <td>19+</td> 1395 <td>(19+)</td> 1396 <td>-</td> 1397 <td class=fn>{@link #setParameters setParameters}</td> 1398 <td></td> 1399 <td></td> 1400 <td></td> 1401 <td>●</td> 1402 <td>●</td> 1403 <td>●</td> 1404 <td>●</td> 1405 <td>●</td> 1406 </tr> 1407 <tr> 1408 <td>-</td> 1409 <td>(16+)</td> 1410 <td>(16+)</td> 1411 <td>16+</td> 1412 <td>(16+)</td> 1413 <td>(16+)</td> 1414 <td>-</td> 1415 <td class=fn>{@link #setVideoScalingMode setVideoScalingMode}</td> 1416 <td>⎆</td> 1417 <td>⎆</td> 1418 <td>⎆</td> 1419 <td>⎆</td> 1420 <td>⎆</td> 1421 <td>⎆</td> 1422 <td>⎆</td> 1423 <td>⎆</td> 1424 </tr> 1425 <tr> 1426 <td>-</td> 1427 <td>-</td> 1428 <td>18+</td> 1429 <td>18+</td> 1430 <td>-</td> 1431 <td>-</td> 1432 <td>-</td> 1433 <td class=fn>{@link #signalEndOfInputStream signalEndOfInputStream}</td> 1434 <td></td> 1435 <td></td> 1436 <td>⎋</td> 1437 <td>⎋</td> 1438 <td>⎋</td> 1439 <td>⎋</td> 1440 <td>⎋</td> 1441 <td>⎋</td> 1442 </tr> 1443 <tr> 1444 <td>-</td> 1445 <td>16+</td> 1446 <td>21+(⇄)</td> 1447 <td>-</td> 1448 <td>-</td> 1449 <td>-</td> 1450 <td>-</td> 1451 <td class=fn>{@link #start start}</td> 1452 <td>●</td> 1453 <td>●</td> 1454 <td>●</td> 1455 <td>●</td> 1456 <td>●</td> 1457 <td>⁕</td> 1458 <td>●</td> 1459 <td>●</td> 1460 </tr> 1461 <tr> 1462 <td>-</td> 1463 <td>-</td> 1464 <td>16+</td> 1465 <td>16+</td> 1466 <td>16+</td> 1467 <td>-</td> 1468 <td>-</td> 1469 <td class=fn>{@link #stop stop}</td> 1470 <td>●</td> 1471 <td>●</td> 1472 <td>●</td> 1473 <td>●</td> 1474 <td>●</td> 1475 <td>●</td> 1476 <td>●</td> 1477 <td>●</td> 1478 </tr> 1479 </tbody> 1480 </table> 1481 */ 1482 final public class MediaCodec { 1483 /** 1484 * Per buffer metadata includes an offset and size specifying 1485 * the range of valid data in the associated codec (output) buffer. 1486 */ 1487 public final static class BufferInfo { 1488 /** 1489 * Update the buffer metadata information. 1490 * 1491 * @param newOffset the start-offset of the data in the buffer. 1492 * @param newSize the amount of data (in bytes) in the buffer. 1493 * @param newTimeUs the presentation timestamp in microseconds. 1494 * @param newFlags buffer flags associated with the buffer. This 1495 * should be a combination of {@link #BUFFER_FLAG_KEY_FRAME} and 1496 * {@link #BUFFER_FLAG_END_OF_STREAM}. 1497 */ set( int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags)1498 public void set( 1499 int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags) { 1500 offset = newOffset; 1501 size = newSize; 1502 presentationTimeUs = newTimeUs; 1503 flags = newFlags; 1504 } 1505 1506 /** 1507 * The start-offset of the data in the buffer. 1508 */ 1509 public int offset; 1510 1511 /** 1512 * The amount of data (in bytes) in the buffer. If this is {@code 0}, 1513 * the buffer has no data in it and can be discarded. The only 1514 * use of a 0-size buffer is to carry the end-of-stream marker. 1515 */ 1516 public int size; 1517 1518 /** 1519 * The presentation timestamp in microseconds for the buffer. 1520 * This is derived from the presentation timestamp passed in 1521 * with the corresponding input buffer. This should be ignored for 1522 * a 0-sized buffer. 1523 */ 1524 public long presentationTimeUs; 1525 1526 /** 1527 * Buffer flags associated with the buffer. A combination of 1528 * {@link #BUFFER_FLAG_KEY_FRAME} and {@link #BUFFER_FLAG_END_OF_STREAM}. 1529 * 1530 * <p>Encoded buffers that are key frames are marked with 1531 * {@link #BUFFER_FLAG_KEY_FRAME}. 1532 * 1533 * <p>The last output buffer corresponding to the input buffer 1534 * marked with {@link #BUFFER_FLAG_END_OF_STREAM} will also be marked 1535 * with {@link #BUFFER_FLAG_END_OF_STREAM}. In some cases this could 1536 * be an empty buffer, whose sole purpose is to carry the end-of-stream 1537 * marker. 1538 */ 1539 @BufferFlag 1540 public int flags; 1541 1542 /** @hide */ 1543 @NonNull dup()1544 public BufferInfo dup() { 1545 BufferInfo copy = new BufferInfo(); 1546 copy.set(offset, size, presentationTimeUs, flags); 1547 return copy; 1548 } 1549 }; 1550 1551 // The follow flag constants MUST stay in sync with their equivalents 1552 // in MediaCodec.h ! 1553 1554 /** 1555 * This indicates that the (encoded) buffer marked as such contains 1556 * the data for a key frame. 1557 * 1558 * @deprecated Use {@link #BUFFER_FLAG_KEY_FRAME} instead. 1559 */ 1560 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 1561 1562 /** 1563 * This indicates that the (encoded) buffer marked as such contains 1564 * the data for a key frame. 1565 */ 1566 public static final int BUFFER_FLAG_KEY_FRAME = 1; 1567 1568 /** 1569 * This indicated that the buffer marked as such contains codec 1570 * initialization / codec specific data instead of media data. 1571 */ 1572 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 1573 1574 /** 1575 * This signals the end of stream, i.e. no buffers will be available 1576 * after this, unless of course, {@link #flush} follows. 1577 */ 1578 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 1579 1580 /** 1581 * This indicates that the buffer only contains part of a frame, 1582 * and the decoder should batch the data until a buffer without 1583 * this flag appears before decoding the frame. 1584 */ 1585 public static final int BUFFER_FLAG_PARTIAL_FRAME = 8; 1586 1587 /** 1588 * This indicates that the buffer contains non-media data for the 1589 * muxer to process. 1590 * 1591 * All muxer data should start with a FOURCC header that determines the type of data. 1592 * 1593 * For example, when it contains Exif data sent to a MediaMuxer track of 1594 * {@link MediaFormat#MIMETYPE_IMAGE_ANDROID_HEIC} type, the data must start with 1595 * Exif header ("Exif\0\0"), followed by the TIFF header (See JEITA CP-3451C Section 4.5.2.) 1596 * 1597 * @hide 1598 */ 1599 public static final int BUFFER_FLAG_MUXER_DATA = 16; 1600 1601 /** @hide */ 1602 @IntDef( 1603 flag = true, 1604 value = { 1605 BUFFER_FLAG_SYNC_FRAME, 1606 BUFFER_FLAG_KEY_FRAME, 1607 BUFFER_FLAG_CODEC_CONFIG, 1608 BUFFER_FLAG_END_OF_STREAM, 1609 BUFFER_FLAG_PARTIAL_FRAME, 1610 BUFFER_FLAG_MUXER_DATA, 1611 }) 1612 @Retention(RetentionPolicy.SOURCE) 1613 public @interface BufferFlag {} 1614 1615 private EventHandler mEventHandler; 1616 private EventHandler mOnFrameRenderedHandler; 1617 private EventHandler mCallbackHandler; 1618 private Callback mCallback; 1619 private OnFrameRenderedListener mOnFrameRenderedListener; 1620 private final Object mListenerLock = new Object(); 1621 private MediaCodecInfo mCodecInfo; 1622 private final Object mCodecInfoLock = new Object(); 1623 1624 private static final int EVENT_CALLBACK = 1; 1625 private static final int EVENT_SET_CALLBACK = 2; 1626 private static final int EVENT_FRAME_RENDERED = 3; 1627 1628 private static final int CB_INPUT_AVAILABLE = 1; 1629 private static final int CB_OUTPUT_AVAILABLE = 2; 1630 private static final int CB_ERROR = 3; 1631 private static final int CB_OUTPUT_FORMAT_CHANGE = 4; 1632 1633 private class EventHandler extends Handler { 1634 private MediaCodec mCodec; 1635 EventHandler(@onNull MediaCodec codec, @NonNull Looper looper)1636 public EventHandler(@NonNull MediaCodec codec, @NonNull Looper looper) { 1637 super(looper); 1638 mCodec = codec; 1639 } 1640 1641 @Override handleMessage(@onNull Message msg)1642 public void handleMessage(@NonNull Message msg) { 1643 switch (msg.what) { 1644 case EVENT_CALLBACK: 1645 { 1646 handleCallback(msg); 1647 break; 1648 } 1649 case EVENT_SET_CALLBACK: 1650 { 1651 mCallback = (MediaCodec.Callback) msg.obj; 1652 break; 1653 } 1654 case EVENT_FRAME_RENDERED: 1655 synchronized (mListenerLock) { 1656 Map<String, Object> map = (Map<String, Object>)msg.obj; 1657 for (int i = 0; ; ++i) { 1658 Object mediaTimeUs = map.get(i + "-media-time-us"); 1659 Object systemNano = map.get(i + "-system-nano"); 1660 if (mediaTimeUs == null || systemNano == null 1661 || mOnFrameRenderedListener == null) { 1662 break; 1663 } 1664 mOnFrameRenderedListener.onFrameRendered( 1665 mCodec, (long)mediaTimeUs, (long)systemNano); 1666 } 1667 break; 1668 } 1669 default: 1670 { 1671 break; 1672 } 1673 } 1674 } 1675 handleCallback(@onNull Message msg)1676 private void handleCallback(@NonNull Message msg) { 1677 if (mCallback == null) { 1678 return; 1679 } 1680 1681 switch (msg.arg1) { 1682 case CB_INPUT_AVAILABLE: 1683 { 1684 int index = msg.arg2; 1685 synchronized(mBufferLock) { 1686 validateInputByteBuffer(mCachedInputBuffers, index); 1687 } 1688 mCallback.onInputBufferAvailable(mCodec, index); 1689 break; 1690 } 1691 1692 case CB_OUTPUT_AVAILABLE: 1693 { 1694 int index = msg.arg2; 1695 BufferInfo info = (MediaCodec.BufferInfo) msg.obj; 1696 synchronized(mBufferLock) { 1697 validateOutputByteBuffer(mCachedOutputBuffers, index, info); 1698 } 1699 mCallback.onOutputBufferAvailable( 1700 mCodec, index, info); 1701 break; 1702 } 1703 1704 case CB_ERROR: 1705 { 1706 mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj); 1707 break; 1708 } 1709 1710 case CB_OUTPUT_FORMAT_CHANGE: 1711 { 1712 mCallback.onOutputFormatChanged(mCodec, 1713 new MediaFormat((Map<String, Object>) msg.obj)); 1714 break; 1715 } 1716 1717 default: 1718 { 1719 break; 1720 } 1721 } 1722 } 1723 } 1724 1725 private boolean mHasSurface = false; 1726 1727 /** 1728 * Instantiate the preferred decoder supporting input data of the given mime type. 1729 * 1730 * The following is a partial list of defined mime types and their semantics: 1731 * <ul> 1732 * <li>"video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm) 1733 * <li>"video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm) 1734 * <li>"video/avc" - H.264/AVC video 1735 * <li>"video/hevc" - H.265/HEVC video 1736 * <li>"video/mp4v-es" - MPEG4 video 1737 * <li>"video/3gpp" - H.263 video 1738 * <li>"audio/3gpp" - AMR narrowband audio 1739 * <li>"audio/amr-wb" - AMR wideband audio 1740 * <li>"audio/mpeg" - MPEG1/2 audio layer III 1741 * <li>"audio/mp4a-latm" - AAC audio (note, this is raw AAC packets, not packaged in LATM!) 1742 * <li>"audio/vorbis" - vorbis audio 1743 * <li>"audio/g711-alaw" - G.711 alaw audio 1744 * <li>"audio/g711-mlaw" - G.711 ulaw audio 1745 * </ul> 1746 * 1747 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findDecoderForFormat} 1748 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1749 * given format. 1750 * 1751 * @param type The mime type of the input data. 1752 * @throws IOException if the codec cannot be created. 1753 * @throws IllegalArgumentException if type is not a valid mime type. 1754 * @throws NullPointerException if type is null. 1755 */ 1756 @NonNull createDecoderByType(@onNull String type)1757 public static MediaCodec createDecoderByType(@NonNull String type) 1758 throws IOException { 1759 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 1760 } 1761 1762 /** 1763 * Instantiate the preferred encoder supporting output data of the given mime type. 1764 * 1765 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findEncoderForFormat} 1766 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1767 * given format. 1768 * 1769 * @param type The desired mime type of the output data. 1770 * @throws IOException if the codec cannot be created. 1771 * @throws IllegalArgumentException if type is not a valid mime type. 1772 * @throws NullPointerException if type is null. 1773 */ 1774 @NonNull createEncoderByType(@onNull String type)1775 public static MediaCodec createEncoderByType(@NonNull String type) 1776 throws IOException { 1777 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 1778 } 1779 1780 /** 1781 * If you know the exact name of the component you want to instantiate 1782 * use this method to instantiate it. Use with caution. 1783 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 1784 * @param name The name of the codec to be instantiated. 1785 * @throws IOException if the codec cannot be created. 1786 * @throws IllegalArgumentException if name is not valid. 1787 * @throws NullPointerException if name is null. 1788 */ 1789 @NonNull createByCodecName(@onNull String name)1790 public static MediaCodec createByCodecName(@NonNull String name) 1791 throws IOException { 1792 return new MediaCodec( 1793 name, false /* nameIsType */, false /* unused */); 1794 } 1795 MediaCodec( @onNull String name, boolean nameIsType, boolean encoder)1796 private MediaCodec( 1797 @NonNull String name, boolean nameIsType, boolean encoder) { 1798 Looper looper; 1799 if ((looper = Looper.myLooper()) != null) { 1800 mEventHandler = new EventHandler(this, looper); 1801 } else if ((looper = Looper.getMainLooper()) != null) { 1802 mEventHandler = new EventHandler(this, looper); 1803 } else { 1804 mEventHandler = null; 1805 } 1806 mCallbackHandler = mEventHandler; 1807 mOnFrameRenderedHandler = mEventHandler; 1808 1809 mBufferLock = new Object(); 1810 1811 native_setup(name, nameIsType, encoder); 1812 } 1813 1814 @Override finalize()1815 protected void finalize() { 1816 native_finalize(); 1817 } 1818 1819 /** 1820 * Returns the codec to its initial (Uninitialized) state. 1821 * 1822 * Call this if an {@link MediaCodec.CodecException#isRecoverable unrecoverable} 1823 * error has occured to reset the codec to its initial state after creation. 1824 * 1825 * @throws CodecException if an unrecoverable error has occured and the codec 1826 * could not be reset. 1827 * @throws IllegalStateException if in the Released state. 1828 */ reset()1829 public final void reset() { 1830 freeAllTrackedBuffers(); // free buffers first 1831 native_reset(); 1832 } 1833 native_reset()1834 private native final void native_reset(); 1835 1836 /** 1837 * Free up resources used by the codec instance. 1838 * 1839 * Make sure you call this when you're done to free up any opened 1840 * component instance instead of relying on the garbage collector 1841 * to do this for you at some point in the future. 1842 */ release()1843 public final void release() { 1844 freeAllTrackedBuffers(); // free buffers first 1845 native_release(); 1846 } 1847 native_release()1848 private native final void native_release(); 1849 1850 /** 1851 * If this codec is to be used as an encoder, pass this flag. 1852 */ 1853 public static final int CONFIGURE_FLAG_ENCODE = 1; 1854 1855 /** @hide */ 1856 @IntDef(flag = true, value = { CONFIGURE_FLAG_ENCODE }) 1857 @Retention(RetentionPolicy.SOURCE) 1858 public @interface ConfigureFlag {} 1859 1860 /** 1861 * Configures a component. 1862 * 1863 * @param format The format of the input data (decoder) or the desired 1864 * format of the output data (encoder). Passing {@code null} 1865 * as {@code format} is equivalent to passing an 1866 * {@link MediaFormat#MediaFormat an empty mediaformat}. 1867 * @param surface Specify a surface on which to render the output of this 1868 * decoder. Pass {@code null} as {@code surface} if the 1869 * codec does not generate raw video output (e.g. not a video 1870 * decoder) and/or if you want to configure the codec for 1871 * {@link ByteBuffer} output. 1872 * @param crypto Specify a crypto object to facilitate secure decryption 1873 * of the media data. Pass {@code null} as {@code crypto} for 1874 * non-secure codecs. 1875 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 1876 * component as an encoder. 1877 * @throws IllegalArgumentException if the surface has been released (or is invalid), 1878 * or the format is unacceptable (e.g. missing a mandatory key), 1879 * or the flags are not set properly 1880 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 1881 * @throws IllegalStateException if not in the Uninitialized state. 1882 * @throws CryptoException upon DRM error. 1883 * @throws CodecException upon codec error. 1884 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @ConfigureFlag int flags)1885 public void configure( 1886 @Nullable MediaFormat format, 1887 @Nullable Surface surface, @Nullable MediaCrypto crypto, 1888 @ConfigureFlag int flags) { 1889 configure(format, surface, crypto, null, flags); 1890 } 1891 1892 /** 1893 * Configure a component to be used with a descrambler. 1894 * @param format The format of the input data (decoder) or the desired 1895 * format of the output data (encoder). Passing {@code null} 1896 * as {@code format} is equivalent to passing an 1897 * {@link MediaFormat#MediaFormat an empty mediaformat}. 1898 * @param surface Specify a surface on which to render the output of this 1899 * decoder. Pass {@code null} as {@code surface} if the 1900 * codec does not generate raw video output (e.g. not a video 1901 * decoder) and/or if you want to configure the codec for 1902 * {@link ByteBuffer} output. 1903 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 1904 * component as an encoder. 1905 * @param descrambler Specify a descrambler object to facilitate secure 1906 * descrambling of the media data, or null for non-secure codecs. 1907 * @throws IllegalArgumentException if the surface has been released (or is invalid), 1908 * or the format is unacceptable (e.g. missing a mandatory key), 1909 * or the flags are not set properly 1910 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 1911 * @throws IllegalStateException if not in the Uninitialized state. 1912 * @throws CryptoException upon DRM error. 1913 * @throws CodecException upon codec error. 1914 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler)1915 public void configure( 1916 @Nullable MediaFormat format, @Nullable Surface surface, 1917 @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler) { 1918 configure(format, surface, null, 1919 descrambler != null ? descrambler.getBinder() : null, flags); 1920 } 1921 configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)1922 private void configure( 1923 @Nullable MediaFormat format, @Nullable Surface surface, 1924 @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, 1925 @ConfigureFlag int flags) { 1926 if (crypto != null && descramblerBinder != null) { 1927 throw new IllegalArgumentException("Can't use crypto and descrambler together!"); 1928 } 1929 1930 String[] keys = null; 1931 Object[] values = null; 1932 1933 if (format != null) { 1934 Map<String, Object> formatMap = format.getMap(); 1935 keys = new String[formatMap.size()]; 1936 values = new Object[formatMap.size()]; 1937 1938 int i = 0; 1939 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 1940 if (entry.getKey().equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 1941 int sessionId = 0; 1942 try { 1943 sessionId = (Integer)entry.getValue(); 1944 } 1945 catch (Exception e) { 1946 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 1947 } 1948 keys[i] = "audio-hw-sync"; 1949 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 1950 } else { 1951 keys[i] = entry.getKey(); 1952 values[i] = entry.getValue(); 1953 } 1954 ++i; 1955 } 1956 } 1957 1958 mHasSurface = surface != null; 1959 1960 native_configure(keys, values, surface, crypto, descramblerBinder, flags); 1961 } 1962 1963 /** 1964 * Dynamically sets the output surface of a codec. 1965 * <p> 1966 * This can only be used if the codec was configured with an output surface. The 1967 * new output surface should have a compatible usage type to the original output surface. 1968 * E.g. codecs may not support switching from a SurfaceTexture (GPU readable) output 1969 * to ImageReader (software readable) output. 1970 * @param surface the output surface to use. It must not be {@code null}. 1971 * @throws IllegalStateException if the codec does not support setting the output 1972 * surface in the current state. 1973 * @throws IllegalArgumentException if the new surface is not of a suitable type for the codec. 1974 */ setOutputSurface(@onNull Surface surface)1975 public void setOutputSurface(@NonNull Surface surface) { 1976 if (!mHasSurface) { 1977 throw new IllegalStateException("codec was not configured for an output surface"); 1978 } 1979 native_setSurface(surface); 1980 } 1981 native_setSurface(@onNull Surface surface)1982 private native void native_setSurface(@NonNull Surface surface); 1983 1984 /** 1985 * Create a persistent input surface that can be used with codecs that normally have an input 1986 * surface, such as video encoders. A persistent input can be reused by subsequent 1987 * {@link MediaCodec} or {@link MediaRecorder} instances, but can only be used by at 1988 * most one codec or recorder instance concurrently. 1989 * <p> 1990 * The application is responsible for calling release() on the Surface when done. 1991 * 1992 * @return an input surface that can be used with {@link #setInputSurface}. 1993 */ 1994 @NonNull createPersistentInputSurface()1995 public static Surface createPersistentInputSurface() { 1996 return native_createPersistentInputSurface(); 1997 } 1998 1999 static class PersistentSurface extends Surface { 2000 @SuppressWarnings("unused") PersistentSurface()2001 PersistentSurface() {} // used by native 2002 2003 @Override release()2004 public void release() { 2005 native_releasePersistentInputSurface(this); 2006 super.release(); 2007 } 2008 2009 private long mPersistentObject; 2010 }; 2011 2012 /** 2013 * Configures the codec (e.g. encoder) to use a persistent input surface in place of input 2014 * buffers. This may only be called after {@link #configure} and before {@link #start}, in 2015 * lieu of {@link #createInputSurface}. 2016 * @param surface a persistent input surface created by {@link #createPersistentInputSurface} 2017 * @throws IllegalStateException if not in the Configured state or does not require an input 2018 * surface. 2019 * @throws IllegalArgumentException if the surface was not created by 2020 * {@link #createPersistentInputSurface}. 2021 */ setInputSurface(@onNull Surface surface)2022 public void setInputSurface(@NonNull Surface surface) { 2023 if (!(surface instanceof PersistentSurface)) { 2024 throw new IllegalArgumentException("not a PersistentSurface"); 2025 } 2026 native_setInputSurface(surface); 2027 } 2028 2029 @NonNull native_createPersistentInputSurface()2030 private static native final PersistentSurface native_createPersistentInputSurface(); native_releasePersistentInputSurface(@onNull Surface surface)2031 private static native final void native_releasePersistentInputSurface(@NonNull Surface surface); native_setInputSurface(@onNull Surface surface)2032 private native final void native_setInputSurface(@NonNull Surface surface); 2033 native_setCallback(@ullable Callback cb)2034 private native final void native_setCallback(@Nullable Callback cb); 2035 native_configure( @ullable String[] keys, @Nullable Object[] values, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2036 private native final void native_configure( 2037 @Nullable String[] keys, @Nullable Object[] values, 2038 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2039 @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags); 2040 2041 /** 2042 * Requests a Surface to use as the input to an encoder, in place of input buffers. This 2043 * may only be called after {@link #configure} and before {@link #start}. 2044 * <p> 2045 * The application is responsible for calling release() on the Surface when 2046 * done. 2047 * <p> 2048 * The Surface must be rendered with a hardware-accelerated API, such as OpenGL ES. 2049 * {@link android.view.Surface#lockCanvas(android.graphics.Rect)} may fail or produce 2050 * unexpected results. 2051 * @throws IllegalStateException if not in the Configured state. 2052 */ 2053 @NonNull createInputSurface()2054 public native final Surface createInputSurface(); 2055 2056 /** 2057 * After successfully configuring the component, call {@code start}. 2058 * <p> 2059 * Call {@code start} also if the codec is configured in asynchronous mode, 2060 * and it has just been flushed, to resume requesting input buffers. 2061 * @throws IllegalStateException if not in the Configured state 2062 * or just after {@link #flush} for a codec that is configured 2063 * in asynchronous mode. 2064 * @throws MediaCodec.CodecException upon codec error. Note that some codec errors 2065 * for start may be attributed to future method calls. 2066 */ start()2067 public final void start() { 2068 native_start(); 2069 synchronized(mBufferLock) { 2070 cacheBuffers(true /* input */); 2071 cacheBuffers(false /* input */); 2072 } 2073 } native_start()2074 private native final void native_start(); 2075 2076 /** 2077 * Finish the decode/encode session, note that the codec instance 2078 * remains active and ready to be {@link #start}ed again. 2079 * To ensure that it is available to other client call {@link #release} 2080 * and don't just rely on garbage collection to eventually do this for you. 2081 * @throws IllegalStateException if in the Released state. 2082 */ stop()2083 public final void stop() { 2084 native_stop(); 2085 freeAllTrackedBuffers(); 2086 2087 synchronized (mListenerLock) { 2088 if (mCallbackHandler != null) { 2089 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 2090 mCallbackHandler.removeMessages(EVENT_CALLBACK); 2091 } 2092 if (mOnFrameRenderedHandler != null) { 2093 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 2094 } 2095 } 2096 } 2097 native_stop()2098 private native final void native_stop(); 2099 2100 /** 2101 * Flush both input and output ports of the component. 2102 * <p> 2103 * Upon return, all indices previously returned in calls to {@link #dequeueInputBuffer 2104 * dequeueInputBuffer} and {@link #dequeueOutputBuffer dequeueOutputBuffer} — or obtained 2105 * via {@link Callback#onInputBufferAvailable onInputBufferAvailable} or 2106 * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} callbacks — become 2107 * invalid, and all buffers are owned by the codec. 2108 * <p> 2109 * If the codec is configured in asynchronous mode, call {@link #start} 2110 * after {@code flush} has returned to resume codec operations. The codec 2111 * will not request input buffers until this has happened. 2112 * <strong>Note, however, that there may still be outstanding {@code onOutputBufferAvailable} 2113 * callbacks that were not handled prior to calling {@code flush}. 2114 * The indices returned via these callbacks also become invalid upon calling {@code flush} and 2115 * should be discarded.</strong> 2116 * <p> 2117 * If the codec is configured in synchronous mode, codec will resume 2118 * automatically if it is configured with an input surface. Otherwise, it 2119 * will resume when {@link #dequeueInputBuffer dequeueInputBuffer} is called. 2120 * 2121 * @throws IllegalStateException if not in the Executing state. 2122 * @throws MediaCodec.CodecException upon codec error. 2123 */ flush()2124 public final void flush() { 2125 synchronized(mBufferLock) { 2126 invalidateByteBuffers(mCachedInputBuffers); 2127 invalidateByteBuffers(mCachedOutputBuffers); 2128 mDequeuedInputBuffers.clear(); 2129 mDequeuedOutputBuffers.clear(); 2130 } 2131 native_flush(); 2132 } 2133 native_flush()2134 private native final void native_flush(); 2135 2136 /** 2137 * Thrown when an internal codec error occurs. 2138 */ 2139 public final static class CodecException extends IllegalStateException { CodecException(int errorCode, int actionCode, @Nullable String detailMessage)2140 CodecException(int errorCode, int actionCode, @Nullable String detailMessage) { 2141 super(detailMessage); 2142 mErrorCode = errorCode; 2143 mActionCode = actionCode; 2144 2145 // TODO get this from codec 2146 final String sign = errorCode < 0 ? "neg_" : ""; 2147 mDiagnosticInfo = 2148 "android.media.MediaCodec.error_" + sign + Math.abs(errorCode); 2149 } 2150 2151 /** 2152 * Returns true if the codec exception is a transient issue, 2153 * perhaps due to resource constraints, and that the method 2154 * (or encoding/decoding) may be retried at a later time. 2155 */ 2156 public boolean isTransient() { 2157 return mActionCode == ACTION_TRANSIENT; 2158 } 2159 2160 /** 2161 * Returns true if the codec cannot proceed further, 2162 * but can be recovered by stopping, configuring, 2163 * and starting again. 2164 */ 2165 public boolean isRecoverable() { 2166 return mActionCode == ACTION_RECOVERABLE; 2167 } 2168 2169 /** 2170 * Retrieve the error code associated with a CodecException 2171 */ 2172 public int getErrorCode() { 2173 return mErrorCode; 2174 } 2175 2176 /** 2177 * Retrieve a developer-readable diagnostic information string 2178 * associated with the exception. Do not show this to end-users, 2179 * since this string will not be localized or generally 2180 * comprehensible to end-users. 2181 */ 2182 public @NonNull String getDiagnosticInfo() { 2183 return mDiagnosticInfo; 2184 } 2185 2186 /** 2187 * This indicates required resource was not able to be allocated. 2188 */ 2189 public static final int ERROR_INSUFFICIENT_RESOURCE = 1100; 2190 2191 /** 2192 * This indicates the resource manager reclaimed the media resource used by the codec. 2193 * <p> 2194 * With this exception, the codec must be released, as it has moved to terminal state. 2195 */ 2196 public static final int ERROR_RECLAIMED = 1101; 2197 2198 /** @hide */ 2199 @IntDef({ 2200 ERROR_INSUFFICIENT_RESOURCE, 2201 ERROR_RECLAIMED, 2202 }) 2203 @Retention(RetentionPolicy.SOURCE) 2204 public @interface ReasonCode {} 2205 2206 /* Must be in sync with android_media_MediaCodec.cpp */ 2207 private final static int ACTION_TRANSIENT = 1; 2208 private final static int ACTION_RECOVERABLE = 2; 2209 2210 private final String mDiagnosticInfo; 2211 private final int mErrorCode; 2212 private final int mActionCode; 2213 } 2214 2215 /** 2216 * Thrown when a crypto error occurs while queueing a secure input buffer. 2217 */ 2218 public final static class CryptoException extends RuntimeException { 2219 public CryptoException(int errorCode, @Nullable String detailMessage) { 2220 super(detailMessage); 2221 mErrorCode = errorCode; 2222 } 2223 2224 /** 2225 * This indicates that the requested key was not found when trying to 2226 * perform a decrypt operation. The operation can be retried after adding 2227 * the correct decryption key. 2228 */ 2229 public static final int ERROR_NO_KEY = 1; 2230 2231 /** 2232 * This indicates that the key used for decryption is no longer 2233 * valid due to license term expiration. The operation can be retried 2234 * after updating the expired keys. 2235 */ 2236 public static final int ERROR_KEY_EXPIRED = 2; 2237 2238 /** 2239 * This indicates that a required crypto resource was not able to be 2240 * allocated while attempting the requested operation. The operation 2241 * can be retried if the app is able to release resources. 2242 */ 2243 public static final int ERROR_RESOURCE_BUSY = 3; 2244 2245 /** 2246 * This indicates that the output protection levels supported by the 2247 * device are not sufficient to meet the requirements set by the 2248 * content owner in the license policy. 2249 */ 2250 public static final int ERROR_INSUFFICIENT_OUTPUT_PROTECTION = 4; 2251 2252 /** 2253 * This indicates that decryption was attempted on a session that is 2254 * not opened, which could be due to a failure to open the session, 2255 * closing the session prematurely, or the session being reclaimed 2256 * by the resource manager. 2257 */ 2258 public static final int ERROR_SESSION_NOT_OPENED = 5; 2259 2260 /** 2261 * This indicates that an operation was attempted that could not be 2262 * supported by the crypto system of the device in its current 2263 * configuration. It may occur when the license policy requires 2264 * device security features that aren't supported by the device, 2265 * or due to an internal error in the crypto system that prevents 2266 * the specified security policy from being met. 2267 */ 2268 public static final int ERROR_UNSUPPORTED_OPERATION = 6; 2269 2270 /** @hide */ 2271 @IntDef({ 2272 ERROR_NO_KEY, 2273 ERROR_KEY_EXPIRED, 2274 ERROR_RESOURCE_BUSY, 2275 ERROR_INSUFFICIENT_OUTPUT_PROTECTION, 2276 ERROR_SESSION_NOT_OPENED, 2277 ERROR_UNSUPPORTED_OPERATION 2278 }) 2279 @Retention(RetentionPolicy.SOURCE) 2280 public @interface CryptoErrorCode {} 2281 2282 /** 2283 * Retrieve the error code associated with a CryptoException 2284 */ 2285 @CryptoErrorCode 2286 public int getErrorCode() { 2287 return mErrorCode; 2288 } 2289 2290 private int mErrorCode; 2291 } 2292 2293 /** 2294 * After filling a range of the input buffer at the specified index 2295 * submit it to the component. Once an input buffer is queued to 2296 * the codec, it MUST NOT be used until it is later retrieved by 2297 * {@link #getInputBuffer} in response to a {@link #dequeueInputBuffer} 2298 * return value or a {@link Callback#onInputBufferAvailable} 2299 * callback. 2300 * <p> 2301 * Many decoders require the actual compressed data stream to be 2302 * preceded by "codec specific data", i.e. setup data used to initialize 2303 * the codec such as PPS/SPS in the case of AVC video or code tables 2304 * in the case of vorbis audio. 2305 * The class {@link android.media.MediaExtractor} provides codec 2306 * specific data as part of 2307 * the returned track format in entries named "csd-0", "csd-1" ... 2308 * <p> 2309 * These buffers can be submitted directly after {@link #start} or 2310 * {@link #flush} by specifying the flag {@link 2311 * #BUFFER_FLAG_CODEC_CONFIG}. However, if you configure the 2312 * codec with a {@link MediaFormat} containing these keys, they 2313 * will be automatically submitted by MediaCodec directly after 2314 * start. Therefore, the use of {@link 2315 * #BUFFER_FLAG_CODEC_CONFIG} flag is discouraged and is 2316 * recommended only for advanced users. 2317 * <p> 2318 * To indicate that this is the final piece of input data (or rather that 2319 * no more input data follows unless the decoder is subsequently flushed) 2320 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 2321 * <p class=note> 2322 * <strong>Note:</strong> Prior to {@link android.os.Build.VERSION_CODES#M}, 2323 * {@code presentationTimeUs} was not propagated to the frame timestamp of (rendered) 2324 * Surface output buffers, and the resulting frame timestamp was undefined. 2325 * Use {@link #releaseOutputBuffer(int, long)} to ensure a specific frame timestamp is set. 2326 * Similarly, since frame timestamps can be used by the destination surface for rendering 2327 * synchronization, <strong>care must be taken to normalize presentationTimeUs so as to not be 2328 * mistaken for a system time. (See {@linkplain #releaseOutputBuffer(int, long) 2329 * SurfaceView specifics}).</strong> 2330 * 2331 * @param index The index of a client-owned input buffer previously returned 2332 * in a call to {@link #dequeueInputBuffer}. 2333 * @param offset The byte offset into the input buffer at which the data starts. 2334 * @param size The number of bytes of valid input data. 2335 * @param presentationTimeUs The presentation timestamp in microseconds for this 2336 * buffer. This is normally the media time at which this 2337 * buffer should be presented (rendered). When using an output 2338 * surface, this will be propagated as the {@link 2339 * SurfaceTexture#getTimestamp timestamp} for the frame (after 2340 * conversion to nanoseconds). 2341 * @param flags A bitmask of flags 2342 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2343 * While not prohibited, most codecs do not use the 2344 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2345 * @throws IllegalStateException if not in the Executing state. 2346 * @throws MediaCodec.CodecException upon codec error. 2347 * @throws CryptoException if a crypto object has been specified in 2348 * {@link #configure} 2349 */ 2350 public final void queueInputBuffer( 2351 int index, 2352 int offset, int size, long presentationTimeUs, int flags) 2353 throws CryptoException { 2354 synchronized(mBufferLock) { 2355 invalidateByteBuffer(mCachedInputBuffers, index); 2356 mDequeuedInputBuffers.remove(index); 2357 } 2358 try { 2359 native_queueInputBuffer( 2360 index, offset, size, presentationTimeUs, flags); 2361 } catch (CryptoException | IllegalStateException e) { 2362 revalidateByteBuffer(mCachedInputBuffers, index); 2363 throw e; 2364 } 2365 } 2366 2367 private native final void native_queueInputBuffer( 2368 int index, 2369 int offset, int size, long presentationTimeUs, int flags) 2370 throws CryptoException; 2371 2372 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 2373 public static final int CRYPTO_MODE_AES_CTR = 1; 2374 public static final int CRYPTO_MODE_AES_CBC = 2; 2375 2376 /** 2377 * Metadata describing the structure of an encrypted input sample. 2378 * <p> 2379 * A buffer's data is considered to be partitioned into "subSamples". Each subSample starts with 2380 * a run of plain, unencrypted bytes followed by a run of encrypted bytes. Either of these runs 2381 * may be empty. If pattern encryption applies, each of the encrypted runs is encrypted only 2382 * partly, according to a repeating pattern of "encrypt" and "skip" blocks. 2383 * {@link #numBytesOfClearData} can be null to indicate that all data is encrypted, and 2384 * {@link #numBytesOfEncryptedData} can be null to indicate that all data is clear. At least one 2385 * of {@link #numBytesOfClearData} and {@link #numBytesOfEncryptedData} must be non-null. 2386 * <p> 2387 * This information encapsulates per-sample metadata as outlined in ISO/IEC FDIS 23001-7:2016 2388 * "Common encryption in ISO base media file format files". 2389 * <p> 2390 * <h3>ISO-CENC Schemes</h3> 2391 * ISO/IEC FDIS 23001-7:2016 defines four possible schemes by which media may be encrypted, 2392 * corresponding to each possible combination of an AES mode with the presence or absence of 2393 * patterned encryption. 2394 * 2395 * <table style="width: 0%"> 2396 * <thead> 2397 * <tr> 2398 * <th> </th> 2399 * <th>AES-CTR</th> 2400 * <th>AES-CBC</th> 2401 * </tr> 2402 * </thead> 2403 * <tbody> 2404 * <tr> 2405 * <th>Without Patterns</th> 2406 * <td>cenc</td> 2407 * <td>cbc1</td> 2408 * </tr><tr> 2409 * <th>With Patterns</th> 2410 * <td>cens</td> 2411 * <td>cbcs</td> 2412 * </tr> 2413 * </tbody> 2414 * </table> 2415 * 2416 * For {@code CryptoInfo}, the scheme is selected implicitly by the combination of the 2417 * {@link #mode} field and the value set with {@link #setPattern}. For the pattern, setting the 2418 * pattern to all zeroes (that is, both {@code blocksToEncrypt} and {@code blocksToSkip} are 2419 * zero) is interpreted as turning patterns off completely. A scheme that does not use patterns 2420 * will be selected, either cenc or cbc1. Setting the pattern to any nonzero value will choose 2421 * one of the pattern-supporting schemes, cens or cbcs. The default pattern if 2422 * {@link #setPattern} is never called is all zeroes. 2423 * <p> 2424 * <h4>HLS SAMPLE-AES Audio</h4> 2425 * HLS SAMPLE-AES audio is encrypted in a manner compatible with the cbcs scheme, except that it 2426 * does not use patterned encryption. However, if {@link #setPattern} is used to set the pattern 2427 * to all zeroes, this will be interpreted as selecting the cbc1 scheme. The cbc1 scheme cannot 2428 * successfully decrypt HLS SAMPLE-AES audio because of differences in how the IVs are handled. 2429 * For this reason, it is recommended that a pattern of {@code 1} encrypted block and {@code 0} 2430 * skip blocks be used with HLS SAMPLE-AES audio. This will trigger decryption to use cbcs mode 2431 * while still decrypting every block. 2432 */ 2433 public final static class CryptoInfo { 2434 /** 2435 * The number of subSamples that make up the buffer's contents. 2436 */ 2437 public int numSubSamples; 2438 /** 2439 * The number of leading unencrypted bytes in each subSample. If null, all bytes are treated 2440 * as encrypted and {@link #numBytesOfEncryptedData} must be specified. 2441 */ 2442 public int[] numBytesOfClearData; 2443 /** 2444 * The number of trailing encrypted bytes in each subSample. If null, all bytes are treated 2445 * as clear and {@link #numBytesOfClearData} must be specified. 2446 */ 2447 public int[] numBytesOfEncryptedData; 2448 /** 2449 * A 16-byte key id 2450 */ 2451 public byte[] key; 2452 /** 2453 * A 16-byte initialization vector 2454 */ 2455 public byte[] iv; 2456 /** 2457 * The type of encryption that has been applied, 2458 * see {@link #CRYPTO_MODE_UNENCRYPTED}, {@link #CRYPTO_MODE_AES_CTR} 2459 * and {@link #CRYPTO_MODE_AES_CBC} 2460 */ 2461 public int mode; 2462 2463 /** 2464 * Metadata describing an encryption pattern for the protected bytes in a subsample. An 2465 * encryption pattern consists of a repeating sequence of crypto blocks comprised of a 2466 * number of encrypted blocks followed by a number of unencrypted, or skipped, blocks. 2467 */ 2468 public final static class Pattern { 2469 /** 2470 * Number of blocks to be encrypted in the pattern. If both this and 2471 * {@link #mSkipBlocks} are zero, pattern encryption is inoperative. 2472 */ 2473 private int mEncryptBlocks; 2474 2475 /** 2476 * Number of blocks to be skipped (left clear) in the pattern. If both this and 2477 * {@link #mEncryptBlocks} are zero, pattern encryption is inoperative. 2478 */ 2479 private int mSkipBlocks; 2480 2481 /** 2482 * Construct a sample encryption pattern given the number of blocks to encrypt and skip 2483 * in the pattern. If both parameters are zero, pattern encryption is inoperative. 2484 */ 2485 public Pattern(int blocksToEncrypt, int blocksToSkip) { 2486 set(blocksToEncrypt, blocksToSkip); 2487 } 2488 2489 /** 2490 * Set the number of blocks to encrypt and skip in a sample encryption pattern. If both 2491 * parameters are zero, pattern encryption is inoperative. 2492 */ 2493 public void set(int blocksToEncrypt, int blocksToSkip) { 2494 mEncryptBlocks = blocksToEncrypt; 2495 mSkipBlocks = blocksToSkip; 2496 } 2497 2498 /** 2499 * Return the number of blocks to skip in a sample encryption pattern. 2500 */ 2501 public int getSkipBlocks() { 2502 return mSkipBlocks; 2503 } 2504 2505 /** 2506 * Return the number of blocks to encrypt in a sample encryption pattern. 2507 */ 2508 public int getEncryptBlocks() { 2509 return mEncryptBlocks; 2510 } 2511 }; 2512 2513 private final Pattern zeroPattern = new Pattern(0, 0); 2514 2515 /** 2516 * The pattern applicable to the protected data in each subsample. 2517 */ 2518 private Pattern pattern; 2519 2520 /** 2521 * Set the subsample count, clear/encrypted sizes, key, IV and mode fields of 2522 * a {@link MediaCodec.CryptoInfo} instance. 2523 */ 2524 public void set( 2525 int newNumSubSamples, 2526 @NonNull int[] newNumBytesOfClearData, 2527 @NonNull int[] newNumBytesOfEncryptedData, 2528 @NonNull byte[] newKey, 2529 @NonNull byte[] newIV, 2530 int newMode) { 2531 numSubSamples = newNumSubSamples; 2532 numBytesOfClearData = newNumBytesOfClearData; 2533 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 2534 key = newKey; 2535 iv = newIV; 2536 mode = newMode; 2537 pattern = zeroPattern; 2538 } 2539 2540 /** 2541 * Set the encryption pattern on a {@link MediaCodec.CryptoInfo} instance. 2542 * See {@link MediaCodec.CryptoInfo.Pattern}. 2543 */ 2544 public void setPattern(Pattern newPattern) { 2545 pattern = newPattern; 2546 } 2547 2548 private void setPattern(int blocksToEncrypt, int blocksToSkip) { 2549 pattern = new Pattern(blocksToEncrypt, blocksToSkip); 2550 } 2551 2552 @Override 2553 public String toString() { 2554 StringBuilder builder = new StringBuilder(); 2555 builder.append(numSubSamples + " subsamples, key ["); 2556 String hexdigits = "0123456789abcdef"; 2557 for (int i = 0; i < key.length; i++) { 2558 builder.append(hexdigits.charAt((key[i] & 0xf0) >> 4)); 2559 builder.append(hexdigits.charAt(key[i] & 0x0f)); 2560 } 2561 builder.append("], iv ["); 2562 for (int i = 0; i < key.length; i++) { 2563 builder.append(hexdigits.charAt((iv[i] & 0xf0) >> 4)); 2564 builder.append(hexdigits.charAt(iv[i] & 0x0f)); 2565 } 2566 builder.append("], clear "); Arrays.toString(numBytesOfClearData)2567 builder.append(Arrays.toString(numBytesOfClearData)); 2568 builder.append(", encrypted "); Arrays.toString(numBytesOfEncryptedData)2569 builder.append(Arrays.toString(numBytesOfEncryptedData)); 2570 return builder.toString(); 2571 } 2572 }; 2573 2574 /** 2575 * Similar to {@link #queueInputBuffer queueInputBuffer} but submits a buffer that is 2576 * potentially encrypted. 2577 * <strong>Check out further notes at {@link #queueInputBuffer queueInputBuffer}.</strong> 2578 * 2579 * @param index The index of a client-owned input buffer previously returned 2580 * in a call to {@link #dequeueInputBuffer}. 2581 * @param offset The byte offset into the input buffer at which the data starts. 2582 * @param info Metadata required to facilitate decryption, the object can be 2583 * reused immediately after this call returns. 2584 * @param presentationTimeUs The presentation timestamp in microseconds for this 2585 * buffer. This is normally the media time at which this 2586 * buffer should be presented (rendered). 2587 * @param flags A bitmask of flags 2588 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2589 * While not prohibited, most codecs do not use the 2590 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2591 * @throws IllegalStateException if not in the Executing state. 2592 * @throws MediaCodec.CodecException upon codec error. 2593 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 2594 * An error code associated with the exception helps identify the 2595 * reason for the failure. 2596 */ queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2597 public final void queueSecureInputBuffer( 2598 int index, 2599 int offset, 2600 @NonNull CryptoInfo info, 2601 long presentationTimeUs, 2602 int flags) throws CryptoException { 2603 synchronized(mBufferLock) { 2604 invalidateByteBuffer(mCachedInputBuffers, index); 2605 mDequeuedInputBuffers.remove(index); 2606 } 2607 try { 2608 native_queueSecureInputBuffer( 2609 index, offset, info, presentationTimeUs, flags); 2610 } catch (CryptoException | IllegalStateException e) { 2611 revalidateByteBuffer(mCachedInputBuffers, index); 2612 throw e; 2613 } 2614 } 2615 native_queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2616 private native final void native_queueSecureInputBuffer( 2617 int index, 2618 int offset, 2619 @NonNull CryptoInfo info, 2620 long presentationTimeUs, 2621 int flags) throws CryptoException; 2622 2623 /** 2624 * Returns the index of an input buffer to be filled with valid data 2625 * or -1 if no such buffer is currently available. 2626 * This method will return immediately if timeoutUs == 0, wait indefinitely 2627 * for the availability of an input buffer if timeoutUs < 0 or wait up 2628 * to "timeoutUs" microseconds if timeoutUs > 0. 2629 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 2630 * @throws IllegalStateException if not in the Executing state, 2631 * or codec is configured in asynchronous mode. 2632 * @throws MediaCodec.CodecException upon codec error. 2633 */ dequeueInputBuffer(long timeoutUs)2634 public final int dequeueInputBuffer(long timeoutUs) { 2635 int res = native_dequeueInputBuffer(timeoutUs); 2636 if (res >= 0) { 2637 synchronized(mBufferLock) { 2638 validateInputByteBuffer(mCachedInputBuffers, res); 2639 } 2640 } 2641 return res; 2642 } 2643 native_dequeueInputBuffer(long timeoutUs)2644 private native final int native_dequeueInputBuffer(long timeoutUs); 2645 2646 /** 2647 * If a non-negative timeout had been specified in the call 2648 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 2649 */ 2650 public static final int INFO_TRY_AGAIN_LATER = -1; 2651 2652 /** 2653 * The output format has changed, subsequent data will follow the new 2654 * format. {@link #getOutputFormat()} returns the new format. Note, that 2655 * you can also use the new {@link #getOutputFormat(int)} method to 2656 * get the format for a specific output buffer. This frees you from 2657 * having to track output format changes. 2658 */ 2659 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 2660 2661 /** 2662 * The output buffers have changed, the client must refer to the new 2663 * set of output buffers returned by {@link #getOutputBuffers} from 2664 * this point on. 2665 * 2666 * <p>Additionally, this event signals that the video scaling mode 2667 * may have been reset to the default.</p> 2668 * 2669 * @deprecated This return value can be ignored as {@link 2670 * #getOutputBuffers} has been deprecated. Client should 2671 * request a current buffer using on of the get-buffer or 2672 * get-image methods each time one has been dequeued. 2673 */ 2674 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 2675 2676 /** @hide */ 2677 @IntDef({ 2678 INFO_TRY_AGAIN_LATER, 2679 INFO_OUTPUT_FORMAT_CHANGED, 2680 INFO_OUTPUT_BUFFERS_CHANGED, 2681 }) 2682 @Retention(RetentionPolicy.SOURCE) 2683 public @interface OutputBufferInfo {} 2684 2685 /** 2686 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 2687 * Returns the index of an output buffer that has been successfully 2688 * decoded or one of the INFO_* constants. 2689 * @param info Will be filled with buffer meta data. 2690 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 2691 * @throws IllegalStateException if not in the Executing state, 2692 * or codec is configured in asynchronous mode. 2693 * @throws MediaCodec.CodecException upon codec error. 2694 */ 2695 @OutputBufferInfo dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)2696 public final int dequeueOutputBuffer( 2697 @NonNull BufferInfo info, long timeoutUs) { 2698 int res = native_dequeueOutputBuffer(info, timeoutUs); 2699 synchronized(mBufferLock) { 2700 if (res == INFO_OUTPUT_BUFFERS_CHANGED) { 2701 cacheBuffers(false /* input */); 2702 } else if (res >= 0) { 2703 validateOutputByteBuffer(mCachedOutputBuffers, res, info); 2704 if (mHasSurface) { 2705 mDequeuedOutputInfos.put(res, info.dup()); 2706 } 2707 } 2708 } 2709 return res; 2710 } 2711 native_dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)2712 private native final int native_dequeueOutputBuffer( 2713 @NonNull BufferInfo info, long timeoutUs); 2714 2715 /** 2716 * If you are done with a buffer, use this call to return the buffer to the codec 2717 * or to render it on the output surface. If you configured the codec with an 2718 * output surface, setting {@code render} to {@code true} will first send the buffer 2719 * to that output surface. The surface will release the buffer back to the codec once 2720 * it is no longer used/displayed. 2721 * 2722 * Once an output buffer is released to the codec, it MUST NOT 2723 * be used until it is later retrieved by {@link #getOutputBuffer} in response 2724 * to a {@link #dequeueOutputBuffer} return value or a 2725 * {@link Callback#onOutputBufferAvailable} callback. 2726 * 2727 * @param index The index of a client-owned output buffer previously returned 2728 * from a call to {@link #dequeueOutputBuffer}. 2729 * @param render If a valid surface was specified when configuring the codec, 2730 * passing true renders this output buffer to the surface. 2731 * @throws IllegalStateException if not in the Executing state. 2732 * @throws MediaCodec.CodecException upon codec error. 2733 */ releaseOutputBuffer(int index, boolean render)2734 public final void releaseOutputBuffer(int index, boolean render) { 2735 BufferInfo info = null; 2736 synchronized(mBufferLock) { 2737 invalidateByteBuffer(mCachedOutputBuffers, index); 2738 mDequeuedOutputBuffers.remove(index); 2739 if (mHasSurface) { 2740 info = mDequeuedOutputInfos.remove(index); 2741 } 2742 } 2743 releaseOutputBuffer(index, render, false /* updatePTS */, 0 /* dummy */); 2744 } 2745 2746 /** 2747 * If you are done with a buffer, use this call to update its surface timestamp 2748 * and return it to the codec to render it on the output surface. If you 2749 * have not specified an output surface when configuring this video codec, 2750 * this call will simply return the buffer to the codec.<p> 2751 * 2752 * The timestamp may have special meaning depending on the destination surface. 2753 * 2754 * <table> 2755 * <tr><th>SurfaceView specifics</th></tr> 2756 * <tr><td> 2757 * If you render your buffer on a {@link android.view.SurfaceView}, 2758 * you can use the timestamp to render the buffer at a specific time (at the 2759 * VSYNC at or after the buffer timestamp). For this to work, the timestamp 2760 * needs to be <i>reasonably close</i> to the current {@link System#nanoTime}. 2761 * Currently, this is set as within one (1) second. A few notes: 2762 * 2763 * <ul> 2764 * <li>the buffer will not be returned to the codec until the timestamp 2765 * has passed and the buffer is no longer used by the {@link android.view.Surface}. 2766 * <li>buffers are processed sequentially, so you may block subsequent buffers to 2767 * be displayed on the {@link android.view.Surface}. This is important if you 2768 * want to react to user action, e.g. stop the video or seek. 2769 * <li>if multiple buffers are sent to the {@link android.view.Surface} to be 2770 * rendered at the same VSYNC, the last one will be shown, and the other ones 2771 * will be dropped. 2772 * <li>if the timestamp is <em>not</em> "reasonably close" to the current system 2773 * time, the {@link android.view.Surface} will ignore the timestamp, and 2774 * display the buffer at the earliest feasible time. In this mode it will not 2775 * drop frames. 2776 * <li>for best performance and quality, call this method when you are about 2777 * two VSYNCs' time before the desired render time. For 60Hz displays, this is 2778 * about 33 msec. 2779 * </ul> 2780 * </td></tr> 2781 * </table> 2782 * 2783 * Once an output buffer is released to the codec, it MUST NOT 2784 * be used until it is later retrieved by {@link #getOutputBuffer} in response 2785 * to a {@link #dequeueOutputBuffer} return value or a 2786 * {@link Callback#onOutputBufferAvailable} callback. 2787 * 2788 * @param index The index of a client-owned output buffer previously returned 2789 * from a call to {@link #dequeueOutputBuffer}. 2790 * @param renderTimestampNs The timestamp to associate with this buffer when 2791 * it is sent to the Surface. 2792 * @throws IllegalStateException if not in the Executing state. 2793 * @throws MediaCodec.CodecException upon codec error. 2794 */ releaseOutputBuffer(int index, long renderTimestampNs)2795 public final void releaseOutputBuffer(int index, long renderTimestampNs) { 2796 BufferInfo info = null; 2797 synchronized(mBufferLock) { 2798 invalidateByteBuffer(mCachedOutputBuffers, index); 2799 mDequeuedOutputBuffers.remove(index); 2800 if (mHasSurface) { 2801 info = mDequeuedOutputInfos.remove(index); 2802 } 2803 } 2804 releaseOutputBuffer( 2805 index, true /* render */, true /* updatePTS */, renderTimestampNs); 2806 } 2807 releaseOutputBuffer( int index, boolean render, boolean updatePTS, long timeNs)2808 private native final void releaseOutputBuffer( 2809 int index, boolean render, boolean updatePTS, long timeNs); 2810 2811 /** 2812 * Signals end-of-stream on input. Equivalent to submitting an empty buffer with 2813 * {@link #BUFFER_FLAG_END_OF_STREAM} set. This may only be used with 2814 * encoders receiving input from a Surface created by {@link #createInputSurface}. 2815 * @throws IllegalStateException if not in the Executing state. 2816 * @throws MediaCodec.CodecException upon codec error. 2817 */ signalEndOfInputStream()2818 public native final void signalEndOfInputStream(); 2819 2820 /** 2821 * Call this after dequeueOutputBuffer signals a format change by returning 2822 * {@link #INFO_OUTPUT_FORMAT_CHANGED}. 2823 * You can also call this after {@link #configure} returns 2824 * successfully to get the output format initially configured 2825 * for the codec. Do this to determine what optional 2826 * configuration parameters were supported by the codec. 2827 * 2828 * @throws IllegalStateException if not in the Executing or 2829 * Configured state. 2830 * @throws MediaCodec.CodecException upon codec error. 2831 */ 2832 @NonNull getOutputFormat()2833 public final MediaFormat getOutputFormat() { 2834 return new MediaFormat(getFormatNative(false /* input */)); 2835 } 2836 2837 /** 2838 * Call this after {@link #configure} returns successfully to 2839 * get the input format accepted by the codec. Do this to 2840 * determine what optional configuration parameters were 2841 * supported by the codec. 2842 * 2843 * @throws IllegalStateException if not in the Executing or 2844 * Configured state. 2845 * @throws MediaCodec.CodecException upon codec error. 2846 */ 2847 @NonNull getInputFormat()2848 public final MediaFormat getInputFormat() { 2849 return new MediaFormat(getFormatNative(true /* input */)); 2850 } 2851 2852 /** 2853 * Returns the output format for a specific output buffer. 2854 * 2855 * @param index The index of a client-owned input buffer previously 2856 * returned from a call to {@link #dequeueInputBuffer}. 2857 * 2858 * @return the format for the output buffer, or null if the index 2859 * is not a dequeued output buffer. 2860 */ 2861 @NonNull getOutputFormat(int index)2862 public final MediaFormat getOutputFormat(int index) { 2863 return new MediaFormat(getOutputFormatNative(index)); 2864 } 2865 2866 @NonNull getFormatNative(boolean input)2867 private native final Map<String, Object> getFormatNative(boolean input); 2868 2869 @NonNull getOutputFormatNative(int index)2870 private native final Map<String, Object> getOutputFormatNative(int index); 2871 2872 // used to track dequeued buffers 2873 private static class BufferMap { 2874 // various returned representations of the codec buffer 2875 private static class CodecBuffer { 2876 private Image mImage; 2877 private ByteBuffer mByteBuffer; 2878 free()2879 public void free() { 2880 if (mByteBuffer != null) { 2881 // all of our ByteBuffers are direct 2882 java.nio.NioUtils.freeDirectBuffer(mByteBuffer); 2883 mByteBuffer = null; 2884 } 2885 if (mImage != null) { 2886 mImage.close(); 2887 mImage = null; 2888 } 2889 } 2890 setImage(@ullable Image image)2891 public void setImage(@Nullable Image image) { 2892 free(); 2893 mImage = image; 2894 } 2895 setByteBuffer(@ullable ByteBuffer buffer)2896 public void setByteBuffer(@Nullable ByteBuffer buffer) { 2897 free(); 2898 mByteBuffer = buffer; 2899 } 2900 } 2901 2902 private final Map<Integer, CodecBuffer> mMap = 2903 new HashMap<Integer, CodecBuffer>(); 2904 remove(int index)2905 public void remove(int index) { 2906 CodecBuffer buffer = mMap.get(index); 2907 if (buffer != null) { 2908 buffer.free(); 2909 mMap.remove(index); 2910 } 2911 } 2912 put(int index, @Nullable ByteBuffer newBuffer)2913 public void put(int index, @Nullable ByteBuffer newBuffer) { 2914 CodecBuffer buffer = mMap.get(index); 2915 if (buffer == null) { // likely 2916 buffer = new CodecBuffer(); 2917 mMap.put(index, buffer); 2918 } 2919 buffer.setByteBuffer(newBuffer); 2920 } 2921 put(int index, @Nullable Image newImage)2922 public void put(int index, @Nullable Image newImage) { 2923 CodecBuffer buffer = mMap.get(index); 2924 if (buffer == null) { // likely 2925 buffer = new CodecBuffer(); 2926 mMap.put(index, buffer); 2927 } 2928 buffer.setImage(newImage); 2929 } 2930 clear()2931 public void clear() { 2932 for (CodecBuffer buffer: mMap.values()) { 2933 buffer.free(); 2934 } 2935 mMap.clear(); 2936 } 2937 } 2938 2939 private ByteBuffer[] mCachedInputBuffers; 2940 private ByteBuffer[] mCachedOutputBuffers; 2941 private final BufferMap mDequeuedInputBuffers = new BufferMap(); 2942 private final BufferMap mDequeuedOutputBuffers = new BufferMap(); 2943 private final Map<Integer, BufferInfo> mDequeuedOutputInfos = 2944 new HashMap<Integer, BufferInfo>(); 2945 final private Object mBufferLock; 2946 invalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)2947 private final void invalidateByteBuffer( 2948 @Nullable ByteBuffer[] buffers, int index) { 2949 if (buffers != null && index >= 0 && index < buffers.length) { 2950 ByteBuffer buffer = buffers[index]; 2951 if (buffer != null) { 2952 buffer.setAccessible(false); 2953 } 2954 } 2955 } 2956 validateInputByteBuffer( @ullable ByteBuffer[] buffers, int index)2957 private final void validateInputByteBuffer( 2958 @Nullable ByteBuffer[] buffers, int index) { 2959 if (buffers != null && index >= 0 && index < buffers.length) { 2960 ByteBuffer buffer = buffers[index]; 2961 if (buffer != null) { 2962 buffer.setAccessible(true); 2963 buffer.clear(); 2964 } 2965 } 2966 } 2967 revalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)2968 private final void revalidateByteBuffer( 2969 @Nullable ByteBuffer[] buffers, int index) { 2970 synchronized(mBufferLock) { 2971 if (buffers != null && index >= 0 && index < buffers.length) { 2972 ByteBuffer buffer = buffers[index]; 2973 if (buffer != null) { 2974 buffer.setAccessible(true); 2975 } 2976 } 2977 } 2978 } 2979 validateOutputByteBuffer( @ullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info)2980 private final void validateOutputByteBuffer( 2981 @Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) { 2982 if (buffers != null && index >= 0 && index < buffers.length) { 2983 ByteBuffer buffer = buffers[index]; 2984 if (buffer != null) { 2985 buffer.setAccessible(true); 2986 buffer.limit(info.offset + info.size).position(info.offset); 2987 } 2988 } 2989 } 2990 invalidateByteBuffers(@ullable ByteBuffer[] buffers)2991 private final void invalidateByteBuffers(@Nullable ByteBuffer[] buffers) { 2992 if (buffers != null) { 2993 for (ByteBuffer buffer: buffers) { 2994 if (buffer != null) { 2995 buffer.setAccessible(false); 2996 } 2997 } 2998 } 2999 } 3000 freeByteBuffer(@ullable ByteBuffer buffer)3001 private final void freeByteBuffer(@Nullable ByteBuffer buffer) { 3002 if (buffer != null /* && buffer.isDirect() */) { 3003 // all of our ByteBuffers are direct 3004 java.nio.NioUtils.freeDirectBuffer(buffer); 3005 } 3006 } 3007 freeByteBuffers(@ullable ByteBuffer[] buffers)3008 private final void freeByteBuffers(@Nullable ByteBuffer[] buffers) { 3009 if (buffers != null) { 3010 for (ByteBuffer buffer: buffers) { 3011 freeByteBuffer(buffer); 3012 } 3013 } 3014 } 3015 freeAllTrackedBuffers()3016 private final void freeAllTrackedBuffers() { 3017 synchronized(mBufferLock) { 3018 freeByteBuffers(mCachedInputBuffers); 3019 freeByteBuffers(mCachedOutputBuffers); 3020 mCachedInputBuffers = null; 3021 mCachedOutputBuffers = null; 3022 mDequeuedInputBuffers.clear(); 3023 mDequeuedOutputBuffers.clear(); 3024 } 3025 } 3026 cacheBuffers(boolean input)3027 private final void cacheBuffers(boolean input) { 3028 ByteBuffer[] buffers = null; 3029 try { 3030 buffers = getBuffers(input); 3031 invalidateByteBuffers(buffers); 3032 } catch (IllegalStateException e) { 3033 // we don't get buffers in async mode 3034 } 3035 if (input) { 3036 mCachedInputBuffers = buffers; 3037 } else { 3038 mCachedOutputBuffers = buffers; 3039 } 3040 } 3041 3042 /** 3043 * Retrieve the set of input buffers. Call this after start() 3044 * returns. After calling this method, any ByteBuffers 3045 * previously returned by an earlier call to this method MUST no 3046 * longer be used. 3047 * 3048 * @deprecated Use the new {@link #getInputBuffer} method instead 3049 * each time an input buffer is dequeued. 3050 * 3051 * <b>Note:</b> As of API 21, dequeued input buffers are 3052 * automatically {@link java.nio.Buffer#clear cleared}. 3053 * 3054 * <em>Do not use this method if using an input surface.</em> 3055 * 3056 * @throws IllegalStateException if not in the Executing state, 3057 * or codec is configured in asynchronous mode. 3058 * @throws MediaCodec.CodecException upon codec error. 3059 */ 3060 @NonNull getInputBuffers()3061 public ByteBuffer[] getInputBuffers() { 3062 if (mCachedInputBuffers == null) { 3063 throw new IllegalStateException(); 3064 } 3065 // FIXME: check codec status 3066 return mCachedInputBuffers; 3067 } 3068 3069 /** 3070 * Retrieve the set of output buffers. Call this after start() 3071 * returns and whenever dequeueOutputBuffer signals an output 3072 * buffer change by returning {@link 3073 * #INFO_OUTPUT_BUFFERS_CHANGED}. After calling this method, any 3074 * ByteBuffers previously returned by an earlier call to this 3075 * method MUST no longer be used. 3076 * 3077 * @deprecated Use the new {@link #getOutputBuffer} method instead 3078 * each time an output buffer is dequeued. This method is not 3079 * supported if codec is configured in asynchronous mode. 3080 * 3081 * <b>Note:</b> As of API 21, the position and limit of output 3082 * buffers that are dequeued will be set to the valid data 3083 * range. 3084 * 3085 * <em>Do not use this method if using an output surface.</em> 3086 * 3087 * @throws IllegalStateException if not in the Executing state, 3088 * or codec is configured in asynchronous mode. 3089 * @throws MediaCodec.CodecException upon codec error. 3090 */ 3091 @NonNull getOutputBuffers()3092 public ByteBuffer[] getOutputBuffers() { 3093 if (mCachedOutputBuffers == null) { 3094 throw new IllegalStateException(); 3095 } 3096 // FIXME: check codec status 3097 return mCachedOutputBuffers; 3098 } 3099 3100 /** 3101 * Returns a {@link java.nio.Buffer#clear cleared}, writable ByteBuffer 3102 * object for a dequeued input buffer index to contain the input data. 3103 * 3104 * After calling this method any ByteBuffer or Image object 3105 * previously returned for the same input index MUST no longer 3106 * be used. 3107 * 3108 * @param index The index of a client-owned input buffer previously 3109 * returned from a call to {@link #dequeueInputBuffer}, 3110 * or received via an onInputBufferAvailable callback. 3111 * 3112 * @return the input buffer, or null if the index is not a dequeued 3113 * input buffer, or if the codec is configured for surface input. 3114 * 3115 * @throws IllegalStateException if not in the Executing state. 3116 * @throws MediaCodec.CodecException upon codec error. 3117 */ 3118 @Nullable getInputBuffer(int index)3119 public ByteBuffer getInputBuffer(int index) { 3120 ByteBuffer newBuffer = getBuffer(true /* input */, index); 3121 synchronized(mBufferLock) { 3122 invalidateByteBuffer(mCachedInputBuffers, index); 3123 mDequeuedInputBuffers.put(index, newBuffer); 3124 } 3125 return newBuffer; 3126 } 3127 3128 /** 3129 * Returns a writable Image object for a dequeued input buffer 3130 * index to contain the raw input video frame. 3131 * 3132 * After calling this method any ByteBuffer or Image object 3133 * previously returned for the same input index MUST no longer 3134 * be used. 3135 * 3136 * @param index The index of a client-owned input buffer previously 3137 * returned from a call to {@link #dequeueInputBuffer}, 3138 * or received via an onInputBufferAvailable callback. 3139 * 3140 * @return the input image, or null if the index is not a 3141 * dequeued input buffer, or not a ByteBuffer that contains a 3142 * raw image. 3143 * 3144 * @throws IllegalStateException if not in the Executing state. 3145 * @throws MediaCodec.CodecException upon codec error. 3146 */ 3147 @Nullable getInputImage(int index)3148 public Image getInputImage(int index) { 3149 Image newImage = getImage(true /* input */, index); 3150 synchronized(mBufferLock) { 3151 invalidateByteBuffer(mCachedInputBuffers, index); 3152 mDequeuedInputBuffers.put(index, newImage); 3153 } 3154 return newImage; 3155 } 3156 3157 /** 3158 * Returns a read-only ByteBuffer for a dequeued output buffer 3159 * index. The position and limit of the returned buffer are set 3160 * to the valid output data. 3161 * 3162 * After calling this method, any ByteBuffer or Image object 3163 * previously returned for the same output index MUST no longer 3164 * be used. 3165 * 3166 * @param index The index of a client-owned output buffer previously 3167 * returned from a call to {@link #dequeueOutputBuffer}, 3168 * or received via an onOutputBufferAvailable callback. 3169 * 3170 * @return the output buffer, or null if the index is not a dequeued 3171 * output buffer, or the codec is configured with an output surface. 3172 * 3173 * @throws IllegalStateException if not in the Executing state. 3174 * @throws MediaCodec.CodecException upon codec error. 3175 */ 3176 @Nullable getOutputBuffer(int index)3177 public ByteBuffer getOutputBuffer(int index) { 3178 ByteBuffer newBuffer = getBuffer(false /* input */, index); 3179 synchronized(mBufferLock) { 3180 invalidateByteBuffer(mCachedOutputBuffers, index); 3181 mDequeuedOutputBuffers.put(index, newBuffer); 3182 } 3183 return newBuffer; 3184 } 3185 3186 /** 3187 * Returns a read-only Image object for a dequeued output buffer 3188 * index that contains the raw video frame. 3189 * 3190 * After calling this method, any ByteBuffer or Image object previously 3191 * returned for the same output index MUST no longer be used. 3192 * 3193 * @param index The index of a client-owned output buffer previously 3194 * returned from a call to {@link #dequeueOutputBuffer}, 3195 * or received via an onOutputBufferAvailable callback. 3196 * 3197 * @return the output image, or null if the index is not a 3198 * dequeued output buffer, not a raw video frame, or if the codec 3199 * was configured with an output surface. 3200 * 3201 * @throws IllegalStateException if not in the Executing state. 3202 * @throws MediaCodec.CodecException upon codec error. 3203 */ 3204 @Nullable getOutputImage(int index)3205 public Image getOutputImage(int index) { 3206 Image newImage = getImage(false /* input */, index); 3207 synchronized(mBufferLock) { 3208 invalidateByteBuffer(mCachedOutputBuffers, index); 3209 mDequeuedOutputBuffers.put(index, newImage); 3210 } 3211 return newImage; 3212 } 3213 3214 /** 3215 * The content is scaled to the surface dimensions 3216 */ 3217 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 3218 3219 /** 3220 * The content is scaled, maintaining its aspect ratio, the whole 3221 * surface area is used, content may be cropped. 3222 * <p class=note> 3223 * This mode is only suitable for content with 1:1 pixel aspect ratio as you cannot 3224 * configure the pixel aspect ratio for a {@link Surface}. 3225 * <p class=note> 3226 * As of {@link android.os.Build.VERSION_CODES#N} release, this mode may not work if 3227 * the video is {@linkplain MediaFormat#KEY_ROTATION rotated} by 90 or 270 degrees. 3228 */ 3229 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 3230 3231 /** @hide */ 3232 @IntDef({ 3233 VIDEO_SCALING_MODE_SCALE_TO_FIT, 3234 VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING, 3235 }) 3236 @Retention(RetentionPolicy.SOURCE) 3237 public @interface VideoScalingMode {} 3238 3239 /** 3240 * If a surface has been specified in a previous call to {@link #configure} 3241 * specifies the scaling mode to use. The default is "scale to fit". 3242 * <p class=note> 3243 * The scaling mode may be reset to the <strong>default</strong> each time an 3244 * {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is received from the codec; therefore, the client 3245 * must call this method after every buffer change event (and before the first output buffer is 3246 * released for rendering) to ensure consistent scaling mode. 3247 * <p class=note> 3248 * Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, this can also be done 3249 * after each {@link #INFO_OUTPUT_FORMAT_CHANGED} event. 3250 * 3251 * @throws IllegalArgumentException if mode is not recognized. 3252 * @throws IllegalStateException if in the Released state. 3253 */ setVideoScalingMode(@ideoScalingMode int mode)3254 public native final void setVideoScalingMode(@VideoScalingMode int mode); 3255 3256 /** 3257 * Get the component name. If the codec was created by createDecoderByType 3258 * or createEncoderByType, what component is chosen is not known beforehand. 3259 * @throws IllegalStateException if in the Released state. 3260 */ 3261 @NonNull getName()3262 public native final String getName(); 3263 3264 /** 3265 * Return Metrics data about the current codec instance. 3266 * 3267 * @return a {@link PersistableBundle} containing the set of attributes and values 3268 * available for the media being handled by this instance of MediaCodec 3269 * The attributes are descibed in {@link MetricsConstants}. 3270 * 3271 * Additional vendor-specific fields may also be present in 3272 * the return value. 3273 */ getMetrics()3274 public PersistableBundle getMetrics() { 3275 PersistableBundle bundle = native_getMetrics(); 3276 return bundle; 3277 } 3278 native_getMetrics()3279 private native PersistableBundle native_getMetrics(); 3280 3281 /** 3282 * Change a video encoder's target bitrate on the fly. The value is an 3283 * Integer object containing the new bitrate in bps. 3284 */ 3285 public static final String PARAMETER_KEY_VIDEO_BITRATE = "video-bitrate"; 3286 3287 /** 3288 * Temporarily suspend/resume encoding of input data. While suspended 3289 * input data is effectively discarded instead of being fed into the 3290 * encoder. This parameter really only makes sense to use with an encoder 3291 * in "surface-input" mode, as the client code has no control over the 3292 * input-side of the encoder in that case. 3293 * The value is an Integer object containing the value 1 to suspend 3294 * or the value 0 to resume. 3295 */ 3296 public static final String PARAMETER_KEY_SUSPEND = "drop-input-frames"; 3297 3298 /** 3299 * Request that the encoder produce a sync frame "soon". 3300 * Provide an Integer with the value 0. 3301 */ 3302 public static final String PARAMETER_KEY_REQUEST_SYNC_FRAME = "request-sync"; 3303 3304 /** 3305 * Communicate additional parameter changes to the component instance. 3306 * <b>Note:</b> Some of these parameter changes may silently fail to apply. 3307 * 3308 * @param params The bundle of parameters to set. 3309 * @throws IllegalStateException if in the Released state. 3310 */ setParameters(@ullable Bundle params)3311 public final void setParameters(@Nullable Bundle params) { 3312 if (params == null) { 3313 return; 3314 } 3315 3316 String[] keys = new String[params.size()]; 3317 Object[] values = new Object[params.size()]; 3318 3319 int i = 0; 3320 for (final String key: params.keySet()) { 3321 keys[i] = key; 3322 values[i] = params.get(key); 3323 ++i; 3324 } 3325 3326 setParameters(keys, values); 3327 } 3328 3329 /** 3330 * Sets an asynchronous callback for actionable MediaCodec events. 3331 * 3332 * If the client intends to use the component in asynchronous mode, 3333 * a valid callback should be provided before {@link #configure} is called. 3334 * 3335 * When asynchronous callback is enabled, the client should not call 3336 * {@link #getInputBuffers}, {@link #getOutputBuffers}, 3337 * {@link #dequeueInputBuffer(long)} or {@link #dequeueOutputBuffer(BufferInfo, long)}. 3338 * <p> 3339 * Also, {@link #flush} behaves differently in asynchronous mode. After calling 3340 * {@code flush}, you must call {@link #start} to "resume" receiving input buffers, 3341 * even if an input surface was created. 3342 * 3343 * @param cb The callback that will run. Use {@code null} to clear a previously 3344 * set callback (before {@link #configure configure} is called and run 3345 * in synchronous mode). 3346 * @param handler Callbacks will happen on the handler's thread. If {@code null}, 3347 * callbacks are done on the default thread (the caller's thread or the 3348 * main thread.) 3349 */ setCallback(@ullable Callback cb, @Nullable Handler handler)3350 public void setCallback(@Nullable /* MediaCodec. */ Callback cb, @Nullable Handler handler) { 3351 if (cb != null) { 3352 synchronized (mListenerLock) { 3353 EventHandler newHandler = getEventHandlerOn(handler, mCallbackHandler); 3354 // NOTE: there are no callbacks on the handler at this time, but check anyways 3355 // even if we were to extend this to be callable dynamically, it must 3356 // be called when codec is flushed, so no messages are pending. 3357 if (newHandler != mCallbackHandler) { 3358 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 3359 mCallbackHandler.removeMessages(EVENT_CALLBACK); 3360 mCallbackHandler = newHandler; 3361 } 3362 } 3363 } else if (mCallbackHandler != null) { 3364 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 3365 mCallbackHandler.removeMessages(EVENT_CALLBACK); 3366 } 3367 3368 if (mCallbackHandler != null) { 3369 // set java callback on main handler 3370 Message msg = mCallbackHandler.obtainMessage(EVENT_SET_CALLBACK, 0, 0, cb); 3371 mCallbackHandler.sendMessage(msg); 3372 3373 // set native handler here, don't post to handler because 3374 // it may cause the callback to be delayed and set in a wrong state. 3375 // Note that native codec may start sending events to the callback 3376 // handler after this returns. 3377 native_setCallback(cb); 3378 } 3379 } 3380 3381 /** 3382 * Sets an asynchronous callback for actionable MediaCodec events on the default 3383 * looper. 3384 * <p> 3385 * Same as {@link #setCallback(Callback, Handler)} with handler set to null. 3386 * @param cb The callback that will run. Use {@code null} to clear a previously 3387 * set callback (before {@link #configure configure} is called and run 3388 * in synchronous mode). 3389 * @see #setCallback(Callback, Handler) 3390 */ setCallback(@ullable Callback cb)3391 public void setCallback(@Nullable /* MediaCodec. */ Callback cb) { 3392 setCallback(cb, null /* handler */); 3393 } 3394 3395 /** 3396 * Listener to be called when an output frame has rendered on the output surface 3397 * 3398 * @see MediaCodec#setOnFrameRenderedListener 3399 */ 3400 public interface OnFrameRenderedListener { 3401 3402 /** 3403 * Called when an output frame has rendered on the output surface. 3404 * <p> 3405 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 3406 * render timing samples, and can be significantly delayed and batched. Some frames may have 3407 * been rendered even if there was no callback generated. 3408 * 3409 * @param codec the MediaCodec instance 3410 * @param presentationTimeUs the presentation time (media time) of the frame rendered. 3411 * This is usually the same as specified in {@link #queueInputBuffer}; however, 3412 * some codecs may alter the media time by applying some time-based transformation, 3413 * such as frame rate conversion. In that case, presentation time corresponds 3414 * to the actual output frame rendered. 3415 * @param nanoTime The system time when the frame was rendered. 3416 * 3417 * @see System#nanoTime 3418 */ onFrameRendered( @onNull MediaCodec codec, long presentationTimeUs, long nanoTime)3419 public void onFrameRendered( 3420 @NonNull MediaCodec codec, long presentationTimeUs, long nanoTime); 3421 } 3422 3423 /** 3424 * Registers a callback to be invoked when an output frame is rendered on the output surface. 3425 * <p> 3426 * This method can be called in any codec state, but will only have an effect in the 3427 * Executing state for codecs that render buffers to the output surface. 3428 * <p> 3429 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 3430 * render timing samples, and can be significantly delayed and batched. Some frames may have 3431 * been rendered even if there was no callback generated. 3432 * 3433 * @param listener the callback that will be run 3434 * @param handler the callback will be run on the handler's thread. If {@code null}, 3435 * the callback will be run on the default thread, which is the looper 3436 * from which the codec was created, or a new thread if there was none. 3437 */ setOnFrameRenderedListener( @ullable OnFrameRenderedListener listener, @Nullable Handler handler)3438 public void setOnFrameRenderedListener( 3439 @Nullable OnFrameRenderedListener listener, @Nullable Handler handler) { 3440 synchronized (mListenerLock) { 3441 mOnFrameRenderedListener = listener; 3442 if (listener != null) { 3443 EventHandler newHandler = getEventHandlerOn(handler, mOnFrameRenderedHandler); 3444 if (newHandler != mOnFrameRenderedHandler) { 3445 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 3446 } 3447 mOnFrameRenderedHandler = newHandler; 3448 } else if (mOnFrameRenderedHandler != null) { 3449 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 3450 } 3451 native_enableOnFrameRenderedListener(listener != null); 3452 } 3453 } 3454 native_enableOnFrameRenderedListener(boolean enable)3455 private native void native_enableOnFrameRenderedListener(boolean enable); 3456 getEventHandlerOn( @ullable Handler handler, @NonNull EventHandler lastHandler)3457 private EventHandler getEventHandlerOn( 3458 @Nullable Handler handler, @NonNull EventHandler lastHandler) { 3459 if (handler == null) { 3460 return mEventHandler; 3461 } else { 3462 Looper looper = handler.getLooper(); 3463 if (lastHandler.getLooper() == looper) { 3464 return lastHandler; 3465 } else { 3466 return new EventHandler(this, looper); 3467 } 3468 } 3469 } 3470 3471 /** 3472 * MediaCodec callback interface. Used to notify the user asynchronously 3473 * of various MediaCodec events. 3474 */ 3475 public static abstract class Callback { 3476 /** 3477 * Called when an input buffer becomes available. 3478 * 3479 * @param codec The MediaCodec object. 3480 * @param index The index of the available input buffer. 3481 */ onInputBufferAvailable(@onNull MediaCodec codec, int index)3482 public abstract void onInputBufferAvailable(@NonNull MediaCodec codec, int index); 3483 3484 /** 3485 * Called when an output buffer becomes available. 3486 * 3487 * @param codec The MediaCodec object. 3488 * @param index The index of the available output buffer. 3489 * @param info Info regarding the available output buffer {@link MediaCodec.BufferInfo}. 3490 */ onOutputBufferAvailable( @onNull MediaCodec codec, int index, @NonNull BufferInfo info)3491 public abstract void onOutputBufferAvailable( 3492 @NonNull MediaCodec codec, int index, @NonNull BufferInfo info); 3493 3494 /** 3495 * Called when the MediaCodec encountered an error 3496 * 3497 * @param codec The MediaCodec object. 3498 * @param e The {@link MediaCodec.CodecException} object describing the error. 3499 */ onError(@onNull MediaCodec codec, @NonNull CodecException e)3500 public abstract void onError(@NonNull MediaCodec codec, @NonNull CodecException e); 3501 3502 /** 3503 * Called when the output format has changed 3504 * 3505 * @param codec The MediaCodec object. 3506 * @param format The new output format. 3507 */ onOutputFormatChanged( @onNull MediaCodec codec, @NonNull MediaFormat format)3508 public abstract void onOutputFormatChanged( 3509 @NonNull MediaCodec codec, @NonNull MediaFormat format); 3510 } 3511 postEventFromNative( int what, int arg1, int arg2, @Nullable Object obj)3512 private void postEventFromNative( 3513 int what, int arg1, int arg2, @Nullable Object obj) { 3514 synchronized (mListenerLock) { 3515 EventHandler handler = mEventHandler; 3516 if (what == EVENT_CALLBACK) { 3517 handler = mCallbackHandler; 3518 } else if (what == EVENT_FRAME_RENDERED) { 3519 handler = mOnFrameRenderedHandler; 3520 } 3521 if (handler != null) { 3522 Message msg = handler.obtainMessage(what, arg1, arg2, obj); 3523 handler.sendMessage(msg); 3524 } 3525 } 3526 } 3527 setParameters(@onNull String[] keys, @NonNull Object[] values)3528 private native final void setParameters(@NonNull String[] keys, @NonNull Object[] values); 3529 3530 /** 3531 * Get the codec info. If the codec was created by createDecoderByType 3532 * or createEncoderByType, what component is chosen is not known beforehand, 3533 * and thus the caller does not have the MediaCodecInfo. 3534 * @throws IllegalStateException if in the Released state. 3535 */ 3536 @NonNull getCodecInfo()3537 public MediaCodecInfo getCodecInfo() { 3538 // Get the codec name first. If the codec is already released, 3539 // IllegalStateException will be thrown here. 3540 String name = getName(); 3541 synchronized (mCodecInfoLock) { 3542 if (mCodecInfo == null) { 3543 // Get the codec info for this codec itself first. Only initialize 3544 // the full codec list if this somehow fails because it can be slow. 3545 mCodecInfo = getOwnCodecInfo(); 3546 if (mCodecInfo == null) { 3547 mCodecInfo = MediaCodecList.getInfoFor(name); 3548 } 3549 } 3550 return mCodecInfo; 3551 } 3552 } 3553 3554 @NonNull getOwnCodecInfo()3555 private native final MediaCodecInfo getOwnCodecInfo(); 3556 3557 @NonNull getBuffers(boolean input)3558 private native final ByteBuffer[] getBuffers(boolean input); 3559 3560 @Nullable getBuffer(boolean input, int index)3561 private native final ByteBuffer getBuffer(boolean input, int index); 3562 3563 @Nullable getImage(boolean input, int index)3564 private native final Image getImage(boolean input, int index); 3565 native_init()3566 private static native final void native_init(); 3567 native_setup( @onNull String name, boolean nameIsType, boolean encoder)3568 private native final void native_setup( 3569 @NonNull String name, boolean nameIsType, boolean encoder); 3570 native_finalize()3571 private native final void native_finalize(); 3572 3573 static { 3574 System.loadLibrary("media_jni"); native_init()3575 native_init(); 3576 } 3577 3578 private long mNativeContext; 3579 3580 /** @hide */ 3581 public static class MediaImage extends Image { 3582 private final boolean mIsReadOnly; 3583 private final int mWidth; 3584 private final int mHeight; 3585 private final int mFormat; 3586 private long mTimestamp; 3587 private final Plane[] mPlanes; 3588 private final ByteBuffer mBuffer; 3589 private final ByteBuffer mInfo; 3590 private final int mXOffset; 3591 private final int mYOffset; 3592 3593 private final static int TYPE_YUV = 1; 3594 3595 private final int mTransform = 0; //Default no transform 3596 private final int mScalingMode = 0; //Default frozen scaling mode 3597 3598 @Override getFormat()3599 public int getFormat() { 3600 throwISEIfImageIsInvalid(); 3601 return mFormat; 3602 } 3603 3604 @Override getHeight()3605 public int getHeight() { 3606 throwISEIfImageIsInvalid(); 3607 return mHeight; 3608 } 3609 3610 @Override getWidth()3611 public int getWidth() { 3612 throwISEIfImageIsInvalid(); 3613 return mWidth; 3614 } 3615 3616 @Override getTransform()3617 public int getTransform() { 3618 throwISEIfImageIsInvalid(); 3619 return mTransform; 3620 } 3621 3622 @Override getScalingMode()3623 public int getScalingMode() { 3624 throwISEIfImageIsInvalid(); 3625 return mScalingMode; 3626 } 3627 3628 @Override getTimestamp()3629 public long getTimestamp() { 3630 throwISEIfImageIsInvalid(); 3631 return mTimestamp; 3632 } 3633 3634 @Override 3635 @NonNull getPlanes()3636 public Plane[] getPlanes() { 3637 throwISEIfImageIsInvalid(); 3638 return Arrays.copyOf(mPlanes, mPlanes.length); 3639 } 3640 3641 @Override close()3642 public void close() { 3643 if (mIsImageValid) { 3644 java.nio.NioUtils.freeDirectBuffer(mBuffer); 3645 mIsImageValid = false; 3646 } 3647 } 3648 3649 /** 3650 * Set the crop rectangle associated with this frame. 3651 * <p> 3652 * The crop rectangle specifies the region of valid pixels in the image, 3653 * using coordinates in the largest-resolution plane. 3654 */ 3655 @Override setCropRect(@ullable Rect cropRect)3656 public void setCropRect(@Nullable Rect cropRect) { 3657 if (mIsReadOnly) { 3658 throw new ReadOnlyBufferException(); 3659 } 3660 super.setCropRect(cropRect); 3661 } 3662 3663 MediaImage( @onNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect)3664 public MediaImage( 3665 @NonNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, 3666 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) { 3667 mFormat = ImageFormat.YUV_420_888; 3668 mTimestamp = timestamp; 3669 mIsImageValid = true; 3670 mIsReadOnly = buffer.isReadOnly(); 3671 mBuffer = buffer.duplicate(); 3672 3673 // save offsets and info 3674 mXOffset = xOffset; 3675 mYOffset = yOffset; 3676 mInfo = info; 3677 3678 // read media-info. See MediaImage2 3679 if (info.remaining() == 104) { 3680 int type = info.getInt(); 3681 if (type != TYPE_YUV) { 3682 throw new UnsupportedOperationException("unsupported type: " + type); 3683 } 3684 int numPlanes = info.getInt(); 3685 if (numPlanes != 3) { 3686 throw new RuntimeException("unexpected number of planes: " + numPlanes); 3687 } 3688 mWidth = info.getInt(); 3689 mHeight = info.getInt(); 3690 if (mWidth < 1 || mHeight < 1) { 3691 throw new UnsupportedOperationException( 3692 "unsupported size: " + mWidth + "x" + mHeight); 3693 } 3694 int bitDepth = info.getInt(); 3695 if (bitDepth != 8) { 3696 throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth); 3697 } 3698 int bitDepthAllocated = info.getInt(); 3699 if (bitDepthAllocated != 8) { 3700 throw new UnsupportedOperationException( 3701 "unsupported allocated bit depth: " + bitDepthAllocated); 3702 } 3703 mPlanes = new MediaPlane[numPlanes]; 3704 for (int ix = 0; ix < numPlanes; ix++) { 3705 int planeOffset = info.getInt(); 3706 int colInc = info.getInt(); 3707 int rowInc = info.getInt(); 3708 int horiz = info.getInt(); 3709 int vert = info.getInt(); 3710 if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) { 3711 throw new UnsupportedOperationException("unexpected subsampling: " 3712 + horiz + "x" + vert + " on plane " + ix); 3713 } 3714 if (colInc < 1 || rowInc < 1) { 3715 throw new UnsupportedOperationException("unexpected strides: " 3716 + colInc + " pixel, " + rowInc + " row on plane " + ix); 3717 } 3718 3719 buffer.clear(); 3720 buffer.position(mBuffer.position() + planeOffset 3721 + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc); 3722 buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8) 3723 + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc); 3724 mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc); 3725 } 3726 } else { 3727 throw new UnsupportedOperationException( 3728 "unsupported info length: " + info.remaining()); 3729 } 3730 3731 if (cropRect == null) { 3732 cropRect = new Rect(0, 0, mWidth, mHeight); 3733 } 3734 cropRect.offset(-xOffset, -yOffset); 3735 super.setCropRect(cropRect); 3736 } 3737 3738 private class MediaPlane extends Plane { MediaPlane(@onNull ByteBuffer buffer, int rowInc, int colInc)3739 public MediaPlane(@NonNull ByteBuffer buffer, int rowInc, int colInc) { 3740 mData = buffer; 3741 mRowInc = rowInc; 3742 mColInc = colInc; 3743 } 3744 3745 @Override getRowStride()3746 public int getRowStride() { 3747 throwISEIfImageIsInvalid(); 3748 return mRowInc; 3749 } 3750 3751 @Override getPixelStride()3752 public int getPixelStride() { 3753 throwISEIfImageIsInvalid(); 3754 return mColInc; 3755 } 3756 3757 @Override 3758 @NonNull getBuffer()3759 public ByteBuffer getBuffer() { 3760 throwISEIfImageIsInvalid(); 3761 return mData; 3762 } 3763 3764 private final int mRowInc; 3765 private final int mColInc; 3766 private final ByteBuffer mData; 3767 } 3768 } 3769 3770 public final static class MetricsConstants 3771 { MetricsConstants()3772 private MetricsConstants() {} 3773 3774 /** 3775 * Key to extract the codec being used 3776 * from the {@link MediaCodec#getMetrics} return value. 3777 * The value is a String. 3778 */ 3779 public static final String CODEC = "android.media.mediacodec.codec"; 3780 3781 /** 3782 * Key to extract the MIME type 3783 * from the {@link MediaCodec#getMetrics} return value. 3784 * The value is a String. 3785 */ 3786 public static final String MIME_TYPE = "android.media.mediacodec.mime"; 3787 3788 /** 3789 * Key to extract what the codec mode 3790 * from the {@link MediaCodec#getMetrics} return value. 3791 * The value is a String. Values will be one of the constants 3792 * {@link #MODE_AUDIO} or {@link #MODE_VIDEO}. 3793 */ 3794 public static final String MODE = "android.media.mediacodec.mode"; 3795 3796 /** 3797 * The value returned for the key {@link #MODE} when the 3798 * codec is a audio codec. 3799 */ 3800 public static final String MODE_AUDIO = "audio"; 3801 3802 /** 3803 * The value returned for the key {@link #MODE} when the 3804 * codec is a video codec. 3805 */ 3806 public static final String MODE_VIDEO = "video"; 3807 3808 /** 3809 * Key to extract the flag indicating whether the codec is running 3810 * as an encoder or decoder from the {@link MediaCodec#getMetrics} return value. 3811 * The value is an integer. 3812 * A 0 indicates decoder; 1 indicates encoder. 3813 */ 3814 public static final String ENCODER = "android.media.mediacodec.encoder"; 3815 3816 /** 3817 * Key to extract the flag indicating whether the codec is running 3818 * in secure (DRM) mode from the {@link MediaCodec#getMetrics} return value. 3819 * The value is an integer. 3820 */ 3821 public static final String SECURE = "android.media.mediacodec.secure"; 3822 3823 /** 3824 * Key to extract the width (in pixels) of the video track 3825 * from the {@link MediaCodec#getMetrics} return value. 3826 * The value is an integer. 3827 */ 3828 public static final String WIDTH = "android.media.mediacodec.width"; 3829 3830 /** 3831 * Key to extract the height (in pixels) of the video track 3832 * from the {@link MediaCodec#getMetrics} return value. 3833 * The value is an integer. 3834 */ 3835 public static final String HEIGHT = "android.media.mediacodec.height"; 3836 3837 /** 3838 * Key to extract the rotation (in degrees) to properly orient the video 3839 * from the {@link MediaCodec#getMetrics} return. 3840 * The value is a integer. 3841 */ 3842 public static final String ROTATION = "android.media.mediacodec.rotation"; 3843 3844 } 3845 } 3846
