1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntDef; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.graphics.ImageFormat; 23 import android.graphics.Rect; 24 import android.graphics.SurfaceTexture; 25 import android.media.MediaCodecInfo.CodecCapabilities; 26 import android.os.Bundle; 27 import android.os.Handler; 28 import android.os.IBinder; 29 import android.os.Looper; 30 import android.os.Message; 31 import android.os.PersistableBundle; 32 import android.view.Surface; 33 34 import java.io.IOException; 35 import java.lang.annotation.Retention; 36 import java.lang.annotation.RetentionPolicy; 37 import java.nio.ByteBuffer; 38 import java.nio.ByteOrder; 39 import java.nio.ReadOnlyBufferException; 40 import java.util.Arrays; 41 import java.util.HashMap; 42 import java.util.Map; 43 44 /** 45 MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components. 46 It is part of the Android low-level multimedia support infrastructure (normally used together 47 with {@link MediaExtractor}, {@link MediaSync}, {@link MediaMuxer}, {@link MediaCrypto}, 48 {@link MediaDrm}, {@link Image}, {@link Surface}, and {@link AudioTrack}.) 49 <p> 50 <center><object style="width: 540px; height: 205px;" type="image/svg+xml" 51 data="../../../images/media/mediacodec_buffers.svg"><img 52 src="../../../images/media/mediacodec_buffers.png" style="width: 540px; height: 205px" 53 alt="MediaCodec buffer flow diagram"></object></center> 54 <p> 55 In broad terms, a codec processes input data to generate output data. It processes data 56 asynchronously and uses a set of input and output buffers. At a simplistic level, you request 57 (or receive) an empty input buffer, fill it up with data and send it to the codec for 58 processing. The codec uses up the data and transforms it into one of its empty output buffers. 59 Finally, you request (or receive) a filled output buffer, consume its contents and release it 60 back to the codec. 61 62 <h3>Data Types</h3> 63 <p> 64 Codecs operate on three kinds of data: compressed data, raw audio data and raw video data. 65 All three kinds of data can be processed using {@link ByteBuffer ByteBuffers}, but you should use 66 a {@link Surface} for raw video data to improve codec performance. Surface uses native video 67 buffers without mapping or copying them to ByteBuffers; thus, it is much more efficient. 68 You normally cannot access the raw video data when using a Surface, but you can use the 69 {@link ImageReader} class to access unsecured decoded (raw) video frames. This may still be more 70 efficient than using ByteBuffers, as some native buffers may be mapped into {@linkplain 71 ByteBuffer#isDirect direct} ByteBuffers. When using ByteBuffer mode, you can access raw video 72 frames using the {@link Image} class and {@link #getInputImage getInput}/{@link #getOutputImage 73 OutputImage(int)}. 74 75 <h4>Compressed Buffers</h4> 76 <p> 77 Input buffers (for decoders) and output buffers (for encoders) contain compressed data according 78 to the {@linkplain MediaFormat#KEY_MIME format's type}. For video types this is normally a single 79 compressed video frame. For audio data this is normally a single access unit (an encoded audio 80 segment typically containing a few milliseconds of audio as dictated by the format type), but 81 this requirement is slightly relaxed in that a buffer may contain multiple encoded access units 82 of audio. In either case, buffers do not start or end on arbitrary byte boundaries, but rather on 83 frame/access unit boundaries unless they are flagged with {@link #BUFFER_FLAG_PARTIAL_FRAME}. 84 85 <h4>Raw Audio Buffers</h4> 86 <p> 87 Raw audio buffers contain entire frames of PCM audio data, which is one sample for each channel 88 in channel order. Each sample is a {@linkplain AudioFormat#ENCODING_PCM_16BIT 16-bit signed 89 integer in native byte order}. 90 91 <pre class=prettyprint> 92 short[] getSamplesForChannel(MediaCodec codec, int bufferId, int channelIx) { 93 ByteBuffer outputBuffer = codec.getOutputBuffer(bufferId); 94 MediaFormat format = codec.getOutputFormat(bufferId); 95 ShortBuffer samples = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer(); 96 int numChannels = formet.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 97 if (channelIx < 0 || channelIx >= numChannels) { 98 return null; 99 } 100 short[] res = new short[samples.remaining() / numChannels]; 101 for (int i = 0; i < res.length; ++i) { 102 res[i] = samples.get(i * numChannels + channelIx); 103 } 104 return res; 105 }</pre> 106 107 <h4>Raw Video Buffers</h4> 108 <p> 109 In ByteBuffer mode video buffers are laid out according to their {@linkplain 110 MediaFormat#KEY_COLOR_FORMAT color format}. You can get the supported color formats as an array 111 from {@link #getCodecInfo}{@code .}{@link MediaCodecInfo#getCapabilitiesForType 112 getCapabilitiesForType(…)}{@code .}{@link CodecCapabilities#colorFormats colorFormats}. 113 Video codecs may support three kinds of color formats: 114 <ul> 115 <li><strong>native raw video format:</strong> This is marked by {@link 116 CodecCapabilities#COLOR_FormatSurface} and it can be used with an input or output Surface.</li> 117 <li><strong>flexible YUV buffers</strong> (such as {@link 118 CodecCapabilities#COLOR_FormatYUV420Flexible}): These can be used with an input/output Surface, 119 as well as in ByteBuffer mode, by using {@link #getInputImage getInput}/{@link #getOutputImage 120 OutputImage(int)}.</li> 121 <li><strong>other, specific formats:</strong> These are normally only supported in ByteBuffer 122 mode. Some color formats are vendor specific. Others are defined in {@link CodecCapabilities}. 123 For color formats that are equivalent to a flexible format, you can still use {@link 124 #getInputImage getInput}/{@link #getOutputImage OutputImage(int)}.</li> 125 </ul> 126 <p> 127 All video codecs support flexible YUV 4:2:0 buffers since {@link 128 android.os.Build.VERSION_CODES#LOLLIPOP_MR1}. 129 130 <h4>Accessing Raw Video ByteBuffers on Older Devices</h4> 131 <p> 132 Prior to {@link android.os.Build.VERSION_CODES#LOLLIPOP} and {@link Image} support, you need to 133 use the {@link MediaFormat#KEY_STRIDE} and {@link MediaFormat#KEY_SLICE_HEIGHT} output format 134 values to understand the layout of the raw output buffers. 135 <p class=note> 136 Note that on some devices the slice-height is advertised as 0. This could mean either that the 137 slice-height is the same as the frame height, or that the slice-height is the frame height 138 aligned to some value (usually a power of 2). Unfortunately, there is no standard and simple way 139 to tell the actual slice height in this case. Furthermore, the vertical stride of the {@code U} 140 plane in planar formats is also not specified or defined, though usually it is half of the slice 141 height. 142 <p> 143 The {@link MediaFormat#KEY_WIDTH} and {@link MediaFormat#KEY_HEIGHT} keys specify the size of the 144 video frames; however, for most encondings the video (picture) only occupies a portion of the 145 video frame. This is represented by the 'crop rectangle'. 146 <p> 147 You need to use the following keys to get the crop rectangle of raw output images from the 148 {@linkplain #getOutputFormat output format}. If these keys are not present, the video occupies the 149 entire video frame.The crop rectangle is understood in the context of the output frame 150 <em>before</em> applying any {@linkplain MediaFormat#KEY_ROTATION rotation}. 151 <table style="width: 0%"> 152 <thead> 153 <tr> 154 <th>Format Key</th> 155 <th>Type</th> 156 <th>Description</th> 157 </tr> 158 </thead> 159 <tbody> 160 <tr> 161 <td>{@code "crop-left"}</td> 162 <td>Integer</td> 163 <td>The left-coordinate (x) of the crop rectangle</td> 164 </tr><tr> 165 <td>{@code "crop-top"}</td> 166 <td>Integer</td> 167 <td>The top-coordinate (y) of the crop rectangle</td> 168 </tr><tr> 169 <td>{@code "crop-right"}</td> 170 <td>Integer</td> 171 <td>The right-coordinate (x) <strong>MINUS 1</strong> of the crop rectangle</td> 172 </tr><tr> 173 <td>{@code "crop-bottom"}</td> 174 <td>Integer</td> 175 <td>The bottom-coordinate (y) <strong>MINUS 1</strong> of the crop rectangle</td> 176 </tr><tr> 177 <td colspan=3> 178 The right and bottom coordinates can be understood as the coordinates of the right-most 179 valid column/bottom-most valid row of the cropped output image. 180 </td> 181 </tr> 182 </tbody> 183 </table> 184 <p> 185 The size of the video frame (before rotation) can be calculated as such: 186 <pre class=prettyprint> 187 MediaFormat format = decoder.getOutputFormat(…); 188 int width = format.getInteger(MediaFormat.KEY_WIDTH); 189 if (format.containsKey("crop-left") && format.containsKey("crop-right")) { 190 width = format.getInteger("crop-right") + 1 - format.getInteger("crop-left"); 191 } 192 int height = format.getInteger(MediaFormat.KEY_HEIGHT); 193 if (format.containsKey("crop-top") && format.containsKey("crop-bottom")) { 194 height = format.getInteger("crop-bottom") + 1 - format.getInteger("crop-top"); 195 } 196 </pre> 197 <p class=note> 198 Also note that the meaning of {@link BufferInfo#offset BufferInfo.offset} was not consistent across 199 devices. On some devices the offset pointed to the top-left pixel of the crop rectangle, while on 200 most devices it pointed to the top-left pixel of the entire frame. 201 202 <h3>States</h3> 203 <p> 204 During its life a codec conceptually exists in one of three states: Stopped, Executing or 205 Released. The Stopped collective state is actually the conglomeration of three states: 206 Uninitialized, Configured and Error, whereas the Executing state conceptually progresses through 207 three sub-states: Flushed, Running and End-of-Stream. 208 <p> 209 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 210 data="../../../images/media/mediacodec_states.svg"><img 211 src="../../../images/media/mediacodec_states.png" style="width: 519px; height: 356px" 212 alt="MediaCodec state diagram"></object></center> 213 <p> 214 When you create a codec using one of the factory methods, the codec is in the Uninitialized 215 state. First, you need to configure it via {@link #configure configure(…)}, which brings 216 it to the Configured state, then call {@link #start} to move it to the Executing state. In this 217 state you can process data through the buffer queue manipulation described above. 218 <p> 219 The Executing state has three sub-states: Flushed, Running and End-of-Stream. Immediately after 220 {@link #start} the codec is in the Flushed sub-state, where it holds all the buffers. As soon 221 as the first input buffer is dequeued, the codec moves to the Running sub-state, where it spends 222 most of its life. When you queue an input buffer with the {@linkplain #BUFFER_FLAG_END_OF_STREAM 223 end-of-stream marker}, the codec transitions to the End-of-Stream sub-state. In this state the 224 codec no longer accepts further input buffers, but still generates output buffers until the 225 end-of-stream is reached on the output. You can move back to the Flushed sub-state at any time 226 while in the Executing state using {@link #flush}. 227 <p> 228 Call {@link #stop} to return the codec to the Uninitialized state, whereupon it may be configured 229 again. When you are done using a codec, you must release it by calling {@link #release}. 230 <p> 231 On rare occasions the codec may encounter an error and move to the Error state. This is 232 communicated using an invalid return value from a queuing operation, or sometimes via an 233 exception. Call {@link #reset} to make the codec usable again. You can call it from any state to 234 move the codec back to the Uninitialized state. Otherwise, call {@link #release} to move to the 235 terminal Released state. 236 237 <h3>Creation</h3> 238 <p> 239 Use {@link MediaCodecList} to create a MediaCodec for a specific {@link MediaFormat}. When 240 decoding a file or a stream, you can get the desired format from {@link 241 MediaExtractor#getTrackFormat MediaExtractor.getTrackFormat}. Inject any specific features that 242 you want to add using {@link MediaFormat#setFeatureEnabled MediaFormat.setFeatureEnabled}, then 243 call {@link MediaCodecList#findDecoderForFormat MediaCodecList.findDecoderForFormat} to get the 244 name of a codec that can handle that specific media format. Finally, create the codec using 245 {@link #createByCodecName}. 246 <p class=note> 247 <strong>Note:</strong> On {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the format to 248 {@code MediaCodecList.findDecoder}/{@code EncoderForFormat} must not contain a {@linkplain 249 MediaFormat#KEY_FRAME_RATE frame rate}. Use 250 <code class=prettyprint>format.setString(MediaFormat.KEY_FRAME_RATE, null)</code> 251 to clear any existing frame rate setting in the format. 252 <p> 253 You can also create the preferred codec for a specific MIME type using {@link 254 #createDecoderByType createDecoder}/{@link #createEncoderByType EncoderByType(String)}. 255 This, however, cannot be used to inject features, and may create a codec that cannot handle the 256 specific desired media format. 257 258 <h4>Creating secure decoders</h4> 259 <p> 260 On versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and earlier, secure codecs might 261 not be listed in {@link MediaCodecList}, but may still be available on the system. Secure codecs 262 that exist can be instantiated by name only, by appending {@code ".secure"} to the name of a 263 regular codec (the name of all secure codecs must end in {@code ".secure"}.) {@link 264 #createByCodecName} will throw an {@code IOException} if the codec is not present on the system. 265 <p> 266 From {@link android.os.Build.VERSION_CODES#LOLLIPOP} onwards, you should use the {@link 267 CodecCapabilities#FEATURE_SecurePlayback} feature in the media format to create a secure decoder. 268 269 <h3>Initialization</h3> 270 <p> 271 After creating the codec, you can set a callback using {@link #setCallback setCallback} if you 272 want to process data asynchronously. Then, {@linkplain #configure configure} the codec using the 273 specific media format. This is when you can specify the output {@link Surface} for video 274 producers – codecs that generate raw video data (e.g. video decoders). This is also when 275 you can set the decryption parameters for secure codecs (see {@link MediaCrypto}). Finally, since 276 some codecs can operate in multiple modes, you must specify whether you want it to work as a 277 decoder or an encoder. 278 <p> 279 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you can query the resulting input and 280 output format in the Configured state. You can use this to verify the resulting configuration, 281 e.g. color formats, before starting the codec. 282 <p> 283 If you want to process raw input video buffers natively with a video consumer – a codec 284 that processes raw video input, such as a video encoder – create a destination Surface for 285 your input data using {@link #createInputSurface} after configuration. Alternately, set up the 286 codec to use a previously created {@linkplain #createPersistentInputSurface persistent input 287 surface} by calling {@link #setInputSurface}. 288 289 <h4 id=CSD><a name="CSD"></a>Codec-specific Data</h4> 290 <p> 291 Some formats, notably AAC audio and MPEG4, H.264 and H.265 video formats require the actual data 292 to be prefixed by a number of buffers containing setup data, or codec specific data. When 293 processing such compressed formats, this data must be submitted to the codec after {@link 294 #start} and before any frame data. Such data must be marked using the flag {@link 295 #BUFFER_FLAG_CODEC_CONFIG} in a call to {@link #queueInputBuffer queueInputBuffer}. 296 <p> 297 Codec-specific data can also be included in the format passed to {@link #configure configure} in 298 ByteBuffer entries with keys "csd-0", "csd-1", etc. These keys are always included in the track 299 {@link MediaFormat} obtained from the {@link MediaExtractor#getTrackFormat MediaExtractor}. 300 Codec-specific data in the format is automatically submitted to the codec upon {@link #start}; 301 you <strong>MUST NOT</strong> submit this data explicitly. If the format did not contain codec 302 specific data, you can choose to submit it using the specified number of buffers in the correct 303 order, according to the format requirements. In case of H.264 AVC, you can also concatenate all 304 codec-specific data and submit it as a single codec-config buffer. 305 <p> 306 Android uses the following codec-specific data buffers. These are also required to be set in 307 the track format for proper {@link MediaMuxer} track configuration. Each parameter set and the 308 codec-specific-data sections marked with (<sup>*</sup>) must start with a start code of 309 {@code "\x00\x00\x00\x01"}. 310 <p> 311 <style>td.NA { background: #ccc; } .mid > tr > td { vertical-align: middle; }</style> 312 <table> 313 <thead> 314 <th>Format</th> 315 <th>CSD buffer #0</th> 316 <th>CSD buffer #1</th> 317 <th>CSD buffer #2</th> 318 </thead> 319 <tbody class=mid> 320 <tr> 321 <td>AAC</td> 322 <td>Decoder-specific information from ESDS<sup>*</sup></td> 323 <td class=NA>Not Used</td> 324 <td class=NA>Not Used</td> 325 </tr> 326 <tr> 327 <td>VORBIS</td> 328 <td>Identification header</td> 329 <td>Setup header</td> 330 <td class=NA>Not Used</td> 331 </tr> 332 <tr> 333 <td>OPUS</td> 334 <td>Identification header</td> 335 <td>Pre-skip in nanosecs<br> 336 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)<br> 337 This overrides the pre-skip value in the identification header.</td> 338 <td>Seek Pre-roll in nanosecs<br> 339 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)</td> 340 </tr> 341 <tr> 342 <td>MPEG-4</td> 343 <td>Decoder-specific information from ESDS<sup>*</sup></td> 344 <td class=NA>Not Used</td> 345 <td class=NA>Not Used</td> 346 </tr> 347 <tr> 348 <td>H.264 AVC</td> 349 <td>SPS (Sequence Parameter Sets<sup>*</sup>)</td> 350 <td>PPS (Picture Parameter Sets<sup>*</sup>)</td> 351 <td class=NA>Not Used</td> 352 </tr> 353 <tr> 354 <td>H.265 HEVC</td> 355 <td>VPS (Video Parameter Sets<sup>*</sup>) +<br> 356 SPS (Sequence Parameter Sets<sup>*</sup>) +<br> 357 PPS (Picture Parameter Sets<sup>*</sup>)</td> 358 <td class=NA>Not Used</td> 359 <td class=NA>Not Used</td> 360 </tr> 361 <tr> 362 <td>VP9</td> 363 <td>VP9 <a href="http://wiki.webmproject.org/vp9-codecprivate">CodecPrivate</a> Data 364 (optional)</td> 365 <td class=NA>Not Used</td> 366 <td class=NA>Not Used</td> 367 </tr> 368 </tbody> 369 </table> 370 371 <p class=note> 372 <strong>Note:</strong> care must be taken if the codec is flushed immediately or shortly 373 after start, before any output buffer or output format change has been returned, as the codec 374 specific data may be lost during the flush. You must resubmit the data using buffers marked with 375 {@link #BUFFER_FLAG_CODEC_CONFIG} after such flush to ensure proper codec operation. 376 <p> 377 Encoders (or codecs that generate compressed data) will create and return the codec specific data 378 before any valid output buffer in output buffers marked with the {@linkplain 379 #BUFFER_FLAG_CODEC_CONFIG codec-config flag}. Buffers containing codec-specific-data have no 380 meaningful timestamps. 381 382 <h3>Data Processing</h3> 383 <p> 384 Each codec maintains a set of input and output buffers that are referred to by a buffer-ID in 385 API calls. After a successful call to {@link #start} the client "owns" neither input nor output 386 buffers. In synchronous mode, call {@link #dequeueInputBuffer dequeueInput}/{@link 387 #dequeueOutputBuffer OutputBuffer(…)} to obtain (get ownership of) an input or output 388 buffer from the codec. In asynchronous mode, you will automatically receive available buffers via 389 the {@link Callback#onInputBufferAvailable MediaCodec.Callback.onInput}/{@link 390 Callback#onOutputBufferAvailable OutputBufferAvailable(…)} callbacks. 391 <p> 392 Upon obtaining an input buffer, fill it with data and submit it to the codec using {@link 393 #queueInputBuffer queueInputBuffer} – or {@link #queueSecureInputBuffer 394 queueSecureInputBuffer} if using decryption. Do not submit multiple input buffers with the same 395 timestamp (unless it is <a href="#CSD">codec-specific data</a> marked as such). 396 <p> 397 The codec in turn will return a read-only output buffer via the {@link 398 Callback#onOutputBufferAvailable onOutputBufferAvailable} callback in asynchronous mode, or in 399 response to a {@link #dequeueOutputBuffer dequeuOutputBuffer} call in synchronous mode. After the 400 output buffer has been processed, call one of the {@link #releaseOutputBuffer 401 releaseOutputBuffer} methods to return the buffer to the codec. 402 <p> 403 While you are not required to resubmit/release buffers immediately to the codec, holding onto 404 input and/or output buffers may stall the codec, and this behavior is device dependent. 405 <strong>Specifically, it is possible that a codec may hold off on generating output buffers until 406 <em>all</em> outstanding buffers have been released/resubmitted.</strong> Therefore, try to 407 hold onto to available buffers as little as possible. 408 <p> 409 Depending on the API version, you can process data in three ways: 410 <table> 411 <thead> 412 <tr> 413 <th>Processing Mode</th> 414 <th>API version <= 20<br>Jelly Bean/KitKat</th> 415 <th>API version >= 21<br>Lollipop and later</th> 416 </tr> 417 </thead> 418 <tbody> 419 <tr> 420 <td>Synchronous API using buffer arrays</td> 421 <td>Supported</td> 422 <td>Deprecated</td> 423 </tr> 424 <tr> 425 <td>Synchronous API using buffers</td> 426 <td class=NA>Not Available</td> 427 <td>Supported</td> 428 </tr> 429 <tr> 430 <td>Asynchronous API using buffers</td> 431 <td class=NA>Not Available</td> 432 <td>Supported</td> 433 </tr> 434 </tbody> 435 </table> 436 437 <h4>Asynchronous Processing using Buffers</h4> 438 <p> 439 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the preferred method is to process data 440 asynchronously by setting a callback before calling {@link #configure configure}. Asynchronous 441 mode changes the state transitions slightly, because you must call {@link #start} after {@link 442 #flush} to transition the codec to the Running sub-state and start receiving input buffers. 443 Similarly, upon an initial call to {@code start} the codec will move directly to the Running 444 sub-state and start passing available input buffers via the callback. 445 <p> 446 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 447 data="../../../images/media/mediacodec_async_states.svg"><img 448 src="../../../images/media/mediacodec_async_states.png" style="width: 516px; height: 353px" 449 alt="MediaCodec state diagram for asynchronous operation"></object></center> 450 <p> 451 MediaCodec is typically used like this in asynchronous mode: 452 <pre class=prettyprint> 453 MediaCodec codec = MediaCodec.createByCodecName(name); 454 MediaFormat mOutputFormat; // member variable 455 codec.setCallback(new MediaCodec.Callback() { 456 {@literal @Override} 457 void onInputBufferAvailable(MediaCodec mc, int inputBufferId) { 458 ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); 459 // fill inputBuffer with valid data 460 … 461 codec.queueInputBuffer(inputBufferId, …); 462 } 463 464 {@literal @Override} 465 void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, …) { 466 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 467 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 468 // bufferFormat is equivalent to mOutputFormat 469 // outputBuffer is ready to be processed or rendered. 470 … 471 codec.releaseOutputBuffer(outputBufferId, …); 472 } 473 474 {@literal @Override} 475 void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { 476 // Subsequent data will conform to new format. 477 // Can ignore if using getOutputFormat(outputBufferId) 478 mOutputFormat = format; // option B 479 } 480 481 {@literal @Override} 482 void onError(…) { 483 … 484 } 485 }); 486 codec.configure(format, …); 487 mOutputFormat = codec.getOutputFormat(); // option B 488 codec.start(); 489 // wait for processing to complete 490 codec.stop(); 491 codec.release();</pre> 492 493 <h4>Synchronous Processing using Buffers</h4> 494 <p> 495 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you should retrieve input and output 496 buffers using {@link #getInputBuffer getInput}/{@link #getOutputBuffer OutputBuffer(int)} and/or 497 {@link #getInputImage getInput}/{@link #getOutputImage OutputImage(int)} even when using the 498 codec in synchronous mode. This allows certain optimizations by the framework, e.g. when 499 processing dynamic content. This optimization is disabled if you call {@link #getInputBuffers 500 getInput}/{@link #getOutputBuffers OutputBuffers()}. 501 502 <p class=note> 503 <strong>Note:</strong> do not mix the methods of using buffers and buffer arrays at the same 504 time. Specifically, only call {@code getInput}/{@code OutputBuffers} directly after {@link 505 #start} or after having dequeued an output buffer ID with the value of {@link 506 #INFO_OUTPUT_FORMAT_CHANGED}. 507 <p> 508 MediaCodec is typically used like this in synchronous mode: 509 <pre> 510 MediaCodec codec = MediaCodec.createByCodecName(name); 511 codec.configure(format, …); 512 MediaFormat outputFormat = codec.getOutputFormat(); // option B 513 codec.start(); 514 for (;;) { 515 int inputBufferId = codec.dequeueInputBuffer(timeoutUs); 516 if (inputBufferId >= 0) { 517 ByteBuffer inputBuffer = codec.getInputBuffer(…); 518 // fill inputBuffer with valid data 519 … 520 codec.queueInputBuffer(inputBufferId, …); 521 } 522 int outputBufferId = codec.dequeueOutputBuffer(…); 523 if (outputBufferId >= 0) { 524 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 525 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 526 // bufferFormat is identical to outputFormat 527 // outputBuffer is ready to be processed or rendered. 528 … 529 codec.releaseOutputBuffer(outputBufferId, …); 530 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 531 // Subsequent data will conform to new format. 532 // Can ignore if using getOutputFormat(outputBufferId) 533 outputFormat = codec.getOutputFormat(); // option B 534 } 535 } 536 codec.stop(); 537 codec.release();</pre> 538 539 <h4>Synchronous Processing using Buffer Arrays (deprecated)</h4> 540 <p> 541 In versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and before, the set of input and 542 output buffers are represented by the {@code ByteBuffer[]} arrays. After a successful call to 543 {@link #start}, retrieve the buffer arrays using {@link #getInputBuffers getInput}/{@link 544 #getOutputBuffers OutputBuffers()}. Use the buffer ID-s as indices into these arrays (when 545 non-negative), as demonstrated in the sample below. Note that there is no inherent correlation 546 between the size of the arrays and the number of input and output buffers used by the system, 547 although the array size provides an upper bound. 548 <pre> 549 MediaCodec codec = MediaCodec.createByCodecName(name); 550 codec.configure(format, …); 551 codec.start(); 552 ByteBuffer[] inputBuffers = codec.getInputBuffers(); 553 ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 554 for (;;) { 555 int inputBufferId = codec.dequeueInputBuffer(…); 556 if (inputBufferId >= 0) { 557 // fill inputBuffers[inputBufferId] with valid data 558 … 559 codec.queueInputBuffer(inputBufferId, …); 560 } 561 int outputBufferId = codec.dequeueOutputBuffer(…); 562 if (outputBufferId >= 0) { 563 // outputBuffers[outputBufferId] is ready to be processed or rendered. 564 … 565 codec.releaseOutputBuffer(outputBufferId, …); 566 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 567 outputBuffers = codec.getOutputBuffers(); 568 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 569 // Subsequent data will conform to new format. 570 MediaFormat format = codec.getOutputFormat(); 571 } 572 } 573 codec.stop(); 574 codec.release();</pre> 575 576 <h4>End-of-stream Handling</h4> 577 <p> 578 When you reach the end of the input data, you must signal it to the codec by specifying the 579 {@link #BUFFER_FLAG_END_OF_STREAM} flag in the call to {@link #queueInputBuffer 580 queueInputBuffer}. You can do this on the last valid input buffer, or by submitting an additional 581 empty input buffer with the end-of-stream flag set. If using an empty buffer, the timestamp will 582 be ignored. 583 <p> 584 The codec will continue to return output buffers until it eventually signals the end of the 585 output stream by specifying the same end-of-stream flag in the {@link BufferInfo} set in {@link 586 #dequeueOutputBuffer dequeueOutputBuffer} or returned via {@link Callback#onOutputBufferAvailable 587 onOutputBufferAvailable}. This can be set on the last valid output buffer, or on an empty buffer 588 after the last valid output buffer. The timestamp of such empty buffer should be ignored. 589 <p> 590 Do not submit additional input buffers after signaling the end of the input stream, unless the 591 codec has been flushed, or stopped and restarted. 592 593 <h4>Using an Output Surface</h4> 594 <p> 595 The data processing is nearly identical to the ByteBuffer mode when using an output {@link 596 Surface}; however, the output buffers will not be accessible, and are represented as {@code null} 597 values. E.g. {@link #getOutputBuffer getOutputBuffer}/{@link #getOutputImage Image(int)} will 598 return {@code null} and {@link #getOutputBuffers} will return an array containing only {@code 599 null}-s. 600 <p> 601 When using an output Surface, you can select whether or not to render each output buffer on the 602 surface. You have three choices: 603 <ul> 604 <li><strong>Do not render the buffer:</strong> Call {@link #releaseOutputBuffer(int, boolean) 605 releaseOutputBuffer(bufferId, false)}.</li> 606 <li><strong>Render the buffer with the default timestamp:</strong> Call {@link 607 #releaseOutputBuffer(int, boolean) releaseOutputBuffer(bufferId, true)}.</li> 608 <li><strong>Render the buffer with a specific timestamp:</strong> Call {@link 609 #releaseOutputBuffer(int, long) releaseOutputBuffer(bufferId, timestamp)}.</li> 610 </ul> 611 <p> 612 Since {@link android.os.Build.VERSION_CODES#M}, the default timestamp is the {@linkplain 613 BufferInfo#presentationTimeUs presentation timestamp} of the buffer (converted to nanoseconds). 614 It was not defined prior to that. 615 <p> 616 Also since {@link android.os.Build.VERSION_CODES#M}, you can change the output Surface 617 dynamically using {@link #setOutputSurface setOutputSurface}. 618 619 <h4>Transformations When Rendering onto Surface</h4> 620 621 If the codec is configured into Surface mode, any crop rectangle, {@linkplain 622 MediaFormat#KEY_ROTATION rotation} and {@linkplain #setVideoScalingMode video scaling 623 mode} will be automatically applied with one exception: 624 <p class=note> 625 Prior to the {@link android.os.Build.VERSION_CODES#M} release, software decoders may not 626 have applied the rotation when being rendered onto a Surface. Unfortunately, there is no standard 627 and simple way to identify software decoders, or if they apply the rotation other than by trying 628 it out. 629 <p> 630 There are also some caveats. 631 <p class=note> 632 Note that the pixel aspect ratio is not considered when displaying the output onto the 633 Surface. This means that if you are using {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT} mode, you 634 must position the output Surface so that it has the proper final display aspect ratio. Conversely, 635 you can only use {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode for content with 636 square pixels (pixel aspect ratio or 1:1). 637 <p class=note> 638 Note also that as of {@link android.os.Build.VERSION_CODES#N} release, {@link 639 #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode may not work correctly for videos rotated 640 by 90 or 270 degrees. 641 <p class=note> 642 When setting the video scaling mode, note that it must be reset after each time the output 643 buffers change. Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, you can 644 do this after each time the output format changes. 645 646 <h4>Using an Input Surface</h4> 647 <p> 648 When using an input Surface, there are no accessible input buffers, as buffers are automatically 649 passed from the input surface to the codec. Calling {@link #dequeueInputBuffer 650 dequeueInputBuffer} will throw an {@code IllegalStateException}, and {@link #getInputBuffers} 651 returns a bogus {@code ByteBuffer[]} array that <strong>MUST NOT</strong> be written into. 652 <p> 653 Call {@link #signalEndOfInputStream} to signal end-of-stream. The input surface will stop 654 submitting data to the codec immediately after this call. 655 <p> 656 657 <h3>Seeking & Adaptive Playback Support</h3> 658 <p> 659 Video decoders (and in general codecs that consume compressed video data) behave differently 660 regarding seek and format change whether or not they support and are configured for adaptive 661 playback. You can check if a decoder supports {@linkplain 662 CodecCapabilities#FEATURE_AdaptivePlayback adaptive playback} via {@link 663 CodecCapabilities#isFeatureSupported CodecCapabilities.isFeatureSupported(String)}. Adaptive 664 playback support for video decoders is only activated if you configure the codec to decode onto a 665 {@link Surface}. 666 667 <h4 id=KeyFrames><a name="KeyFrames"></a>Stream Boundary and Key Frames</h4> 668 <p> 669 It is important that the input data after {@link #start} or {@link #flush} starts at a suitable 670 stream boundary: the first frame must a key frame. A <em>key frame</em> can be decoded 671 completely on its own (for most codecs this means an I-frame), and no frames that are to be 672 displayed after a key frame refer to frames before the key frame. 673 <p> 674 The following table summarizes suitable key frames for various video formats. 675 <table> 676 <thead> 677 <tr> 678 <th>Format</th> 679 <th>Suitable key frame</th> 680 </tr> 681 </thead> 682 <tbody class=mid> 683 <tr> 684 <td>VP9/VP8</td> 685 <td>a suitable intraframe where no subsequent frames refer to frames prior to this frame.<br> 686 <i>(There is no specific name for such key frame.)</i></td> 687 </tr> 688 <tr> 689 <td>H.265 HEVC</td> 690 <td>IDR or CRA</td> 691 </tr> 692 <tr> 693 <td>H.264 AVC</td> 694 <td>IDR</td> 695 </tr> 696 <tr> 697 <td>MPEG-4<br>H.263<br>MPEG-2</td> 698 <td>a suitable I-frame where no subsequent frames refer to frames prior to this frame.<br> 699 <i>(There is no specific name for such key frame.)</td> 700 </tr> 701 </tbody> 702 </table> 703 704 <h4>For decoders that do not support adaptive playback (including when not decoding onto a 705 Surface)</h4> 706 <p> 707 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 708 seek) you <strong>MUST</strong> flush the decoder. Since all output buffers are immediately 709 revoked at the point of the flush, you may want to first signal then wait for the end-of-stream 710 before you call {@code flush}. It is important that the input data after a flush starts at a 711 suitable stream boundary/key frame. 712 <p class=note> 713 <strong>Note:</strong> the format of the data submitted after a flush must not change; {@link 714 #flush} does not support format discontinuities; for that, a full {@link #stop} - {@link 715 #configure configure(…)} - {@link #start} cycle is necessary. 716 717 <p class=note> 718 <strong>Also note:</strong> if you flush the codec too soon after {@link #start} – 719 generally, before the first output buffer or output format change is received – you 720 will need to resubmit the codec-specific-data to the codec. See the <a 721 href="#CSD">codec-specific-data section</a> for more info. 722 723 <h4>For decoders that support and are configured for adaptive playback</h4> 724 <p> 725 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 726 seek) it is <em>not necessary</em> to flush the decoder; however, input data after the 727 discontinuity must start at a suitable stream boundary/key frame. 728 <p> 729 For some video formats - namely H.264, H.265, VP8 and VP9 - it is also possible to change the 730 picture size or configuration mid-stream. To do this you must package the entire new 731 codec-specific configuration data together with the key frame into a single buffer (including 732 any start codes), and submit it as a <strong>regular</strong> input buffer. 733 <p> 734 You will receive an {@link #INFO_OUTPUT_FORMAT_CHANGED} return value from {@link 735 #dequeueOutputBuffer dequeueOutputBuffer} or a {@link Callback#onOutputBufferAvailable 736 onOutputFormatChanged} callback just after the picture-size change takes place and before any 737 frames with the new size have been returned. 738 <p class=note> 739 <strong>Note:</strong> just as the case for codec-specific data, be careful when calling 740 {@link #flush} shortly after you have changed the picture size. If you have not received 741 confirmation of the picture size change, you will need to repeat the request for the new picture 742 size. 743 744 <h3>Error handling</h3> 745 <p> 746 The factory methods {@link #createByCodecName createByCodecName} and {@link #createDecoderByType 747 createDecoder}/{@link #createEncoderByType EncoderByType} throw {@code IOException} on failure 748 which you must catch or declare to pass up. MediaCodec methods throw {@code 749 IllegalStateException} when the method is called from a codec state that does not allow it; this 750 is typically due to incorrect application API usage. Methods involving secure buffers may throw 751 {@link CryptoException}, which has further error information obtainable from {@link 752 CryptoException#getErrorCode}. 753 <p> 754 Internal codec errors result in a {@link CodecException}, which may be due to media content 755 corruption, hardware failure, resource exhaustion, and so forth, even when the application is 756 correctly using the API. The recommended action when receiving a {@code CodecException} 757 can be determined by calling {@link CodecException#isRecoverable} and {@link 758 CodecException#isTransient}: 759 <ul> 760 <li><strong>recoverable errors:</strong> If {@code isRecoverable()} returns true, then call 761 {@link #stop}, {@link #configure configure(…)}, and {@link #start} to recover.</li> 762 <li><strong>transient errors:</strong> If {@code isTransient()} returns true, then resources are 763 temporarily unavailable and the method may be retried at a later time.</li> 764 <li><strong>fatal errors:</strong> If both {@code isRecoverable()} and {@code isTransient()} 765 return false, then the {@code CodecException} is fatal and the codec must be {@linkplain #reset 766 reset} or {@linkplain #release released}.</li> 767 </ul> 768 <p> 769 Both {@code isRecoverable()} and {@code isTransient()} do not return true at the same time. 770 771 <h2 id=History><a name="History"></a>Valid API Calls and API History</h2> 772 <p> 773 This sections summarizes the valid API calls in each state and the API history of the MediaCodec 774 class. For API version numbers, see {@link android.os.Build.VERSION_CODES}. 775 776 <style> 777 .api > tr > th, .api > tr > td { text-align: center; padding: 4px 4px; } 778 .api > tr > th { vertical-align: bottom; } 779 .api > tr > td { vertical-align: middle; } 780 .sml > tr > th, .sml > tr > td { text-align: center; padding: 2px 4px; } 781 .fn { text-align: left; } 782 .fn > code > a { font: 14px/19px Roboto Condensed, sans-serif; } 783 .deg45 { 784 white-space: nowrap; background: none; border: none; vertical-align: bottom; 785 width: 30px; height: 83px; 786 } 787 .deg45 > div { 788 transform: skew(-45deg, 0deg) translate(1px, -67px); 789 transform-origin: bottom left 0; 790 width: 30px; height: 20px; 791 } 792 .deg45 > div > div { border: 1px solid #ddd; background: #999; height: 90px; width: 42px; } 793 .deg45 > div > div > div { transform: skew(45deg, 0deg) translate(-55px, 55px) rotate(-45deg); } 794 </style> 795 796 <table align="right" style="width: 0%"> 797 <thead> 798 <tr><th>Symbol</th><th>Meaning</th></tr> 799 </thead> 800 <tbody class=sml> 801 <tr><td>●</td><td>Supported</td></tr> 802 <tr><td>⁕</td><td>Semantics changed</td></tr> 803 <tr><td>○</td><td>Experimental support</td></tr> 804 <tr><td>[ ]</td><td>Deprecated</td></tr> 805 <tr><td>⎋</td><td>Restricted to surface input mode</td></tr> 806 <tr><td>⎆</td><td>Restricted to surface output mode</td></tr> 807 <tr><td>▧</td><td>Restricted to ByteBuffer input mode</td></tr> 808 <tr><td>↩</td><td>Restricted to synchronous mode</td></tr> 809 <tr><td>⇄</td><td>Restricted to asynchronous mode</td></tr> 810 <tr><td>( )</td><td>Can be called, but shouldn't</td></tr> 811 </tbody> 812 </table> 813 814 <table style="width: 100%;"> 815 <thead class=api> 816 <tr> 817 <th class=deg45><div><div style="background:#4285f4"><div>Uninitialized</div></div></div></th> 818 <th class=deg45><div><div style="background:#f4b400"><div>Configured</div></div></div></th> 819 <th class=deg45><div><div style="background:#e67c73"><div>Flushed</div></div></div></th> 820 <th class=deg45><div><div style="background:#0f9d58"><div>Running</div></div></div></th> 821 <th class=deg45><div><div style="background:#f7cb4d"><div>End of Stream</div></div></div></th> 822 <th class=deg45><div><div style="background:#db4437"><div>Error</div></div></div></th> 823 <th class=deg45><div><div style="background:#666"><div>Released</div></div></div></th> 824 <th></th> 825 <th colspan="8">SDK Version</th> 826 </tr> 827 <tr> 828 <th colspan="7">State</th> 829 <th>Method</th> 830 <th>16</th> 831 <th>17</th> 832 <th>18</th> 833 <th>19</th> 834 <th>20</th> 835 <th>21</th> 836 <th>22</th> 837 <th>23</th> 838 </tr> 839 </thead> 840 <tbody class=api> 841 <tr> 842 <td></td> 843 <td></td> 844 <td></td> 845 <td></td> 846 <td></td> 847 <td></td> 848 <td></td> 849 <td class=fn>{@link #createByCodecName createByCodecName}</td> 850 <td>●</td> 851 <td>●</td> 852 <td>●</td> 853 <td>●</td> 854 <td>●</td> 855 <td>●</td> 856 <td>●</td> 857 <td>●</td> 858 </tr> 859 <tr> 860 <td></td> 861 <td></td> 862 <td></td> 863 <td></td> 864 <td></td> 865 <td></td> 866 <td></td> 867 <td class=fn>{@link #createDecoderByType createDecoderByType}</td> 868 <td>●</td> 869 <td>●</td> 870 <td>●</td> 871 <td>●</td> 872 <td>●</td> 873 <td>●</td> 874 <td>●</td> 875 <td>●</td> 876 </tr> 877 <tr> 878 <td></td> 879 <td></td> 880 <td></td> 881 <td></td> 882 <td></td> 883 <td></td> 884 <td></td> 885 <td class=fn>{@link #createEncoderByType createEncoderByType}</td> 886 <td>●</td> 887 <td>●</td> 888 <td>●</td> 889 <td>●</td> 890 <td>●</td> 891 <td>●</td> 892 <td>●</td> 893 <td>●</td> 894 </tr> 895 <tr> 896 <td></td> 897 <td></td> 898 <td></td> 899 <td></td> 900 <td></td> 901 <td></td> 902 <td></td> 903 <td class=fn>{@link #createPersistentInputSurface createPersistentInputSurface}</td> 904 <td></td> 905 <td></td> 906 <td></td> 907 <td></td> 908 <td></td> 909 <td></td> 910 <td></td> 911 <td>●</td> 912 </tr> 913 <tr> 914 <td>16+</td> 915 <td>-</td> 916 <td>-</td> 917 <td>-</td> 918 <td>-</td> 919 <td>-</td> 920 <td>-</td> 921 <td class=fn>{@link #configure configure}</td> 922 <td>●</td> 923 <td>●</td> 924 <td>●</td> 925 <td>●</td> 926 <td>●</td> 927 <td>⁕</td> 928 <td>●</td> 929 <td>●</td> 930 </tr> 931 <tr> 932 <td>-</td> 933 <td>18+</td> 934 <td>-</td> 935 <td>-</td> 936 <td>-</td> 937 <td>-</td> 938 <td>-</td> 939 <td class=fn>{@link #createInputSurface createInputSurface}</td> 940 <td></td> 941 <td></td> 942 <td>⎋</td> 943 <td>⎋</td> 944 <td>⎋</td> 945 <td>⎋</td> 946 <td>⎋</td> 947 <td>⎋</td> 948 </tr> 949 <tr> 950 <td>-</td> 951 <td>-</td> 952 <td>16+</td> 953 <td>16+</td> 954 <td>(16+)</td> 955 <td>-</td> 956 <td>-</td> 957 <td class=fn>{@link #dequeueInputBuffer dequeueInputBuffer}</td> 958 <td>●</td> 959 <td>●</td> 960 <td>▧</td> 961 <td>▧</td> 962 <td>▧</td> 963 <td>⁕▧↩</td> 964 <td>▧↩</td> 965 <td>▧↩</td> 966 </tr> 967 <tr> 968 <td>-</td> 969 <td>-</td> 970 <td>16+</td> 971 <td>16+</td> 972 <td>16+</td> 973 <td>-</td> 974 <td>-</td> 975 <td class=fn>{@link #dequeueOutputBuffer dequeueOutputBuffer}</td> 976 <td>●</td> 977 <td>●</td> 978 <td>●</td> 979 <td>●</td> 980 <td>●</td> 981 <td>⁕↩</td> 982 <td>↩</td> 983 <td>↩</td> 984 </tr> 985 <tr> 986 <td>-</td> 987 <td>-</td> 988 <td>16+</td> 989 <td>16+</td> 990 <td>16+</td> 991 <td>-</td> 992 <td>-</td> 993 <td class=fn>{@link #flush flush}</td> 994 <td>●</td> 995 <td>●</td> 996 <td>●</td> 997 <td>●</td> 998 <td>●</td> 999 <td>●</td> 1000 <td>●</td> 1001 <td>●</td> 1002 </tr> 1003 <tr> 1004 <td>18+</td> 1005 <td>18+</td> 1006 <td>18+</td> 1007 <td>18+</td> 1008 <td>18+</td> 1009 <td>18+</td> 1010 <td>-</td> 1011 <td class=fn>{@link #getCodecInfo getCodecInfo}</td> 1012 <td></td> 1013 <td></td> 1014 <td>●</td> 1015 <td>●</td> 1016 <td>●</td> 1017 <td>●</td> 1018 <td>●</td> 1019 <td>●</td> 1020 </tr> 1021 <tr> 1022 <td>-</td> 1023 <td>-</td> 1024 <td>(21+)</td> 1025 <td>21+</td> 1026 <td>(21+)</td> 1027 <td>-</td> 1028 <td>-</td> 1029 <td class=fn>{@link #getInputBuffer getInputBuffer}</td> 1030 <td></td> 1031 <td></td> 1032 <td></td> 1033 <td></td> 1034 <td></td> 1035 <td>●</td> 1036 <td>●</td> 1037 <td>●</td> 1038 </tr> 1039 <tr> 1040 <td>-</td> 1041 <td>-</td> 1042 <td>16+</td> 1043 <td>(16+)</td> 1044 <td>(16+)</td> 1045 <td>-</td> 1046 <td>-</td> 1047 <td class=fn>{@link #getInputBuffers getInputBuffers}</td> 1048 <td>●</td> 1049 <td>●</td> 1050 <td>●</td> 1051 <td>●</td> 1052 <td>●</td> 1053 <td>[⁕↩]</td> 1054 <td>[↩]</td> 1055 <td>[↩]</td> 1056 </tr> 1057 <tr> 1058 <td>-</td> 1059 <td>21+</td> 1060 <td>(21+)</td> 1061 <td>(21+)</td> 1062 <td>(21+)</td> 1063 <td>-</td> 1064 <td>-</td> 1065 <td class=fn>{@link #getInputFormat getInputFormat}</td> 1066 <td></td> 1067 <td></td> 1068 <td></td> 1069 <td></td> 1070 <td></td> 1071 <td>●</td> 1072 <td>●</td> 1073 <td>●</td> 1074 </tr> 1075 <tr> 1076 <td>-</td> 1077 <td>-</td> 1078 <td>(21+)</td> 1079 <td>21+</td> 1080 <td>(21+)</td> 1081 <td>-</td> 1082 <td>-</td> 1083 <td class=fn>{@link #getInputImage getInputImage}</td> 1084 <td></td> 1085 <td></td> 1086 <td></td> 1087 <td></td> 1088 <td></td> 1089 <td>○</td> 1090 <td>●</td> 1091 <td>●</td> 1092 </tr> 1093 <tr> 1094 <td>18+</td> 1095 <td>18+</td> 1096 <td>18+</td> 1097 <td>18+</td> 1098 <td>18+</td> 1099 <td>18+</td> 1100 <td>-</td> 1101 <td class=fn>{@link #getName getName}</td> 1102 <td></td> 1103 <td></td> 1104 <td>●</td> 1105 <td>●</td> 1106 <td>●</td> 1107 <td>●</td> 1108 <td>●</td> 1109 <td>●</td> 1110 </tr> 1111 <tr> 1112 <td>-</td> 1113 <td>-</td> 1114 <td>(21+)</td> 1115 <td>21+</td> 1116 <td>21+</td> 1117 <td>-</td> 1118 <td>-</td> 1119 <td class=fn>{@link #getOutputBuffer getOutputBuffer}</td> 1120 <td></td> 1121 <td></td> 1122 <td></td> 1123 <td></td> 1124 <td></td> 1125 <td>●</td> 1126 <td>●</td> 1127 <td>●</td> 1128 </tr> 1129 <tr> 1130 <td>-</td> 1131 <td>-</td> 1132 <td>16+</td> 1133 <td>16+</td> 1134 <td>16+</td> 1135 <td>-</td> 1136 <td>-</td> 1137 <td class=fn>{@link #getOutputBuffers getOutputBuffers}</td> 1138 <td>●</td> 1139 <td>●</td> 1140 <td>●</td> 1141 <td>●</td> 1142 <td>●</td> 1143 <td>[⁕↩]</td> 1144 <td>[↩]</td> 1145 <td>[↩]</td> 1146 </tr> 1147 <tr> 1148 <td>-</td> 1149 <td>21+</td> 1150 <td>16+</td> 1151 <td>16+</td> 1152 <td>16+</td> 1153 <td>-</td> 1154 <td>-</td> 1155 <td class=fn>{@link #getOutputFormat()}</td> 1156 <td>●</td> 1157 <td>●</td> 1158 <td>●</td> 1159 <td>●</td> 1160 <td>●</td> 1161 <td>●</td> 1162 <td>●</td> 1163 <td>●</td> 1164 </tr> 1165 <tr> 1166 <td>-</td> 1167 <td>-</td> 1168 <td>(21+)</td> 1169 <td>21+</td> 1170 <td>21+</td> 1171 <td>-</td> 1172 <td>-</td> 1173 <td class=fn>{@link #getOutputFormat(int)}</td> 1174 <td></td> 1175 <td></td> 1176 <td></td> 1177 <td></td> 1178 <td></td> 1179 <td>●</td> 1180 <td>●</td> 1181 <td>●</td> 1182 </tr> 1183 <tr> 1184 <td>-</td> 1185 <td>-</td> 1186 <td>(21+)</td> 1187 <td>21+</td> 1188 <td>21+</td> 1189 <td>-</td> 1190 <td>-</td> 1191 <td class=fn>{@link #getOutputImage getOutputImage}</td> 1192 <td></td> 1193 <td></td> 1194 <td></td> 1195 <td></td> 1196 <td></td> 1197 <td>○</td> 1198 <td>●</td> 1199 <td>●</td> 1200 </tr> 1201 <tr> 1202 <td>-</td> 1203 <td>-</td> 1204 <td>-</td> 1205 <td>16+</td> 1206 <td>(16+)</td> 1207 <td>-</td> 1208 <td>-</td> 1209 <td class=fn>{@link #queueInputBuffer queueInputBuffer}</td> 1210 <td>●</td> 1211 <td>●</td> 1212 <td>●</td> 1213 <td>●</td> 1214 <td>●</td> 1215 <td>⁕</td> 1216 <td>●</td> 1217 <td>●</td> 1218 </tr> 1219 <tr> 1220 <td>-</td> 1221 <td>-</td> 1222 <td>-</td> 1223 <td>16+</td> 1224 <td>(16+)</td> 1225 <td>-</td> 1226 <td>-</td> 1227 <td class=fn>{@link #queueSecureInputBuffer queueSecureInputBuffer}</td> 1228 <td>●</td> 1229 <td>●</td> 1230 <td>●</td> 1231 <td>●</td> 1232 <td>●</td> 1233 <td>⁕</td> 1234 <td>●</td> 1235 <td>●</td> 1236 </tr> 1237 <tr> 1238 <td>16+</td> 1239 <td>16+</td> 1240 <td>16+</td> 1241 <td>16+</td> 1242 <td>16+</td> 1243 <td>16+</td> 1244 <td>16+</td> 1245 <td class=fn>{@link #release release}</td> 1246 <td>●</td> 1247 <td>●</td> 1248 <td>●</td> 1249 <td>●</td> 1250 <td>●</td> 1251 <td>●</td> 1252 <td>●</td> 1253 <td>●</td> 1254 </tr> 1255 <tr> 1256 <td>-</td> 1257 <td>-</td> 1258 <td>-</td> 1259 <td>16+</td> 1260 <td>16+</td> 1261 <td>-</td> 1262 <td>-</td> 1263 <td class=fn>{@link #releaseOutputBuffer(int, boolean)}</td> 1264 <td>●</td> 1265 <td>●</td> 1266 <td>●</td> 1267 <td>●</td> 1268 <td>●</td> 1269 <td>⁕</td> 1270 <td>●</td> 1271 <td>⁕</td> 1272 </tr> 1273 <tr> 1274 <td>-</td> 1275 <td>-</td> 1276 <td>-</td> 1277 <td>21+</td> 1278 <td>21+</td> 1279 <td>-</td> 1280 <td>-</td> 1281 <td class=fn>{@link #releaseOutputBuffer(int, long)}</td> 1282 <td></td> 1283 <td></td> 1284 <td></td> 1285 <td></td> 1286 <td></td> 1287 <td>⎆</td> 1288 <td>⎆</td> 1289 <td>⎆</td> 1290 </tr> 1291 <tr> 1292 <td>21+</td> 1293 <td>21+</td> 1294 <td>21+</td> 1295 <td>21+</td> 1296 <td>21+</td> 1297 <td>21+</td> 1298 <td>-</td> 1299 <td class=fn>{@link #reset reset}</td> 1300 <td></td> 1301 <td></td> 1302 <td></td> 1303 <td></td> 1304 <td></td> 1305 <td>●</td> 1306 <td>●</td> 1307 <td>●</td> 1308 </tr> 1309 <tr> 1310 <td>21+</td> 1311 <td>-</td> 1312 <td>-</td> 1313 <td>-</td> 1314 <td>-</td> 1315 <td>-</td> 1316 <td>-</td> 1317 <td class=fn>{@link #setCallback(Callback) setCallback}</td> 1318 <td></td> 1319 <td></td> 1320 <td></td> 1321 <td></td> 1322 <td></td> 1323 <td>●</td> 1324 <td>●</td> 1325 <td>{@link #setCallback(Callback, Handler) ⁕}</td> 1326 </tr> 1327 <tr> 1328 <td>-</td> 1329 <td>23+</td> 1330 <td>-</td> 1331 <td>-</td> 1332 <td>-</td> 1333 <td>-</td> 1334 <td>-</td> 1335 <td class=fn>{@link #setInputSurface setInputSurface}</td> 1336 <td></td> 1337 <td></td> 1338 <td></td> 1339 <td></td> 1340 <td></td> 1341 <td></td> 1342 <td></td> 1343 <td>⎋</td> 1344 </tr> 1345 <tr> 1346 <td>23+</td> 1347 <td>23+</td> 1348 <td>23+</td> 1349 <td>23+</td> 1350 <td>23+</td> 1351 <td>(23+)</td> 1352 <td>(23+)</td> 1353 <td class=fn>{@link #setOnFrameRenderedListener setOnFrameRenderedListener}</td> 1354 <td></td> 1355 <td></td> 1356 <td></td> 1357 <td></td> 1358 <td></td> 1359 <td></td> 1360 <td></td> 1361 <td>○ ⎆</td> 1362 </tr> 1363 <tr> 1364 <td>-</td> 1365 <td>23+</td> 1366 <td>23+</td> 1367 <td>23+</td> 1368 <td>23+</td> 1369 <td>-</td> 1370 <td>-</td> 1371 <td class=fn>{@link #setOutputSurface setOutputSurface}</td> 1372 <td></td> 1373 <td></td> 1374 <td></td> 1375 <td></td> 1376 <td></td> 1377 <td></td> 1378 <td></td> 1379 <td>⎆</td> 1380 </tr> 1381 <tr> 1382 <td>19+</td> 1383 <td>19+</td> 1384 <td>19+</td> 1385 <td>19+</td> 1386 <td>19+</td> 1387 <td>(19+)</td> 1388 <td>-</td> 1389 <td class=fn>{@link #setParameters setParameters}</td> 1390 <td></td> 1391 <td></td> 1392 <td></td> 1393 <td>●</td> 1394 <td>●</td> 1395 <td>●</td> 1396 <td>●</td> 1397 <td>●</td> 1398 </tr> 1399 <tr> 1400 <td>-</td> 1401 <td>(16+)</td> 1402 <td>(16+)</td> 1403 <td>16+</td> 1404 <td>(16+)</td> 1405 <td>(16+)</td> 1406 <td>-</td> 1407 <td class=fn>{@link #setVideoScalingMode setVideoScalingMode}</td> 1408 <td>⎆</td> 1409 <td>⎆</td> 1410 <td>⎆</td> 1411 <td>⎆</td> 1412 <td>⎆</td> 1413 <td>⎆</td> 1414 <td>⎆</td> 1415 <td>⎆</td> 1416 </tr> 1417 <tr> 1418 <td>-</td> 1419 <td>-</td> 1420 <td>18+</td> 1421 <td>18+</td> 1422 <td>-</td> 1423 <td>-</td> 1424 <td>-</td> 1425 <td class=fn>{@link #signalEndOfInputStream signalEndOfInputStream}</td> 1426 <td></td> 1427 <td></td> 1428 <td>⎋</td> 1429 <td>⎋</td> 1430 <td>⎋</td> 1431 <td>⎋</td> 1432 <td>⎋</td> 1433 <td>⎋</td> 1434 </tr> 1435 <tr> 1436 <td>-</td> 1437 <td>16+</td> 1438 <td>21+(⇄)</td> 1439 <td>-</td> 1440 <td>-</td> 1441 <td>-</td> 1442 <td>-</td> 1443 <td class=fn>{@link #start start}</td> 1444 <td>●</td> 1445 <td>●</td> 1446 <td>●</td> 1447 <td>●</td> 1448 <td>●</td> 1449 <td>⁕</td> 1450 <td>●</td> 1451 <td>●</td> 1452 </tr> 1453 <tr> 1454 <td>-</td> 1455 <td>-</td> 1456 <td>16+</td> 1457 <td>16+</td> 1458 <td>16+</td> 1459 <td>-</td> 1460 <td>-</td> 1461 <td class=fn>{@link #stop stop}</td> 1462 <td>●</td> 1463 <td>●</td> 1464 <td>●</td> 1465 <td>●</td> 1466 <td>●</td> 1467 <td>●</td> 1468 <td>●</td> 1469 <td>●</td> 1470 </tr> 1471 </tbody> 1472 </table> 1473 */ 1474 final public class MediaCodec { 1475 /** 1476 * Per buffer metadata includes an offset and size specifying 1477 * the range of valid data in the associated codec (output) buffer. 1478 */ 1479 public final static class BufferInfo { 1480 /** 1481 * Update the buffer metadata information. 1482 * 1483 * @param newOffset the start-offset of the data in the buffer. 1484 * @param newSize the amount of data (in bytes) in the buffer. 1485 * @param newTimeUs the presentation timestamp in microseconds. 1486 * @param newFlags buffer flags associated with the buffer. This 1487 * should be a combination of {@link #BUFFER_FLAG_KEY_FRAME} and 1488 * {@link #BUFFER_FLAG_END_OF_STREAM}. 1489 */ set( int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags)1490 public void set( 1491 int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags) { 1492 offset = newOffset; 1493 size = newSize; 1494 presentationTimeUs = newTimeUs; 1495 flags = newFlags; 1496 } 1497 1498 /** 1499 * The start-offset of the data in the buffer. 1500 */ 1501 public int offset; 1502 1503 /** 1504 * The amount of data (in bytes) in the buffer. If this is {@code 0}, 1505 * the buffer has no data in it and can be discarded. The only 1506 * use of a 0-size buffer is to carry the end-of-stream marker. 1507 */ 1508 public int size; 1509 1510 /** 1511 * The presentation timestamp in microseconds for the buffer. 1512 * This is derived from the presentation timestamp passed in 1513 * with the corresponding input buffer. This should be ignored for 1514 * a 0-sized buffer. 1515 */ 1516 public long presentationTimeUs; 1517 1518 /** 1519 * Buffer flags associated with the buffer. A combination of 1520 * {@link #BUFFER_FLAG_KEY_FRAME} and {@link #BUFFER_FLAG_END_OF_STREAM}. 1521 * 1522 * <p>Encoded buffers that are key frames are marked with 1523 * {@link #BUFFER_FLAG_KEY_FRAME}. 1524 * 1525 * <p>The last output buffer corresponding to the input buffer 1526 * marked with {@link #BUFFER_FLAG_END_OF_STREAM} will also be marked 1527 * with {@link #BUFFER_FLAG_END_OF_STREAM}. In some cases this could 1528 * be an empty buffer, whose sole purpose is to carry the end-of-stream 1529 * marker. 1530 */ 1531 @BufferFlag 1532 public int flags; 1533 1534 /** @hide */ 1535 @NonNull dup()1536 public BufferInfo dup() { 1537 BufferInfo copy = new BufferInfo(); 1538 copy.set(offset, size, presentationTimeUs, flags); 1539 return copy; 1540 } 1541 }; 1542 1543 // The follow flag constants MUST stay in sync with their equivalents 1544 // in MediaCodec.h ! 1545 1546 /** 1547 * This indicates that the (encoded) buffer marked as such contains 1548 * the data for a key frame. 1549 * 1550 * @deprecated Use {@link #BUFFER_FLAG_KEY_FRAME} instead. 1551 */ 1552 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 1553 1554 /** 1555 * This indicates that the (encoded) buffer marked as such contains 1556 * the data for a key frame. 1557 */ 1558 public static final int BUFFER_FLAG_KEY_FRAME = 1; 1559 1560 /** 1561 * This indicated that the buffer marked as such contains codec 1562 * initialization / codec specific data instead of media data. 1563 */ 1564 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 1565 1566 /** 1567 * This signals the end of stream, i.e. no buffers will be available 1568 * after this, unless of course, {@link #flush} follows. 1569 */ 1570 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 1571 1572 /** 1573 * This indicates that the buffer only contains part of a frame, 1574 * and the decoder should batch the data until a buffer without 1575 * this flag appears before decoding the frame. 1576 */ 1577 public static final int BUFFER_FLAG_PARTIAL_FRAME = 8; 1578 1579 /** @hide */ 1580 @IntDef( 1581 flag = true, 1582 value = { 1583 BUFFER_FLAG_SYNC_FRAME, 1584 BUFFER_FLAG_KEY_FRAME, 1585 BUFFER_FLAG_CODEC_CONFIG, 1586 BUFFER_FLAG_END_OF_STREAM, 1587 BUFFER_FLAG_PARTIAL_FRAME, 1588 }) 1589 @Retention(RetentionPolicy.SOURCE) 1590 public @interface BufferFlag {} 1591 1592 private EventHandler mEventHandler; 1593 private EventHandler mOnFrameRenderedHandler; 1594 private EventHandler mCallbackHandler; 1595 private Callback mCallback; 1596 private OnFrameRenderedListener mOnFrameRenderedListener; 1597 private Object mListenerLock = new Object(); 1598 1599 private static final int EVENT_CALLBACK = 1; 1600 private static final int EVENT_SET_CALLBACK = 2; 1601 private static final int EVENT_FRAME_RENDERED = 3; 1602 1603 private static final int CB_INPUT_AVAILABLE = 1; 1604 private static final int CB_OUTPUT_AVAILABLE = 2; 1605 private static final int CB_ERROR = 3; 1606 private static final int CB_OUTPUT_FORMAT_CHANGE = 4; 1607 1608 private class EventHandler extends Handler { 1609 private MediaCodec mCodec; 1610 EventHandler(@onNull MediaCodec codec, @NonNull Looper looper)1611 public EventHandler(@NonNull MediaCodec codec, @NonNull Looper looper) { 1612 super(looper); 1613 mCodec = codec; 1614 } 1615 1616 @Override handleMessage(@onNull Message msg)1617 public void handleMessage(@NonNull Message msg) { 1618 switch (msg.what) { 1619 case EVENT_CALLBACK: 1620 { 1621 handleCallback(msg); 1622 break; 1623 } 1624 case EVENT_SET_CALLBACK: 1625 { 1626 mCallback = (MediaCodec.Callback) msg.obj; 1627 break; 1628 } 1629 case EVENT_FRAME_RENDERED: 1630 synchronized (mListenerLock) { 1631 Map<String, Object> map = (Map<String, Object>)msg.obj; 1632 for (int i = 0; ; ++i) { 1633 Object mediaTimeUs = map.get(i + "-media-time-us"); 1634 Object systemNano = map.get(i + "-system-nano"); 1635 if (mediaTimeUs == null || systemNano == null 1636 || mOnFrameRenderedListener == null) { 1637 break; 1638 } 1639 mOnFrameRenderedListener.onFrameRendered( 1640 mCodec, (long)mediaTimeUs, (long)systemNano); 1641 } 1642 break; 1643 } 1644 default: 1645 { 1646 break; 1647 } 1648 } 1649 } 1650 handleCallback(@onNull Message msg)1651 private void handleCallback(@NonNull Message msg) { 1652 if (mCallback == null) { 1653 return; 1654 } 1655 1656 switch (msg.arg1) { 1657 case CB_INPUT_AVAILABLE: 1658 { 1659 int index = msg.arg2; 1660 synchronized(mBufferLock) { 1661 validateInputByteBuffer(mCachedInputBuffers, index); 1662 } 1663 mCallback.onInputBufferAvailable(mCodec, index); 1664 break; 1665 } 1666 1667 case CB_OUTPUT_AVAILABLE: 1668 { 1669 int index = msg.arg2; 1670 BufferInfo info = (MediaCodec.BufferInfo) msg.obj; 1671 synchronized(mBufferLock) { 1672 validateOutputByteBuffer(mCachedOutputBuffers, index, info); 1673 } 1674 mCallback.onOutputBufferAvailable( 1675 mCodec, index, info); 1676 break; 1677 } 1678 1679 case CB_ERROR: 1680 { 1681 mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj); 1682 break; 1683 } 1684 1685 case CB_OUTPUT_FORMAT_CHANGE: 1686 { 1687 mCallback.onOutputFormatChanged(mCodec, 1688 new MediaFormat((Map<String, Object>) msg.obj)); 1689 break; 1690 } 1691 1692 default: 1693 { 1694 break; 1695 } 1696 } 1697 } 1698 } 1699 1700 private boolean mHasSurface = false; 1701 1702 /** 1703 * Instantiate the preferred decoder supporting input data of the given mime type. 1704 * 1705 * The following is a partial list of defined mime types and their semantics: 1706 * <ul> 1707 * <li>"video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm) 1708 * <li>"video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm) 1709 * <li>"video/avc" - H.264/AVC video 1710 * <li>"video/hevc" - H.265/HEVC video 1711 * <li>"video/mp4v-es" - MPEG4 video 1712 * <li>"video/3gpp" - H.263 video 1713 * <li>"audio/3gpp" - AMR narrowband audio 1714 * <li>"audio/amr-wb" - AMR wideband audio 1715 * <li>"audio/mpeg" - MPEG1/2 audio layer III 1716 * <li>"audio/mp4a-latm" - AAC audio (note, this is raw AAC packets, not packaged in LATM!) 1717 * <li>"audio/vorbis" - vorbis audio 1718 * <li>"audio/g711-alaw" - G.711 alaw audio 1719 * <li>"audio/g711-mlaw" - G.711 ulaw audio 1720 * </ul> 1721 * 1722 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findDecoderForFormat} 1723 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1724 * given format. 1725 * 1726 * @param type The mime type of the input data. 1727 * @throws IOException if the codec cannot be created. 1728 * @throws IllegalArgumentException if type is not a valid mime type. 1729 * @throws NullPointerException if type is null. 1730 */ 1731 @NonNull createDecoderByType(@onNull String type)1732 public static MediaCodec createDecoderByType(@NonNull String type) 1733 throws IOException { 1734 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 1735 } 1736 1737 /** 1738 * Instantiate the preferred encoder supporting output data of the given mime type. 1739 * 1740 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findEncoderForFormat} 1741 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1742 * given format. 1743 * 1744 * @param type The desired mime type of the output data. 1745 * @throws IOException if the codec cannot be created. 1746 * @throws IllegalArgumentException if type is not a valid mime type. 1747 * @throws NullPointerException if type is null. 1748 */ 1749 @NonNull createEncoderByType(@onNull String type)1750 public static MediaCodec createEncoderByType(@NonNull String type) 1751 throws IOException { 1752 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 1753 } 1754 1755 /** 1756 * If you know the exact name of the component you want to instantiate 1757 * use this method to instantiate it. Use with caution. 1758 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 1759 * @param name The name of the codec to be instantiated. 1760 * @throws IOException if the codec cannot be created. 1761 * @throws IllegalArgumentException if name is not valid. 1762 * @throws NullPointerException if name is null. 1763 */ 1764 @NonNull createByCodecName(@onNull String name)1765 public static MediaCodec createByCodecName(@NonNull String name) 1766 throws IOException { 1767 return new MediaCodec( 1768 name, false /* nameIsType */, false /* unused */); 1769 } 1770 MediaCodec( @onNull String name, boolean nameIsType, boolean encoder)1771 private MediaCodec( 1772 @NonNull String name, boolean nameIsType, boolean encoder) { 1773 Looper looper; 1774 if ((looper = Looper.myLooper()) != null) { 1775 mEventHandler = new EventHandler(this, looper); 1776 } else if ((looper = Looper.getMainLooper()) != null) { 1777 mEventHandler = new EventHandler(this, looper); 1778 } else { 1779 mEventHandler = null; 1780 } 1781 mCallbackHandler = mEventHandler; 1782 mOnFrameRenderedHandler = mEventHandler; 1783 1784 mBufferLock = new Object(); 1785 1786 native_setup(name, nameIsType, encoder); 1787 } 1788 1789 @Override finalize()1790 protected void finalize() { 1791 native_finalize(); 1792 } 1793 1794 /** 1795 * Returns the codec to its initial (Uninitialized) state. 1796 * 1797 * Call this if an {@link MediaCodec.CodecException#isRecoverable unrecoverable} 1798 * error has occured to reset the codec to its initial state after creation. 1799 * 1800 * @throws CodecException if an unrecoverable error has occured and the codec 1801 * could not be reset. 1802 * @throws IllegalStateException if in the Released state. 1803 */ reset()1804 public final void reset() { 1805 freeAllTrackedBuffers(); // free buffers first 1806 native_reset(); 1807 } 1808 native_reset()1809 private native final void native_reset(); 1810 1811 /** 1812 * Free up resources used by the codec instance. 1813 * 1814 * Make sure you call this when you're done to free up any opened 1815 * component instance instead of relying on the garbage collector 1816 * to do this for you at some point in the future. 1817 */ release()1818 public final void release() { 1819 freeAllTrackedBuffers(); // free buffers first 1820 native_release(); 1821 } 1822 native_release()1823 private native final void native_release(); 1824 1825 /** 1826 * If this codec is to be used as an encoder, pass this flag. 1827 */ 1828 public static final int CONFIGURE_FLAG_ENCODE = 1; 1829 1830 /** @hide */ 1831 @IntDef(flag = true, value = { CONFIGURE_FLAG_ENCODE }) 1832 @Retention(RetentionPolicy.SOURCE) 1833 public @interface ConfigureFlag {} 1834 1835 /** 1836 * Configures a component. 1837 * 1838 * @param format The format of the input data (decoder) or the desired 1839 * format of the output data (encoder). Passing {@code null} 1840 * as {@code format} is equivalent to passing an 1841 * {@link MediaFormat#MediaFormat an empty mediaformat}. 1842 * @param surface Specify a surface on which to render the output of this 1843 * decoder. Pass {@code null} as {@code surface} if the 1844 * codec does not generate raw video output (e.g. not a video 1845 * decoder) and/or if you want to configure the codec for 1846 * {@link ByteBuffer} output. 1847 * @param crypto Specify a crypto object to facilitate secure decryption 1848 * of the media data. Pass {@code null} as {@code crypto} for 1849 * non-secure codecs. 1850 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 1851 * component as an encoder. 1852 * @throws IllegalArgumentException if the surface has been released (or is invalid), 1853 * or the format is unacceptable (e.g. missing a mandatory key), 1854 * or the flags are not set properly 1855 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 1856 * @throws IllegalStateException if not in the Uninitialized state. 1857 * @throws CryptoException upon DRM error. 1858 * @throws CodecException upon codec error. 1859 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @ConfigureFlag int flags)1860 public void configure( 1861 @Nullable MediaFormat format, 1862 @Nullable Surface surface, @Nullable MediaCrypto crypto, 1863 @ConfigureFlag int flags) { 1864 configure(format, surface, crypto, null, flags); 1865 } 1866 1867 /** 1868 * Configure a component to be used with a descrambler. 1869 * @param format The format of the input data (decoder) or the desired 1870 * format of the output data (encoder). Passing {@code null} 1871 * as {@code format} is equivalent to passing an 1872 * {@link MediaFormat#MediaFormat an empty mediaformat}. 1873 * @param surface Specify a surface on which to render the output of this 1874 * decoder. Pass {@code null} as {@code surface} if the 1875 * codec does not generate raw video output (e.g. not a video 1876 * decoder) and/or if you want to configure the codec for 1877 * {@link ByteBuffer} output. 1878 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 1879 * component as an encoder. 1880 * @param descrambler Specify a descrambler object to facilitate secure 1881 * descrambling of the media data, or null for non-secure codecs. 1882 * @throws IllegalArgumentException if the surface has been released (or is invalid), 1883 * or the format is unacceptable (e.g. missing a mandatory key), 1884 * or the flags are not set properly 1885 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 1886 * @throws IllegalStateException if not in the Uninitialized state. 1887 * @throws CryptoException upon DRM error. 1888 * @throws CodecException upon codec error. 1889 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler)1890 public void configure( 1891 @Nullable MediaFormat format, @Nullable Surface surface, 1892 @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler) { 1893 configure(format, surface, null, 1894 descrambler != null ? descrambler.getBinder() : null, flags); 1895 } 1896 configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IBinder descramblerBinder, @ConfigureFlag int flags)1897 private void configure( 1898 @Nullable MediaFormat format, @Nullable Surface surface, 1899 @Nullable MediaCrypto crypto, @Nullable IBinder descramblerBinder, 1900 @ConfigureFlag int flags) { 1901 if (crypto != null && descramblerBinder != null) { 1902 throw new IllegalArgumentException("Can't use crypto and descrambler together!"); 1903 } 1904 1905 String[] keys = null; 1906 Object[] values = null; 1907 1908 if (format != null) { 1909 Map<String, Object> formatMap = format.getMap(); 1910 keys = new String[formatMap.size()]; 1911 values = new Object[formatMap.size()]; 1912 1913 int i = 0; 1914 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 1915 if (entry.getKey().equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 1916 int sessionId = 0; 1917 try { 1918 sessionId = (Integer)entry.getValue(); 1919 } 1920 catch (Exception e) { 1921 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 1922 } 1923 keys[i] = "audio-hw-sync"; 1924 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 1925 } else { 1926 keys[i] = entry.getKey(); 1927 values[i] = entry.getValue(); 1928 } 1929 ++i; 1930 } 1931 } 1932 1933 mHasSurface = surface != null; 1934 1935 native_configure(keys, values, surface, crypto, descramblerBinder, flags); 1936 } 1937 1938 /** 1939 * Dynamically sets the output surface of a codec. 1940 * <p> 1941 * This can only be used if the codec was configured with an output surface. The 1942 * new output surface should have a compatible usage type to the original output surface. 1943 * E.g. codecs may not support switching from a SurfaceTexture (GPU readable) output 1944 * to ImageReader (software readable) output. 1945 * @param surface the output surface to use. It must not be {@code null}. 1946 * @throws IllegalStateException if the codec does not support setting the output 1947 * surface in the current state. 1948 * @throws IllegalArgumentException if the new surface is not of a suitable type for the codec. 1949 */ setOutputSurface(@onNull Surface surface)1950 public void setOutputSurface(@NonNull Surface surface) { 1951 if (!mHasSurface) { 1952 throw new IllegalStateException("codec was not configured for an output surface"); 1953 } 1954 native_setSurface(surface); 1955 } 1956 native_setSurface(@onNull Surface surface)1957 private native void native_setSurface(@NonNull Surface surface); 1958 1959 /** 1960 * Create a persistent input surface that can be used with codecs that normally have an input 1961 * surface, such as video encoders. A persistent input can be reused by subsequent 1962 * {@link MediaCodec} or {@link MediaRecorder} instances, but can only be used by at 1963 * most one codec or recorder instance concurrently. 1964 * <p> 1965 * The application is responsible for calling release() on the Surface when done. 1966 * 1967 * @return an input surface that can be used with {@link #setInputSurface}. 1968 */ 1969 @NonNull createPersistentInputSurface()1970 public static Surface createPersistentInputSurface() { 1971 return native_createPersistentInputSurface(); 1972 } 1973 1974 static class PersistentSurface extends Surface { 1975 @SuppressWarnings("unused") PersistentSurface()1976 PersistentSurface() {} // used by native 1977 1978 @Override release()1979 public void release() { 1980 native_releasePersistentInputSurface(this); 1981 super.release(); 1982 } 1983 1984 private long mPersistentObject; 1985 }; 1986 1987 /** 1988 * Configures the codec (e.g. encoder) to use a persistent input surface in place of input 1989 * buffers. This may only be called after {@link #configure} and before {@link #start}, in 1990 * lieu of {@link #createInputSurface}. 1991 * @param surface a persistent input surface created by {@link #createPersistentInputSurface} 1992 * @throws IllegalStateException if not in the Configured state or does not require an input 1993 * surface. 1994 * @throws IllegalArgumentException if the surface was not created by 1995 * {@link #createPersistentInputSurface}. 1996 */ setInputSurface(@onNull Surface surface)1997 public void setInputSurface(@NonNull Surface surface) { 1998 if (!(surface instanceof PersistentSurface)) { 1999 throw new IllegalArgumentException("not a PersistentSurface"); 2000 } 2001 native_setInputSurface(surface); 2002 } 2003 2004 @NonNull native_createPersistentInputSurface()2005 private static native final PersistentSurface native_createPersistentInputSurface(); native_releasePersistentInputSurface(@onNull Surface surface)2006 private static native final void native_releasePersistentInputSurface(@NonNull Surface surface); native_setInputSurface(@onNull Surface surface)2007 private native final void native_setInputSurface(@NonNull Surface surface); 2008 native_setCallback(@ullable Callback cb)2009 private native final void native_setCallback(@Nullable Callback cb); 2010 native_configure( @ullable String[] keys, @Nullable Object[] values, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IBinder descramblerBinder, @ConfigureFlag int flags)2011 private native final void native_configure( 2012 @Nullable String[] keys, @Nullable Object[] values, 2013 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2014 @Nullable IBinder descramblerBinder, @ConfigureFlag int flags); 2015 2016 /** 2017 * Requests a Surface to use as the input to an encoder, in place of input buffers. This 2018 * may only be called after {@link #configure} and before {@link #start}. 2019 * <p> 2020 * The application is responsible for calling release() on the Surface when 2021 * done. 2022 * <p> 2023 * The Surface must be rendered with a hardware-accelerated API, such as OpenGL ES. 2024 * {@link android.view.Surface#lockCanvas(android.graphics.Rect)} may fail or produce 2025 * unexpected results. 2026 * @throws IllegalStateException if not in the Configured state. 2027 */ 2028 @NonNull createInputSurface()2029 public native final Surface createInputSurface(); 2030 2031 /** 2032 * After successfully configuring the component, call {@code start}. 2033 * <p> 2034 * Call {@code start} also if the codec is configured in asynchronous mode, 2035 * and it has just been flushed, to resume requesting input buffers. 2036 * @throws IllegalStateException if not in the Configured state 2037 * or just after {@link #flush} for a codec that is configured 2038 * in asynchronous mode. 2039 * @throws MediaCodec.CodecException upon codec error. Note that some codec errors 2040 * for start may be attributed to future method calls. 2041 */ start()2042 public final void start() { 2043 native_start(); 2044 synchronized(mBufferLock) { 2045 cacheBuffers(true /* input */); 2046 cacheBuffers(false /* input */); 2047 } 2048 } native_start()2049 private native final void native_start(); 2050 2051 /** 2052 * Finish the decode/encode session, note that the codec instance 2053 * remains active and ready to be {@link #start}ed again. 2054 * To ensure that it is available to other client call {@link #release} 2055 * and don't just rely on garbage collection to eventually do this for you. 2056 * @throws IllegalStateException if in the Released state. 2057 */ stop()2058 public final void stop() { 2059 native_stop(); 2060 freeAllTrackedBuffers(); 2061 2062 synchronized (mListenerLock) { 2063 if (mCallbackHandler != null) { 2064 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 2065 mCallbackHandler.removeMessages(EVENT_CALLBACK); 2066 } 2067 if (mOnFrameRenderedHandler != null) { 2068 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 2069 } 2070 } 2071 } 2072 native_stop()2073 private native final void native_stop(); 2074 2075 /** 2076 * Flush both input and output ports of the component. 2077 * <p> 2078 * Upon return, all indices previously returned in calls to {@link #dequeueInputBuffer 2079 * dequeueInputBuffer} and {@link #dequeueOutputBuffer dequeueOutputBuffer} — or obtained 2080 * via {@link Callback#onInputBufferAvailable onInputBufferAvailable} or 2081 * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} callbacks — become 2082 * invalid, and all buffers are owned by the codec. 2083 * <p> 2084 * If the codec is configured in asynchronous mode, call {@link #start} 2085 * after {@code flush} has returned to resume codec operations. The codec 2086 * will not request input buffers until this has happened. 2087 * <strong>Note, however, that there may still be outstanding {@code onOutputBufferAvailable} 2088 * callbacks that were not handled prior to calling {@code flush}. 2089 * The indices returned via these callbacks also become invalid upon calling {@code flush} and 2090 * should be discarded.</strong> 2091 * <p> 2092 * If the codec is configured in synchronous mode, codec will resume 2093 * automatically if it is configured with an input surface. Otherwise, it 2094 * will resume when {@link #dequeueInputBuffer dequeueInputBuffer} is called. 2095 * 2096 * @throws IllegalStateException if not in the Executing state. 2097 * @throws MediaCodec.CodecException upon codec error. 2098 */ flush()2099 public final void flush() { 2100 synchronized(mBufferLock) { 2101 invalidateByteBuffers(mCachedInputBuffers); 2102 invalidateByteBuffers(mCachedOutputBuffers); 2103 mDequeuedInputBuffers.clear(); 2104 mDequeuedOutputBuffers.clear(); 2105 } 2106 native_flush(); 2107 } 2108 native_flush()2109 private native final void native_flush(); 2110 2111 /** 2112 * Thrown when an internal codec error occurs. 2113 */ 2114 public final static class CodecException extends IllegalStateException { CodecException(int errorCode, int actionCode, @Nullable String detailMessage)2115 CodecException(int errorCode, int actionCode, @Nullable String detailMessage) { 2116 super(detailMessage); 2117 mErrorCode = errorCode; 2118 mActionCode = actionCode; 2119 2120 // TODO get this from codec 2121 final String sign = errorCode < 0 ? "neg_" : ""; 2122 mDiagnosticInfo = 2123 "android.media.MediaCodec.error_" + sign + Math.abs(errorCode); 2124 } 2125 2126 /** 2127 * Returns true if the codec exception is a transient issue, 2128 * perhaps due to resource constraints, and that the method 2129 * (or encoding/decoding) may be retried at a later time. 2130 */ 2131 public boolean isTransient() { 2132 return mActionCode == ACTION_TRANSIENT; 2133 } 2134 2135 /** 2136 * Returns true if the codec cannot proceed further, 2137 * but can be recovered by stopping, configuring, 2138 * and starting again. 2139 */ 2140 public boolean isRecoverable() { 2141 return mActionCode == ACTION_RECOVERABLE; 2142 } 2143 2144 /** 2145 * Retrieve the error code associated with a CodecException 2146 */ 2147 public int getErrorCode() { 2148 return mErrorCode; 2149 } 2150 2151 /** 2152 * Retrieve a developer-readable diagnostic information string 2153 * associated with the exception. Do not show this to end-users, 2154 * since this string will not be localized or generally 2155 * comprehensible to end-users. 2156 */ 2157 public @NonNull String getDiagnosticInfo() { 2158 return mDiagnosticInfo; 2159 } 2160 2161 /** 2162 * This indicates required resource was not able to be allocated. 2163 */ 2164 public static final int ERROR_INSUFFICIENT_RESOURCE = 1100; 2165 2166 /** 2167 * This indicates the resource manager reclaimed the media resource used by the codec. 2168 * <p> 2169 * With this exception, the codec must be released, as it has moved to terminal state. 2170 */ 2171 public static final int ERROR_RECLAIMED = 1101; 2172 2173 /** @hide */ 2174 @IntDef({ 2175 ERROR_INSUFFICIENT_RESOURCE, 2176 ERROR_RECLAIMED, 2177 }) 2178 @Retention(RetentionPolicy.SOURCE) 2179 public @interface ReasonCode {} 2180 2181 /* Must be in sync with android_media_MediaCodec.cpp */ 2182 private final static int ACTION_TRANSIENT = 1; 2183 private final static int ACTION_RECOVERABLE = 2; 2184 2185 private final String mDiagnosticInfo; 2186 private final int mErrorCode; 2187 private final int mActionCode; 2188 } 2189 2190 /** 2191 * Thrown when a crypto error occurs while queueing a secure input buffer. 2192 */ 2193 public final static class CryptoException extends RuntimeException { 2194 public CryptoException(int errorCode, @Nullable String detailMessage) { 2195 super(detailMessage); 2196 mErrorCode = errorCode; 2197 } 2198 2199 /** 2200 * This indicates that the requested key was not found when trying to 2201 * perform a decrypt operation. The operation can be retried after adding 2202 * the correct decryption key. 2203 */ 2204 public static final int ERROR_NO_KEY = 1; 2205 2206 /** 2207 * This indicates that the key used for decryption is no longer 2208 * valid due to license term expiration. The operation can be retried 2209 * after updating the expired keys. 2210 */ 2211 public static final int ERROR_KEY_EXPIRED = 2; 2212 2213 /** 2214 * This indicates that a required crypto resource was not able to be 2215 * allocated while attempting the requested operation. The operation 2216 * can be retried if the app is able to release resources. 2217 */ 2218 public static final int ERROR_RESOURCE_BUSY = 3; 2219 2220 /** 2221 * This indicates that the output protection levels supported by the 2222 * device are not sufficient to meet the requirements set by the 2223 * content owner in the license policy. 2224 */ 2225 public static final int ERROR_INSUFFICIENT_OUTPUT_PROTECTION = 4; 2226 2227 /** 2228 * This indicates that decryption was attempted on a session that is 2229 * not opened, which could be due to a failure to open the session, 2230 * closing the session prematurely, or the session being reclaimed 2231 * by the resource manager. 2232 */ 2233 public static final int ERROR_SESSION_NOT_OPENED = 5; 2234 2235 /** 2236 * This indicates that an operation was attempted that could not be 2237 * supported by the crypto system of the device in its current 2238 * configuration. It may occur when the license policy requires 2239 * device security features that aren't supported by the device, 2240 * or due to an internal error in the crypto system that prevents 2241 * the specified security policy from being met. 2242 */ 2243 public static final int ERROR_UNSUPPORTED_OPERATION = 6; 2244 2245 /** @hide */ 2246 @IntDef({ 2247 ERROR_NO_KEY, 2248 ERROR_KEY_EXPIRED, 2249 ERROR_RESOURCE_BUSY, 2250 ERROR_INSUFFICIENT_OUTPUT_PROTECTION, 2251 ERROR_SESSION_NOT_OPENED, 2252 ERROR_UNSUPPORTED_OPERATION 2253 }) 2254 @Retention(RetentionPolicy.SOURCE) 2255 public @interface CryptoErrorCode {} 2256 2257 /** 2258 * Retrieve the error code associated with a CryptoException 2259 */ 2260 @CryptoErrorCode 2261 public int getErrorCode() { 2262 return mErrorCode; 2263 } 2264 2265 private int mErrorCode; 2266 } 2267 2268 /** 2269 * After filling a range of the input buffer at the specified index 2270 * submit it to the component. Once an input buffer is queued to 2271 * the codec, it MUST NOT be used until it is later retrieved by 2272 * {@link #getInputBuffer} in response to a {@link #dequeueInputBuffer} 2273 * return value or a {@link Callback#onInputBufferAvailable} 2274 * callback. 2275 * <p> 2276 * Many decoders require the actual compressed data stream to be 2277 * preceded by "codec specific data", i.e. setup data used to initialize 2278 * the codec such as PPS/SPS in the case of AVC video or code tables 2279 * in the case of vorbis audio. 2280 * The class {@link android.media.MediaExtractor} provides codec 2281 * specific data as part of 2282 * the returned track format in entries named "csd-0", "csd-1" ... 2283 * <p> 2284 * These buffers can be submitted directly after {@link #start} or 2285 * {@link #flush} by specifying the flag {@link 2286 * #BUFFER_FLAG_CODEC_CONFIG}. However, if you configure the 2287 * codec with a {@link MediaFormat} containing these keys, they 2288 * will be automatically submitted by MediaCodec directly after 2289 * start. Therefore, the use of {@link 2290 * #BUFFER_FLAG_CODEC_CONFIG} flag is discouraged and is 2291 * recommended only for advanced users. 2292 * <p> 2293 * To indicate that this is the final piece of input data (or rather that 2294 * no more input data follows unless the decoder is subsequently flushed) 2295 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 2296 * <p class=note> 2297 * <strong>Note:</strong> Prior to {@link android.os.Build.VERSION_CODES#M}, 2298 * {@code presentationTimeUs} was not propagated to the frame timestamp of (rendered) 2299 * Surface output buffers, and the resulting frame timestamp was undefined. 2300 * Use {@link #releaseOutputBuffer(int, long)} to ensure a specific frame timestamp is set. 2301 * Similarly, since frame timestamps can be used by the destination surface for rendering 2302 * synchronization, <strong>care must be taken to normalize presentationTimeUs so as to not be 2303 * mistaken for a system time. (See {@linkplain #releaseOutputBuffer(int, long) 2304 * SurfaceView specifics}).</strong> 2305 * 2306 * @param index The index of a client-owned input buffer previously returned 2307 * in a call to {@link #dequeueInputBuffer}. 2308 * @param offset The byte offset into the input buffer at which the data starts. 2309 * @param size The number of bytes of valid input data. 2310 * @param presentationTimeUs The presentation timestamp in microseconds for this 2311 * buffer. This is normally the media time at which this 2312 * buffer should be presented (rendered). When using an output 2313 * surface, this will be propagated as the {@link 2314 * SurfaceTexture#getTimestamp timestamp} for the frame (after 2315 * conversion to nanoseconds). 2316 * @param flags A bitmask of flags 2317 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2318 * While not prohibited, most codecs do not use the 2319 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2320 * @throws IllegalStateException if not in the Executing state. 2321 * @throws MediaCodec.CodecException upon codec error. 2322 * @throws CryptoException if a crypto object has been specified in 2323 * {@link #configure} 2324 */ 2325 public final void queueInputBuffer( 2326 int index, 2327 int offset, int size, long presentationTimeUs, int flags) 2328 throws CryptoException { 2329 synchronized(mBufferLock) { 2330 invalidateByteBuffer(mCachedInputBuffers, index); 2331 mDequeuedInputBuffers.remove(index); 2332 } 2333 try { 2334 native_queueInputBuffer( 2335 index, offset, size, presentationTimeUs, flags); 2336 } catch (CryptoException | IllegalStateException e) { 2337 revalidateByteBuffer(mCachedInputBuffers, index); 2338 throw e; 2339 } 2340 } 2341 2342 private native final void native_queueInputBuffer( 2343 int index, 2344 int offset, int size, long presentationTimeUs, int flags) 2345 throws CryptoException; 2346 2347 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 2348 public static final int CRYPTO_MODE_AES_CTR = 1; 2349 public static final int CRYPTO_MODE_AES_CBC = 2; 2350 2351 /** 2352 * Metadata describing the structure of a (at least partially) encrypted 2353 * input sample. 2354 * A buffer's data is considered to be partitioned into "subSamples", 2355 * each subSample starts with a (potentially empty) run of plain, 2356 * unencrypted bytes followed by a (also potentially empty) run of 2357 * encrypted bytes. If pattern encryption applies, each of the latter runs 2358 * is encrypted only partly, according to a repeating pattern of "encrypt" 2359 * and "skip" blocks. numBytesOfClearData can be null to indicate that all 2360 * data is encrypted. This information encapsulates per-sample metadata as 2361 * outlined in ISO/IEC FDIS 23001-7:2011 "Common encryption in ISO base 2362 * media file format files". 2363 */ 2364 public final static class CryptoInfo { 2365 /** 2366 * The number of subSamples that make up the buffer's contents. 2367 */ 2368 public int numSubSamples; 2369 /** 2370 * The number of leading unencrypted bytes in each subSample. 2371 */ 2372 public int[] numBytesOfClearData; 2373 /** 2374 * The number of trailing encrypted bytes in each subSample. 2375 */ 2376 public int[] numBytesOfEncryptedData; 2377 /** 2378 * A 16-byte key id 2379 */ 2380 public byte[] key; 2381 /** 2382 * A 16-byte initialization vector 2383 */ 2384 public byte[] iv; 2385 /** 2386 * The type of encryption that has been applied, 2387 * see {@link #CRYPTO_MODE_UNENCRYPTED}, {@link #CRYPTO_MODE_AES_CTR} 2388 * and {@link #CRYPTO_MODE_AES_CBC} 2389 */ 2390 public int mode; 2391 2392 /** 2393 * Metadata describing an encryption pattern for the protected bytes in 2394 * a subsample. An encryption pattern consists of a repeating sequence 2395 * of crypto blocks comprised of a number of encrypted blocks followed 2396 * by a number of unencrypted, or skipped, blocks. 2397 */ 2398 public final static class Pattern { 2399 /** 2400 * Number of blocks to be encrypted in the pattern. If zero, pattern 2401 * encryption is inoperative. 2402 */ 2403 private int mEncryptBlocks; 2404 2405 /** 2406 * Number of blocks to be skipped (left clear) in the pattern. If zero, 2407 * pattern encryption is inoperative. 2408 */ 2409 private int mSkipBlocks; 2410 2411 /** 2412 * Construct a sample encryption pattern given the number of blocks to 2413 * encrypt and skip in the pattern. 2414 */ 2415 public Pattern(int blocksToEncrypt, int blocksToSkip) { 2416 set(blocksToEncrypt, blocksToSkip); 2417 } 2418 2419 /** 2420 * Set the number of blocks to encrypt and skip in a sample encryption 2421 * pattern. 2422 */ 2423 public void set(int blocksToEncrypt, int blocksToSkip) { 2424 mEncryptBlocks = blocksToEncrypt; 2425 mSkipBlocks = blocksToSkip; 2426 } 2427 2428 /** 2429 * Return the number of blocks to skip in a sample encryption pattern. 2430 */ 2431 public int getSkipBlocks() { 2432 return mSkipBlocks; 2433 } 2434 2435 /** 2436 * Return the number of blocks to encrypt in a sample encryption pattern. 2437 */ 2438 public int getEncryptBlocks() { 2439 return mEncryptBlocks; 2440 } 2441 }; 2442 2443 /** 2444 * The pattern applicable to the protected data in each subsample. 2445 */ 2446 private Pattern pattern; 2447 2448 /** 2449 * Set the subsample count, clear/encrypted sizes, key, IV and mode fields of 2450 * a {@link MediaCodec.CryptoInfo} instance. 2451 */ 2452 public void set( 2453 int newNumSubSamples, 2454 @NonNull int[] newNumBytesOfClearData, 2455 @NonNull int[] newNumBytesOfEncryptedData, 2456 @NonNull byte[] newKey, 2457 @NonNull byte[] newIV, 2458 int newMode) { 2459 numSubSamples = newNumSubSamples; 2460 numBytesOfClearData = newNumBytesOfClearData; 2461 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 2462 key = newKey; 2463 iv = newIV; 2464 mode = newMode; 2465 pattern = new Pattern(0, 0); 2466 } 2467 2468 /** 2469 * Set the encryption pattern on a {@link MediaCodec.CryptoInfo} instance. 2470 * See {@link MediaCodec.CryptoInfo.Pattern}. 2471 */ 2472 public void setPattern(Pattern newPattern) { 2473 pattern = newPattern; 2474 } 2475 2476 @Override 2477 public String toString() { 2478 StringBuilder builder = new StringBuilder(); 2479 builder.append(numSubSamples + " subsamples, key ["); 2480 String hexdigits = "0123456789abcdef"; 2481 for (int i = 0; i < key.length; i++) { 2482 builder.append(hexdigits.charAt((key[i] & 0xf0) >> 4)); 2483 builder.append(hexdigits.charAt(key[i] & 0x0f)); 2484 } 2485 builder.append("], iv ["); 2486 for (int i = 0; i < key.length; i++) { 2487 builder.append(hexdigits.charAt((iv[i] & 0xf0) >> 4)); 2488 builder.append(hexdigits.charAt(iv[i] & 0x0f)); 2489 } 2490 builder.append("], clear "); Arrays.toString(numBytesOfClearData)2491 builder.append(Arrays.toString(numBytesOfClearData)); 2492 builder.append(", encrypted "); Arrays.toString(numBytesOfEncryptedData)2493 builder.append(Arrays.toString(numBytesOfEncryptedData)); 2494 return builder.toString(); 2495 } 2496 }; 2497 2498 /** 2499 * Similar to {@link #queueInputBuffer queueInputBuffer} but submits a buffer that is 2500 * potentially encrypted. 2501 * <strong>Check out further notes at {@link #queueInputBuffer queueInputBuffer}.</strong> 2502 * 2503 * @param index The index of a client-owned input buffer previously returned 2504 * in a call to {@link #dequeueInputBuffer}. 2505 * @param offset The byte offset into the input buffer at which the data starts. 2506 * @param info Metadata required to facilitate decryption, the object can be 2507 * reused immediately after this call returns. 2508 * @param presentationTimeUs The presentation timestamp in microseconds for this 2509 * buffer. This is normally the media time at which this 2510 * buffer should be presented (rendered). 2511 * @param flags A bitmask of flags 2512 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2513 * While not prohibited, most codecs do not use the 2514 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2515 * @throws IllegalStateException if not in the Executing state. 2516 * @throws MediaCodec.CodecException upon codec error. 2517 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 2518 * An error code associated with the exception helps identify the 2519 * reason for the failure. 2520 */ queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2521 public final void queueSecureInputBuffer( 2522 int index, 2523 int offset, 2524 @NonNull CryptoInfo info, 2525 long presentationTimeUs, 2526 int flags) throws CryptoException { 2527 synchronized(mBufferLock) { 2528 invalidateByteBuffer(mCachedInputBuffers, index); 2529 mDequeuedInputBuffers.remove(index); 2530 } 2531 try { 2532 native_queueSecureInputBuffer( 2533 index, offset, info, presentationTimeUs, flags); 2534 } catch (CryptoException | IllegalStateException e) { 2535 revalidateByteBuffer(mCachedInputBuffers, index); 2536 throw e; 2537 } 2538 } 2539 native_queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2540 private native final void native_queueSecureInputBuffer( 2541 int index, 2542 int offset, 2543 @NonNull CryptoInfo info, 2544 long presentationTimeUs, 2545 int flags) throws CryptoException; 2546 2547 /** 2548 * Returns the index of an input buffer to be filled with valid data 2549 * or -1 if no such buffer is currently available. 2550 * This method will return immediately if timeoutUs == 0, wait indefinitely 2551 * for the availability of an input buffer if timeoutUs < 0 or wait up 2552 * to "timeoutUs" microseconds if timeoutUs > 0. 2553 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 2554 * @throws IllegalStateException if not in the Executing state, 2555 * or codec is configured in asynchronous mode. 2556 * @throws MediaCodec.CodecException upon codec error. 2557 */ dequeueInputBuffer(long timeoutUs)2558 public final int dequeueInputBuffer(long timeoutUs) { 2559 int res = native_dequeueInputBuffer(timeoutUs); 2560 if (res >= 0) { 2561 synchronized(mBufferLock) { 2562 validateInputByteBuffer(mCachedInputBuffers, res); 2563 } 2564 } 2565 return res; 2566 } 2567 native_dequeueInputBuffer(long timeoutUs)2568 private native final int native_dequeueInputBuffer(long timeoutUs); 2569 2570 /** 2571 * If a non-negative timeout had been specified in the call 2572 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 2573 */ 2574 public static final int INFO_TRY_AGAIN_LATER = -1; 2575 2576 /** 2577 * The output format has changed, subsequent data will follow the new 2578 * format. {@link #getOutputFormat()} returns the new format. Note, that 2579 * you can also use the new {@link #getOutputFormat(int)} method to 2580 * get the format for a specific output buffer. This frees you from 2581 * having to track output format changes. 2582 */ 2583 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 2584 2585 /** 2586 * The output buffers have changed, the client must refer to the new 2587 * set of output buffers returned by {@link #getOutputBuffers} from 2588 * this point on. 2589 * 2590 * <p>Additionally, this event signals that the video scaling mode 2591 * may have been reset to the default.</p> 2592 * 2593 * @deprecated This return value can be ignored as {@link 2594 * #getOutputBuffers} has been deprecated. Client should 2595 * request a current buffer using on of the get-buffer or 2596 * get-image methods each time one has been dequeued. 2597 */ 2598 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 2599 2600 /** @hide */ 2601 @IntDef({ 2602 INFO_TRY_AGAIN_LATER, 2603 INFO_OUTPUT_FORMAT_CHANGED, 2604 INFO_OUTPUT_BUFFERS_CHANGED, 2605 }) 2606 @Retention(RetentionPolicy.SOURCE) 2607 public @interface OutputBufferInfo {} 2608 2609 /** 2610 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 2611 * Returns the index of an output buffer that has been successfully 2612 * decoded or one of the INFO_* constants. 2613 * @param info Will be filled with buffer meta data. 2614 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 2615 * @throws IllegalStateException if not in the Executing state, 2616 * or codec is configured in asynchronous mode. 2617 * @throws MediaCodec.CodecException upon codec error. 2618 */ 2619 @OutputBufferInfo dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)2620 public final int dequeueOutputBuffer( 2621 @NonNull BufferInfo info, long timeoutUs) { 2622 int res = native_dequeueOutputBuffer(info, timeoutUs); 2623 synchronized(mBufferLock) { 2624 if (res == INFO_OUTPUT_BUFFERS_CHANGED) { 2625 cacheBuffers(false /* input */); 2626 } else if (res >= 0) { 2627 validateOutputByteBuffer(mCachedOutputBuffers, res, info); 2628 if (mHasSurface) { 2629 mDequeuedOutputInfos.put(res, info.dup()); 2630 } 2631 } 2632 } 2633 return res; 2634 } 2635 native_dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)2636 private native final int native_dequeueOutputBuffer( 2637 @NonNull BufferInfo info, long timeoutUs); 2638 2639 /** 2640 * If you are done with a buffer, use this call to return the buffer to the codec 2641 * or to render it on the output surface. If you configured the codec with an 2642 * output surface, setting {@code render} to {@code true} will first send the buffer 2643 * to that output surface. The surface will release the buffer back to the codec once 2644 * it is no longer used/displayed. 2645 * 2646 * Once an output buffer is released to the codec, it MUST NOT 2647 * be used until it is later retrieved by {@link #getOutputBuffer} in response 2648 * to a {@link #dequeueOutputBuffer} return value or a 2649 * {@link Callback#onOutputBufferAvailable} callback. 2650 * 2651 * @param index The index of a client-owned output buffer previously returned 2652 * from a call to {@link #dequeueOutputBuffer}. 2653 * @param render If a valid surface was specified when configuring the codec, 2654 * passing true renders this output buffer to the surface. 2655 * @throws IllegalStateException if not in the Executing state. 2656 * @throws MediaCodec.CodecException upon codec error. 2657 */ releaseOutputBuffer(int index, boolean render)2658 public final void releaseOutputBuffer(int index, boolean render) { 2659 BufferInfo info = null; 2660 synchronized(mBufferLock) { 2661 invalidateByteBuffer(mCachedOutputBuffers, index); 2662 mDequeuedOutputBuffers.remove(index); 2663 if (mHasSurface) { 2664 info = mDequeuedOutputInfos.remove(index); 2665 } 2666 } 2667 releaseOutputBuffer(index, render, false /* updatePTS */, 0 /* dummy */); 2668 } 2669 2670 /** 2671 * If you are done with a buffer, use this call to update its surface timestamp 2672 * and return it to the codec to render it on the output surface. If you 2673 * have not specified an output surface when configuring this video codec, 2674 * this call will simply return the buffer to the codec.<p> 2675 * 2676 * The timestamp may have special meaning depending on the destination surface. 2677 * 2678 * <table> 2679 * <tr><th>SurfaceView specifics</th></tr> 2680 * <tr><td> 2681 * If you render your buffer on a {@link android.view.SurfaceView}, 2682 * you can use the timestamp to render the buffer at a specific time (at the 2683 * VSYNC at or after the buffer timestamp). For this to work, the timestamp 2684 * needs to be <i>reasonably close</i> to the current {@link System#nanoTime}. 2685 * Currently, this is set as within one (1) second. A few notes: 2686 * 2687 * <ul> 2688 * <li>the buffer will not be returned to the codec until the timestamp 2689 * has passed and the buffer is no longer used by the {@link android.view.Surface}. 2690 * <li>buffers are processed sequentially, so you may block subsequent buffers to 2691 * be displayed on the {@link android.view.Surface}. This is important if you 2692 * want to react to user action, e.g. stop the video or seek. 2693 * <li>if multiple buffers are sent to the {@link android.view.Surface} to be 2694 * rendered at the same VSYNC, the last one will be shown, and the other ones 2695 * will be dropped. 2696 * <li>if the timestamp is <em>not</em> "reasonably close" to the current system 2697 * time, the {@link android.view.Surface} will ignore the timestamp, and 2698 * display the buffer at the earliest feasible time. In this mode it will not 2699 * drop frames. 2700 * <li>for best performance and quality, call this method when you are about 2701 * two VSYNCs' time before the desired render time. For 60Hz displays, this is 2702 * about 33 msec. 2703 * </ul> 2704 * </td></tr> 2705 * </table> 2706 * 2707 * Once an output buffer is released to the codec, it MUST NOT 2708 * be used until it is later retrieved by {@link #getOutputBuffer} in response 2709 * to a {@link #dequeueOutputBuffer} return value or a 2710 * {@link Callback#onOutputBufferAvailable} callback. 2711 * 2712 * @param index The index of a client-owned output buffer previously returned 2713 * from a call to {@link #dequeueOutputBuffer}. 2714 * @param renderTimestampNs The timestamp to associate with this buffer when 2715 * it is sent to the Surface. 2716 * @throws IllegalStateException if not in the Executing state. 2717 * @throws MediaCodec.CodecException upon codec error. 2718 */ releaseOutputBuffer(int index, long renderTimestampNs)2719 public final void releaseOutputBuffer(int index, long renderTimestampNs) { 2720 BufferInfo info = null; 2721 synchronized(mBufferLock) { 2722 invalidateByteBuffer(mCachedOutputBuffers, index); 2723 mDequeuedOutputBuffers.remove(index); 2724 if (mHasSurface) { 2725 info = mDequeuedOutputInfos.remove(index); 2726 } 2727 } 2728 releaseOutputBuffer( 2729 index, true /* render */, true /* updatePTS */, renderTimestampNs); 2730 } 2731 releaseOutputBuffer( int index, boolean render, boolean updatePTS, long timeNs)2732 private native final void releaseOutputBuffer( 2733 int index, boolean render, boolean updatePTS, long timeNs); 2734 2735 /** 2736 * Signals end-of-stream on input. Equivalent to submitting an empty buffer with 2737 * {@link #BUFFER_FLAG_END_OF_STREAM} set. This may only be used with 2738 * encoders receiving input from a Surface created by {@link #createInputSurface}. 2739 * @throws IllegalStateException if not in the Executing state. 2740 * @throws MediaCodec.CodecException upon codec error. 2741 */ signalEndOfInputStream()2742 public native final void signalEndOfInputStream(); 2743 2744 /** 2745 * Call this after dequeueOutputBuffer signals a format change by returning 2746 * {@link #INFO_OUTPUT_FORMAT_CHANGED}. 2747 * You can also call this after {@link #configure} returns 2748 * successfully to get the output format initially configured 2749 * for the codec. Do this to determine what optional 2750 * configuration parameters were supported by the codec. 2751 * 2752 * @throws IllegalStateException if not in the Executing or 2753 * Configured state. 2754 * @throws MediaCodec.CodecException upon codec error. 2755 */ 2756 @NonNull getOutputFormat()2757 public final MediaFormat getOutputFormat() { 2758 return new MediaFormat(getFormatNative(false /* input */)); 2759 } 2760 2761 /** 2762 * Call this after {@link #configure} returns successfully to 2763 * get the input format accepted by the codec. Do this to 2764 * determine what optional configuration parameters were 2765 * supported by the codec. 2766 * 2767 * @throws IllegalStateException if not in the Executing or 2768 * Configured state. 2769 * @throws MediaCodec.CodecException upon codec error. 2770 */ 2771 @NonNull getInputFormat()2772 public final MediaFormat getInputFormat() { 2773 return new MediaFormat(getFormatNative(true /* input */)); 2774 } 2775 2776 /** 2777 * Returns the output format for a specific output buffer. 2778 * 2779 * @param index The index of a client-owned input buffer previously 2780 * returned from a call to {@link #dequeueInputBuffer}. 2781 * 2782 * @return the format for the output buffer, or null if the index 2783 * is not a dequeued output buffer. 2784 */ 2785 @NonNull getOutputFormat(int index)2786 public final MediaFormat getOutputFormat(int index) { 2787 return new MediaFormat(getOutputFormatNative(index)); 2788 } 2789 2790 @NonNull getFormatNative(boolean input)2791 private native final Map<String, Object> getFormatNative(boolean input); 2792 2793 @NonNull getOutputFormatNative(int index)2794 private native final Map<String, Object> getOutputFormatNative(int index); 2795 2796 // used to track dequeued buffers 2797 private static class BufferMap { 2798 // various returned representations of the codec buffer 2799 private static class CodecBuffer { 2800 private Image mImage; 2801 private ByteBuffer mByteBuffer; 2802 free()2803 public void free() { 2804 if (mByteBuffer != null) { 2805 // all of our ByteBuffers are direct 2806 java.nio.NioUtils.freeDirectBuffer(mByteBuffer); 2807 mByteBuffer = null; 2808 } 2809 if (mImage != null) { 2810 mImage.close(); 2811 mImage = null; 2812 } 2813 } 2814 setImage(@ullable Image image)2815 public void setImage(@Nullable Image image) { 2816 free(); 2817 mImage = image; 2818 } 2819 setByteBuffer(@ullable ByteBuffer buffer)2820 public void setByteBuffer(@Nullable ByteBuffer buffer) { 2821 free(); 2822 mByteBuffer = buffer; 2823 } 2824 } 2825 2826 private final Map<Integer, CodecBuffer> mMap = 2827 new HashMap<Integer, CodecBuffer>(); 2828 remove(int index)2829 public void remove(int index) { 2830 CodecBuffer buffer = mMap.get(index); 2831 if (buffer != null) { 2832 buffer.free(); 2833 mMap.remove(index); 2834 } 2835 } 2836 put(int index, @Nullable ByteBuffer newBuffer)2837 public void put(int index, @Nullable ByteBuffer newBuffer) { 2838 CodecBuffer buffer = mMap.get(index); 2839 if (buffer == null) { // likely 2840 buffer = new CodecBuffer(); 2841 mMap.put(index, buffer); 2842 } 2843 buffer.setByteBuffer(newBuffer); 2844 } 2845 put(int index, @Nullable Image newImage)2846 public void put(int index, @Nullable Image newImage) { 2847 CodecBuffer buffer = mMap.get(index); 2848 if (buffer == null) { // likely 2849 buffer = new CodecBuffer(); 2850 mMap.put(index, buffer); 2851 } 2852 buffer.setImage(newImage); 2853 } 2854 clear()2855 public void clear() { 2856 for (CodecBuffer buffer: mMap.values()) { 2857 buffer.free(); 2858 } 2859 mMap.clear(); 2860 } 2861 } 2862 2863 private ByteBuffer[] mCachedInputBuffers; 2864 private ByteBuffer[] mCachedOutputBuffers; 2865 private final BufferMap mDequeuedInputBuffers = new BufferMap(); 2866 private final BufferMap mDequeuedOutputBuffers = new BufferMap(); 2867 private final Map<Integer, BufferInfo> mDequeuedOutputInfos = 2868 new HashMap<Integer, BufferInfo>(); 2869 final private Object mBufferLock; 2870 invalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)2871 private final void invalidateByteBuffer( 2872 @Nullable ByteBuffer[] buffers, int index) { 2873 if (buffers != null && index >= 0 && index < buffers.length) { 2874 ByteBuffer buffer = buffers[index]; 2875 if (buffer != null) { 2876 buffer.setAccessible(false); 2877 } 2878 } 2879 } 2880 validateInputByteBuffer( @ullable ByteBuffer[] buffers, int index)2881 private final void validateInputByteBuffer( 2882 @Nullable ByteBuffer[] buffers, int index) { 2883 if (buffers != null && index >= 0 && index < buffers.length) { 2884 ByteBuffer buffer = buffers[index]; 2885 if (buffer != null) { 2886 buffer.setAccessible(true); 2887 buffer.clear(); 2888 } 2889 } 2890 } 2891 revalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)2892 private final void revalidateByteBuffer( 2893 @Nullable ByteBuffer[] buffers, int index) { 2894 synchronized(mBufferLock) { 2895 if (buffers != null && index >= 0 && index < buffers.length) { 2896 ByteBuffer buffer = buffers[index]; 2897 if (buffer != null) { 2898 buffer.setAccessible(true); 2899 } 2900 } 2901 } 2902 } 2903 validateOutputByteBuffer( @ullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info)2904 private final void validateOutputByteBuffer( 2905 @Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) { 2906 if (buffers != null && index >= 0 && index < buffers.length) { 2907 ByteBuffer buffer = buffers[index]; 2908 if (buffer != null) { 2909 buffer.setAccessible(true); 2910 buffer.limit(info.offset + info.size).position(info.offset); 2911 } 2912 } 2913 } 2914 invalidateByteBuffers(@ullable ByteBuffer[] buffers)2915 private final void invalidateByteBuffers(@Nullable ByteBuffer[] buffers) { 2916 if (buffers != null) { 2917 for (ByteBuffer buffer: buffers) { 2918 if (buffer != null) { 2919 buffer.setAccessible(false); 2920 } 2921 } 2922 } 2923 } 2924 freeByteBuffer(@ullable ByteBuffer buffer)2925 private final void freeByteBuffer(@Nullable ByteBuffer buffer) { 2926 if (buffer != null /* && buffer.isDirect() */) { 2927 // all of our ByteBuffers are direct 2928 java.nio.NioUtils.freeDirectBuffer(buffer); 2929 } 2930 } 2931 freeByteBuffers(@ullable ByteBuffer[] buffers)2932 private final void freeByteBuffers(@Nullable ByteBuffer[] buffers) { 2933 if (buffers != null) { 2934 for (ByteBuffer buffer: buffers) { 2935 freeByteBuffer(buffer); 2936 } 2937 } 2938 } 2939 freeAllTrackedBuffers()2940 private final void freeAllTrackedBuffers() { 2941 synchronized(mBufferLock) { 2942 freeByteBuffers(mCachedInputBuffers); 2943 freeByteBuffers(mCachedOutputBuffers); 2944 mCachedInputBuffers = null; 2945 mCachedOutputBuffers = null; 2946 mDequeuedInputBuffers.clear(); 2947 mDequeuedOutputBuffers.clear(); 2948 } 2949 } 2950 cacheBuffers(boolean input)2951 private final void cacheBuffers(boolean input) { 2952 ByteBuffer[] buffers = null; 2953 try { 2954 buffers = getBuffers(input); 2955 invalidateByteBuffers(buffers); 2956 } catch (IllegalStateException e) { 2957 // we don't get buffers in async mode 2958 } 2959 if (input) { 2960 mCachedInputBuffers = buffers; 2961 } else { 2962 mCachedOutputBuffers = buffers; 2963 } 2964 } 2965 2966 /** 2967 * Retrieve the set of input buffers. Call this after start() 2968 * returns. After calling this method, any ByteBuffers 2969 * previously returned by an earlier call to this method MUST no 2970 * longer be used. 2971 * 2972 * @deprecated Use the new {@link #getInputBuffer} method instead 2973 * each time an input buffer is dequeued. 2974 * 2975 * <b>Note:</b> As of API 21, dequeued input buffers are 2976 * automatically {@link java.nio.Buffer#clear cleared}. 2977 * 2978 * <em>Do not use this method if using an input surface.</em> 2979 * 2980 * @throws IllegalStateException if not in the Executing state, 2981 * or codec is configured in asynchronous mode. 2982 * @throws MediaCodec.CodecException upon codec error. 2983 */ 2984 @NonNull getInputBuffers()2985 public ByteBuffer[] getInputBuffers() { 2986 if (mCachedInputBuffers == null) { 2987 throw new IllegalStateException(); 2988 } 2989 // FIXME: check codec status 2990 return mCachedInputBuffers; 2991 } 2992 2993 /** 2994 * Retrieve the set of output buffers. Call this after start() 2995 * returns and whenever dequeueOutputBuffer signals an output 2996 * buffer change by returning {@link 2997 * #INFO_OUTPUT_BUFFERS_CHANGED}. After calling this method, any 2998 * ByteBuffers previously returned by an earlier call to this 2999 * method MUST no longer be used. 3000 * 3001 * @deprecated Use the new {@link #getOutputBuffer} method instead 3002 * each time an output buffer is dequeued. This method is not 3003 * supported if codec is configured in asynchronous mode. 3004 * 3005 * <b>Note:</b> As of API 21, the position and limit of output 3006 * buffers that are dequeued will be set to the valid data 3007 * range. 3008 * 3009 * <em>Do not use this method if using an output surface.</em> 3010 * 3011 * @throws IllegalStateException if not in the Executing state, 3012 * or codec is configured in asynchronous mode. 3013 * @throws MediaCodec.CodecException upon codec error. 3014 */ 3015 @NonNull getOutputBuffers()3016 public ByteBuffer[] getOutputBuffers() { 3017 if (mCachedOutputBuffers == null) { 3018 throw new IllegalStateException(); 3019 } 3020 // FIXME: check codec status 3021 return mCachedOutputBuffers; 3022 } 3023 3024 /** 3025 * Returns a {@link java.nio.Buffer#clear cleared}, writable ByteBuffer 3026 * object for a dequeued input buffer index to contain the input data. 3027 * 3028 * After calling this method any ByteBuffer or Image object 3029 * previously returned for the same input index MUST no longer 3030 * be used. 3031 * 3032 * @param index The index of a client-owned input buffer previously 3033 * returned from a call to {@link #dequeueInputBuffer}, 3034 * or received via an onInputBufferAvailable callback. 3035 * 3036 * @return the input buffer, or null if the index is not a dequeued 3037 * input buffer, or if the codec is configured for surface input. 3038 * 3039 * @throws IllegalStateException if not in the Executing state. 3040 * @throws MediaCodec.CodecException upon codec error. 3041 */ 3042 @Nullable getInputBuffer(int index)3043 public ByteBuffer getInputBuffer(int index) { 3044 ByteBuffer newBuffer = getBuffer(true /* input */, index); 3045 synchronized(mBufferLock) { 3046 invalidateByteBuffer(mCachedInputBuffers, index); 3047 mDequeuedInputBuffers.put(index, newBuffer); 3048 } 3049 return newBuffer; 3050 } 3051 3052 /** 3053 * Returns a writable Image object for a dequeued input buffer 3054 * index to contain the raw input video frame. 3055 * 3056 * After calling this method any ByteBuffer or Image object 3057 * previously returned for the same input index MUST no longer 3058 * be used. 3059 * 3060 * @param index The index of a client-owned input buffer previously 3061 * returned from a call to {@link #dequeueInputBuffer}, 3062 * or received via an onInputBufferAvailable callback. 3063 * 3064 * @return the input image, or null if the index is not a 3065 * dequeued input buffer, or not a ByteBuffer that contains a 3066 * raw image. 3067 * 3068 * @throws IllegalStateException if not in the Executing state. 3069 * @throws MediaCodec.CodecException upon codec error. 3070 */ 3071 @Nullable getInputImage(int index)3072 public Image getInputImage(int index) { 3073 Image newImage = getImage(true /* input */, index); 3074 synchronized(mBufferLock) { 3075 invalidateByteBuffer(mCachedInputBuffers, index); 3076 mDequeuedInputBuffers.put(index, newImage); 3077 } 3078 return newImage; 3079 } 3080 3081 /** 3082 * Returns a read-only ByteBuffer for a dequeued output buffer 3083 * index. The position and limit of the returned buffer are set 3084 * to the valid output data. 3085 * 3086 * After calling this method, any ByteBuffer or Image object 3087 * previously returned for the same output index MUST no longer 3088 * be used. 3089 * 3090 * @param index The index of a client-owned output buffer previously 3091 * returned from a call to {@link #dequeueOutputBuffer}, 3092 * or received via an onOutputBufferAvailable callback. 3093 * 3094 * @return the output buffer, or null if the index is not a dequeued 3095 * output buffer, or the codec is configured with an output surface. 3096 * 3097 * @throws IllegalStateException if not in the Executing state. 3098 * @throws MediaCodec.CodecException upon codec error. 3099 */ 3100 @Nullable getOutputBuffer(int index)3101 public ByteBuffer getOutputBuffer(int index) { 3102 ByteBuffer newBuffer = getBuffer(false /* input */, index); 3103 synchronized(mBufferLock) { 3104 invalidateByteBuffer(mCachedOutputBuffers, index); 3105 mDequeuedOutputBuffers.put(index, newBuffer); 3106 } 3107 return newBuffer; 3108 } 3109 3110 /** 3111 * Returns a read-only Image object for a dequeued output buffer 3112 * index that contains the raw video frame. 3113 * 3114 * After calling this method, any ByteBuffer or Image object previously 3115 * returned for the same output index MUST no longer be used. 3116 * 3117 * @param index The index of a client-owned output buffer previously 3118 * returned from a call to {@link #dequeueOutputBuffer}, 3119 * or received via an onOutputBufferAvailable callback. 3120 * 3121 * @return the output image, or null if the index is not a 3122 * dequeued output buffer, not a raw video frame, or if the codec 3123 * was configured with an output surface. 3124 * 3125 * @throws IllegalStateException if not in the Executing state. 3126 * @throws MediaCodec.CodecException upon codec error. 3127 */ 3128 @Nullable getOutputImage(int index)3129 public Image getOutputImage(int index) { 3130 Image newImage = getImage(false /* input */, index); 3131 synchronized(mBufferLock) { 3132 invalidateByteBuffer(mCachedOutputBuffers, index); 3133 mDequeuedOutputBuffers.put(index, newImage); 3134 } 3135 return newImage; 3136 } 3137 3138 /** 3139 * The content is scaled to the surface dimensions 3140 */ 3141 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 3142 3143 /** 3144 * The content is scaled, maintaining its aspect ratio, the whole 3145 * surface area is used, content may be cropped. 3146 * <p class=note> 3147 * This mode is only suitable for content with 1:1 pixel aspect ratio as you cannot 3148 * configure the pixel aspect ratio for a {@link Surface}. 3149 * <p class=note> 3150 * As of {@link android.os.Build.VERSION_CODES#N} release, this mode may not work if 3151 * the video is {@linkplain MediaFormat#KEY_ROTATION rotated} by 90 or 270 degrees. 3152 */ 3153 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 3154 3155 /** @hide */ 3156 @IntDef({ 3157 VIDEO_SCALING_MODE_SCALE_TO_FIT, 3158 VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING, 3159 }) 3160 @Retention(RetentionPolicy.SOURCE) 3161 public @interface VideoScalingMode {} 3162 3163 /** 3164 * If a surface has been specified in a previous call to {@link #configure} 3165 * specifies the scaling mode to use. The default is "scale to fit". 3166 * <p class=note> 3167 * The scaling mode may be reset to the <strong>default</strong> each time an 3168 * {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is received from the codec; therefore, the client 3169 * must call this method after every buffer change event (and before the first output buffer is 3170 * released for rendering) to ensure consistent scaling mode. 3171 * <p class=note> 3172 * Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, this can also be done 3173 * after each {@link #INFO_OUTPUT_FORMAT_CHANGED} event. 3174 * 3175 * @throws IllegalArgumentException if mode is not recognized. 3176 * @throws IllegalStateException if in the Released state. 3177 */ setVideoScalingMode(@ideoScalingMode int mode)3178 public native final void setVideoScalingMode(@VideoScalingMode int mode); 3179 3180 /** 3181 * Get the component name. If the codec was created by createDecoderByType 3182 * or createEncoderByType, what component is chosen is not known beforehand. 3183 * @throws IllegalStateException if in the Released state. 3184 */ 3185 @NonNull getName()3186 public native final String getName(); 3187 3188 /** 3189 * Return Metrics data about the current codec instance. 3190 * 3191 * @return a {@link PersistableBundle} containing the set of attributes and values 3192 * available for the media being handled by this instance of MediaCodec 3193 * The attributes are descibed in {@link MetricsConstants}. 3194 * 3195 * Additional vendor-specific fields may also be present in 3196 * the return value. 3197 */ getMetrics()3198 public PersistableBundle getMetrics() { 3199 PersistableBundle bundle = native_getMetrics(); 3200 return bundle; 3201 } 3202 native_getMetrics()3203 private native PersistableBundle native_getMetrics(); 3204 3205 /** 3206 * Change a video encoder's target bitrate on the fly. The value is an 3207 * Integer object containing the new bitrate in bps. 3208 */ 3209 public static final String PARAMETER_KEY_VIDEO_BITRATE = "video-bitrate"; 3210 3211 /** 3212 * Temporarily suspend/resume encoding of input data. While suspended 3213 * input data is effectively discarded instead of being fed into the 3214 * encoder. This parameter really only makes sense to use with an encoder 3215 * in "surface-input" mode, as the client code has no control over the 3216 * input-side of the encoder in that case. 3217 * The value is an Integer object containing the value 1 to suspend 3218 * or the value 0 to resume. 3219 */ 3220 public static final String PARAMETER_KEY_SUSPEND = "drop-input-frames"; 3221 3222 /** 3223 * Request that the encoder produce a sync frame "soon". 3224 * Provide an Integer with the value 0. 3225 */ 3226 public static final String PARAMETER_KEY_REQUEST_SYNC_FRAME = "request-sync"; 3227 3228 /** 3229 * Communicate additional parameter changes to the component instance. 3230 * <b>Note:</b> Some of these parameter changes may silently fail to apply. 3231 * 3232 * @param params The bundle of parameters to set. 3233 * @throws IllegalStateException if in the Released state. 3234 */ setParameters(@ullable Bundle params)3235 public final void setParameters(@Nullable Bundle params) { 3236 if (params == null) { 3237 return; 3238 } 3239 3240 String[] keys = new String[params.size()]; 3241 Object[] values = new Object[params.size()]; 3242 3243 int i = 0; 3244 for (final String key: params.keySet()) { 3245 keys[i] = key; 3246 values[i] = params.get(key); 3247 ++i; 3248 } 3249 3250 setParameters(keys, values); 3251 } 3252 3253 /** 3254 * Sets an asynchronous callback for actionable MediaCodec events. 3255 * 3256 * If the client intends to use the component in asynchronous mode, 3257 * a valid callback should be provided before {@link #configure} is called. 3258 * 3259 * When asynchronous callback is enabled, the client should not call 3260 * {@link #getInputBuffers}, {@link #getOutputBuffers}, 3261 * {@link #dequeueInputBuffer(long)} or {@link #dequeueOutputBuffer(BufferInfo, long)}. 3262 * <p> 3263 * Also, {@link #flush} behaves differently in asynchronous mode. After calling 3264 * {@code flush}, you must call {@link #start} to "resume" receiving input buffers, 3265 * even if an input surface was created. 3266 * 3267 * @param cb The callback that will run. Use {@code null} to clear a previously 3268 * set callback (before {@link #configure configure} is called and run 3269 * in synchronous mode). 3270 * @param handler Callbacks will happen on the handler's thread. If {@code null}, 3271 * callbacks are done on the default thread (the caller's thread or the 3272 * main thread.) 3273 */ setCallback(@ullable Callback cb, @Nullable Handler handler)3274 public void setCallback(@Nullable /* MediaCodec. */ Callback cb, @Nullable Handler handler) { 3275 if (cb != null) { 3276 synchronized (mListenerLock) { 3277 EventHandler newHandler = getEventHandlerOn(handler, mCallbackHandler); 3278 // NOTE: there are no callbacks on the handler at this time, but check anyways 3279 // even if we were to extend this to be callable dynamically, it must 3280 // be called when codec is flushed, so no messages are pending. 3281 if (newHandler != mCallbackHandler) { 3282 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 3283 mCallbackHandler.removeMessages(EVENT_CALLBACK); 3284 mCallbackHandler = newHandler; 3285 } 3286 } 3287 } else if (mCallbackHandler != null) { 3288 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 3289 mCallbackHandler.removeMessages(EVENT_CALLBACK); 3290 } 3291 3292 if (mCallbackHandler != null) { 3293 // set java callback on main handler 3294 Message msg = mCallbackHandler.obtainMessage(EVENT_SET_CALLBACK, 0, 0, cb); 3295 mCallbackHandler.sendMessage(msg); 3296 3297 // set native handler here, don't post to handler because 3298 // it may cause the callback to be delayed and set in a wrong state. 3299 // Note that native codec may start sending events to the callback 3300 // handler after this returns. 3301 native_setCallback(cb); 3302 } 3303 } 3304 3305 /** 3306 * Sets an asynchronous callback for actionable MediaCodec events on the default 3307 * looper. 3308 * <p> 3309 * Same as {@link #setCallback(Callback, Handler)} with handler set to null. 3310 * @param cb The callback that will run. Use {@code null} to clear a previously 3311 * set callback (before {@link #configure configure} is called and run 3312 * in synchronous mode). 3313 * @see #setCallback(Callback, Handler) 3314 */ setCallback(@ullable Callback cb)3315 public void setCallback(@Nullable /* MediaCodec. */ Callback cb) { 3316 setCallback(cb, null /* handler */); 3317 } 3318 3319 /** 3320 * Listener to be called when an output frame has rendered on the output surface 3321 * 3322 * @see MediaCodec#setOnFrameRenderedListener 3323 */ 3324 public interface OnFrameRenderedListener { 3325 3326 /** 3327 * Called when an output frame has rendered on the output surface. 3328 * <p> 3329 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 3330 * render timing samples, and can be significantly delayed and batched. Some frames may have 3331 * been rendered even if there was no callback generated. 3332 * 3333 * @param codec the MediaCodec instance 3334 * @param presentationTimeUs the presentation time (media time) of the frame rendered. 3335 * This is usually the same as specified in {@link #queueInputBuffer}; however, 3336 * some codecs may alter the media time by applying some time-based transformation, 3337 * such as frame rate conversion. In that case, presentation time corresponds 3338 * to the actual output frame rendered. 3339 * @param nanoTime The system time when the frame was rendered. 3340 * 3341 * @see System#nanoTime 3342 */ onFrameRendered( @onNull MediaCodec codec, long presentationTimeUs, long nanoTime)3343 public void onFrameRendered( 3344 @NonNull MediaCodec codec, long presentationTimeUs, long nanoTime); 3345 } 3346 3347 /** 3348 * Registers a callback to be invoked when an output frame is rendered on the output surface. 3349 * <p> 3350 * This method can be called in any codec state, but will only have an effect in the 3351 * Executing state for codecs that render buffers to the output surface. 3352 * <p> 3353 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 3354 * render timing samples, and can be significantly delayed and batched. Some frames may have 3355 * been rendered even if there was no callback generated. 3356 * 3357 * @param listener the callback that will be run 3358 * @param handler the callback will be run on the handler's thread. If {@code null}, 3359 * the callback will be run on the default thread, which is the looper 3360 * from which the codec was created, or a new thread if there was none. 3361 */ setOnFrameRenderedListener( @ullable OnFrameRenderedListener listener, @Nullable Handler handler)3362 public void setOnFrameRenderedListener( 3363 @Nullable OnFrameRenderedListener listener, @Nullable Handler handler) { 3364 synchronized (mListenerLock) { 3365 mOnFrameRenderedListener = listener; 3366 if (listener != null) { 3367 EventHandler newHandler = getEventHandlerOn(handler, mOnFrameRenderedHandler); 3368 if (newHandler != mOnFrameRenderedHandler) { 3369 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 3370 } 3371 mOnFrameRenderedHandler = newHandler; 3372 } else if (mOnFrameRenderedHandler != null) { 3373 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 3374 } 3375 native_enableOnFrameRenderedListener(listener != null); 3376 } 3377 } 3378 native_enableOnFrameRenderedListener(boolean enable)3379 private native void native_enableOnFrameRenderedListener(boolean enable); 3380 getEventHandlerOn( @ullable Handler handler, @NonNull EventHandler lastHandler)3381 private EventHandler getEventHandlerOn( 3382 @Nullable Handler handler, @NonNull EventHandler lastHandler) { 3383 if (handler == null) { 3384 return mEventHandler; 3385 } else { 3386 Looper looper = handler.getLooper(); 3387 if (lastHandler.getLooper() == looper) { 3388 return lastHandler; 3389 } else { 3390 return new EventHandler(this, looper); 3391 } 3392 } 3393 } 3394 3395 /** 3396 * MediaCodec callback interface. Used to notify the user asynchronously 3397 * of various MediaCodec events. 3398 */ 3399 public static abstract class Callback { 3400 /** 3401 * Called when an input buffer becomes available. 3402 * 3403 * @param codec The MediaCodec object. 3404 * @param index The index of the available input buffer. 3405 */ onInputBufferAvailable(@onNull MediaCodec codec, int index)3406 public abstract void onInputBufferAvailable(@NonNull MediaCodec codec, int index); 3407 3408 /** 3409 * Called when an output buffer becomes available. 3410 * 3411 * @param codec The MediaCodec object. 3412 * @param index The index of the available output buffer. 3413 * @param info Info regarding the available output buffer {@link MediaCodec.BufferInfo}. 3414 */ onOutputBufferAvailable( @onNull MediaCodec codec, int index, @NonNull BufferInfo info)3415 public abstract void onOutputBufferAvailable( 3416 @NonNull MediaCodec codec, int index, @NonNull BufferInfo info); 3417 3418 /** 3419 * Called when the MediaCodec encountered an error 3420 * 3421 * @param codec The MediaCodec object. 3422 * @param e The {@link MediaCodec.CodecException} object describing the error. 3423 */ onError(@onNull MediaCodec codec, @NonNull CodecException e)3424 public abstract void onError(@NonNull MediaCodec codec, @NonNull CodecException e); 3425 3426 /** 3427 * Called when the output format has changed 3428 * 3429 * @param codec The MediaCodec object. 3430 * @param format The new output format. 3431 */ onOutputFormatChanged( @onNull MediaCodec codec, @NonNull MediaFormat format)3432 public abstract void onOutputFormatChanged( 3433 @NonNull MediaCodec codec, @NonNull MediaFormat format); 3434 } 3435 postEventFromNative( int what, int arg1, int arg2, @Nullable Object obj)3436 private void postEventFromNative( 3437 int what, int arg1, int arg2, @Nullable Object obj) { 3438 synchronized (mListenerLock) { 3439 EventHandler handler = mEventHandler; 3440 if (what == EVENT_CALLBACK) { 3441 handler = mCallbackHandler; 3442 } else if (what == EVENT_FRAME_RENDERED) { 3443 handler = mOnFrameRenderedHandler; 3444 } 3445 if (handler != null) { 3446 Message msg = handler.obtainMessage(what, arg1, arg2, obj); 3447 handler.sendMessage(msg); 3448 } 3449 } 3450 } 3451 setParameters(@onNull String[] keys, @NonNull Object[] values)3452 private native final void setParameters(@NonNull String[] keys, @NonNull Object[] values); 3453 3454 /** 3455 * Get the codec info. If the codec was created by createDecoderByType 3456 * or createEncoderByType, what component is chosen is not known beforehand, 3457 * and thus the caller does not have the MediaCodecInfo. 3458 * @throws IllegalStateException if in the Released state. 3459 */ 3460 @NonNull getCodecInfo()3461 public MediaCodecInfo getCodecInfo() { 3462 return MediaCodecList.getInfoFor(getName()); 3463 } 3464 3465 @NonNull getBuffers(boolean input)3466 private native final ByteBuffer[] getBuffers(boolean input); 3467 3468 @Nullable getBuffer(boolean input, int index)3469 private native final ByteBuffer getBuffer(boolean input, int index); 3470 3471 @Nullable getImage(boolean input, int index)3472 private native final Image getImage(boolean input, int index); 3473 native_init()3474 private static native final void native_init(); 3475 native_setup( @onNull String name, boolean nameIsType, boolean encoder)3476 private native final void native_setup( 3477 @NonNull String name, boolean nameIsType, boolean encoder); 3478 native_finalize()3479 private native final void native_finalize(); 3480 3481 static { 3482 System.loadLibrary("media_jni"); native_init()3483 native_init(); 3484 } 3485 3486 private long mNativeContext; 3487 3488 /** @hide */ 3489 public static class MediaImage extends Image { 3490 private final boolean mIsReadOnly; 3491 private final int mWidth; 3492 private final int mHeight; 3493 private final int mFormat; 3494 private long mTimestamp; 3495 private final Plane[] mPlanes; 3496 private final ByteBuffer mBuffer; 3497 private final ByteBuffer mInfo; 3498 private final int mXOffset; 3499 private final int mYOffset; 3500 3501 private final static int TYPE_YUV = 1; 3502 3503 @Override getFormat()3504 public int getFormat() { 3505 throwISEIfImageIsInvalid(); 3506 return mFormat; 3507 } 3508 3509 @Override getHeight()3510 public int getHeight() { 3511 throwISEIfImageIsInvalid(); 3512 return mHeight; 3513 } 3514 3515 @Override getWidth()3516 public int getWidth() { 3517 throwISEIfImageIsInvalid(); 3518 return mWidth; 3519 } 3520 3521 @Override getTimestamp()3522 public long getTimestamp() { 3523 throwISEIfImageIsInvalid(); 3524 return mTimestamp; 3525 } 3526 3527 @Override 3528 @NonNull getPlanes()3529 public Plane[] getPlanes() { 3530 throwISEIfImageIsInvalid(); 3531 return Arrays.copyOf(mPlanes, mPlanes.length); 3532 } 3533 3534 @Override close()3535 public void close() { 3536 if (mIsImageValid) { 3537 java.nio.NioUtils.freeDirectBuffer(mBuffer); 3538 mIsImageValid = false; 3539 } 3540 } 3541 3542 /** 3543 * Set the crop rectangle associated with this frame. 3544 * <p> 3545 * The crop rectangle specifies the region of valid pixels in the image, 3546 * using coordinates in the largest-resolution plane. 3547 */ 3548 @Override setCropRect(@ullable Rect cropRect)3549 public void setCropRect(@Nullable Rect cropRect) { 3550 if (mIsReadOnly) { 3551 throw new ReadOnlyBufferException(); 3552 } 3553 super.setCropRect(cropRect); 3554 } 3555 3556 MediaImage( @onNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect)3557 public MediaImage( 3558 @NonNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, 3559 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) { 3560 mFormat = ImageFormat.YUV_420_888; 3561 mTimestamp = timestamp; 3562 mIsImageValid = true; 3563 mIsReadOnly = buffer.isReadOnly(); 3564 mBuffer = buffer.duplicate(); 3565 3566 // save offsets and info 3567 mXOffset = xOffset; 3568 mYOffset = yOffset; 3569 mInfo = info; 3570 3571 // read media-info. See MediaImage2 3572 if (info.remaining() == 104) { 3573 int type = info.getInt(); 3574 if (type != TYPE_YUV) { 3575 throw new UnsupportedOperationException("unsupported type: " + type); 3576 } 3577 int numPlanes = info.getInt(); 3578 if (numPlanes != 3) { 3579 throw new RuntimeException("unexpected number of planes: " + numPlanes); 3580 } 3581 mWidth = info.getInt(); 3582 mHeight = info.getInt(); 3583 if (mWidth < 1 || mHeight < 1) { 3584 throw new UnsupportedOperationException( 3585 "unsupported size: " + mWidth + "x" + mHeight); 3586 } 3587 int bitDepth = info.getInt(); 3588 if (bitDepth != 8) { 3589 throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth); 3590 } 3591 int bitDepthAllocated = info.getInt(); 3592 if (bitDepthAllocated != 8) { 3593 throw new UnsupportedOperationException( 3594 "unsupported allocated bit depth: " + bitDepthAllocated); 3595 } 3596 mPlanes = new MediaPlane[numPlanes]; 3597 for (int ix = 0; ix < numPlanes; ix++) { 3598 int planeOffset = info.getInt(); 3599 int colInc = info.getInt(); 3600 int rowInc = info.getInt(); 3601 int horiz = info.getInt(); 3602 int vert = info.getInt(); 3603 if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) { 3604 throw new UnsupportedOperationException("unexpected subsampling: " 3605 + horiz + "x" + vert + " on plane " + ix); 3606 } 3607 if (colInc < 1 || rowInc < 1) { 3608 throw new UnsupportedOperationException("unexpected strides: " 3609 + colInc + " pixel, " + rowInc + " row on plane " + ix); 3610 } 3611 3612 buffer.clear(); 3613 buffer.position(mBuffer.position() + planeOffset 3614 + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc); 3615 buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8) 3616 + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc); 3617 mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc); 3618 } 3619 } else { 3620 throw new UnsupportedOperationException( 3621 "unsupported info length: " + info.remaining()); 3622 } 3623 3624 if (cropRect == null) { 3625 cropRect = new Rect(0, 0, mWidth, mHeight); 3626 } 3627 cropRect.offset(-xOffset, -yOffset); 3628 super.setCropRect(cropRect); 3629 } 3630 3631 private class MediaPlane extends Plane { MediaPlane(@onNull ByteBuffer buffer, int rowInc, int colInc)3632 public MediaPlane(@NonNull ByteBuffer buffer, int rowInc, int colInc) { 3633 mData = buffer; 3634 mRowInc = rowInc; 3635 mColInc = colInc; 3636 } 3637 3638 @Override getRowStride()3639 public int getRowStride() { 3640 throwISEIfImageIsInvalid(); 3641 return mRowInc; 3642 } 3643 3644 @Override getPixelStride()3645 public int getPixelStride() { 3646 throwISEIfImageIsInvalid(); 3647 return mColInc; 3648 } 3649 3650 @Override 3651 @NonNull getBuffer()3652 public ByteBuffer getBuffer() { 3653 throwISEIfImageIsInvalid(); 3654 return mData; 3655 } 3656 3657 private final int mRowInc; 3658 private final int mColInc; 3659 private final ByteBuffer mData; 3660 } 3661 } 3662 3663 public final static class MetricsConstants 3664 { MetricsConstants()3665 private MetricsConstants() {} 3666 3667 /** 3668 * Key to extract the codec being used 3669 * from the {@link MediaCodec#getMetrics} return value. 3670 * The value is a String. 3671 */ 3672 public static final String CODEC = "android.media.mediacodec.codec"; 3673 3674 /** 3675 * Key to extract the MIME type 3676 * from the {@link MediaCodec#getMetrics} return value. 3677 * The value is a String. 3678 */ 3679 public static final String MIME_TYPE = "android.media.mediacodec.mime"; 3680 3681 /** 3682 * Key to extract what the codec mode 3683 * from the {@link MediaCodec#getMetrics} return value. 3684 * The value is a String. Values will be one of the constants 3685 * {@link #MODE_AUDIO} or {@link #MODE_VIDEO}. 3686 */ 3687 public static final String MODE = "android.media.mediacodec.mode"; 3688 3689 /** 3690 * The value returned for the key {@link #MODE} when the 3691 * codec is a audio codec. 3692 */ 3693 public static final String MODE_AUDIO = "audio"; 3694 3695 /** 3696 * The value returned for the key {@link #MODE} when the 3697 * codec is a video codec. 3698 */ 3699 public static final String MODE_VIDEO = "video"; 3700 3701 /** 3702 * Key to extract the flag indicating whether the codec is running 3703 * as an encoder or decoder from the {@link MediaCodec#getMetrics} return value. 3704 * The value is an integer. 3705 * A 0 indicates decoder; 1 indicates encoder. 3706 */ 3707 public static final String ENCODER = "android.media.mediacodec.encoder"; 3708 3709 /** 3710 * Key to extract the flag indicating whether the codec is running 3711 * in secure (DRM) mode from the {@link MediaCodec#getMetrics} return value. 3712 * The value is an integer. 3713 */ 3714 public static final String SECURE = "android.media.mediacodec.secure"; 3715 3716 /** 3717 * Key to extract the width (in pixels) of the video track 3718 * from the {@link MediaCodec#getMetrics} return value. 3719 * The value is an integer. 3720 */ 3721 public static final String WIDTH = "android.media.mediacodec.width"; 3722 3723 /** 3724 * Key to extract the height (in pixels) of the video track 3725 * from the {@link MediaCodec#getMetrics} return value. 3726 * The value is an integer. 3727 */ 3728 public static final String HEIGHT = "android.media.mediacodec.height"; 3729 3730 /** 3731 * Key to extract the rotation (in degrees) to properly orient the video 3732 * from the {@link MediaCodec#getMetrics} return. 3733 * The value is a integer. 3734 */ 3735 public static final String ROTATION = "android.media.mediacodec.rotation"; 3736 3737 } 3738 } 3739
