基于 SurfaceView、AudioTrack、MediaCodec 和 MediaExtractor 解码 MP4 播放

一. 前言

上篇文章介绍了 基于Camera、AudioRecord 、MediaCodec 和 MediaMuxer 录制 MP4 , 录制的过程是这样的，那么相应的播放过程就是上述过程的逆过程，本篇文章将介绍如何通过 MediaExtractor 分离视频流和音频流，再通过 MediaCodec 解码，将数据传递给 SurfaceView 播放视频，给 AudioTrack 播放音频。

MediaExtractor

MediaExtractor 是 MediaMuxer 的逆过程，主要用于音视频混合数据的分离，并获取相应的音频轨对应的音频格式，和视频轨和视频格式。

1. 初始化

MediaExtractor extractor = new MediaExtractor();

2. 设置数据源

 extractor.setDataSource(...);

3. 找到音频轨或视频轨的媒体格式

 int numTracks = extractor.getTrackCount();
 for (int i = 0; i < numTracks; ++i) {
   MediaFormat format = extractor.getTrackFormat(i);
   String mime = format.getString(MediaFormat.KEY_MIME);
   if (weAreInterestedInThisTrack) {
     extractor.selectTrack(i);
   }
 }

4. 读取数据

 ByteBuffer inputBuffer = ByteBuffer.allocate(...)
 while (extractor.readSampleData(inputBuffer, ...) >= 0) {
   int trackIndex = extractor.getSampleTrackIndex();
   long presentationTimeUs = extractor.getSampleTime();// 获取 pts
   ...
   extractor.advance();//获取下一帧
 }

5. 释放资源

 extractor.release();
 extractor = null;

二. 解码播放 MP4

1. 音频处理

找到音频轨和媒体格式

MediaExtractor audioExtractor = new MediaExtractor();
        MediaCodec audioCodec = null;
        try {
            audioExtractor.setDataSource(mFileDescriptor);
        } catch (IOException e) {
            e.printStackTrace();
        }
        for (int i = 0; i < audioExtractor.getTrackCount(); i++) {
            MediaFormat mediaFormat = audioExtractor.getTrackFormat(i);
            String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
            if (mimeType.startsWith("audio/")) {
                audioExtractor.selectTrack(i);
                ...

根据 MediaFormat 创建 AudioTrack

for (int i = 0; i < audioExtractor.getTrackCount(); i++) {
            MediaFormat mediaFormat = audioExtractor.getTrackFormat(i);
            String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
            if (mimeType.startsWith("audio/")) {
                audioExtractor.selectTrack(i);
                int audioChannels = mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
                int audioSampleRate = mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
                int maxInputSize = mediaFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE);
                int minBufferSize = AudioTrack.getMinBufferSize(audioSampleRate, audioChannels == 1 ? AudioFormat.CHANNEL_OUT_MONO : AudioFormat.CHANNEL_OUT_STEREO, AudioFormat.ENCODING_PCM_16BIT);
                mAudioInputBufferSize = minBufferSize > 0 ? minBufferSize * 4 : maxInputSize;
                int frameSizeInBytes = audioChannels * 2;
                mAudioInputBufferSize = (mAudioInputBufferSize / frameSizeInBytes) * frameSizeInBytes;
                mAudioTrack = new AudioTrack(AudioManager.STREAM_MUSIC,
                        44100,
                        (audioChannels == 1 ? AudioFormat.CHANNEL_OUT_MONO : AudioFormat.CHANNEL_OUT_STEREO),
                        AudioFormat.ENCODING_PCM_16BIT,
                        mAudioInputBufferSize,
                        AudioTrack.MODE_STREAM);
                mAudioTrack.play();
                
                ...

创建编解码器并开始解码

 //
                try {
                    audioCodec = MediaCodec.createDecoderByType(mimeType);
                } catch (IOException e) {
                    e.printStackTrace();
                }
                audioCodec.configure(mediaFormat, null, null, 0);
                break;
            }
        }
        if (audioCodec == null) {
            return;
        }
        audioCodec.start();
        MediaCodec.BufferInfo decodeBufferInfo = new MediaCodec.BufferInfo();
        while (mIsPlaying) {
            int inputIndex = audioCodec.dequeueInputBuffer(10_000);
            if (inputIndex < 0) {
                mIsPlaying = false;
            }
            ByteBuffer inputBuffer = audioCodec.getInputBuffer(inputIndex);
            inputBuffer.clear();
            int sampleSize = audioExtractor.readSampleData(inputBuffer, 0);
            if (sampleSize > 0) {
                audioCodec.queueInputBuffer(inputIndex, 0, sampleSize, audioExtractor.getSampleTime(), 0);
                audioExtractor.advance();
            } else {
                mIsPlaying = false;
            }

            int outputIndex = audioCodec.dequeueOutputBuffer(decodeBufferInfo, 10_000);
            ByteBuffer outputBuffer;
            byte[] chunkPCM;
            while (outputIndex >= 0) {
                outputBuffer = audioCodec.getOutputBuffer(outputIndex);
                chunkPCM = new byte[decodeBufferInfo.size];
                outputBuffer.get(chunkPCM);
                outputBuffer.clear();
                mAudioTrack.write(chunkPCM, 0, decodeBufferInfo.size);
                audioCodec.releaseOutputBuffer(outputIndex, false);
                outputIndex = audioCodec.dequeueOutputBuffer(decodeBufferInfo, 10_000);

            }

        }

2. 视频处理

找到音视频轨并打开解码器

MediaExtractor videoExtractor = new MediaExtractor();
        MediaCodec videoCodec = null;
        long startWhen = 0;
        try {
            videoExtractor.setDataSource(mAssetFileDescriptor);
        } catch (IOException e) {
            e.printStackTrace();
        }
        boolean firstFrame = false;
        for (int i = 0; i < videoExtractor.getTrackCount(); i++) {
            MediaFormat mediaFormat = videoExtractor.getTrackFormat(i);
            String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
            if (mimeType.startsWith("video/")) {
                videoExtractor.selectTrack(i);
                try {
                    videoCodec = MediaCodec.createDecoderByType(mimeType);
                } catch (IOException e) {
                    e.printStackTrace();
                }
                videoCodec.configure(mediaFormat, mSurface, null, 0);
                break;
            }
        }
        if (videoCodec == null) {
            return;
        }
        videoCodec.start();

解码，渲染数据化

while (!mEOF) {
            MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
            ByteBuffer[] inputBuffers = videoCodec.getInputBuffers();
            int inputIndex = videoCodec.dequeueInputBuffer(10_000);
            if (inputIndex > 0) {
                ByteBuffer byteBuffer = inputBuffers[inputIndex];
                int sampleSize = videoExtractor.readSampleData(byteBuffer, 0);
                if (sampleSize > 0) {
                    videoCodec.queueInputBuffer(inputIndex, 0, sampleSize, videoExtractor.getSampleTime(), 0);
                    videoExtractor.advance();
                } else {
                    videoCodec.queueInputBuffer(inputIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
                }
            }

            int outputIndex = videoCodec.dequeueOutputBuffer(bufferInfo, 10_000);
            switch (outputIndex) {
                case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
                    videoCodec.getOutputBuffers();
                    break;
                case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
                    break;
                case MediaCodec.INFO_TRY_AGAIN_LATER:
                    break;
                default:
                    if (!firstFrame) {
                        startWhen = System.currentTimeMillis();
                        firstFrame = true;
                    }
                    long sleepTime = (bufferInfo.presentationTimeUs / 1000) - (System.currentTimeMillis() - startWhen);
                    if (sleepTime > 0) {
                        SystemClock.sleep(sleepTime);
                    }
                    videoCodec.releaseOutputBuffer(outputIndex, true);
                    break;

            }
            if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
                mEOF = true;
                break;
            }
        }

流程图如图所示：

github Demo