Mediacodec 如何硬件解码到纹理的

Java
 MediaCodec codec = MediaCodec.createByCodecName(name);
 codec.configure(format, …);
 MediaFormat outputFormat = codec.getOutputFormat(); // option B
 codec.start();
 for (;;) {
  int inputBufferId = codec.dequeueInputBuffer(timeoutUs);
  if (inputBufferId >= 0) {
    ByteBuffer inputBuffer = codec.getInputBuffer(…);
    // fill inputBuffer with valid data
    …
    codec.queueInputBuffer(inputBufferId, …);
  }
  int outputBufferId = codec.dequeueOutputBuffer(…);
  if (outputBufferId >= 0) {
    ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId);
    MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A
    // bufferFormat is identical to outputFormat
    // outputBuffer is ready to be processed or rendered.
    …
    codec.releaseOutputBuffer(outputBufferId, …);
  } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
    // Subsequent data will conform to new format.
    // Can ignore if using getOutputFormat(outputBufferId)
    outputFormat = codec.getOutputFormat(); // option B
  }
 }
 codec.stop();
 codec.release();

Java
public void releaseOutputBuffer (int index,
                boolean render)
/*
If you are done with a buffer, use this call to return the buffer to the codec or to render it on the output surface. If you configured the codec with an output surface, setting render to true will first send the buffer to that output surface. The surface will release the buffer back to the codec once it is no longer used/displayed. Once an output buffer is released to the codec, it MUST NOT be used until it is later retrieved by getOutputBuffer(int) in response to a dequeueOutputBuffer(MediaCodec.BufferInfo, long) return value or a Callback#onOutputBufferAvailable callback.

Parameters
indexint: The index of a client-owned output buffer previously returned from a call to dequeueOutputBuffer(MediaCodec.BufferInfo, long).

renderboolean: If a valid surface was specified when configuring the codec, passing true renders this output buffer to the surface.
*/

C++
status_t JMediaCodec::releaseOutputBuffer(
        size_t index, bool render, bool updatePTS, int64_t timestampNs) {
    if (updatePTS) {
        return mCodec->renderOutputBufferAndRelease(index, timestampNs);
    }
    return render
        ? mCodec->renderOutputBufferAndRelease(index)
        : mCodec->releaseOutputBuffer(index);
}

C++
status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) {
    sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
    msg->setSize("index", index);
    msg->setInt32("render", true);
    msg->setInt64("timestampNs", timestampNs);

    sp<AMessage> response;
    return PostAndAwaitResponse(msg, &response);
}

C++
void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
    switch (msg->what()) {
    ....
    case kWhatReleaseOutputBuffer:
        {
            sp<AReplyToken> replyID;
            CHECK(msg->senderAwaitsResponse(&replyID));
            status_t err = onReleaseOutputBuffer(msg);

            PostReplyWithError(replyID, err);
            break;
        }
        .........
    }
}

C++
tatus_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
    size_t index;
    CHECK(msg->findSize("index", &index));

    int32_t render;
    if (!msg->findInt32("render", &render)) {
        render = 0;
    }

    if (index >= mPortBuffers[kPortIndexOutput].size()) {
        return -ERANGE;
    }

    BufferInfo *info = &mPortBuffers[kPortIndexOutput][index];

    if (info->mData == nullptr || !info->mOwnedByClient) {
        return -EACCES;
    }

    // synchronization boundary for getBufferAndFormat
    sp<MediaCodecBuffer> buffer;
  
    if (render && buffer->size() != 0) {
        ......
        status_t err = mBufferChannel->renderOutputBuffer(buffer, renderTimeNs);

        if (err == NO_INIT) {
            ALOGE("rendering to non-initilized(obsolete) surface");
            return err;
        }
        if (err != OK) {
            ALOGI("rendring output error %d", err);
        }
    }
    return OK;
}
 

C++
status_t ACodecBufferChannel::renderOutputBuffer(
        const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
    std::shared_ptr<const std::vector<const BufferInfo>> array(
            std::atomic_load(&mOutputBuffers));
    BufferInfoIterator it = findClientBuffer(array, buffer);
    if (it == array->end()) {
        return -ENOENT;
    }

    ALOGV("renderOutputBuffer #%d", it->mBufferId);
    sp<AMessage> msg = mOutputBufferDrained->dup();
    msg->setObject("buffer", buffer);
    msg->setInt32("buffer-id", it->mBufferId);
    msg->setInt32("render", true);
    msg->setInt64("timestampNs", timestampNs);
    msg->post();
    return OK;
}

C++
ACodecBufferChannel::ACodecBufferChannel(
        const sp<AMessage> &inputBufferFilled, const sp<AMessage> &outputBufferDrained)
    : mInputBufferFilled(inputBufferFilled),
      mOutputBufferDrained(outputBufferDrained),
      mHeapSeqNum(-1) {
}
std::shared_ptr<BufferChannelBase> ACodec::getBufferChannel() {
    if (!mBufferChannel) {
        mBufferChannel = std::make_shared<ACodecBufferChannel>(
                new AMessage(kWhatInputBufferFilled, this),
                new AMessage(kWhatOutputBufferDrained, this));
    }
    return mBufferChannel;
}
status_t MediaCodec::init(const AString &name) {

    ..............
    mBufferChannel = mCodec->getBufferChannel();
    mBufferChannel->setCallback(
            std::unique_ptr<CodecBase::BufferCallback>(
                    new BufferCallback(new AMessage(kWhatCodecNotify, this))));
}

C++
bool ACodec::BaseState::onMessageReceived(const sp<AMessage> &msg) {
    switch (msg->what()) {
        case kWhatOutputBufferDrained:
        {
            onOutputBufferDrained(msg);
            break;
        }
}

C++

void ACodec::BaseState::onOutputBufferDrained(const sp<AMessage> &msg) {
    IOMX::buffer_id bufferID;
    CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID));
    sp<RefBase> obj;
    CHECK(msg->findObject("buffer", &obj));
    sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
    int32_t discarded = 0;
    msg->findInt32("discarded", &discarded);


    info->mData = buffer;
    int32_t render;
    .........
     err = mCodec->mNativeWindow->queueBuffer(
            mCodec->mNativeWindow.get(), info->mGraphicBuffer.get(), info->mFenceFd);
    .........
}
 

C++

int Surface::queueBuffer(android_native_buffer_t* buffer, int fenceFd) {
    ATRACE_CALL();
    ALOGV("Surface::queueBuffer");
    Mutex::Autolock lock(mMutex);
    int64_t timestamp;
    bool isAutoTimestamp = false;
    .........
    int i = getSlotFromBufferLocked(buffer);
    nsecs_t now = systemTime();
    status_t err = mGraphicBufferProducer->queueBuffer(i, input, &output);
    mLastQueueDuration = systemTime() - now;
    if (err != OK)  {
        ALOGE("queueBuffer: error queuing buffer to SurfaceTexture, %d", err);
    }
    ............

    return err;
}
 

C++

status_t BufferQueueProducer::queueBuffer(int slot,
        const QueueBufferInput &input, QueueBufferOutput *output) {
    ATRACE_CALL();
    ATRACE_BUFFER_INDEX(slot);
    ..........................
       
        item.mAcquireCalled = mSlots[slot].mAcquireCalled;
        item.mGraphicBuffer = mSlots[slot].mGraphicBuffer;
        item.mCrop = crop;
        item.mTransform = transform &
                ~static_cast<uint32_t>(NATIVE_WINDOW_TRANSFORM_INVERSE_DISPLAY);
        item.mTransformToDisplayInverse =
                (transform & NATIVE_WINDOW_TRANSFORM_INVERSE_DISPLAY) != 0;
        item.mScalingMode = static_cast<uint32_t>(scalingMode);
        item.mTimestamp = requestedPresentTimestamp;
        item.mIsAutoTimestamp = isAutoTimestamp;
        item.mDataSpace = dataSpace;
        item.mHdrMetadata = hdrMetadata;
        item.mFrameNumber = currentFrameNumber;
        item.mSlot = slot;
        item.mFence = acquireFence;
        item.mFenceTime = acquireFenceTime;
        item.mIsDroppable = mCore->mAsyncMode ||
                (mConsumerIsSurfaceFlinger && mCore->mQueueBufferCanDrop) ||
                (mCore->mLegacyBufferDrop && mCore->mQueueBufferCanDrop) ||
                (mCore->mSharedBufferMode && mCore->mSharedBufferSlot == slot);
        item.mSurfaceDamage = surfaceDamage;
        item.mQueuedBuffer = true;
        item.mAutoRefresh = mCore->mSharedBufferMode && mCore->mAutoRefresh;
        item.mApi = mCore->mConnectedApi;

        mStickyTransform = stickyTransform;

        output->bufferReplaced = false;
        if (mCore->mQueue.empty()) {
            // When the queue is empty, we can ignore mDequeueBufferCannotBlock
            // and simply queue this buffer
            mCore->mQueue.push_back(item);
            frameAvailableListener = mCore->mConsumerListener;
        } else {
                .......
                mCore->mQueue.push_back(item);
                frameAvailableListener = mCore->mConsumerListener;
                .......
        }

          ........
        if (frameAvailableListener != nullptr) {
            frameAvailableListener->onFrameAvailable(item);
        }
    }

    return NO_ERROR;
}
 

C++
status_t SurfaceTexture::updateTexImage() {
    ATRACE_CALL();
    SFT_LOGV("updateTexImage");
    Mutex::Autolock lock(mMutex);

    if (mAbandoned) {
        SFT_LOGE("updateTexImage: SurfaceTexture is abandoned!");
        return NO_INIT;
    }

    return mEGLConsumer.updateTexImage(*this);
}

C++
status_t EGLConsumer::updateTexImage(SurfaceTexture& st){   
     BufferItem item;

    // Acquire the next buffer.
    // In asynchronous mode the list is guaranteed to be one buffer
    // deep, while in synchronous mode we use the oldest buffer.
    err = st.acquireBufferLocked(&item, 0);
   .......
    // Release the previous buffer.
    err = updateAndReleaseLocked(item, nullptr, st);
   ........
    // Bind the new buffer to the GL texture, and wait until it's ready.
    return bindTextureImageLocked(st);
 }

C++
status_t ConsumerBase::acquireBufferLocked(BufferItem *item,
        nsecs_t presentWhen, uint64_t maxFrameNumber) {
    if (mAbandoned) {
        CB_LOGE("acquireBufferLocked: ConsumerBase is abandoned!");
        return NO_INIT;
    }

    status_t err = mConsumer->acquireBuffer(item, presentWhen, maxFrameNumber);
    if (err != NO_ERROR) {
        return err;
    }

    if (item->mGraphicBuffer != nullptr) {
        if (mSlots[item->mSlot].mGraphicBuffer != nullptr) {
            freeBufferLocked(item->mSlot);
        }
        mSlots[item->mSlot].mGraphicBuffer = item->mGraphicBuffer;
    }

    mSlots[item->mSlot].mFrameNumber = item->mFrameNumber;
    mSlots[item->mSlot].mFence = item->mFence;

    CB_LOGV("acquireBufferLocked: -> slot=%d/%" PRIu64,
            item->mSlot, item->mFrameNumber);

    return OK;
}

C++

status_t EGLConsumer::updateAndReleaseLocked(const BufferItem& item, PendingRelease* pendingRelease,
                                             SurfaceTexture& st) {
    status_t err = NO_ERROR;

    int slot = item.mSlot;
    ........
    err = mEglSlots[slot].mEglImage->createIfNeeded(mEglDisplay);
    if (err != NO_ERROR) {
        EGC_LOGW("updateAndRelease: unable to createImage on display=%p slot=%d", mEglDisplay,
                 slot);
        st.releaseBufferLocked(slot, st.mSlots[slot].mGraphicBuffer, mEglDisplay, EGL_NO_SYNC_KHR);
        return UNKNOWN_ERROR;
    }
    ...................
     sp<EglImage> nextTextureImage = mEglSlots[slot].mEglImage;
    mCurrentTextureImage = nextTextureImage;
    return err;
}
status_t EGLConsumer::EglImage::createIfNeeded(EGLDisplay eglDisplay, bool forceCreation) {
    // If there's an image and it's no longer valid, destroy it.
 
   .......
   mEglImage = createImage(mEglDisplay, mGraphicBuffer);
   ........

    return OK;
}
EGLImageKHR EGLConsumer::EglImage::createImage(EGLDisplay dpy,
                                               const sp<GraphicBuffer>& graphicBuffer) {
    EGLClientBuffer cbuf = static_cast<EGLClientBuffer>(graphicBuffer->getNativeBuffer());
    const bool createProtectedImage =
            (graphicBuffer->getUsage() & GRALLOC_USAGE_PROTECTED) && hasEglProtectedContent();
    EGLint attrs[] = {
            EGL_IMAGE_PRESERVED_KHR,
            EGL_TRUE,
            createProtectedImage ? EGL_PROTECTED_CONTENT_EXT : EGL_NONE,
            createProtectedImage ? EGL_TRUE : EGL_NONE,
            EGL_NONE,
    };
    eglInitialize(dpy, nullptr, nullptr);
    EGLImageKHR image =
            eglCreateImageKHR(dpy, EGL_NO_CONTEXT, EGL_NATIVE_BUFFER_ANDROID, cbuf, attrs);
    if (image == EGL_NO_IMAGE_KHR) {
        EGLint error = eglGetError();
        ALOGE("error creating EGLImage: %#x", error);
        eglTerminate(dpy);
    }
    return image;
}
 

C++

EGLAPI EGLImageKHR EGLAPIENTRY eglCreateImageKHR(EGLDisplay dpy, EGLContext ctx, EGLenum target, EGLClientBuffer buffer, const EGLint *attrib_list)
{
        return egl::CreateImageKHR(dpy, ctx, target, buffer, attrib_list);
}

C++

EGLImageKHR CreateImageKHR(EGLDisplay dpy, EGLContext ctx, EGLenum target, EGLClientBuffer buffer, const EGLint *attrib_list)
{
        TRACE("(EGLDisplay dpy = %p, EGLContext ctx = %p, EGLenum target = 0x%X, buffer = %p, const EGLint attrib_list = %p)", dpy, ctx, target, buffer, attrib_list);

        egl::Display *display = egl::Display::get(dpy);
        egl::Context *context = static_cast<egl::Context*>(ctx);

        .................
        GLuint name = static_cast<GLuint>(reinterpret_cast<uintptr_t>(buffer));

        if(name == 0)
        {
                return error(EGL_BAD_PARAMETER, EGL_NO_IMAGE_KHR);
        }
        if(target == EGL_NATIVE_BUFFER_ANDROID)
        {
                ANativeWindowBuffer *nativeBuffer = reinterpret_cast<ANativeWindowBuffer*>(buffer);

                if(!nativeBuffer || GLPixelFormatFromAndroid(nativeBuffer->format) == GL_NONE)
                {
                        ALOGW("%s badness unsupported HAL format=%x", __FUNCTION__, nativeBuffer ? nativeBuffer->format : 0);
                        return error(EGL_BAD_ATTRIBUTE, EGL_NO_IMAGE_KHR);
                }

                Image *image = new AndroidNativeImage(nativeBuffer);
                EGLImageKHR eglImage = display->createSharedImage(image);

                return success(eglImage);
        }

        return success(eglImage);
}
EGLImageKHR Display::createSharedImage(Image *image)
{
        return reinterpret_cast<EGLImageKHR>((intptr_t)mSharedImageNameSpace.allocate(image));
}

C++

status_t EGLConsumer::bindTextureImageLocked(SurfaceTexture& st) {
    if (mEglDisplay == EGL_NO_DISPLAY) {
        ALOGE("bindTextureImage: invalid display");
        return INVALID_OPERATION;
    }

  
    glBindTexture(st.mTexTarget, st.mTexName);
    if (st.mCurrentTexture == BufferQueue::INVALID_BUFFER_SLOT && mCurrentTextureImage == nullptr) {
        EGC_LOGE("bindTextureImage: no currently-bound texture");
        return NO_INIT;
    }
    ........................

    mCurrentTextureImage->bindToTextureTarget(st.mTexTarget);

    ........................
    // Wait for the new buffer to be ready.
    return doGLFenceWaitLocked(st);
}
 

C++

GL_APICALL void GL_APIENTRY glBindTexture(GLenum target, GLuint texture)
{
        return es2::BindTexture(target, texture);
}

void BindTexture(GLenum target, GLuint texture)
{
        TRACE("(GLenum target = 0x%X, GLuint texture = %d)", target, texture);

        es2::Context *context = es2::getContext();

        if(context)
        {
                switch(target)
                {
         
                case GL_TEXTURE_EXTERNAL_OES:
                        context->bindTextureExternal(texture);
                        break;
 
                }
        }
}

void Context::bindTextureExternal(GLuint texture)
{
        mResourceManager->checkTextureAllocation(texture, TEXTURE_EXTERNAL);

        mState.samplerTexture[TEXTURE_EXTERNAL][mState.activeSampler] = getTexture(texture);
}

void ResourceManager::checkTextureAllocation(GLuint texture, TextureType type)
{
        if(!getTexture(texture) && texture != 0)
        {
                Texture *textureObject;

                if(type == TEXTURE_2D)
                {
                        textureObject = new Texture2D(texture);
                }
                else if(type == TEXTURE_CUBE)
                {
                        textureObject = new TextureCubeMap(texture);
                }
                else
                {
                        UNREACHABLE(type);
                        return;
                }

                mTextureMap[texture] = textureObject;
                textureObject->addRef();
        }
}
 

C++
void EGLConsumer::EglImage::bindToTextureTarget(uint32_t texTarget) {
    glEGLImageTargetTexture2DOES(texTarget, static_cast<GLeglImageOES>(mEglImage));
}
GL_APICALL void GL_APIENTRY glEGLImageTargetTexture2DOES(GLenum target, GLeglImageOES image)
{
        return es2::EGLImageTargetTexture2DOES(target, image);
}

void EGLImageTargetTexture2DOES(GLenum target, GLeglImageOES image)
{
     
        es2::Context *context = es2::getContext();

        if(context)
        {
                es2::Texture2D *texture = nullptr;

                switch(target)
                {
                case GL_TEXTURE_2D:           texture = context->getTexture2D();       break;
                case GL_TEXTURE_EXTERNAL_OES: texture = context->getTextureExternal(); break;
                default:                      UNREACHABLE(target);
                }

             

                egl::Image *eglImage = context->getSharedImage(image);

                if(!eglImage)
                {
                        return error(GL_INVALID_OPERATION);
                }

                texture->setSharedImage(eglImage);
        }
}
 

C++
  mResourceManager->checkTextureAllocation(texture, TEXTURE_EXTERNAL);

        mState.samplerTexture[TEXTURE_EXTERNAL][mState.activeSampler] = getTexture(texture);