diff --git a/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vavcore_vulkan_bridge.cpp b/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vavcore_vulkan_bridge.cpp index 8b4b2fb..ef1badc 100644 --- a/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vavcore_vulkan_bridge.cpp +++ b/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vavcore_vulkan_bridge.cpp @@ -182,6 +182,10 @@ void VavCoreVulkanBridge::CloseVideoFile() { m_currentPositionUs = 0; m_frameNumber = 0; + // Reset drain mode state + m_isDraining = false; + m_drainedFrameCount = 0; + LOGI("Video file closed"); } @@ -232,6 +236,10 @@ bool VavCoreVulkanBridge::Stop() { m_currentPositionUs = 0; m_frameNumber = 0; + // Reset drain mode state + m_isDraining = false; + m_drainedFrameCount = 0; + if (m_player) { vavcore_reset(m_player); } @@ -271,16 +279,32 @@ bool VavCoreVulkanBridge::ProcessNextFrame() { } // Decode next frame to Vulkan surface (GPU zero-copy pipeline) + // 16-Frame Buffering Pattern: + // - Normal mode: target_surface=non-NULL → read packet and decode + // - Drain mode: target_surface=NULL → flush buffered frames (no packet reading) VavCoreVideoFrame frame = {}; - LOGI("Calling vavcore_decode_to_surface..."); + LOGI("Calling vavcore_decode_to_surface (draining=%s)...", m_isDraining ? "true" : "false"); + + // Use target_surface to signal drain mode to VavCore + // Normal mode: Pass dummy non-NULL pointer (VkImage is managed internally by VavCore) + // Drain mode: Pass NULL to trigger buffered frame flush + void* target_surface = m_isDraining ? nullptr : (void*)0x1; + VavCoreResult result = vavcore_decode_to_surface(m_player, VAVCORE_SURFACE_VULKAN_IMAGE, - nullptr, // target_surface (not needed for Vulkan) + target_surface, &frame); LOGI("vavcore_decode_to_surface returned: %d", result); - if (result == VAVCORE_END_OF_STREAM) { - LOGI("End of stream reached"); + // Handle 16-Frame Buffering Pattern results + if (result == VAVCORE_PACKET_ACCEPTED) { + // Priming phase: packet accepted but no frame output yet + // This is normal during the first 16 frames (buffering phase) + LOGI("Packet accepted - buffering phase (no frame output yet)"); + return true; // Continue processing, not an error + } else if (result == VAVCORE_END_OF_STREAM) { + // All buffered frames consumed - draining complete + LOGI("End of stream reached - all buffered frames consumed"); SetPlaybackState(PlaybackState::STOPPED); return false; } else if (result != VAVCORE_SUCCESS) { @@ -740,9 +764,12 @@ void VavCoreVulkanBridge::PlaybackThreadMain() { LOGI("Playback thread started"); int frameCount = 0; + const uint32_t MAX_DRAIN_ATTEMPTS = 16; // Maximum buffered frames + while (ShouldContinuePlayback()) { frameCount++; - LOGI("=== Playback Loop Iteration #%d START ===", frameCount); + LOGI("=== Playback Loop Iteration #%d START (draining=%s) ===", + frameCount, m_isDraining ? "true" : "false"); auto frameStart = std::chrono::steady_clock::now(); // Process next frame @@ -751,10 +778,32 @@ void VavCoreVulkanBridge::PlaybackThreadMain() { LOGI("ProcessNextFrame() returned: %s", success ? "true" : "false"); if (!success) { - LOGI("End of video or decode error, stopping playback"); - // Set state to stopped and break the loop - SetPlaybackState(PlaybackState::STOPPED); - break; + // Check if we should enter drain mode + if (!m_isDraining) { + LOGI("End of file detected - entering drain mode to flush buffered frames"); + m_isDraining = true; + m_drainedFrameCount = 0; + + // Continue to drain buffered frames + continue; + } else { + // Already draining and got failure - all frames consumed + LOGI("Drain complete - all buffered frames consumed"); + SetPlaybackState(PlaybackState::STOPPED); + break; + } + } + + // Check drain attempt limit + if (m_isDraining) { + m_drainedFrameCount++; + LOGI("Drained frame %u/%u", m_drainedFrameCount, MAX_DRAIN_ATTEMPTS); + + if (m_drainedFrameCount >= MAX_DRAIN_ATTEMPTS) { + LOGI("Maximum drain attempts reached (%u frames)", MAX_DRAIN_ATTEMPTS); + SetPlaybackState(PlaybackState::STOPPED); + break; + } } // Calculate frame timing diff --git a/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vavcore_vulkan_bridge.h b/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vavcore_vulkan_bridge.h index 78143f4..0b82683 100644 --- a/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vavcore_vulkan_bridge.h +++ b/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vavcore_vulkan_bridge.h @@ -165,6 +165,10 @@ private: uint64_t m_renderedFrameCount = 0; uint64_t m_droppedFrameCount = 0; + // 16-Frame Buffering Pattern support (MediaCodec latency hiding) + bool m_isDraining = false; // True when draining buffered frames (EOS reached) + uint32_t m_drainedFrameCount = 0; // Number of frames drained from buffer + // Continuous playback thread std::thread m_playbackThread; std::atomic m_shouldContinuePlayback{false}; diff --git a/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecAsyncHandler.cpp b/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecAsyncHandler.cpp index af90135..42f5d45 100644 --- a/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecAsyncHandler.cpp +++ b/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecAsyncHandler.cpp @@ -56,6 +56,9 @@ void MediaCodecAsyncHandler::Cleanup() { while (!m_async_input_buffer_queue.empty()) { m_async_input_buffer_queue.pop(); } + while (!m_pending_output_buffers.empty()) { + m_pending_output_buffers.pop(); + } // Reset hidden queue pattern state m_prebuffering = true; @@ -101,33 +104,30 @@ bool MediaCodecAsyncHandler::InitializeAsyncMode() { }; m_async_callbacks.onOutputBufferAvailable = [this](int32_t index, AMediaCodecBufferInfo* bufferInfo) { - // Output buffer available - process in callback + // DEADLOCK FIX: Do NOT call MediaCodec APIs from callback thread + // Instead, store index and bufferInfo in queue for processing by decode thread try { - VideoFrame frame; - if (ProcessAsyncOutputFrame(index, bufferInfo, frame)) { - std::lock_guard lock(m_async_mutex); + std::lock_guard lock(m_async_mutex); - // Hidden Queue Pattern: Check buffer size limit to prevent overflow - if (m_async_output_queue.size() >= MAX_BUFFER_SIZE) { - LogWarning("Frame queue full (size=" + std::to_string(m_async_output_queue.size()) + - "/" + std::to_string(MAX_BUFFER_SIZE) + ") - dropping frame (timestamp=" + - std::to_string(bufferInfo->presentationTimeUs) + "us)"); - // Frame resources already released by ProcessAsyncOutputFrame - // This prevents unbounded queue growth when consumer is slower than producer - return; - } + // Copy bufferInfo (callback pointer is ephemeral and will be invalidated) + PendingOutputBuffer pending; + pending.index = index; + pending.bufferInfo = *bufferInfo; // Deep copy - AsyncFrameData async_data; - async_data.frame = std::make_unique(std::move(frame)); - async_data.timestamp_us = bufferInfo->presentationTimeUs; - // TODO: NDK 26 does not expose keyframe flag in AMediaCodecBufferInfo - // Keyframe detection needs to be done via other means (e.g., frame analysis) - async_data.is_keyframe = false; // Placeholder - keyframe flag not available in NDK 26 - async_data.decode_start_time = std::chrono::steady_clock::now(); - - m_async_output_queue.push(std::move(async_data)); - m_async_condition.notify_one(); + // Check queue size limit to prevent overflow + if (m_pending_output_buffers.size() >= MAX_BUFFER_SIZE) { + LogWarning("Pending output buffer queue full (size=" + std::to_string(m_pending_output_buffers.size()) + + "/" + std::to_string(MAX_BUFFER_SIZE) + ") - dropping buffer (index=" + + std::to_string(index) + ", timestamp=" + std::to_string(bufferInfo->presentationTimeUs) + "us)"); + // Release buffer immediately without rendering to prevent MediaCodec stall + AMediaCodec_releaseOutputBuffer(m_codec, index, false); + return; } + + m_pending_output_buffers.push(pending); + LogInfo("Output buffer stored in pending queue: index=" + std::to_string(index) + + ", pending queue size=" + std::to_string(m_pending_output_buffers.size())); + m_async_condition.notify_one(); } catch (const std::exception& e) { LogError("Exception in onOutputBufferAvailable: " + std::string(e.what())); } catch (...) { @@ -218,6 +218,9 @@ void MediaCodecAsyncHandler::CleanupAsyncMode() { while (!m_async_input_buffer_queue.empty()) { m_async_input_buffer_queue.pop(); } + while (!m_pending_output_buffers.empty()) { + m_pending_output_buffers.pop(); + } LogInfo("Async mode cleanup complete"); } @@ -310,10 +313,52 @@ bool MediaCodecAsyncHandler::DecodeFrameAsync(const uint8_t* packet_data, size_t return false; } - LogInfo("DecodeFrameAsync: Input buffer queued successfully"); + LogInfo("DecodeFrameAsync: Input buffer queued successfully, now processing pending outputs..."); + + // DEADLOCK FIX: Process pending output buffers (defer MediaCodec API calls out of callback) + // Callbacks store output indices in pending queue, decode thread processes them here + { + std::unique_lock lock(m_async_mutex); + + // Process all pending output buffers + while (!m_pending_output_buffers.empty()) { + PendingOutputBuffer pending = m_pending_output_buffers.front(); + m_pending_output_buffers.pop(); + lock.unlock(); // Release lock while calling MediaCodec APIs + + LogInfo("DecodeFrameAsync: Processing pending output buffer index=" + std::to_string(pending.index)); + + // Process frame outside callback context (safe to call releaseOutputBuffer here) + VideoFrame frame; + if (ProcessAsyncOutputFrame(pending.index, &pending.bufferInfo, frame)) { + // Frame processed successfully - add to output queue + std::lock_guard queue_lock(m_async_mutex); + + // Hidden Queue Pattern: Check buffer size limit + if (m_async_output_queue.size() >= MAX_BUFFER_SIZE) { + LogWarning("Frame queue full (size=" + std::to_string(m_async_output_queue.size()) + + "/" + std::to_string(MAX_BUFFER_SIZE) + ") - dropping frame"); + // Frame resources already released by ProcessAsyncOutputFrame + lock.lock(); + continue; + } + + AsyncFrameData async_data; + async_data.frame = std::make_unique(std::move(frame)); + async_data.timestamp_us = pending.bufferInfo.presentationTimeUs; + async_data.is_keyframe = false; + async_data.decode_start_time = std::chrono::steady_clock::now(); + + m_async_output_queue.push(std::move(async_data)); + LogInfo("DecodeFrameAsync: Frame added to output queue (size=" + + std::to_string(m_async_output_queue.size()) + ")"); + } + + lock.lock(); // Re-acquire lock for next iteration + } + } // Check if output frame is already available in queue (non-blocking) - // MediaCodec async callbacks will populate the queue when frames are ready { std::lock_guard lock(m_async_mutex); diff --git a/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecAsyncHandler.h b/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecAsyncHandler.h index 9dcc5bf..1322835 100644 --- a/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecAsyncHandler.h +++ b/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecAsyncHandler.h @@ -118,6 +118,14 @@ private: // Async input buffer index queue std::queue m_async_input_buffer_queue; + // Pending output buffer indices (stored by callback, processed by decode thread) + // This avoids deadlock by deferring MediaCodec API calls out of callback context + struct PendingOutputBuffer { + int32_t index; + AMediaCodecBufferInfo bufferInfo; // Copy of buffer info (callback pointer is ephemeral) + }; + std::queue m_pending_output_buffers; + // Async callbacks MediaCodecAsyncCallbacks m_async_callbacks;