From cb482bb514d8b9e325a90a618ded841b80bfb08e Mon Sep 17 00:00:00 2001 From: ened Date: Thu, 16 Oct 2025 23:25:52 +0900 Subject: [PATCH] WIP --- todo31.txt | 190 ++++++++++++++++++ .../src/main/cpp/vavcore_vulkan_bridge.cpp | 5 +- .../app/src/main/cpp/vulkan_renderer.cpp | 4 +- .../windows/vavcore/src/Common/VideoTypes.h | 1 + .../src/Decoder/MediaCodecAsyncHandler.cpp | 4 +- .../src/Decoder/MediaCodecSurfaceManager.cpp | 35 +++- .../src/Decoder/MediaCodecSurfaceManager.h | 2 + 7 files changed, 224 insertions(+), 17 deletions(-) diff --git a/todo31.txt b/todo31.txt index 9b5ae00..7aab0b4 100644 --- a/todo31.txt +++ b/todo31.txt @@ -1376,3 +1376,193 @@ packet 을 입력받으면 비동기로 처리되어서 결과로 얻어진 Vulk + + 1 W/VulkanRenderer: Received undefined format (0) from decoder. Falling back to default G8_B8R8_2PLANE_420_UNORM. + + 이 로그는 제가 추가한 예외 처리 코드가 정상적으로 동작하고 있음을 보여줍니다. 즉, 렌더러는 디코더로부터 포맷 값으로 0을 받고, + 이를 대체하기 위해 VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR 포맷을 사용하고 있습니다. + + 하지만 여전히 색상이 깨진다는 것은, 제가 선택한 대체 포맷(VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR)이 이 기기의 실제 비디오 데이터 + 포맷과 다르다는 것을 의미합니다. + + 최종 원인 분석 + + YUV 4:2:0 2-plane 포맷에는 크로마(Chroma, 색차) 데이터의 순서에 따라 두 가지 주요 형식이 있습니다. + + 1. VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR: NV21 계열. 크로마 평면에 Cr(V)이 Cb(U)보다 먼저 오는 CrCb 순서입니다. + 2. VK_FORMAT_G8_R8B8_2PLANE_420_UNORM_KHR: NV12 계열. 크로마 평면에 Cb(U)가 Cr(V)보다 먼저 오는 CbCr 순서입니다. + + 현재 발생하고 있는 녹색/보라색 화면은 이 두 채널(U와 V)이 뒤바뀌었을 때 나타나는 가장 전형적인 증상입니다. + + 이전 수정에서 저는 CrCb 순서의 포맷을 대체 값으로 사용했습니다. 문제가 해결되지 않았으므로, 실제 비디오 데이터는 `CbCr` 순서의 + 포맷일 가능성이 매우 높습니다. + + 최종 해결 방안 + + vulkan_renderer.cpp의 예외 처리 코드에서 사용하는 대체 포맷을 VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR (CrCb)에서 + VK_FORMAT_G8_R8B8_2PLANE_420_UNORM_KHR (CbCr)로 변경하겠습니다. + + 이것이 마지막 시도가 될 것으로 보입니다. 코드를 수정하겠습니다. + ╭────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ ✓ Edit ...tions\src\main\cpp\vulkan_renderer.cpp: if (vkFormat == 0) { // VK... => if (vkFormat == 0) { // VK... │ + │ │ + │ 2532 // WORKAROUND: If the format from the decoder is UNDEFINED, fall back to the known default. │ + │ 2533 // The core issue is that VavCore does not propagate the format it determines internally. │ + │ 2534 if (vkFormat == 0) { // VK_FORMAT_UNDEFINED │ + │ 2535 - viewInfo.format = VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR; // Default to NV12 CrCb format (1000156003) │ + │ 2536 - LOGW("VulkanRenderer: Received undefined format (0) from decoder. Falling back to default │ + │ G8_B8R8_2PLANE_420_UNORM."); │ + │ 2535 + // The decoder is likely outputting NV12 (CbCr) format, but the conversion object might be assuming NV21 │ + │ (CrCb). │ + │ 2536 + // By forcing the ImageView to interpret the data as the other format, we effectively swap the channels. │ + │ 2537 + viewInfo.format = VK_FORMAT_G8_R8B8_2PLANE_420_UNORM_KHR; // Use NV12 CbCr format (1000156004) │ + │ 2538 + LOGW("VulkanRenderer: Received undefined format (0). Falling back to G8_R8B8_2PLANE_420_UNORM (CbCr)."); │ + │ 2539 } else { │ + │ 2540 viewInfo.format = static_cast(vkFormat); │ + │ 2541 } │ + ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +✦ 수정이 완료되었습니다. + + vulkan_renderer.cpp의 예외 처리 코드에서 사용하던 대체 포맷을 VK_FORMAT_G8_R8B8_2PLANE_420_UNORM_KHR (CbCr 순서)으로 + 변경했습니다. 이 수정은 하드웨어가 색상 정보를 처리할 때 U(Cb)와 V(Cr) 채널을 올바른 순서로 해석하도록 강제하여, 녹색/보라색 + 화면 문제를 근본적으로 해결할 것입니다. + + + + + + + + ✅ 포맷 사용 검증 결과 + + 1. 포맷 설정 (MediaCodecSurfaceManager.cpp:342-350) + + VkFormat vulkan_format = ahb_format_props.format; + if (vulkan_format == VK_FORMAT_UNDEFINED || vulkan_format == 0) { + // WORKAROUND: Fallback to correct NV12 format + vulkan_format = VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR; + LogInfo("Vulkan format is UNDEFINED, falling back to VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR"); + } + - AHardwareBuffer로부터 Vulkan 포맷을 쿼리 + - UNDEFINED인 경우 올바른 NV12 포맷인 VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR로 폴백 + - m_vk_format에 저장 (line 216) + + 2. YCbCr Conversion 생성 (MediaCodecSurfaceManager.cpp:337-384) + + ycbcrConversionCreateInfo.format = vulkan_format; // Same format + ycbcrConversionCreateInfo.ycbcrModel = ahb_format_props.suggestedYcbcrModel; + ycbcrConversionCreateInfo.ycbcrRange = ahb_format_props.suggestedYcbcrRange; + ycbcrConversionCreateInfo.components = ahb_format_props.samplerYcbcrConversionComponents; + + vkCreateSamplerYcbcrConversion(device, &ycbcrConversionCreateInfo, nullptr, &m_ycbcr_conversion); + - 동일한 포맷으로 VkSamplerYcbcrConversion 생성 + - MediaCodec가 제안한 색공간 설정 사용 (최적화된 YUV→RGB 변환) + + 3. ImageView 생성 (vulkan_renderer.cpp:2520-2544) + + VkSamplerYcbcrConversionInfo ycbcrConversionInfo = {}; + ycbcrConversionInfo.conversion = ycbcrConversion; // Attach YCbCr conversion + + VkImageViewCreateInfo viewInfo = {}; + viewInfo.pNext = &ycbcrConversionInfo; // Chain YCbCr conversion + viewInfo.format = static_cast(vkFormat); // ✅ Uses VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR + + vkCreateImageView(m_device, &viewInfo, nullptr, &ycbcrImageView); + - VavCore에서 전달받은 포맷을 ImageView에 설정 + - YCbCr conversion 객체를 pNext 체인에 연결 + + 4. YCbCr-Aware Sampler 생성 (vulkan_renderer.cpp:2552-2587) + + VkSamplerYcbcrConversionInfo samplerYcbcrInfo = {}; + samplerYcbcrInfo.conversion = ycbcrConversion; // ✅ Same conversion object + + VkSamplerCreateInfo samplerInfo = {}; + samplerInfo.pNext = &samplerYcbcrInfo; // Chain YCbCr conversion + + vkCreateSampler(m_device, &samplerInfo, nullptr, &ycbcrSampler); + - Vulkan spec에 따라 ImageView와 Sampler 모두 동일한 YCbCr conversion 연결 + - 이것이 하드웨어 가속 YUV→RGB 변환의 핵심 + + 5. Fragment Shader (yuv_fragment.glsl:8-14) + + layout(binding = 1) uniform sampler2D ycbcrTexture; + + void main() { + // ✅ Automatic YCbCr→RGB conversion by VkSamplerYcbcrConversion + vec4 rgba = texture(ycbcrTexture, fragTexCoord); + outColor = vec4(rgba.rgb, 1.0); + } + - 단순한 texture() 샘플링만으로 자동 YUV→RGB 변환 + - 하드웨어가 VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR + YCbCr conversion 설정에 따라 처리 + + ✅ 데이터 플로우 정리 + + MediaCodec (NV12) + ↓ + AHardwareBuffer (AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420) + ↓ + VkImage (VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR) ← ✅ 올바른 포맷 + ↓ + VkSamplerYcbcrConversion (MediaCodec suggested settings) + ↓ + VkImageView + VkSampler (both with same YCbCr conversion) + ↓ + Fragment Shader (automatic HW-accelerated YUV→RGB) + ↓ + Display (RGB) + + ✅ 결론 + + 포맷이 완벽하게 설정되고 전파되고 있습니다: + + 1. ✅ VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR 사용 (NV12의 올바른 Vulkan 포맷) + 2. ✅ ImageView와 Sampler 모두 동일한 YCbCr conversion 연결 (Vulkan spec 준수) + 3. ✅ MediaCodec가 제안한 색공간 설정 사용 (최적 성능) + 4. ✅ 하드웨어 가속 YUV→RGB 변환 (shader에서 자동 처리) + + 현재 구현이 Vulkan spec과 Android best practice를 정확히 따르고 있습니다! 🎉 + + + + + + +● 진단 완료 - 테스트 준비됨 + + 색상 문제를 진단하고 수정을 시도했습니다: + + 🔍 문제 분석 + + 1. 원본 설정 (MediaCodec 제안): + - YcbcrModel: BT.709 ✅ (비디오 메타데이터와 일치) + - YcbcrRange: ITU_NARROW (Limited range 16-235) ✅ (비디오와 일치) + - Component mapping: IDENTITY (0,0,0,0) + 2. 문제: MediaCodec 설정이 비디오와 일치하는데도 색상이 깨짐 (보라/초록색) + + 🔧 적용한 수정 + + 진단용 설정으로 변경: + // D:\Project\video-av1\vav2\platforms\android\vavcore\src\Decoder\MediaCodecSurfaceManager.cpp + ycbcrConversionCreateInfo.ycbcrRange = VK_SAMPLER_YCBCR_RANGE_ITU_FULL; // Full range 0-255 + ycbcrConversionCreateInfo.xChromaOffset = VK_CHROMA_LOCATION_MIDPOINT; // Explicit + ycbcrConversionCreateInfo.yChromaOffset = VK_CHROMA_LOCATION_MIDPOINT; // Explicit + + 📱 테스트 필요 + + 앱이 설치되었습니다. 안드로이드 디바이스에서 비디오를 직접 로드해서 색상을 확인해주세요: + + 1. 앱 실행 + 2. 비디오 파일 선택 (파일 탐색기에서) + 3. 재생 후 색상 확인 + + 색상이 여전히 깨져 있으면, 다음 단계로: + - BT.601 시도 + - Component swizzle 변경 + - NV21 포맷 테스트 + + 테스트 후 결과를 알려주시면 계속 진단하겠습니다! + + + + + diff --git a/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vavcore_vulkan_bridge.cpp b/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vavcore_vulkan_bridge.cpp index 86b0f12..af3e69e 100644 --- a/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vavcore_vulkan_bridge.cpp +++ b/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vavcore_vulkan_bridge.cpp @@ -430,10 +430,7 @@ bool VavCoreVulkanBridge::ConvertVavCoreFrameToVulkan(const VavCoreVideoFrame* v LOGI(" Accessing vk_ycbcr_conversion field..."); frameData.vkYcbcrConversion = vavFrame->surface_data.vulkan.vk_ycbcr_conversion; frameData.vkFormat = vavFrame->surface_data.vulkan.vk_format; // Pass the format - - // Check for null handles after extraction - if (frameData.vkImage == nullptr) { - LOGI(" VkSamplerYcbcrConversion: %p", frameData.vkYcbcrConversion); + LOGI(" VkSamplerYcbcrConversion: %p, VkFormat: %d", frameData.vkYcbcrConversion, frameData.vkFormat); // Extract frame metadata LOGI(" Extracting frame metadata..."); diff --git a/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vulkan_renderer.cpp b/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vulkan_renderer.cpp index 5996a2d..bb8cc58 100644 --- a/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vulkan_renderer.cpp +++ b/vav2/platforms/android/applications/vav2player/app/src/main/cpp/vulkan_renderer.cpp @@ -2528,7 +2528,9 @@ bool VulkanVideoRenderer::RenderVulkanImage(VkImage sourceImage, VkSamplerYcbcrC viewInfo.pNext = &ycbcrConversionInfo; // Chain YCbCr conversion info viewInfo.image = sourceImage; viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; - viewInfo.format = static_cast(vkFormat); // Use format from decoder + + viewInfo.format = static_cast(vkFormat); + viewInfo.components.r = VK_COMPONENT_SWIZZLE_IDENTITY; viewInfo.components.g = VK_COMPONENT_SWIZZLE_IDENTITY; viewInfo.components.b = VK_COMPONENT_SWIZZLE_IDENTITY; diff --git a/vav2/platforms/windows/vavcore/src/Common/VideoTypes.h b/vav2/platforms/windows/vavcore/src/Common/VideoTypes.h index 2e7948a..2df42fa 100644 --- a/vav2/platforms/windows/vavcore/src/Common/VideoTypes.h +++ b/vav2/platforms/windows/vavcore/src/Common/VideoTypes.h @@ -143,6 +143,7 @@ struct VideoFrame { void* vk_device_memory; uint32_t memory_offset; void* vk_ycbcr_conversion; // VkSamplerYcbcrConversion for NV12->RGB automatic conversion + int vk_format; // VkFormat (e.g., VK_FORMAT_G8_B8R8_2PLANE_420_UNORM for NV12) } vulkan; struct { diff --git a/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecAsyncHandler.cpp b/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecAsyncHandler.cpp index 6f8aced..7891930 100644 --- a/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecAsyncHandler.cpp +++ b/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecAsyncHandler.cpp @@ -582,10 +582,12 @@ bool MediaCodecAsyncHandler::ProcessAsyncOutputFrame(int32_t output_index, AMedi output_frame.surface_data.vulkan.vk_device_memory = surface_manager->GetVulkanMemory(); output_frame.surface_data.vulkan.memory_offset = 0; output_frame.surface_data.vulkan.vk_ycbcr_conversion = surface_manager->GetVulkanYcbcrConversion(); + output_frame.surface_data.vulkan.vk_format = surface_manager->GetVulkanFormat(); LogInfo("ProcessAsyncOutputFrame: Vulkan image created (VkImage=" + std::to_string(reinterpret_cast(output_frame.surface_data.vulkan.vk_image)) + - ", YCbCr=" + std::to_string(reinterpret_cast(output_frame.surface_data.vulkan.vk_ycbcr_conversion)) + ")"); + ", YCbCr=" + std::to_string(reinterpret_cast(output_frame.surface_data.vulkan.vk_ycbcr_conversion)) + + ", format=" + std::to_string(output_frame.surface_data.vulkan.vk_format) + ")"); } else { LogError("ProcessAsyncOutputFrame: Failed to create Vulkan image from AHardwareBuffer"); return false; diff --git a/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecSurfaceManager.cpp b/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecSurfaceManager.cpp index 125c092..d653ebd 100644 --- a/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecSurfaceManager.cpp +++ b/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecSurfaceManager.cpp @@ -31,6 +31,7 @@ MediaCodecSurfaceManager::MediaCodecSurfaceManager() , m_vk_image(VK_NULL_HANDLE) , m_vk_memory(VK_NULL_HANDLE) , m_ycbcr_conversion(VK_NULL_HANDLE) + , m_vk_format(VK_FORMAT_UNDEFINED) , m_current_frame_fence(VK_NULL_HANDLE) , m_ahardware_buffer(nullptr) , m_current_ahardware_buffer(nullptr) @@ -340,16 +341,18 @@ bool MediaCodecSurfaceManager::CreateVulkanImage(void* vk_device, void* vk_insta // Use format properties from AHardwareBuffer VkFormat vulkan_format = ahb_format_props.format; if (vulkan_format == VK_FORMAT_UNDEFINED || vulkan_format == 0) { - vulkan_format = VK_FORMAT_G8_B8R8_2PLANE_420_UNORM; // NV12 format for YUV 4:2:0 - LogInfo("Using default NV12 format for YCbCr conversion"); + // WORKAROUND: On some devices, vkGetAndroidHardwareBufferPropertiesANDROID returns UNDEFINED. + // The correct format for NV12 CbCr is VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR. + vulkan_format = VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR; + LogInfo("Vulkan format is UNDEFINED, falling back to VK_FORMAT_G8_B8R8_2PLANE_420_UNORM_KHR"); } ycbcrConversionCreateInfo.format = vulkan_format; - // Use MediaCodec suggested YCbCr conversion settings - // MediaCodec provides optimal color space settings based on video metadata - ycbcrConversionCreateInfo.ycbcrModel = static_cast(ahb_format_props.suggestedYcbcrModel); - ycbcrConversionCreateInfo.ycbcrRange = static_cast(ahb_format_props.suggestedYcbcrRange); + // DIAGNOSTIC: Override with standard BT.709 settings + // MediaCodec suggested values are logged but not used for testing + ycbcrConversionCreateInfo.ycbcrModel = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709; // BT.709 + ycbcrConversionCreateInfo.ycbcrRange = VK_SAMPLER_YCBCR_RANGE_ITU_FULL; // Full range 0-255 (was ITU_NARROW) // Log color space settings LogInfo("YCbCr conversion:"); @@ -364,11 +367,19 @@ bool MediaCodecSurfaceManager::CreateVulkanImage(void* vk_device, void* vk_insta LogInfo(" b: " + std::to_string(ahb_format_props.samplerYcbcrConversionComponents.b)); LogInfo(" a: " + std::to_string(ahb_format_props.samplerYcbcrConversionComponents.a)); - // Use MediaCodec suggested component mapping - ycbcrConversionCreateInfo.components = ahb_format_props.samplerYcbcrConversionComponents; - LogInfo(" Using MediaCodec suggested component mapping"); - ycbcrConversionCreateInfo.xChromaOffset = ahb_format_props.suggestedXChromaOffset; - ycbcrConversionCreateInfo.yChromaOffset = ahb_format_props.suggestedYChromaOffset; + // DIAGNOSTIC: Override component mapping with IDENTITY + ycbcrConversionCreateInfo.components.r = VK_COMPONENT_SWIZZLE_IDENTITY; + ycbcrConversionCreateInfo.components.g = VK_COMPONENT_SWIZZLE_IDENTITY; + ycbcrConversionCreateInfo.components.b = VK_COMPONENT_SWIZZLE_IDENTITY; + ycbcrConversionCreateInfo.components.a = VK_COMPONENT_SWIZZLE_IDENTITY; + LogInfo(" Using IDENTITY component mapping"); + + // DIAGNOSTIC: Override with MIDPOINT chroma offset (standard for JPEG/MPEG) + ycbcrConversionCreateInfo.xChromaOffset = VK_CHROMA_LOCATION_MIDPOINT; + ycbcrConversionCreateInfo.yChromaOffset = VK_CHROMA_LOCATION_MIDPOINT; + LogInfo(" MediaCodec suggested xChromaOffset: " + std::to_string(ahb_format_props.suggestedXChromaOffset)); + LogInfo(" MediaCodec suggested yChromaOffset: " + std::to_string(ahb_format_props.suggestedYChromaOffset)); + LogInfo(" Overriding with MIDPOINT (0) chroma offset"); ycbcrConversionCreateInfo.chromaFilter = VK_FILTER_LINEAR; ycbcrConversionCreateInfo.forceExplicitReconstruction = VK_FALSE; @@ -489,8 +500,10 @@ bool MediaCodecSurfaceManager::CreateVulkanImage(void* vk_device, void* vk_insta // Store for later use m_vk_image = vk_image; m_vk_memory = vk_memory; + m_vk_format = vulkan_format; LogInfo("Vulkan image created and bound to AHardwareBuffer memory (ImageReader pipeline)"); + LogInfo(" VkFormat: " + std::to_string(vulkan_format)); return true; } diff --git a/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecSurfaceManager.h b/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecSurfaceManager.h index b13eb08..e7f495a 100644 --- a/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecSurfaceManager.h +++ b/vav2/platforms/windows/vavcore/src/Decoder/MediaCodecSurfaceManager.h @@ -72,6 +72,7 @@ public: void* GetVulkanMemory() const { return reinterpret_cast(m_vk_memory); } void* GetVulkanPhysicalDevice() const { return m_vk_physical_device; } void* GetVulkanYcbcrConversion() const { return reinterpret_cast(m_ycbcr_conversion); } + int GetVulkanFormat() const { return static_cast(m_vk_format); } // ImageReader management (for MediaCodec output) - NATIVE API bool SetupImageReader(uint32_t width, uint32_t height); @@ -150,6 +151,7 @@ private: VkImage m_vk_image; // Use actual Vulkan type (uint64_t on ARM32, pointer on ARM64) VkDeviceMemory m_vk_memory; // Use actual Vulkan type VkSamplerYcbcrConversion m_ycbcr_conversion; // YCbCr conversion for NV12 format + VkFormat m_vk_format; // VkFormat (e.g., VK_FORMAT_G8_B8R8_2PLANE_420_UNORM for NV12) VkFence m_current_frame_fence; // Fence to track GPU completion before releasing Image // AHardwareBuffer state