using Godot; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; // VavCore 데이터 구조체들 [StructLayout(LayoutKind.Sequential)] public struct VavCoreVideoFrame { // Legacy CPU fields (for backward compatibility) public IntPtr y_plane; // uint8_t* public IntPtr u_plane; // uint8_t* public IntPtr v_plane; // uint8_t* public int y_stride; // Y plane stride public int u_stride; // U plane stride public int v_stride; // V plane stride // Frame metadata public int width; // Frame width public int height; // Frame height public ulong timestamp_us; // Timestamp in microseconds public ulong frame_number; // Frame sequence number // Surface type and data (we'll use CPU mode for now) public int surface_type; // VavCoreSurfaceType (0 = CPU) // Union data - we'll only use the first 64 bytes for CPU data [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)] public ulong[] surface_data; // Union as array of ulongs } public partial class VavCorePlayer : Control { // VavCore DLL Import - Use addons plugin path private const string VavCoreDll = "addons/VavCoreGodot/bin/VavCore.dll"; [DllImport(VavCoreDll, CallingConvention = CallingConvention.Cdecl)] private static extern int vavcore_initialize(); [DllImport(VavCoreDll, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr vavcore_create_player(); [DllImport(VavCoreDll, CallingConvention = CallingConvention.Cdecl)] private static extern int vavcore_open_file(IntPtr player, string filePath); [DllImport(VavCoreDll, CallingConvention = CallingConvention.Cdecl)] private static extern bool vavcore_is_open(IntPtr player); [DllImport(VavCoreDll, CallingConvention = CallingConvention.Cdecl)] private static extern void vavcore_destroy_player(IntPtr player); [DllImport(VavCoreDll, CallingConvention = CallingConvention.Cdecl)] private static extern void vavcore_cleanup(); [DllImport(VavCoreDll, CallingConvention = CallingConvention.Cdecl)] private static extern void vavcore_close_file(IntPtr player); [DllImport(VavCoreDll, CallingConvention = CallingConvention.Cdecl)] private static extern int vavcore_decode_frame(IntPtr player, out VavCoreVideoFrame frame); [DllImport(VavCoreDll, CallingConvention = CallingConvention.Cdecl)] private static extern int vavcore_decode_next_frame(IntPtr player, out VavCoreVideoFrame frame); [DllImport(VavCoreDll, CallingConvention = CallingConvention.Cdecl)] private static extern void vavcore_free_frame(ref VavCoreVideoFrame frame); // VavCore Player 인스턴스 private IntPtr _vavCorePlayer = IntPtr.Zero; private string _currentVideoPath = string.Empty; private bool _isInitialized = false; // Godot 노드들 private TextureRect _videoTexture; private ShaderMaterial _yuvShaderMaterial; // GPU 텍스처들 private ImageTexture _yTexture; private ImageTexture _uTexture; private ImageTexture _vTexture; // 텍스처 캐싱 최적화 private int _cachedFrameWidth = 0; private int _cachedFrameHeight = 0; private bool _texturesInitialized = false; // Phase 1 최적화: Image 객체 재사용 private Image _cachedYUVImage; private byte[] _cachedYUVData; private ImageTexture _cachedYUVTexture; // Phase 1 최적화: 셰이더 파라미터 캐싱 private struct CachedShaderParams { public int width, height; public int y_size, u_size, v_size; public int y_offset, u_offset, v_offset; } private CachedShaderParams _lastShaderParams; // Phase 1 최적화: 프레임 카운터 private int _frameCounter = 0; // Phase 2 Memory Pool: Image와 Texture 재사용 시스템 private class MemoryPool { private readonly Queue _imagePool = new Queue(); private readonly Queue _texturePool = new Queue(); private readonly Queue _dataBufferPool = new Queue(); private readonly object _poolLock = new object(); // Pool 크기 제한 private const int MAX_POOL_SIZE = 10; // 통계 private int _imagePoolHits = 0; private int _imagePoolMisses = 0; private int _texturePoolHits = 0; private int _texturePoolMisses = 0; private int _bufferPoolHits = 0; private int _bufferPoolMisses = 0; public Image GetImage(int width, int height, Image.Format format) { lock (_poolLock) { if (_imagePool.Count > 0) { var image = _imagePool.Dequeue(); // 이미지 크기가 일치하는지 확인 if (image.GetWidth() == width && image.GetHeight() == height && image.GetFormat() == format) { _imagePoolHits++; return image; } else { // 크기가 다르면 새로 생성하고 기존 이미지는 버림 image?.Dispose(); } } _imagePoolMisses++; return Image.CreateEmpty(width, height, false, format); } } public void ReturnImage(Image image) { if (image == null) return; lock (_poolLock) { if (_imagePool.Count < MAX_POOL_SIZE) { _imagePool.Enqueue(image); } else { image.Dispose(); } } } public ImageTexture GetTexture() { lock (_poolLock) { if (_texturePool.Count > 0) { _texturePoolHits++; return _texturePool.Dequeue(); } _texturePoolMisses++; return new ImageTexture(); } } public void ReturnTexture(ImageTexture texture) { if (texture == null) return; lock (_poolLock) { if (_texturePool.Count < MAX_POOL_SIZE) { _texturePool.Enqueue(texture); } else { texture?.Dispose(); } } } public byte[] GetDataBuffer(int size) { lock (_poolLock) { if (_dataBufferPool.Count > 0) { var buffer = _dataBufferPool.Dequeue(); if (buffer.Length >= size) { _bufferPoolHits++; return buffer; } } _bufferPoolMisses++; return new byte[size]; } } public void ReturnDataBuffer(byte[] buffer) { if (buffer == null) return; lock (_poolLock) { if (_dataBufferPool.Count < MAX_POOL_SIZE) { _dataBufferPool.Enqueue(buffer); } } } public void PrintStatistics() { lock (_poolLock) { double imageHitRate = _imagePoolHits + _imagePoolMisses > 0 ? (double)_imagePoolHits / (_imagePoolHits + _imagePoolMisses) * 100 : 0; double textureHitRate = _texturePoolHits + _texturePoolMisses > 0 ? (double)_texturePoolHits / (_texturePoolHits + _texturePoolMisses) * 100 : 0; double bufferHitRate = _bufferPoolHits + _bufferPoolMisses > 0 ? (double)_bufferPoolHits / (_bufferPoolHits + _bufferPoolMisses) * 100 : 0; GD.Print($"Memory Pool Stats - Image: {imageHitRate:F1}% hit rate ({_imagePoolHits}/{_imagePoolHits + _imagePoolMisses})"); GD.Print($"Memory Pool Stats - Texture: {textureHitRate:F1}% hit rate ({_texturePoolHits}/{_texturePoolHits + _texturePoolMisses})"); GD.Print($"Memory Pool Stats - Buffer: {bufferHitRate:F1}% hit rate ({_bufferPoolHits}/{_bufferPoolHits + _bufferPoolMisses})"); } } public void Clear() { lock (_poolLock) { while (_imagePool.Count > 0) { _imagePool.Dequeue()?.Dispose(); } while (_texturePool.Count > 0) { _texturePool.Dequeue()?.Dispose(); } _dataBufferPool.Clear(); _imagePoolHits = _imagePoolMisses = 0; _texturePoolHits = _texturePoolMisses = 0; _bufferPoolHits = _bufferPoolMisses = 0; } } } private MemoryPool _memoryPool = new MemoryPool(); // Phase 2 고급 성능 모니터링 및 적응형 품질 조정 private class AdvancedPerformanceMonitor { private Queue _decodingTimes = new Queue(); private Queue _renderingTimes = new Queue(); private Queue _totalFrameTimes = new Queue(); private Queue _queueSizes = new Queue(); private const int SAMPLE_SIZE = 60; // 2초 (30fps * 2) private DateTime _lastFrameTime = DateTime.Now; private DateTime _lastDecodeTime = DateTime.Now; private DateTime _lastRenderTime = DateTime.Now; // 적응형 품질 조정 상태 private int _consecutiveSlowFrames = 0; private int _consecutiveFastFrames = 0; private bool _qualityReductionActive = false; private const int SLOW_FRAME_THRESHOLD = 5; private const int FAST_FRAME_THRESHOLD = 30; public void RecordDecodeTime() { var now = DateTime.Now; double decodeTime = (now - _lastDecodeTime).TotalMilliseconds; _lastDecodeTime = now; _decodingTimes.Enqueue(decodeTime); if (_decodingTimes.Count > SAMPLE_SIZE) { _decodingTimes.Dequeue(); } } public void RecordRenderTime() { var now = DateTime.Now; double renderTime = (now - _lastRenderTime).TotalMilliseconds; _lastRenderTime = now; _renderingTimes.Enqueue(renderTime); if (_renderingTimes.Count > SAMPLE_SIZE) { _renderingTimes.Dequeue(); } } public void RecordTotalFrameTime() { var now = DateTime.Now; double frameTime = (now - _lastFrameTime).TotalMilliseconds; _lastFrameTime = now; _totalFrameTimes.Enqueue(frameTime); if (_totalFrameTimes.Count > SAMPLE_SIZE) { _totalFrameTimes.Dequeue(); } // 적응형 품질 조정 로직 CheckForQualityAdjustment(frameTime); } public void RecordQueueSize(int queueSize) { _queueSizes.Enqueue(queueSize); if (_queueSizes.Count > SAMPLE_SIZE) { _queueSizes.Dequeue(); } } private void CheckForQualityAdjustment(double frameTime) { // const double TARGET_FRAME_TIME = 33.33; // 30fps target (참조용) const double SLOW_THRESHOLD = 40.0; // 25fps (너무 느림) const double FAST_THRESHOLD = 25.0; // 40fps (충분히 빠름) if (frameTime > SLOW_THRESHOLD) { _consecutiveSlowFrames++; _consecutiveFastFrames = 0; } else if (frameTime < FAST_THRESHOLD) { _consecutiveFastFrames++; _consecutiveSlowFrames = 0; } else { _consecutiveSlowFrames = 0; _consecutiveFastFrames = 0; } } public bool ShouldReduceQuality() { if (_consecutiveSlowFrames >= SLOW_FRAME_THRESHOLD && !_qualityReductionActive) { _qualityReductionActive = true; _consecutiveSlowFrames = 0; return true; } return false; } public bool ShouldRestoreQuality() { if (_consecutiveFastFrames >= FAST_FRAME_THRESHOLD && _qualityReductionActive) { _qualityReductionActive = false; _consecutiveFastFrames = 0; return true; } return false; } public bool ShouldSkipFrame() { if (_totalFrameTimes.Count >= 10) // 최소 10샘플 필요 { double avgTime = _totalFrameTimes.Skip(_totalFrameTimes.Count - 10).Average(); double avgQueueSize = _queueSizes.Count > 0 ? _queueSizes.Average() : 0; // 큐가 거의 비어있고 프레임 시간이 너무 오래 걸리는 경우 return avgTime > 45.0 && avgQueueSize < 2.0; // 22fps 이하이고 큐가 거의 빈 경우 } return false; } public PerformanceStats GetStats() { return new PerformanceStats { AverageDecodeTime = _decodingTimes.Count > 0 ? _decodingTimes.Average() : 0.0, AverageRenderTime = _renderingTimes.Count > 0 ? _renderingTimes.Average() : 0.0, AverageTotalTime = _totalFrameTimes.Count > 0 ? _totalFrameTimes.Average() : 0.0, AverageQueueSize = _queueSizes.Count > 0 ? _queueSizes.Average() : 0.0, CurrentFPS = _totalFrameTimes.Count > 0 ? 1000.0 / _totalFrameTimes.Average() : 0.0, QualityReductionActive = _qualityReductionActive, ConsecutiveSlowFrames = _consecutiveSlowFrames, ConsecutiveFastFrames = _consecutiveFastFrames }; } } public struct PerformanceStats { public double AverageDecodeTime; public double AverageRenderTime; public double AverageTotalTime; public double AverageQueueSize; public double CurrentFPS; public bool QualityReductionActive; public int ConsecutiveSlowFrames; public int ConsecutiveFastFrames; } private AdvancedPerformanceMonitor _performanceMonitor = new AdvancedPerformanceMonitor(); // 재생 상태 관리 private bool _isPlaying = false; private bool _isPaused = false; private Godot.Timer _playbackTimer; private double _targetFrameRate = 30.0; // 기본 30fps // Phase 2 멀티스레딩 디코딩 private Thread _decodingThread; private CancellationTokenSource _cancellationTokenSource; private ConcurrentQueue _frameQueue; private readonly object _frameQueueLock = new object(); private volatile bool _isDecodingActive = false; private const int MAX_QUEUED_FRAMES = 5; // 최대 버퍼링 프레임 수 private AutoResetEvent _frameAvailableEvent = new AutoResetEvent(false); // Phase 2: 멀티스레딩 모드 활성화 private bool _useMultithreading = true; public override void _Ready() { GD.Print("VavCorePlayer: Initializing..."); // Phase 2: 멀티스레딩 초기화 InitializeMultithreading(); // VavCore 라이브러리 초기화 InitializeVavCore(); // 비디오 출력용 TextureRect 생성 SetupVideoTexture(); } private void InitializeMultithreading() { _frameQueue = new ConcurrentQueue(); _cancellationTokenSource = new CancellationTokenSource(); GD.Print("VavCorePlayer: Phase 2 multithreading initialized"); } private void InitializeVavCore() { try { GD.Print("VavCorePlayer: Initializing VavCore library..."); // DLL 경로 확인 - addons 플러그인 경로 사용 string dllPath = System.IO.Path.Combine(System.Environment.CurrentDirectory, "addons/VavCoreGodot/bin/VavCore.dll"); GD.Print($"VavCorePlayer: Looking for DLL at: {dllPath}"); GD.Print($"VavCorePlayer: DLL exists: {System.IO.File.Exists(dllPath)}"); GD.Print("VavCorePlayer: Calling vavcore_initialize()..."); int initResult = vavcore_initialize(); GD.Print($"VavCorePlayer: vavcore_initialize() returned: {initResult}"); if (initResult == 0) // VAVCORE_SUCCESS = 0 { _isInitialized = true; GD.Print("VavCorePlayer: VavCore initialized successfully!"); // VavCore 플레이어 인스턴스 생성 GD.Print("VavCorePlayer: Creating VavCore player instance..."); _vavCorePlayer = vavcore_create_player(); GD.Print($"VavCorePlayer: vavcore_create_player() returned: {_vavCorePlayer}"); if (_vavCorePlayer != IntPtr.Zero) { GD.Print("VavCorePlayer: VavCore player created successfully!"); } else { GD.PrintErr("VavCorePlayer: Failed to create VavCore player!"); _isInitialized = false; } } else { GD.PrintErr("VavCorePlayer: Failed to initialize VavCore!"); } } catch (Exception ex) { GD.PrintErr($"VavCorePlayer: Exception during initialization: {ex.Message}"); GD.PrintErr($"VavCorePlayer: Exception type: {ex.GetType().Name}"); GD.PrintErr($"VavCorePlayer: Stack trace: {ex.StackTrace}"); } } private void SetupVideoTexture() { // 비디오 출력용 TextureRect 노드 생성 _videoTexture = new TextureRect(); _videoTexture.Name = "VideoTexture"; // Fill the entire parent control _videoTexture.SetAnchorsAndOffsetsPreset(Control.LayoutPreset.FullRect); _videoTexture.ExpandMode = TextureRect.ExpandModeEnum.FitWidthProportional; _videoTexture.StretchMode = TextureRect.StretchModeEnum.KeepAspectCentered; // 가시성 강제 설정 _videoTexture.Visible = true; _videoTexture.Modulate = Colors.White; // 완전 불투명 _videoTexture.ZIndex = 100; // 최상위에 표시 // 투명 배경으로 설정 _videoTexture.SelfModulate = Colors.White; // GPU 셰이더 설정 SetupGPUShader(); AddChild(_videoTexture); // 재생 타이머 설정 _playbackTimer = new Godot.Timer(); _playbackTimer.Name = "PlaybackTimer"; _playbackTimer.WaitTime = 1.0 / _targetFrameRate; // 30fps = ~0.033초 // Note: 타이머 콜백은 재생 모드에 따라 동적으로 연결됩니다 AddChild(_playbackTimer); // 디버그 정보 출력 GD.Print($"VavCorePlayer: Video texture setup complete"); GD.Print($"VavCorePlayer: TextureRect position: {_videoTexture.Position}"); GD.Print($"VavCorePlayer: TextureRect size: {_videoTexture.Size}"); GD.Print($"VavCorePlayer: TextureRect visible: {_videoTexture.Visible}"); GD.Print($"VavCorePlayer: GPU shader setup complete"); } public bool LoadVideo(string videoPath) { if (!_isInitialized || _vavCorePlayer == IntPtr.Zero) { GD.PrintErr("VavCorePlayer: Not initialized!"); return false; } try { GD.Print($"VavCorePlayer: Loading video: {videoPath}"); // Godot 리소스 경로를 실제 파일 경로로 변환 string realPath = ProjectSettings.GlobalizePath(videoPath); GD.Print($"VavCorePlayer: Real path: {realPath}"); int result = vavcore_open_file(_vavCorePlayer, realPath); if (result == 0) // VAVCORE_SUCCESS = 0 { _currentVideoPath = realPath; // 성공시 경로 저장 // 텍스처 캐시 초기화 _texturesInitialized = false; _cachedFrameWidth = 0; _cachedFrameHeight = 0; GD.Print("VavCorePlayer: Video loaded successfully!"); // 비디오가 로드되었는지 확인 bool isOpen = vavcore_is_open(_vavCorePlayer); GD.Print($"VavCorePlayer: Video is open: {isOpen}"); return true; } else { GD.PrintErr("VavCorePlayer: Failed to load video!"); return false; } } catch (Exception ex) { GD.PrintErr($"VavCorePlayer: Exception during video loading: {ex.Message}"); return false; } } public bool IsVideoLoaded() { if (!_isInitialized || _vavCorePlayer == IntPtr.Zero) return false; try { return vavcore_is_open(_vavCorePlayer); } catch (Exception ex) { GD.PrintErr($"VavCorePlayer: Exception checking video status: {ex.Message}"); return false; } } public override void _ExitTree() { GD.Print("VavCorePlayer: Cleaning up..."); // Phase 2: Cleanup multithreading resources first StopBackgroundDecoding(); _cancellationTokenSource?.Cancel(); _cancellationTokenSource?.Dispose(); _frameAvailableEvent?.Dispose(); // Phase 2: Memory Pool cleanup _memoryPool?.Clear(); GD.Print("VavCorePlayer: Memory Pool cleared"); // VavCore 리소스 정리 if (_vavCorePlayer != IntPtr.Zero) { vavcore_close_file(_vavCorePlayer); vavcore_destroy_player(_vavCorePlayer); _vavCorePlayer = IntPtr.Zero; } if (_isInitialized) { vavcore_cleanup(); _isInitialized = false; } GD.Print("VavCorePlayer: Cleanup complete"); } // 재생 제어 메서드들 public void StartPlayback() { if (!IsVideoLoaded()) { GD.PrintErr("VavCorePlayer: Cannot start playback - no video loaded"); return; } // Phase 2: Choose playback method based on multithreading setting if (_useMultithreading) { StartPlaybackMultithreaded(); } else { // 단일 스레드 모드: 상태 설정 및 콜백 연결 _isPlaying = true; _isPaused = false; _playbackTimer.Timeout += OnPlaybackTimerTimeout; _playbackTimer.Start(); } GD.Print($"VavCorePlayer: Playback started (Multithreading: {_useMultithreading})"); } public void PausePlayback() { _isPaused = true; // Phase 2: Handle both single-threaded and multithreaded modes if (_useMultithreading) { // In multithreaded mode, the background thread will check _isPaused GD.Print("VavCorePlayer: Pausing multithreaded playback"); } else { _playbackTimer.Stop(); } GD.Print("VavCorePlayer: Playback paused"); } public void StopPlayback() { _isPlaying = false; _isPaused = false; // Phase 2: Handle both single-threaded and multithreaded modes if (_useMultithreading) { StopPlaybackMultithreaded(); } else { _playbackTimer.Stop(); _playbackTimer.Timeout -= OnPlaybackTimerTimeout; } // 처음부터 다시 재생하기 위해 비디오를 다시 로드 if (_vavCorePlayer != IntPtr.Zero && !string.IsNullOrEmpty(_currentVideoPath)) { GD.Print("VavCorePlayer: Reloading video to reset position"); // 현재 파일 경로 저장 string currentPath = _currentVideoPath; // 비디오 다시 로드 (내부적으로 seek to beginning과 동일한 효과) int result = vavcore_open_file(_vavCorePlayer, currentPath); if (result == 0) { GD.Print("VavCorePlayer: Video reset to beginning successfully"); } else { GD.PrintErr($"VavCorePlayer: Failed to reset video position: {result}"); } } GD.Print("VavCorePlayer: Playback stopped"); } public bool IsPlaying() { return _isPlaying && !_isPaused; } // 타이머 콜백 - 매 프레임마다 호출됨 (Phase 1 최적화 적용) private void OnPlaybackTimerTimeout() { if (!_isPlaying || _isPaused || !IsVideoLoaded()) return; try { // Phase 2: 성능 모니터링 _performanceMonitor.RecordTotalFrameTime(); // Phase 2: 적응형 프레임 스킵 if (_performanceMonitor.ShouldSkipFrame()) { var stats = _performanceMonitor.GetStats(); GD.Print($"VavCorePlayer: Frame SKIPPED - FPS: {stats.CurrentFPS:F1}, Queue: {stats.AverageQueueSize:F1}"); return; } DecodeAndDisplayNextFrame(); // 성능 정보 주기적 출력 (60프레임마다) _frameCounter++; if (_frameCounter % 60 == 0) { var stats = _performanceMonitor.GetStats(); GD.Print($"VavCorePlayer: SINGLE-THREAD PERFORMANCE"); GD.Print($" FPS: {stats.CurrentFPS:F1} | Total: {stats.AverageTotalTime:F1}ms"); } } catch (Exception ex) { GD.PrintErr($"VavCorePlayer: Error during frame decode: {ex.Message}"); StopPlayback(); } } // 다음 프레임 디코딩 및 표시 private void DecodeAndDisplayNextFrame() { VavCoreVideoFrame frame = new VavCoreVideoFrame(); // 구조체 초기화 frame.surface_data = new ulong[16]; int result = vavcore_decode_next_frame(_vavCorePlayer, out frame); GD.Print($"VavCorePlayer: Decode result: {result}"); if (result == 0) // VAVCORE_SUCCESS { GD.Print($"VavCorePlayer: Decoded frame {frame.frame_number} ({frame.width}x{frame.height})"); GD.Print($"VavCorePlayer: Y-plane: {frame.y_plane}, U-plane: {frame.u_plane}, V-plane: {frame.v_plane}"); GD.Print($"VavCorePlayer: Y-stride: {frame.y_stride}, U-stride: {frame.u_stride}, V-stride: {frame.v_stride}"); // YUV 데이터가 유효한지 확인 if (frame.y_plane != IntPtr.Zero && frame.width > 0 && frame.height > 0) { // GPU에서 YUV 데이터를 직접 처리하여 표시 DisplayFrameGPU(frame); } else { GD.PrintErr("VavCorePlayer: Invalid frame data received"); } // 프레임 메모리 해제 vavcore_free_frame(ref frame); } else if (result == 1) // VAVCORE_END_OF_STREAM { GD.Print("VavCorePlayer: End of video reached"); StopPlayback(); } else { GD.PrintErr($"VavCorePlayer: Frame decode failed with error: {result}"); StopPlayback(); } } // 폴백 이미지 생성 (GPU 셰이더 사용 불가시) private Image CreateFallbackImage(VavCoreVideoFrame frame) { // GPU 셰이더가 사용 가능한 경우 null 반환 (셰이더가 처리) if (_yuvShaderMaterial != null) { return null; } // GPU 셰이더 사용 불가시 CPU 방식으로 폴백 GD.PrintErr("VavCorePlayer: GPU shader not available, using CPU fallback"); // 간단한 그레이스케일 변환 (성능 최적화) try { var image = Image.CreateEmpty(frame.width, frame.height, false, Image.Format.Rgb8); unsafe { byte* yPtr = (byte*)frame.y_plane.ToPointer(); for (int y = 0; y < frame.height; y++) { for (int x = 0; x < frame.width; x++) { int yIndex = y * frame.y_stride + x; byte yVal = yPtr[yIndex]; // Y 값만 사용하여 그레이스케일로 표시 (고속 처리) float gray = yVal / 255.0f; var color = new Color(gray, gray, gray, 1.0f); image.SetPixel(x, y, color); } } } return image; } catch (Exception ex) { GD.PrintErr($"VavCorePlayer: Fallback image creation error: {ex.Message}"); return null; } } // GPU 셰이더 설정 private void SetupGPUShader() { try { // YUV to RGB 셰이더 로드 var shader = GD.Load("res://shaders/yuv_to_rgb.gdshader"); if (shader == null) { GD.PrintErr("VavCorePlayer: Failed to load YUV shader"); return; } // 셰이더 머티리얼 생성 _yuvShaderMaterial = new ShaderMaterial(); _yuvShaderMaterial.Shader = shader; GD.Print("VavCorePlayer: GPU shader loaded successfully"); } catch (Exception ex) { GD.PrintErr($"VavCorePlayer: Error setting up GPU shader: {ex.Message}"); } } // GPU에서 YUV 프레임을 직접 처리하여 표시 private void DisplayFrameGPU(VavCoreVideoFrame frame) { try { // TextureRect 크기가 0이면 다시 설정 if (_videoTexture.Size.X <= 0 || _videoTexture.Size.Y <= 0) { GD.Print("VavCorePlayer: TextureRect size is 0, forcing resize..."); // 부모 컨테이너 크기 확인 var parentSize = GetParent().Size; GD.Print($"VavCorePlayer: Parent size: {parentSize}"); // 강제로 크기 설정 _videoTexture.Size = parentSize; _videoTexture.Position = Vector2.Zero; GD.Print($"VavCorePlayer: TextureRect resized to: {_videoTexture.Size}"); } // YUV 데이터를 GPU 텍스처로 변환 bool success = CreateYUVTextures(frame); if (success && _yuvShaderMaterial != null) { // 셰이더에 YUV 텍스처 할당 _yuvShaderMaterial.SetShaderParameter("y_texture", _yTexture); _yuvShaderMaterial.SetShaderParameter("u_texture", _uTexture); _yuvShaderMaterial.SetShaderParameter("v_texture", _vTexture); // 메쉬 기반 렌더링을 위한 더미 텍스처 생성 var dummyImage = Image.CreateEmpty(frame.width, frame.height, false, Image.Format.Rgb8); var dummyTexture = ImageTexture.CreateFromImage(dummyImage); // TextureRect에 셰이더 머티리얼 적용 _videoTexture.Texture = dummyTexture; _videoTexture.Material = _yuvShaderMaterial; // GPU frame displayed successfully } else { // GPU 처리 실패시 빨간색 에러 표시 var errorImage = Image.CreateEmpty(frame.width, frame.height, false, Image.Format.Rgb8); for (int y = 0; y < frame.height; y++) { for (int x = 0; x < frame.width; x++) { errorImage.SetPixel(x, y, Colors.Red); } } var errorTexture = ImageTexture.CreateFromImage(errorImage); _videoTexture.Texture = errorTexture; _videoTexture.Material = null; // 셰이더 비활성화 GD.PrintErr($"VavCorePlayer: GPU YUV processing failed, showing error image"); } } catch (Exception ex) { GD.PrintErr($"VavCorePlayer: Error displaying GPU frame: {ex.Message}"); } } // YUV 데이터로 GPU 텍스처 생성 private bool CreateYUVTextures(VavCoreVideoFrame frame) { if (frame.y_plane == IntPtr.Zero || frame.width <= 0 || frame.height <= 0) { GD.PrintErr("VavCorePlayer: Invalid frame data for GPU texture creation"); return false; } if (frame.u_plane == IntPtr.Zero || frame.v_plane == IntPtr.Zero) { GD.PrintErr("VavCorePlayer: Missing UV plane data for GPU texture"); return false; } // YUV420P 메모리 연속성 분석 long yAddr = (long)frame.y_plane; long uAddr = (long)frame.u_plane; long vAddr = (long)frame.v_plane; long ySize = frame.width * frame.height; long uSize = (frame.width / 2) * (frame.height / 2); long vSize = (frame.width / 2) * (frame.height / 2); GD.Print($"VavCorePlayer: Memory layout analysis:"); GD.Print($" Y plane: 0x{yAddr:X} (size: {ySize} bytes)"); GD.Print($" U plane: 0x{uAddr:X} (size: {uSize} bytes)"); GD.Print($" V plane: 0x{vAddr:X} (size: {vSize} bytes)"); GD.Print($" Y->U gap: {uAddr - (yAddr + ySize)} bytes"); GD.Print($" U->V gap: {vAddr - (uAddr + uSize)} bytes"); // 연속 메모리 공간인지 확인 bool isContiguous = (uAddr == yAddr + ySize) && (vAddr == uAddr + uSize); bool hasOptimalStrides = frame.y_stride == frame.width && frame.u_stride == (frame.width / 2) && frame.v_stride == (frame.width / 2); GD.Print($"VavCorePlayer: YUV planes contiguous: {isContiguous}"); GD.Print($"VavCorePlayer: Optimal strides: {hasOptimalStrides}"); if (isContiguous && hasOptimalStrides) { GD.Print("VavCorePlayer: Attempting single-block YUV420P copy!"); return CreateSingleBlockYUVTexture(frame); } try { // Phase 1 최적화: 텍스처 캐싱 최적화 (해상도가 바뀌었거나 첫 번째 프레임인 경우에만 재생성) bool needsResize = !_texturesInitialized || _cachedFrameWidth != frame.width || _cachedFrameHeight != frame.height; var startTime = DateTime.Now; if (needsResize) { _cachedFrameWidth = frame.width; _cachedFrameHeight = frame.height; _texturesInitialized = true; // Y 평면 텍스처 생성 var yImage = CreatePlaneImageBlockCopy(frame.y_plane, frame.width, frame.height, frame.y_stride); if (yImage != null) { _yTexture = ImageTexture.CreateFromImage(yImage); } else { GD.PrintErr("VavCorePlayer: Failed to create Y plane image"); return false; } // U 평면 텍스처 생성 int uvWidth = frame.width / 2; int uvHeight = frame.height / 2; var uImage = CreatePlaneImageBlockCopy(frame.u_plane, uvWidth, uvHeight, frame.u_stride); if (uImage != null) { _uTexture = ImageTexture.CreateFromImage(uImage); } else { GD.PrintErr("VavCorePlayer: Failed to create U plane image"); return false; } // V 평면 텍스처 생성 var vImage = CreatePlaneImageBlockCopy(frame.v_plane, uvWidth, uvHeight, frame.v_stride); if (vImage != null) { _vTexture = ImageTexture.CreateFromImage(vImage); } else { GD.PrintErr("VavCorePlayer: Failed to create V plane image"); return false; } } else { // Phase 1 최적화: 텍스처 업데이트만 수행 (훨씬 빠름!) var yImage = CreatePlaneImageBlockCopy(frame.y_plane, frame.width, frame.height, frame.y_stride); var uImage = CreatePlaneImageBlockCopy(frame.u_plane, frame.width / 2, frame.height / 2, frame.u_stride); var vImage = CreatePlaneImageBlockCopy(frame.v_plane, frame.width / 2, frame.height / 2, frame.v_stride); if (yImage != null && uImage != null && vImage != null) { _yTexture.Update(yImage); _uTexture.Update(uImage); _vTexture.Update(vImage); } else { GD.PrintErr("VavCorePlayer: Failed to update textures"); return false; } } // GPU YUV textures processed successfully return true; } catch (Exception ex) { GD.PrintErr($"VavCorePlayer: GPU texture creation error: {ex.Message}"); GD.PrintErr($"VavCorePlayer: Stack trace: {ex.StackTrace}"); return false; } } // 단일 평면 데이터로 이미지 생성 (최적화된 버전) private Image CreatePlaneImage(IntPtr planeData, int width, int height, int stride) { try { // R8 포맷 사용 (단일 체널, 8-bit) var image = Image.CreateEmpty(width, height, false, Image.Format.R8); GD.Print($"VavCorePlayer: Creating plane image - Size: {width}x{height}, Stride: {stride}, Format: R8"); unsafe { byte* srcPtr = (byte*)planeData.ToPointer(); // 직접 픽셀 데이터 복사 (고속 처리) for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { int srcIndex = y * stride + x; byte value = srcPtr[srcIndex]; // R8 포맷: 빨간 채널에만 값 설정 var color = new Color(value / 255.0f, 0.0f, 0.0f, 1.0f); image.SetPixel(x, y, color); } } } GD.Print($"VavCorePlayer: Plane image created successfully - Format: {image.GetFormat()}"); return image; } catch (Exception ex) { GD.PrintErr($"VavCorePlayer: Error creating plane image: {ex.Message}"); return null; } } // 블록 메모리 복사를 위한 최고속 평면 이미지 생성 private Image CreatePlaneImageBlockCopy(IntPtr planeData, int width, int height, int stride) { try { // 케이스 1: 스트라이드가 폭과 같은 경우 - 전체 블록 복사 if (stride == width) { int totalBytes = width * height; var imageData = new byte[totalBytes]; unsafe { byte* srcPtr = (byte*)planeData.ToPointer(); fixed (byte* dstPtr = imageData) { // 전체 메모리 블록을 한 번에 복사 (최고속) Buffer.MemoryCopy(srcPtr, dstPtr, totalBytes, totalBytes); } } var image = Image.CreateFromData(width, height, false, Image.Format.R8, imageData); return image; } // 케이스 2: 스트라이드가 폭보다 큰 경우 - 라인별 복사 (하지만 memcpy 사용) else { var imageData = new byte[width * height]; unsafe { byte* srcPtr = (byte*)planeData.ToPointer(); fixed (byte* dstPtr = imageData) { // 라인별 고속 메모리 복사 for (int y = 0; y < height; y++) { byte* srcLine = srcPtr + (y * stride); byte* dstLine = dstPtr + (y * width); // 한 라인을 memcpy로 고속 복사 Buffer.MemoryCopy(srcLine, dstLine, width, width); } } } var image = Image.CreateFromData(width, height, false, Image.Format.R8, imageData); GD.Print($"VavCorePlayer: Line memcpy completed - {height} lines of {width} bytes"); return image; } } catch (Exception ex) { GD.PrintErr($"VavCorePlayer: Error in block copy: {ex.Message}"); // 폴백: 셰이더 기반 스트라이드 처리 시도 return CreatePlaneImageWithShaderStride(planeData, width, height, stride); } } // 셰이더에서 스트라이드를 처리하는 방식 (실험적) private Image CreatePlaneImageWithShaderStride(IntPtr planeData, int width, int height, int stride) { try { GD.Print($"VavCorePlayer: Shader stride mode - Creating {stride}x{height} texture for {width}x{height} content"); // 스트라이드 전체 데이터를 텍스처로 업로드 int totalBytes = stride * height; var imageData = new byte[totalBytes]; unsafe { byte* srcPtr = (byte*)planeData.ToPointer(); fixed (byte* dstPtr = imageData) { // 전체 스트라이드 데이터를 블록 복사 Buffer.MemoryCopy(srcPtr, dstPtr, totalBytes, totalBytes); } } // 스트라이드 크기로 텍스처 생성 (셰이더에서 UV 좌표 조정 필요) var image = Image.CreateFromData(stride, height, false, Image.Format.R8, imageData); GD.Print($"VavCorePlayer: Shader stride texture created - {stride}x{height} (actual content: {width}x{height})"); return image; } catch (Exception ex) { GD.PrintErr($"VavCorePlayer: Error in shader stride mode: {ex.Message}"); // 최종 폴백: 기존 방식 return CreatePlaneImage(planeData, width, height, stride); } } private bool CreateSingleBlockYUVTexture(VavCoreVideoFrame frame) { try { var startTime = DateTime.Now; long ySize = frame.width * frame.height; long uSize = (frame.width / 2) * (frame.height / 2); long vSize = (frame.width / 2) * (frame.height / 2); long totalSize = ySize + uSize + vSize; // Phase 2 최적화: Memory Pool 사용 _cachedYUVData = _memoryPool.GetDataBuffer((int)totalSize); _cachedYUVImage = _memoryPool.GetImage((int)totalSize, 1, Image.Format.R8); // 메모리 복사 (단일 Buffer.MemoryCopy만 사용) unsafe { byte* srcPtr = (byte*)frame.y_plane.ToPointer(); fixed (byte* dstPtr = _cachedYUVData) { Buffer.MemoryCopy(srcPtr, dstPtr, totalSize, totalSize); } } // Image는 이미 _cachedYUVData를 참조하므로 별도의 SetData 불필요 // 텍스처 업데이트 최적화: Memory Pool 사용 if (_cachedYUVTexture == null) { _cachedYUVTexture = _memoryPool.GetTexture(); _cachedYUVTexture = ImageTexture.CreateFromImage(_cachedYUVImage); } else { _cachedYUVTexture.Update(_cachedYUVImage); } // Phase 1 최적화: 셰이더 파라미터 캐싱 (변경된 것만 업데이트) UpdateShaderParametersIfChanged(frame, ySize, uSize, vSize); // 텍스처는 매번 업데이트 (프레임 데이터이므로) _yuvShaderMaterial.SetShaderParameter("yuv_texture", _cachedYUVTexture); // Phase 2: Memory Pool 통계 출력 (60프레임마다) if (_frameCounter % 60 == 0) { _memoryPool.PrintStatistics(); } return true; } catch (Exception ex) { GD.PrintErr($"VavCorePlayer: Single-block copy failed: {ex.Message}"); GD.PrintErr($"VavCorePlayer: Falling back to 3-block method"); return false; } } // Phase 1 최적화: 셰이더 파라미터 캐싱 메서드 private void UpdateShaderParametersIfChanged(VavCoreVideoFrame frame, long ySize, long uSize, long vSize) { var currentParams = new CachedShaderParams { width = frame.width, height = frame.height, y_size = (int)ySize, u_size = (int)uSize, v_size = (int)vSize, y_offset = 0, u_offset = (int)ySize, v_offset = (int)(ySize + uSize) }; // 파라미터가 변경된 경우에만 업데이트 if (!currentParams.Equals(_lastShaderParams)) { _yuvShaderMaterial.SetShaderParameter("y_offset", currentParams.y_offset); _yuvShaderMaterial.SetShaderParameter("u_offset", currentParams.u_offset); _yuvShaderMaterial.SetShaderParameter("v_offset", currentParams.v_offset); _yuvShaderMaterial.SetShaderParameter("y_size", currentParams.y_size); _yuvShaderMaterial.SetShaderParameter("u_size", currentParams.u_size); _yuvShaderMaterial.SetShaderParameter("v_size", currentParams.v_size); _yuvShaderMaterial.SetShaderParameter("frame_width", currentParams.width); _yuvShaderMaterial.SetShaderParameter("frame_height", currentParams.height); _lastShaderParams = currentParams; } } // Phase 2 멀티스레딩 디코딩 메서드들 private void StartBackgroundDecoding() { if (_isDecodingActive || _vavCorePlayer == IntPtr.Zero) { GD.Print($"VavCorePlayer: Cannot start background decoding - Active: {_isDecodingActive}, Player: {_vavCorePlayer}"); return; } GD.Print("VavCorePlayer: Starting background decoding thread..."); _isDecodingActive = true; _decodingThread = new Thread(BackgroundDecodingLoop) { Name = "VavCore-Decoder", IsBackground = true }; _decodingThread.Start(); GD.Print("VavCorePlayer: Background decoding thread started"); } private void StopBackgroundDecoding() { if (!_isDecodingActive) return; _isDecodingActive = false; _cancellationTokenSource?.Cancel(); _frameAvailableEvent.Set(); // Wake up any waiting threads if (_decodingThread != null && _decodingThread.IsAlive) { if (!_decodingThread.Join(1000)) // Wait 1 second { GD.PrintErr("VavCorePlayer: Decoding thread did not stop gracefully"); } } // Clear frame queue while (_frameQueue.TryDequeue(out _)) { } GD.Print("VavCorePlayer: Background decoding thread stopped"); } private void BackgroundDecodingLoop() { var token = _cancellationTokenSource.Token; GD.Print("VavCorePlayer: Background decoding loop STARTED"); try { while (_isDecodingActive && !token.IsCancellationRequested) { // Check if queue is full if (_frameQueue.Count >= MAX_QUEUED_FRAMES) { Thread.Sleep(1); // Brief pause if queue is full continue; } // Phase 2: Record decode time start _performanceMonitor.RecordDecodeTime(); // Decode next frame var frame = new VavCoreVideoFrame(); frame.surface_data = new ulong[16]; int result = vavcore_decode_next_frame(_vavCorePlayer, out frame); if (result == 0) // Success { _frameQueue.Enqueue(frame); _frameAvailableEvent.Set(); // Signal main thread that frame is available // Phase 2: Record queue size for performance monitoring _performanceMonitor.RecordQueueSize(_frameQueue.Count); } else if (result == 1) // End of stream { GD.Print("VavCorePlayer: Background decoder reached end of stream"); break; } else // Error { GD.PrintErr($"VavCorePlayer: Background decode error: {result}"); Thread.Sleep(5); // Brief pause on error } } } catch (Exception ex) { GD.PrintErr($"VavCorePlayer: Background decoding thread error: {ex.Message}"); } finally { _isDecodingActive = false; GD.Print("VavCorePlayer: Background decoding loop ended"); } } // Modified playback methods to use multithreading public void StartPlaybackMultithreaded() { if (_vavCorePlayer == IntPtr.Zero || !vavcore_is_open(_vavCorePlayer)) { GD.PrintErr("VavCorePlayer: Cannot start playback - no video loaded"); return; } _isPlaying = true; _isPaused = false; // Start background decoding thread StartBackgroundDecoding(); // Start main thread timer for rendering (connect to the right callback) _playbackTimer.Timeout += OnMultithreadedPlaybackTimer; _playbackTimer.Start(); GD.Print("VavCorePlayer: Multithreaded playback started"); } public void StopPlaybackMultithreaded() { if (!_isPlaying) return; _isPlaying = false; _playbackTimer.Stop(); // Stop background decoding StopBackgroundDecoding(); // Disconnect the timer callback _playbackTimer.Timeout -= OnMultithreadedPlaybackTimer; GD.Print("VavCorePlayer: Multithreaded playback stopped"); } // Phase 2: Advanced multithreaded timer callback with performance monitoring private void OnMultithreadedPlaybackTimer() { if (!_isPlaying || _isPaused) return; // Phase 2: Record total frame time _performanceMonitor.RecordTotalFrameTime(); // Phase 2: Check for adaptive quality adjustments var stats = _performanceMonitor.GetStats(); if (_performanceMonitor.ShouldReduceQuality()) { GD.Print($"VavCorePlayer: QUALITY REDUCTION triggered - FPS: {stats.CurrentFPS:F1}, Queue: {stats.AverageQueueSize:F1}"); } else if (_performanceMonitor.ShouldRestoreQuality()) { GD.Print($"VavCorePlayer: QUALITY RESTORATION triggered - FPS: {stats.CurrentFPS:F1}, Queue: {stats.AverageQueueSize:F1}"); } // Phase 2: Check for frame skipping if (_performanceMonitor.ShouldSkipFrame()) { GD.Print($"VavCorePlayer: FRAME SKIP triggered - FPS: {stats.CurrentFPS:F1}, Queue: {stats.AverageQueueSize:F1}"); return; } // Try to get a frame from the queue if (_frameQueue.TryDequeue(out VavCoreVideoFrame frame)) { // Phase 2: Record render time start _performanceMonitor.RecordRenderTime(); // Render the frame (same as before) DisplayFrameGPU(frame); // Free the frame after displaying vavcore_free_frame(ref frame); // Phase 2: Print detailed stats every 60 frames (2 seconds) _frameCounter++; if (_frameCounter % 60 == 0) { var currentStats = _performanceMonitor.GetStats(); GD.Print($"VavCorePlayer: PERFORMANCE STATS"); GD.Print($" FPS: {currentStats.CurrentFPS:F1} | Decode: {currentStats.AverageDecodeTime:F1}ms | Render: {currentStats.AverageRenderTime:F1}ms"); GD.Print($" Total: {currentStats.AverageTotalTime:F1}ms | Queue: {currentStats.AverageQueueSize:F1} | Quality Reduction: {currentStats.QualityReductionActive}"); } } else { // No frame available - could indicate decoding is falling behind if (_frameCounter % 30 == 0) // Only log every 30 callbacks to reduce spam { GD.Print("VavCorePlayer: No frame available in queue"); } } } // Phase 2 UI 통합: 공개 메서드들 public void SetMultithreadingMode(bool enabled) { // Only allow changing when not playing if (_isPlaying) { GD.PrintErr("VavCorePlayer: Cannot change multithreading mode while playing"); return; } _useMultithreading = enabled; GD.Print($"VavCorePlayer: Multithreading mode set to: {enabled}"); } public bool IsMultithreadingEnabled() { return _useMultithreading; } }