using System; using System.Threading; using System.Threading.Tasks; using Vav1Player.Container; using Vav1Player.Decoder; namespace Vav1Player.Video { /// /// Video decoder pipeline that continuously decodes frames from file reader to frame buffer /// public class VideoDecoderPipeline : IDisposable { private readonly VideoFileReader _fileReader; private readonly Dav1dDecoder _decoder; private readonly FrameBuffer _frameBuffer; private readonly CancellationTokenSource _cancellationTokenSource; private readonly Task _decodingTask; private volatile bool _disposed = false; private volatile bool _isPaused = false; private int _frameCounter = 0; public bool IsRunning => !_decodingTask.IsCompleted && !_disposed; public bool IsPaused => _isPaused; public VideoFileReader FileReader => _fileReader; public FrameBuffer FrameBuffer => _frameBuffer; public int DecodedFrameCount => _frameCounter; public VideoDecoderPipeline(VideoFileReader fileReader, Dav1dDecoder decoder, FrameBuffer frameBuffer) { _fileReader = fileReader ?? throw new ArgumentNullException(nameof(fileReader)); _decoder = decoder ?? throw new ArgumentNullException(nameof(decoder)); _frameBuffer = frameBuffer ?? throw new ArgumentNullException(nameof(frameBuffer)); // Initialize decoder with av1C configuration if available InitializeDecoderWithConfig(); _cancellationTokenSource = new CancellationTokenSource(); _decodingTask = Task.Run(DecodingLoop, _cancellationTokenSource.Token); System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Started decoding pipeline"); } private void InitializeDecoderWithConfig() { var trackInfo = _fileReader.TrackInfo; if (trackInfo?.Av1ConfigurationRecord != null) { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] AV1 configuration available: {trackInfo.Av1ConfigurationRecord.Length} bytes"); // Determine the configuration format based on file extension var extension = System.IO.Path.GetExtension(_fileReader.FilePath).ToLowerInvariant(); byte[]? sequenceOBUs = null; if (extension == ".mp4") { // MP4 uses av1C format (ISO BMFF) sequenceOBUs = ParseAv1ConfigurationRecord(trackInfo.Av1ConfigurationRecord); System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Parsed MP4 av1C configuration"); } else if (extension == ".webm" || extension == ".mkv") { // WebM/MKV uses raw AV1 OBUs in CodecPrivate sequenceOBUs = ParseMatroskaCodecPrivate(trackInfo.Av1ConfigurationRecord); System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Parsed WebM CodecPrivate configuration"); } if (sequenceOBUs != null && sequenceOBUs.Length > 0) { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Extracted sequence OBUs: {sequenceOBUs.Length} bytes"); // Log first few bytes for debugging var hexData = string.Join(" ", sequenceOBUs.Take(Math.Min(16, sequenceOBUs.Length)).Select(b => b.ToString("X2"))); System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Sequence OBU data: {hexData}"); // Send sequence header to decoder if (_decoder.DecodeFrame(sequenceOBUs, out var _)) { System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Successfully initialized decoder with sequence header"); } else { System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Failed to initialize decoder with sequence header"); } } else { System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] No sequence header found in configuration - relying on stream data"); } } else { System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] No AV1 configuration available - using first frame for initialization"); } } private byte[]? ParseMatroskaCodecPrivate(byte[] codecPrivate) { try { if (codecPrivate == null || codecPrivate.Length == 0) return null; System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Parsing WebM CodecPrivate: {codecPrivate.Length} bytes"); // Log raw CodecPrivate data for debugging var hexData = string.Join(" ", codecPrivate.Take(Math.Min(32, codecPrivate.Length)).Select(b => b.ToString("X2"))); System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] CodecPrivate data: {hexData}"); // According to Matroska AV1 spec: // CodecPrivate consists of 4 octets similar to ISOBMFF AV1CodecConfigurationRecord // However, some encoders include sequence header OBUs after the 4-byte config if (codecPrivate.Length == 4) { // Standard compliant: only 4-byte configuration System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Standard 4-byte CodecPrivate (no sequence header included)"); ParseAv1ConfigBytes(codecPrivate); return null; // No sequence header in CodecPrivate, will come from first keyframe } else if (codecPrivate.Length > 4) { // Non-standard but common: includes sequence header after 4-byte config System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Extended CodecPrivate with potential sequence header"); // Parse the 4-byte configuration first var configBytes = new byte[4]; Array.Copy(codecPrivate, 0, configBytes, 0, 4); ParseAv1ConfigBytes(configBytes); // Check if remaining data contains sequence header OBU if (codecPrivate.Length > 4) { // Use proper OBU parser to validate and extract sequence header var obuInfo = ParseObuHeader(codecPrivate, 4); if (obuInfo != null && obuInfo.Value.obuType == 1) // Sequence Header OBU { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Found sequence header OBU at offset 4"); System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Original OBU size: {obuInfo.Value.obuSize}, has size field: {obuInfo.Value.hasSizeField}"); // The original OBU might have incorrect size field, reconstruct it properly byte obuHeaderByte = codecPrivate[4]; // 0x0A (type 1, has_size_field=1) // Calculate the actual payload size from CodecPrivate int actualPayloadSize = codecPrivate.Length - 4 - 1 - 1; // total - offset - header - size_field System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Reconstructing OBU with correct size field: {actualPayloadSize} bytes"); // Reconstruct the OBU with correct size var sequenceOBU = new List(); sequenceOBU.Add(obuHeaderByte); // OBU header (0x0A) sequenceOBU.Add((byte)actualPayloadSize); // Corrected size field (simple case, < 128) // Add the payload (skip original header and size field) for (int i = 6; i < codecPrivate.Length; i++) { sequenceOBU.Add(codecPrivate[i]); } var reconstructedOBU = sequenceOBU.ToArray(); var sequenceHex = string.Join(" ", reconstructedOBU.Take(Math.Min(16, reconstructedOBU.Length)).Select(b => b.ToString("X2"))); System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Reconstructed sequence OBU: {sequenceHex} (length: {reconstructedOBU.Length})"); return reconstructedOBU; } else { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Invalid OBU at offset 4: type={obuInfo?.obuType ?? -1}"); } } } // Fallback: non-standard format, search for sequence header for (int i = 0; i < codecPrivate.Length; i++) { var obuInfo = ParseObuHeader(codecPrivate, i); if (obuInfo != null && obuInfo.Value.obuType == 1) // Sequence Header OBU { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Found valid sequence header OBU at offset {i} (fallback search)"); System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] OBU size: {obuInfo.Value.obuSize}, has size field: {obuInfo.Value.hasSizeField}"); // Extract the complete OBU including header and size field int obuLength = obuInfo.Value.nextPosition - i; var sequenceOBU = new byte[obuLength]; Array.Copy(codecPrivate, i, sequenceOBU, 0, obuLength); var sequenceHex = string.Join(" ", sequenceOBU.Take(Math.Min(16, sequenceOBU.Length)).Select(b => b.ToString("X2"))); System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Extracted sequence OBU: {sequenceHex} (length: {obuLength})"); return sequenceOBU; } } System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] No sequence header found in CodecPrivate"); return null; } catch (Exception ex) { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Error parsing WebM CodecPrivate: {ex.Message}"); return null; } } private void ParseAv1ConfigBytes(byte[] configBytes) { if (configBytes.Length != 4) return; // Parse 4-byte AV1 configuration according to spec byte byte0 = configBytes[0]; byte byte1 = configBytes[1]; byte byte2 = configBytes[2]; byte byte3 = configBytes[3]; bool marker = (byte0 & 0x80) != 0; int version = byte0 & 0x7F; int seqProfile = (byte1 >> 5) & 0x07; int seqLevelIdx = byte1 & 0x1F; System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] AV1 Config - Marker: {marker}, Version: {version}, Profile: {seqProfile}, Level: {seqLevelIdx}"); } private void ValidateAv1Keyframe(byte[] frameData, long sampleIndex) { if (frameData == null || frameData.Length == 0) return; System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Validating AV1 keyframe at sample {sampleIndex}"); bool hasSequenceHeader = false; bool hasFrameHeader = false; bool hasFrameOBU = false; int obuCount = 0; // Parse OBUs in the keyframe using proper AV1 spec parsing int position = 0; while (position < frameData.Length && obuCount < 10) { var obuInfo = ParseObuHeader(frameData, position); if (obuInfo == null) break; obuCount++; string obuTypeName = GetOBUTypeName(obuInfo.Value.obuType); if (obuCount <= 5) // Log first few OBUs { System.Diagnostics.Debug.WriteLine($" OBU {obuCount}: Type {obuInfo.Value.obuType} ({obuTypeName}), HasSize: {obuInfo.Value.hasSizeField}, Size: {obuInfo.Value.obuSize}"); } // Track important OBU types for spec validation if (obuInfo.Value.obuType == 1) hasSequenceHeader = true; if (obuInfo.Value.obuType == 3) hasFrameHeader = true; if (obuInfo.Value.obuType == 6) hasFrameOBU = true; // Advance to next OBU position = obuInfo.Value.nextPosition; } // Validate Matroska AV1 spec requirements for keyframes if (!hasSequenceHeader) { System.Diagnostics.Debug.WriteLine($" ⚠️ SPEC VIOLATION: Keyframe at sample {sampleIndex} missing Sequence Header OBU"); } // Either Frame Header (Type 3) OR Frame OBU (Type 6) is required, not both if (!hasFrameHeader && !hasFrameOBU) { System.Diagnostics.Debug.WriteLine($" ⚠️ SPEC VIOLATION: Keyframe at sample {sampleIndex} missing Frame Header (Type 3) or Frame OBU (Type 6)"); } if (hasSequenceHeader && (hasFrameHeader || hasFrameOBU)) { System.Diagnostics.Debug.WriteLine($" ✅ Keyframe at sample {sampleIndex} appears spec compliant"); } } private (int obuType, bool hasSizeField, uint obuSize, int nextPosition)? ParseObuHeader(byte[] data, int position) { if (position >= data.Length) return null; // Parse OBU header according to AV1 spec byte obuHeader = data[position]; // Validate forbidden bit (must be 0) if ((obuHeader & 0x80) != 0) { System.Diagnostics.Debug.WriteLine($" ⚠️ SPEC VIOLATION: OBU forbidden bit is 1 at position {position}"); return null; } int obuType = (obuHeader >> 3) & 0x0F; bool extensionFlag = (obuHeader & 0x04) != 0; bool hasSizeField = (obuHeader & 0x02) != 0; position++; // Move past OBU header // Skip extension header if present if (extensionFlag && position < data.Length) { position++; // Skip extension byte } uint obuSize = 0; if (hasSizeField) { // Parse LEB128 size field according to AV1 spec var leb128Result = ParseLeb128(data, position); if (leb128Result == null) return null; obuSize = leb128Result.Value.value; position = leb128Result.Value.nextPosition; } else { // If no size field, OBU extends to end of current data obuSize = (uint)(data.Length - position); } // Low Overhead Bitstream Format requires has_size_field = 1 if (!hasSizeField) { System.Diagnostics.Debug.WriteLine($" ⚠️ SPEC VIOLATION: Low Overhead Format requires obu_has_size_field=1, but found 0"); } return (obuType, hasSizeField, obuSize, position + (int)obuSize); } private (uint value, int nextPosition)? ParseLeb128(byte[] data, int position) { uint value = 0; int shift = 0; while (position < data.Length && shift < 35) // Max 5 bytes for uint32 { byte b = data[position++]; value |= (uint)(b & 0x7F) << shift; if ((b & 0x80) == 0) // No continuation bit { return (value, position); } shift += 7; } System.Diagnostics.Debug.WriteLine($" ⚠️ Invalid LEB128 encoding at position {position}"); return null; } private List SplitOBUsFromData(byte[] data) { var obuList = new List(); int position = 0; while (position < data.Length) { var obuInfo = ParseObuHeader(data, position); if (obuInfo == null) break; int obuStart = position; int obuEnd = obuInfo.Value.nextPosition; // Extract this OBU data including header and payload int obuLength = obuEnd - obuStart; if (obuLength > 0 && obuEnd <= data.Length) { byte[] obuData = new byte[obuLength]; Array.Copy(data, obuStart, obuData, 0, obuLength); obuList.Add(obuData); System.Diagnostics.Debug.WriteLine($"[SplitOBUs] Extracted OBU Type {obuInfo.Value.obuType}: {obuLength} bytes"); } position = obuEnd; } System.Diagnostics.Debug.WriteLine($"[SplitOBUs] Split data into {obuList.Count} OBUs"); return obuList; } private string GetOBUTypeName(int obuType) { return obuType switch { 0 => "Reserved", 1 => "Sequence Header", 2 => "Temporal Delimiter", 3 => "Frame Header", 4 => "Tile Group", 5 => "Metadata", 6 => "Frame", 7 => "Redundant Frame Header", 8 => "Tile List", 15 => "Padding", _ => $"Unknown({obuType})" }; } private byte[]? ParseAv1ConfigurationRecord(byte[] av1C) { try { if (av1C.Length < 4) return null; // av1C format according to AV1 codec ISO BMFF specification: // - 1 byte: marker + version (bit 7 = marker, bits 6-0 = version) // - 1 byte: seq_profile (3 bits) + seq_level_idx_0 (5 bits) // - 1 byte: various flags // - 1 byte: chroma/color info // - N bytes: configOBUs (sequence header and metadata OBUs) byte marker_version = av1C[0]; if ((marker_version & 0x80) == 0) { System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Invalid av1C: missing marker bit"); return null; } // Skip the 4-byte configuration header to get to configOBUs if (av1C.Length <= 4) { System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] av1C contains no configOBUs"); return null; } var configOBUs = new byte[av1C.Length - 4]; Array.Copy(av1C, 4, configOBUs, 0, configOBUs.Length); System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Extracted configOBUs: {configOBUs.Length} bytes"); // Log first few bytes for debugging var hexData = string.Join(" ", configOBUs.Take(Math.Min(16, configOBUs.Length)).Select(b => b.ToString("X2"))); System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] ConfigOBUs data: {hexData}"); return configOBUs; } catch (Exception ex) { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Error parsing av1C: {ex.Message}"); return null; } } /// /// Pause the decoding pipeline /// public void Pause() { _isPaused = true; System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Paused"); } /// /// Resume the decoding pipeline /// public void Resume() { _isPaused = false; System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Resumed"); } /// /// Seek to a specific time position /// public async Task SeekAsync(TimeSpan time) { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Seeking to {time}"); // Clear the frame buffer _frameBuffer.Clear(); // Seek in the file reader var success = await _fileReader.SeekToTimeAsync(time, _cancellationTokenSource.Token); if (success) { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Seek successful, resumed at sample {_fileReader.CurrentSampleIndex}"); } else { System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Seek failed"); } return success; } /// /// Main decoding loop that runs continuously /// private async Task DecodingLoop() { try { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Starting decoding loop for {_fileReader.TotalSamples} samples"); while (!_cancellationTokenSource.Token.IsCancellationRequested) { // Pause handling while (_isPaused && !_cancellationTokenSource.Token.IsCancellationRequested) { await Task.Delay(10, _cancellationTokenSource.Token); } // Check if we have space in the buffer var bufferStats = _frameBuffer.GetStats(); if (bufferStats.FrameCount >= bufferStats.MaxFrameCount * 0.9) // 90% full { // Buffer is nearly full, wait a bit await Task.Delay(10, _cancellationTokenSource.Token); continue; } // Read next chunk from file var chunk = await _fileReader.ReadNextChunkAsync(_cancellationTokenSource.Token); if (chunk == null) { // End of file reached _frameBuffer.MarkEndOfStream(); System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] End of stream reached"); break; } System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Processing {chunk}"); // Process the chunk based on container format await ProcessVideoChunk(chunk); } } catch (OperationCanceledException) { System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Decoding loop cancelled"); } catch (Exception ex) { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Decoding loop error: {ex.Message}"); } finally { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Decoding loop finished. Decoded {_frameCounter} frames"); } } /// /// Process a video chunk based on container format /// private async Task ProcessVideoChunk(VideoDataChunk chunk) { try { byte[] decodingData; // Handle different container formats var extension = System.IO.Path.GetExtension(_fileReader.FilePath).ToLowerInvariant(); if (extension == ".mp4") { // MP4: Parse AV1 sample to extract OBUs var obuList = Av1BitstreamParser.ParseMp4Sample(chunk.Data); if (obuList.Count == 0) { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] No OBUs found in MP4 sample"); return; } // Log OBU types for first few chunks to understand structure if (chunk.SampleIndex < 3) // Only log first 3 chunks to avoid spam { for (int i = 0; i < obuList.Count; i++) { Av1BitstreamParser.LogOBUInfo(obuList[i], $"[VideoDecoderPipeline] Chunk {chunk.SampleIndex} OBU {i}: "); } } // Combine all OBUs for decoding decodingData = Av1BitstreamParser.CombineOBUs(obuList); System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] MP4: Extracted {obuList.Count} OBUs, combined size: {decodingData.Length}"); } else { // WebM/MKV: Use data directly as AV1 OBUs in "Low Overhead Bitstream Format" System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Matroska: Using data directly, size: {chunk.Data.Length}"); // For WebM keyframes with multiple OBUs, handle them separately if (chunk.IsKeyFrame && chunk.Data.Length > 1000) // Large keyframes likely have multiple OBUs { ValidateAv1Keyframe(chunk.Data, chunk.SampleIndex); // Split OBUs and process separately for dav1d var obuList = SplitOBUsFromData(chunk.Data); bool decodingSuccess = false; foreach (var obuData in obuList) { if (obuData.Length > 0) { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Processing individual OBU: {obuData.Length} bytes"); if (_decoder.DecodeFrame(obuData, out var frameResult)) { if (frameResult.HasValue) { var frame = frameResult.Value; System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Decoded frame #{_frameCounter}: {frame.Width}x{frame.Height}"); // Create video frame with timing information var videoFrame = new VideoFrame(frame, chunk.PresentationTimeMs, _frameCounter, chunk.IsKeyFrame); // Add to frame buffer var enqueued = await _frameBuffer.TryEnqueueAsync(videoFrame, _cancellationTokenSource.Token); if (enqueued) { _frameCounter++; decodingSuccess = true; // Log buffer status periodically if (_frameCounter % 10 == 0) { var stats = _frameBuffer.GetStats(); System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Buffer: {stats}"); } } else { // Buffer is full, dispose the frame videoFrame.Dispose(); System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Buffer full, dropped frame"); } break; // Successfully decoded, stop processing more OBUs } } } } if (!decodingSuccess) { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Failed to decode keyframe with split OBUs"); } return; // Skip regular decoding for keyframes } else { decodingData = chunk.Data; } } // Decode the frame if (_decoder.DecodeFrame(decodingData, out var decodedFrame)) { if (decodedFrame.HasValue) { var frame = decodedFrame.Value; System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Decoded frame #{_frameCounter}: {frame.Width}x{frame.Height}"); // Create video frame with timing information var videoFrame = new VideoFrame(frame, chunk.PresentationTimeMs, _frameCounter, chunk.IsKeyFrame); // Add to frame buffer var enqueued = await _frameBuffer.TryEnqueueAsync(videoFrame, _cancellationTokenSource.Token); if (enqueued) { _frameCounter++; // Log buffer status periodically if (_frameCounter % 10 == 0) { var stats = _frameBuffer.GetStats(); System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Buffer: {stats}"); } } else { // Buffer is full, dispose the frame videoFrame.Dispose(); System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Buffer full, dropped frame"); } } } else { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Failed to decode chunk of size {decodingData.Length}"); } } catch (Exception ex) { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Error processing chunk: {ex.Message}"); } } public void Dispose() { if (_disposed) return; System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Disposing"); _disposed = true; _cancellationTokenSource.Cancel(); try { _decodingTask.Wait(TimeSpan.FromSeconds(5)); } catch (Exception ex) { System.Diagnostics.Debug.WriteLine($"[VideoDecoderPipeline] Error waiting for decoding task: {ex.Message}"); } _cancellationTokenSource.Dispose(); System.Diagnostics.Debug.WriteLine("[VideoDecoderPipeline] Disposed"); } } }