diff --git a/.github/workflows/release-ci.yml b/.github/workflows/release-ci.yml index 400994f..4d9d9e5 100644 --- a/.github/workflows/release-ci.yml +++ b/.github/workflows/release-ci.yml @@ -270,7 +270,7 @@ jobs: make -C toolchain/v echo "$GITHUB_WORKSPACE/toolchain/v" >> "$GITHUB_PATH" - name: Install native build dependencies - run: sudo apt-get update && sudo apt-get install -y build-essential cmake libglfw3-dev libvulkan-dev libvulkan-volk-dev pkg-config + run: sudo apt-get update && sudo apt-get install -y build-essential cmake libglfw3-dev libvulkan-dev libvulkan-volk-dev pkg-config mesa-vulkan-drivers xvfb - name: Install headers matching Vulkan master run: | set -euo pipefail @@ -295,6 +295,9 @@ jobs: cd "$GITHUB_WORKSPACE/source/v_vulkan_video" v -new-compiler -cc tcc test . v -new-compiler -cc tcc -o "$RUNNER_TEMP/v_vulkan_video" . + timeout 20s xvfb-run -a "$RUNNER_TEMP/v_vulkan_video" --list-gpus res/20240917_095400.mp4 > "$RUNNER_TEMP/v3-gpus.log" 2>&1 + cat "$RUNNER_TEMP/v3-gpus.log" + grep -F 'Vulkan devices for H.264' "$RUNNER_TEMP/v3-gpus.log" windows: name: Windows / ${{ matrix.compiler }} @@ -419,6 +422,10 @@ jobs: if ($LASTEXITCODE -ne 0 -or $HelpText -notmatch 'Usage:') { throw 'The Windows player did not print its command-line help.' } + $GpuText = (& .\dist\vkvideo-windows-x64\v_vulkan_video.exe --list-gpus .\dist\vkvideo-windows-x64\res\20240917_095400.mp4 | Out-String) + if ($LASTEXITCODE -ne 0 -or $GpuText -notmatch 'Vulkan devices for H.264') { + throw 'The Windows player could not enumerate Vulkan devices.' + } $Flags = @('-cc', 'msvc', '-cflags', '/DWIN32_LEAN_AND_MEAN') if ($Compiler -eq 'v3') { $Flags += '-new-compiler' } v @Flags test . diff --git a/BUILDING.md b/BUILDING.md index 4ca8007..b2206e9 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -62,10 +62,13 @@ a platform-independent archive. ./v_vulkan_video [--list-gpus] [--gpu INDEX] [--decode-output-mode MODE] [video.mp4] ``` -GPU compatibility is evaluated against the input stream's actual H.264 -profile. Without `--gpu`, the first fully compatible presentation/decode device -is selected. Invalid media and unsupported or out-of-range devices return a -clean non-zero exit status with a diagnostic instead of a panic. +GPU compatibility is evaluated against the input stream's H.264 profile, +coded extent, DPB slots, active references, output mode, and image formats. +Without `--gpu`, the first fully compatible presentation/decode device +is selected. Unsupported codecs and profiles, missing slice parameter-set +references, truncated MP4 samples, and incompatible or out-of-range devices +return a non-zero exit status with a diagnostic. Arbitrarily corrupted SPS/PPS +bitstreams are not fully validated by the pinned H.264 parser. `--decode-output-mode auto|coincident|distinct` selects an advertised DPB and output-image mode; `auto` is the default. diff --git a/PLATFORM_SUPPORT.md b/PLATFORM_SUPPORT.md index 449291c..58f844c 100644 --- a/PLATFORM_SUPPORT.md +++ b/PLATFORM_SUPPORT.md @@ -12,9 +12,15 @@ A normal Vulkan graphics driver does not necessarily provide Vulkan Video. | macOS | Unsupported for video decode | The UI bindings can be built for macOS, but this application requires Vulkan Video H.264 decode. Do not treat a MoltenVK graphics-capable system as proof of Vulkan Video support. | Linux V3 compilation is tracked by CI as an experimental, non-release-gating -job. With V master at `efae23e85b`, the complete player builds with TinyCC and its -three software-only test files pass. Playback with V3 on supported hardware -has not yet been validated, so stable V remains the release compiler. +job. With V master at `b99970bd438a7bdcdfbe38f74d9364db801d5439`, the complete +player builds with TinyCC and its three software-only test files pass. On the +Linux GTX 1060, V3/TinyCC decoded the ID-7 fixture (5 frames) and multislice +fixture (24 frames) byte-for-byte identically to FFmpeg and exited cleanly on +Escape. The player uses the binding's loader initialization and passes the +swapchain semaphore by value. Raw Volk initialization can collide with Linux +TinyCC's exported dispatch variables; a mutable handle parameter was lowered +to its address by this compiler. Stable V remains the release compiler while +broader V3 hardware and platform coverage is completed. On Windows, source-built V master at `b99970bd438a7bdcdfbe38f74d9364db801d5439` passed the V3/MSVC package build, all three software test files, `--help`, @@ -44,6 +50,27 @@ The application currently decodes H.264/AVC video carried in MP4. It supports 8-bit 4:2:0 progressive Baseline, Main, and High profiles when the driver reports a compatible Vulkan Video profile. Other codecs, chroma formats, bit depths, and interlaced streams are rejected with an explanatory error. +Picture-order-count types 0, 1, and 2 are calculated for progressive frames. +Separate top and bottom order counts are supplied to Vulkan references. +The DPB applies sliding-window marking and explicit MMCO 1–6, including +long-term references. The bundled 360p stream exercises MMCO 1 on the tested +Linux GPU. The external `MR2_TANDBERG_E` conformance stream exercises MMCO 5 +and long-term operations 3, 4, and 6; `FRExt_MMCO4_Sony_B` exercises long-term +operations 2, 3, 4, and 6. On the Linux GTX 1060, decoded NV12 output matched +FFmpeg byte for byte for all 300 Tandberg frames and all 60 Sony frames. The +bundled four-slice and ID-7 fixtures also matched for all 24 and 5 frames. +The Sony comparison exposed a scaling-list bug: the pinned H.264 parser set +the SPS list-presence flags but left the list values at zero. The player now +[fills the validated lists](h264_parameter_sets.v#L77) before creating +[Vulkan session parameters](decoder_session.v#L303). + +AVC samples with 1, 2, or 4 byte NAL length prefixes are accepted. The parser +skips metadata-only samples, checks every slice in a sample belongs to the +same picture, and reports invalid slice references before creating a Vulkan +device. SPS/PPS preflight checks truncated syntax and the pinned parser's +fixed-array limits. The pinned H.264 module's weighted-prediction reader is +corrected in this application's checked slice reader. Device selection also +checks the stream's H.264 level against the GPU's reported maximum. B-frame streams are decoded in codec order and retained in a bounded image queue until they become next in presentation order. The queue size is derived @@ -53,16 +80,19 @@ have completed. The bundled Big Buck Bunny fixtures cover this path at 360p, 720p, and 1080p. Unsupported media, missing Vulkan Video extensions, and incompatible GPU -profiles produce orderly diagnostics and a non-zero exit status. Unexpected -failures after Vulkan device creation (for example, allocation, swapchain, or -queue-submission failures) remain fatal because teardown from partially -recorded or submitted command buffers is not yet modeled as recoverable. These -driver/runtime failures are tracked as post-release lifecycle hardening rather -than being conflated with malformed-input handling. +profiles produce orderly diagnostics and a non-zero exit status. The pinned +H.264 parser and this application's preflight do not validate every semantic +relationship inside arbitrarily corrupted SPS/PPS bitstreams. Failures after +Vulkan device creation (for example, allocation, swapchain, or queue-submission +failures) remain fatal because teardown from +partially recorded or submitted command buffers is not yet modeled as +recoverable. These driver/runtime failures are tracked as post-release +lifecycle hardening rather than being conflated with malformed-input handling. Hardware is selected by capability rather than vendor name: the device must provide graphics/presentation, the required Vulkan Video extensions, an H.264 -decode queue, and a supported decode output format. The decoded-picture-buffer +decode queue, coded extent and reference limits, and output/DPB formats with +the required image usages. The decoded-picture-buffer and output-image mode is chosen from the modes reported by the driver. Use `--decode-output-mode coincident` or `--decode-output-mode distinct` to force a specific advertised path during compatibility testing; `auto` remains the @@ -82,6 +112,50 @@ synchronization, and presentation still require a real Vulkan Video device. ## Hardware validation checklist +To repeat the H.264 reference-marking smoke check, download the +[MR2 Tandberg stream](https://dev.gentoo.org/~lu_zero/fate/h264-conformance/MR2_TANDBERG_E.264) +and the +[FRExt Sony stream](https://dev.gentoo.org/~lu_zero/fate/h264-conformance/FRext/FRExt_MMCO4_Sony_B.264), +then remux them to MP4 without transcoding: + +```sh +ffmpeg -r 30 -i MR2_TANDBERG_E.264 -c:v copy MR2_TANDBERG_E.mp4 +ffmpeg -r 25 -i FRExt_MMCO4_Sony_B.264 -c:v copy FRExt_MMCO4_Sony_B.mp4 +./v_vulkan_video MR2_TANDBERG_E.mp4 +./v_vulkan_video FRExt_MMCO4_Sony_B.mp4 +``` + +The files are external conformance media and are not included in the repository. +Inspect `memory_management_control_operation` with FFmpeg's `trace_headers` +bitstream filter to confirm which operations each stream contains. + +To compare decoded pixels, set `VV_DUMP_NV12_DIR` to an empty directory before +running the player. The [readback path](frame_readback.v#L48) copies the first +playback loop's decoded images to display-order `N.nv12` files. Run the +[comparison script](scripts/compare_nv12.py#L25) against the original H.264 +elementary stream for the Sony case: FFmpeg's MP4 remux has nonmonotonic +timestamps and drops frames when exporting raw video. + +```sh +VV_DUMP_NV12_DIR=/tmp/sony-nv12 ./v_vulkan_video FRExt_MMCO4_Sony_B.mp4 +python3 scripts/compare_nv12.py FRExt_MMCO4_Sony_B.264 /tmp/sony-nv12 +``` + +The readback waits for decode fences, invalidates mapped memory, and writes +only the first playback loop. It costs extra GPU memory and stalls that loop; +leave the environment variable unset for normal playback. An exact match +checks decoded NV12 bytes and display order on this GPU. It does not verify +the YCbCr-to-RGB rendering or another driver's decode implementation. + +The rendered window was checked separately on the same GPU. An X11 capture of +the bundled ID-7 color-bar fixture (BT.601 limited-range fallback) was compared +with FFmpeg's RGB output at nine interior pixels. A generated H.264 +`smptehdbars` clip signaling BT.709 and full range was compared at eight +interior pixels, with FFmpeg's scale filter explicitly set to BT.709/full +input. The largest per-channel difference was 2 in 8-bit RGB in both checks. +These spot checks cover the two indicated conversion paths, not every output +pixel, chroma edge, display compositor, or GPU driver. + Before calling a platform supported for release, run at least: - playback through multiple loops; diff --git a/QUICKSTART.md b/QUICKSTART.md index 0080864..ca2f71b 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -32,9 +32,10 @@ Install the [V compiler](https://github.com/vlang/v), then from this checkout: ``` Use `./scripts/build_linux.sh --compiler v3` to build the full player with V3. -Current V master builds the complete player with V3 and TinyCC on Linux, but V3 -playback has not yet been validated on supported hardware. Use the stable -compiler for release builds. The V3 build uses TinyCC by default; pass +Current V master builds the complete player with V3 and TinyCC on Linux, and +the ID-7 and multislice fixtures have passed decoded-frame comparisons with +FFmpeg on a GTX 1060. Broader V3 hardware coverage remains incomplete. Use the +stable compiler for release builds. The V3 build uses TinyCC by default; pass `--cc gcc` if more detailed C diagnostics are needed. The wrapper also offers `--linkage static` and `--glfw bundled --glfw-version 3.4`; run it with `--help` for all choices. diff --git a/README.md b/README.md index a6944de..494f3e1 100644 --- a/README.md +++ b/README.md @@ -86,10 +86,10 @@ and follow the chapters that match the problem you are solving. v test . ``` -The software-only tests cover MP4 metadata and validation, H.264 picture order, -malformed and truncated inputs, playback timing, looping, and command-line -parsing. Vulkan decode, synchronization, resize, and presentation still -require hardware with Vulkan Video support. +The software-only tests cover MP4 metadata and validation, H.264 parameter sets, +multi-slice pictures, picture order, malformed and truncated inputs, playback +timing, looping, and command-line parsing. Vulkan decode, synchronization, +resize, and presentation still require hardware with Vulkan Video support. ## License diff --git a/app.v b/app.v index e06e49a..88cbeb6 100644 --- a/app.v +++ b/app.v @@ -126,9 +126,11 @@ fn video_render_transform(metadata VideoMetadata, extent vk.Extent2D) VideoRende result.values[6] = 0 } } + mut scale_x := f32(1) mut scale_y := f32(1) - if metadata.display_width > 0 && metadata.display_height > 0 && extent.width > 0 && extent.height > 0 { + if metadata.display_width > 0 && metadata.display_height > 0 && extent.width > 0 + && extent.height > 0 { video_aspect := f32(metadata.display_width) / f32(metadata.display_height) surface_aspect := f32(extent.width) / f32(extent.height) if video_aspect > surface_aspect { @@ -176,8 +178,10 @@ fn (mut app VideoDecodeApp) initialize() bool { return false } h264_profile_idc := app.video_player.h264_profile_idc() + decode_requirements := app.video_player.decode_requirements() video_metadata := app.video_player.metadata() - diagnostics := app.device_context.h264_decode_gpu_diagnostics_for_output_mode(h264_profile_idc, app.decode_output_mode) + diagnostics := app.device_context.h264_decode_gpu_diagnostics_for_output_mode(decode_requirements, + app.decode_output_mode) if app.list_gpus { println('Vulkan devices for H.264 ${h264_profile_name(h264_profile_idc)} Profile:') for index, diagnostic in diagnostics { @@ -193,14 +197,16 @@ fn (mut app VideoDecodeApp) initialize() bool { app.abort_initialization() return false } - if !app.device_context.is_h264_decode_gpu_compatible_for_output_mode(app.preferred_gpu_index, h264_profile_idc, app.decode_output_mode) { + if !app.device_context.is_h264_decode_gpu_compatible_for_output_mode(app.preferred_gpu_index, + decode_requirements, app.decode_output_mode) { eprintln('GPU [${app.preferred_gpu_index}] cannot play this video: ${diagnostics[app.preferred_gpu_index]}') app.abort_initialization() return false } gpu_index = u32(app.preferred_gpu_index) } else { - gpu_index = app.device_context.find_h264_decode_gpu_for_output_mode(h264_profile_idc, app.decode_output_mode) or { + gpu_index = app.device_context.find_h264_decode_gpu_for_output_mode(decode_requirements, + app.decode_output_mode) or { mode_requirement := if app.decode_output_mode == .automatic { 'a supported DPB/output mode' } else { @@ -241,7 +247,8 @@ fn (mut app VideoDecodeApp) initialize() bool { poolSizeCount: u32(descriptor_pool_sizes.len) pPoolSizes: descriptor_pool_sizes.data } - check_vk(vk.create_descriptor_pool(app.device_context.vk_device, &descriptor_pool_ci, unsafe { nil }, &app.descriptor_pool), 'Could not create descriptor pool') + check_vk(vk.create_descriptor_pool(app.device_context.vk_device, &descriptor_pool_ci, + unsafe { nil }, &app.descriptor_pool), 'Could not create descriptor pool') mut sampler_ci := vk.SamplerCreateInfo{ magFilter: vk.Filter.linear @@ -255,7 +262,8 @@ fn (mut app VideoDecodeApp) initialize() bool { conversion: app.device_context.sampler_ycbcr_conversion } sampler_ci.pNext = &sampler_conversion_info - check_vk(vk.create_sampler(app.device_context.vk_device, &sampler_ci, unsafe { nil }, &app.sampler), 'Could not create video sampler') + check_vk(vk.create_sampler(app.device_context.vk_device, &sampler_ci, unsafe { nil }, + &app.sampler), 'Could not create video sampler') // ImGui // # IMGUI_CHECKVERSION(); @@ -290,7 +298,8 @@ fn (mut app VideoDecodeApp) initialize() bool { bindingCount: u32(ds_layouts.len) pBindings: ds_layouts.data } - check_vk(vk.create_descriptor_set_layout(app.device_context.vk_device, &ds_layout_ci, unsafe { nil }, &app.ds_layout), 'Could not create descriptor-set layout') + check_vk(vk.create_descriptor_set_layout(app.device_context.vk_device, &ds_layout_ci, + unsafe { nil }, &app.ds_layout), 'Could not create descriptor-set layout') app.initialize_render_pass() app.initialize_pipeline() @@ -303,8 +312,10 @@ fn (mut app VideoDecodeApp) initialize() bool { } semaphore_ci := vk.SemaphoreCreateInfo{} - check_vk(vk.create_semaphore(vk_device, &semaphore_ci, unsafe { nil }, &app.sem_render_complete), 'Could not create render-complete semaphore') - check_vk(vk.create_semaphore(vk_device, &semaphore_ci, unsafe { nil }, &app.sem_present_complete), 'Could not create presentation semaphore') + check_vk(vk.create_semaphore(vk_device, &semaphore_ci, unsafe { nil }, &app.sem_render_complete), + 'Could not create render-complete semaphore') + check_vk(vk.create_semaphore(vk_device, &semaphore_ci, unsafe { nil }, + &app.sem_present_complete), 'Could not create presentation semaphore') $if debug { eprintln('Application synchronization initialized') } @@ -360,7 +371,8 @@ fn (mut app VideoDecodeApp) run() { graph.ensure_cap(300) } mut prev_time := time.now() - for !glfw.window_should_close(app.window_p) && glfw.get_key(app.window_p, glfw.key_escape) == glfw.release { + for !glfw.window_should_close(app.window_p) + && glfw.get_key(app.window_p, glfw.key_escape) == glfw.release { glfw.poll_events() // Keep playback timing in nanoseconds. Converting a sub-second frame @@ -369,7 +381,7 @@ fn (mut app VideoDecodeApp) run() { time_elapsed_ns := math.min(time.since(prev_time).nanoseconds(), i64(500_000_000)) prev_time = time.now() - mut res := app.device_context.swapchain.acquire_next_image(mut app.sem_present_complete) + mut res := app.device_context.swapchain.acquire_next_image(app.sem_present_complete) if res == vk.Result.error_out_of_date_khr { app.recreate_swapchain() continue @@ -408,7 +420,8 @@ fn (mut app VideoDecodeApp) run() { descriptorType: .combined_image_sampler pImageInfo: &image_info } - vk.update_descriptor_sets(app.device_context.vk_device, 1, &write_descriptor, 0, unsafe { nil }) + vk.update_descriptor_sets(app.device_context.vk_device, 1, &write_descriptor, 0, + unsafe { nil }) } // Finish the Dear ImGui frame before starting the next one. The previous @@ -437,8 +450,11 @@ fn (mut app VideoDecodeApp) run() { mut video_transform := video_render_transform(metadata, extent) if !isnil(output_view) { vk.cmd_bind_pipeline(frame.command_buffer, .graphics, app.pipeline) - vk.cmd_bind_descriptor_sets(frame.command_buffer, .graphics, app.pipeline_layout, 0, 1, &frame.descriptor_set, 0, unsafe { nil }) - vk.cmd_push_constants(frame.command_buffer, app.pipeline_layout, vk.ShaderStageFlags(vk.ShaderStageFlagBits.vertex), 0, u32(sizeof(VideoRenderTransform)), &video_transform) + vk.cmd_bind_descriptor_sets(frame.command_buffer, .graphics, app.pipeline_layout, 0, 1, + &frame.descriptor_set, 0, unsafe { nil }) + vk.cmd_push_constants(frame.command_buffer, app.pipeline_layout, + vk.ShaderStageFlags(vk.ShaderStageFlagBits.vertex), 0, + u32(sizeof(VideoRenderTransform)), &video_transform) viewport := vk.Viewport{ width: f32(extent.width) height: f32(extent.height) @@ -468,12 +484,14 @@ fn (mut app VideoDecodeApp) run() { signalSemaphoreCount: 1 pSignalSemaphores: &app.sem_render_complete } - res = vk.queue_submit(app.device_context.get_queue(.graphics), 1, &submit_info, frame.queue_submit_fence) + res = vk.queue_submit(app.device_context.get_queue(.graphics), 1, &submit_info, + frame.queue_submit_fence) if res != vk.Result.success { panic('Could not submit graphics command buffer: ${res}') } present_result := app.device_context.present([app.sem_render_complete]) - if present_result == vk.Result.error_out_of_date_khr || present_result == vk.Result.suboptimal_khr { + if present_result == vk.Result.error_out_of_date_khr + || present_result == vk.Result.suboptimal_khr { app.recreate_swapchain() } else if present_result != vk.Result.success { panic('Could not present a swapchain image: ${present_result}') @@ -596,7 +614,8 @@ fn (mut app VideoDecodeApp) abort_initialization() { fn (mut app VideoDecodeApp) teardown_per_frame(mut frame_info FrameInfo) { vk_device := app.device_context.get_vk_device() if !isnil(frame_info.descriptor_set) { - result := vk.free_descriptor_sets(vk_device, app.descriptor_pool, 1, &frame_info.descriptor_set) + result := vk.free_descriptor_sets(vk_device, app.descriptor_pool, 1, + &frame_info.descriptor_set) assert result == .success frame_info.descriptor_set = unsafe { nil } } @@ -687,7 +706,8 @@ fn (mut app VideoDecodeApp) initialize_render_pass() { pDependencies: &dependency } - check_vk(vk.create_render_pass(vk_device, &rp_info, unsafe { nil }, &app.render_pass), 'Could not create render pass') + check_vk(vk.create_render_pass(vk_device, &rp_info, unsafe { nil }, &app.render_pass), + 'Could not create render pass') } @[heap] @@ -703,7 +723,8 @@ fn (mut app VideoDecodeApp) create_shader_module(shader_data []u32) vk.ShaderMod pCode: unsafe { shader_data.data } } mut shader_module := vk.ShaderModule(0) - check_vk(vk.create_shader_module(vk_device, &module_ci, unsafe { nil }, &shader_module), 'Could not create shader module') + check_vk(vk.create_shader_module(vk_device, &module_ci, unsafe { nil }, &shader_module), + 'Could not create shader module') return shader_module } @@ -723,7 +744,8 @@ fn (mut app VideoDecodeApp) initialize_pipeline() { pushConstantRangeCount: 1 pPushConstantRanges: &push_constant_range } - check_vk(vk.create_pipeline_layout(vk_device, &pipeline_layout_ci, unsafe { nil }, &app.pipeline_layout), 'Could not create graphics pipeline layout') + check_vk(vk.create_pipeline_layout(vk_device, &pipeline_layout_ci, unsafe { nil }, + &app.pipeline_layout), 'Could not create graphics pipeline layout') mut vertex_input_ci := vk.PipelineVertexInputStateCreateInfo{} mut input_assembly_ci := vk.PipelineInputAssemblyStateCreateInfo{ @@ -785,7 +807,8 @@ fn (mut app VideoDecodeApp) initialize_pipeline() { renderPass: app.render_pass } - check_vk(vk.create_graphics_pipelines(vk_device, unsafe { nil }, 1, &pipeline_ci, unsafe { nil }, &app.pipeline), 'Could not create graphics pipeline') + check_vk(vk.create_graphics_pipelines(vk_device, unsafe { nil }, 1, &pipeline_ci, + unsafe { nil }, &app.pipeline), 'Could not create graphics pipeline') for sstage in shader_stages { vk.destroy_shader_module(vk_device, sstage.module, unsafe { nil }) @@ -811,7 +834,8 @@ fn (mut app VideoDecodeApp) initialize_framebuffers() { layers: 1 } mut fb := unsafe { nil } - check_vk(vk.create_framebuffer(vk_device, &framebuffer_ci, unsafe { nil }, &fb), 'Could not create swapchain framebuffer ${i}') + check_vk(vk.create_framebuffer(vk_device, &framebuffer_ci, unsafe { nil }, &fb), + 'Could not create swapchain framebuffer ${i}') app.frames[i].framebuffer = fb app.init_per_frame(mut app.frames[i]) } @@ -823,19 +847,22 @@ fn (mut app VideoDecodeApp) init_per_frame(mut frame_info FrameInfo) { fence_ci := vk.FenceCreateInfo{ flags: vk.FenceCreateFlags(vk.FenceCreateFlagBits.signaled) } - check_vk(vk.create_fence(vk_device, &fence_ci, unsafe { nil }, &frame_info.queue_submit_fence), 'Could not create frame fence') + check_vk(vk.create_fence(vk_device, &fence_ci, unsafe { nil }, &frame_info.queue_submit_fence), + 'Could not create frame fence') command_pool_ci := vk.CommandPoolCreateInfo{ flags: vk.CommandPoolCreateFlags(vk.CommandPoolCreateFlagBits.transient) } - check_vk(vk.create_command_pool(vk_device, &command_pool_ci, unsafe { nil }, &frame_info.command_pool), 'Could not create graphics command pool') + check_vk(vk.create_command_pool(vk_device, &command_pool_ci, unsafe { nil }, + &frame_info.command_pool), 'Could not create graphics command pool') command_buffer_allocate_info := vk.CommandBufferAllocateInfo{ commandPool: frame_info.command_pool level: vk.CommandBufferLevel.primary commandBufferCount: 1 } - check_vk(vk.allocate_command_buffers(vk_device, &command_buffer_allocate_info, &frame_info.command_buffer), 'Could not allocate graphics command buffer') + check_vk(vk.allocate_command_buffers(vk_device, &command_buffer_allocate_info, + &frame_info.command_buffer), 'Could not allocate graphics command buffer') frame_info.queue_index = 0 descriptor_set_allocate_info := vk.DescriptorSetAllocateInfo{ @@ -843,7 +870,8 @@ fn (mut app VideoDecodeApp) init_per_frame(mut frame_info FrameInfo) { descriptorSetCount: 1 pSetLayouts: &app.ds_layout } - result := vk.allocate_descriptor_sets(vk_device, &descriptor_set_allocate_info, &frame_info.descriptor_set) + result := vk.allocate_descriptor_sets(vk_device, &descriptor_set_allocate_info, + &frame_info.descriptor_set) if result != .success || isnil(frame_info.descriptor_set) { panic('Could not allocate per-frame descriptor set: ${result}') } diff --git a/decoder_session.v b/decoder_session.v index c8d5fab..333739e 100644 --- a/decoder_session.v +++ b/decoder_session.v @@ -28,7 +28,8 @@ fn (mut d Decoder) initialize(mut app VideoDecodeApp) { } d.properties.caps.pNext = &d.properties.decode_caps - mut res := vk.get_physical_device_video_capabilities_khr(dev_ctx.get_gpu_current(), &d.settings.profile_info, mut &d.properties.caps) + mut res := vk.get_physical_device_video_capabilities_khr(dev_ctx.get_gpu_current(), + &d.settings.profile_info, mut &d.properties.caps) if res != vk.Result.success { panic('Vulkan device does not expose H.264 ${h264_profile_name(d.video_data.h264_profile_idc)} Profile decode capabilities: ${res}') } @@ -45,7 +46,8 @@ fn (mut d Decoder) initialize(mut app VideoDecodeApp) { capability_flags := d.properties.decode_caps.flags supports_coincide := (capability_flags & vk.VideoDecodeCapabilityFlagsKHR(vk.VideoDecodeCapabilityFlagBitsKHR.dpb_and_output_coincide)) != 0 supports_distinct := (capability_flags & vk.VideoDecodeCapabilityFlagsKHR(vk.VideoDecodeCapabilityFlagBitsKHR.dpb_and_output_distinct)) != 0 - selected_output_mode := select_decode_output_mode(app.decode_output_mode, supports_coincide, supports_distinct) or { panic(err) } + selected_output_mode := select_decode_output_mode(app.decode_output_mode, supports_coincide, + supports_distinct) or { panic(err) } d.properties.dpb_and_output_coincide = selected_output_mode == .coincident println('Decode image mode: ${if d.properties.dpb_and_output_coincide { 'coincident DPB/output' @@ -53,7 +55,8 @@ fn (mut d Decoder) initialize(mut app VideoDecodeApp) { 'distinct DPB/output' }}') output_usage := vk.ImageUsageFlags(u32(vk.ImageUsageFlagBits.video_decode_dst) | u32(vk.ImageUsageFlagBits.transfer_src)) - d.properties.format_props = query_video_format(dev_ctx.get_gpu_current(), &d.settings.profile_list_info, output_usage) or { + d.properties.format_props = query_video_format(dev_ctx.get_gpu_current(), + &d.settings.profile_list_info, output_usage) or { panic('No Vulkan Video decode-output format supports transfer to the display image') } dpb_usage := if d.properties.dpb_and_output_coincide { @@ -61,14 +64,17 @@ fn (mut d Decoder) initialize(mut app VideoDecodeApp) { } else { vk.ImageUsageFlags(vk.ImageUsageFlagBits.video_decode_dpb) } - d.properties.dpb_format_props = query_video_format(dev_ctx.get_gpu_current(), &d.settings.profile_list_info, dpb_usage) or { + d.properties.dpb_format_props = query_video_format(dev_ctx.get_gpu_current(), + &d.settings.profile_list_info, dpb_usage) or { panic('No Vulkan Video DPB format supports the required decode mode') } d.properties.usage_dpb = dpb_usage num_memory_frames := u64(d.video_data.num_dpb_slots) - mut aligned_frame_size := U64(d.video_data.max_memory_frame_size_bytes).align_to(d.properties.caps.minBitstreamBufferOffsetAlignment) - aligned_frame_size = U64(aligned_frame_size).align_to(d.properties.caps.minBitstreamBufferSizeAlignment) + mut aligned_frame_size := + U64(d.video_data.max_memory_frame_size_bytes).align_to(d.properties.caps.minBitstreamBufferOffsetAlignment) + aligned_frame_size = + U64(aligned_frame_size).align_to(d.properties.caps.minBitstreamBufferSizeAlignment) d.video_data.max_memory_frame_size_bytes = aligned_frame_size video_decoder_queue_family_index := dev_ctx.get_decoder_queue_family_index() buffer_size := d.video_data.max_memory_frame_size_bytes * num_memory_frames @@ -108,26 +114,29 @@ fn (mut d Decoder) initialize(mut app VideoDecodeApp) { pVideoProfile: &d.settings.profile_info pictureFormat: d.properties.format_props.format maxCodedExtent: vk.Extent2D{ - width: math.min(d.video_data.width, d.properties.caps.maxCodedExtent.width) - height: math.min(d.video_data.height, d.properties.caps.maxCodedExtent.height) + width: d.video_data.width_padd + height: d.video_data.height_padd } referencePictureFormat: d.properties.dpb_format_props.format maxDpbSlots: d.video_data.num_dpb_slots maxActiveReferencePictures: d.video_data.max_reference_pictures pStdHeaderVersion: &d.properties.caps.stdHeaderVersion } - res = vk.create_video_session_khr(dev_ctx.vk_device, &session_ci, unsafe { nil }, &d.video_session) + res = vk.create_video_session_khr(dev_ctx.vk_device, &session_ci, unsafe { nil }, + &d.video_session) if res != vk.Result.success { panic('Could not create the Vulkan Video H.264 decode session: ${res}') } mut requirement_count := u32(0) mut n := unsafe { nil } - vk.get_video_session_memory_requirements_khr(dev_ctx.vk_device, d.video_session, &requirement_count, mut n) + vk.get_video_session_memory_requirements_khr(dev_ctx.vk_device, d.video_session, + &requirement_count, mut n) mut requirements := []vk.VideoSessionMemoryRequirementsKHR{len: int(requirement_count), init: vk.VideoSessionMemoryRequirementsKHR{}} mut requirements_data := requirements.data - res = vk.get_video_session_memory_requirements_khr(dev_ctx.vk_device, d.video_session, &requirement_count, mut requirements_data) + res = vk.get_video_session_memory_requirements_khr(dev_ctx.vk_device, d.video_session, + &requirement_count, mut requirements_data) if res != vk.Result.success { panic('Could not query Vulkan Video session memory requirements: ${res}') } @@ -140,7 +149,8 @@ fn (mut d Decoder) initialize(mut app VideoDecodeApp) { // Video-session memory is opaque driver storage. The requirement's // memoryTypeBits is authoritative; some drivers expose a dedicated type // without VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT for this binding. - memory_type_index := dev_ctx.memory_allocator.get_memory_type(req.memoryRequirements.memoryTypeBits, vk.MemoryPropertyFlags(0)) + memory_type_index := dev_ctx.memory_allocator.get_memory_type(req.memoryRequirements.memoryTypeBits, + vk.MemoryPropertyFlags(0)) if memory_type_index == max_u32 { panic('No compatible Vulkan memory type exists for video-session binding ${req.memoryBindIndex}') } @@ -166,11 +176,13 @@ fn (mut d Decoder) initialize(mut app VideoDecodeApp) { // Resolve this extension command from the device directly. When another // shared object links libvulkan, ELF symbol interposition can otherwise // make Volk's same-named global dispatch slot unreliable. - bind_session_memory_fn := vk.PFN_vkBindVideoSessionMemoryKHR(vk.get_device_proc_addr(dev_ctx.vk_device, c'vkBindVideoSessionMemoryKHR')) + bind_session_memory_fn := vk.PFN_vkBindVideoSessionMemoryKHR(vk.get_device_proc_addr(dev_ctx.vk_device, + c'vkBindVideoSessionMemoryKHR')) if isnil(voidptr(bind_session_memory_fn)) { panic('vkGetDeviceProcAddr returned null for vkBindVideoSessionMemoryKHR') } - res = bind_session_memory_fn(dev_ctx.vk_device, d.video_session, u32(bind_session_memory_infos.len), bind_session_memory_infos.data) + res = bind_session_memory_fn(dev_ctx.vk_device, d.video_session, + u32(bind_session_memory_infos.len), bind_session_memory_infos.data) if res != vk.Result.success { panic('Could not bind Vulkan Video session memory: ${res}') } @@ -228,8 +240,8 @@ fn (mut d Decoder) prepare_decoded_picture_buffer(device vk.Device, mut allocato imageType: vk.ImageType._2d format: d.properties.dpb_format_props.format extent: vk.Extent3D{ - width: d.video_data.width - height: d.video_data.height + width: d.video_data.width_padd + height: d.video_data.height_padd depth: 1 } mipLevels: 1 @@ -301,12 +313,20 @@ fn (mut d Decoder) create_video_session_parameters(device vk.Device) { for j in 0 .. pps.use_default_scaling_matrix_4x4_flag.len { video_scaling_list_pps[i].use_default_scaling_matrix_mask |= u16(pps.use_default_scaling_matrix_4x4_flag[j]) << j } + for j in 0 .. pps.use_default_scaling_matrix_8x8_flag.len { + video_scaling_list_pps[i].use_default_scaling_matrix_mask |= u16(pps.use_default_scaling_matrix_8x8_flag[j]) << ( + j + 6) + } mut list_idx := 0 mut el_idx := 0 - for list_idx < vk.std_video_h264_scaling_list_4x4_num_lists && list_idx < pps.scaling_list_4x4.len { - for el_idx < vk.std_video_h264_scaling_list_4x4_num_elements && el_idx < pps.scaling_list_4x4[0].len { + for list_idx < vk.std_video_h264_scaling_list_4x4_num_lists + && list_idx < pps.scaling_list_4x4.len { + el_idx = 0 + for el_idx < vk.std_video_h264_scaling_list_4x4_num_elements + && el_idx < pps.scaling_list_4x4[0].len { unsafe { - C.vv_set_h264_scaling_list_4x4(&video_scaling_list_pps[i], u32(list_idx), u32(el_idx), u8(pps.scaling_list_4x4[list_idx][el_idx])) + C.vv_set_h264_scaling_list_4x4(&video_scaling_list_pps[i], u32(list_idx), + u32(el_idx), u8(pps.scaling_list_4x4[list_idx][el_idx])) } el_idx++ } @@ -314,10 +334,14 @@ fn (mut d Decoder) create_video_session_parameters(device vk.Device) { } list_idx = 0 el_idx = 0 - for list_idx < vk.std_video_h264_scaling_list_8x8_num_lists && list_idx < pps.scaling_list_8x8.len { - for el_idx < vk.std_video_h264_scaling_list_8x8_num_elements && el_idx < pps.scaling_list_8x8[0].len { + for list_idx < vk.std_video_h264_scaling_list_8x8_num_lists + && list_idx < pps.scaling_list_8x8.len { + el_idx = 0 + for el_idx < vk.std_video_h264_scaling_list_8x8_num_elements + && el_idx < pps.scaling_list_8x8[0].len { unsafe { - C.vv_set_h264_scaling_list_8x8(&video_scaling_list_pps[i], u32(list_idx), u32(el_idx), u8(pps.scaling_list_8x8[list_idx][el_idx])) + C.vv_set_h264_scaling_list_8x8(&video_scaling_list_pps[i], u32(list_idx), + u32(el_idx), u8(pps.scaling_list_8x8[list_idx][el_idx])) } el_idx++ } @@ -368,7 +392,7 @@ fn (mut d Decoder) create_video_session_parameters(device vk.Device) { sps := unsafe { &h264.SequenceParameterSet(&d.video_data.sps_bytes[sps_offset]) } video_sequence_parameter_set[i] = vk.StdVideoH264SequenceParameterSet{ - flags: vk.StdVideoH264SpsFlags{ + flags: vk.StdVideoH264SpsFlags{ constraint_set0_flag: sps.constraint_set0_flag constraint_set1_flag: sps.constraint_set1_flag constraint_set2_flag: sps.constraint_set2_flag @@ -388,8 +412,9 @@ fn (mut d Decoder) create_video_session_parameters(device vk.Device) { } // Note: There is no 0 in StdVideoH264ProfileIdc enum profile_idc: unsafe { vk.StdVideoH264ProfileIdc(sps.profile_idc) } - level_idc: unsafe { vk.StdVideoH264LevelIdc(sps.level_idc) } - chroma_format_idc: get_chroma_format(sps.profile_idc, sps.chroma_format_idc) + level_idc: std_h264_level_idc(sps.level_idc) + chroma_format_idc: get_chroma_format(sps.profile_idc, + sps.chroma_format_idc) seq_parameter_set_id: u8(sps.seq_parameter_set_id) bit_depth_luma_minus8: u8(sps.bit_depth_luma_minus8) bit_depth_chroma_minus8: u8(sps.bit_depth_chroma_minus8) @@ -449,9 +474,9 @@ fn (mut d Decoder) create_video_session_parameters(device vk.Device) { hrd := &sps.hrd video_hrd_parameters[i] = vk.StdVideoH264HrdParameters{ - cpb_cnt_minus1: u8(hrd.cpb_cnt_minus1) - bit_rate_scale: u8(hrd.bit_rate_scale) - cpb_size_scale: u8(hrd.cpb_size_scale) + cpb_cnt_minus1: u8(hrd.cpb_cnt_minus1) + bit_rate_scale: u8(hrd.bit_rate_scale) + cpb_size_scale: u8(hrd.cpb_size_scale) // reserved1: u8(0) // bit_rate_value_minus1: [u32(0)] // cpb_size_value_minus1: [u32(0)] @@ -476,13 +501,21 @@ fn (mut d Decoder) create_video_session_parameters(device vk.Device) { for j in 0 .. sps.use_default_scaling_matrix_4x4_flag.len { video_scaling_list_sps[i].use_default_scaling_matrix_mask |= u16(sps.use_default_scaling_matrix_4x4_flag[j]) << j } + for j in 0 .. sps.use_default_scaling_matrix_8x8_flag.len { + video_scaling_list_sps[i].use_default_scaling_matrix_mask |= u16(sps.use_default_scaling_matrix_8x8_flag[j]) << ( + j + 6) + } mut list_idx := 0 mut el_idx := 0 - for list_idx < vk.std_video_h264_scaling_list_4x4_num_lists && list_idx < sps.scaling_list_4x4.len { - for el_idx < vk.std_video_h264_scaling_list_4x4_num_elements && el_idx < sps.scaling_list_4x4[0].len { + for list_idx < vk.std_video_h264_scaling_list_4x4_num_lists + && list_idx < sps.scaling_list_4x4.len { + el_idx = 0 + for el_idx < vk.std_video_h264_scaling_list_4x4_num_elements + && el_idx < sps.scaling_list_4x4[0].len { unsafe { - C.vv_set_h264_scaling_list_4x4(&video_scaling_list_sps[i], u32(list_idx), u32(el_idx), u8(sps.scaling_list_4x4[list_idx][el_idx])) + C.vv_set_h264_scaling_list_4x4(&video_scaling_list_sps[i], u32(list_idx), + u32(el_idx), u8(sps.scaling_list_4x4[list_idx][el_idx])) } el_idx++ } @@ -491,10 +524,14 @@ fn (mut d Decoder) create_video_session_parameters(device vk.Device) { list_idx = 0 el_idx = 0 - for list_idx < vk.std_video_h264_scaling_list_8x8_num_lists && list_idx < sps.scaling_list_8x8.len { - for el_idx < vk.std_video_h264_scaling_list_8x8_num_elements && el_idx < sps.scaling_list_8x8[0].len { + for list_idx < vk.std_video_h264_scaling_list_8x8_num_lists + && list_idx < sps.scaling_list_8x8.len { + el_idx = 0 + for el_idx < vk.std_video_h264_scaling_list_8x8_num_elements + && el_idx < sps.scaling_list_8x8[0].len { unsafe { - C.vv_set_h264_scaling_list_8x8(&video_scaling_list_sps[i], u32(list_idx), u32(el_idx), u8(sps.scaling_list_8x8[list_idx][el_idx])) + C.vv_set_h264_scaling_list_8x8(&video_scaling_list_sps[i], u32(list_idx), + u32(el_idx), u8(sps.scaling_list_8x8[list_idx][el_idx])) } el_idx++ } @@ -520,7 +557,8 @@ fn (mut d Decoder) create_video_session_parameters(device vk.Device) { videoSession: d.video_session } - res := vk.create_video_session_parameters_khr(device, &video_session_parameters_ci, unsafe { nil }, &d.video_session_parameters) + res := vk.create_video_session_parameters_khr(device, &video_session_parameters_ci, + unsafe { nil }, &d.video_session_parameters) if res != vk.Result.success { panic('Could not create H.264 video-session parameters: ${res}') } diff --git a/device_context.v b/device_context.v index 11291da..b9e4202 100644 --- a/device_context.v +++ b/device_context.v @@ -5,32 +5,22 @@ import antono2.vkmemalloc as vkmem import antono2.glfw import math -// import bindings.volk - -// #flag linux -I$env('VULKAN_SDK')/include -// #flag windows -I$env('VULKAN_SDK')/Include -// #flag windows -I$env('VULKAN_SDK')/Include/Volk - -// #flag -DVOLK_IMPLEMENTATION -// #include "volk.h" struct DeviceContext { mut: - vk_debug_utils vk.DebugUtilsMessengerEXT = unsafe { nil } - use_gpu_index int - gpus []vk.PhysicalDevice - queue_families []QueueFamilyProperties - queue_family_indices []u32 - video_decode_family u32 = vk.queue_family_ignored - graphics_queue vk.Queue = unsafe { nil } - video_decode_queue vk.Queue = unsafe { nil } - physical_device_memory_props []vk.PhysicalDeviceMemoryProperties2 - features2 vk.PhysicalDeviceFeatures2 = vk.PhysicalDeviceFeatures2{} - vulkan12_features vk.PhysicalDeviceVulkan12Features = vk.PhysicalDeviceVulkan12Features{} - vulkan13_features vk.PhysicalDeviceVulkan13Features = vk.PhysicalDeviceVulkan13Features{} - video_profile_info vk.VideoProfileInfoKHR - video_decode_h264 VideoDecodeH264 - video_capabilities vk.VideoCapabilitiesKHR = vk.VideoCapabilitiesKHR{} - video_decode_capabilities vk.VideoDecodeCapabilitiesKHR + vk_debug_utils vk.DebugUtilsMessengerEXT = unsafe { nil } + use_gpu_index int + gpus []vk.PhysicalDevice + queue_families []QueueFamilyProperties + video_decode_family u32 = vk.queue_family_ignored + graphics_queue vk.Queue = unsafe { nil } + video_decode_queue vk.Queue = unsafe { nil } + features2 vk.PhysicalDeviceFeatures2 = vk.PhysicalDeviceFeatures2{} + vulkan12_features vk.PhysicalDeviceVulkan12Features = vk.PhysicalDeviceVulkan12Features{} + vulkan13_features vk.PhysicalDeviceVulkan13Features = vk.PhysicalDeviceVulkan13Features{} + video_profile_info vk.VideoProfileInfoKHR + video_decode_h264 VideoDecodeH264 + video_capabilities vk.VideoCapabilitiesKHR = vk.VideoCapabilitiesKHR{} + video_decode_capabilities vk.VideoDecodeCapabilitiesKHR pub mut: vk_instance vk.Instance = unsafe { nil } vk_device vk.Device = unsafe { nil } @@ -48,53 +38,65 @@ mut: capabilities vk.VideoDecodeH264CapabilitiesKHR = vk.VideoDecodeH264CapabilitiesKHR{} } +struct VideoDecodeRequirements { + profile_idc u32 + level_idc u32 + width u32 + height u32 + dpb_slots u32 + references u32 +} + +// H.264 stores the level number in the SPS (40 means level 4.0), while +// StdVideoH264LevelIdc is an ordinal enum (level 4.0 is 10). +fn std_h264_level_idc(level_idc u32) vk.StdVideoH264LevelIdc { + return match level_idc { + 10 { ._1_0 } + 11 { ._1_1 } + 12 { ._1_2 } + 13 { ._1_3 } + 20 { ._2_0 } + 21 { ._2_1 } + 22 { ._2_2 } + 30 { ._3_0 } + 31 { ._3_1 } + 32 { ._3_2 } + 40 { ._4_0 } + 41 { ._4_1 } + 42 { ._4_2 } + 50 { ._5_0 } + 51 { ._5_1 } + 52 { ._5_2 } + 60 { ._6_0 } + 61 { ._6_1 } + 62 { ._6_2 } + else { .invalid } + } +} + +fn h264_level_issue(required u32, supported vk.StdVideoH264LevelIdc) string { + level := std_h264_level_idc(required) + if level == .invalid { + return 'H.264 level_idc ${required} is unsupported' + } + if u32(supported) > u32(vk.StdVideoH264LevelIdc._6_2) { + return 'device reported an invalid H.264 level limit' + } + if u32(level) > u32(supported) { + levels := [u32(10), 11, 12, 13, 20, 21, 22, 30, 31, 32, 40, 41, 42, 50, 51, 52, 60, 61, + 62] + maximum := levels[int(u32(supported))] + return 'video requires H.264 level ${required / 10}.${required % 10}; device supports up to ${maximum / 10}.${maximum % 10}' + } + return '' +} + struct QueueFamilyProperties { mut: properties vk.QueueFamilyProperties2 properties_video vk.QueueFamilyVideoPropertiesKHR } -struct GPUBufferDesc { -pub mut: - size vk.DeviceSize - usage vk.BufferUsageFlags - memory_property vk.MemoryPropertyFlagBits = vk.MemoryPropertyFlagBits.device_local -} - -struct GPUImageDesc { -pub mut: - extent vk.Extent3D - array_size u32 = 1 - mip_levels u32 = 1 - image_type vk.ImageType = vk.ImageType._2d - format vk.Format = vk.Format.undefined - sample_count u32 = 1 - usage vk.ImageUsageFlags - memory_property vk.MemoryPropertyFlagBits = vk.MemoryPropertyFlagBits.device_local -} - -struct GPUBuffer { -pub mut: - buffer vk.Buffer = unsafe { nil } - memory vk.DeviceMemory = unsafe { nil } - device_address vk.DeviceAddress - p_mapped voidptr = unsafe { nil } - - desc GPUBufferDesc -} - -struct GPUImage { -pub mut: - image vk.Image = unsafe { nil } - image_view vk.ImageView = unsafe { nil } - memory vk.DeviceMemory = unsafe { nil } - - device_address vk.DeviceAddress - p_mapped voidptr = unsafe { nil } - - desc GPUImageDesc -} - enum QueueType { graphics video_decode @@ -117,7 +119,8 @@ fn (mut ctx DeviceContext) shutdown() { ctx.swapchain.shutdown() } if !isnil(ctx.sampler_ycbcr_conversion) { - vk.destroy_sampler_ycbcr_conversion(ctx.vk_device, ctx.sampler_ycbcr_conversion, unsafe { nil }) + vk.destroy_sampler_ycbcr_conversion(ctx.vk_device, ctx.sampler_ycbcr_conversion, + unsafe { nil }) ctx.sampler_ycbcr_conversion = unsafe { nil } } ctx.memory_allocator.destroy() @@ -137,7 +140,8 @@ fn (mut ctx DeviceContext) shutdown_instance_resources() { } $if debug ? { if !isnil(ctx.vk_debug_utils) { - vk.destroy_debug_utils_messenger_ext(ctx.vk_instance, ctx.vk_debug_utils, unsafe { nil }) + vk.destroy_debug_utils_messenger_ext(ctx.vk_instance, ctx.vk_debug_utils, + unsafe { nil }) ctx.vk_debug_utils = unsafe { nil } } } @@ -175,19 +179,21 @@ fn (mut ctx DeviceContext) initialize_device(use_gpu_index u32, h264_profile_idc queue_family := ctx.queue_families[i].properties.queueFamilyProperties mut supports_present := vk.Bool32(0) - vk.get_physical_device_surface_support_khr(gpu, u32(i), ctx.swapchain.surface, &supports_present) - if queue_family.queueCount > 0 && (queue_family.queueFlags & vk.QueueFlags(vk.QueueFlagBits.graphics)) != 0 && supports_present == vk._true { + vk.get_physical_device_surface_support_khr(gpu, u32(i), ctx.swapchain.surface, + &supports_present) + if queue_family.queueCount > 0 + && (queue_family.queueFlags & vk.QueueFlags(vk.QueueFlagBits.graphics)) != 0 + && supports_present == vk._true { if ctx.graphics_family == vk.queue_family_ignored { ctx.graphics_family = u32(i) - ctx.queue_family_indices << ctx.graphics_family } } - if queue_family.queueCount > 0 && (queue_family.queueFlags & vk.QueueFlags(vk.QueueFlagBits.video_decode)) != 0 { + if queue_family.queueCount > 0 + && (queue_family.queueFlags & vk.QueueFlags(vk.QueueFlagBits.video_decode)) != 0 { if ctx.video_decode_family == vk.queue_family_ignored { // H264 if (int(ctx.queue_families[i].properties_video.videoCodecOperations) & int(vk.VideoCodecOperationFlagBitsKHR.decode_h264)) != 0 { ctx.video_decode_family = u32(i) - ctx.queue_family_indices << ctx.video_decode_family } } } @@ -221,9 +227,8 @@ fn (mut ctx DeviceContext) initialize_device(use_gpu_index u32, h264_profile_idc } } - mut active_device_extensions := [vk.khr_swapchain_extension_name, - vk.khr_video_queue_extension_name, vk.khr_video_decode_queue_extension_name, - vk.khr_video_decode_h264_extension_name] + mut active_device_extensions := [vk.khr_swapchain_extension_name, vk.khr_video_queue_extension_name, + vk.khr_video_decode_queue_extension_name, vk.khr_video_decode_h264_extension_name] memory_budget_supported := vkmem.supports_memory_budget(gpu) if memory_budget_supported { active_device_extensions << vk.ext_memory_budget_extension_name @@ -253,7 +258,7 @@ fn (mut ctx DeviceContext) initialize_device(use_gpu_index u32, h264_profile_idc } panic(error_msg) } - C.volkLoadDevice(ctx.vk_device) + vk.load_device_commands(ctx.vk_device) vk.get_device_queue(ctx.vk_device, ctx.graphics_family, 0, &ctx.graphics_queue) if isnil(ctx.graphics_queue) { @@ -277,13 +282,16 @@ fn (mut ctx DeviceContext) initialize_device(use_gpu_index u32, h264_profile_idc ctx.video_decode_capabilities.pNext = &ctx.video_decode_h264.capabilities - capabilities_result := vk.get_physical_device_video_capabilities_khr(gpu, &ctx.video_profile_info, mut &ctx.video_capabilities) + capabilities_result := vk.get_physical_device_video_capabilities_khr(gpu, + &ctx.video_profile_info, mut &ctx.video_capabilities) if capabilities_result != .success { panic('Could not query selected GPU H.264 video capabilities: ${capabilities_result}') } - ctx.video_decode_bitstream_alignment = math.max(ctx.video_decode_bitstream_alignment, ctx.video_capabilities.minBitstreamBufferOffsetAlignment) - ctx.video_decode_bitstream_alignment = math.max(ctx.video_decode_bitstream_alignment, ctx.video_capabilities.minBitstreamBufferSizeAlignment) + ctx.video_decode_bitstream_alignment = math.max(ctx.video_decode_bitstream_alignment, + ctx.video_capabilities.minBitstreamBufferOffsetAlignment) + ctx.video_decode_bitstream_alignment = math.max(ctx.video_decode_bitstream_alignment, + ctx.video_capabilities.minBitstreamBufferSizeAlignment) allocator_create_info := vkmem.AllocatorCreateInfo{ physical_device: ctx.get_gpu_current() @@ -297,7 +305,8 @@ fn (mut ctx DeviceContext) initialize_device(use_gpu_index u32, h264_profile_idc pProfiles: &ctx.video_profile_info } decode_output_usage := vk.ImageUsageFlags(u32(vk.ImageUsageFlagBits.video_decode_dst) | u32(vk.ImageUsageFlagBits.transfer_src)) - decode_output_format := query_video_format(ctx.get_gpu_current(), &profile_list, decode_output_usage) or { + decode_output_format := query_video_format(ctx.get_gpu_current(), &profile_list, + decode_output_usage) or { panic('No Vulkan Video decode-output format supports transfer to the display image') } @@ -321,7 +330,8 @@ fn (mut ctx DeviceContext) initialize_device(use_gpu_index u32, h264_profile_idc forceExplicitReconstruction: 0 } - conversion_result := vk.create_sampler_ycbcr_conversion(ctx.vk_device, &sampler_ycbcr_conversion_ci, unsafe { nil }, &ctx.sampler_ycbcr_conversion) + conversion_result := vk.create_sampler_ycbcr_conversion(ctx.vk_device, + &sampler_ycbcr_conversion_ci, unsafe { nil }, &ctx.sampler_ycbcr_conversion) if conversion_result != .success { panic('Could not create metadata-aware Vulkan YCbCr conversion: ${conversion_result}') } @@ -402,11 +412,11 @@ fn missing_device_extensions(gpu vk.PhysicalDevice, required_extensions []&u8) [ return missing } -fn (ctx DeviceContext) h264_decode_gpu_diagnostics(h264_profile_idc u32) []string { - return ctx.h264_decode_gpu_diagnostics_for_output_mode(h264_profile_idc, .automatic) +fn (ctx DeviceContext) h264_decode_gpu_diagnostics(requirements VideoDecodeRequirements) []string { + return ctx.h264_decode_gpu_diagnostics_for_output_mode(requirements, .automatic) } -fn (ctx DeviceContext) h264_decode_gpu_diagnostics_for_output_mode(h264_profile_idc u32, +fn (ctx DeviceContext) h264_decode_gpu_diagnostics_for_output_mode(requirements VideoDecodeRequirements, output_mode DecodeOutputMode) []string { required_extensions := [vk.khr_swapchain_extension_name, vk.khr_video_queue_extension_name, vk.khr_video_decode_queue_extension_name, vk.khr_video_decode_h264_extension_name] @@ -421,14 +431,18 @@ fn (ctx DeviceContext) h264_decode_gpu_diagnostics_for_output_mode(h264_profile_ missing := missing_device_extensions(gpu, required_extensions) if missing.len > 0 { diagnostics << '${name}: missing ${missing.join(', ')}' - } else if !gpu_supports_h264_profile(gpu, h264_profile_idc) { - diagnostics << '${name}: H.264 ${h264_profile_name(h264_profile_idc)} Profile, 8-bit 4:2:0 progressive decode is not supported' - } else if !gpu_supports_h264_output_mode(gpu, h264_profile_idc, output_mode) { - diagnostics << '${name}: requested ${decode_output_mode_name(output_mode)} DPB/output mode is unavailable; supports ${gpu_h264_output_mode_names(gpu, h264_profile_idc)}' - } else if device_has_required_queues(ctx, gpu) { - diagnostics << '${name}: compatible (DPB/output: ${gpu_h264_output_mode_names(gpu, h264_profile_idc)})' - } else { + } else if !gpu_supports_h264_profile(gpu, requirements.profile_idc) { + diagnostics << '${name}: H.264 ${h264_profile_name(requirements.profile_idc)} Profile, 8-bit 4:2:0 progressive decode is not supported' + } else if !gpu_supports_h264_output_mode(gpu, requirements.profile_idc, output_mode) { + diagnostics << '${name}: requested ${decode_output_mode_name(output_mode)} DPB/output mode is unavailable; supports ${gpu_h264_output_mode_names(gpu, + requirements.profile_idc)}' + } else if !device_has_required_queues(ctx, gpu) { diagnostics << '${name}: required extensions/profile exist, but no compatible graphics, presentation, and decode queue combination was found' + } else if gpu_h264_stream_issue(gpu, requirements, output_mode) != '' { + diagnostics << '${name}: ${gpu_h264_stream_issue(gpu, requirements, output_mode)}' + } else { + diagnostics << '${name}: compatible (DPB/output: ${gpu_h264_output_mode_names(gpu, + requirements.profile_idc)})' } } return diagnostics @@ -438,12 +452,12 @@ fn (ctx DeviceContext) gpu_count() int { return ctx.gpus.len } -fn (ctx DeviceContext) is_h264_decode_gpu_compatible(gpu_index int, h264_profile_idc u32) bool { - return ctx.is_h264_decode_gpu_compatible_for_output_mode(gpu_index, h264_profile_idc, .automatic) +fn (ctx DeviceContext) is_h264_decode_gpu_compatible(gpu_index int, requirements VideoDecodeRequirements) bool { + return ctx.is_h264_decode_gpu_compatible_for_output_mode(gpu_index, requirements, .automatic) } fn (ctx DeviceContext) is_h264_decode_gpu_compatible_for_output_mode(gpu_index int, - h264_profile_idc u32, + requirements VideoDecodeRequirements, output_mode DecodeOutputMode) bool { if gpu_index < 0 || gpu_index >= ctx.gpus.len { return false @@ -453,8 +467,9 @@ fn (ctx DeviceContext) is_h264_decode_gpu_compatible_for_output_mode(gpu_index i vk.khr_video_decode_queue_extension_name, vk.khr_video_decode_h264_extension_name] return device_supports_extensions(gpu, required_extensions) && device_has_required_queues(ctx, gpu) - && gpu_supports_h264_profile(gpu, h264_profile_idc) - && gpu_supports_h264_output_mode(gpu, h264_profile_idc, output_mode) + && gpu_supports_h264_profile(gpu, requirements.profile_idc) + && gpu_supports_h264_output_mode(gpu, requirements.profile_idc, output_mode) + && gpu_h264_stream_issue(gpu, requirements, output_mode) == '' } fn device_has_required_queues(ctx &DeviceContext, gpu vk.PhysicalDevice) bool { @@ -464,7 +479,9 @@ fn device_has_required_queues(ctx &DeviceContext, gpu vk.PhysicalDevice) bool { mut video_props := []vk.QueueFamilyVideoPropertiesKHR{len: int(family_count), init: vk.QueueFamilyVideoPropertiesKHR{}} mut family_props := []vk.QueueFamilyProperties2{len: int(family_count), init: vk.QueueFamilyProperties2{}} for i in 0 .. family_count { - unsafe { family_props[i].pNext = &video_props[i] } + unsafe { + family_props[i].pNext = &video_props[i] + } } mut props_data := family_props.data vk.get_physical_device_queue_family_properties2(gpu, &family_count, mut props_data) @@ -476,26 +493,25 @@ fn device_has_required_queues(ctx &DeviceContext, gpu vk.PhysicalDevice) bool { continue } mut supports_present := vk.Bool32(0) - vk.get_physical_device_surface_support_khr(gpu, u32(i), ctx.swapchain.surface, &supports_present) - has_graphics_and_present = has_graphics_and_present || ((queue.queueFlags & vk.QueueFlags(vk.QueueFlagBits.graphics)) != 0 && supports_present == vk._true) - has_h264_decode = has_h264_decode || ((queue.queueFlags & vk.QueueFlags(vk.QueueFlagBits.video_decode)) != 0 && (video_props[i].videoCodecOperations & vk.VideoCodecOperationFlagsKHR(vk.VideoCodecOperationFlagBitsKHR.decode_h264)) != 0) + vk.get_physical_device_surface_support_khr(gpu, u32(i), ctx.swapchain.surface, + &supports_present) + has_graphics_and_present = has_graphics_and_present + || ((queue.queueFlags & vk.QueueFlags(vk.QueueFlagBits.graphics)) != 0 + && supports_present == vk._true) + has_h264_decode = has_h264_decode + || ((queue.queueFlags & vk.QueueFlags(vk.QueueFlagBits.video_decode)) != 0&& (video_props[i].videoCodecOperations & vk.VideoCodecOperationFlagsKHR(vk.VideoCodecOperationFlagBitsKHR.decode_h264)) != 0) } return has_graphics_and_present && has_h264_decode } -fn (ctx DeviceContext) find_h264_decode_gpu(h264_profile_idc u32) ?u32 { - return ctx.find_h264_decode_gpu_for_output_mode(h264_profile_idc, .automatic) +fn (ctx DeviceContext) find_h264_decode_gpu(requirements VideoDecodeRequirements) ?u32 { + return ctx.find_h264_decode_gpu_for_output_mode(requirements, .automatic) } -fn (ctx DeviceContext) find_h264_decode_gpu_for_output_mode(h264_profile_idc u32, +fn (ctx DeviceContext) find_h264_decode_gpu_for_output_mode(requirements VideoDecodeRequirements, output_mode DecodeOutputMode) ?u32 { - required_extensions := [vk.khr_swapchain_extension_name, vk.khr_video_queue_extension_name, - vk.khr_video_decode_queue_extension_name, vk.khr_video_decode_h264_extension_name] - for gpu_index, gpu in ctx.gpus { - if device_supports_extensions(gpu, required_extensions) - && device_has_required_queues(ctx, gpu) - && gpu_supports_h264_profile(gpu, h264_profile_idc) - && gpu_supports_h264_output_mode(gpu, h264_profile_idc, output_mode) { + for gpu_index, _ in ctx.gpus { + if ctx.is_h264_decode_gpu_compatible_for_output_mode(gpu_index, requirements, output_mode) { return u32(gpu_index) } } @@ -507,6 +523,69 @@ fn gpu_supports_h264_profile(gpu vk.PhysicalDevice, h264_profile_idc u32) bool { return true } +fn gpu_h264_stream_issue(gpu vk.PhysicalDevice, requirements VideoDecodeRequirements, + requested_mode DecodeOutputMode) string { + mut h264_profile := vk.VideoDecodeH264ProfileInfoKHR{ + stdProfileIdc: unsafe { vk.StdVideoH264ProfileIdc(requirements.profile_idc) } + pictureLayout: .progressive + } + mut profile := vk.VideoProfileInfoKHR{ + pNext: &h264_profile + videoCodecOperation: .decode_h264 + chromaSubsampling: vk.VideoChromaSubsamplingFlagsKHR(vk.VideoChromaSubsamplingFlagBitsKHR._420) + lumaBitDepth: vk.VideoComponentBitDepthFlagsKHR(vk.VideoComponentBitDepthFlagBitsKHR._8) + chromaBitDepth: vk.VideoComponentBitDepthFlagsKHR(vk.VideoComponentBitDepthFlagBitsKHR._8) + } + mut h264_caps := vk.VideoDecodeH264CapabilitiesKHR{} + mut decode_caps := vk.VideoDecodeCapabilitiesKHR{ + pNext: &h264_caps + } + mut caps := vk.VideoCapabilitiesKHR{ + pNext: &decode_caps + } + if vk.get_physical_device_video_capabilities_khr(gpu, &profile, mut &caps) != .success { + return 'could not query H.264 decode capabilities' + } + level_issue := h264_level_issue(requirements.level_idc, h264_caps.maxLevelIdc) + if level_issue != '' { + return level_issue + } + if requirements.width < caps.minCodedExtent.width + || requirements.height < caps.minCodedExtent.height + || requirements.width > caps.maxCodedExtent.width + || requirements.height > caps.maxCodedExtent.height { + return 'coded extent ${requirements.width}x${requirements.height} is outside supported ${caps.minCodedExtent.width}x${caps.minCodedExtent.height}–${caps.maxCodedExtent.width}x${caps.maxCodedExtent.height}' + } + if requirements.dpb_slots > caps.maxDpbSlots { + return 'video requires ${requirements.dpb_slots} DPB slots; device supports ${caps.maxDpbSlots}' + } + if requirements.references > caps.maxActiveReferencePictures { + return 'video requires ${requirements.references} active references; device supports ${caps.maxActiveReferencePictures}' + } + supports_coincident := (decode_caps.flags & vk.VideoDecodeCapabilityFlagsKHR(vk.VideoDecodeCapabilityFlagBitsKHR.dpb_and_output_coincide)) != 0 + supports_distinct := (decode_caps.flags & vk.VideoDecodeCapabilityFlagsKHR(vk.VideoDecodeCapabilityFlagBitsKHR.dpb_and_output_distinct)) != 0 + mode := select_decode_output_mode(requested_mode, supports_coincident, supports_distinct) or { + return 'requested DPB/output mode is unavailable' + } + profile_list := vk.VideoProfileListInfoKHR{ + profileCount: 1 + pProfiles: &profile + } + output_usage := vk.ImageUsageFlags(u32(vk.ImageUsageFlagBits.video_decode_dst) | u32(vk.ImageUsageFlagBits.transfer_src)) + _ := query_video_format(gpu, &profile_list, output_usage) or { + return 'no decode-output format supports transfer to the display image' + } + dpb_usage := if mode == .coincident { + vk.ImageUsageFlags(u32(vk.ImageUsageFlagBits.video_decode_dpb) | u32(vk.ImageUsageFlagBits.video_decode_dst) | u32(vk.ImageUsageFlagBits.transfer_src)) + } else { + vk.ImageUsageFlags(vk.ImageUsageFlagBits.video_decode_dpb) + } + _ := query_video_format(gpu, &profile_list, dpb_usage) or { + return 'no DPB format supports the selected decode mode' + } + return '' +} + fn gpu_h264_decode_capability_flags(gpu vk.PhysicalDevice, h264_profile_idc u32) ?vk.VideoDecodeCapabilityFlagsKHR { mut h264_profile := vk.VideoDecodeH264ProfileInfoKHR{ @@ -566,112 +645,6 @@ fn (mut ctx DeviceContext) initialize_swapchain(window_p &glfw.Window, desired_f return ctx.swapchain.initialize(window_p, desired_format) } -fn (ctx DeviceContext) create_buffer(desc &GPUBufferDesc, buffer &GPUBuffer) { - // TODO - panic('Not implemented') -} - -fn (mut ctx DeviceContext) create_image(desc &GPUImageDesc, mut image &GPUImage) { - mut n := unsafe { nil } - mut image_ci := vk.ImageCreateInfo{ - flags: 0 - imageType: desc.image_type - format: desc.format - extent: desc.extent - mipLevels: desc.mip_levels - arrayLayers: desc.array_size - samples: vk.SampleCountFlagBits._1 - tiling: vk.ImageTiling.optimal - usage: u32(desc.usage) - sharingMode: vk.SharingMode.exclusive - queueFamilyIndexCount: 0 - pQueueFamilyIndices: unsafe { nil } - initialLayout: vk.ImageLayout.undefined - } - image_ci.usage = vk.ImageUsageFlags(u32(vk.BufferUsageFlagBits.transfer_src) | u32(vk.BufferUsageFlagBits.transfer_dst)) - - mut profile_list_info := vk.VideoProfileListInfoKHR{ - profileCount: 1 - pProfiles: &ctx.video_profile_info - } - if desc.usage & (u32(vk.ImageUsageFlagBits.video_decode_dst) | u32(vk.ImageUsageFlagBits.video_decode_src) | u32(vk.ImageUsageFlagBits.video_decode_dpb)) != 0 { - image_ci.pNext = &profile_list_info - - mut video_format_info := vk.PhysicalDeviceVideoFormatInfoKHR{ - pNext: &profile_list_info - imageUsage: image_ci.usage - } - mut format_count := u32(0) - vk.get_physical_device_video_format_properties_khr(ctx.get_gpu_current(), &video_format_info, &format_count, mut n) - if format_count > 0 { - mut video_formats := []vk.VideoFormatPropertiesKHR{len: int(format_count)} - res := vk.get_physical_device_video_format_properties_khr(ctx.get_gpu_current(), &video_format_info, &format_count, mut video_formats[0]) - if res != vk.Result.success { - panic('Could not get device video format properties') - } - } - } - - if ctx.queue_family_indices.len > 1 { - image_ci.sharingMode = vk.SharingMode.concurrent - image_ci.queueFamilyIndexCount = u32(ctx.queue_family_indices.len) - image_ci.pQueueFamilyIndices = ctx.queue_family_indices.data - } - - vk.create_image(ctx.vk_device, &image_ci, unsafe { nil }, &image.image) - - mut reqs := vk.MemoryRequirements2{} - mut info := vk.ImageMemoryRequirementsInfo2{ - image: image.image - } - vk.get_image_memory_requirements2(ctx.vk_device, &info, mut &reqs) - - mut alloc_info := vk.MemoryAllocateInfo{ - allocationSize: reqs.memoryRequirements.size - memoryTypeIndex: ctx.get_memory_type_index(reqs, vk.MemoryPropertyFlags(desc.memory_property)) - } - vk.allocate_memory(ctx.vk_device, &alloc_info, unsafe { nil }, &image.memory) - vk.bind_image_memory(ctx.vk_device, image.image, image.memory, 0) - - mut view_ci := vk.ImageViewCreateInfo{ - flags: 0 - image: image.image - viewType: vk.ImageViewType._2d - format: image_ci.format - // NOTE: int(identity) = 0 - components: vk.ComponentMapping{ - r: vk.ComponentSwizzle.identity - g: vk.ComponentSwizzle.identity - b: vk.ComponentSwizzle.identity - a: vk.ComponentSwizzle.identity - } - subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - baseMipLevel: 0 - levelCount: image_ci.mipLevels - baseArrayLayer: 0 - layerCount: image_ci.arrayLayers - } - } - - view_ci.viewType = match desc.image_type { - ._1d { - vk.ImageViewType._1d - } - ._2d { - vk.ImageViewType._2d - } - ._3d { - vk.ImageViewType._3d - } - else { - vk.ImageViewType._2d - } - } - - vk.create_image_view(ctx.vk_device, &view_ci, unsafe { nil }, &image.image_view) -} - fn (ctx DeviceContext) submit(queue_type QueueType, mut p_submit_info vk.SubmitInfo, mut wait_fence vk.Fence) { mut queue := match queue_type { .graphics { ctx.graphics_queue } @@ -705,8 +678,8 @@ fn (ctx DeviceContext) get_queue(type QueueType) vk.Queue { } fn (mut ctx DeviceContext) initialize_vk_instance() bool { - if C.volkInitialize() != vk.Result.success { - panic('Could not volkInitialize()') + if vk.initialize_loader() != vk.Result.success { + panic('Could not initialize Vulkan loader') } mut n := unsafe { nil } mut instance_extension_count := u32(0) @@ -714,7 +687,8 @@ fn (mut ctx DeviceContext) initialize_vk_instance() bool { mut instance_extensions := []vk.ExtensionProperties{len: int(instance_extension_count)} // Temporary variable to make data mutable mut instance_extensions_data := instance_extensions.data - vk.enumerate_instance_extension_properties(unsafe { nil }, &instance_extension_count, mut instance_extensions_data) + vk.enumerate_instance_extension_properties(unsafe { nil }, &instance_extension_count, mut + instance_extensions_data) mut active_instance_extensions := []&u8{} mut glfw_required_count := u32(0) @@ -765,9 +739,10 @@ fn (mut ctx DeviceContext) initialize_vk_instance() bool { if res != .success { panic('Could not create vkInstance') } - C.volkLoadInstance(ctx.vk_instance) + vk.load_instance_commands(ctx.vk_instance) $if debug ? { - resdbg := vk.create_debug_utils_messenger_ext(ctx.vk_instance, &debug_utils_create_info, unsafe { nil }, &ctx.vk_debug_utils) + resdbg := vk.create_debug_utils_messenger_ext(ctx.vk_instance, &debug_utils_create_info, + unsafe { nil }, &ctx.vk_debug_utils) if resdbg != vk.Result.success { panic('Could not create DebugUtilsMessengerEXT') } @@ -782,35 +757,6 @@ fn (mut ctx DeviceContext) enumerate_gpus() { int_gpu_count := int(gpu_count) ctx.gpus = unsafe { []vk.PhysicalDevice{len: int_gpu_count} } vk.enumerate_physical_devices(ctx.vk_instance, &gpu_count, ctx.gpus.data) - - ctx.physical_device_memory_props = []vk.PhysicalDeviceMemoryProperties2{len: int(gpu_count), init: vk.PhysicalDeviceMemoryProperties2{ - memoryProperties: vk.PhysicalDeviceMemoryProperties{ - memoryTypes: [vk.max_memory_types]vk.MemoryType{} - memoryHeaps: [vk.max_memory_heaps]vk.MemoryHeap{} - } - }} - for i in 0 .. gpu_count { - vk.get_physical_device_memory_properties2(ctx.gpus[i], mut &ctx.physical_device_memory_props[i]) - } -} - -fn (ctx DeviceContext) get_memory_type_index(reqs vk.MemoryRequirements2, flags vk.MemoryPropertyFlags) u32 { - mut request_bits := reqs.memoryRequirements.memoryTypeBits - memory_props := ctx.physical_device_memory_props[ctx.use_gpu_index].memoryProperties - for i in 0 .. memory_props.memoryTypeCount { - // Match wanted memory properties - if (int(request_bits) & 1) != 0 { - if (memory_props.memoryTypes[i].propertyFlags & flags) == flags { - return i - } - } - request_bits >>= 1 - } - return max_u32 -} - -fn (ctx DeviceContext) get_gpu(gpu_index int) vk.PhysicalDevice { - return ctx.gpus[gpu_index] } fn (ctx DeviceContext) get_gpu_current() vk.PhysicalDevice { @@ -818,7 +764,6 @@ fn (ctx DeviceContext) get_gpu_current() vk.PhysicalDevice { return ctx.gpus[ctx.use_gpu_index] } -// TODO: Do we want getters/setters? fn (ctx DeviceContext) get_vk_device() vk.Device { return ctx.vk_device } diff --git a/docs/guide/01-input-and-order.md b/docs/guide/01-input-and-order.md index 1398741..9b1c146 100644 --- a/docs/guide/01-input-and-order.md +++ b/docs/guide/01-input-and-order.md @@ -7,8 +7,8 @@ assumed profile. ## Trace the input -[`VideoPlayer.prepare`](../../video_player.v#L415) calls -[`Decoder.parse_mp4_data`](../../mp4_parser.v#L100). The parser finds an H.264 +[`VideoPlayer.prepare`](../../video_player.v#L579) calls +[`Decoder.parse_mp4_data`](../../mp4_parser.v#L252). The parser finds an H.264 track, checks its timescale and samples, reads SPS and PPS data, and records picture dimensions, profile, timing, references, and display metadata. MP4 gives sample offsets and durations; H.264 headers give codec rules such as @@ -18,13 +18,44 @@ efficiency: an invalid file should not leave a half-created GPU decoder. The implementation also checks file size and reads at absolute sample offsets through [`read_callback`](../../mp4_parser.v#L17). Tests cover -[non-MP4 input](../../video_player_test.v#L214), -[truncation](../../video_player_test.v#L253), and -[short reads](../../video_player_test.v#L233). In another project, input +[non-MP4 input](../../video_player_test.v#L557), +[truncation](../../video_player_test.v#L596), and +[short reads](../../video_player_test.v#L576). In another project, input could instead be a network segment or a camera stream. The boundary remains useful: turn untrusted bytes into validated stream requirements before asking the device to allocate resources. +The [SPS preflight](../../h264_parameter_sets.v#L225) and +[PPS preflight](../../h264_parameter_sets.v#L300) check syntax length, reference +counts, bit widths, and fixed-array limits before calling the pinned H.264 +parser. Unsupported slice groups fail with a clear error. The parser +[resolves parameter sets by H.264 ID](../../h264_slice.v#L5), and the player +[maps those IDs to serialized offsets](../../video_player.v#L442) when recording +[decode commands](../../player_decode.v#L49). A legal nonzero ID therefore +does not have to equal its position in the MP4 parameter-set list. The +[ID mapping test](../../video_player_test.v#L170) and +[truncation and mutation test](../../video_player_test.v#L190) exercise these +boundaries without a GPU. A [five-frame fixture test](../../video_player_test.v#L672) +also parses SPS, PPS, and slices whose IDs are all 7 while each parameter +set is the first entry in its MP4 list. + +MP4 AVC samples store each NAL with a length prefix. The demux binding does +not expose the avcC prefix width, so the parser [detects a complete 1, 2, or +4 byte layout](../../mp4_parser.v#L90) in the first sample and uses that width +for both [parsing](../../mp4_parser.v#L528) and +[GPU upload](../../player_decode.v#L562). Samples containing only metadata +are [left out of the picture list](../../mp4_parser.v#L714), keeping slice +headers aligned with decode indices. The +[checked slice reader](../../h264_slice.v#L122) honors weighted prediction +reference counts and rejects invalid reference marking; the pinned H.264 +dependency's reader does not consume the full weighted table. +The parser [counts every slice's Annex B bytes](../../mp4_parser.v#L578) +before sizing the upload buffer; several short MP4 length prefixes can expand +into several four-byte start codes. +It [checks each later slice](../../mp4_parser.v#L131) against the first slice's +picture identity. The [four-slice fixture test](../../video_player_test.v#L504) +exercises a complete access unit. + ## Two orders, two jobs H.264 B-pictures can be displayed before a reference picture that must be @@ -38,15 +69,28 @@ flowchart LR B --> C[Display: 0, 1, 2, 3, 4, 5, 6] ``` -[`parse_mp4_data`](../../mp4_parser.v#L501) assigns a display order to each +[the display-order pass](../../mp4_parser.v#L728) assigns a display order to each picture while preserving decode order for the decoder. The DPB retains reference pictures for the codec; the output-image queue retains decoded pictures waiting for presentation. Those are different lifetimes. The -[`presentation_buffer_size`](../../video_player.v#L253) calculation looks at the +[`presentation_buffer_size`](../../video_player.v#L397) calculation looks at the stream's display-order sequence to bound the waiting queue. The fixture-based -tests assert the [early sequence](../../video_player_test.v#L192) and +tests assert the [early sequence](../../video_player_test.v#L520) and [required queue depth](../../playback_timeline_test.v#L59). +An H.264 MMCO 5 picture resets reference-picture numbering after it is decoded. +The parser [detects the operation](../../mp4_parser.v#L190) and +[starts a new display-order group](../../mp4_parser.v#L221), while retaining +the picture's original count for the decode command. The +[ordering test](../../video_player_test.v#L300) covers a reset followed by a +picture-order-count wrap. This separation matters whenever a codec resets its +reference state without starting a new file or decoder session. + +The parser also computes [POC type 1](../../mp4_parser.v#L157) from the SPS +reference cycle, reference status, and slice deltas. The +[unit case](../../video_player_test.v#L219) shows why a nonreference picture can +have a different count from a reference picture with the same frame number. + **Invariant:** decode input advances in codec order; presentation advances only when the next display-order picture is ready. The next display-order number does not have to equal the current decode index. diff --git a/docs/guide/02-capabilities.md b/docs/guide/02-capabilities.md index 0c35f2f..7c2ff56 100644 --- a/docs/guide/02-capabilities.md +++ b/docs/guide/02-capabilities.md @@ -7,17 +7,39 @@ The chosen MP4 may require a profile that another file does not. ## The selection path -After parsing, [`VideoDecodeApp.initialize`](../../app.v#L179) obtains the stream's -profile and asks [`h264_decode_gpu_diagnostics_for_output_mode`](../../device_context.v#L409) -for diagnostics for every GPU. [`--list-gpus`](../../app.v#L182) exposes those -diagnostics to the user. A forced [`--gpu` index](../../app.v#L197) is checked +After parsing, [`VideoDecodeApp.initialize`](../../app.v#L147) obtains the stream's +[decode requirements](../../video_player.v#L606): profile, level, coded extent, +DPB slots, and active references. It asks +[`h264_decode_gpu_diagnostics_for_output_mode`](../../device_context.v#L419) +for diagnostics for every GPU. [`--list-gpus`](../../app.v#L187) exposes those +diagnostics to the user. A forced [`--gpu` index](../../app.v#L202) is checked against the same requirements; otherwise the first compatible device is chosen. Errors name the missing capability instead of assuming that a graphics-capable GPU can decode. -[`initialize_device`](../../device_context.v#L150) then chooses queue families and +The [stream probe](../../device_context.v#L526) checks the queried H.264 level, +coded extent, DPB and active reference limits, selected output mode, and output/DPB +formats with the image usages the session will actually create. It runs before +logical device creation, so a second GPU can be tried when the first one +cannot satisfy the particular video. + +The SPS stores level 4.0 as `level_idc = 40`; Vulkan's +[`StdVideoH264LevelIdc` conversion](../../device_context.v#L52) maps it to +the enum value used in session parameters. The +[level check](../../device_context.v#L77) compares that enum against the +device's `maxLevelIdc`, and reports the required and supported levels before +creating a session. + +The [SPS macroblock dimensions](../../mp4_parser.v#L143) supply the coded +extent, which can be larger than the visible image after H.264 cropping. +The [session extent](../../decoder_session.v#L116) and +[decode picture resources](../../player_decode.v#L469) use those coded +dimensions; the [display copy](../../player_decode.v#L272) uses the visible +dimensions. + +[`initialize_device`](../../device_context.v#L154) then chooses queue families and creates the logical device with the required extensions. It builds a -[VideoProfileInfoKHR](../../device_context.v#L267) for progressive 8-bit 4:2:0 H.264 and chains H.264 +[VideoProfileInfoKHR](../../device_context.v#L272) for progressive 8-bit 4:2:0 H.264 and chains H.264 profile and capability structs through `pNext`. Vulkan Video format queries use the same profile. A format is useful only if it supports the image usages required by the next step, including transfer out of the decoded picture in @@ -27,18 +49,18 @@ The player accepts the two advertised DPB/output modes. In *coincident* mode, the decoded output is a DPB image. In *distinct* mode, the output and DPB images are separate. `auto` prefers coincident and falls back to distinct; forced modes aid driver validation and fail if unsupported. The choice is made -by [`select_decode_output_mode`](../../video_player.v#L17), with software tests -for [automatic fallback](../../video_player_test.v#L23) and -[forced modes](../../video_player_test.v#L28). +by [`select_decode_output_mode`](../../video_player.v#L18), with software tests +for [automatic fallback](../../video_player_test.v#L334) and +[forced modes](../../video_player_test.v#L339). ```mermaid flowchart TD Stream[Parsed H.264 profile and dimensions] --> Extensions[Required extensions] Extensions --> Queues[Presenting graphics and H.264 decode queues] Queues --> Profile[Profile capabilities and limits] - Profile --> Formats[Output and DPB formats with required usages] - Formats --> Mode[Coincident or distinct output mode] - Mode --> Device[Create device and decoder] + Profile --> Mode[Coincident or distinct output mode] + Mode --> Formats[Output and DPB formats with required usages] + Formats --> Device[Create device and decoder] ``` **Invariant:** the profile and image usage used for capability queries must diff --git a/docs/guide/03-decode-resources.md b/docs/guide/03-decode-resources.md index 800e5bb..e55ac76 100644 --- a/docs/guide/03-decode-resources.md +++ b/docs/guide/03-decode-resources.md @@ -7,27 +7,45 @@ remain valid while commands are in flight. ## Build the session from parsed limits -[`Decoder.initialize`](../../decoder_session.v#L8) asks the device for H.264 -capabilities and supported formats. It checks the parsed DPB slot and active -reference counts against device limits. It allocates an aligned bitstream -buffer, [creates a `VideoSessionKHR`](../../decoder_session.v#L119), queries its opaque memory requirements, +[`Decoder.initialize`](../../decoder_session.v#L8) asks the selected device for H.264 +capabilities and supported formats. Device selection has already +[checked the stream's extent and reference limits](../../device_context.v#L526). +Session setup confirms those values and allocates an aligned bitstream +buffer, [creates a `VideoSessionKHR`](../../decoder_session.v#L125), queries its opaque memory requirements, binds that memory, and creates session parameters from SPS/PPS data. The alignment used for each upload slot comes from the queried minimum bitstream -offset and size alignments. [`write_video_frame`](../../player_decode.v#L521) fills +offset and size alignments. [`write_video_frame`](../../player_decode.v#L562) fills one slot with the next access unit and its slice offsets. The DPB stores reference pictures for later H.264 predictions. A picture that is no longer needed as a reference may free a DPB slot even if its display time has not arrived. Conversely, a displayed picture may still be referenced by later decode operations. This is why the app copies decode results into a -separate bounded pool of [`OutputImage`](../../video_player.v#L173) objects, -[allocated after decoder setup](../../video_player.v#L521). The +separate bounded pool of [`OutputImage`](../../video_player.v#L311) objects, +[allocated after decoder setup](../../video_player.v#L849). The graphics side samples that pool, not a DPB slot whose codec lifetime it does not control. +The [DPB marking helper](../../video_player.v#L218) applies sliding-window +marking and MMCO 1–6, including long-term indices. The player invokes it +[after recording the decode](../../player_decode.v#L160), so the current +picture still sees the old references. MMCO 5 clears older references and +renumbers the current picture. For a long-term reference, the +[Vulkan slot information](../../player_decode.v#L476) carries its index in +`FrameNum` and sets `used_for_long_term_reference`. The +[software marking test](../../video_player_test.v#L239) covers removal, +conversion, and long-term limits. + +Progressive H.264 pictures still have separate top and bottom order counts. +The [decode command](../../player_decode.v#L107) passes both to Vulkan, and +the [DPB slot data](../../player_decode.v#L75) retains both for later references. +An MMCO 5 picture uses its original counts for the current decode, then +[normalizes its stored reference](../../video_player.v#L173) for subsequent +pictures. + In coincident mode, the copy source is the current DPB image. In distinct mode, the copy source is a separate decode-output image; the DPB remains -reference storage. [`copy_decoded_frame_to_output`](../../player_decode.v#L187) +reference storage. [`copy_decoded_frame_to_output`](../../player_decode.v#L205) selects the correct source and restores its decode layout after the copy. ```mermaid @@ -56,5 +74,5 @@ Those rules are related but have different owners. For another project, calculate the bound from reorder depth, concurrent uploads, and frames still sampled by graphics. A fixed pool is safe only if the producer pauses when no image is reusable. Here -[`VideoPlayer.update`](../../player_presentation.v#L127) checks the free pool before +[the free-pool check](../../player_presentation.v#L128) pauses before starting another decode. diff --git a/docs/guide/04-synchronization.md b/docs/guide/04-synchronization.md index 0120c9b..0831169 100644 --- a/docs/guide/04-synchronization.md +++ b/docs/guide/04-synchronization.md @@ -6,21 +6,21 @@ chapter follows the synchronization path and the output-image reuse rule. ## Follow one submission -[`update_decode_video`](../../player_decode.v#L81) waits on the fence for the +[`update_decode_video`](../../player_decode.v#L90) waits on the fence for the bitstream upload slot it is about to reuse, then writes and flushes the encoded bytes. It records the video command buffer: prepare image layouts, begin and perform decode, copy the result to an output image, and restore the source for later decode use. The output image is tagged with display order and placed in the ready queue. -[`VideoPlayer.update`](../../player_presentation.v#L156) submits that command buffer on -the video queue and signals a semaphore. It then submits a graphics command +[the video queue submission](../../player_presentation.v#L173) sends that command buffer to +the decode queue and signals a semaphore. It then submits a graphics command buffer that waits on the semaphore and transitions the copied image for fragment sampling. The app's render command buffer waits for the player's event before drawing from the selected output view. The swapchain also has -its own [acquire](../../app.v#L373) and -[render-complete](../../app.v#L463) semaphores, managed in -[`VideoDecodeApp.run`](../../app.v#L355). +its own [acquire](../../app.v#L386) and +[render-complete](../../app.v#L480) semaphores, managed in +[`VideoDecodeApp.run`](../../app.v#L367). ```mermaid sequenceDiagram @@ -35,8 +35,8 @@ sequenceDiagram Graphics->>Graphics: Sample image and render swapchain ``` -Image barriers in [`video_decode_pre_barrier`](../../player_decode.v#L340) and -[`copy_decoded_frame_to_output`](../../player_decode.v#L187) describe access and +Image barriers in [`video_decode_pre_barrier`](../../player_decode.v#L359) and +[`copy_decoded_frame_to_output`](../../player_decode.v#L205) describe access and layout transitions. The semaphore orders work between queues. These solve different problems: a layout name alone does not wait for a prior queue's writes, and a semaphore alone does not describe the next image layout. @@ -48,7 +48,7 @@ submission still samples it. [`retire_current_output`](../../player_presentation marks that image for later reuse; [`reclaim_output_textures`](../../player_presentation.v#L14) returns it to the free pool only after enough later render submissions have begun. This design relies on the app's one fence per swapchain image and -ordered graphics submissions. [Resize](../../app.v#L485) waits for the device to become idle +ordered graphics submissions. [Resize](../../app.v#L504) waits for the device to become idle before rebuilding swapchain-dependent resources. **Invariant:** neither a mapped bitstream region nor an output image is diff --git a/docs/guide/05-presentation.md b/docs/guide/05-presentation.md index 7bbc549..327dda7 100644 --- a/docs/guide/05-presentation.md +++ b/docs/guide/05-presentation.md @@ -6,7 +6,7 @@ ratio and rotation metadata. ## Display order and time -[`VideoPlayer.update_presentation`](../../player_presentation.v#L72) asks for the next +[`VideoPlayer.update_presentation`](../../player_presentation.v#L73) asks for the next display-order image. It initially waits until enough reordered pictures are ready; then it advances only when the current duration is due and that next image exists. At end of stream it keeps the final picture visible while the @@ -29,23 +29,23 @@ reordered image can cause a burst of frames when it arrives. ## Color and geometry are media data -[`parse_mp4_data`](../../mp4_parser.v#L152) stores the track matrix and -[H.264 video usability metadata](../../mp4_parser.v#L203). -[`ycbcr_model_for_video`](../../device_context.v#L338) +[the MP4 track parse](../../mp4_parser.v#L304) stores the track matrix and +[H.264 video usability metadata](../../mp4_parser.v#L379). +[`ycbcr_model_for_video`](../../device_context.v#L348) chooses a Vulkan YCbCr model from signaled matrix coefficients, with a resolution-based fallback when no description is present. The Vulkan sampler conversion uses the video's full or limited range. The app draws into an -[`UNORM swapchain`](../../app.v#L220) so display-encoded YCbCr conversion is +[`UNORM swapchain`](../../app.v#L230) so display-encoded YCbCr conversion is not encoded as sRGB a second time. [`video_render_transform`](../../app.v#L91) applies quarter-turn rotation and letterboxing based on display dimensions. Those dimensions incorporate sample aspect ratio before rotation. The picture can therefore have a coded width, a display width, and a window width that differ. Tests cover -[track matrices](../../video_player_test.v#L47), -[sample aspect ratio](../../video_player_test.v#L54), -[conversion choices](../../video_player_test.v#L84), and -[portrait letterboxing](../../video_player_test.v#L314). +[track matrices](../../video_player_test.v#L423), +[sample aspect ratio](../../video_player_test.v#L378), +[conversion choices](../../video_player_test.v#L395), and +[portrait letterboxing](../../video_player_test.v#L657). ```mermaid flowchart LR diff --git a/docs/guide/06-lifecycle-and-verification.md b/docs/guide/06-lifecycle-and-verification.md index bfd620e..4fa2daf 100644 --- a/docs/guide/06-lifecycle-and-verification.md +++ b/docs/guide/06-lifecycle-and-verification.md @@ -9,16 +9,16 @@ without reparsing the MP4. ## Trace ownership [`main`](../../main.v#L71) owns the `VideoDecodeApp` lifetime. -[`VideoDecodeApp.initialize`](../../app.v#L145) creates the window, instance, +[`VideoDecodeApp.initialize`](../../app.v#L147) creates the window, instance, surface, compatible device, swapchain, descriptors, pipeline, ImGui backend, -and player resources. [`VideoDecodeApp.run`](../../app.v#L355) handles out-of-date -swapchain results and calls [`recreate_swapchain`](../../app.v#L485), which waits +and player resources. [`VideoDecodeApp.run`](../../app.v#L367) handles out-of-date +swapchain results and calls [`recreate_swapchain`](../../app.v#L504), which waits for idle, releases per-frame resources, resizes, and recreates the dependent -resources. [`shutdown`](../../app.v#L534) waits for the device, releases player and +resources. [`shutdown`](../../app.v#L553) waits for the device, releases player and graphics resources, then destroys the device, window, and GLFW state. Input and capability failures during initialization use -[`abort_initialization`](../../app.v#L587) and close the input file. Later Vulkan +[`abort_initialization`](../../app.v#L606) and close the input file. Later Vulkan allocation or submission failures can still be fatal; the repository does not claim to recover from every partial GPU initialization. The precise support boundary is in [Supported media and failure behavior](../../PLATFORM_SUPPORT.md#supported-media-and-failure-behavior). @@ -37,7 +37,7 @@ flowchart TD | Check | What it establishes | What it cannot establish | | --- | --- | --- | -| [`v test .`](../../README.md#tests) | CLI parsing, MP4 validation, metadata, reorder depth, and timeline rules for the test fixtures. | Driver video commands, image barriers, or visible output. | +| [`v test .`](../../README.md#tests) | CLI parsing, MP4 and parameter-set validation, multi-slice access units, metadata, reorder depth, and timeline rules. | Driver video commands, image barriers, or visible output. | | [Root executable build](../../BUILDING.md#shared-dear-imgui-default) | V/C bindings, native linking, and package entry point. | Compatible hardware or correct playback. | | [`--list-gpus VIDEO`](../../README.md#run) | The current driver advertises the required capabilities for that stream. | That a full decode and resize session succeeds. | | [Playback and resize on a supported GPU](../../PLATFORM_SUPPORT.md#hardware-validation-checklist) | The tested media and driver complete the actual path. | Other codecs, GPUs, operating systems, or long-running stability. | @@ -48,6 +48,22 @@ The [README](../../README.md#tests) has the software command; lists release checks. A useful development loop is to run software tests for every parser or timing change, then use short media fixtures with B-frames, rotation, and different color metadata on an actual decode-capable GPU. +The [four-slice fixture](../../res/README.md#test-media) exercises access-unit +assembly and picture-consistency checks; its +[parser test](../../video_player_test.v#L504) runs without a GPU. +The [reference-marking conformance streams](../../PLATFORM_SUPPORT.md#hardware-validation-checklist) +exercise MMCO 5 and long-term operations on real hardware after remuxing to +MP4. The optional [NV12 readback](../../frame_readback.v#L48) and +[comparison script](../../scripts/compare_nv12.py#L25) test decoded bytes +against FFmpeg in display order. This exposed custom H.264 scaling lists +whose values were lost by the pinned parser; the player now +[populates those lists](../../h264_parameter_sets.v#L77), and a +[software test](../../video_player_test.v#L695) covers SPS and PPS examples. +The four Linux GPU comparisons in +[Platform Support](../../PLATFORM_SUPPORT.md#supported-media-and-failure-behavior) +matched byte for byte. A clean validation-layer run checks API use; pixel +comparison checks those decoded streams on the tested GPU. Neither establishes +correctness on all drivers or in the final color-converted window image. ## Transfer the approach diff --git a/docs/learning-path.md b/docs/learning-path.md index 291aaa9..680220a 100644 --- a/docs/learning-path.md +++ b/docs/learning-path.md @@ -40,11 +40,11 @@ flowchart LR ``` The application enters at [`main`](../main.v#L71). The drawing loop is -[`VideoDecodeApp.run`](../app.v#L355), and GPU selection is in -[`find_h264_decode_gpu_for_output_mode`](../device_context.v#L490). -[`parse_mp4_data`](../mp4_parser.v#L100) reads the input, +[`VideoDecodeApp.run`](../app.v#L367), and GPU selection is in +[`find_h264_decode_gpu_for_output_mode`](../device_context.v#L511). +[`parse_mp4_data`](../mp4_parser.v#L252) reads the input, [`update_decode_video`](../player_decode.v#L7) records decode work, and -[`update_presentation`](../player_presentation.v#L72) manages output images. +[`update_presentation`](../player_presentation.v#L73) manages output images. [`PlaybackTimeline`](../playback_timeline.v#L10) tracks media time. These are source file responsibilities within one V package, not public modules. @@ -65,18 +65,18 @@ the idea understandable without reading every Vulkan call. ## A route through one picture -1. [`prepare`](../video_player.v#L415) calls the - [MP4 parser](../mp4_parser.v#L100), which rejects unsupported streams. +1. [`prepare`](../video_player.v#L579) calls the + [MP4 parser](../mp4_parser.v#L252), which rejects unsupported streams. Its profile and dimensions inform GPU selection. -2. [`initialize_device`](../device_context.v#L150) selects graphics and H.264 +2. [`initialize_device`](../device_context.v#L154) selects graphics and H.264 decode queues, queries the video profile, and chooses a compatible format. 3. [`Decoder.initialize`](../decoder_session.v#L8) creates the session, - bitstream buffer, and DPB images. [`VideoPlayer.initialize`](../video_player.v#L521) + bitstream buffer, and DPB images. [`VideoPlayer.initialize`](../video_player.v#L626) allocates the bounded output-image pool. 4. [`update_decode_video`](../player_decode.v#L7) uploads an access unit, records decode and copy commands, and tags the copied output with display order. -5. [`update_presentation`](../player_presentation.v#L72) chooses the next display-order - image when its duration is due. [`VideoDecodeApp.run`](../app.v#L355) samples it +5. [`update_presentation`](../player_presentation.v#L73) chooses the next display-order + image when its duration is due. [`VideoDecodeApp.run`](../app.v#L367) samples it and presents the swapchain image. The [queue submission trace](guide/04-synchronization.md#follow-one-submission) diff --git a/frame_readback.v b/frame_readback.v new file mode 100644 index 0000000..d2b2c77 --- /dev/null +++ b/frame_readback.v @@ -0,0 +1,154 @@ +module main + +import os +import antono2.vulkan as vk +import antono2.vkmemalloc as vkmem + +// Opt-in validation path: one host-visible NV12 buffer per in-flight decode. +struct FrameReadback { +mut: + buffer vk.Buffer + allocation vkmem.AllocationInfo + mapped byteptr = unsafe { nil } + decode_index int + display_order int + pending bool +} + +fn (mut vp VideoPlayer) initialize_frame_readback() { + vp.frame_readback_dir = os.getenv('VV_DUMP_NV12_DIR') + if vp.frame_readback_dir == '' { + return + } + os.mkdir_all(vp.frame_readback_dir) or { panic('Could not create readback directory: ${err}') } + width := vp.decoder.video_data.width + height := vp.decoder.video_data.height + if width % 2 != 0 || height % 2 != 0 { + panic('NV12 readback requires even picture dimensions') + } + buffer_size := u64(width) * u64(height) * 3 / 2 + vp.frame_readbacks = []FrameReadback{len: vp.video_frames.len} + for mut frame in vp.frame_readbacks { + buffer_ci := vk.BufferCreateInfo{ + size: buffer_size + usage: vk.BufferUsageFlags(vk.BufferUsageFlagBits.transfer_dst) + } + result := vp.app.device_context.memory_allocator.create_buffer_with_options(&buffer_ci, vkmem.AllocationOptions{ + usage: .readback + }, &frame.buffer, mut frame.allocation) + check_vk(result, 'Could not allocate decoded-frame readback buffer') + mut mapped := voidptr(unsafe { nil }) + check_vk(vp.app.device_context.memory_allocator.map(mut frame.allocation, &mapped), + 'Could not map decoded-frame readback buffer') + frame.mapped = unsafe { byteptr(mapped) } + } + eprintln('Decoded NV12 readback enabled: ${vp.frame_readback_dir}') +} + +fn (mut vp VideoPlayer) record_frame_readback(command_buffer vk.CommandBuffer, source_image vk.Image) { + if vp.frame_readback_dir == '' || vp.frame_readback_done { + return + } + mut frame := &vp.frame_readbacks[vp.current_upload_index] + width := vp.decoder.video_data.width + height := vp.decoder.video_data.height + regions := [ + vk.BufferImageCopy2{ + bufferOffset: 0 + imageSubresource: vk.ImageSubresourceLayers{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.plane0) + layerCount: 1 + } + imageExtent: vk.Extent3D{ + width: width + height: height + depth: 1 + } + }, + vk.BufferImageCopy2{ + bufferOffset: u64(width) * u64(height) + imageSubresource: vk.ImageSubresourceLayers{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.plane1) + layerCount: 1 + } + imageExtent: vk.Extent3D{ + width: width / 2 + height: height / 2 + depth: 1 + } + }, + ] + copy_info := vk.CopyImageToBufferInfo2{ + srcImage: source_image + srcImageLayout: .transfer_src_optimal + dstBuffer: frame.buffer + regionCount: u32(regions.len) + pRegions: regions.data + } + vk.cmd_copy_image_to_buffer2(command_buffer, ©_info) + family := vp.app.device_context.get_decoder_queue_family_index() + mut barrier := vk.BufferMemoryBarrier2{ + srcStageMask: vk.pipeline_stage_2_transfer_bit + srcAccessMask: vk.access_2_transfer_write_bit + dstStageMask: vk.pipeline_stage_2_host_bit + dstAccessMask: vk.access_2_host_read_bit + srcQueueFamilyIndex: family + dstQueueFamilyIndex: family + buffer: frame.buffer + offset: 0 + size: vk.whole_size + } + dependency := vk.DependencyInfo{ + bufferMemoryBarrierCount: 1 + pBufferMemoryBarriers: &barrier + } + vk.cmd_pipeline_barrier2(command_buffer, &dependency) + frame.decode_index = vp.current_frame + frame.display_order = vp.decoder.video_data.frame_infos[vp.current_frame].display_order + frame.pending = true +} + +fn (mut vp VideoPlayer) write_frame_readback(slot int) ! { + if vp.frame_readback_dir == '' || !vp.frame_readbacks[slot].pending { + return + } + mut frame := &vp.frame_readbacks[slot] + size := u64(vp.decoder.video_data.width) * u64(vp.decoder.video_data.height) * 3 / 2 + check_vk(vp.app.device_context.memory_allocator.invalidate_range(frame.allocation, 0, size), + 'Could not invalidate decoded-frame readback') + path := os.join_path(vp.frame_readback_dir, '${frame.display_order}.nv12') + os.write_file_array(path, unsafe { frame.mapped.vbytes(int(size)) })! + frame.pending = false +} + +fn (mut vp VideoPlayer) finish_frame_readback() ! { + if vp.frame_readback_dir == '' || vp.frame_readback_done { + return + } + for i, frame in vp.frame_readbacks { + if !frame.pending { + continue + } + fence := vp.video_frames[i].in_flight_fence + check_vk(vk.wait_for_fences(vp.app.device_context.vk_device, 1, &fence, vk._true, max_u64), + 'Could not wait for decoded-frame readback') + vp.write_frame_readback(i)! + } + vp.frame_readback_done = true + eprintln('Decoded NV12 frames written to ${vp.frame_readback_dir}') +} + +fn (mut vp VideoPlayer) release_frame_readback() { + for mut frame in vp.frame_readbacks { + if !isnil(frame.mapped) { + vp.app.device_context.memory_allocator.unmap(mut frame.allocation) + frame.mapped = unsafe { nil } + } + if !isnil(frame.buffer) { + vk.destroy_buffer(vp.app.device_context.vk_device, frame.buffer, unsafe { nil }) + frame.buffer = unsafe { nil } + } + _ = vp.app.device_context.memory_allocator.release(mut frame.allocation) + } + vp.frame_readbacks.clear() +} diff --git a/h264_parameter_sets.v b/h264_parameter_sets.v new file mode 100644 index 0000000..21c2e19 --- /dev/null +++ b/h264_parameter_sets.v @@ -0,0 +1,352 @@ +module main + +import antono2.h264 + +// The pinned H.264 parser reads past truncated RBSPs as zero bits and has +// fixed-size arrays for POC cycles and HRD entries. Check those boundaries +// before passing untrusted MP4 parameter sets to it. +struct CheckedH264Bits { + data []u8 +mut: + position int +} + +fn (mut bits CheckedH264Bits) read(count int) !u32 { + if count < 0 || count > 32 || bits.position + count > bits.data.len * 8 { + return error('truncated H.264 parameter set') + } + mut value := u32(0) + for _ in 0 .. count { + value = (value << 1) | u32((bits.data[bits.position / 8] >> (7 - bits.position % 8)) & 1) + bits.position++ + } + return value +} + +fn (mut bits CheckedH264Bits) ue() !u32 { + mut zeros := 0 + for bits.read(1)! == 0 { + zeros++ + if zeros > 31 { + return error('invalid H.264 Exp-Golomb value') + } + } + return (u32(1) << zeros) - 1 + bits.read(zeros)! +} + +fn (mut bits CheckedH264Bits) se() !int { + value := bits.ue()! + if value > 0x7fffffff { + return error('H.264 signed Exp-Golomb value is too large') + } + return if value & 1 != 0 { int((value + 1) / 2) } else { -int(value / 2) } +} + +fn (mut bits CheckedH264Bits) scaling_list(size int) ! { + mut last := 8 + mut next := 8 + for _ in 0 .. size { + if next != 0 { + next = (last + bits.se()! + 256) % 256 + } + last = if next == 0 { last } else { next } + } +} + +fn (mut bits CheckedH264Bits) scaling_list_values(size int) !([]u8, bool) { + mut values := []u8{len: size} + mut last := 8 + mut next := 8 + mut use_default := false + for i in 0 .. size { + if next != 0 { + next = (last + bits.se()! + 256) % 256 + if i == 0 && next == 0 { + use_default = true + } + } + values[i] = u8(if next == 0 { last } else { next }) + last = int(values[i]) + } + return values, use_default +} + +// The pinned parser records scaling-list presence but its fixed-array slice +// writes do not persist the list values. Re-read the validated syntax into the +// actual SPS arrays used to build Vulkan session parameters. +fn populate_sps_scaling_lists(payload []u8, mut sps h264.SequenceParameterSet) ! { + mut bits := CheckedH264Bits{ + data: payload + } + profile := bits.read(8)! + _ = bits.read(16)! + _ = bits.ue()! + if profile != 100 { + return + } + chroma := bits.ue()! + if chroma == 3 { + _ = bits.read(1)! + } + _ = bits.ue()! + _ = bits.ue()! + _ = bits.read(1)! + if bits.read(1)! == 0 { + return + } + for i in 0 .. 8 { + if bits.read(1)! == 0 { + continue + } + values, use_default := bits.scaling_list_values(if i < 6 { 16 } else { 64 })! + if i < 6 { + for j, value in values { + sps.scaling_list_4x4[i][j] = int(value) + } + sps.use_default_scaling_matrix_4x4_flag[i] = if use_default { u32(1) } else { u32(0) } + } else { + for j, value in values { + sps.scaling_list_8x8[i - 6][j] = int(value) + } + sps.use_default_scaling_matrix_8x8_flag[i - 6] = if use_default { + u32(1) + } else { + u32(0) + } + } + } +} + +fn populate_pps_scaling_lists(payload []u8, mut pps h264.PictureParameterSet) ! { + mut bits := CheckedH264Bits{ + data: payload + } + _ = bits.ue()! + _ = bits.ue()! + _ = bits.read(2)! + _ = bits.ue()! // slice groups: preflight rejected nonzero values + _ = bits.ue()! + _ = bits.ue()! + _ = bits.read(3)! + _ = bits.se()! + _ = bits.se()! + _ = bits.se()! + _ = bits.read(3)! + if !bits.has_more_rbsp_data()! { + return + } + transform_8x8 := bits.read(1)! + if bits.read(1)! == 0 { + return + } + for i in 0 .. 6 + int(transform_8x8) * 2 { + if bits.read(1)! == 0 { + continue + } + values, use_default := bits.scaling_list_values(if i < 6 { 16 } else { 64 })! + if i < 6 { + for j, value in values { + pps.scaling_list_4x4[i][j] = int(value) + } + pps.use_default_scaling_matrix_4x4_flag[i] = if use_default { u32(1) } else { u32(0) } + } else { + for j, value in values { + pps.scaling_list_8x8[i - 6][j] = int(value) + } + pps.use_default_scaling_matrix_8x8_flag[i - 6] = if use_default { + u32(1) + } else { + u32(0) + } + } + } +} + +fn (mut bits CheckedH264Bits) hrd() ! { + count := bits.ue()! + if count > 32 { + return error('H.264 HRD has more than 32 entries') + } + _ = bits.read(4)! + _ = bits.read(4)! + for _ in 0 .. int(count) { + _ = bits.ue()! + _ = bits.ue()! + _ = bits.read(1)! + } + _ = bits.read(20)! +} + +fn (mut bits CheckedH264Bits) vui() ! { + if bits.read(1)! != 0 { + if bits.read(8)! == 255 { + _ = bits.read(32)! + } + } + if bits.read(1)! != 0 { + _ = bits.read(1)! + } + if bits.read(1)! != 0 { + _ = bits.read(4)! + if bits.read(1)! != 0 { + _ = bits.read(24)! + } + } + if bits.read(1)! != 0 { + _ = bits.ue()! + _ = bits.ue()! + } + if bits.read(1)! != 0 { + _ = bits.read(32)! + _ = bits.read(32)! + _ = bits.read(1)! + } + mut has_hrd := false + if bits.read(1)! != 0 { + bits.hrd()! + has_hrd = true + } + if bits.read(1)! != 0 { + bits.hrd()! + has_hrd = true + } + if has_hrd { + _ = bits.read(1)! + } + _ = bits.read(1)! + if bits.read(1)! != 0 { + _ = bits.read(1)! + for _ in 0 .. 6 { + _ = bits.ue()! + } + } +} + +fn validate_sps_rbsp(payload []u8) ! { + mut bits := CheckedH264Bits{ + data: payload + } + profile := bits.read(8)! + if profile !in [u32(66), 77, 100] { + return error('H.264 profile_idc ${profile} is unsupported; supported profiles are Baseline, Main, and High 8-bit 4:2:0') + } + _ = bits.read(8)! // constraint flags and reserved bits + _ = bits.read(8)! // level_idc + sps_id := bits.ue()! + if sps_id > 31 { + return error('H.264 SPS id exceeds 31') + } + if profile == 100 { + chroma := bits.ue()! + if chroma == 3 { + _ = bits.read(1)! + } + _ = bits.ue()! + _ = bits.ue()! + _ = bits.read(1)! + if bits.read(1)! != 0 { + for i in 0 .. 8 { + if bits.read(1)! != 0 { + bits.scaling_list(if i < 6 { 16 } else { 64 })! + } + } + } + } + if bits.ue()! > 12 { + return error('invalid H.264 frame-number bit width') + } + poc_type := bits.ue()! + if poc_type == 0 { + if bits.ue()! > 12 { + return error('invalid H.264 picture-order-count bit width') + } + } else if poc_type == 1 { + _ = bits.read(1)! + _ = bits.se()! + _ = bits.se()! + cycle_count := bits.ue()! + if cycle_count > 256 { + return error('H.264 POC cycle has more than 256 entries') + } + for _ in 0 .. int(cycle_count) { + _ = bits.se()! + } + } else if poc_type != 2 { + return error('invalid H.264 picture-order-count type ${poc_type}') + } + if bits.ue()! >= slot_count { + return error('H.264 reference count exceeds decoder slots') + } + _ = bits.read(1)! + _ = bits.ue()! // width + _ = bits.ue()! // height + if bits.read(1)! == 0 { + _ = bits.read(1)! + } + _ = bits.read(1)! + if bits.read(1)! != 0 { + for _ in 0 .. 4 { + _ = bits.ue()! + } + } + if bits.read(1)! != 0 { + bits.vui()! + } + if bits.read(1)! != 1 { + return error('H.264 SPS has no RBSP stop bit') + } +} + +fn validate_pps_rbsp(payload []u8) ! { + mut bits := CheckedH264Bits{ + data: payload + } + pps_id := bits.ue()! + sps_id := bits.ue()! + if pps_id > 255 || sps_id > 31 { + return error('H.264 PPS or SPS id is out of range') + } + _ = bits.read(2)! + if bits.ue()! != 0 { + return error('H.264 slice groups are not supported') + } + if bits.ue()! >= 64 || bits.ue()! >= 64 { + return error('H.264 reference-list count exceeds 64') + } + _ = bits.read(1)! // weighted prediction + _ = bits.read(2)! // weighted biprediction + _ = bits.se()! // initial QP + _ = bits.se()! // initial QS + _ = bits.se()! // chroma QP offset + _ = bits.read(3)! // deblocking, intra prediction, redundant count + if bits.has_more_rbsp_data()! { + transform_8x8 := bits.read(1)! + if bits.read(1)! != 0 { + for i in 0 .. 6 + int(transform_8x8) * 2 { + if bits.read(1)! != 0 { + bits.scaling_list(if i < 6 { 16 } else { 64 })! + } + } + } + _ = bits.se()! + } + if bits.read(1)! != 1 { + return error('H.264 PPS has no RBSP stop bit') + } +} + +fn (bits CheckedH264Bits) has_more_rbsp_data() !bool { + if bits.position >= bits.data.len * 8 { + return error('truncated H.264 parameter set') + } + mut lookahead := bits + if lookahead.read(1)! == 0 { + return true + } + for lookahead.position < lookahead.data.len * 8 { + if lookahead.read(1)! != 0 { + return true + } + } + return false +} diff --git a/h264_slice.v b/h264_slice.v new file mode 100644 index 0000000..671f8ed --- /dev/null +++ b/h264_slice.v @@ -0,0 +1,206 @@ +module main + +import antono2.h264 + +fn h264_pps_by_id(pps_array []h264.PictureParameterSet, id u32) !h264.PictureParameterSet { + for pps in pps_array { + if pps.pic_parameter_set_id == id { + return pps + } + } + return error('H.264 slice references missing PPS ${id}') +} + +fn h264_sps_by_id(sps_array []h264.SequenceParameterSet, id u32) !h264.SequenceParameterSet { + for sps in sps_array { + if sps.seq_parameter_set_id == id { + return sps + } + } + return error('H.264 PPS references missing SPS ${id}') +} + +// The pinned h264 module omits the final weighted reference entry and ignores +// slice-level reference-count overrides. Both shift the following MMCO syntax. +fn read_weight_table(mut sh h264.SliceHeader, sps &h264.SequenceParameterSet, + pps &h264.PictureParameterSet, mut bits h264.Bitstream) ! { + sh.pwt.luma_log2_weight_denom = bits.ue() + if sps.chroma_format_idc != 0 { + sh.pwt.chroma_log2_weight_denom = bits.ue() + } + l0_count := if sh.num_ref_idx_active_override_flag != 0 { + sh.num_ref_idx_l0_active_minus1 + 1 + } else { + pps.num_ref_idx_l0_active_minus1 + 1 + } + if l0_count > 64 { + return error('H.264 weighted reference count exceeds 64') + } + for i in 0 .. l0_count { + sh.pwt.luma_weight_l0_flag[i] = bits.u1() + if sh.pwt.luma_weight_l0_flag[i] != 0 { + sh.pwt.luma_weight_l0[i] = bits.se() + sh.pwt.luma_offset_l0[i] = bits.se() + } + if sps.chroma_format_idc != 0 { + sh.pwt.chroma_weight_l0_flag[i] = bits.u1() + if sh.pwt.chroma_weight_l0_flag[i] != 0 { + for j in 0 .. 2 { + sh.pwt.chroma_weight_l0[i][j] = bits.se() + sh.pwt.chroma_offset_l0[i][j] = bits.se() + } + } + } + } + if sh.is_slice_type(.b) { + l1_count := if sh.num_ref_idx_active_override_flag != 0 { + sh.num_ref_idx_l1_active_minus1 + 1 + } else { + pps.num_ref_idx_l1_active_minus1 + 1 + } + if l1_count > 64 { + return error('H.264 weighted reference count exceeds 64') + } + for i in 0 .. l1_count { + sh.pwt.luma_weight_l1_flag[i] = bits.u1() + if sh.pwt.luma_weight_l1_flag[i] != 0 { + sh.pwt.luma_weight_l1[i] = bits.se() + sh.pwt.luma_offset_l1[i] = bits.se() + } + if sps.chroma_format_idc != 0 { + sh.pwt.chroma_weight_l1_flag[i] = bits.u1() + if sh.pwt.chroma_weight_l1_flag[i] != 0 { + for j in 0 .. 2 { + sh.pwt.chroma_weight_l1[i][j] = bits.se() + sh.pwt.chroma_offset_l1[i][j] = bits.se() + } + } + } + } + } +} + +fn read_reference_marking(mut sh h264.SliceHeader, + nal &h264.NetworkAbstractionLayerHeader, mut bits h264.Bitstream) ! { + if nal.type == .coded_slice_idr { + sh.drpm.no_output_of_prior_pics_flag = bits.u1() + sh.drpm.long_term_reference_flag = bits.u1() + return + } + sh.drpm.adaptive_ref_pic_marking_mode_flag = bits.u1() + if sh.drpm.adaptive_ref_pic_marking_mode_flag == 0 { + return + } + for i in 0 .. sh.drpm.memory_management_control_operation.len { + if bits.eof() { + return error('truncated H.264 reference marking') + } + op := bits.ue() + if op > 6 { + return error('invalid H.264 memory-management operation ${op}') + } + sh.drpm.memory_management_control_operation[i] = op + if op in [u32(1), 3] { + sh.drpm.difference_of_pic_nums_minus1[i] = bits.ue() + } + if op == 2 { + sh.drpm.long_term_pic_num[i] = bits.ue() + } + if op in [u32(3), 6] { + sh.drpm.long_term_frame_idx[i] = bits.ue() + } + if op == 4 { + sh.drpm.max_long_term_frame_idx_plus1[i] = bits.ue() + } + if op == 0 { + return + } + } + return error('H.264 reference marking has more than 64 operations') +} + +fn read_slice_header_checked(nal &h264.NetworkAbstractionLayerHeader, + pps_array []h264.PictureParameterSet, sps_array []h264.SequenceParameterSet, + mut bits h264.Bitstream) !h264.SliceHeader { + mut sh := h264.SliceHeader{} + sh.first_mb_in_slice = bits.ue() + sh.slice_type = bits.ue() + if sh.slice_type > 9 { + return error('invalid H.264 slice type ${sh.slice_type}') + } + sh.pic_parameter_set_id = bits.ue() + pps := h264_pps_by_id(pps_array, sh.pic_parameter_set_id)! + sps := h264_sps_by_id(sps_array, pps.seq_parameter_set_id)! + sh.frame_num = bits.u(sps.log2_max_frame_num_minus4 + 4) + if sps.frame_mbs_only_flag == 0 { + sh.field_pic_flag = bits.u1() + if sh.field_pic_flag != 0 { + sh.bottom_field_flag = bits.u1() + } + } + if nal.type == .coded_slice_idr { + sh.idr_pic_id = bits.ue() + } + if sps.pic_order_cnt_type == 0 { + sh.pic_order_cnt_lsb = bits.u(sps.log2_max_pic_order_cnt_lsb_minus4 + 4) + if pps.pic_order_present_flag != 0 && sh.field_pic_flag == 0 { + sh.delta_pic_order_cnt_bottom = bits.se() + } + } else if sps.pic_order_cnt_type == 1 && sps.delta_pic_order_always_zero_flag == 0 { + sh.delta_pic_order_cnt[0] = bits.se() + if pps.pic_order_present_flag != 0 && sh.field_pic_flag == 0 { + sh.delta_pic_order_cnt[1] = bits.se() + } + } + if pps.redundant_pic_cnt_present_flag != 0 { + sh.redundant_pic_cnt = bits.ue() + } + if sh.is_slice_type(.b) { + sh.direct_spatial_mv_pred_flag = bits.u1() + } + if sh.is_slice_type(.p) || sh.is_slice_type(.sp) || sh.is_slice_type(.b) { + sh.num_ref_idx_active_override_flag = bits.u1() + if sh.num_ref_idx_active_override_flag != 0 { + sh.num_ref_idx_l0_active_minus1 = bits.ue() + if sh.num_ref_idx_l0_active_minus1 >= 64 { + return error('H.264 L0 reference count exceeds 64') + } + if sh.is_slice_type(.b) { + sh.num_ref_idx_l1_active_minus1 = bits.ue() + if sh.num_ref_idx_l1_active_minus1 >= 64 { + return error('H.264 L1 reference count exceeds 64') + } + } + } + } + sh.read_ref_pic_list_reordering(mut bits) + if (pps.weighted_pred_flag != 0 && (sh.is_slice_type(.p) || sh.is_slice_type(.sp))) + || (pps.weighted_bipred_idc == 1 && sh.is_slice_type(.b)) { + read_weight_table(mut sh, &sps, &pps, mut bits)! + } + if nal.idc != .priority_disposable { + read_reference_marking(mut sh, nal, mut bits)! + } + if pps.entropy_coding_mode_flag != 0 && !sh.is_slice_type(.i) && !sh.is_slice_type(.si) { + sh.cabac_init_idc = bits.ue() + } + sh.slice_qp_delta = bits.se() + if sh.is_slice_type(.sp) || sh.is_slice_type(.si) { + if sh.is_slice_type(.sp) { + sh.sp_for_switch_flag = bits.u1() + } + sh.slice_qs_delta = bits.se() + } + if pps.deblocking_filter_control_present_flag != 0 { + sh.disable_deblocking_filter_idc = bits.ue() + if sh.disable_deblocking_filter_idc != 1 { + sh.slice_alpha_c0_offset_div2 = bits.se() + sh.slice_beta_offset_div2 = bits.se() + } + } + if pps.num_slice_groups_minus1 > 0 && pps.slice_group_map_type >= 3 + && pps.slice_group_map_type <= 5 { + return error('H.264 slice groups are not supported') + } + return sh +} diff --git a/mp4_parser.v b/mp4_parser.v index c6d8f03..2c836d3 100644 --- a/mp4_parser.v +++ b/mp4_parser.v @@ -76,25 +76,177 @@ fn remove_emulation_prevention_bytes(ebsp byteptr, size int) []u8 { return rbsp } -type BytePtr = byteptr +fn read_nal_length(bytes []u8, offset int, length_size int) !u32 { + if length_size !in [1, 2, 4] || offset < 0 || offset + length_size > bytes.len { + return error('truncated or unsupported H.264 NAL length') + } + mut length := u32(0) + for i in 0 .. length_size { + length = (length << 8) | u32(bytes[offset + i]) + } + return length +} -fn (p BytePtr) to_varray[T](len u32) []T { - if isnil(p) && len > 0 { - panic('Nil to_varray for len: ${len}') +fn detect_nal_length_size(sample []u8) !u32 { + for length_size in [4, 2, 1] { + mut offset := 0 + mut valid := sample.len > 0 + for offset < sample.len { + if offset + length_size > sample.len { + valid = false + break + } + nal_size := read_nal_length(sample, offset, length_size) or { + valid = false + break + } + if nal_size < 2 || u64(nal_size) > u64(sample.len - offset - length_size) { + valid = false + break + } + nal_header := sample[offset + length_size] + if nal_header & 0x80 != 0 || nal_header & 0x1f == 0 { + valid = false + break + } + offset += length_size + int(nal_size) + } + if valid && offset == sample.len { + return u32(length_size) + } } - if len <= 0 { - return []T{} + return error('MP4 sample has no valid H.264 NAL length prefix') +} + +fn validate_slice_parameter_sets(payload []u8, pps_array []h264.PictureParameterSet, sps_array []h264.SequenceParameterSet) ! { + mut bits := h264.Bitstream{} + bits.init(payload) + bits.ue() // first_mb_in_slice + bits.ue() // slice_type + pps_id := bits.ue() + pps := h264_pps_by_id(pps_array, pps_id)! + _ = h264_sps_by_id(sps_array, pps.seq_parameter_set_id)! +} + +fn validate_same_picture(first &h264.SliceHeader, next &h264.SliceHeader, + first_nal &h264.NetworkAbstractionLayerHeader, next_nal &h264.NetworkAbstractionLayerHeader) ! { + if first.pic_parameter_set_id != next.pic_parameter_set_id || first.frame_num != next.frame_num + || first.pic_order_cnt_lsb != next.pic_order_cnt_lsb + || first.delta_pic_order_cnt_bottom != next.delta_pic_order_cnt_bottom + || first.delta_pic_order_cnt != next.delta_pic_order_cnt + || first.idr_pic_id != next.idr_pic_id || first_nal.type != next_nal.type + || (first_nal.idc == .priority_disposable) != (next_nal.idc == .priority_disposable) { + return error('H.264 sample contains slices from different pictures') + } +} + +fn progressive_h264_dimensions(sps &h264.SequenceParameterSet) !(u32, u32, u32, u32) { + // This player only reaches here with progressive 4:2:0 SPSs, whose crop + // units are two luma samples in each direction. + padded_width := (u64(sps.pic_width_in_mbs_minus1) + 1) * 16 + padded_height := (u64(sps.pic_height_in_map_units_minus1) + 1) * 16 + crop_width := (u64(sps.frame_crop_left_offset) + u64(sps.frame_crop_right_offset)) * 2 + crop_height := (u64(sps.frame_crop_top_offset) + u64(sps.frame_crop_bottom_offset)) * 2 + if padded_width > 0xffffffff || padded_height > 0xffffffff || crop_width >= padded_width + || crop_height >= padded_height { + return error('invalid H.264 SPS dimensions or crop offsets') } - // TODO: doesn't trigger on to_varray[u8] - $if T is u8 { - return p.vbytes(int(len)) + return u32(padded_width - crop_width), u32(padded_height - crop_height), u32(padded_width), u32(padded_height) +} + +fn poc_type1_fields(sps &h264.SequenceParameterSet, header &h264.SliceHeader, + frame_num_offset u32, is_reference bool) !(int, int) { + mut abs_frame_num := if sps.num_ref_frames_in_pic_order_cnt_cycle == 0 { + u32(0) + } else { + frame_num_offset + header.frame_num } - mut ret := []T{cap: int(len)} - // vmemcpy(ret.data, p, len * sizeof(T)) - for i in 0 .. len { - ret << *unsafe { &T(p + (i * sizeof(T))) } + if !is_reference && abs_frame_num > 0 { + abs_frame_num-- + } + mut expected_poc := 0 + if abs_frame_num > 0 { + cycle_length := int(sps.num_ref_frames_in_pic_order_cnt_cycle) + if cycle_length <= 0 || cycle_length > sps.offset_for_ref_frame.len { + return error('invalid H.264 picture-order-count cycle length ${cycle_length}') + } + mut cycle_delta := 0 + for i in 0 .. cycle_length { + cycle_delta += sps.offset_for_ref_frame[i] + } + expected_poc = (int(abs_frame_num) - 1) / cycle_length * cycle_delta + for i in 0 .. ((int(abs_frame_num) - 1) % cycle_length + 1) { + expected_poc += sps.offset_for_ref_frame[i] + } + } + if !is_reference { + expected_poc += sps.offset_for_non_ref_pic + } + top := expected_poc + header.delta_pic_order_cnt[0] + bottom := top + sps.offset_for_top_to_bottom_field + header.delta_pic_order_cnt[1] + return top, bottom +} + +fn slice_has_mmco5(slice_header &h264.SliceHeader, is_idr bool, ref_idc h264.NAL_REF_IDC) bool { + if is_idr || ref_idc == .priority_disposable + || slice_header.drpm.adaptive_ref_pic_marking_mode_flag == 0 { + return false + } + for operation in slice_header.drpm.memory_management_control_operation { + if operation == 5 { + return true + } + if operation == 0 { + break + } + } + return false +} + +struct PictureOrderCountType0State { +mut: + prev_msb int + prev_lsb int + cycle int = -1 +} + +struct PictureOrderCountType0Result { + decode_poc int + display_poc int + top int + bottom int + cycle int +} + +fn (mut state PictureOrderCountType0State) advance(pic_lsb int, delta_bottom int, + max_lsb int, is_idr bool, is_reference bool, has_mmco5 bool) PictureOrderCountType0Result { + if is_idr { + state.prev_msb = 0 + state.prev_lsb = 0 + state.cycle++ + } + mut msb := state.prev_msb + if pic_lsb < state.prev_lsb && state.prev_lsb - pic_lsb >= max_lsb / 2 { + msb += max_lsb + } else if pic_lsb > state.prev_lsb && pic_lsb - state.prev_lsb > max_lsb / 2 { + msb -= max_lsb + } + top := msb + pic_lsb + bottom := top + delta_bottom + if has_mmco5 { + state.cycle++ + } + if is_reference { + state.prev_msb = if has_mmco5 { 0 } else { msb } + state.prev_lsb = if has_mmco5 { top } else { pic_lsb } + } + return PictureOrderCountType0Result{ + decode_poc: top + display_poc: if has_mmco5 { 0 } else { math.min(top, bottom) } + top: top + bottom: bottom + cycle: state.cycle } - return ret } fn (mut d Decoder) parse_mp4_data(file_path string) ! { @@ -158,20 +310,38 @@ fn (mut d Decoder) parse_mp4_data(file_path string) ! { mut sps_array := []h264.SequenceParameterSet{} mut data_sps := minimp4.mp4d_read_sps(&mp4, ntrack, count_sps, &num_bytes_sps) for !isnil(data_sps) { + if num_bytes_sps <= 1 { + return error('H.264 track contains an invalid sequence parameter set') + } + sps_header := unsafe { byteptr(data_sps)[0] } + if sps_header & 0x80 != 0 || sps_header & 0x1f != 7 { + return error('H.264 track contains an invalid SPS NAL header') + } mut nal := h264.NetworkAbstractionLayerHeader{} mut nal_header_bs := h264.Bitstream{} nal_header_bs.init(unsafe { data_sps.vbytes(1) }) nal.read_nal_header(mut &nal_header_bs) - if num_bytes_sps <= 1 { - return error('H.264 track contains an invalid sequence parameter set') - } mut nal_payload_rbsp_data := unsafe { remove_emulation_prevention_bytes(byteptr(data_sps) + 1, num_bytes_sps - 1) } + validate_sps_rbsp(nal_payload_rbsp_data)! mut nal_payload_bs := h264.Bitstream{} nal_payload_bs.init(nal_payload_rbsp_data) mut sps := h264.SequenceParameterSet{} sps.read_sps(mut nal_payload_bs) + populate_sps_scaling_lists(nal_payload_rbsp_data, mut sps)! + for prior in sps_array { + if prior.seq_parameter_set_id == sps.seq_parameter_set_id { + return error('duplicate H.264 SPS id ${sps.seq_parameter_set_id}') + } + } + if std_h264_level_idc(sps.level_idc) == .invalid { + return error('H.264 level_idc ${sps.level_idc} is unsupported') + } + if sps.level_idc == 11 && sps.constraint_set3_flag != 0 && sps.profile_idc in [u32(66), 77] { + return error('H.264 level 1b is unsupported') + } + d.video_data.h264_level_idc = math.max[u32](d.video_data.h264_level_idc, sps.level_idc) if sps.profile_idc !in [u32(66), 77, 100] { return error('H.264 profile_idc ${sps.profile_idc} is unsupported; supported profiles are Baseline, Main, and High 8-bit 4:2:0') } @@ -189,21 +359,29 @@ fn (mut d Decoder) parse_mp4_data(file_path string) ! { if sps.frame_mbs_only_flag == 0 { return error('interlaced H.264 video is not currently supported') } - // Data validation - // https://stackoverflow.com/questions/6394874/fetching-the-dimensions-of-a-h264video-stream - width := ((sps.pic_width_in_mbs_minus1 + 1) * 16) - (sps.frame_crop_left_offset * 2) - (sps.frame_crop_right_offset * 2) - height := ((2 - sps.frame_mbs_only_flag) * (sps.pic_height_in_map_units_minus1 + 1) * 16) - (sps.frame_crop_top_offset * 2) - (sps.frame_crop_bottom_offset * 2) + if sps.pic_order_cnt_type > 2 { + return error('invalid H.264 picture-order-count type ${sps.pic_order_cnt_type}') + } + if sps.num_ref_frames >= slot_count { + return error('H.264 stream requires ${sps.num_ref_frames} references; this player supports at most ${slot_count - 1}') + } + if sps.log2_max_frame_num_minus4 > 12 || sps.log2_max_pic_order_cnt_lsb_minus4 > 12 { + return error('invalid H.264 frame or picture-order-count bit width') + } + width, height, padded_width, padded_height := progressive_h264_dimensions(&sps)! mp4_width := unsafe { track.sampleDescription.video.width } mp4_height := unsafe { track.sampleDescription.video.height } if mp4_width != width || mp4_height != height { eprintln('Warning: MP4 dimensions ${mp4_width}x${mp4_height} differ from H.264 SPS display dimensions ${width}x${height}') } - d.video_data.width_padd = (sps.pic_width_in_mbs_minus1 + 1) * 16 - d.video_data.height_padd = (sps.pic_height_in_map_units_minus1 + 1) * 16 + d.video_data.width_padd = padded_width + d.video_data.height_padd = padded_height if sps.vui_parameters_present_flag != 0 { - d.video_data.metadata.sar_width, d.video_data.metadata.sar_height = sample_aspect_ratio(sps.vui.aspect_ratio_idc, sps.vui.sar_width, sps.vui.sar_height) + d.video_data.metadata.sar_width, d.video_data.metadata.sar_height = sample_aspect_ratio(sps.vui.aspect_ratio_idc, + sps.vui.sar_width, sps.vui.sar_height) d.video_data.metadata.video_full_range = sps.vui.video_full_range_flag != 0 - d.video_data.metadata.colour_description_present = d.video_data.metadata.colour_description_present + d.video_data.metadata.colour_description_present = + d.video_data.metadata.colour_description_present || sps.vui.color_description_present_flag != 0 d.video_data.metadata.colour_primaries = u8(sps.vui.colour_primaries) d.video_data.metadata.transfer_function = u8(sps.vui.transfer_characteristics) @@ -211,9 +389,11 @@ fn (mut d Decoder) parse_mp4_data(file_path string) ! { } // x ^ ((x ^ y) & -(x < y)) // max(x, y) // d.video_data.num_dpb_slots = d.video_data.num_dpb_slots ^ ((d.video_data.num_dpb_slots ^ (sps.num_ref_frames * 2 + 1)) & -u32(d.video_data.num_dpb_slots < (sps.num_ref_frames * 2 + 1))) - d.video_data.num_dpb_slots = math.max[u32](d.video_data.num_dpb_slots, sps.num_ref_frames + 1) + d.video_data.num_dpb_slots = math.max[u32](d.video_data.num_dpb_slots, sps.num_ref_frames + + 1) d.video_data.sps_bytes << unsafe { byteptr(&sps).vbytes(int(sizeof(sps))) } sps_array << sps + d.video_data.sps_storage_index[sps.seq_parameter_set_id] = u8(sps_array.len) d.video_data.sps_count++ count_sps++ @@ -230,24 +410,37 @@ fn (mut d Decoder) parse_mp4_data(file_path string) ! { mut data_pps := minimp4.mp4d_read_pps(&mp4, ntrack, count_pps, &size_pps) for !isnil(data_pps) { + if size_pps <= 1 { + return error('H.264 track contains an invalid picture parameter set') + } + pps_header := unsafe { byteptr(data_pps)[0] } + if pps_header & 0x80 != 0 || pps_header & 0x1f != 8 { + return error('H.264 track contains an invalid PPS NAL header') + } mut nal := h264.NetworkAbstractionLayerHeader{} mut nal_header_bs := h264.Bitstream{} nal_header_bs.init(unsafe { data_pps.vbytes(1) }) nal.read_nal_header(mut nal_header_bs) - if size_pps <= 1 { - return error('H.264 track contains an invalid picture parameter set') - } pps_payload_rbsp_data := unsafe { remove_emulation_prevention_bytes(byteptr(data_pps) + 1, size_pps - 1) } + validate_pps_rbsp(pps_payload_rbsp_data)! mut pps_payload_bs := h264.Bitstream{} pps_payload_bs.init(pps_payload_rbsp_data) mut pps := h264.PictureParameterSet{} pps.read_pps(mut pps_payload_bs) + populate_pps_scaling_lists(pps_payload_rbsp_data, mut pps)! + for prior in pps_array { + if prior.pic_parameter_set_id == pps.pic_parameter_set_id { + return error('duplicate H.264 PPS id ${pps.pic_parameter_set_id}') + } + } + _ = h264_sps_by_id(sps_array, pps.seq_parameter_set_id)! d.video_data.pps_bytes.ensure_cap(d.video_data.pps_bytes.len + int(sizeof(pps))) d.video_data.pps_bytes << unsafe { byteptr(&pps).vbytes(int(sizeof(pps))) } pps_array << pps + d.video_data.pps_storage_index[pps.pic_parameter_set_id] = u16(pps_array.len) d.video_data.pps_count++ count_pps++ @@ -266,9 +459,7 @@ fn (mut d Decoder) parse_mp4_data(file_path string) ! { println('Display metadata: coded ${d.video_data.metadata.coded_width}x${d.video_data.metadata.coded_height}, display ${d.video_data.metadata.display_width}x${d.video_data.metadata.display_height}, SAR ${d.video_data.metadata.sar_width}:${d.video_data.metadata.sar_height}, rotation ${d.video_data.metadata.rotation_degrees}°') timescale_rcp := 1.0 / f64(track.timescale) - mut prev_pic_order_cnt_lsb := u32(0) - mut prev_pic_order_cnt_msb := u32(0) - mut poc_cycle := -1 + mut poc_state := PictureOrderCountType0State{} mut prev_frame_num := u32(0) mut prev_frame_offset := u32(0) @@ -280,7 +471,6 @@ fn (mut d Decoder) parse_mp4_data(file_path string) ! { d.video_data.frame_infos = []DecoderVideoDataFrameInfo{cap: int(track.sample_count)} d.video_data.slice_header_bytes.ensure_cap(int(track.sample_count * sizeof(h264.SliceHeader))) - d.video_data.slice_header_count = track.sample_count mut file := d.video_data.file mut sample_index := u32(0) @@ -294,7 +484,8 @@ fn (mut d Decoder) parse_mp4_data(file_path string) ! { // minimp4 returns timestamp before duration. These were previously passed // in reverse order, causing later frames to use their growing timestamp as // a duration and making playback progressively slower. - offset := minimp4.mp4d_frame_offset(&mp4, ntrack, sample_index, &frame_bytes_num_to_do, ×tamp, &duration) + offset := minimp4.mp4d_frame_offset(&mp4, ntrack, sample_index, &frame_bytes_num_to_do, + ×tamp, &duration) // The upload buffer must fit any slice contained in the complete MP4 // sample, including samples with leading non-slice NAL units. max_frame_size_bytes = math.max[u64](max_frame_size_bytes, frame_bytes_num_to_do) @@ -323,33 +514,43 @@ fn (mut d Decoder) parse_mp4_data(file_path string) ! { return error('MP4 sample ${sample_index} is truncated: expected ${expected_frame_bytes} bytes, read ${frame_bytes_num_to_do}') } input_file_position = offset + frame_bytes_num_to_do + if sample_index == 0 { + d.video_data.nal_length_size = detect_nal_length_size(src_buffer)! + } + length_size := int(d.video_data.nal_length_size) + mut found_slice := false + mut first_slice_header := h264.SliceHeader{} + mut first_slice_nal := h264.NetworkAbstractionLayerHeader{} for frame_bytes_num_to_do > 0 { - if frame_bytes_num_to_do < 4 { + if frame_bytes_num_to_do < d.video_data.nal_length_size { return error('MP4 sample ${sample_index} has a truncated H.264 NAL length') } - // mut size := unsafe{ (u32(*&src_buffer[src_buffer_idx+0]) << 24) | (u32(*&src_buffer[src_buffer_idx+1]) << 16) | (u32(*&src_buffer[src_buffer_idx+2]) << 8) | *&src_buffer[src_buffer_idx+3] } - mut size := unsafe { - (u32(src_buffer[src_buffer_idx + 0]) << 24) | (u32(src_buffer[src_buffer_idx + 1]) << 16) | (u32(src_buffer[src_buffer_idx + 2]) << 8) | src_buffer[src_buffer_idx + 3] - } - size += 4 - if size < 4 || frame_bytes_num_to_do < size { - return error('MP4 sample ${sample_index} has an invalid H.264 NAL size ${size - 4}') + nal_size := read_nal_length(src_buffer, src_buffer_idx, length_size)! + if nal_size > frame_bytes_num_to_do - d.video_data.nal_length_size { + return error('MP4 sample ${sample_index} has an invalid H.264 NAL size ${nal_size}') } + size := nal_size + d.video_data.nal_length_size - length_prefixed_data_offset := src_buffer_idx + 4 - length_prefixed_data_size := size - 4 + length_prefixed_data_offset := src_buffer_idx + length_size + length_prefixed_data_size := nal_size if length_prefixed_data_size <= 1 { return error('MP4 sample ${sample_index} contains an empty H.264 NAL unit') } + nal_header := src_buffer[length_prefixed_data_offset] + if nal_header & 0x80 != 0 || nal_header & 0x1f == 0 { + return error('MP4 sample ${sample_index} contains an invalid H.264 NAL header') + } mut nal := h264.NetworkAbstractionLayerHeader{} mut nal_header_bs := h264.Bitstream{} - nal_header_bs.init(src_buffer[length_prefixed_data_offset..length_prefixed_data_offset + 1]) + nal_header_bs.init(src_buffer[length_prefixed_data_offset.. + length_prefixed_data_offset + 1]) nal.read_nal_header(mut nal_header_bs) slfrom := length_prefixed_data_offset + 1 nal_payload_rbsp_data := unsafe { - remove_emulation_prevention_bytes(byteptr(src_buffer.data) + slfrom, int(length_prefixed_data_size - 1)) + remove_emulation_prevention_bytes(byteptr(src_buffer.data) + slfrom, + int(length_prefixed_data_size - 1)) } mut nal_payload_bs := h264.Bitstream{} nal_payload_bs.init(nal_payload_rbsp_data) @@ -370,69 +571,83 @@ fn (mut d Decoder) parse_mp4_data(file_path string) ! { } } + // All slices in this MP4 sample belong to one access unit. The + // first slice supplies picture metadata; every slice needs space in + // the Annex B upload (whose start code can exceed the MP4 prefix). + nal_start_code := h264.NalStartCode{}.value + data_frame.size += u64(nal_start_code.len) + u64(nal_size) + if found_slice { + validate_slice_parameter_sets(nal_payload_rbsp_data, pps_array, sps_array) or { + return error('MP4 sample ${sample_index}: ${err}') + } + next_slice_header := read_slice_header_checked(&nal, pps_array, sps_array, mut + nal_payload_bs) or { return error('MP4 sample ${sample_index}: ${err}') } + validate_same_picture(&first_slice_header, &next_slice_header, &first_slice_nal, + &nal) or { return error('MP4 sample ${sample_index}: ${err}') } + frame_bytes_num_to_do -= size + src_buffer_idx += int(size) + continue + } + /* * Decode Picture Order Count * (tig) see ITU-T H.264 (08/2021) pp.113 * */ // tig: see Rec. ITU-T H.264 (08/2021) p.66 (7-1) - mut slice_header := h264.SliceHeader{} - slice_header.read_slice_header(&nal, pps_array, sps_array, mut nal_payload_bs) - if slice_header.pic_parameter_set_id >= u32(pps_array.len) { - return error('MP4 sample ${sample_index} references missing H.264 PPS ${slice_header.pic_parameter_set_id}') - } - pps := pps_array[slice_header.pic_parameter_set_id] - if pps.seq_parameter_set_id >= u32(sps_array.len) { - return error('MP4 sample ${sample_index} references missing H.264 SPS ${pps.seq_parameter_set_id}') + validate_slice_parameter_sets(nal_payload_rbsp_data, pps_array, sps_array) or { + return error('MP4 sample ${sample_index}: ${err}') } - sps := sps_array[pps.seq_parameter_set_id] + mut slice_header := read_slice_header_checked(&nal, pps_array, sps_array, mut + nal_payload_bs) or { return error('MP4 sample ${sample_index}: ${err}') } + first_slice_header = slice_header + first_slice_nal = nal + data_frame.has_mmco5 = slice_has_mmco5(&slice_header, is_idr, nal.idc) + pps := h264_pps_by_id(pps_array, slice_header.pic_parameter_set_id)! + sps := h264_sps_by_id(sps_array, pps.seq_parameter_set_id)! max_frame_num := u32(1) << (sps.log2_max_frame_num_minus4 + 4) - max_pic_order_cnt_lsb := u32(1) << (sps.log2_max_pic_order_cnt_lsb_minus4 + 4) - pic_order_cnt_lsb := u32(slice_header.pic_order_cnt_lsb) - mut pic_order_cnt_msb := u32(0) + max_pic_order_cnt_lsb := int(u32(1) << (sps.log2_max_pic_order_cnt_lsb_minus4 + 4)) mut frame_num_offset := u32(0) - mut tmp_pic_order_cout := u32(0) + mut tmp_pic_order_cout := 0 match sps.pic_order_cnt_type { 0 { - // TYPE 0 - // Rec. ITU-T H.264 (08/2021) page 114 - // Use the NAL unit type, not idr flag + // The parser already rejected interlaced SPSs, so this is a frame. + result := poc_state.advance(int(slice_header.pic_order_cnt_lsb), + int(slice_header.delta_pic_order_cnt_bottom), max_pic_order_cnt_lsb, + is_idr, nal.idc != .priority_disposable, data_frame.has_mmco5) + data_frame.top_field_order_cnt = result.top + data_frame.bottom_field_order_cnt = result.bottom + data_frame.decode_poc = result.decode_poc + data_frame.poc = result.display_poc + data_frame.gop = result.cycle + } + // match 0 + 1 { if is_idr { - prev_pic_order_cnt_msb = 0 - prev_pic_order_cnt_lsb = 0 - poc_cycle++ - } - if pic_order_cnt_lsb < prev_pic_order_cnt_lsb && (prev_pic_order_cnt_lsb - pic_order_cnt_lsb) >= max_pic_order_cnt_lsb / 2 { - pic_order_cnt_msb = prev_pic_order_cnt_msb + max_pic_order_cnt_lsb - } else if pic_order_cnt_lsb > prev_pic_order_cnt_lsb && (pic_order_cnt_lsb - prev_pic_order_cnt_lsb) > max_pic_order_cnt_lsb / 2 { - pic_order_cnt_msb = prev_pic_order_cnt_msb - max_pic_order_cnt_lsb + frame_num_offset = 0 + } else if prev_frame_num > slice_header.frame_num { + frame_num_offset = prev_frame_offset + max_frame_num } else { - pic_order_cnt_msb = prev_pic_order_cnt_msb - } - // Top and bottom field order count in case the picture is a field - if slice_header.field_pic_flag == 0 || slice_header.bottom_field_flag == 0 { - data_frame.top_field_order_cnt = pic_order_cnt_msb + pic_order_cnt_lsb + frame_num_offset = prev_frame_offset } - if slice_header.field_pic_flag == 0 { - data_frame.bottom_field_order_cnt = data_frame.top_field_order_cnt + u32(slice_header.delta_pic_order_cnt_bottom) - } else if slice_header.bottom_field_flag != 0 { - data_frame.bottom_field_order_cnt = pic_order_cnt_msb + slice_header.pic_order_cnt_lsb + data_frame.top_field_order_cnt, data_frame.bottom_field_order_cnt = poc_type1_fields(&sps, + &slice_header, frame_num_offset, nal.idc != .priority_disposable)! + data_frame.decode_poc = math.min(data_frame.top_field_order_cnt, + data_frame.bottom_field_order_cnt) + data_frame.poc = if data_frame.has_mmco5 { 0 } else { data_frame.decode_poc } + if is_idr || data_frame.has_mmco5 { + poc_state.cycle++ } - - // Same as top field order count - data_frame.poc = int(pic_order_cnt_msb + pic_order_cnt_lsb) - data_frame.gop = poc_cycle - - // TODO: memory_management_control_operation equal to 5 - if nal.idc != h264.NAL_REF_IDC.priority_disposable { - prev_pic_order_cnt_msb = pic_order_cnt_msb - prev_pic_order_cnt_lsb = pic_order_cnt_lsb + data_frame.gop = poc_state.cycle + prev_frame_offset = if data_frame.has_mmco5 { u32(0) } else { frame_num_offset } + prev_frame_num = if data_frame.has_mmco5 { + u32(0) + } else { + slice_header.frame_num } } - - // match 0 2 { if is_idr { frame_num_offset = 0 @@ -441,28 +656,34 @@ fn (mut d Decoder) parse_mp4_data(file_path string) ! { } else { frame_num_offset = prev_frame_offset } - prev_frame_offset = frame_num_offset - prev_frame_num = slice_header.frame_num + prev_frame_offset = if data_frame.has_mmco5 { u32(0) } else { frame_num_offset } + prev_frame_num = if data_frame.has_mmco5 { + u32(0) + } else { + slice_header.frame_num + } if is_idr { tmp_pic_order_cout = 0 } else if nal.idc == h264.NAL_REF_IDC.priority_disposable { - tmp_pic_order_cout = 2 * (frame_num_offset + slice_header.frame_num) - 1 + tmp_pic_order_cout = 2 * int(frame_num_offset + slice_header.frame_num) - 1 } else { - tmp_pic_order_cout = 2 * (frame_num_offset + slice_header.frame_num) + tmp_pic_order_cout = 2 * int(frame_num_offset + slice_header.frame_num) } // (tig) Ignore bottom or top fields, as we assume progressive. // If it were otherwise - for interleaved - either the top or the bottom // field shall be set, depending on whether the current picture is the // top or bottom field, as indicated by bottom_field_flag - data_frame.poc = int(tmp_pic_order_cout) - if tmp_pic_order_cout == 0 { - poc_cycle++ + data_frame.decode_poc = int(tmp_pic_order_cout) + data_frame.top_field_order_cnt = tmp_pic_order_cout + data_frame.bottom_field_order_cnt = tmp_pic_order_cout + data_frame.poc = if data_frame.has_mmco5 { 0 } else { data_frame.decode_poc } + if tmp_pic_order_cout == 0 || data_frame.has_mmco5 { + poc_state.cycle++ } - data_frame.gop = poc_cycle + data_frame.gop = poc_state.cycle } - // match 2 else { return error('H.264 picture-order-count type ${sps.pic_order_cnt_type} is not supported') @@ -472,30 +693,36 @@ fn (mut d Decoder) parse_mp4_data(file_path string) ! { // Accept frame beginning NAL unit data_frame.nal_ref_idc = u32(nal.idc) data_frame.nal_unit_type = u8(nal.type) - // TODO: h264.nal_start_code as pub const. error, imported types must start with a capital letter, but const can't be upper case - // data_frame.size = sizeof(h264.nal_start_code) + size - 4 - nal_start_code := h264.NalStartCode{}.value - data_frame.size = u64(nal_start_code.len) + size - 4 data_frame.reference_priority = u32(nal.idc) data_frame.decode_time_ns = i64(f64(timestamp) * timescale_rcp * 1_000_000_000.0) data_frame.display_time_ns = i64(f64(timestamp) * timescale_rcp * 1_000_000_000.0) - data_frame.duration_ns = math.max[i64](1, i64(f64(duration) * timescale_rcp * 1_000_000_000.0)) + data_frame.duration_ns = math.max[i64](1, + i64(f64(duration) * timescale_rcp * 1_000_000_000.0)) d.video_data.slice_header_bytes << unsafe { byteptr(&slice_header).vbytes(int(sizeof(slice_header))) } - - // for frame_bytes_num_to_do > 0 - break + found_slice = true + frame_bytes_num_to_do -= size + src_buffer_idx += int(size) } // x ^ ((x ^ y) & -(x < y)) // max(x, y) // max_frame_size_bytes = max_frame_size_bytes ^ ((max_frame_size_bytes ^ data_frame.size) & -(u64(max_frame_size_bytes < data_frame.size))) max_frame_size_bytes = math.max[u64](max_frame_size_bytes, data_frame.size) - d.video_data.frame_infos << data_frame + if found_slice { + d.video_data.frame_infos << data_frame + } else if d.video_data.frame_infos.len > 0 { + // A metadata-only sample consumes time without producing a picture. + last := d.video_data.frame_infos.len - 1 + d.video_data.frame_infos[last].duration_ns += i64(f64(duration) * timescale_rcp * 1_000_000_000.0) + } sample_index++ } // for sample_index < track.sample_count + if d.video_data.frame_infos.len == 0 { + return error('H.264 video track contains no decodable slices') + } // Fills array values with their index 0..len d.video_data.frame_display_order = []u64{len: d.video_data.frame_infos.len, init: index} @@ -508,7 +735,8 @@ fn (mut d Decoder) parse_mp4_data(file_path string) ! { mut j := i for j > 0 { previous_index := d.video_data.frame_display_order[j - 1] - if compare_frame_display_order(&d.video_data.frame_infos[previous_index], &frame_to_insert) <= 0 { + if compare_frame_display_order(&d.video_data.frame_infos[previous_index], + &frame_to_insert) <= 0 { break } d.video_data.frame_display_order[j] = previous_index diff --git a/player_decode.v b/player_decode.v index 30f6865..0e4d028 100644 --- a/player_decode.v +++ b/player_decode.v @@ -22,7 +22,8 @@ fn (mut vp VideoPlayer) update_decode_video() ! { vk.reset_command_buffer(command_buffer_info.graphics_command_buffer, 0) vk.begin_command_buffer(command_buffer_info.graphics_command_buffer, &begin_command_buffer) if vp.is_stopped { - vk.cmd_set_event(command_buffer_info.graphics_command_buffer, vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands)) + vk.cmd_set_event(command_buffer_info.graphics_command_buffer, vp.event_video_player, + vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands)) return } @@ -40,18 +41,22 @@ fn (mut vp VideoPlayer) update_decode_video() ! { assert !isnil(vp.decoder.get_sps()) slice_header = unsafe { - &h264.SliceHeader(byteptr(usize(vp.decoder.get_slice_header()) + usize(vp.current_frame) * sizeof(h264.SliceHeader))) + &h264.SliceHeader(byteptr(usize(vp.decoder.get_slice_header()) + + usize(vp.current_frame) * sizeof(h264.SliceHeader))) } pps = unsafe { - &h264.PictureParameterSet(byteptr(usize(vp.decoder.get_pps()) + usize(slice_header.pic_parameter_set_id) * sizeof(h264.PictureParameterSet))) + &h264.PictureParameterSet(byteptr(usize(vp.decoder.get_pps()) + + usize(vp.decoder.video_data.pps_storage_offset(slice_header.pic_parameter_set_id)!))) } sps = unsafe { - &h264.SequenceParameterSet(byteptr(usize(vp.decoder.get_sps()) + usize(pps.seq_parameter_set_id) * sizeof(h264.SequenceParameterSet))) + &h264.SequenceParameterSet(byteptr(usize(vp.decoder.get_sps()) + + usize(vp.decoder.video_data.sps_storage_offset(pps.seq_parameter_set_id)!))) } } mut decode_ope := DecoderVideoDecodeOperation{} - if vp.current_frame == 0 || has_flag[VideoPlayerFlags](vp.flags, VideoPlayerFlags.e_decoder_reset) { + if vp.current_frame == 0 + || has_flag[VideoPlayerFlags](vp.flags, VideoPlayerFlags.e_decoder_reset) { decode_ope.flags = u32(DecoderVideoDecodeOperationFlags.e_session_reset) vp.flags &= ~u32(VideoPlayerFlags.e_decoder_reset) } @@ -61,11 +66,14 @@ fn (mut vp VideoPlayer) update_decode_video() ! { // picture sequence here. if frame_info.nal_unit_type == u8(h264.NAL_UNIT_TYPE.coded_slice_idr) { vp.dpb.reference_usage.clear() + vp.dpb.max_long_term_index = -1 } dpb_slot_num := int(vp.decoder.video_data.num_dpb_slots) vp.dpb.current_slot = vp.dpb.acquire_decode_slot(dpb_slot_num) - vp.dpb.poc_status[vp.dpb.current_slot] = int(frame_info.poc) + vp.dpb.long_term[vp.dpb.current_slot] = false + vp.dpb.poc_status[vp.dpb.current_slot] = frame_info.top_field_order_cnt + vp.dpb.bottom_poc_status[vp.dpb.current_slot] = frame_info.bottom_field_order_cnt vp.dpb.frame_num_status[vp.dpb.current_slot] = int(slice_header.frame_num) // Index variable on initialization comes in handy @@ -80,36 +88,27 @@ fn (mut vp VideoPlayer) update_decode_video() ! { upload_fence := vp.video_frames[use_frame_index].in_flight_fence res_wait := vk.wait_for_fences(dev_ctx.vk_device, 1, &upload_fence, vk._true, max_u64) check_vk(res_wait, 'Could not wait for decoded-frame fence') + vp.write_frame_readback(use_frame_index)! res_reset := vk.reset_fences(dev_ctx.vk_device, 1, &upload_fence) check_vk(res_reset, 'Could not reset decoded-frame fence') vp.video_frames[use_frame_index].gpu_bitstream_size = 0 vp.video_frames[use_frame_index].slice_offsets.clear() mut use_frame := &vp.video_frames[use_frame_index] vp.write_video_frame(mut use_frame)! - if use_frame.gpu_bitstream_size == 0 { - // MP4 samples may contain only metadata/non-slice NAL units. They do not - // form a Vulkan decode operation and must not be submitted with range 0. - vk.end_command_buffer(video_command_buffer) - if vp.current_frame + 1 < vp.decoder.video_data.frame_infos.len { - vp.current_frame++ - } else { - vp.decode_finished = true - } - vk.cmd_set_event(command_buffer_info.graphics_command_buffer, vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands)) - return - } mut flush_result := vk.Result.error_unknown lock vp.decoder { - flush_result = vp.app.device_context.memory_allocator.flush_range(vp.decoder.gpu_bitstream_allocation, use_frame.gpu_bitstream_offset, use_frame.gpu_bitstream_size) + flush_result = vp.app.device_context.memory_allocator.flush_range(vp.decoder.gpu_bitstream_allocation, + use_frame.gpu_bitstream_offset, use_frame.gpu_bitstream_size) } check_vk(flush_result, 'Could not flush the Vulkan Video bitstream buffer') decode_ope.stream_offset = use_frame.gpu_bitstream_offset decode_ope.stream_size = use_frame.gpu_bitstream_size - decode_ope.poc[0] = frame_info.poc - decode_ope.poc[1] = frame_info.poc + decode_ope.poc[0] = frame_info.top_field_order_cnt + decode_ope.poc[1] = frame_info.bottom_field_order_cnt decode_ope.frame_type = frame_info.frame_type decode_ope.reference_priority = frame_info.reference_priority + decode_ope.current_mmco5 = frame_info.has_mmco5 decode_ope.decoded_frame_index = vp.current_frame decode_ope.slice_header = slice_header decode_ope.pps = pps @@ -119,7 +118,25 @@ fn (mut vp VideoPlayer) update_decode_video() ! { decode_ope.dpb_reference_slots = vp.dpb.reference_usage.data // Pointer to data of fixed size array decode_ope.dpb_poc = &vp.dpb.poc_status[0] + decode_ope.dpb_bottom_poc = &vp.dpb.bottom_poc_status[0] decode_ope.dpb_frame_num = &vp.dpb.frame_num_status[0] + decode_ope.dpb_long_term = &vp.dpb.long_term[0] + decode_ope.dpb_long_term_index = &vp.dpb.long_term_index[0] + if frame_info.nal_unit_type == u8(h264.NAL_UNIT_TYPE.coded_slice_idr) { + decode_ope.current_long_term = slice_header.drpm.long_term_reference_flag != 0 + decode_ope.current_long_index = 0 + } else if frame_info.reference_priority > 0 + && slice_header.drpm.adaptive_ref_pic_marking_mode_flag != 0 { + for i in 0 .. slice_header.drpm.memory_management_control_operation.len { + op := slice_header.drpm.memory_management_control_operation[i] + if op == 0 { break + } + if op == 6 { + decode_ope.current_long_term = true + decode_ope.current_long_index = int(slice_header.drpm.long_term_frame_idx[i]) + } + } + } decode_ope.dpb_slot_num = u32(dpb_slot_num) decode_ope.p_dpbs = dpbs.data @@ -138,12 +155,12 @@ fn (mut vp VideoPlayer) update_decode_video() ! { vp.output_textures_ready << output_index output_queued = true - if frame_info.reference_priority > 0 { - vp.dpb.reference_usage << vp.dpb.current_slot - for vp.dpb.reference_usage.len > int(vp.decoder.video_data.max_reference_pictures) { - vp.dpb.reference_usage.delete(0) - } - } + // Reference marking takes effect after decoding; this picture used the + // previous DPB when the Vulkan command was recorded. + vp.dpb.mark_after_decode(slice_header, + frame_info.nal_unit_type == u8(h264.NAL_UNIT_TYPE.coded_slice_idr), + frame_info.reference_priority > 0, int(vp.decoder.video_data.max_reference_pictures), int(u32(1) << ( + sps.log2_max_frame_num_minus4 + 4)))! vk.end_command_buffer(video_command_buffer) mut frame_count := 0 rlock vp.decoder { @@ -181,7 +198,8 @@ fn (mut vp VideoPlayer) update_decode_video() ! { vp.output_textures[output_index].layout = .shader_read_only_optimal // Signal the application command buffer after the decode queue completes. - vk.cmd_set_event(command_buffer_info.graphics_command_buffer, vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands)) + vk.cmd_set_event(command_buffer_info.graphics_command_buffer, vp.event_video_player, + vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands)) } fn (mut vp VideoPlayer) copy_decoded_frame_to_output(command_buffer vk.CommandBuffer, output_index int) { @@ -282,6 +300,7 @@ fn (mut vp VideoPlayer) copy_decoded_frame_to_output(command_buffer vk.CommandBu pRegions: regions.data } vk.cmd_copy_image2(command_buffer, ©_info) + vp.record_frame_readback(command_buffer, source_image) output.layout = .transfer_dst_optimal output.is_new = false @@ -341,7 +360,8 @@ fn (mut vp VideoPlayer) video_decode_pre_barrier(video_command_buffer vk.Command mut image_barriers := []vk.ImageMemoryBarrier2{} decode_family := vp.app.device_context.get_decoder_queue_family_index() mut current_state := &vp.dpb.resource_state[vp.dpb.current_slot] - if current_state.layout != .video_decode_dpb_khr || current_state.flag != vk.access_2_video_decode_write_bit_khr { + if current_state.layout != .video_decode_dpb_khr + || current_state.flag != vk.access_2_video_decode_write_bit_khr { barrier := vk.ImageMemoryBarrier2{ srcStageMask: vk.pipeline_stage_2_video_decode_bit_khr srcAccessMask: current_state.flag @@ -364,7 +384,9 @@ fn (mut vp VideoPlayer) video_decode_pre_barrier(video_command_buffer vk.Command current_state.layout = barrier.newLayout current_state.flag = barrier.dstAccessMask } - if !vp.decoder.properties.dpb_and_output_coincide && (vp.decode_output_state.layout != .video_decode_dst_khr || vp.decode_output_state.flag != vk.access_2_video_decode_write_bit_khr) { + if !vp.decoder.properties.dpb_and_output_coincide + && (vp.decode_output_state.layout != .video_decode_dst_khr + || vp.decode_output_state.flag != vk.access_2_video_decode_write_bit_khr) { output_barrier := vk.ImageMemoryBarrier2{ srcStageMask: vk.pipeline_stage_2_all_commands_bit srcAccessMask: vp.decode_output_state.flag @@ -387,7 +409,8 @@ fn (mut vp VideoPlayer) video_decode_pre_barrier(video_command_buffer vk.Command } for ref_index in vp.dpb.reference_usage { mut ref_state := &vp.dpb.resource_state[ref_index] - if ref_state.layout != .video_decode_dpb_khr || ref_state.flag != vk.access_2_video_decode_read_bit_khr { + if ref_state.layout != .video_decode_dpb_khr + || ref_state.flag != vk.access_2_video_decode_read_bit_khr { barrier := vk.ImageMemoryBarrier2{ srcStageMask: vk.pipeline_stage_2_video_decode_bit_khr srcAccessMask: ref_state.flag @@ -444,13 +467,18 @@ fn (mut vp VideoPlayer) video_decode_core(operation &DecoderVideoDecodeOperation for i in 0 .. int(operation.dpb_slot_num) { pictures[i] = vk.VideoPictureResourceInfoKHR{ codedExtent: vk.Extent2D{ - width: vp.decoder.video_data.width - height: vp.decoder.video_data.height + width: vp.decoder.video_data.width_padd + height: vp.decoder.video_data.height_padd } baseArrayLayer: 0 imageViewBinding: vp.dpb.image[i].view } - C.vv_set_h264_reference_info(&reference_infos[i], u16(unsafe { operation.dpb_frame_num[i] }), unsafe { operation.dpb_poc[i] }, unsafe { operation.dpb_poc[i] }) + C.vv_set_h264_reference_info(&reference_infos[i], u16(if unsafe { operation.dpb_long_term[i] } { + unsafe { operation.dpb_long_term_index[i] } + } else { + unsafe { operation.dpb_frame_num[i] } + }), unsafe { operation.dpb_poc[i] }, unsafe { operation.dpb_bottom_poc[i] }) + reference_infos[i].flags.used_for_long_term_reference = u32(unsafe { operation.dpb_long_term[i] }) h264_slots[i] = vk.VideoDecodeH264DpbSlotInfoKHR{ pStdReferenceInfo: unsafe { &reference_infos[i] } } @@ -460,6 +488,19 @@ fn (mut vp VideoPlayer) video_decode_core(operation &DecoderVideoDecodeOperation pPictureResource: unsafe { &pictures[i] } } } + if operation.current_mmco5 { + minimum := if operation.poc[0] < operation.poc[1] { + operation.poc[0] + } else { + operation.poc[1] + } + C.vv_set_h264_reference_info(&reference_infos[operation.current_dpb], 0, + operation.poc[0] - minimum, operation.poc[1] - minimum) + } else if operation.current_long_term { + C.vv_set_h264_reference_info(&reference_infos[operation.current_dpb], + u16(operation.current_long_index), operation.poc[0], operation.poc[1]) + reference_infos[operation.current_dpb].flags.used_for_long_term_reference = 1 + } mut active_slots := [slot_count]vk.VideoReferenceSlotInfoKHR{} for i in 0 .. int(operation.dpb_reference_count) { @@ -492,8 +533,8 @@ fn (mut vp VideoPlayer) video_decode_core(operation &DecoderVideoDecodeOperation } else { vk.VideoPictureResourceInfoKHR{ codedExtent: vk.Extent2D{ - width: vp.decoder.video_data.width - height: vp.decoder.video_data.height + width: vp.decoder.video_data.width_padd + height: vp.decoder.video_data.height_padd } baseArrayLayer: 0 imageViewBinding: vp.decode_output_image.view @@ -521,16 +562,17 @@ fn (mut vp VideoPlayer) video_decode_core(operation &DecoderVideoDecodeOperation fn (mut vp VideoPlayer) write_video_frame(mut frame VideoPlayerDecodeStreamFrame) ! { data_frame := vp.decoder.video_data.frame_infos[vp.current_frame] mut frame_bytes_num_to_do := data_frame.frame_bytes_num + length_size := int(vp.decoder.video_data.nal_length_size) lock vp.decoder { vp.decoder.video_data.file.seek(data_frame.src_offset, .start) or { return error('could not seek to MP4 frame ${vp.current_frame}: ${err}') } } for frame_bytes_num_to_do > 0 { - if frame_bytes_num_to_do < 4 { + if frame_bytes_num_to_do < u64(length_size) { return error('MP4 frame ${vp.current_frame} has a truncated H.264 NAL length') } - mut src_buffer := []u8{len: 4} + mut src_buffer := []u8{len: length_size} mut length_bytes_read := 0 lock vp.decoder { length_bytes_read = vp.decoder.video_data.file.read(mut src_buffer) or { @@ -538,13 +580,13 @@ fn (mut vp VideoPlayer) write_video_frame(mut frame VideoPlayerDecodeStreamFrame } } if length_bytes_read != src_buffer.len { - return error('short read of H.264 NAL length in MP4 frame ${vp.current_frame}: expected 4 bytes, read ${length_bytes_read}') + return error('short read of H.264 NAL length in MP4 frame ${vp.current_frame}: expected ${length_size} bytes, read ${length_bytes_read}') } - mut size := u32(src_buffer[0]) << 24 | u32(src_buffer[1]) << 16 | u32(src_buffer[2]) << 8 | src_buffer[3] - size += 4 - if size < 4 || frame_bytes_num_to_do < size { - return error('MP4 frame ${vp.current_frame} has an invalid H.264 NAL size ${size - 4}') + nal_size := read_nal_length(src_buffer, 0, length_size)! + if u64(nal_size) > frame_bytes_num_to_do - u64(length_size) { + return error('MP4 frame ${vp.current_frame} has an invalid H.264 NAL size ${nal_size}') } + size := u64(nal_size) + u64(length_size) mut file := File(os.File{}) mut nal_header_byte := u8(0) lock vp.decoder { @@ -556,18 +598,19 @@ fn (mut vp VideoPlayer) write_video_frame(mut frame VideoPlayerDecodeStreamFrame mut nal := h264.NetworkAbstractionLayerHeader{} nal.read_nal_header(mut bs) // Skip over any frame data that is not idr slice or non-idr slice - if nal.type != h264.NAL_UNIT_TYPE.coded_slice_idr && nal.type != h264.NAL_UNIT_TYPE.coded_slice_non_idr { + if nal.type != h264.NAL_UNIT_TYPE.coded_slice_idr + && nal.type != h264.NAL_UNIT_TYPE.coded_slice_non_idr { frame_bytes_num_to_do -= size lock vp.decoder { - vp.decoder.video_data.file.seek(size - 4, .current) or { + vp.decoder.video_data.file.seek(nal_size, .current) or { return error('could not skip non-slice NAL in MP4 frame ${vp.current_frame}: ${err}') } } continue } - if frame.gpu_bitstream_size + size <= frame.gpu_bitstream_capacity { - nal_start_code := h264.NalStartCode{}.value + nal_start_code := h264.NalStartCode{}.value + if frame.gpu_bitstream_size + u64(nal_start_code.len) + u64(nal_size) <= frame.gpu_bitstream_capacity { frame.slice_offsets << u32(frame.gpu_bitstream_size) dst_buffer := unsafe { frame.gpu_bitstream_slice_mapped_memory_address + frame.gpu_bitstream_size @@ -576,27 +619,32 @@ fn (mut vp VideoPlayer) write_video_frame(mut frame VideoPlayerDecodeStreamFrame unsafe { vmemcpy(dst_buffer, nal_start_code.data, nal_start_code.len) } bytes_read := vp.decoder.video_data.file.read_into_ptr(unsafe { dst_buffer + nal_start_code.len - }, int(size - 4)) or { + }, int(nal_size)) or { return error('could not read H.264 NAL payload in MP4 frame ${vp.current_frame}: ${err}') } - if bytes_read != int(size - 4) { - return error('short read of H.264 NAL payload in MP4 frame ${vp.current_frame}: expected ${size - 4} bytes, read ${bytes_read}') + if bytes_read != int(nal_size) { + return error('short read of H.264 NAL payload in MP4 frame ${vp.current_frame}: expected ${nal_size} bytes, read ${bytes_read}') } } - frame.gpu_bitstream_size += u64(nal_start_code.len) + size - 4 + frame.gpu_bitstream_size += u64(nal_start_code.len) + u64(nal_size) } else { return error('encoded access unit ${vp.current_frame} requires more than its ${frame.gpu_bitstream_capacity}-byte aligned bitstream-buffer capacity (written=${frame.gpu_bitstream_size}, next_nal=${size})') } frame_bytes_num_to_do -= size } + if frame.gpu_bitstream_size == 0 { + return error('MP4 frame ${vp.current_frame} no longer contains a decodable H.264 slice') + } lock vp.decoder { - aligned_size := U64(frame.gpu_bitstream_size).align_to(vp.decoder.properties.caps.minBitstreamBufferSizeAlignment) + aligned_size := + U64(frame.gpu_bitstream_size).align_to(vp.decoder.properties.caps.minBitstreamBufferSizeAlignment) if aligned_size > frame.gpu_bitstream_capacity { return error('aligned access unit ${vp.current_frame} exceeds its ${frame.gpu_bitstream_capacity}-byte bitstream-buffer capacity') } if aligned_size > frame.gpu_bitstream_size { unsafe { - vmemset(frame.gpu_bitstream_slice_mapped_memory_address + frame.gpu_bitstream_size, 0, isize(aligned_size - frame.gpu_bitstream_size)) + vmemset(frame.gpu_bitstream_slice_mapped_memory_address + frame.gpu_bitstream_size, + 0, isize(aligned_size - frame.gpu_bitstream_size)) } } frame.gpu_bitstream_size = aligned_size diff --git a/player_presentation.v b/player_presentation.v index d6e108f..87ef600 100644 --- a/player_presentation.v +++ b/player_presentation.v @@ -65,6 +65,7 @@ fn (mut vp VideoPlayer) restart_decode_cycle() { vp.decode_finished = false vp.waiting_for_loop_start = true vp.dpb.reference_usage.clear() + vp.dpb.max_long_term_index = -1 vp.flags |= u32(VideoPlayerFlags.e_decoder_reset) vp.playback_timeline.reset() } @@ -144,8 +145,12 @@ fn (mut vp VideoPlayer) update(graphics_cmd_buffer vk.CommandBuffer, time_elapse } vp.update_presentation() - vk.cmd_wait_events(graphics_cmd_buffer, 1, &vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands), vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands), 0, unsafe { nil }, 0, unsafe { nil }, 0, unsafe { nil }) - vk.cmd_reset_event(graphics_cmd_buffer, vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.bottom_of_pipe)) + vk.cmd_wait_events(graphics_cmd_buffer, 1, &vp.event_video_player, + vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands), + vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands), 0, unsafe { nil }, 0, + unsafe { nil }, 0, unsafe { nil }) + vk.cmd_reset_event(graphics_cmd_buffer, vp.event_video_player, + vk.PipelineStageFlags(vk.PipelineStageFlagBits.bottom_of_pipe)) // Finish recording the command buffer and submit dev_ctx := vp.app.device_context @@ -165,7 +170,8 @@ fn (mut vp VideoPlayer) update(graphics_cmd_buffer vk.CommandBuffer, time_elapse pSignalSemaphores: &semaphore } upload_fence := vp.video_frames[vp.current_upload_index].in_flight_fence - res_video := vk.queue_submit(dev_ctx.get_queue(.video_decode), 1, &sumbit_info_video, upload_fence) + res_video := vk.queue_submit(dev_ctx.get_queue(.video_decode), 1, &sumbit_info_video, + upload_fence) check_vk(res_video, 'Could not submit Vulkan Video decode command') mut sumbit_info_graphics := vk.SubmitInfo{ @@ -175,6 +181,10 @@ fn (mut vp VideoPlayer) update(graphics_cmd_buffer vk.CommandBuffer, time_elapse commandBufferCount: 1 pCommandBuffers: &command_buffer_info.graphics_command_buffer } - res_graphics := vk.queue_submit(dev_ctx.get_queue(.graphics), 1, &sumbit_info_graphics, unsafe { nil }) + res_graphics := vk.queue_submit(dev_ctx.get_queue(.graphics), 1, &sumbit_info_graphics, + unsafe { nil }) check_vk(res_graphics, 'Could not submit decoded frame for graphics use') + if vp.decode_finished { + vp.finish_frame_readback() or { panic('Could not save decoded frames: ${err}') } + } } diff --git a/res/H264_multislice_320x180_1s.mp4 b/res/H264_multislice_320x180_1s.mp4 new file mode 100644 index 0000000..84cbdb1 Binary files /dev/null and b/res/H264_multislice_320x180_1s.mp4 differ diff --git a/res/H264_parameter_id_7_160x96_1s.mp4 b/res/H264_parameter_id_7_160x96_1s.mp4 new file mode 100644 index 0000000..6ba3e5e Binary files /dev/null and b/res/H264_parameter_id_7_160x96_1s.mp4 differ diff --git a/res/README.md b/res/README.md index b4f115b..1f77740 100644 --- a/res/README.md +++ b/res/README.md @@ -1,5 +1,30 @@ # Test media +`H264_parameter_id_7_160x96_1s.mp4` is a five-frame generated test pattern +whose SPS and PPS both use ID 7. It verifies that an H.264 parameter-set ID +does not have to equal its position in the MP4 parameter-set list. Its SHA-256 +digest is `48b32703d0b4f9f1bdfabde4635d5ae8f5804acb97635950da22e57e7da8cd59`. +Regenerate it with: + +```sh +ffmpeg -f lavfi -i 'testsrc2=size=160x96:rate=5' -frames:v 5 \ + -c:v libx264 -x264-params 'sps-id=7:keyint=5:min-keyint=5:scenecut=0' \ + -pix_fmt yuv420p H264_parameter_id_7_160x96_1s.mp4 +``` + +`H264_multislice_320x180_1s.mp4` is a generated test pattern with 24 frames +and four H.264 slices per frame. It exercises complete access-unit upload and +per-slice picture-consistency checks. Its SHA-256 digest is +`04daf20c9a1a903cf3ba258696d7365446056732ec25f9b6745e309355124db8`. +Regenerate it with: + +```sh +ffmpeg -f lavfi -i 'testsrc2=size=320x180:rate=24:duration=1' \ + -c:v libx264 -preset veryfast -crf 28 -pix_fmt yuv420p \ + -profile:v high -g 12 -bf 2 -x264-params 'slices=4:aud=1:scenecut=0' \ + -an -movflags +faststart H264_multislice_320x180_1s.mp4 +``` + `20240917_095400.mp4` is a self-recorded video supplied by the project owner for use as the player's default and regression fixture. The published copy is a metadata-stripped transcode containing only the video stream. It preserves diff --git a/scripts/compare_nv12.py b/scripts/compare_nv12.py new file mode 100644 index 0000000..5ad215e --- /dev/null +++ b/scripts/compare_nv12.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Compare display-order NV12 GPU dumps with FFmpeg's decoded frames.""" + +import argparse +import json +from pathlib import Path +import subprocess +import sys + + +def video_dimensions(path: Path) -> tuple[int, int]: + result = subprocess.run( + [ + "ffprobe", "-v", "error", "-select_streams", "v:0", + "-show_entries", "stream=width,height", "-of", "json", str(path), + ], + check=True, + capture_output=True, + text=True, + ) + stream = json.loads(result.stdout)["streams"][0] + return stream["width"], stream["height"] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("reference", type=Path, help="MP4 or original H.264 elementary stream") + parser.add_argument("dump_dir", type=Path, help="VV_DUMP_NV12_DIR from one playback loop") + parser.add_argument("--tolerance", type=int, default=0, help="maximum allowed difference per byte") + args = parser.parse_args() + if not 0 <= args.tolerance <= 255: + parser.error("--tolerance must be between 0 and 255") + width, height = video_dimensions(args.reference) + if width % 2 or height % 2: + parser.error("NV12 comparison requires even dimensions") + frame_size = width * height * 3 // 2 + frames = sorted(args.dump_dir.glob("*.nv12"), key=lambda path: int(path.stem)) + if not frames or [int(path.stem) for path in frames] != list(range(len(frames))): + parser.error("dump directory must contain contiguous 0.nv12, 1.nv12, ... files") + + command = [ + "ffmpeg", "-hide_banner", "-loglevel", "error", "-i", str(args.reference), + "-fps_mode", "passthrough", "-pix_fmt", "nv12", "-f", "rawvideo", "pipe:1", + ] + mismatched = 0 + max_difference = 0 + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) as process: + assert process.stdout is not None + for index, path in enumerate(frames): + gpu = path.read_bytes() + reference = process.stdout.read(frame_size) + if len(gpu) != frame_size or len(reference) != frame_size: + print(f"Frame {index}: expected {frame_size} bytes, got GPU={len(gpu)} FFmpeg={len(reference)}", file=sys.stderr) + process.terminate() + return 1 + difference = max(abs(a - b) for a, b in zip(gpu, reference)) + max_difference = max(max_difference, difference) + if difference > args.tolerance: + mismatched += 1 + if mismatched <= 5: + print(f"Frame {index}: maximum byte difference {difference}") + if process.stdout.read(1): + print("FFmpeg produced more frames than the GPU capture", file=sys.stderr) + process.terminate() + return 1 + if process.wait() != 0: + print("FFmpeg could not decode the reference input", file=sys.stderr) + return 1 + print(f"Compared {len(frames)} frames at {width}x{height}; mismatched={mismatched}, maximum byte difference={max_difference}") + return 1 if mismatched else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/swapchain.v b/swapchain.v index 2e3c6bc..dad7db3 100644 --- a/swapchain.v +++ b/swapchain.v @@ -168,7 +168,7 @@ fn (sc Swapchain) get_handle() vk.SwapchainKHR { return sc.swapchain } -fn (mut sc Swapchain) acquire_next_image(mut sem_present_complete vk.Semaphore) vk.Result { +fn (mut sc Swapchain) acquire_next_image(sem_present_complete vk.Semaphore) vk.Result { device_context := sc.app.device_context mut vk_device := device_context.get_vk_device() mut index := u32(0) diff --git a/video_player.c.v b/video_player.c.v index e83c8cf..e7fd5b3 100644 --- a/video_player.c.v +++ b/video_player.c.v @@ -16,11 +16,3 @@ fn C.vv_set_h264_reference_info(voidptr, u16, i32, i32) $if windows { #define VK_USE_PLATFORM_WIN32_KHR } -/* -#include "Volk/volk.h" - -// TODO: Check if needed -#undef ERROR -#undef min -#undef max -*/ diff --git a/video_player.v b/video_player.v index 00efc6c..4e2078a 100644 --- a/video_player.v +++ b/video_player.v @@ -4,6 +4,7 @@ import antono2.vulkan as vk import os import math import antono2.vkmemalloc as vkmem +import antono2.h264 const max_texture_count = 64 const slot_count = 17 @@ -61,6 +62,9 @@ mut: current_frame int flags u32 video_frames []VideoPlayerDecodeStreamFrame + frame_readbacks []FrameReadback + frame_readback_dir string + frame_readback_done bool decode_output_image Image decode_output_state DPBResourceState playback_timeline PlaybackTimeline @@ -137,20 +141,22 @@ enum VideoPlayerFlags as u32 { struct DPB { pub mut: - image [slot_count]Image - resource_state [slot_count]DPBResourceState - poc_status [slot_count]int - frame_num_status [slot_count]int - reference_usage []u8 - next_ref u8 - next_slot u8 - current_slot u8 + image [slot_count]Image + resource_state [slot_count]DPBResourceState + poc_status [slot_count]int + bottom_poc_status [slot_count]int + frame_num_status [slot_count]int + reference_usage []u8 + long_term [slot_count]bool + long_term_index [slot_count]int + max_long_term_index int = -1 + next_ref u8 + next_slot u8 + current_slot u8 } // Choose a slot which is not referenced by the picture being decoded. When all -// slots are references, expire the oldest short-term reference first (the H.264 -// sliding-window default). Explicit MMCO and long-term references are handled -// separately as stream metadata becomes available. +// slots are references, expire the oldest short-term reference first. fn (mut dpb DPB) acquire_decode_slot(slot_limit int) u8 { assert slot_limit > 0 && slot_limit <= slot_count for slot in 0 .. slot_limit { @@ -164,6 +170,138 @@ fn (mut dpb DPB) acquire_decode_slot(slot_limit int) u8 { return expired_slot } +fn (mut dpb DPB) finish_mmco5() { + dpb.reference_usage.clear() + dpb.max_long_term_index = -1 + minimum := math.min(dpb.poc_status[dpb.current_slot], dpb.bottom_poc_status[dpb.current_slot]) + dpb.poc_status[dpb.current_slot] -= minimum + dpb.bottom_poc_status[dpb.current_slot] -= minimum + dpb.frame_num_status[dpb.current_slot] = 0 +} + +fn (mut dpb DPB) remove_reference(slot u8) { + for i, active in dpb.reference_usage { + if active == slot { + dpb.reference_usage.delete(i) + break + } + } + dpb.long_term[slot] = false +} + +fn (mut dpb DPB) remove_long_term_index(index int) { + for slot in dpb.reference_usage.clone() { + if dpb.long_term[slot] && dpb.long_term_index[slot] == index { + dpb.remove_reference(slot) + } + } +} + +fn (dpb &DPB) short_term_slot(pic_num int, curr_frame_num int, max_frame_num int) ?u8 { + for slot in dpb.reference_usage { + if dpb.long_term[slot] { + continue + } + frame_num := dpb.frame_num_status[slot] + frame_num_wrap := if frame_num > curr_frame_num { + frame_num - max_frame_num + } else { + frame_num + } + if frame_num_wrap == pic_num { + return slot + } + } + return none +} + +fn (mut dpb DPB) mark_after_decode(header &h264.SliceHeader, is_idr bool, + is_reference bool, max_references int, max_frame_num int) ! { + if !is_reference { + return + } + dpb.long_term[dpb.current_slot] = false + if is_idr { + dpb.reference_usage.clear() + dpb.max_long_term_index = if header.drpm.long_term_reference_flag != 0 { 0 } else { -1 } + if header.drpm.long_term_reference_flag != 0 { + dpb.long_term[dpb.current_slot] = true + dpb.long_term_index[dpb.current_slot] = 0 + } + } else if header.drpm.adaptive_ref_pic_marking_mode_flag != 0 { + for i in 0 .. header.drpm.memory_management_control_operation.len { + op := header.drpm.memory_management_control_operation[i] + if op == 0 { + break + } + pic_num_x := int(header.frame_num) - int(header.drpm.difference_of_pic_nums_minus1[i]) - + 1 + if op in [u32(1), 3] { + if pic_num_x < -max_frame_num { + return error('invalid H.264 MMCO short-term picture number') + } + } + match op { + 1 { + if slot := dpb.short_term_slot(pic_num_x, int(header.frame_num), max_frame_num) { + dpb.remove_reference(slot) + } + } + 2 { + dpb.remove_long_term_index(int(header.drpm.long_term_pic_num[i])) + } + 3 { + slot := dpb.short_term_slot(pic_num_x, int(header.frame_num), max_frame_num) or { + return error('H.264 MMCO 3 refers to an unavailable short-term picture') + } + index := int(header.drpm.long_term_frame_idx[i]) + if index > dpb.max_long_term_index { + return error('H.264 MMCO 3 long-term index ${index} exceeds ${dpb.max_long_term_index}') + } + dpb.remove_long_term_index(index) + dpb.long_term[slot] = true + dpb.long_term_index[slot] = index + } + 4 { + dpb.max_long_term_index = int(header.drpm.max_long_term_frame_idx_plus1[i]) - 1 + for slot in dpb.reference_usage.clone() { + if dpb.long_term[slot] + && dpb.long_term_index[slot] > dpb.max_long_term_index { + dpb.remove_reference(slot) + } + } + } + 5 { + dpb.finish_mmco5() + } + 6 { + index := int(header.drpm.long_term_frame_idx[i]) + if index > dpb.max_long_term_index { + return error('H.264 MMCO 6 long-term index ${index} exceeds ${dpb.max_long_term_index}') + } + dpb.remove_long_term_index(index) + dpb.long_term[dpb.current_slot] = true + dpb.long_term_index[dpb.current_slot] = index + } + else { + return error('invalid H.264 MMCO ${op} at frame_num ${header.frame_num}, operation ${i}') + } + } + } + } else if dpb.reference_usage.len >= max_references { + for slot in dpb.reference_usage { + if !dpb.long_term[slot] { + dpb.remove_reference(slot) + break + } + } + } + dpb.reference_usage << dpb.current_slot + if dpb.reference_usage.len > max_references { + return error('H.264 reference marking exceeds ${max_references} active pictures') + } +} + struct DPBResourceState { pub mut: flag vk.AccessFlags2 @@ -222,8 +360,10 @@ pub mut: frame_bytes_num u64 size u64 poc int - bottom_field_order_cnt u32 - top_field_order_cnt u32 + decode_poc int + has_mmco5 bool + bottom_field_order_cnt int + top_field_order_cnt int gop int display_order int decode_time_ns i64 @@ -236,12 +376,16 @@ pub mut: } fn compare_frame_display_order(a &DecoderVideoDataFrameInfo, b &DecoderVideoDataFrameInfo) int { - key_a := u64(a.gop) << 32 | u64(a.poc) - key_b := u64(b.gop) << 32 | u64(b.poc) - if key_a < key_b { + if a.gop < b.gop { + return -1 + } + if a.gop > b.gop { + return 1 + } + if a.poc < b.poc { return -1 } - if key_a > key_b { + if a.poc > b.poc { return 1 } return 0 @@ -267,16 +411,19 @@ fn presentation_buffer_size(display_orders []int) int { struct DecoderVideoFileProperties { pub mut: - file os.File - file_open bool - h264_profile_idc u32 - width_padd u32 - height_padd u32 - width u32 - height u32 - sps_count u32 - pps_count u32 - slice_header_count u32 + file os.File + file_open bool + h264_profile_idc u32 + h264_level_idc u32 + width_padd u32 + height_padd u32 + width u32 + height u32 + sps_count u32 + pps_count u32 + nal_length_size u32 + sps_storage_index [32]u8 + pps_storage_index [256]u16 frame_infos []DecoderVideoDataFrameInfo max_memory_frame_size_bytes u64 @@ -292,6 +439,20 @@ pub mut: metadata VideoMetadata } +fn (data &DecoderVideoFileProperties) sps_storage_offset(id u32) !int { + if id >= u32(data.sps_storage_index.len) || data.sps_storage_index[id] == 0 { + return error('H.264 references missing SPS ${id}') + } + return int(data.sps_storage_index[id] - 1) * int(sizeof(h264.SequenceParameterSet)) +} + +fn (data &DecoderVideoFileProperties) pps_storage_offset(id u32) !int { + if id >= u32(data.pps_storage_index.len) || data.pps_storage_index[id] == 0 { + return error('H.264 references missing PPS ${id}') + } + return int(data.pps_storage_index[id] - 1) * int(sizeof(h264.PictureParameterSet)) +} + struct VideoMetadata { pub mut: coded_width u32 @@ -375,10 +536,8 @@ fn (mut metadata VideoMetadata) update_display_dimensions() { struct DecoderDpbImage { pub mut: - image vk.Image - view vk.ImageView - // TODO: Refactor Allocator to Decoder - allocator vkmem.Allocator + image vk.Image + view vk.ImageView allocation_info vkmem.AllocationInfo } @@ -396,9 +555,15 @@ pub mut: poc [2]int current_dpb u32 dpb_reference_count u32 - dpb_reference_slots &u8 = unsafe { nil } - dpb_poc &int = unsafe { nil } - dpb_frame_num &int = unsafe { nil } + dpb_reference_slots &u8 = unsafe { nil } + dpb_poc &int = unsafe { nil } + dpb_bottom_poc &int = unsafe { nil } + dpb_frame_num &int = unsafe { nil } + dpb_long_term &bool = unsafe { nil } + dpb_long_term_index &int = unsafe { nil } + current_long_term bool + current_long_index int + current_mmco5 bool dpb_slot_num u32 p_dpbs vk.Image p_dpb_views vk.ImageView @@ -411,7 +576,6 @@ enum DecoderVideoDecodeOperationFlags { e_session_reset = 1 } -// TODO: May be worth to use interface types, but interfaces IApp containg sub interface IDeviceContext "error: `&video_decode_app.VideoDecodeApp` incorrectly implements field `device_context` of interface `examples.video_decode_app.video_player.IApp`, expected `video_player.IDeviceContext`, got `video_decode_app.DeviceContext`", no matter what's in the interface fn (mut vp VideoPlayer) prepare(path string) ! { // Do not propagate parser errors from inside the lock: cleanup must be able // to reacquire it and close a partially opened input file. @@ -439,6 +603,20 @@ fn (vp &VideoPlayer) h264_profile_idc() u32 { } } +fn (vp &VideoPlayer) decode_requirements() VideoDecodeRequirements { + rlock vp.decoder { + data := vp.decoder.video_data + return VideoDecodeRequirements{ + profile_idc: data.h264_profile_idc + level_idc: data.h264_level_idc + width: data.width_padd + height: data.height_padd + dpb_slots: data.num_dpb_slots + references: data.num_dpb_slots - 1 + } + } +} + fn (vp &VideoPlayer) metadata() VideoMetadata { rlock vp.decoder { return vp.decoder.video_data.metadata @@ -468,7 +646,8 @@ fn (mut vp VideoPlayer) initialize(mut app VideoDecodeApp) { allocation_info: image.allocation_info } } - vp.presentation_buffer_count = presentation_buffer_size(vp.decoder.video_data.frame_infos.map(it.display_order)) + vp.presentation_buffer_count = + presentation_buffer_size(vp.decoder.video_data.frame_infos.map(it.display_order)) } vk_device := app.device_context.vk_device @@ -476,17 +655,20 @@ fn (mut vp VideoPlayer) initialize(mut app VideoDecodeApp) { fence_ci := vk.FenceCreateInfo{ flags: vk.FenceCreateFlags(vk.FenceCreateFlagBits.signaled) } - res := vk.create_fence(vk_device, &fence_ci, unsafe { nil }, &vp.video_frames[i].in_flight_fence) + res := vk.create_fence(vk_device, &fence_ci, unsafe { nil }, + &vp.video_frames[i].in_flight_fence) check_vk(res, 'Could not create video-frame fence ${i}') } mut command_pool_ci := vk.CommandPoolCreateInfo{ flags: vk.CommandPoolCreateFlags(vk.CommandPoolCreateFlagBits.reset_command_buffer) } command_pool_ci.queueFamilyIndex = app.device_context.graphics_family - mut res := vk.create_command_pool(vk_device, &command_pool_ci, unsafe { nil }, &vp.graphics_command_pool) + mut res := vk.create_command_pool(vk_device, &command_pool_ci, unsafe { nil }, + &vp.graphics_command_pool) check_vk(res, 'Could not create video-player graphics command pool') command_pool_ci.queueFamilyIndex = app.device_context.get_decoder_queue_family_index() - res = vk.create_command_pool(vk_device, &command_pool_ci, unsafe { nil }, &vp.video_command_pool) + res = vk.create_command_pool(vk_device, &command_pool_ci, unsafe { nil }, + &vp.video_command_pool) check_vk(res, 'Could not create video-decode command pool') vp.command_buffer_infos = []CommandBufferInfo{len: app.device_context.swapchain.image_views.len} @@ -521,6 +703,7 @@ fn (mut vp VideoPlayer) initialize(mut app VideoDecodeApp) { for _ in 0 .. output_texture_count { vp.create_output_image() } + vp.initialize_frame_readback() println('Presentation queue: ${vp.presentation_buffer_count} reorder images, ${output_texture_count} images total') if !vp.decoder.properties.dpb_and_output_coincide { vp.create_decode_output_image() @@ -531,7 +714,8 @@ fn (mut vp VideoPlayer) recreate_swapchain_resources() { vk_device := vp.app.device_context.vk_device for mut info in vp.command_buffer_infos { if !isnil(info.graphics_command_buffer) { - vk.free_command_buffers(vk_device, vp.graphics_command_pool, 1, &info.graphics_command_buffer) + vk.free_command_buffers(vk_device, vp.graphics_command_pool, 1, + &info.graphics_command_buffer) } if !isnil(info.video_command_buffer) { vk.free_command_buffers(vk_device, vp.video_command_pool, 1, &info.video_command_buffer) @@ -547,12 +731,14 @@ fn (mut vp VideoPlayer) recreate_swapchain_resources() { commandBufferCount: 1 commandPool: vp.graphics_command_pool } - mut result := vk.allocate_command_buffers(vk_device, &alloc_info, &info.graphics_command_buffer) + mut result := vk.allocate_command_buffers(vk_device, &alloc_info, + &info.graphics_command_buffer) check_vk(result, 'Could not reallocate video-player graphics command buffer') alloc_info.commandPool = vp.video_command_pool result = vk.allocate_command_buffers(vk_device, &alloc_info, &info.video_command_buffer) check_vk(result, 'Could not reallocate video-decode command buffer') - result = vk.create_semaphore(vk_device, &vk.SemaphoreCreateInfo{}, unsafe { nil }, &info.sem_video_to_gfx) + result = vk.create_semaphore(vk_device, &vk.SemaphoreCreateInfo{}, unsafe { nil }, + &info.sem_video_to_gfx) check_vk(result, 'Could not recreate video-to-graphics semaphore') } // The caller waits for device idle before rebuilding the swapchain, so every @@ -598,6 +784,7 @@ fn (mut vp VideoPlayer) shutdown() { frame.in_flight_fence = unsafe { nil } } } + vp.release_frame_readback() lock vp.decoder { for mut output in vp.output_textures { if !isnil(output.texture.view) { @@ -622,7 +809,8 @@ fn (mut vp VideoPlayer) shutdown() { vk.destroy_image(vk_device, vp.decode_output_image.image, unsafe { nil }) vp.decode_output_image.image = unsafe { nil } } - _ = vp.app.device_context.memory_allocator.release(mut vp.decode_output_image.allocation_info) + _ = + vp.app.device_context.memory_allocator.release(mut vp.decode_output_image.allocation_info) for mut dpb in vp.decoder.info.images_dpb { if !isnil(dpb.view) { vk.destroy_image_view(vk_device, dpb.view, unsafe { nil }) @@ -641,7 +829,8 @@ fn (mut vp VideoPlayer) shutdown() { } _ = vp.app.device_context.memory_allocator.release(mut vp.decoder.gpu_bitstream_allocation) if !isnil(vp.decoder.video_session_parameters) { - vk.destroy_video_session_parameters_khr(vk_device, vp.decoder.video_session_parameters, unsafe { nil }) + vk.destroy_video_session_parameters_khr(vk_device, vp.decoder.video_session_parameters, + unsafe { nil }) vp.decoder.video_session_parameters = unsafe { nil } } if !isnil(vp.decoder.video_session) { @@ -724,8 +913,8 @@ fn (mut vp VideoPlayer) create_decode_output_image() { imageType: ._2d format: vp.decoder.properties.format_props.format extent: vk.Extent3D{ - width: vp.decoder.video_data.width - height: vp.decoder.video_data.height + width: vp.decoder.video_data.width_padd + height: vp.decoder.video_data.height_padd depth: 1 } mipLevels: 1 @@ -752,7 +941,8 @@ fn (mut vp VideoPlayer) create_decode_output_image() { layerCount: 1 } } - result = vk.create_image_view(dev_ctx.vk_device, &view_ci, unsafe { nil }, &vp.decode_output_image.view) + result = vk.create_image_view(dev_ctx.vk_device, &view_ci, unsafe { nil }, + &vp.decode_output_image.view) check_vk(result, 'Could not create distinct video decode-output image view') } @@ -764,13 +954,15 @@ fn query_video_format(gpu vk.PhysicalDevice, profile_list &vk.VideoProfileListIn } mut count := u32(0) mut no_formats := unsafe { nil } - mut result := vk.get_physical_device_video_format_properties_khr(gpu, &format_info, &count, mut no_formats) + mut result := vk.get_physical_device_video_format_properties_khr(gpu, &format_info, &count, mut + no_formats) if result != .success || count == 0 { return none } mut formats := []vk.VideoFormatPropertiesKHR{len: int(count), init: vk.VideoFormatPropertiesKHR{}} mut formats_data := formats.data - result = vk.get_physical_device_video_format_properties_khr(gpu, &format_info, &count, mut formats_data) + result = vk.get_physical_device_video_format_properties_khr(gpu, &format_info, &count, mut + formats_data) if result != .success || count == 0 { return none } diff --git a/video_player_test.v b/video_player_test.v index 8057251..5ff4206 100644 --- a/video_player_test.v +++ b/video_player_test.v @@ -1,6 +1,8 @@ module main import antono2.minimp4 +import antono2.h264 +import encoding.hex import os import antono2.vulkan as vk @@ -20,6 +22,315 @@ fn test_dpb_acquire_expires_oldest_reference_when_full() { assert dpb.reference_usage == [u8(0), 1] } +fn test_mmco5_discards_old_references_after_decode_and_renumbers_current_picture() { + mut dpb := DPB{ + current_slot: 2 + reference_usage: [u8(0), 1] + } + dpb.poc_status[2] = 14 + dpb.bottom_poc_status[2] = 14 + dpb.frame_num_status[2] = 7 + dpb.finish_mmco5() + assert dpb.reference_usage.len == 0 + assert dpb.poc_status[2] == 0 + assert dpb.bottom_poc_status[2] == 0 + assert dpb.frame_num_status[2] == 0 + assert dpb.poc_status[0] == 0 +} + +fn test_progressive_field_order_counts_remain_distinct() { + mut state := PictureOrderCountType0State{} + result := state.advance(6, -2, 16, true, true, false) + assert result.top == 6 + assert result.bottom == 4 + assert result.display_poc == 4 + mut dpb := DPB{ + current_slot: 1 + } + dpb.poc_status[1] = 14 + dpb.bottom_poc_status[1] = 16 + dpb.finish_mmco5() + assert dpb.poc_status[1] == 0 + assert dpb.bottom_poc_status[1] == 2 +} + +fn test_nal_length_prefixes_and_invalid_prefixes() { + for width in [1, 2, 4] { + mut sample := []u8{len: width} + sample[width - 1] = 2 + sample << [u8(0x65), 0x80] + assert detect_nal_length_size(sample)! == u32(width) + assert read_nal_length(sample, 0, width)! == 2 + } + if _ := detect_nal_length_size([u8(0), 2, 0xe5, 0x80]) { + assert false, 'forbidden NAL header was accepted' + } +} + +fn test_h264_sps_levels_map_to_vulkan_ordinals() { + assert std_h264_level_idc(10) == ._1_0 + assert std_h264_level_idc(31) == ._3_1 + assert std_h264_level_idc(40) == ._4_0 + assert std_h264_level_idc(62) == ._6_2 + assert std_h264_level_idc(49) == .invalid + assert h264_level_issue(31, ._4_0) == '' + assert h264_level_issue(42, ._4_0).contains('requires H.264 level 4.2') +} + +fn test_parameter_set_preflight_rejects_truncation_and_oversized_arrays() { + mut truncated := false + validate_sps_rbsp([u8(0x42), 0, 0x1f]) or { + assert err.msg().contains('truncated') + truncated = true + } + assert truncated + mut oversized := false + validate_sps_rbsp([u8(0x42), 0, 0x1f, 0xd3, 0, 0x81, 0x40]) or { + assert err.msg().contains('more than 256') + oversized = true + } + assert oversized + validate_pps_rbsp([u8(0xc5)]) or { + assert err.msg().contains('slice groups') + return + } + assert false, 'unsupported H.264 slice groups were accepted' +} + +fn test_sps_dimensions_reject_overflow_and_crop_past_picture() { + mut sps := h264.SequenceParameterSet{ + pic_width_in_mbs_minus1: 19 + pic_height_in_map_units_minus1: 11 + } + width, height, padded_width, padded_height := progressive_h264_dimensions(&sps) or { + panic(err) + } + assert width == 320 && height == 192 + assert padded_width == 320 && padded_height == 192 + sps.frame_crop_left_offset = 160 + mut rejected := false + progressive_h264_dimensions(&sps) or { + assert err.msg().contains('crop offsets') + rejected = true + } + assert rejected + sps.frame_crop_left_offset = 0 + sps.pic_width_in_mbs_minus1 = 0xffffffff + rejected = false + progressive_h264_dimensions(&sps) or { + assert err.msg().contains('dimensions') + rejected = true + } + assert rejected +} + +fn test_extra_slice_must_belong_to_first_picture() { + first := h264.SliceHeader{ + pic_parameter_set_id: 1 + frame_num: 3 + pic_order_cnt_lsb: 6 + } + mut next := first + first_nal := h264.NetworkAbstractionLayerHeader{ + idc: .priority_high + type: .coded_slice_non_idr + } + next_nal := first_nal + validate_same_picture(&first, &next, &first_nal, &next_nal) or { panic(err) } + next.frame_num = 4 + validate_same_picture(&first, &next, &first_nal, &next_nal) or { + assert err.msg().contains('different pictures') + return + } + assert false, 'a slice from another picture was accepted' +} + +fn test_slice_parameter_set_references_are_validated_before_full_parse() { + mut pps := h264.PictureParameterSet{} + pps.seq_parameter_set_id = 0 + validate_slice_parameter_sets([u8(0xd0)], [pps], [h264.SequenceParameterSet{}]) or { + assert err.msg().contains('missing PPS 1') + return + } + assert false, 'missing PPS was accepted' +} + +fn test_slice_reader_rejects_invalid_slice_type_before_parameter_lookup() { + mut bits := h264.Bitstream{} + bits.init([u8(0x8b)]) // first_mb_in_slice = 0, slice_type = 10 + nal := h264.NetworkAbstractionLayerHeader{} + read_slice_header_checked(&nal, []h264.PictureParameterSet{}, []h264.SequenceParameterSet{}, mut + bits) or { + assert err.msg().contains('invalid H.264 slice type') + return + } + assert false, 'invalid slice type was accepted' +} + +fn test_slice_parameter_set_ids_need_not_match_array_offsets() { + pps := h264.PictureParameterSet{ + pic_parameter_set_id: 3 + seq_parameter_set_id: 7 + } + sps := h264.SequenceParameterSet{ + seq_parameter_set_id: 7 + } + validate_slice_parameter_sets([u8(0xb2), 0], [pps], [sps]) or { panic(err) } + assert h264_pps_by_id([pps], 3)!.seq_parameter_set_id == 7 + mut data := DecoderVideoFileProperties{} + data.sps_storage_index[7] = 2 + data.pps_storage_index[3] = 2 + assert data.sps_storage_offset(7)! == int(sizeof(h264.SequenceParameterSet)) + assert data.pps_storage_offset(3)! == int(sizeof(h264.PictureParameterSet)) + if _ := data.pps_storage_offset(1) { + assert false, 'missing PPS id resolved to a serialized offset' + } +} + +fn test_parameter_set_preflight_handles_truncated_and_mutated_headers() { + sps_ebsp := [u8(0x64), 0, 0x0c, 0xac, 0xd9, 0x41, 0x41, 0x9f, 0x9f, 0x01, 0x10, 0, 0, 0x03, + 0, 0x10, 0, 0, 0x03, 0x03, 0, 0xf1, 0x42, 0x99, 0x60] + sps := unsafe { remove_emulation_prevention_bytes(sps_ebsp.data, sps_ebsp.len) } + pps := [u8(0xef), 0x89, 0xcb] + validate_sps_rbsp(sps) or { panic(err) } + validate_pps_rbsp(pps) or { panic(err) } + for prefix in 0 .. sps.len { + if _ := validate_sps_rbsp(sps[..prefix]) { + assert false, 'truncated SPS prefix ${prefix} was accepted' + } + } + for prefix in 0 .. pps.len { + if _ := validate_pps_rbsp(pps[..prefix]) { + assert false, 'truncated PPS prefix ${prefix} was accepted' + } + } + for bit in 0 .. sps.len * 8 { + mut changed := sps.clone() + changed[bit / 8] ^= u8(1 << (bit % 8)) + validate_sps_rbsp(changed) or { continue } + } + for bit in 0 .. pps.len * 8 { + mut changed := pps.clone() + changed[bit / 8] ^= u8(1 << (bit % 8)) + validate_pps_rbsp(changed) or { continue } + } +} + +fn test_picture_order_count_type_one_uses_cycle_and_nonreference_offset() { + mut sps := h264.SequenceParameterSet{ + pic_order_cnt_type: 1 + num_ref_frames_in_pic_order_cnt_cycle: 2 + offset_for_non_ref_pic: -1 + offset_for_top_to_bottom_field: 1 + } + sps.offset_for_ref_frame[0] = 2 + sps.offset_for_ref_frame[1] = 2 + mut header := h264.SliceHeader{ + frame_num: 3 + } + header.delta_pic_order_cnt[0] = -1 + top, bottom := poc_type1_fields(&sps, &header, 0, true) or { panic(err) } + assert top == 5 && bottom == 6 + header.frame_num = 4 + nonref_top, nonref_bottom := poc_type1_fields(&sps, &header, 0, false) or { panic(err) } + assert nonref_top == 4 && nonref_bottom == 5 +} + +fn test_dpb_applies_short_and_long_term_reference_marking() { + mut dpb := DPB{ + reference_usage: [u8(0), 1] + current_slot: 2 + max_long_term_index: 1 + } + dpb.frame_num_status[0] = 3 + dpb.frame_num_status[1] = 4 + mut header := h264.SliceHeader{ + frame_num: 5 + } + header.drpm.adaptive_ref_pic_marking_mode_flag = 1 + header.drpm.memory_management_control_operation[0] = 1 + header.drpm.difference_of_pic_nums_minus1[0] = 0 + dpb.frame_num_status[2] = 5 + dpb.mark_after_decode(&header, false, true, 3, 16) or { panic(err) } + assert dpb.reference_usage == [u8(0), 2] + + dpb.current_slot = 1 + dpb.frame_num_status[1] = 6 + header = h264.SliceHeader{ + frame_num: 6 + } + header.drpm.adaptive_ref_pic_marking_mode_flag = 1 + header.drpm.memory_management_control_operation[0] = 3 + header.drpm.difference_of_pic_nums_minus1[0] = 2 + header.drpm.long_term_frame_idx[0] = 1 + header.drpm.memory_management_control_operation[1] = 6 + header.drpm.long_term_frame_idx[1] = 0 + dpb.mark_after_decode(&header, false, true, 3, 16) or { panic(err) } + assert dpb.reference_usage == [u8(0), 2, 1] + assert dpb.long_term[0] && dpb.long_term_index[0] == 1 + assert !dpb.long_term[2] + assert dpb.long_term[1] && dpb.long_term_index[1] == 0 + + dpb.current_slot = 3 + header = h264.SliceHeader{ + frame_num: 7 + } + header.drpm.adaptive_ref_pic_marking_mode_flag = 1 + header.drpm.memory_management_control_operation[0] = 2 + header.drpm.long_term_pic_num[0] = 0 + header.drpm.memory_management_control_operation[1] = 4 + header.drpm.max_long_term_frame_idx_plus1[1] = 1 + dpb.mark_after_decode(&header, false, true, 3, 16) or { panic(err) } + assert dpb.reference_usage == [u8(2), 3] + assert !dpb.long_term[1] && !dpb.long_term[0] +} + +fn test_mmco5_is_found_only_in_a_reference_non_idr_slice() { + mut slice_header := h264.SliceHeader{} + slice_header.drpm.adaptive_ref_pic_marking_mode_flag = 1 + slice_header.drpm.memory_management_control_operation[0] = 1 + slice_header.drpm.memory_management_control_operation[1] = 5 + assert slice_has_mmco5(&slice_header, false, .priority_high) + assert !slice_has_mmco5(&slice_header, true, .priority_high) + assert !slice_has_mmco5(&slice_header, false, .priority_disposable) + slice_header.drpm.memory_management_control_operation[0] = 0 + assert !slice_has_mmco5(&slice_header, false, .priority_high) +} + +fn test_mmco5_starts_a_new_display_group_and_preserves_pre_reset_decode_poc() { + mut state := PictureOrderCountType0State{} + idr := state.advance(0, 0, 16, true, true, false) + assert idr.decode_poc == 0 && idr.display_poc == 0 && idr.cycle == 0 + previous := state.advance(6, 0, 16, false, true, false) + assert previous.decode_poc == 6 && previous.cycle == 0 + reset := state.advance(14, 0, 16, false, true, true) + assert reset.decode_poc == 14 + assert reset.display_poc == 0 && reset.cycle == 1 + assert state.prev_msb == 0 && state.prev_lsb == 14 + next := state.advance(1, 0, 16, false, true, false) + assert next.decode_poc == 17 && next.display_poc == 17 && next.cycle == 1 + assert compare_frame_display_order(&DecoderVideoDataFrameInfo{ + gop: previous.cycle + poc: previous.display_poc + }, &DecoderVideoDataFrameInfo{ + gop: reset.cycle + poc: reset.display_poc + }) < 0 +} + +fn test_display_order_compares_signed_picture_counts_within_a_group() { + before := DecoderVideoDataFrameInfo{ + gop: 2 + poc: -2 + } + after := DecoderVideoDataFrameInfo{ + gop: 2 + poc: 1 + } + assert compare_frame_display_order(&before, &after) < 0 + assert compare_frame_display_order(&after, &before) > 0 +} + fn test_decode_output_mode_selection_prefers_coincident_in_auto_mode() { assert select_decode_output_mode(.automatic, true, true)! == .coincident assert select_decode_output_mode(.automatic, false, true)! == .distinct @@ -164,15 +475,16 @@ fn test_parser_accepts_available_h264_resolution_and_rate_samples() { assert decoder.video_data.h264_profile_idc == 100 assert decoder.video_data.frame_infos.len > 250 assert decoder.video_data.total_duration >= 9_000_000_000 + for frame in decoder.video_data.frame_infos { + assert frame.size <= decoder.video_data.max_memory_frame_size_bytes + } decoder.video_data.file.close() } } fn test_parser_accepts_supported_elephants_dream_720p_sample() { mut decoder := Decoder{} - decoder.parse_mp4_data('${v_modroot}/res/Elephants_Dream_720p30_8s_CC-BY.mp4') or { - panic(err) - } + decoder.parse_mp4_data('${v_modroot}/res/Elephants_Dream_720p30_8s_CC-BY.mp4') or { panic(err) } defer { decoder.video_data.file.close() } @@ -189,11 +501,25 @@ fn test_parser_accepts_supported_elephants_dream_720p_sample() { } } -fn test_parser_orders_type_zero_b_frames_within_their_gop() { +fn test_parser_accepts_four_slices_per_picture() { mut decoder := Decoder{} - decoder.parse_mp4_data('${v_modroot}/res/Big_Buck_Bunny_360_10s_1MB.mp4') or { - panic(err) + decoder.parse_mp4_data('${v_modroot}/res/H264_multislice_320x180_1s.mp4') or { panic(err) } + defer { + decoder.video_data.file.close() + } + assert decoder.video_data.frame_infos.len == 24 + assert decoder.video_data.h264_level_idc == 12 + assert decoder.video_data.sps_storage_offset(0)! == 0 + assert decoder.video_data.pps_storage_offset(0)! == 0 + for frame in decoder.video_data.frame_infos { + assert frame.size > 0 + assert frame.size <= decoder.video_data.max_memory_frame_size_bytes } +} + +fn test_parser_orders_type_zero_b_frames_within_their_gop() { + mut decoder := Decoder{} + decoder.parse_mp4_data('${v_modroot}/res/Big_Buck_Bunny_360_10s_1MB.mp4') or { panic(err) } defer { decoder.video_data.file.close() } @@ -211,6 +537,23 @@ fn test_parser_orders_type_zero_b_frames_within_their_gop() { } } +fn test_parsed_reference_marking_operations_are_valid() { + mut decoder := Decoder{} + decoder.parse_mp4_data('${v_modroot}/res/Big_Buck_Bunny_360_10s_1MB.mp4') or { panic(err) } + defer { decoder.video_data.file.close() } + for i in 0 .. decoder.video_data.frame_infos.len { + header := unsafe { + &h264.SliceHeader(byteptr(decoder.video_data.slice_header_bytes.data) + + i * sizeof(h264.SliceHeader)) + } + for op in header.drpm.memory_management_control_operation { + assert op <= 6, 'sample ${i} has invalid MMCO ${op}' + if op == 0 { break + } + } + } +} + fn test_parser_rejects_non_mp4_input_as_an_error() { temp_path := os.join_path(os.temp_dir(), 'vkvideo-not-an-mp4-${os.getpid()}.txt') os.write_file(temp_path, 'This is deliberately not an MP4 file.') or { panic(err) } @@ -325,3 +668,55 @@ fn test_render_transform_rotates_minus_90_and_letterboxes_portrait_video() { assert transform.values[8] > 0.31 && transform.values[8] < 0.32 assert transform.values[9] == 1 } + +fn test_parser_keeps_nonzero_parameter_set_ids_for_runtime_lookup() { + mut decoder := Decoder{} + decoder.parse_mp4_data('${v_modroot}/res/H264_parameter_id_7_160x96_1s.mp4') or { panic(err) } + defer { decoder.video_data.file.close() } + assert decoder.video_data.frame_infos.len == 5 + assert decoder.video_data.sps_count == 1 + assert decoder.video_data.pps_count == 1 + assert decoder.video_data.sps_storage_offset(7)! == 0 + assert decoder.video_data.pps_storage_offset(7)! == 0 + sps := unsafe { &h264.SequenceParameterSet(decoder.video_data.sps_bytes.data) } + pps := unsafe { &h264.PictureParameterSet(decoder.video_data.pps_bytes.data) } + assert sps.seq_parameter_set_id == 7 + assert pps.pic_parameter_set_id == 7 + assert pps.seq_parameter_set_id == 7 + for i in 0 .. decoder.video_data.frame_infos.len { + header := unsafe { + &h264.SliceHeader(byteptr(decoder.video_data.slice_header_bytes.data) + + i * sizeof(h264.SliceHeader)) + } + assert header.pic_parameter_set_id == 7 + } +} + +fn test_custom_h264_scaling_lists_reach_vulkan_parameter_structs() { + // SPS from the FRExt_MMCO4_Sony_B conformance stream. Its eight custom + // lists exposed the pinned parser's fixed-array slice-write bug. + sps_nal := + hex.decode('2764001fad9464763b8ac4444a323b1dc5622225191d8ee2b11114222b373669a844566e6cd35088acdcd9a69444cd1b9bc57c9f93f9bf27c9e4e4cd251a4689c9ebe4fd7f27ebe4f5c9a906c694160964')! + sps_rbsp := unsafe { remove_emulation_prevention_bytes(byteptr(sps_nal.data) + 1, + sps_nal.len - 1) } + validate_sps_rbsp(sps_rbsp)! + mut sps_bits := h264.Bitstream{} + sps_bits.init(sps_rbsp) + mut sps := h264.SequenceParameterSet{} + sps.read_sps(mut sps_bits) + populate_sps_scaling_lists(sps_rbsp, mut sps)! + assert sps.scaling_list_4x4[0][..4] == [i32(6), 12, 12, 19] + assert sps.scaling_list_8x8[0][..4] == [i32(6), 10, 10, 13] + assert sps.scaling_list_8x8[1][0] != 0 + + // A minimal PPS with one custom 4x4 list of sixteen eights. + pps_rbsp := hex.decode('ce3c7fffe0c0')! + validate_pps_rbsp(pps_rbsp)! + mut pps_bits := h264.Bitstream{} + pps_bits.init(pps_rbsp) + mut pps := h264.PictureParameterSet{} + pps.read_pps(mut pps_bits) + populate_pps_scaling_lists(pps_rbsp, mut pps)! + assert pps.pic_scaling_matrix_present_flag == 1 + assert pps.scaling_list_4x4[0][..4] == [i32(8), 8, 8, 8] +}