diff --git a/.github/workflows/release-ci.yml b/.github/workflows/release-ci.yml index c9dba25..b54c5ac 100644 --- a/.github/workflows/release-ci.yml +++ b/.github/workflows/release-ci.yml @@ -367,6 +367,10 @@ jobs: Enter-VsDevShell -VsInstallPath $InstallPath -SkipAutomaticLocation -DevCmdArguments '-arch=x64 -host_arch=x64' ./scripts/build_windows.ps1 -Compiler stable $env:Path = "$(Resolve-Path dist\vkvideo-windows-x64);$env:Path" + $HelpText = (& .\dist\vkvideo-windows-x64\v_vulkan_video.exe --help | Out-String) + if ($LASTEXITCODE -ne 0 -or $HelpText -notmatch 'Usage:') { + throw 'The Windows player did not print its command-line help.' + } v -cc msvc test . ./scripts/verify_windows_package.ps1 - name: Upload Windows package diff --git a/BUILDING.md b/BUILDING.md index 7b7e1b6..59af968 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -59,13 +59,15 @@ a platform-independent archive. ## Player command line ```sh -./v_vulkan_video [--list-gpus] [--gpu INDEX] [video.mp4] +./v_vulkan_video [--list-gpus] [--gpu INDEX] [--decode-output-mode MODE] [video.mp4] ``` GPU compatibility is evaluated against the input stream's actual H.264 profile. Without `--gpu`, the first fully compatible presentation/decode device is selected. Invalid media and unsupported or out-of-range devices return a clean non-zero exit status with a diagnostic instead of a panic. +`--decode-output-mode auto|coincident|distinct` selects an advertised DPB and +output-image mode; `auto` is the default. ## Ubuntu 24.04 binary package diff --git a/PLATFORM_SUPPORT.md b/PLATFORM_SUPPORT.md index 0b25bd7..a4a2163 100644 --- a/PLATFORM_SUPPORT.md +++ b/PLATFORM_SUPPORT.md @@ -34,6 +34,8 @@ Video-capable GPU is available. The remaining Windows work is: The completed MSVC build and GTX 765M unsupported-device test do not satisfy these playback checks. +## Supported media and failure behavior + The application currently decodes H.264/AVC video carried in MP4. It supports 8-bit 4:2:0 progressive Baseline, Main, and High profiles when the driver reports a compatible Vulkan Video profile. Other codecs, chroma formats, @@ -74,6 +76,8 @@ This deliberately does not advertise Vulkan Video extensions or emulate video commands. Decoded-picture-buffer operation, image transitions, queue synchronization, and presentation still require a real Vulkan Video device. +## Hardware validation checklist + Before calling a platform supported for release, run at least: - playback through multiple loops; diff --git a/README.md b/README.md index 6b7ba9d..a6944de 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,11 @@ To compile and run directly from a source checkout, target the repository root: v run . [--list-gpus] [--gpu INDEX] [--decode-output-mode MODE] [video.mp4] ``` -`examples/video_decode_app` is the application's importable module rather than -a standalone `main` package, so it is not a direct `v run` target. +The player is one V application. `main.v` handles command-line options; +`app.v` owns the window and rendering loop; `device_context.v` selects the GPU. +`mp4_parser.v`, `decoder_session.v`, `player_decode.v`, and +`player_presentation.v` implement the media path. Shared player state and +resource ownership are in `video_player.v`. Without a video path, the bundled self-recorded and metadata-sanitized sample is used. `--list-gpus` reports compatibility against the selected video's @@ -69,6 +72,14 @@ workflow. See [PLATFORM_SUPPORT.md](PLATFORM_SUPPORT.md) for tested hardware, known limitations, and the release-validation matrix. +## Learn from the implementation + +The [design guide](docs/learning-path.md) traces an encoded picture through +MP4 parsing, GPU capability selection, Vulkan Video decoding, synchronization, +and presentation. It explains the underlying decisions, their tradeoffs, and +how to apply the same concepts in another project. Start with its system map +and follow the chapters that match the problem you are solving. + ## Tests ```sh diff --git a/examples/video_decode_app/app.v b/app.v similarity index 86% rename from examples/video_decode_app/app.v rename to app.v index 306b387..b7a16a8 100644 --- a/examples/video_decode_app/app.v +++ b/app.v @@ -1,4 +1,4 @@ -module video_decode_app +module main import antono2.vulkan as vk import antono2.glfw @@ -25,10 +25,10 @@ import os // #flag -DCIMGUI_NO_EXPORT=no // #flag -DIMGUI_DISABLE_WIN32_FUNCTIONS=yes // #flag -DIMGUI_DISABLE_OSX_FUNCTIONS=yes -pub const v_modroot = @VMODROOT +const v_modroot = @VMODROOT @[heap] -pub struct VideoDecodeApp { +struct VideoDecodeApp { mut: reference_slots []int dpb_slot_graph [18][]int @@ -60,7 +60,7 @@ fn check_vk(result vk.Result, operation string) { } @[heap] -pub struct FrameInfo { +struct FrameInfo { pub mut: command_pool vk.CommandPool command_buffer vk.CommandBuffer @@ -72,7 +72,7 @@ pub mut: // Three vec4 values match the vertex shader's push-constant block without // relying on compiler-specific struct padding. -pub struct VideoRenderTransform { +struct VideoRenderTransform { pub mut: values [12]f32 } @@ -142,7 +142,7 @@ fn video_render_transform(metadata VideoMetadata, extent vk.Extent2D) VideoRende return result } -pub fn (mut app VideoDecodeApp) initialize() bool { +fn (mut app VideoDecodeApp) initialize() bool { println('Initializing') if !glfw.initialize() { println('Could not initialize glfw') @@ -228,26 +228,26 @@ pub fn (mut app VideoDecodeApp) initialize() bool { descriptor_pool_sizes := [ vk.DescriptorPoolSize{ - type: vk.DescriptorType.uniform_buffer + type: vk.DescriptorType.uniform_buffer descriptorCount: 1000 }, vk.DescriptorPoolSize{ - type: vk.DescriptorType.combined_image_sampler + type: vk.DescriptorType.combined_image_sampler descriptorCount: 1000 }, ] descriptor_pool_ci := vk.DescriptorPoolCreateInfo{ - flags: vk.DescriptorPoolCreateFlags(vk.DescriptorPoolCreateFlagBits.free_descriptor_set) - maxSets: 100 + flags: vk.DescriptorPoolCreateFlags(vk.DescriptorPoolCreateFlagBits.free_descriptor_set) + maxSets: 100 poolSizeCount: u32(descriptor_pool_sizes.len) - pPoolSizes: descriptor_pool_sizes.data + pPoolSizes: descriptor_pool_sizes.data } check_vk(vk.create_descriptor_pool(app.device_context.vk_device, &descriptor_pool_ci, unsafe { nil }, &app.descriptor_pool), 'Could not create descriptor pool') mut sampler_ci := vk.SamplerCreateInfo{ - magFilter: vk.Filter.linear - minFilter: vk.Filter.linear - mipmapMode: vk.SamplerMipmapMode.linear + magFilter: vk.Filter.linear + minFilter: vk.Filter.linear + mipmapMode: vk.SamplerMipmapMode.linear addressModeU: vk.SamplerAddressMode.clamp_to_edge addressModeV: vk.SamplerAddressMode.clamp_to_edge addressModeW: vk.SamplerAddressMode.clamp_to_edge @@ -274,22 +274,22 @@ pub fn (mut app VideoDecodeApp) initialize() bool { } ds_layouts := [ vk.DescriptorSetLayoutBinding{ - binding: 0 - descriptorType: vk.DescriptorType.uniform_buffer + binding: 0 + descriptorType: vk.DescriptorType.uniform_buffer descriptorCount: 1 - stageFlags: vk.ShaderStageFlags(vk.ShaderStageFlagBits.all_graphics) + stageFlags: vk.ShaderStageFlags(vk.ShaderStageFlagBits.all_graphics) }, vk.DescriptorSetLayoutBinding{ - binding: 1 - descriptorType: vk.DescriptorType.combined_image_sampler - descriptorCount: 1 - stageFlags: vk.ShaderStageFlags(vk.ShaderStageFlagBits.all_graphics) + binding: 1 + descriptorType: vk.DescriptorType.combined_image_sampler + descriptorCount: 1 + stageFlags: vk.ShaderStageFlags(vk.ShaderStageFlagBits.all_graphics) pImmutableSamplers: &app.sampler }, ] ds_layout_ci := vk.DescriptorSetLayoutCreateInfo{ bindingCount: u32(ds_layouts.len) - pBindings: ds_layouts.data + pBindings: ds_layouts.data } check_vk(vk.create_descriptor_set_layout(app.device_context.vk_device, &ds_layout_ci, unsafe { nil }, &app.ds_layout), 'Could not create descriptor-set layout') @@ -316,17 +316,17 @@ pub fn (mut app VideoDecodeApp) initialize() bool { fn (mut app VideoDecodeApp) initialize_imgui_vulkan_backend() { dev_ctx := app.device_context mut vk_info := impl_vulkan.InitInfo{ - api_version: vk.api_version_1_3 - instance: dev_ctx.vk_instance - physical_device: dev_ctx.get_gpu_current() - device: dev_ctx.vk_device - queue_family: dev_ctx.graphics_family - queue: dev_ctx.get_queue(.graphics) - descriptor_pool: app.descriptor_pool - min_image_count: 2 - image_count: dev_ctx.swapchain.image_count + api_version: vk.api_version_1_3 + instance: dev_ctx.vk_instance + physical_device: dev_ctx.get_gpu_current() + device: dev_ctx.vk_device + queue_family: dev_ctx.graphics_family + queue: dev_ctx.get_queue(.graphics) + descriptor_pool: app.descriptor_pool + min_image_count: 2 + image_count: dev_ctx.swapchain.image_count pipeline_info_main: impl_vulkan.PipelineInfo{ - render_pass: app.render_pass + render_pass: app.render_pass msaa_samples: vk.SampleCountFlagBits._1 } } @@ -340,7 +340,7 @@ fn (mut app VideoDecodeApp) initialize_imgui_vulkan_backend() { app.imgui_image_count = dev_ctx.swapchain.image_count } -pub fn loader_function_callback(function_name &char, user_data voidptr) voidptr { +fn loader_function_callback(function_name &char, user_data voidptr) voidptr { app := unsafe { &VideoDecodeApp(user_data) } // ImGui requests both instance- and device-level commands through this one // callback. Querying the instance first follows vkGetInstanceProcAddr's @@ -352,7 +352,7 @@ pub fn loader_function_callback(function_name &char, user_data voidptr) voidptr return vk.get_device_proc_addr(app.device_context.vk_device, function_name) } -pub fn (mut app VideoDecodeApp) run() { +fn (mut app VideoDecodeApp) run() { $if debug { eprintln('Entering playback loop') } @@ -398,16 +398,16 @@ pub fn (mut app VideoDecodeApp) run() { app.video_player.update(frame.command_buffer, time_elapsed_ns) output_view := app.video_player.current_output_view() mut image_info := vk.DescriptorImageInfo{ - imageView: output_view + imageView: output_view imageLayout: .shader_read_only_optimal } if !isnil(output_view) { write_descriptor := vk.WriteDescriptorSet{ - dstSet: frame.descriptor_set - dstBinding: 1 + dstSet: frame.descriptor_set + dstBinding: 1 descriptorCount: 1 - descriptorType: .combined_image_sampler - pImageInfo: &image_info + descriptorType: .combined_image_sampler + pImageInfo: &image_info } vk.update_descriptor_sets(app.device_context.vk_device, 1, &write_descriptor, 0, unsafe { nil }) } @@ -421,13 +421,13 @@ pub fn (mut app VideoDecodeApp) run() { clear_value.color.float32[2] = 0.10 clear_value.color.float32[3] = 1.0 render_pass_begin := vk.RenderPassBeginInfo{ - renderPass: app.render_pass - framebuffer: frame.framebuffer - renderArea: vk.Rect2D{ + renderPass: app.render_pass + framebuffer: frame.framebuffer + renderArea: vk.Rect2D{ extent: app.device_context.swapchain.extent_2d } clearValueCount: 1 - pClearValues: &clear_value + pClearValues: &clear_value } vk.cmd_begin_render_pass(frame.command_buffer, &render_pass_begin, .inline) extent := app.device_context.swapchain.extent_2d @@ -441,8 +441,8 @@ pub fn (mut app VideoDecodeApp) run() { vk.cmd_bind_descriptor_sets(frame.command_buffer, .graphics, app.pipeline_layout, 0, 1, &frame.descriptor_set, 0, unsafe { nil }) vk.cmd_push_constants(frame.command_buffer, app.pipeline_layout, vk.ShaderStageFlags(vk.ShaderStageFlagBits.vertex), 0, u32(sizeof(VideoRenderTransform)), &video_transform) viewport := vk.Viewport{ - width: f32(extent.width) - height: f32(extent.height) + width: f32(extent.width) + height: f32(extent.height) maxDepth: 1 } scissor := vk.Rect2D{ @@ -461,13 +461,13 @@ pub fn (mut app VideoDecodeApp) run() { } mut wait_stage := vk.PipelineStageFlags(vk.PipelineStageFlagBits.color_attachment_output) submit_info := vk.SubmitInfo{ - waitSemaphoreCount: 1 - pWaitSemaphores: &app.sem_present_complete - pWaitDstStageMask: &wait_stage - commandBufferCount: 1 - pCommandBuffers: &frame.command_buffer + waitSemaphoreCount: 1 + pWaitSemaphores: &app.sem_present_complete + pWaitDstStageMask: &wait_stage + commandBufferCount: 1 + pCommandBuffers: &frame.command_buffer signalSemaphoreCount: 1 - pSignalSemaphores: &app.sem_render_complete + pSignalSemaphores: &app.sem_render_complete } res = vk.queue_submit(app.device_context.get_queue(.graphics), 1, &submit_info, frame.queue_submit_fence) if res != vk.Result.success { @@ -497,7 +497,7 @@ fn (mut app VideoDecodeApp) recreate_swapchain() bool { } app.frames.clear() if !app.device_context.swapchain.resize(vk.Extent2D{ - width: u32(width) + width: u32(width) height: u32(height) }) { return false @@ -531,7 +531,7 @@ fn key_function_callback(window_p &glfw.Window, key int, _scancode int, action i } } -pub fn (mut app VideoDecodeApp) shutdown() { +fn (mut app VideoDecodeApp) shutdown() { vk_device := app.device_context.get_vk_device() vk.device_wait_idle(vk_device) app.video_player.shutdown() @@ -649,50 +649,50 @@ fn (mut app VideoDecodeApp) initialize_render_pass() { mut attachment := vk.AttachmentDescription{ // flags: vk.AttachmentDescriptionFlags(vk.AttachmentDescriptionFlagBits.may_alias) - format: dev_ctx.swapchain.surface_format.format - samples: vk.SampleCountFlagBits._1 - loadOp: vk.AttachmentLoadOp.clear - storeOp: vk.AttachmentStoreOp.store - stencilLoadOp: vk.AttachmentLoadOp.dont_care + format: dev_ctx.swapchain.surface_format.format + samples: vk.SampleCountFlagBits._1 + loadOp: vk.AttachmentLoadOp.clear + storeOp: vk.AttachmentStoreOp.store + stencilLoadOp: vk.AttachmentLoadOp.dont_care stencilStoreOp: vk.AttachmentStoreOp.dont_care - initialLayout: vk.ImageLayout.undefined - finalLayout: vk.ImageLayout.present_src_khr + initialLayout: vk.ImageLayout.undefined + finalLayout: vk.ImageLayout.present_src_khr } mut color_ref := vk.AttachmentReference{ attachment: 0 - layout: vk.ImageLayout.color_attachment_optimal + layout: vk.ImageLayout.color_attachment_optimal } mut subpass := vk.SubpassDescription{ - pipelineBindPoint: vk.PipelineBindPoint.graphics + pipelineBindPoint: vk.PipelineBindPoint.graphics colorAttachmentCount: 1 - pColorAttachments: &color_ref + pColorAttachments: &color_ref } mut dependency := vk.SubpassDependency{ - srcSubpass: vk.subpass_external - dstSubpass: 0 - srcStageMask: vk.PipelineStageFlags(vk.PipelineStageFlagBits.color_attachment_output) - dstStageMask: vk.PipelineStageFlags(vk.PipelineStageFlagBits.color_attachment_output) + srcSubpass: vk.subpass_external + dstSubpass: 0 + srcStageMask: vk.PipelineStageFlags(vk.PipelineStageFlagBits.color_attachment_output) + dstStageMask: vk.PipelineStageFlags(vk.PipelineStageFlagBits.color_attachment_output) srcAccessMask: 0 dstAccessMask: vk.AccessFlags(u32(vk.AccessFlagBits.color_attachment_read) | u32(vk.AccessFlagBits.color_attachment_write)) } rp_info := vk.RenderPassCreateInfo{ attachmentCount: 1 - pAttachments: &attachment - subpassCount: 1 - pSubpasses: &subpass + pAttachments: &attachment + subpassCount: 1 + pSubpasses: &subpass dependencyCount: 1 - pDependencies: &dependency + pDependencies: &dependency } check_vk(vk.create_render_pass(vk_device, &rp_info, unsafe { nil }, &app.render_pass), 'Could not create render pass') } @[heap] -pub struct ShaderModuleHeap { +struct ShaderModuleHeap { pub mut: shader_module vk.ShaderModule } @@ -701,7 +701,7 @@ fn (mut app VideoDecodeApp) create_shader_module(shader_data []u32) vk.ShaderMod vk_device := app.device_context.vk_device module_ci := vk.ShaderModuleCreateInfo{ codeSize: usize(shader_data.len) * sizeof(u32) - pCode: unsafe { shader_data.data } + pCode: unsafe { shader_data.data } } mut shader_module := vk.ShaderModule(0) check_vk(vk.create_shader_module(vk_device, &module_ci, unsafe { nil }, &shader_module), 'Could not create shader module') @@ -716,13 +716,13 @@ fn (mut app VideoDecodeApp) initialize_pipeline() { } mut push_constant_range := vk.PushConstantRange{ stageFlags: vk.ShaderStageFlags(vk.ShaderStageFlagBits.vertex) - size: u32(sizeof(VideoRenderTransform)) + size: u32(sizeof(VideoRenderTransform)) } pipeline_layout_ci := vk.PipelineLayoutCreateInfo{ - setLayoutCount: 1 - pSetLayouts: &app.ds_layout + setLayoutCount: 1 + pSetLayouts: &app.ds_layout pushConstantRangeCount: 1 - pPushConstantRanges: &push_constant_range + pPushConstantRanges: &push_constant_range } check_vk(vk.create_pipeline_layout(vk_device, &pipeline_layout_ci, unsafe { nil }, &app.pipeline_layout), 'Could not create graphics pipeline layout') @@ -731,7 +731,7 @@ fn (mut app VideoDecodeApp) initialize_pipeline() { topology: vk.PrimitiveTopology.triangle_strip } mut raster_ci := vk.PipelineRasterizationStateCreateInfo{ - cullMode: vk.CullModeFlags(vk.CullModeFlagBits.back) + cullMode: vk.CullModeFlags(vk.CullModeFlagBits.back) frontFace: vk.FrontFace.counter_clockwise lineWidth: 1.0 } @@ -744,11 +744,11 @@ fn (mut app VideoDecodeApp) initialize_pipeline() { } mut blend_ci := vk.PipelineColorBlendStateCreateInfo{ attachmentCount: 1 - pAttachments: &blend_attachment + pAttachments: &blend_attachment } mut viewport_ci := vk.PipelineViewportStateCreateInfo{ viewportCount: 1 - scissorCount: 1 + scissorCount: 1 } mut depth_stencil_ci := vk.PipelineDepthStencilStateCreateInfo{} mut multisample_ci := vk.PipelineMultisampleStateCreateInfo{ @@ -757,33 +757,33 @@ fn (mut app VideoDecodeApp) initialize_pipeline() { mut dynamic_states := [vk.DynamicState.viewport, vk.DynamicState.scissor] mut dynamic_ci := vk.PipelineDynamicStateCreateInfo{ dynamicStateCount: u32(dynamic_states.len) - pDynamicStates: dynamic_states.data + pDynamicStates: dynamic_states.data } mut shader_stages := [ vk.PipelineShaderStageCreateInfo{ - stage: vk.ShaderStageFlagBits.vertex + stage: vk.ShaderStageFlagBits.vertex module: app.create_shader_module(g_vertex_shader) - pName: c'main' + pName: c'main' }, vk.PipelineShaderStageCreateInfo{ - stage: vk.ShaderStageFlagBits.fragment + stage: vk.ShaderStageFlagBits.fragment module: app.create_shader_module(g_fragment_shader) - pName: c'main' + pName: c'main' }, ] mut pipeline_ci := vk.GraphicsPipelineCreateInfo{ - stageCount: u32(shader_stages.len) - pStages: shader_stages.data - pVertexInputState: &vertex_input_ci + stageCount: u32(shader_stages.len) + pStages: shader_stages.data + pVertexInputState: &vertex_input_ci pInputAssemblyState: &input_assembly_ci - pViewportState: &viewport_ci + pViewportState: &viewport_ci pRasterizationState: &raster_ci - pMultisampleState: &multisample_ci - pDepthStencilState: &depth_stencil_ci - pColorBlendState: &blend_ci - pDynamicState: &dynamic_ci - layout: app.pipeline_layout - renderPass: app.render_pass + pMultisampleState: &multisample_ci + pDepthStencilState: &depth_stencil_ci + pColorBlendState: &blend_ci + pDynamicState: &dynamic_ci + layout: app.pipeline_layout + renderPass: app.render_pass } check_vk(vk.create_graphics_pipelines(vk_device, unsafe { nil }, 1, &pipeline_ci, unsafe { nil }, &app.pipeline), 'Could not create graphics pipeline') @@ -804,12 +804,12 @@ fn (mut app VideoDecodeApp) initialize_framebuffers() { for i in 0 .. count { view := swapchain.image_views[i] framebuffer_ci := vk.FramebufferCreateInfo{ - renderPass: app.render_pass + renderPass: app.render_pass attachmentCount: 1 - pAttachments: &view - width: extent.width - height: extent.height - layers: 1 + pAttachments: &view + width: extent.width + height: extent.height + layers: 1 } mut fb := unsafe { nil } check_vk(vk.create_framebuffer(vk_device, &framebuffer_ci, unsafe { nil }, &fb), 'Could not create swapchain framebuffer ${i}') @@ -832,17 +832,17 @@ fn (mut app VideoDecodeApp) init_per_frame(mut frame_info FrameInfo) { check_vk(vk.create_command_pool(vk_device, &command_pool_ci, unsafe { nil }, &frame_info.command_pool), 'Could not create graphics command pool') command_buffer_allocate_info := vk.CommandBufferAllocateInfo{ - commandPool: frame_info.command_pool - level: vk.CommandBufferLevel.primary + commandPool: frame_info.command_pool + level: vk.CommandBufferLevel.primary commandBufferCount: 1 } check_vk(vk.allocate_command_buffers(vk_device, &command_buffer_allocate_info, &frame_info.command_buffer), 'Could not allocate graphics command buffer') frame_info.queue_index = 0 descriptor_set_allocate_info := vk.DescriptorSetAllocateInfo{ - descriptorPool: app.descriptor_pool + descriptorPool: app.descriptor_pool descriptorSetCount: 1 - pSetLayouts: &app.ds_layout + pSetLayouts: &app.ds_layout } result := vk.allocate_descriptor_sets(vk_device, &descriptor_set_allocate_info, &frame_info.descriptor_set) if result != .success || isnil(frame_info.descriptor_set) { diff --git a/decoder_session.v b/decoder_session.v new file mode 100644 index 0000000..c8d5fab --- /dev/null +++ b/decoder_session.v @@ -0,0 +1,533 @@ +module main + +import antono2.vulkan as vk +import math +import antono2.vkmemalloc as vkmem +import antono2.h264 + +fn (mut d Decoder) initialize(mut app VideoDecodeApp) { + mut dev_ctx := app.device_context + d.properties.decode_h264_caps = vk.VideoDecodeH264CapabilitiesKHR{} + d.properties.decode_caps = vk.VideoDecodeCapabilitiesKHR{ + pNext: &d.properties.decode_h264_caps + } + d.properties.caps = vk.VideoCapabilitiesKHR{ + pNext: &d.properties.decode_caps + } + + d.settings.decode_h264_profile_info = vk.VideoDecodeH264ProfileInfoKHR{ + stdProfileIdc: unsafe { vk.StdVideoH264ProfileIdc(d.video_data.h264_profile_idc) } + pictureLayout: vk.VideoDecodeH264PictureLayoutFlagBitsKHR.progressive + } + d.settings.profile_info = vk.VideoProfileInfoKHR{ + pNext: &d.settings.decode_h264_profile_info + videoCodecOperation: vk.VideoCodecOperationFlagBitsKHR.decode_h264 + chromaSubsampling: vk.VideoChromaSubsamplingFlagsKHR(vk.VideoChromaSubsamplingFlagBitsKHR._420) + lumaBitDepth: vk.VideoComponentBitDepthFlagsKHR(vk.VideoComponentBitDepthFlagBitsKHR._8) + chromaBitDepth: vk.VideoComponentBitDepthFlagsKHR(vk.VideoComponentBitDepthFlagBitsKHR._8) + } + d.properties.caps.pNext = &d.properties.decode_caps + + mut res := vk.get_physical_device_video_capabilities_khr(dev_ctx.get_gpu_current(), &d.settings.profile_info, mut &d.properties.caps) + if res != vk.Result.success { + panic('Vulkan device does not expose H.264 ${h264_profile_name(d.video_data.h264_profile_idc)} Profile decode capabilities: ${res}') + } + + // Construct the complete wrapper so its required Vulkan sType default is + // applied. Mutating fields of the zero-initialized embedded value leaves + // sType at zero with some V compiler/code-generation paths; NVIDIA's driver + // may then fault when this chain is passed to vkCreateImage. + d.settings.profile_list_info = vk.VideoProfileListInfoKHR{ + profileCount: 1 + pProfiles: &d.settings.profile_info + } + + capability_flags := d.properties.decode_caps.flags + supports_coincide := (capability_flags & vk.VideoDecodeCapabilityFlagsKHR(vk.VideoDecodeCapabilityFlagBitsKHR.dpb_and_output_coincide)) != 0 + supports_distinct := (capability_flags & vk.VideoDecodeCapabilityFlagsKHR(vk.VideoDecodeCapabilityFlagBitsKHR.dpb_and_output_distinct)) != 0 + selected_output_mode := select_decode_output_mode(app.decode_output_mode, supports_coincide, supports_distinct) or { panic(err) } + d.properties.dpb_and_output_coincide = selected_output_mode == .coincident + println('Decode image mode: ${if d.properties.dpb_and_output_coincide { + 'coincident DPB/output' + } else { + 'distinct DPB/output' + }}') + output_usage := vk.ImageUsageFlags(u32(vk.ImageUsageFlagBits.video_decode_dst) | u32(vk.ImageUsageFlagBits.transfer_src)) + d.properties.format_props = query_video_format(dev_ctx.get_gpu_current(), &d.settings.profile_list_info, output_usage) or { + panic('No Vulkan Video decode-output format supports transfer to the display image') + } + dpb_usage := if d.properties.dpb_and_output_coincide { + vk.ImageUsageFlags(u32(vk.ImageUsageFlagBits.video_decode_dpb) | u32(vk.ImageUsageFlagBits.video_decode_dst) | u32(vk.ImageUsageFlagBits.transfer_src)) + } else { + vk.ImageUsageFlags(vk.ImageUsageFlagBits.video_decode_dpb) + } + d.properties.dpb_format_props = query_video_format(dev_ctx.get_gpu_current(), &d.settings.profile_list_info, dpb_usage) or { + panic('No Vulkan Video DPB format supports the required decode mode') + } + d.properties.usage_dpb = dpb_usage + + num_memory_frames := u64(d.video_data.num_dpb_slots) + mut aligned_frame_size := U64(d.video_data.max_memory_frame_size_bytes).align_to(d.properties.caps.minBitstreamBufferOffsetAlignment) + aligned_frame_size = U64(aligned_frame_size).align_to(d.properties.caps.minBitstreamBufferSizeAlignment) + d.video_data.max_memory_frame_size_bytes = aligned_frame_size + video_decoder_queue_family_index := dev_ctx.get_decoder_queue_family_index() + buffer_size := d.video_data.max_memory_frame_size_bytes * num_memory_frames + buffer_ci := vk.BufferCreateInfo{ + pNext: &d.settings.profile_list_info + flags: 0 + size: buffer_size + usage: vk.BufferUsageFlags(vk.BufferUsageFlagBits.video_decode_src) + sharingMode: vk.SharingMode.exclusive + queueFamilyIndexCount: 0 + pQueueFamilyIndices: unsafe { nil } + } + res = app.device_context.memory_allocator.create_buffer_with_options(&buffer_ci, vkmem.AllocationOptions{ + usage: .upload + }, &d.gpu_bitstream_buffer, mut d.gpu_bitstream_allocation) + if res != vk.Result.success { + panic('Could not create the Vulkan Video bitstream buffer: ${res}') + } + // Host visible + mut p_data := unsafe { nil } + // Use the same live allocator instance that created the allocation. dev_ctx + // was copied before create_buffer() updated the allocator's block table. + res = app.device_context.memory_allocator.map(mut d.gpu_bitstream_allocation, &p_data) + if res != vk.Result.success { + panic('Could not map the Vulkan Video bitstream buffer: ${res}') + } + if d.video_data.num_dpb_slots > d.properties.caps.maxDpbSlots { + panic('Video requires ${d.video_data.num_dpb_slots} DPB slots, but this device supports ${d.properties.caps.maxDpbSlots}') + } + d.video_data.max_reference_pictures = d.video_data.num_dpb_slots - 1 + if d.video_data.max_reference_pictures > d.properties.caps.maxActiveReferencePictures { + panic('Video requires ${d.video_data.max_reference_pictures} active references, but this device supports ${d.properties.caps.maxActiveReferencePictures}') + } + + session_ci := vk.VideoSessionCreateInfoKHR{ + queueFamilyIndex: video_decoder_queue_family_index + pVideoProfile: &d.settings.profile_info + pictureFormat: d.properties.format_props.format + maxCodedExtent: vk.Extent2D{ + width: math.min(d.video_data.width, d.properties.caps.maxCodedExtent.width) + height: math.min(d.video_data.height, d.properties.caps.maxCodedExtent.height) + } + referencePictureFormat: d.properties.dpb_format_props.format + maxDpbSlots: d.video_data.num_dpb_slots + maxActiveReferencePictures: d.video_data.max_reference_pictures + pStdHeaderVersion: &d.properties.caps.stdHeaderVersion + } + res = vk.create_video_session_khr(dev_ctx.vk_device, &session_ci, unsafe { nil }, &d.video_session) + if res != vk.Result.success { + panic('Could not create the Vulkan Video H.264 decode session: ${res}') + } + + mut requirement_count := u32(0) + mut n := unsafe { nil } + vk.get_video_session_memory_requirements_khr(dev_ctx.vk_device, d.video_session, &requirement_count, mut n) + mut requirements := []vk.VideoSessionMemoryRequirementsKHR{len: int(requirement_count), init: vk.VideoSessionMemoryRequirementsKHR{}} + + mut requirements_data := requirements.data + res = vk.get_video_session_memory_requirements_khr(dev_ctx.vk_device, d.video_session, &requirement_count, mut requirements_data) + if res != vk.Result.success { + panic('Could not query Vulkan Video session memory requirements: ${res}') + } + d.session_memory_allocations.ensure_cap(int(requirement_count)) + // V's length-only array initialization zeroes C structs and therefore loses + // the Vulkan binding's required sType default. Construct every element. + mut bind_session_memory_infos := []vk.BindVideoSessionMemoryInfoKHR{len: int(requirement_count), init: vk.BindVideoSessionMemoryInfoKHR{}} + for i in 0 .. requirement_count { + req := requirements[i] + // Video-session memory is opaque driver storage. The requirement's + // memoryTypeBits is authoritative; some drivers expose a dedicated type + // without VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT for this binding. + memory_type_index := dev_ctx.memory_allocator.get_memory_type(req.memoryRequirements.memoryTypeBits, vk.MemoryPropertyFlags(0)) + if memory_type_index == max_u32 { + panic('No compatible Vulkan memory type exists for video-session binding ${req.memoryBindIndex}') + } + allocate_info := vk.MemoryAllocateInfo{ + allocationSize: req.memoryRequirements.size + memoryTypeIndex: memory_type_index + } + mut session_memory := vk.DeviceMemory(unsafe { nil }) + res = vk.allocate_memory(dev_ctx.vk_device, &allocate_info, unsafe { nil }, &session_memory) + if res != vk.Result.success { + panic('Could not allocate ${req.memoryRequirements.size} bytes for Vulkan Video session binding ${req.memoryBindIndex}: ${res}') + } + d.session_memory_allocations << session_memory + + bind_session_memory_infos[i] = vk.BindVideoSessionMemoryInfoKHR{ + memoryBindIndex: req.memoryBindIndex + memory: session_memory + memoryOffset: 0 + memorySize: req.memoryRequirements.size + } + } + + // Resolve this extension command from the device directly. When another + // shared object links libvulkan, ELF symbol interposition can otherwise + // make Volk's same-named global dispatch slot unreliable. + bind_session_memory_fn := vk.PFN_vkBindVideoSessionMemoryKHR(vk.get_device_proc_addr(dev_ctx.vk_device, c'vkBindVideoSessionMemoryKHR')) + if isnil(voidptr(bind_session_memory_fn)) { + panic('vkGetDeviceProcAddr returned null for vkBindVideoSessionMemoryKHR') + } + res = bind_session_memory_fn(dev_ctx.vk_device, d.video_session, u32(bind_session_memory_infos.len), bind_session_memory_infos.data) + if res != vk.Result.success { + panic('Could not bind Vulkan Video session memory: ${res}') + } + + d.create_video_session_parameters(dev_ctx.vk_device) + + $if debug { + if (d.properties.decode_caps.flags & vk.VideoDecodeCapabilityFlagsKHR(vk.VideoDecodeCapabilityFlagBitsKHR.dpb_and_output_coincide)) > 0 { + println('NOTE: video decode: dpb and output coincide') + } else { + println('NOTE: video decode: dpb and output do NOT coincide') + } + if (d.properties.decode_caps.flags & vk.VideoDecodeCapabilityFlagsKHR(vk.VideoDecodeCapabilityFlagBitsKHR.dpb_and_output_distinct)) > 0 { + println('NOTE: video decode: dpb and output distinct') + } else { + println('NOTE: video decode: dpb and output NOT distinct') + } + } + + d.prepare_decoded_picture_buffer(dev_ctx.vk_device, mut app.device_context.memory_allocator) + + d.info.memory_frames = []DecoderVideoMemoryFrameInfo{len: int(num_memory_frames), init: DecoderVideoMemoryFrameInfo{ + data_frame_info: unsafe { nil } + gpu_bitstream_slice_mapped_memory_address: unsafe { nil } + decoding_frame_index: -1 + }} + mut i := u64(0) + for mut frame in d.info.memory_frames { + frame.gpu_bitstream_capacity = d.video_data.max_memory_frame_size_bytes + frame.gpu_bitstream_offset = i * d.video_data.max_memory_frame_size_bytes + frame.gpu_bitstream_size = 0 + frame.gpu_bitstream_slice_mapped_memory_address = unsafe { + byteptr(p_data) + frame.gpu_bitstream_offset + } + i++ + } + // Runtime sample uploads use absolute MP4 offsets. Do not perform an + // unrelated seek here that can turn an otherwise completed initialization + // into a fatal media-I/O panic. +} + +fn (mut d Decoder) prepare_decoded_picture_buffer(device vk.Device, mut allocator vkmem.Allocator) { + // Allocate an image array to store decoded pictures in - + // num_dpb_slots already includes one slot for the picture currently decoded. + // + // we know there will be at max 17 images (16+1) as 16 is the max by the standard. + // + // Decoded-picture-buffer images should remain in device-local memory. + dpb_image_count := int(d.video_data.num_dpb_slots) + d.info.images_dpb = []DecoderDpbImage{len: dpb_image_count} + + image_ci := vk.ImageCreateInfo{ + pNext: &d.settings.profile_list_info + flags: 0 + imageType: vk.ImageType._2d + format: d.properties.dpb_format_props.format + extent: vk.Extent3D{ + width: d.video_data.width + height: d.video_data.height + depth: 1 + } + mipLevels: 1 + arrayLayers: 1 + samples: vk.SampleCountFlagBits._1 + tiling: vk.ImageTiling.optimal + usage: d.properties.usage_dpb + sharingMode: vk.SharingMode.exclusive + queueFamilyIndexCount: 0 + pQueueFamilyIndices: unsafe { nil } + initialLayout: vk.ImageLayout.undefined + } + + mut image_view_ci := vk.ImageViewCreateInfo{ + flags: 0 + image: unsafe { nil } + viewType: vk.ImageViewType._2d + format: image_ci.format + components: vk.ComponentMapping{} + subresourceRange: vk.ImageSubresourceRange{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) + baseMipLevel: 0 + levelCount: 1 + baseArrayLayer: 0 + layerCount: 1 + } + } + + mut dpb_index := 0 + mut res := vk.Result.error_unknown + for mut dpb in d.info.images_dpb { + res = allocator.create_image_with_options(&image_ci, vkmem.AllocationOptions{ + usage: .gpu_only + }, &dpb.image, mut dpb.allocation_info) + if res != vk.Result.success { + panic('Could not create decoded-picture-buffer image ${dpb_index}: ${res}') + } + image_view_ci.image = dpb.image + res = vk.create_image_view(device, &image_view_ci, unsafe { nil }, &dpb.view) + if res != vk.Result.success { + panic('Could not create decoded-picture-buffer image view ${dpb_index}: ${res}') + } + // VK_EXT_debug_utils is enabled only for debug builds. Calling its + // device command unconditionally jumps through a null dispatch slot in + // release packages. + $if debug ? { + name := 'myDPBImage${dpb_index}' + name_info := vk.DebugUtilsObjectNameInfoEXT{ + objectType: vk.ObjectType.image + objectHandle: u64(voidptr(dpb.image)) + pObjectName: name.str // C string + } + vk.set_debug_utils_object_name_ext(device, &name_info) + } + dpb_index++ + } // for d.info.images_dpb +} + +fn (mut d Decoder) create_video_session_parameters(device vk.Device) { + mut video_picture_parameter_sets := []vk.StdVideoH264PictureParameterSet{len: int(d.video_data.pps_count)} + mut video_scaling_list_pps := []vk.StdVideoH264ScalingLists{len: int(d.video_data.pps_count)} + for i in 0 .. int(d.video_data.pps_count) { + pps_offset := i * int(sizeof(h264.PictureParameterSet)) + pps := unsafe { &h264.PictureParameterSet(&d.video_data.pps_bytes[pps_offset]) } + video_scaling_list_pps[i] = vk.StdVideoH264ScalingLists{} + for j in 0 .. pps.pic_scaling_list_present_flag.len { + video_scaling_list_pps[i].scaling_list_present_mask |= u16(pps.pic_scaling_list_present_flag[j]) << j + } + for j in 0 .. pps.use_default_scaling_matrix_4x4_flag.len { + video_scaling_list_pps[i].use_default_scaling_matrix_mask |= u16(pps.use_default_scaling_matrix_4x4_flag[j]) << j + } + mut list_idx := 0 + mut el_idx := 0 + for list_idx < vk.std_video_h264_scaling_list_4x4_num_lists && list_idx < pps.scaling_list_4x4.len { + for el_idx < vk.std_video_h264_scaling_list_4x4_num_elements && el_idx < pps.scaling_list_4x4[0].len { + unsafe { + C.vv_set_h264_scaling_list_4x4(&video_scaling_list_pps[i], u32(list_idx), u32(el_idx), u8(pps.scaling_list_4x4[list_idx][el_idx])) + } + el_idx++ + } + list_idx++ + } + list_idx = 0 + el_idx = 0 + for list_idx < vk.std_video_h264_scaling_list_8x8_num_lists && list_idx < pps.scaling_list_8x8.len { + for el_idx < vk.std_video_h264_scaling_list_8x8_num_elements && el_idx < pps.scaling_list_8x8[0].len { + unsafe { + C.vv_set_h264_scaling_list_8x8(&video_scaling_list_pps[i], u32(list_idx), u32(el_idx), u8(pps.scaling_list_8x8[list_idx][el_idx])) + } + el_idx++ + } + list_idx++ + } + + video_picture_parameter_sets[i] = vk.StdVideoH264PictureParameterSet{ + flags: vk.StdVideoH264PpsFlags{ + transform_8x8_mode_flag: u32(pps.transform_8x8_mode_flag) + redundant_pic_cnt_present_flag: pps.redundant_pic_cnt_present_flag + constrained_intra_pred_flag: pps.constrained_intra_pred_flag + deblocking_filter_control_present_flag: pps.deblocking_filter_control_present_flag + weighted_pred_flag: pps.weighted_pred_flag + bottom_field_pic_order_in_frame_present_flag: pps.pic_order_present_flag + entropy_coding_mode_flag: pps.entropy_coding_mode_flag + pic_scaling_matrix_present_flag: pps.pic_scaling_matrix_present_flag + } + seq_parameter_set_id: u8(pps.seq_parameter_set_id) + pic_parameter_set_id: u8(pps.pic_parameter_set_id) + num_ref_idx_l0_default_active_minus1: u8(pps.num_ref_idx_l0_active_minus1) + num_ref_idx_l1_default_active_minus1: u8(pps.num_ref_idx_l1_active_minus1) + weighted_bipred_idc: unsafe { vk.StdVideoH264WeightedBipredIdc(pps.weighted_bipred_idc) } + pic_init_qp_minus26: i8(pps.pic_init_qp_minus26) + pic_init_qs_minus26: i8(pps.pic_init_qs_minus26) + chroma_qp_index_offset: i8(pps.chroma_qp_index_offset) + second_chroma_qp_index_offset: i8(pps.second_chroma_qp_index_offset) + pScalingLists: unsafe { &video_scaling_list_pps[i] } + } + } // for d.video_data.pps_count + + mut video_sequence_parameter_set := []vk.StdVideoH264SequenceParameterSet{len: int(d.video_data.sps_count)} + mut video_sequence_parameter_set_vui := []vk.StdVideoH264SequenceParameterSetVui{len: int(d.video_data.sps_count)} + mut video_scaling_list_sps := []vk.StdVideoH264ScalingLists{len: int(d.video_data.sps_count)} + mut video_hrd_parameters := []vk.StdVideoH264HrdParameters{len: int(d.video_data.sps_count)} + get_chroma_format := fn (profile u32, chroma u32) vk.StdVideoH264ChromaFormatIdc { + if profile < unsafe { int(vk.StdVideoH264ProfileIdc.high) } { + // If profile is less than HIGH chroma format will not be explicitly given. (A.2) + // If chroma format is not present, it shall be inferred to be equal to 1 (4:2:0) (7.4.2.1.1) + return vk.StdVideoH264ChromaFormatIdc._420 + } else { + // If profile is greater than HIGH, then we assume chroma to be explicitly specified + return unsafe { vk.StdVideoH264ChromaFormatIdc(chroma) } + } + } + + for i in 0 .. int(d.video_data.sps_count) { + sps_offset := i * int(sizeof(h264.SequenceParameterSet)) + sps := unsafe { &h264.SequenceParameterSet(&d.video_data.sps_bytes[sps_offset]) } + + video_sequence_parameter_set[i] = vk.StdVideoH264SequenceParameterSet{ + flags: vk.StdVideoH264SpsFlags{ + constraint_set0_flag: sps.constraint_set0_flag + constraint_set1_flag: sps.constraint_set1_flag + constraint_set2_flag: sps.constraint_set2_flag + constraint_set3_flag: sps.constraint_set3_flag + constraint_set4_flag: sps.constraint_set4_flag + constraint_set5_flag: sps.constraint_set5_flag + direct_8x8_inference_flag: sps.direct_8x8_inference_flag + mb_adaptive_frame_field_flag: sps.mb_adaptive_frame_field_flag + frame_mbs_only_flag: sps.frame_mbs_only_flag + delta_pic_order_always_zero_flag: sps.delta_pic_order_always_zero_flag + separate_colour_plane_flag: sps.separate_colour_plane_flag + gaps_in_frame_num_value_allowed_flag: sps.gaps_in_frame_num_value_allowed_flag + qpprime_y_zero_transform_bypass_flag: sps.qpprime_y_zero_transform_bypass_flag + frame_cropping_flag: sps.frame_cropping_flag + seq_scaling_matrix_present_flag: sps.seq_scaling_matrix_present_flag + vui_parameters_present_flag: sps.vui_parameters_present_flag + } + // Note: There is no 0 in StdVideoH264ProfileIdc enum + profile_idc: unsafe { vk.StdVideoH264ProfileIdc(sps.profile_idc) } + level_idc: unsafe { vk.StdVideoH264LevelIdc(sps.level_idc) } + chroma_format_idc: get_chroma_format(sps.profile_idc, sps.chroma_format_idc) + seq_parameter_set_id: u8(sps.seq_parameter_set_id) + bit_depth_luma_minus8: u8(sps.bit_depth_luma_minus8) + bit_depth_chroma_minus8: u8(sps.bit_depth_chroma_minus8) + log2_max_frame_num_minus4: u8(sps.log2_max_frame_num_minus4) + pic_order_cnt_type: unsafe { vk.StdVideoH264PocType(sps.pic_order_cnt_type) } + offset_for_non_ref_pic: sps.offset_for_non_ref_pic + log2_max_pic_order_cnt_lsb_minus4: u8(sps.log2_max_pic_order_cnt_lsb_minus4) + num_ref_frames_in_pic_order_cnt_cycle: u8(sps.num_ref_frames_in_pic_order_cnt_cycle) + max_num_ref_frames: u8(sps.num_ref_frames) + reserved1: 0 + pic_width_in_mbs_minus1: sps.pic_width_in_mbs_minus1 + pic_height_in_map_units_minus1: sps.pic_height_in_map_units_minus1 + frame_crop_left_offset: sps.frame_crop_left_offset + frame_crop_right_offset: sps.frame_crop_right_offset + frame_crop_top_offset: sps.frame_crop_top_offset + frame_crop_bottom_offset: sps.frame_crop_bottom_offset + reserved2: 0 + pOffsetForRefFrame: unsafe { nil } + pScalingLists: unsafe { &video_scaling_list_sps[i] } + pSequenceParameterSetVui: unsafe { &video_sequence_parameter_set_vui[i] } + } + + // VUI stands for "Video Usability Information" + vui := &sps.vui + + video_sequence_parameter_set_vui[i] = vk.StdVideoH264SequenceParameterSetVui{ + flags: vk.StdVideoH264SpsVuiFlags{ + aspect_ratio_info_present_flag: vui.aspect_ratio_info_present_flag + overscan_info_present_flag: vui.overscan_info_present_flag + overscan_appropriate_flag: vui.overscan_appropriate_flag + video_signal_type_present_flag: vui.video_signal_type_present_flag + video_full_range_flag: vui.video_full_range_flag + color_description_present_flag: vui.color_description_present_flag + chroma_loc_info_present_flag: vui.chroma_loc_info_present_flag + timing_info_present_flag: vui.timing_info_present_flag + fixed_frame_rate_flag: vui.fixed_frame_rate_flag + bitstream_restriction_flag: vui.bitstream_restriction_flag + nal_hrd_parameters_present_flag: vui.nal_hrd_parameters_present_flag + vcl_hrd_parameters_present_flag: vui.vcl_hrd_parameters_present_flag + } + aspect_ratio_idc: unsafe { vk.StdVideoH264AspectRatioIdc(vui.aspect_ratio_idc) } + sar_width: u16(vui.sar_width) + sar_height: u16(vui.sar_height) + video_format: u8(vui.video_format) + colour_primaries: u8(vui.colour_primaries) + transfer_characteristics: u8(vui.transfer_characteristics) + matrix_coefficients: u8(vui.matrix_coefficients) + num_units_in_tick: vui.num_units_in_tick + time_scale: vui.time_scale + max_num_reorder_frames: u8(vui.num_reorder_frames) + max_dec_frame_buffering: u8(vui.max_dec_frame_buffering) + chroma_sample_loc_type_top_field: u8(vui.chroma_sample_loc_type_top_field) + chroma_sample_loc_type_bottom_field: u8(vui.chroma_sample_loc_type_bottom_field) + reserved1: 0 + pHrdParameters: unsafe { &video_hrd_parameters[i] } + } + + hrd := &sps.hrd + video_hrd_parameters[i] = vk.StdVideoH264HrdParameters{ + cpb_cnt_minus1: u8(hrd.cpb_cnt_minus1) + bit_rate_scale: u8(hrd.bit_rate_scale) + cpb_size_scale: u8(hrd.cpb_size_scale) + // reserved1: u8(0) + // bit_rate_value_minus1: [u32(0)] + // cpb_size_value_minus1: [u32(0)] + // cbr_flag: [u8(0)] + initial_cpb_removal_delay_length_minus1: hrd.initial_cpb_removal_delay_length_minus1 + cpb_removal_delay_length_minus1: hrd.cpb_removal_delay_length_minus1 + dpb_output_delay_length_minus1: hrd.dpb_output_delay_length_minus1 + time_offset_length: hrd.time_offset_length + } + + for j in 0 .. vk.std_video_h264_cpb_cnt_list_size { + video_hrd_parameters[i].bit_rate_value_minus1[j] = hrd.bit_rate_value_minus1[j] + video_hrd_parameters[i].cpb_size_value_minus1[j] = hrd.cpb_size_value_minus1[j] + video_hrd_parameters[i].cbr_flag[j] = u8(hrd.cbr_flag[j]) + } + + // Fill scaling lists + video_scaling_list_sps[i] = vk.StdVideoH264ScalingLists{} + for j in 0 .. sps.seq_scaling_list_present_flag.len { + video_scaling_list_sps[i].scaling_list_present_mask |= u16(sps.seq_scaling_list_present_flag[j]) << j + } + for j in 0 .. sps.use_default_scaling_matrix_4x4_flag.len { + video_scaling_list_sps[i].use_default_scaling_matrix_mask |= u16(sps.use_default_scaling_matrix_4x4_flag[j]) << j + } + + mut list_idx := 0 + mut el_idx := 0 + for list_idx < vk.std_video_h264_scaling_list_4x4_num_lists && list_idx < sps.scaling_list_4x4.len { + for el_idx < vk.std_video_h264_scaling_list_4x4_num_elements && el_idx < sps.scaling_list_4x4[0].len { + unsafe { + C.vv_set_h264_scaling_list_4x4(&video_scaling_list_sps[i], u32(list_idx), u32(el_idx), u8(sps.scaling_list_4x4[list_idx][el_idx])) + } + el_idx++ + } + list_idx++ + } + + list_idx = 0 + el_idx = 0 + for list_idx < vk.std_video_h264_scaling_list_8x8_num_lists && list_idx < sps.scaling_list_8x8.len { + for el_idx < vk.std_video_h264_scaling_list_8x8_num_elements && el_idx < sps.scaling_list_8x8[0].len { + unsafe { + C.vv_set_h264_scaling_list_8x8(&video_scaling_list_sps[i], u32(list_idx), u32(el_idx), u8(sps.scaling_list_8x8[list_idx][el_idx])) + } + el_idx++ + } + list_idx++ + } + } // for d.video_data.sps_count + + mut session_parameters_add_info := vk.VideoDecodeH264SessionParametersAddInfoKHR{ + stdSPSCount: d.video_data.sps_count + pStdSPSs: video_sequence_parameter_set.data + stdPPSCount: d.video_data.pps_count + pStdPPSs: video_picture_parameter_sets.data + } + mut video_decode_session_parameters_ci := vk.VideoDecodeH264SessionParametersCreateInfoKHR{ + maxStdSPSCount: d.video_data.sps_count + maxStdPPSCount: d.video_data.pps_count + pParametersAddInfo: &session_parameters_add_info + } + video_session_parameters_ci := vk.VideoSessionParametersCreateInfoKHR{ + pNext: &video_decode_session_parameters_ci + flags: 0 + videoSessionParametersTemplate: unsafe { nil } + videoSession: d.video_session + } + + res := vk.create_video_session_parameters_khr(device, &video_session_parameters_ci, unsafe { nil }, &d.video_session_parameters) + if res != vk.Result.success { + panic('Could not create H.264 video-session parameters: ${res}') + } +} + +type U64 = u64 + +fn (sz U64) align_to(alignment usize) u64 { + return ((sz - 1) / alignment + 1) * alignment +} diff --git a/examples/video_decode_app/device_context.v b/device_context.v similarity index 82% rename from examples/video_decode_app/device_context.v rename to device_context.v index ce38282..11291da 100644 --- a/examples/video_decode_app/device_context.v +++ b/device_context.v @@ -1,4 +1,4 @@ -module video_decode_app +module main import antono2.vulkan as vk import antono2.vkmemalloc as vkmem @@ -13,18 +13,18 @@ import math // #flag -DVOLK_IMPLEMENTATION // #include "volk.h" -pub struct DeviceContext { +struct DeviceContext { mut: vk_debug_utils vk.DebugUtilsMessengerEXT = unsafe { nil } use_gpu_index int gpus []vk.PhysicalDevice queue_families []QueueFamilyProperties queue_family_indices []u32 - video_decode_family u32 = vk.queue_family_ignored + video_decode_family u32 = vk.queue_family_ignored graphics_queue vk.Queue = unsafe { nil } video_decode_queue vk.Queue = unsafe { nil } physical_device_memory_props []vk.PhysicalDeviceMemoryProperties2 - features2 vk.PhysicalDeviceFeatures2 = vk.PhysicalDeviceFeatures2{} + features2 vk.PhysicalDeviceFeatures2 = vk.PhysicalDeviceFeatures2{} vulkan12_features vk.PhysicalDeviceVulkan12Features = vk.PhysicalDeviceVulkan12Features{} vulkan13_features vk.PhysicalDeviceVulkan13Features = vk.PhysicalDeviceVulkan13Features{} video_profile_info vk.VideoProfileInfoKHR @@ -33,18 +33,18 @@ mut: video_decode_capabilities vk.VideoDecodeCapabilitiesKHR pub mut: vk_instance vk.Instance = unsafe { nil } - vk_device vk.Device = unsafe { nil } + vk_device vk.Device = unsafe { nil } swapchain Swapchain // @[required] sampler vk.Sampler = unsafe { nil } memory_allocator vkmem.Allocator sampler_ycbcr_conversion vk.SamplerYcbcrConversion video_decode_bitstream_alignment vk.DeviceSize = 1 - graphics_family u32 = vk.queue_family_ignored + graphics_family u32 = vk.queue_family_ignored } struct VideoDecodeH264 { mut: - profile vk.VideoDecodeH264ProfileInfoKHR = vk.VideoDecodeH264ProfileInfoKHR{} + profile vk.VideoDecodeH264ProfileInfoKHR = vk.VideoDecodeH264ProfileInfoKHR{} capabilities vk.VideoDecodeH264CapabilitiesKHR = vk.VideoDecodeH264CapabilitiesKHR{} } @@ -54,28 +54,28 @@ mut: properties_video vk.QueueFamilyVideoPropertiesKHR } -pub struct GPUBufferDesc { +struct GPUBufferDesc { pub mut: size vk.DeviceSize usage vk.BufferUsageFlags memory_property vk.MemoryPropertyFlagBits = vk.MemoryPropertyFlagBits.device_local } -pub struct GPUImageDesc { +struct GPUImageDesc { pub mut: extent vk.Extent3D - array_size u32 = 1 - mip_levels u32 = 1 + array_size u32 = 1 + mip_levels u32 = 1 image_type vk.ImageType = vk.ImageType._2d - format vk.Format = vk.Format.undefined - sample_count u32 = 1 + format vk.Format = vk.Format.undefined + sample_count u32 = 1 usage vk.ImageUsageFlags memory_property vk.MemoryPropertyFlagBits = vk.MemoryPropertyFlagBits.device_local } -pub struct GPUBuffer { +struct GPUBuffer { pub mut: - buffer vk.Buffer = unsafe { nil } + buffer vk.Buffer = unsafe { nil } memory vk.DeviceMemory = unsafe { nil } device_address vk.DeviceAddress p_mapped voidptr = unsafe { nil } @@ -83,10 +83,10 @@ pub mut: desc GPUBufferDesc } -pub struct GPUImage { +struct GPUImage { pub mut: - image vk.Image = unsafe { nil } - image_view vk.ImageView = unsafe { nil } + image vk.Image = unsafe { nil } + image_view vk.ImageView = unsafe { nil } memory vk.DeviceMemory = unsafe { nil } device_address vk.DeviceAddress @@ -95,24 +95,24 @@ pub mut: desc GPUImageDesc } -pub enum QueueType { +enum QueueType { graphics video_decode } -pub fn vulkan_debug_callback(messageSeverity vk.DebugUtilsMessageSeverityFlagBitsEXT, messageType vk.DebugUtilsMessageTypeFlagsEXT, data &vk.DebugUtilsMessengerCallbackDataEXT, userData voidptr) vk.Bool32 { +fn vulkan_debug_callback(messageSeverity vk.DebugUtilsMessageSeverityFlagBitsEXT, messageType vk.DebugUtilsMessageTypeFlagsEXT, data &vk.DebugUtilsMessengerCallbackDataEXT, userData voidptr) vk.Bool32 { if int(messageSeverity) & int(vk.DebugUtilsMessageSeverityFlagBitsEXT.error) != 0 { println(unsafe { data.pMessage.vstring() }) } return 0 } -pub fn (mut ctx DeviceContext) initialize() { +fn (mut ctx DeviceContext) initialize() { ctx.initialize_vk_instance() ctx.enumerate_gpus() } -pub fn (mut ctx DeviceContext) shutdown() { +fn (mut ctx DeviceContext) shutdown() { if !isnil(ctx.vk_device) { ctx.swapchain.shutdown() } @@ -130,7 +130,7 @@ pub fn (mut ctx DeviceContext) shutdown() { // Release resources created before logical-device selection. This is also used // when no installed GPU supports the requested Vulkan Video profile. -pub fn (mut ctx DeviceContext) shutdown_instance_resources() { +fn (mut ctx DeviceContext) shutdown_instance_resources() { if !isnil(ctx.swapchain.surface) && !isnil(ctx.vk_instance) { vk.destroy_surface_khr(ctx.vk_instance, ctx.swapchain.surface, unsafe { nil }) ctx.swapchain.surface = unsafe { nil } @@ -147,7 +147,7 @@ pub fn (mut ctx DeviceContext) shutdown_instance_resources() { } } -pub fn (mut ctx DeviceContext) initialize_device(use_gpu_index u32, h264_profile_idc u32, +fn (mut ctx DeviceContext) initialize_device(use_gpu_index u32, h264_profile_idc u32, metadata VideoMetadata) bool { mut n := unsafe { nil } mut family_props_count := u32(0) @@ -207,17 +207,17 @@ pub fn (mut ctx DeviceContext) initialize_device(use_gpu_index u32, h264_profile mut device_queue_ci := [ vk.DeviceQueueCreateInfo{ queueFamilyIndex: ctx.graphics_family - queueCount: 1 + queueCount: 1 pQueuePriorities: &default_prior - flags: 0 // If not 0, use vk.get_device_queue2 below + flags: 0 // If not 0, use vk.get_device_queue2 below }, ] if ctx.video_decode_family != ctx.graphics_family { device_queue_ci << vk.DeviceQueueCreateInfo{ queueFamilyIndex: ctx.video_decode_family - queueCount: 1 + queueCount: 1 pQueuePriorities: &default_prior - flags: 0 // If not 0, use vk.get_device_queue2 below + flags: 0 // If not 0, use vk.get_device_queue2 below } } @@ -235,10 +235,10 @@ pub fn (mut ctx DeviceContext) initialize_device(use_gpu_index u32, h264_profile ctx.vulkan13_features.pNext = &sampler_ycbcr_conversion_features mut device_create_info := vk.DeviceCreateInfo{ - pNext: &ctx.features2 - queueCreateInfoCount: u32(device_queue_ci.len) - pQueueCreateInfos: device_queue_ci.data - enabledExtensionCount: u32(active_device_extensions.len) + pNext: &ctx.features2 + queueCreateInfoCount: u32(device_queue_ci.len) + pQueueCreateInfos: device_queue_ci.data + enabledExtensionCount: u32(active_device_extensions.len) ppEnabledExtensionNames: active_device_extensions.data } @@ -286,15 +286,15 @@ pub fn (mut ctx DeviceContext) initialize_device(use_gpu_index u32, h264_profile ctx.video_decode_bitstream_alignment = math.max(ctx.video_decode_bitstream_alignment, ctx.video_capabilities.minBitstreamBufferSizeAlignment) allocator_create_info := vkmem.AllocatorCreateInfo{ - physical_device: ctx.get_gpu_current() - device: ctx.get_vk_device() + physical_device: ctx.get_gpu_current() + device: ctx.get_vk_device() memory_budget_enabled: memory_budget_supported } ctx.memory_allocator = vkmem.new(allocator_create_info) profile_list := vk.VideoProfileListInfoKHR{ profileCount: 1 - pProfiles: &ctx.video_profile_info + pProfiles: &ctx.video_profile_info } decode_output_usage := vk.ImageUsageFlags(u32(vk.ImageUsageFlagBits.video_decode_dst) | u32(vk.ImageUsageFlagBits.transfer_src)) decode_output_format := query_video_format(ctx.get_gpu_current(), &profile_list, decode_output_usage) or { @@ -302,22 +302,22 @@ pub fn (mut ctx DeviceContext) initialize_device(use_gpu_index u32, h264_profile } mut sampler_ycbcr_conversion_ci := vk.SamplerYcbcrConversionCreateInfo{ - format: decode_output_format.format - ycbcrModel: ycbcr_model_for_video(metadata) - ycbcrRange: if metadata.video_full_range { + format: decode_output_format.format + ycbcrModel: ycbcr_model_for_video(metadata) + ycbcrRange: if metadata.video_full_range { vk.SamplerYcbcrRange.itu_full } else { vk.SamplerYcbcrRange.itu_narrow } - components: vk.ComponentMapping{ + components: vk.ComponentMapping{ r: vk.ComponentSwizzle.identity g: vk.ComponentSwizzle.identity b: vk.ComponentSwizzle.identity a: vk.ComponentSwizzle.identity } - xChromaOffset: vk.ChromaLocation.midpoint - yChromaOffset: vk.ChromaLocation.midpoint - chromaFilter: vk.Filter.nearest + xChromaOffset: vk.ChromaLocation.midpoint + yChromaOffset: vk.ChromaLocation.midpoint + chromaFilter: vk.Filter.nearest forceExplicitReconstruction: 0 } @@ -402,11 +402,11 @@ fn missing_device_extensions(gpu vk.PhysicalDevice, required_extensions []&u8) [ return missing } -pub fn (ctx DeviceContext) h264_decode_gpu_diagnostics(h264_profile_idc u32) []string { +fn (ctx DeviceContext) h264_decode_gpu_diagnostics(h264_profile_idc u32) []string { return ctx.h264_decode_gpu_diagnostics_for_output_mode(h264_profile_idc, .automatic) } -pub fn (ctx DeviceContext) h264_decode_gpu_diagnostics_for_output_mode(h264_profile_idc u32, +fn (ctx DeviceContext) h264_decode_gpu_diagnostics_for_output_mode(h264_profile_idc u32, output_mode DecodeOutputMode) []string { required_extensions := [vk.khr_swapchain_extension_name, vk.khr_video_queue_extension_name, vk.khr_video_decode_queue_extension_name, vk.khr_video_decode_h264_extension_name] @@ -434,15 +434,15 @@ pub fn (ctx DeviceContext) h264_decode_gpu_diagnostics_for_output_mode(h264_prof return diagnostics } -pub fn (ctx DeviceContext) gpu_count() int { +fn (ctx DeviceContext) gpu_count() int { return ctx.gpus.len } -pub fn (ctx DeviceContext) is_h264_decode_gpu_compatible(gpu_index int, h264_profile_idc u32) bool { +fn (ctx DeviceContext) is_h264_decode_gpu_compatible(gpu_index int, h264_profile_idc u32) bool { return ctx.is_h264_decode_gpu_compatible_for_output_mode(gpu_index, h264_profile_idc, .automatic) } -pub fn (ctx DeviceContext) is_h264_decode_gpu_compatible_for_output_mode(gpu_index int, +fn (ctx DeviceContext) is_h264_decode_gpu_compatible_for_output_mode(gpu_index int, h264_profile_idc u32, output_mode DecodeOutputMode) bool { if gpu_index < 0 || gpu_index >= ctx.gpus.len { @@ -483,11 +483,11 @@ fn device_has_required_queues(ctx &DeviceContext, gpu vk.PhysicalDevice) bool { return has_graphics_and_present && has_h264_decode } -pub fn (ctx DeviceContext) find_h264_decode_gpu(h264_profile_idc u32) ?u32 { +fn (ctx DeviceContext) find_h264_decode_gpu(h264_profile_idc u32) ?u32 { return ctx.find_h264_decode_gpu_for_output_mode(h264_profile_idc, .automatic) } -pub fn (ctx DeviceContext) find_h264_decode_gpu_for_output_mode(h264_profile_idc u32, +fn (ctx DeviceContext) find_h264_decode_gpu_for_output_mode(h264_profile_idc u32, output_mode DecodeOutputMode) ?u32 { required_extensions := [vk.khr_swapchain_extension_name, vk.khr_video_queue_extension_name, vk.khr_video_decode_queue_extension_name, vk.khr_video_decode_h264_extension_name] @@ -514,11 +514,11 @@ fn gpu_h264_decode_capability_flags(gpu vk.PhysicalDevice, pictureLayout: vk.VideoDecodeH264PictureLayoutFlagBitsKHR.progressive } profile := vk.VideoProfileInfoKHR{ - pNext: &h264_profile + pNext: &h264_profile videoCodecOperation: vk.VideoCodecOperationFlagBitsKHR.decode_h264 - chromaSubsampling: vk.VideoChromaSubsamplingFlagsKHR(vk.VideoChromaSubsamplingFlagBitsKHR._420) - lumaBitDepth: vk.VideoComponentBitDepthFlagsKHR(vk.VideoComponentBitDepthFlagBitsKHR._8) - chromaBitDepth: vk.VideoComponentBitDepthFlagsKHR(vk.VideoComponentBitDepthFlagBitsKHR._8) + chromaSubsampling: vk.VideoChromaSubsamplingFlagsKHR(vk.VideoChromaSubsamplingFlagBitsKHR._420) + lumaBitDepth: vk.VideoComponentBitDepthFlagsKHR(vk.VideoComponentBitDepthFlagBitsKHR._8) + chromaBitDepth: vk.VideoComponentBitDepthFlagsKHR(vk.VideoComponentBitDepthFlagBitsKHR._8) } mut h264_caps := vk.VideoDecodeH264CapabilitiesKHR{} mut decode_caps := vk.VideoDecodeCapabilitiesKHR{ @@ -562,43 +562,43 @@ fn gpu_h264_output_mode_names(gpu vk.PhysicalDevice, h264_profile_idc u32) strin return if modes.len == 0 { 'none' } else { modes.join(', ') } } -pub fn (mut ctx DeviceContext) initialize_swapchain(window_p &glfw.Window, desired_format vk.Format) bool { +fn (mut ctx DeviceContext) initialize_swapchain(window_p &glfw.Window, desired_format vk.Format) bool { return ctx.swapchain.initialize(window_p, desired_format) } -pub fn (ctx DeviceContext) create_buffer(desc &GPUBufferDesc, buffer &GPUBuffer) { +fn (ctx DeviceContext) create_buffer(desc &GPUBufferDesc, buffer &GPUBuffer) { // TODO panic('Not implemented') } -pub fn (mut ctx DeviceContext) create_image(desc &GPUImageDesc, mut image &GPUImage) { +fn (mut ctx DeviceContext) create_image(desc &GPUImageDesc, mut image &GPUImage) { mut n := unsafe { nil } mut image_ci := vk.ImageCreateInfo{ - flags: 0 - imageType: desc.image_type - format: desc.format - extent: desc.extent - mipLevels: desc.mip_levels - arrayLayers: desc.array_size - samples: vk.SampleCountFlagBits._1 - tiling: vk.ImageTiling.optimal - usage: u32(desc.usage) - sharingMode: vk.SharingMode.exclusive + flags: 0 + imageType: desc.image_type + format: desc.format + extent: desc.extent + mipLevels: desc.mip_levels + arrayLayers: desc.array_size + samples: vk.SampleCountFlagBits._1 + tiling: vk.ImageTiling.optimal + usage: u32(desc.usage) + sharingMode: vk.SharingMode.exclusive queueFamilyIndexCount: 0 - pQueueFamilyIndices: unsafe { nil } - initialLayout: vk.ImageLayout.undefined + pQueueFamilyIndices: unsafe { nil } + initialLayout: vk.ImageLayout.undefined } image_ci.usage = vk.ImageUsageFlags(u32(vk.BufferUsageFlagBits.transfer_src) | u32(vk.BufferUsageFlagBits.transfer_dst)) mut profile_list_info := vk.VideoProfileListInfoKHR{ profileCount: 1 - pProfiles: &ctx.video_profile_info + pProfiles: &ctx.video_profile_info } if desc.usage & (u32(vk.ImageUsageFlagBits.video_decode_dst) | u32(vk.ImageUsageFlagBits.video_decode_src) | u32(vk.ImageUsageFlagBits.video_decode_dpb)) != 0 { image_ci.pNext = &profile_list_info mut video_format_info := vk.PhysicalDeviceVideoFormatInfoKHR{ - pNext: &profile_list_info + pNext: &profile_list_info imageUsage: image_ci.usage } mut format_count := u32(0) @@ -627,30 +627,30 @@ pub fn (mut ctx DeviceContext) create_image(desc &GPUImageDesc, mut image &GPUIm vk.get_image_memory_requirements2(ctx.vk_device, &info, mut &reqs) mut alloc_info := vk.MemoryAllocateInfo{ - allocationSize: reqs.memoryRequirements.size + allocationSize: reqs.memoryRequirements.size memoryTypeIndex: ctx.get_memory_type_index(reqs, vk.MemoryPropertyFlags(desc.memory_property)) } vk.allocate_memory(ctx.vk_device, &alloc_info, unsafe { nil }, &image.memory) vk.bind_image_memory(ctx.vk_device, image.image, image.memory, 0) mut view_ci := vk.ImageViewCreateInfo{ - flags: 0 - image: image.image - viewType: vk.ImageViewType._2d - format: image_ci.format + flags: 0 + image: image.image + viewType: vk.ImageViewType._2d + format: image_ci.format // NOTE: int(identity) = 0 - components: vk.ComponentMapping{ + components: vk.ComponentMapping{ r: vk.ComponentSwizzle.identity g: vk.ComponentSwizzle.identity b: vk.ComponentSwizzle.identity a: vk.ComponentSwizzle.identity } subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - baseMipLevel: 0 - levelCount: image_ci.mipLevels + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) + baseMipLevel: 0 + levelCount: image_ci.mipLevels baseArrayLayer: 0 - layerCount: image_ci.arrayLayers + layerCount: image_ci.arrayLayers } } @@ -672,7 +672,7 @@ pub fn (mut ctx DeviceContext) create_image(desc &GPUImageDesc, mut image &GPUIm vk.create_image_view(ctx.vk_device, &view_ci, unsafe { nil }, &image.image_view) } -pub fn (ctx DeviceContext) submit(queue_type QueueType, mut p_submit_info vk.SubmitInfo, mut wait_fence vk.Fence) { +fn (ctx DeviceContext) submit(queue_type QueueType, mut p_submit_info vk.SubmitInfo, mut wait_fence vk.Fence) { mut queue := match queue_type { .graphics { ctx.graphics_queue } .video_decode { ctx.video_decode_queue } @@ -681,30 +681,30 @@ pub fn (ctx DeviceContext) submit(queue_type QueueType, mut p_submit_info vk.Sub vk.queue_submit(queue, 1, p_submit_info, wait_fence) } -pub fn (mut ctx DeviceContext) present(wait_semaphores []vk.Semaphore) vk.Result { +fn (mut ctx DeviceContext) present(wait_semaphores []vk.Semaphore) vk.Result { mut handle := ctx.swapchain.get_handle() mut index := ctx.swapchain.get_current_index() mut present := vk.PresentInfoKHR{ waitSemaphoreCount: u32(wait_semaphores.len) - pWaitSemaphores: wait_semaphores.data - swapchainCount: 1 - pSwapchains: &handle - pImageIndices: &index - pResults: unsafe { nil } + pWaitSemaphores: wait_semaphores.data + swapchainCount: 1 + pSwapchains: &handle + pImageIndices: &index + pResults: unsafe { nil } } result := vk.queue_present_khr(ctx.graphics_queue, &present) vk.queue_wait_idle(ctx.graphics_queue) return result } -pub fn (ctx DeviceContext) get_queue(type QueueType) vk.Queue { +fn (ctx DeviceContext) get_queue(type QueueType) vk.Queue { return match type { .graphics { ctx.graphics_queue } .video_decode { ctx.video_decode_queue } } } -pub fn (mut ctx DeviceContext) initialize_vk_instance() bool { +fn (mut ctx DeviceContext) initialize_vk_instance() bool { if C.volkInitialize() != vk.Result.success { panic('Could not volkInitialize()') } @@ -740,23 +740,23 @@ pub fn (mut ctx DeviceContext) initialize_vk_instance() bool { mut app_info := vk.ApplicationInfo{ pApplicationName: c'Example' - pEngineName: c'Example' - engineVersion: vk.api_version_1_3 - apiVersion: vk.api_version_1_3 + pEngineName: c'Example' + engineVersion: vk.api_version_1_3 + apiVersion: vk.api_version_1_3 } mut instance_create_info := vk.InstanceCreateInfo{ - pApplicationInfo: &app_info - enabledLayerCount: u32(active_instance_layers.len) - ppEnabledLayerNames: &&u8(active_instance_layers.data) - enabledExtensionCount: u32(active_instance_extensions.len) + pApplicationInfo: &app_info + enabledLayerCount: u32(active_instance_layers.len) + ppEnabledLayerNames: &&u8(active_instance_layers.data) + enabledExtensionCount: u32(active_instance_extensions.len) ppEnabledExtensionNames: &&u8(active_instance_extensions.data) } mut debug_utils_create_info := vk.DebugUtilsMessengerCreateInfoEXT{} $if debug ? { debug_utils_create_info = vk.DebugUtilsMessengerCreateInfoEXT{ messageSeverity: vk.DebugUtilsMessageSeverityFlagsEXT(vk.DebugUtilsMessageSeverityFlagBitsEXT.error) | vk.DebugUtilsMessageSeverityFlagsEXT(vk.DebugUtilsMessageSeverityFlagBitsEXT.warning) - messageType: vk.DebugUtilsMessageTypeFlagsEXT(vk.DebugUtilsMessageTypeFlagBitsEXT.validation) | vk.DebugUtilsMessageTypeFlagsEXT(vk.DebugUtilsMessageTypeFlagBitsEXT.performance) + messageType: vk.DebugUtilsMessageTypeFlagsEXT(vk.DebugUtilsMessageTypeFlagBitsEXT.validation) | vk.DebugUtilsMessageTypeFlagsEXT(vk.DebugUtilsMessageTypeFlagBitsEXT.performance) pfnUserCallback: vulkan_debug_callback } instance_create_info.pNext = &debug_utils_create_info @@ -775,7 +775,7 @@ pub fn (mut ctx DeviceContext) initialize_vk_instance() bool { return true } -pub fn (mut ctx DeviceContext) enumerate_gpus() { +fn (mut ctx DeviceContext) enumerate_gpus() { mut gpu_count := u32(0) vk.enumerate_physical_devices(ctx.vk_instance, &gpu_count, unsafe { nil }) @@ -794,7 +794,7 @@ pub fn (mut ctx DeviceContext) enumerate_gpus() { } } -pub fn (ctx DeviceContext) get_memory_type_index(reqs vk.MemoryRequirements2, flags vk.MemoryPropertyFlags) u32 { +fn (ctx DeviceContext) get_memory_type_index(reqs vk.MemoryRequirements2, flags vk.MemoryPropertyFlags) u32 { mut request_bits := reqs.memoryRequirements.memoryTypeBits memory_props := ctx.physical_device_memory_props[ctx.use_gpu_index].memoryProperties for i in 0 .. memory_props.memoryTypeCount { @@ -809,25 +809,25 @@ pub fn (ctx DeviceContext) get_memory_type_index(reqs vk.MemoryRequirements2, fl return max_u32 } -pub fn (ctx DeviceContext) get_gpu(gpu_index int) vk.PhysicalDevice { +fn (ctx DeviceContext) get_gpu(gpu_index int) vk.PhysicalDevice { return ctx.gpus[gpu_index] } -pub fn (ctx DeviceContext) get_gpu_current() vk.PhysicalDevice { +fn (ctx DeviceContext) get_gpu_current() vk.PhysicalDevice { assert ctx.gpus.len > ctx.use_gpu_index && !isnil(ctx.gpus[ctx.use_gpu_index]) return ctx.gpus[ctx.use_gpu_index] } // TODO: Do we want getters/setters? -pub fn (ctx DeviceContext) get_vk_device() vk.Device { +fn (ctx DeviceContext) get_vk_device() vk.Device { return ctx.vk_device } -pub fn (ctx DeviceContext) wait_for_idle() { +fn (ctx DeviceContext) wait_for_idle() { vk.queue_wait_idle(ctx.graphics_queue) vk.queue_wait_idle(ctx.video_decode_queue) } -pub fn (ctx DeviceContext) get_decoder_queue_family_index() u32 { +fn (ctx DeviceContext) get_decoder_queue_family_index() u32 { return ctx.video_decode_family } diff --git a/docs/guide/01-input-and-order.md b/docs/guide/01-input-and-order.md new file mode 100644 index 0000000..1398741 --- /dev/null +++ b/docs/guide/01-input-and-order.md @@ -0,0 +1,65 @@ +# 1. Validate input and distinguish picture orders + +The first useful boundary is between an encoded file and the requirements it +places on a decoder. This player reads the MP4 before selecting a GPU. That +lets device selection use the stream's actual H.264 profile rather than an +assumed profile. + +## Trace the input + +[`VideoPlayer.prepare`](../../video_player.v#L415) calls +[`Decoder.parse_mp4_data`](../../mp4_parser.v#L100). The parser finds an H.264 +track, checks its timescale and samples, reads SPS and PPS data, and records +picture dimensions, profile, timing, references, and display metadata. MP4 +gives sample offsets and durations; H.264 headers give codec rules such as +picture order count (POC) and reference status. The parser rejects unsupported +formats before Vulkan session creation. This matters for errors as well as +efficiency: an invalid file should not leave a half-created GPU decoder. + +The implementation also checks file size and reads at absolute sample offsets +through [`read_callback`](../../mp4_parser.v#L17). Tests cover +[non-MP4 input](../../video_player_test.v#L214), +[truncation](../../video_player_test.v#L253), and +[short reads](../../video_player_test.v#L233). In another project, input +could instead be a network segment or a camera stream. The boundary remains +useful: turn untrusted bytes into validated stream requirements before asking +the device to allocate resources. + +## Two orders, two jobs + +H.264 B-pictures can be displayed before a reference picture that must be +decoded first. The bundled 360p fixture starts with decode-order pictures +whose display orders are `0, 6, 3, 1, 2, 4, 5`. A loop that displays each +picture immediately after decoding would show jumps in time. + +```mermaid +flowchart LR + A[Decode: 0, 6, 3, 1, 2, 4, 5] --> B[Retain ready output images] + B --> C[Display: 0, 1, 2, 3, 4, 5, 6] +``` + +[`parse_mp4_data`](../../mp4_parser.v#L501) assigns a display order to each +picture while preserving decode order for the decoder. The DPB retains +reference pictures for the codec; the output-image queue retains decoded +pictures waiting for presentation. Those are different lifetimes. The +[`presentation_buffer_size`](../../video_player.v#L253) calculation looks at the +stream's display-order sequence to bound the waiting queue. The fixture-based +tests assert the [early sequence](../../video_player_test.v#L192) and +[required queue depth](../../playback_timeline_test.v#L59). + +**Invariant:** decode input advances in codec order; presentation advances +only when the next display-order picture is ready. The next display-order +number does not have to equal the current decode index. + +## Apply the boundary elsewhere + +| Input design | Useful when | Work it adds | +| --- | --- | --- | +| Parse the complete local file first, as here | A short or seekable file can be scanned before playback. | Startup scanning and metadata memory. | +| Incremental parser and bounded reorder queue | Live or long-form streaming. | Backpressure, incomplete access units, format changes, and recovery from dropped data. | +| External demux and codec library | Broad container or codec support matters more than direct control of the Vulkan path. | A second API and explicit ownership of decoded frames and timestamps. | + +For a streaming player, do not copy this file parser wholesale. Keep the +contract it demonstrates: parsed access units carry enough information to +select a compatible decoder, retain references, schedule presentation, and +report malformed input without corrupting GPU state. diff --git a/docs/guide/02-capabilities.md b/docs/guide/02-capabilities.md new file mode 100644 index 0000000..0c35f2f --- /dev/null +++ b/docs/guide/02-capabilities.md @@ -0,0 +1,61 @@ +# 2. Select a device from the video's requirements + +“Supports Vulkan” is too broad a device test. A player needs a graphics queue +that can present to its window, an H.264 decode queue, the Vulkan Video +extensions, a compatible profile and picture layout, and usable image formats. +The chosen MP4 may require a profile that another file does not. + +## The selection path + +After parsing, [`VideoDecodeApp.initialize`](../../app.v#L179) obtains the stream's +profile and asks [`h264_decode_gpu_diagnostics_for_output_mode`](../../device_context.v#L409) +for diagnostics for every GPU. [`--list-gpus`](../../app.v#L182) exposes those +diagnostics to the user. A forced [`--gpu` index](../../app.v#L197) is checked +against the same requirements; otherwise the first +compatible device is chosen. Errors name the missing capability instead of +assuming that a graphics-capable GPU can decode. + +[`initialize_device`](../../device_context.v#L150) then chooses queue families and +creates the logical device with the required extensions. It builds a +[VideoProfileInfoKHR](../../device_context.v#L267) for progressive 8-bit 4:2:0 H.264 and chains H.264 +profile and capability structs through `pNext`. Vulkan Video format queries +use the same profile. A format is useful only if it supports the image usages +required by the next step, including transfer out of the decoded picture in +this design. + +The player accepts the two advertised DPB/output modes. In *coincident* mode, +the decoded output is a DPB image. In *distinct* mode, the output and DPB +images are separate. `auto` prefers coincident and falls back to distinct; +forced modes aid driver validation and fail if unsupported. The choice is made +by [`select_decode_output_mode`](../../video_player.v#L17), with software tests +for [automatic fallback](../../video_player_test.v#L23) and +[forced modes](../../video_player_test.v#L28). + +```mermaid +flowchart TD + Stream[Parsed H.264 profile and dimensions] --> Extensions[Required extensions] + Extensions --> Queues[Presenting graphics and H.264 decode queues] + Queues --> Profile[Profile capabilities and limits] + Profile --> Formats[Output and DPB formats with required usages] + Formats --> Mode[Coincident or distinct output mode] + Mode --> Device[Create device and decoder] +``` + +**Invariant:** the profile and image usage used for capability queries must +match the resources later created. A successful generic Vulkan device +creation is not proof that a video session or its images will work. + +## Apply the selection pattern elsewhere + +Build a requirement record from the content first: codec, profile, bit depth, +chroma, coded extent, and any presentation requirements. Probe candidates +against that record. For an editor handling many clips, the policy might be +“choose one device that handles every clip” or “select a backend per clip.” For +a game that uses video only for optional cutscenes, software decode could be +a fallback. Each policy changes when and how compatibility failures should +be reported. + +The application has one window and chooses one GPU. Multi-GPU transfer, +software fallback, and runtime format changes are outside its current design. +See [platform support](../../PLATFORM_SUPPORT.md) before treating a successful +build as playback support. diff --git a/docs/guide/03-decode-resources.md b/docs/guide/03-decode-resources.md new file mode 100644 index 0000000..800e5bb --- /dev/null +++ b/docs/guide/03-decode-resources.md @@ -0,0 +1,60 @@ +# 3. Give codec references and display images separate lifetimes + +Vulkan Video makes the application describe and own resources that a +high-level media API might hide. The important design question is which +resource belongs to the codec, which belongs to presentation, and what must +remain valid while commands are in flight. + +## Build the session from parsed limits + +[`Decoder.initialize`](../../decoder_session.v#L8) asks the device for H.264 +capabilities and supported formats. It checks the parsed DPB slot and active +reference counts against device limits. It allocates an aligned bitstream +buffer, [creates a `VideoSessionKHR`](../../decoder_session.v#L119), queries its opaque memory requirements, +binds that memory, and creates session parameters from SPS/PPS data. The +alignment used for each upload slot comes from the queried minimum bitstream +offset and size alignments. [`write_video_frame`](../../player_decode.v#L521) fills +one slot with the next access unit and its slice offsets. + +The DPB stores reference pictures for later H.264 predictions. A picture that +is no longer needed as a reference may free a DPB slot even if its display +time has not arrived. Conversely, a displayed picture may still be referenced +by later decode operations. This is why the app copies decode results into a +separate bounded pool of [`OutputImage`](../../video_player.v#L173) objects, +[allocated after decoder setup](../../video_player.v#L521). The +graphics side samples that pool, not a DPB slot whose codec lifetime it does +not control. + +In coincident mode, the copy source is the current DPB image. In distinct +mode, the copy source is a separate decode-output image; the DPB remains +reference storage. [`copy_decoded_frame_to_output`](../../player_decode.v#L187) +selects the correct source and restores its decode layout after the copy. + +```mermaid +flowchart LR + Bits[Aligned bitstream slot] --> Session[Video session] + Params[SPS/PPS session parameters] --> Session + Session --> DPB[DPB reference images] + Session --> Distinct[Separate decode output when required] + DPB --> Copy[Copy into display-owned image] + Distinct --> Copy + Copy --> Ready[Ready output queue] +``` + +**Invariant:** a DPB slot cannot be overwritten while the codec still needs +it, and an output image cannot be reused while graphics may still sample it. +Those rules are related but have different owners. + +## Other ownership choices + +| Choice | Where it can fit | Constraint | +| --- | --- | --- | +| Copy to a separate display pool, as here | A compact player that wants an explicit presentation queue and simple sampling ownership. | Extra image memory and a per-picture copy. | +| Share decoded images directly with rendering | A pipeline whose decoder and renderer can coordinate image lifetime precisely. | Reference, display, queue, and descriptor lifetimes must all be tracked together. | +| Use a higher-level decoder API | Broad codec support or faster integration. | Less direct control of Vulkan Video session setup and possibly an interop copy. | + +For another project, calculate the bound from reorder depth, concurrent +uploads, and frames still sampled by graphics. A fixed pool is safe only if +the producer pauses when no image is reusable. Here +[`VideoPlayer.update`](../../player_presentation.v#L127) checks the free pool before +starting another decode. diff --git a/docs/guide/04-synchronization.md b/docs/guide/04-synchronization.md new file mode 100644 index 0000000..0120c9b --- /dev/null +++ b/docs/guide/04-synchronization.md @@ -0,0 +1,67 @@ +# 4. Make queue order and image reuse explicit + +Video decode and graphics may use different Vulkan queues. Recording commands +in source order does not, by itself, make data visible across queues. This +chapter follows the synchronization path and the output-image reuse rule. + +## Follow one submission + +[`update_decode_video`](../../player_decode.v#L81) waits on the fence for the +bitstream upload slot it is about to reuse, then writes and flushes the encoded +bytes. It records the video command buffer: prepare image layouts, begin and +perform decode, copy the result to an output image, and restore the source +for later decode use. The output image is tagged with display order and placed +in the ready queue. + +[`VideoPlayer.update`](../../player_presentation.v#L156) submits that command buffer on +the video queue and signals a semaphore. It then submits a graphics command +buffer that waits on the semaphore and transitions the copied image for +fragment sampling. The app's render command buffer waits for the player's +event before drawing from the selected output view. The swapchain also has +its own [acquire](../../app.v#L373) and +[render-complete](../../app.v#L463) semaphores, managed in +[`VideoDecodeApp.run`](../../app.v#L355). + +```mermaid +sequenceDiagram + participant CPU + participant Video as Video queue + participant Graphics as Graphics queue + CPU->>CPU: Wait for upload-slot fence + CPU->>Video: Decode and copy to output image + Video-->>Graphics: Signal decode semaphore + Graphics->>Graphics: Transition output for shader read + Graphics-->>Graphics: Signal player event + Graphics->>Graphics: Sample image and render swapchain +``` + +Image barriers in [`video_decode_pre_barrier`](../../player_decode.v#L340) and +[`copy_decoded_frame_to_output`](../../player_decode.v#L187) describe access and +layout transitions. The semaphore orders work between queues. These solve +different problems: a layout name alone does not wait for a prior queue's +writes, and a semaphore alone does not describe the next image layout. + +## Retire rather than immediately recycle + +Presentation may replace the current output view while an earlier graphics +submission still samples it. [`retire_current_output`](../../player_presentation.v#L29) +marks that image for later reuse; [`reclaim_output_textures`](../../player_presentation.v#L14) +returns it to the free pool only after enough later render submissions have +begun. This design relies on the app's one fence per swapchain image and +ordered graphics submissions. [Resize](../../app.v#L485) waits for the device to become idle +before rebuilding swapchain-dependent resources. + +**Invariant:** neither a mapped bitstream region nor an output image is +reused until the submissions that consume it are complete. Preserve this rule +if you change the number of frames in flight or introduce another queue. + +## Alternatives for another renderer + +A timeline semaphore can attach explicit completion values to frames and +make image retirement independent of a fixed swapchain image count. A render +graph can own barriers and resource lifetimes if it models video decode and +cross-queue dependencies. A single-queue design can simplify ordering on +devices that support the required operations together, but it still needs +correct image access and layout transitions. Pick one completion model and +make every reusable resource refer to it; mixing an old frame-count heuristic +with a new submission scheme invites premature reuse. diff --git a/docs/guide/05-presentation.md b/docs/guide/05-presentation.md new file mode 100644 index 0000000..7bbc549 --- /dev/null +++ b/docs/guide/05-presentation.md @@ -0,0 +1,68 @@ +# 5. Turn decoded pictures into correctly displayed frames + +Correct decode is only part of playback. Pictures need a display-order gate, +a media clock, color conversion, and a transform that respects pixel aspect +ratio and rotation metadata. + +## Display order and time + +[`VideoPlayer.update_presentation`](../../player_presentation.v#L72) asks for the next +display-order image. It initially waits until enough reordered pictures are +ready; then it advances only when the current duration is due and that next +image exists. At end of stream it keeps the final picture visible while the +next loop's early pictures are decoded. It resets decoder reference state for +the next cycle rather than showing pictures in decode order. + +[`PlaybackTimeline`](../../playback_timeline.v#L10) accumulates elapsed nanoseconds, +subtracts a picture's duration when the picture is presented, and caps a long +stall at 500 ms. Its tests cover [fixed](../../playback_timeline_test.v#L3) +and [variable](../../playback_timeline_test.v#L15) rates, +[missing next pictures](../../playback_timeline_test.v#L39), +[stalls](../../playback_timeline_test.v#L28), and +[reset](../../playback_timeline_test.v#L50). A project requiring +audio sync would need an explicit master clock and a policy for late pictures; +this player has no audio path. + +**Invariant:** a picture's duration is consumed when it is presented, not +merely when the wall clock passes it. Otherwise a temporarily missing +reordered image can cause a burst of frames when it arrives. + +## Color and geometry are media data + +[`parse_mp4_data`](../../mp4_parser.v#L152) stores the track matrix and +[H.264 video usability metadata](../../mp4_parser.v#L203). +[`ycbcr_model_for_video`](../../device_context.v#L338) +chooses a Vulkan YCbCr model from signaled matrix coefficients, with a +resolution-based fallback when no description is present. The Vulkan sampler +conversion uses the video's full or limited range. The app draws into an +[`UNORM swapchain`](../../app.v#L220) so display-encoded YCbCr conversion is +not encoded as sRGB a second time. + +[`video_render_transform`](../../app.v#L91) applies quarter-turn rotation and +letterboxing based on display dimensions. Those dimensions incorporate sample +aspect ratio before rotation. The picture can therefore have a coded width, +a display width, and a window width that differ. Tests cover +[track matrices](../../video_player_test.v#L47), +[sample aspect ratio](../../video_player_test.v#L54), +[conversion choices](../../video_player_test.v#L84), and +[portrait letterboxing](../../video_player_test.v#L314). + +```mermaid +flowchart LR + Encoded[Coded size and YCbCr samples] --> Metadata[SAR, matrix, range, rotation] + Metadata --> Convert[YCbCr sampler conversion] + Metadata --> Transform[Rotation and fit to window] + Convert --> Draw[Fragment sampling] + Transform --> Draw +``` + +## Adaptation choices + +For an editor, preserve source timestamps and expose a seek clock instead of +looping automatically. For live video, decide whether latency or every +picture matters more: a late picture might be dropped, but a referenced +picture may still need decoding. For HDR or wide-gamut material, a simple +UNORM output and the current matrix mapping are insufficient; the project +needs an explicit transfer-function, gamut, and display pipeline. Those are +product decisions layered on top of the same separation between decode +order, media time, and presentation. diff --git a/docs/guide/06-lifecycle-and-verification.md b/docs/guide/06-lifecycle-and-verification.md new file mode 100644 index 0000000..bfd620e --- /dev/null +++ b/docs/guide/06-lifecycle-and-verification.md @@ -0,0 +1,58 @@ +# 6. Own the lifecycle and test the right boundary + +The player creates resources in layers. Input validation comes before GPU +selection; instance and surface come before presentation checks; the logical +device comes before session, images, and descriptors. Shutdown follows those +dependencies in reverse. Resize rebuilds resources tied to the swapchain +without reparsing the MP4. + +## Trace ownership + +[`main`](../../main.v#L71) owns the `VideoDecodeApp` lifetime. +[`VideoDecodeApp.initialize`](../../app.v#L145) creates the window, instance, +surface, compatible device, swapchain, descriptors, pipeline, ImGui backend, +and player resources. [`VideoDecodeApp.run`](../../app.v#L355) handles out-of-date +swapchain results and calls [`recreate_swapchain`](../../app.v#L485), which waits +for idle, releases per-frame resources, resizes, and recreates the dependent +resources. [`shutdown`](../../app.v#L534) waits for the device, releases player and +graphics resources, then destroys the device, window, and GLFW state. + +Input and capability failures during initialization use +[`abort_initialization`](../../app.v#L587) and close the input file. Later Vulkan +allocation or submission failures can still be fatal; the repository does not +claim to recover from every partial GPU initialization. The precise support +boundary is in [Supported media and failure behavior](../../PLATFORM_SUPPORT.md#supported-media-and-failure-behavior). + +```mermaid +flowchart TD + Parse[Validate MP4] --> Instance[Instance, surface, GPU probe] + Instance --> Device[Device and swapchain] + Device --> Decoder[Session, DPB and output images] + Decoder --> Frames[Per-frame descriptors and commands] + Frames --> Resize[Resize: wait, release dependent objects, rebuild] + Frames --> Shutdown[Shutdown: wait, release in reverse order] +``` + +## What the checks establish + +| Check | What it establishes | What it cannot establish | +| --- | --- | --- | +| [`v test .`](../../README.md#tests) | CLI parsing, MP4 validation, metadata, reorder depth, and timeline rules for the test fixtures. | Driver video commands, image barriers, or visible output. | +| [Root executable build](../../BUILDING.md#shared-dear-imgui-default) | V/C bindings, native linking, and package entry point. | Compatible hardware or correct playback. | +| [`--list-gpus VIDEO`](../../README.md#run) | The current driver advertises the required capabilities for that stream. | That a full decode and resize session succeeds. | +| [Playback and resize on a supported GPU](../../PLATFORM_SUPPORT.md#hardware-validation-checklist) | The tested media and driver complete the actual path. | Other codecs, GPUs, operating systems, or long-running stability. | + +The [README](../../README.md#tests) has the software command; +[Platform Support](../../PLATFORM_SUPPORT.md) has the hardware matrix, and the +[hardware checklist](../../PLATFORM_SUPPORT.md#hardware-validation-checklist) +lists release checks. A useful development loop is to run software tests for +every parser or timing change, then use short media fixtures with B-frames, +rotation, and different color metadata on an actual decode-capable GPU. + +## Transfer the approach + +Keep a testable media clock and parser separate from hardware submission. +Record which failures can be returned cleanly and which still abort. When +adding seeking, stream changes, audio, or multiple windows, draw a new +resource-lifetime diagram first: each feature adds a new point at which a +buffer, reference picture, or display image may still be in use. diff --git a/docs/learning-path.md b/docs/learning-path.md new file mode 100644 index 0000000..291aaa9 --- /dev/null +++ b/docs/learning-path.md @@ -0,0 +1,94 @@ +# Design guide: building a Vulkan Video player + +This guide uses `v_vulkan_video` to explain decisions that recur in media and +GPU projects: validating input before choosing a device, retaining codec +references while presenting in a different order, coordinating queues, and +owning resources across resize and shutdown. It is for readers who know basic +V and Vulkan graphics but are new to Vulkan Video. The code is an application, +not a reusable player library. + +The implementation plays progressive 8-bit 4:2:0 H.264/AVC in MP4. It is one +design for one codec and container. The alternatives in this guide describe +possible designs for other projects, not features already implemented here. +Start with the [README](../README.md#run) to run the player, or the +[Quick Start](../QUICKSTART.md) to set up a build. + +## System map + +An *access unit* is the encoded data for one picture. The *decoded picture +buffer* (DPB) retains pictures that later pictures may reference. *Decode +order* is the order the codec needs; *display order* is the order viewers see. +An *output image* here is a separate image copied from the decode result and +kept until the graphics queue has finished sampling it. + +H.264 *sequence parameter sets* (SPS) and *picture parameter sets* (PPS) +describe decoding rules for a stream and its pictures. *Picture order count* +(POC) helps reconstruct display order. *YCbCr* represents luma and chroma +components; the renderer converts them to display color while sampling. + +```mermaid +flowchart LR + File[MP4 file] --> Parser[Track, SPS/PPS and picture metadata] + Parser --> Probe[GPU and video-profile probe] + Parser --> Upload[Encoded access units] + Probe --> Session[Vulkan Video session and DPB] + Upload --> Session + Session --> Copy[Copy to bounded output-image queue] + Copy --> Clock[Display-order clock] + Clock --> Render[YCbCr sampling and aspect/rotation transform] + Render --> Swapchain[Present] +``` + +The application enters at [`main`](../main.v#L71). The drawing loop is +[`VideoDecodeApp.run`](../app.v#L355), and GPU selection is in +[`find_h264_decode_gpu_for_output_mode`](../device_context.v#L490). +[`parse_mp4_data`](../mp4_parser.v#L100) reads the input, +[`update_decode_video`](../player_decode.v#L7) records decode work, and +[`update_presentation`](../player_presentation.v#L72) manages output images. +[`PlaybackTimeline`](../playback_timeline.v#L10) tracks media time. These are +source file responsibilities within one V package, not public modules. + +## Read by decision + +| Chapter | Question to take to another project | +| --- | --- | +| [1. Input and picture order](guide/01-input-and-order.md) | What must be known about the stream before allocation, and why can decode and display order differ? | +| [2. Device and format selection](guide/02-capabilities.md) | How should an application turn actual media requirements into a device choice? | +| [3. Decode resources](guide/03-decode-resources.md) | Which images and buffers does a decoder own, and how do the supported output modes change them? | +| [4. Queues and image lifetime](guide/04-synchronization.md) | When is a decoded image safe to copy, sample, and reuse? | +| [5. Presentation and metadata](guide/05-presentation.md) | How do picture order, media time, color, rotation, and window size become a displayed frame? | +| [6. Lifecycle and verification](guide/06-lifecycle-and-verification.md) | Which checks prove parser behavior, and which still require real hardware? | + +Each chapter traces the implementation, states an invariant, and compares +alternatives. The code links are navigation aids; the explanation should make +the idea understandable without reading every Vulkan call. + +## A route through one picture + +1. [`prepare`](../video_player.v#L415) calls the + [MP4 parser](../mp4_parser.v#L100), which rejects unsupported streams. + Its profile and dimensions inform GPU selection. +2. [`initialize_device`](../device_context.v#L150) selects graphics and H.264 + decode queues, queries the video profile, and chooses a compatible format. +3. [`Decoder.initialize`](../decoder_session.v#L8) creates the session, + bitstream buffer, and DPB images. [`VideoPlayer.initialize`](../video_player.v#L521) + allocates the bounded output-image pool. +4. [`update_decode_video`](../player_decode.v#L7) uploads an access unit, records + decode and copy commands, and tags the copied output with display order. +5. [`update_presentation`](../player_presentation.v#L72) chooses the next display-order + image when its duration is due. [`VideoDecodeApp.run`](../app.v#L355) samples it + and presents the swapchain image. + +The [queue submission trace](guide/04-synchronization.md#follow-one-submission) +explains why these steps cannot be collapsed into a simple “decode, then draw” +call on every device. + +## Evidence and limits + +[`v test .`](../README.md#tests) exercises parser, ordering, timing, CLI, and +metadata rules without requiring a Vulkan Video GPU. It does not execute +video commands or prove cross-queue synchronization. See +[tested platforms](../PLATFORM_SUPPORT.md) and the +[hardware checklist](../PLATFORM_SUPPORT.md#hardware-validation-checklist). +When adapting the design, repeat those checks with the codec, +driver, devices, and media that your product actually supports. diff --git a/examples/video_decode_app/README.txt b/examples/video_decode_app/README.txt deleted file mode 100644 index 01507b7..0000000 --- a/examples/video_decode_app/README.txt +++ /dev/null @@ -1,13 +0,0 @@ -Run - - ./run.sh [--list-gpus] [--gpu INDEX] [video.mp4] - -The bundled sample is used when the path is omitted. Debug builds enable the -Khronos validation layer and print Vulkan validation messages. - -`--list-gpus` reports every Vulkan device and whether it can decode the input -video's H.264 profile. `--gpu INDEX` selects one of those devices explicitly. - -Requires - -sudo apt install libimgui-dev diff --git a/examples/video_decode_app/video_player.v b/examples/video_decode_app/video_player.v deleted file mode 100644 index 8547865..0000000 --- a/examples/video_decode_app/video_player.v +++ /dev/null @@ -1,2641 +0,0 @@ -module video_decode_app - -import antono2.vulkan as vk -import os -import math -import antono2.vkmemalloc as vkmem -import antono2.minimp4 -import antono2.h264 - -pub const max_texture_count = 64 -pub const slot_count = 17 - -pub enum DecodeOutputMode { - automatic - coincident - distinct -} - -fn select_decode_output_mode(requested DecodeOutputMode, supports_coincident bool, - supports_distinct bool) !DecodeOutputMode { - return match requested { - .automatic { - if supports_coincident { - DecodeOutputMode.coincident - } else if supports_distinct { - DecodeOutputMode.distinct - } else { - return error('Vulkan Video device supports neither coincident nor distinct DPB/output images') - } - } - .coincident { - if !supports_coincident { - return error('Vulkan Video device does not support forced coincident DPB/output images') - } - DecodeOutputMode.coincident - } - .distinct { - if !supports_distinct { - return error('Vulkan Video device does not support forced distinct DPB/output images') - } - DecodeOutputMode.distinct - } - } -} - -pub struct VideoPlayer { -mut: - decode_operation DecoderVideoDecodeOperation - output_textures []OutputImage - output_textures_free []int - output_textures_ready []int - output_textures_retired []RetiredOutputImage - current_output_index int = -1 - next_display_order int - presentation_buffer_count int = 1 - decode_finished bool - waiting_for_loop_start bool - render_serial u64 - is_stopped bool - is_looping bool = true - dpb_slot_used []int - dpb DPB - current_frame int - flags u32 - video_frames []VideoPlayerDecodeStreamFrame - decode_output_image Image - decode_output_state DPBResourceState - playback_timeline PlaybackTimeline - current_upload_index int - graphics_command_pool vk.CommandPool = unsafe { nil } - video_command_pool vk.CommandPool = unsafe { nil } -pub mut: - app &VideoDecodeApp = unsafe { nil } - decoder shared Decoder - event_video_player vk.Event - command_buffer_infos []CommandBufferInfo -} - -pub struct Decoder { -pub mut: - properties DecoderQueryProperties - settings DecoderSettings - video_data DecoderVideoFileProperties - gpu_bitstream_buffer vk.Buffer = unsafe { nil } - gpu_bitstream_allocation vkmem.AllocationInfo - session_memory_allocations []vk.DeviceMemory - video_session vk.VideoSessionKHR = unsafe { nil } - video_session_parameters vk.VideoSessionParametersKHR = unsafe { nil } - info DecoderInfo -} - -pub struct DecoderDpbState { -pub mut: - slotindex int - frame_num int - reference_info vk.StdVideoDecodeH264ReferenceInfo -} - -pub struct DecoderInfo { -pub mut: - memory_frames []DecoderVideoMemoryFrameInfo - images_dpb []DecoderDpbImage - dpb_state []DecoderDpbState -} - -pub struct DecoderVideoMemoryFrameInfo { -pub mut: - data_frame_info &DecoderVideoDataFrameInfo - gpu_bitstream_offset u64 - gpu_bitstream_capacity u64 - gpu_bitstream_size u64 - gpu_bitstream_slice_mapped_memory_address &u8 - decoding_frame_index int = -1 -} - -pub struct VideoPlayerDecodeStreamFrame { -pub mut: - gpu_bitstream_capacity u64 - gpu_bitstream_offset u64 - gpu_bitstream_size u64 - gpu_bitstream_slice_mapped_memory_address byteptr - slice_offsets []u32 - in_flight_fence vk.Fence = unsafe { nil } -} - -pub struct CommandBufferInfo { -pub mut: - video_command_buffer vk.CommandBuffer - graphics_command_buffer vk.CommandBuffer - sem_video_to_gfx vk.Semaphore -} - -pub enum VideoPlayerFlags as u32 { - e_none = 0 - e_playing = 1 << 1 - e_decoder_reset = 1 << 3 - e_need_resolve = 1 << 4 -} - -pub struct DPB { -pub mut: - image [slot_count]Image - resource_state [slot_count]DPBResourceState - poc_status [slot_count]int - frame_num_status [slot_count]int - reference_usage []u8 - next_ref u8 - next_slot u8 - current_slot u8 -} - -// Choose a slot which is not referenced by the picture being decoded. When all -// slots are references, expire the oldest short-term reference first (the H.264 -// sliding-window default). Explicit MMCO and long-term references are handled -// separately as stream metadata becomes available. -fn (mut dpb DPB) acquire_decode_slot(slot_limit int) u8 { - assert slot_limit > 0 && slot_limit <= slot_count - for slot in 0 .. slot_limit { - if u8(slot) !in dpb.reference_usage { - return u8(slot) - } - } - assert dpb.reference_usage.len > 0 - expired_slot := dpb.reference_usage[0] - dpb.reference_usage.delete(0) - return expired_slot -} - -pub struct DPBResourceState { -pub mut: - flag vk.AccessFlags2 - layout vk.ImageLayout -} - -pub struct OutputImage { -pub mut: - display_order int = -1 - texture Image - duration_ns i64 - layout vk.ImageLayout - is_new bool = true -} - -struct RetiredOutputImage { - index int - reusable_after_serial u64 -} - -pub struct Image { -pub mut: - image vk.Image - view vk.ImageView - allocation vkmem.Allocator - allocation_info vkmem.AllocationInfo -} - -pub struct DecoderQueryProperties { -pub mut: - decode_h264_caps vk.VideoDecodeH264CapabilitiesKHR - decode_caps vk.VideoDecodeCapabilitiesKHR - caps vk.VideoCapabilitiesKHR - format_props vk.VideoFormatPropertiesKHR - dpb_format_props vk.VideoFormatPropertiesKHR - dpb_and_output_coincide bool - usage_dpb vk.ImageUsageFlags -} - -pub struct DecoderSettings { -pub mut: - decode_h264_profile_info vk.VideoDecodeH264ProfileInfoKHR - profile_info vk.VideoProfileInfoKHR - profile_list_info vk.VideoProfileListInfoKHR -} - -pub enum DecoderFrameType as u8 { - e_unknown = 0 - e_intra - e_predictive -} - -pub struct DecoderVideoDataFrameInfo { -pub mut: - src_offset u64 - frame_bytes_num u64 - size u64 - poc int - bottom_field_order_cnt u32 - top_field_order_cnt u32 - gop int - display_order int - decode_time_ns i64 - display_time_ns i64 - duration_ns i64 - nal_unit_type u8 - frame_type DecoderFrameType - nal_ref_idc u32 - reference_priority u32 -} - -fn compare_frame_display_order(a &DecoderVideoDataFrameInfo, b &DecoderVideoDataFrameInfo) int { - key_a := u64(a.gop) << 32 | u64(a.poc) - key_b := u64(b.gop) << 32 | u64(b.poc) - if key_a < key_b { - return -1 - } - if key_a > key_b { - return 1 - } - return 0 -} - -// presentation_buffer_size returns the number of decoded pictures which may -// have to wait while an earlier display-order picture is still being decoded. -// One additional slot is reserved for that missing picture itself. -fn presentation_buffer_size(display_orders []int) int { - mut waiting := map[int]bool{} - mut next_display_order := 0 - mut max_waiting := 0 - for display_order in display_orders { - waiting[display_order] = true - for waiting[next_display_order] { - waiting.delete(next_display_order) - next_display_order++ - } - max_waiting = math.max(max_waiting, waiting.len) - } - return max_waiting + 1 -} - -pub struct DecoderVideoFileProperties { -pub mut: - file os.File - file_open bool - h264_profile_idc u32 - width_padd u32 - height_padd u32 - width u32 - height u32 - sps_count u32 - pps_count u32 - slice_header_count u32 - - frame_infos []DecoderVideoDataFrameInfo - max_memory_frame_size_bytes u64 - num_dpb_slots u32 - max_reference_pictures u32 - - sps_bytes []u8 - pps_bytes []u8 - slice_header_bytes []u8 - frame_display_order []u64 - - total_duration i64 - metadata VideoMetadata -} - -pub struct VideoMetadata { -pub mut: - coded_width u32 - coded_height u32 - display_width u32 - display_height u32 - sar_width u32 = 1 - sar_height u32 = 1 - rotation_degrees int - track_matrix [9]i32 - video_full_range bool - colour_description_present bool - colour_primaries u8 - transfer_function u8 - matrix_coefficients u8 -} - -fn rotation_from_track_matrix(matrix [9]i32) int { - a := matrix[0] - b := matrix[1] - c := matrix[3] - d := matrix[4] - if a < 0 && d < 0 { - return 180 - } - if b > 0 && c < 0 { - return -90 - } - if b < 0 && c > 0 { - return 90 - } - return 0 -} - -fn h264_profile_name(profile_idc u32) string { - return match profile_idc { - 66 { 'Baseline' } - 77 { 'Main' } - 100 { 'High' } - else { 'Unknown (${profile_idc})' } - } -} - -fn sample_aspect_ratio(aspect_ratio_idc u32, extended_width u32, extended_height u32) (u32, u32) { - if aspect_ratio_idc == 255 { - if extended_width > 0 && extended_height > 0 { - return extended_width, extended_height - } - return 1, 1 - } - return match aspect_ratio_idc { - 1 { u32(1), u32(1) } - 2 { u32(12), u32(11) } - 3 { u32(10), u32(11) } - 4 { u32(16), u32(11) } - 5 { u32(40), u32(33) } - 6 { u32(24), u32(11) } - 7 { u32(20), u32(11) } - 8 { u32(32), u32(11) } - 9 { u32(80), u32(33) } - 10 { u32(18), u32(11) } - 11 { u32(15), u32(11) } - 12 { u32(64), u32(33) } - 13 { u32(160), u32(99) } - 14 { u32(4), u32(3) } - 15 { u32(3), u32(2) } - 16 { u32(2), u32(1) } - else { u32(1), u32(1) } - } -} - -fn (mut metadata VideoMetadata) update_display_dimensions() { - mut width := u64(metadata.coded_width) * u64(metadata.sar_width) / u64(metadata.sar_height) - mut height := u64(metadata.coded_height) - if metadata.rotation_degrees in [90, -90] { - width, height = height, width - } - metadata.display_width = u32(width) - metadata.display_height = u32(height) -} - -pub struct DecoderDpbImage { -pub mut: - image vk.Image - view vk.ImageView - // TODO: Refactor Allocator to Decoder - allocator vkmem.Allocator - allocation_info vkmem.AllocationInfo -} - -pub struct DecoderVideoDecodeOperation { -pub mut: - flags u32 - stream_offset u64 - stream_size u64 - frame_type DecoderFrameType = DecoderFrameType.e_intra - reference_priority u32 - decoded_frame_index int - slice_header voidptr = unsafe { nil } - pps voidptr = unsafe { nil } - sps voidptr = unsafe { nil } - poc [2]int - current_dpb u32 - dpb_reference_count u32 - dpb_reference_slots &u8 = unsafe { nil } - dpb_poc &int = unsafe { nil } - dpb_frame_num &int = unsafe { nil } - dpb_slot_num u32 - p_dpbs vk.Image - p_dpb_views vk.ImageView - slice_count u32 - slice_offsets &u32 = unsafe { nil } -} - -pub enum DecoderVideoDecodeOperationFlags { - e_none = 0 - e_session_reset = 1 -} - -// TODO: May be worth to use interface types, but interfaces IApp containg sub interface IDeviceContext "error: `&video_decode_app.VideoDecodeApp` incorrectly implements field `device_context` of interface `examples.video_decode_app.video_player.IApp`, expected `video_player.IDeviceContext`, got `video_decode_app.DeviceContext`", no matter what's in the interface -pub fn (mut vp VideoPlayer) prepare(path string) ! { - // Do not propagate parser errors from inside the lock: cleanup must be able - // to reacquire it and close a partially opened input file. - mut parse_error := '' - lock vp.decoder { - vp.decoder.parse_mp4_data(path) or { parse_error = err.msg() } - } - if parse_error != '' { - return error(parse_error) - } -} - -pub fn (mut vp VideoPlayer) close_input() { - lock vp.decoder { - if vp.decoder.video_data.file_open { - vp.decoder.video_data.file.close() - vp.decoder.video_data.file_open = false - } - } -} - -pub fn (vp &VideoPlayer) h264_profile_idc() u32 { - rlock vp.decoder { - return vp.decoder.video_data.h264_profile_idc - } -} - -pub fn (vp &VideoPlayer) metadata() VideoMetadata { - rlock vp.decoder { - return vp.decoder.video_data.metadata - } -} - -pub fn (mut vp VideoPlayer) initialize(mut app VideoDecodeApp) { - vp.app = app - lock vp.decoder { - vp.decoder.initialize(mut app) - vp.video_frames = []VideoPlayerDecodeStreamFrame{len: vp.decoder.info.memory_frames.len} - for i, frame in vp.decoder.info.memory_frames { - vp.video_frames[i] = VideoPlayerDecodeStreamFrame{ - gpu_bitstream_capacity: frame.gpu_bitstream_capacity - gpu_bitstream_offset: frame.gpu_bitstream_offset - gpu_bitstream_size: frame.gpu_bitstream_size - gpu_bitstream_slice_mapped_memory_address: byteptr(frame.gpu_bitstream_slice_mapped_memory_address) - } - } - if vp.decoder.info.images_dpb.len > vp.dpb.image.len { - panic('Video stream requires too many DPB images') - } - for i, image in vp.decoder.info.images_dpb { - vp.dpb.image[i] = Image{ - image: image.image - view: image.view - allocation_info: image.allocation_info - } - } - vp.presentation_buffer_count = presentation_buffer_size(vp.decoder.video_data.frame_infos.map(it.display_order)) - } - - vk_device := app.device_context.vk_device - for i in 0 .. vp.decoder.info.memory_frames.len { - fence_ci := vk.FenceCreateInfo{ - flags: vk.FenceCreateFlags(vk.FenceCreateFlagBits.signaled) - } - res := vk.create_fence(vk_device, &fence_ci, unsafe { nil }, &vp.video_frames[i].in_flight_fence) - check_vk(res, 'Could not create video-frame fence ${i}') - } - mut command_pool_ci := vk.CommandPoolCreateInfo{ - flags: vk.CommandPoolCreateFlags(vk.CommandPoolCreateFlagBits.reset_command_buffer) - } - command_pool_ci.queueFamilyIndex = app.device_context.graphics_family - mut res := vk.create_command_pool(vk_device, &command_pool_ci, unsafe { nil }, &vp.graphics_command_pool) - check_vk(res, 'Could not create video-player graphics command pool') - command_pool_ci.queueFamilyIndex = app.device_context.get_decoder_queue_family_index() - res = vk.create_command_pool(vk_device, &command_pool_ci, unsafe { nil }, &vp.video_command_pool) - check_vk(res, 'Could not create video-decode command pool') - - vp.command_buffer_infos = []CommandBufferInfo{len: app.device_context.swapchain.image_views.len} - for mut info in vp.command_buffer_infos { - mut alloc_info := vk.CommandBufferAllocateInfo{ - level: .primary - commandBufferCount: 1 - } - alloc_info.commandPool = vp.graphics_command_pool - res = vk.allocate_command_buffers(vk_device, &alloc_info, &info.graphics_command_buffer) - check_vk(res, 'Could not allocate video-player graphics command buffer') - alloc_info.commandPool = vp.video_command_pool - res = vk.allocate_command_buffers(vk_device, &alloc_info, &info.video_command_buffer) - check_vk(res, 'Could not allocate video-decode command buffer') - semaphore_ci := vk.SemaphoreCreateInfo{} - res = vk.create_semaphore(vk_device, &semaphore_ci, unsafe { nil }, &info.sem_video_to_gfx) - check_vk(res, 'Could not create video-to-graphics semaphore') - } - event_ci := vk.EventCreateInfo{} - res = vk.create_event(vk_device, &event_ci, unsafe { nil }, &vp.event_video_player) - check_vk(res, 'Could not create video-player event') - output_texture_count := vp.presentation_buffer_count + vp.command_buffer_infos.len + 2 - if output_texture_count > max_texture_count { - panic('Video requires ${output_texture_count} presentation images, but the player supports at most ${max_texture_count}') - } - decode_family := app.device_context.get_decoder_queue_family_index() - if decode_family != app.device_context.graphics_family { - println('Display image queues: decode family ${decode_family}, graphics family ${app.device_context.graphics_family} (concurrent)') - } else { - println('Display image queue family: ${decode_family} (exclusive)') - } - for _ in 0 .. output_texture_count { - vp.create_output_image() - } - println('Presentation queue: ${vp.presentation_buffer_count} reorder images, ${output_texture_count} images total') - if !vp.decoder.properties.dpb_and_output_coincide { - vp.create_decode_output_image() - } -} - -pub fn (mut vp VideoPlayer) recreate_swapchain_resources() { - vk_device := vp.app.device_context.vk_device - for mut info in vp.command_buffer_infos { - if !isnil(info.graphics_command_buffer) { - vk.free_command_buffers(vk_device, vp.graphics_command_pool, 1, &info.graphics_command_buffer) - } - if !isnil(info.video_command_buffer) { - vk.free_command_buffers(vk_device, vp.video_command_pool, 1, &info.video_command_buffer) - } - if !isnil(info.sem_video_to_gfx) { - vk.destroy_semaphore(vk_device, info.sem_video_to_gfx, unsafe { nil }) - } - } - vp.command_buffer_infos = []CommandBufferInfo{len: vp.app.device_context.swapchain.image_views.len} - for mut info in vp.command_buffer_infos { - mut alloc_info := vk.CommandBufferAllocateInfo{ - level: .primary - commandBufferCount: 1 - commandPool: vp.graphics_command_pool - } - mut result := vk.allocate_command_buffers(vk_device, &alloc_info, &info.graphics_command_buffer) - check_vk(result, 'Could not reallocate video-player graphics command buffer') - alloc_info.commandPool = vp.video_command_pool - result = vk.allocate_command_buffers(vk_device, &alloc_info, &info.video_command_buffer) - check_vk(result, 'Could not reallocate video-decode command buffer') - result = vk.create_semaphore(vk_device, &vk.SemaphoreCreateInfo{}, unsafe { nil }, &info.sem_video_to_gfx) - check_vk(result, 'Could not recreate video-to-graphics semaphore') - } - // The caller waits for device idle before rebuilding the swapchain, so every - // retired presentation image can be reused immediately. Grow the pool if the - // replacement swapchain has more images than the old one. - for retired in vp.output_textures_retired { - vp.output_textures_free << retired.index - } - vp.output_textures_retired.clear() - required_output_count := vp.presentation_buffer_count + vp.command_buffer_infos.len + 2 - if required_output_count > max_texture_count { - panic('Resized swapchain requires ${required_output_count} presentation images, but the player supports at most ${max_texture_count}') - } - for vp.output_textures.len < required_output_count { - vp.create_output_image() - } -} - -pub fn (mut vp VideoPlayer) shutdown() { - vk_device := vp.app.device_context.vk_device - for mut info in vp.command_buffer_infos { - if !isnil(info.sem_video_to_gfx) { - vk.destroy_semaphore(vk_device, info.sem_video_to_gfx, unsafe { nil }) - info.sem_video_to_gfx = unsafe { nil } - } - } - vp.command_buffer_infos.clear() - if !isnil(vp.graphics_command_pool) { - vk.destroy_command_pool(vk_device, vp.graphics_command_pool, unsafe { nil }) - vp.graphics_command_pool = unsafe { nil } - } - if !isnil(vp.video_command_pool) { - vk.destroy_command_pool(vk_device, vp.video_command_pool, unsafe { nil }) - vp.video_command_pool = unsafe { nil } - } - if !isnil(vp.event_video_player) { - vk.destroy_event(vk_device, vp.event_video_player, unsafe { nil }) - vp.event_video_player = unsafe { nil } - } - for mut frame in vp.video_frames { - if !isnil(frame.in_flight_fence) { - vk.destroy_fence(vk_device, frame.in_flight_fence, unsafe { nil }) - frame.in_flight_fence = unsafe { nil } - } - } - lock vp.decoder { - for mut output in vp.output_textures { - if !isnil(output.texture.view) { - vk.destroy_image_view(vk_device, output.texture.view, unsafe { nil }) - output.texture.view = unsafe { nil } - } - if !isnil(output.texture.image) { - vk.destroy_image(vk_device, output.texture.image, unsafe { nil }) - output.texture.image = unsafe { nil } - } - _ = vp.app.device_context.memory_allocator.release(mut output.texture.allocation_info) - } - vp.output_textures.clear() - vp.output_textures_free.clear() - vp.output_textures_ready.clear() - vp.output_textures_retired.clear() - if !isnil(vp.decode_output_image.view) { - vk.destroy_image_view(vk_device, vp.decode_output_image.view, unsafe { nil }) - vp.decode_output_image.view = unsafe { nil } - } - if !isnil(vp.decode_output_image.image) { - vk.destroy_image(vk_device, vp.decode_output_image.image, unsafe { nil }) - vp.decode_output_image.image = unsafe { nil } - } - _ = vp.app.device_context.memory_allocator.release(mut vp.decode_output_image.allocation_info) - for mut dpb in vp.decoder.info.images_dpb { - if !isnil(dpb.view) { - vk.destroy_image_view(vk_device, dpb.view, unsafe { nil }) - dpb.view = unsafe { nil } - } - if !isnil(dpb.image) { - vk.destroy_image(vk_device, dpb.image, unsafe { nil }) - dpb.image = unsafe { nil } - } - _ = vp.app.device_context.memory_allocator.release(mut dpb.allocation_info) - } - if !isnil(vp.decoder.gpu_bitstream_buffer) { - vp.app.device_context.memory_allocator.unmap(mut vp.decoder.gpu_bitstream_allocation) - vk.destroy_buffer(vk_device, vp.decoder.gpu_bitstream_buffer, unsafe { nil }) - vp.decoder.gpu_bitstream_buffer = unsafe { nil } - } - _ = vp.app.device_context.memory_allocator.release(mut vp.decoder.gpu_bitstream_allocation) - if !isnil(vp.decoder.video_session_parameters) { - vk.destroy_video_session_parameters_khr(vk_device, vp.decoder.video_session_parameters, unsafe { nil }) - vp.decoder.video_session_parameters = unsafe { nil } - } - if !isnil(vp.decoder.video_session) { - vk.destroy_video_session_khr(vk_device, vp.decoder.video_session, unsafe { nil }) - vp.decoder.video_session = unsafe { nil } - } - for memory in vp.decoder.session_memory_allocations { - if !isnil(memory) { - vk.free_memory(vk_device, memory, unsafe { nil }) - } - } - vp.decoder.session_memory_allocations.clear() - } -} - -fn (mut vp VideoPlayer) create_output_image() { - mut dev_ctx := vp.app.device_context - mut output := OutputImage{} - queue_families := [dev_ctx.get_decoder_queue_family_index(), dev_ctx.graphics_family] - queues_differ := queue_families[0] != queue_families[1] - queue_family_data := if queues_differ { - queue_families.data - } else { - unsafe { &u32(nil) } - } - mut image_ci := vk.ImageCreateInfo{ - pNext: unsafe { nil } - flags: 0 - imageType: ._2d - format: vp.decoder.properties.format_props.format - extent: vk.Extent3D{ - width: vp.decoder.video_data.width - height: vp.decoder.video_data.height - depth: 1 - } - mipLevels: 1 - arrayLayers: 1 - samples: ._1 - tiling: .optimal - usage: vk.ImageUsageFlags(u32(vk.ImageUsageFlagBits.transfer_dst) | u32(vk.ImageUsageFlagBits.sampled)) - sharingMode: .exclusive - queueFamilyIndexCount: 0 - pQueueFamilyIndices: unsafe { nil } - initialLayout: .undefined - } - if queues_differ { - image_ci.sharingMode = .concurrent - image_ci.queueFamilyIndexCount = u32(queue_families.len) - image_ci.pQueueFamilyIndices = queue_family_data - } - mut res := vp.app.device_context.memory_allocator.create_image_with_options(&image_ci, vkmem.AllocationOptions{ - usage: .gpu_only - }, &output.texture.image, mut output.texture.allocation_info) - check_vk(res, 'Could not create sampled video output image') - mut conversion_info := vk.SamplerYcbcrConversionInfo{ - conversion: dev_ctx.sampler_ycbcr_conversion - } - view_ci := vk.ImageViewCreateInfo{ - pNext: &conversion_info - image: output.texture.image - viewType: ._2d - format: image_ci.format - subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - levelCount: 1 - layerCount: 1 - } - } - res = vk.create_image_view(dev_ctx.vk_device, &view_ci, unsafe { nil }, &output.texture.view) - check_vk(res, 'Could not create sampled video output image view') - vp.output_textures << output - vp.output_textures_free << vp.output_textures.len - 1 -} - -fn (mut vp VideoPlayer) create_decode_output_image() { - mut dev_ctx := vp.app.device_context - image_ci := vk.ImageCreateInfo{ - pNext: &vp.decoder.settings.profile_list_info - flags: 0 - imageType: ._2d - format: vp.decoder.properties.format_props.format - extent: vk.Extent3D{ - width: vp.decoder.video_data.width - height: vp.decoder.video_data.height - depth: 1 - } - mipLevels: 1 - arrayLayers: 1 - samples: ._1 - tiling: .optimal - usage: vk.ImageUsageFlags(u32(vk.ImageUsageFlagBits.video_decode_dst) | u32(vk.ImageUsageFlagBits.transfer_src)) - sharingMode: .exclusive - queueFamilyIndexCount: 0 - pQueueFamilyIndices: unsafe { nil } - initialLayout: .undefined - } - mut result := vp.app.device_context.memory_allocator.create_image_with_options(&image_ci, vkmem.AllocationOptions{ - usage: .gpu_only - }, &vp.decode_output_image.image, mut vp.decode_output_image.allocation_info) - check_vk(result, 'Could not create distinct video decode-output image') - view_ci := vk.ImageViewCreateInfo{ - image: vp.decode_output_image.image - viewType: ._2d - format: image_ci.format - subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - levelCount: 1 - layerCount: 1 - } - } - result = vk.create_image_view(dev_ctx.vk_device, &view_ci, unsafe { nil }, &vp.decode_output_image.view) - check_vk(result, 'Could not create distinct video decode-output image view') -} - -fn query_video_format(gpu vk.PhysicalDevice, profile_list &vk.VideoProfileListInfoKHR, - usage vk.ImageUsageFlags) ?vk.VideoFormatPropertiesKHR { - format_info := vk.PhysicalDeviceVideoFormatInfoKHR{ - pNext: unsafe { profile_list } - imageUsage: usage - } - mut count := u32(0) - mut no_formats := unsafe { nil } - mut result := vk.get_physical_device_video_format_properties_khr(gpu, &format_info, &count, mut no_formats) - if result != .success || count == 0 { - return none - } - mut formats := []vk.VideoFormatPropertiesKHR{len: int(count), init: vk.VideoFormatPropertiesKHR{}} - mut formats_data := formats.data - result = vk.get_physical_device_video_format_properties_khr(gpu, &format_info, &count, mut formats_data) - if result != .success || count == 0 { - return none - } - return formats[0] -} - -fn (mut d Decoder) initialize(mut app VideoDecodeApp) { - mut dev_ctx := app.device_context - d.properties.decode_h264_caps = vk.VideoDecodeH264CapabilitiesKHR{} - d.properties.decode_caps = vk.VideoDecodeCapabilitiesKHR{ - pNext: &d.properties.decode_h264_caps - } - d.properties.caps = vk.VideoCapabilitiesKHR{ - pNext: &d.properties.decode_caps - } - - d.settings.decode_h264_profile_info = vk.VideoDecodeH264ProfileInfoKHR{ - stdProfileIdc: unsafe { vk.StdVideoH264ProfileIdc(d.video_data.h264_profile_idc) } - pictureLayout: vk.VideoDecodeH264PictureLayoutFlagBitsKHR.progressive - } - d.settings.profile_info = vk.VideoProfileInfoKHR{ - pNext: &d.settings.decode_h264_profile_info - videoCodecOperation: vk.VideoCodecOperationFlagBitsKHR.decode_h264 - chromaSubsampling: vk.VideoChromaSubsamplingFlagsKHR(vk.VideoChromaSubsamplingFlagBitsKHR._420) - lumaBitDepth: vk.VideoComponentBitDepthFlagsKHR(vk.VideoComponentBitDepthFlagBitsKHR._8) - chromaBitDepth: vk.VideoComponentBitDepthFlagsKHR(vk.VideoComponentBitDepthFlagBitsKHR._8) - } - d.properties.caps.pNext = &d.properties.decode_caps - - mut res := vk.get_physical_device_video_capabilities_khr(dev_ctx.get_gpu_current(), &d.settings.profile_info, mut &d.properties.caps) - if res != vk.Result.success { - panic('Vulkan device does not expose H.264 ${h264_profile_name(d.video_data.h264_profile_idc)} Profile decode capabilities: ${res}') - } - - // Construct the complete wrapper so its required Vulkan sType default is - // applied. Mutating fields of the zero-initialized embedded value leaves - // sType at zero with some V compiler/code-generation paths; NVIDIA's driver - // may then fault when this chain is passed to vkCreateImage. - d.settings.profile_list_info = vk.VideoProfileListInfoKHR{ - profileCount: 1 - pProfiles: &d.settings.profile_info - } - - capability_flags := d.properties.decode_caps.flags - supports_coincide := (capability_flags & vk.VideoDecodeCapabilityFlagsKHR(vk.VideoDecodeCapabilityFlagBitsKHR.dpb_and_output_coincide)) != 0 - supports_distinct := (capability_flags & vk.VideoDecodeCapabilityFlagsKHR(vk.VideoDecodeCapabilityFlagBitsKHR.dpb_and_output_distinct)) != 0 - selected_output_mode := select_decode_output_mode(app.decode_output_mode, supports_coincide, supports_distinct) or { panic(err) } - d.properties.dpb_and_output_coincide = selected_output_mode == .coincident - println('Decode image mode: ${if d.properties.dpb_and_output_coincide { - 'coincident DPB/output' - } else { - 'distinct DPB/output' - }}') - output_usage := vk.ImageUsageFlags(u32(vk.ImageUsageFlagBits.video_decode_dst) | u32(vk.ImageUsageFlagBits.transfer_src)) - d.properties.format_props = query_video_format(dev_ctx.get_gpu_current(), &d.settings.profile_list_info, output_usage) or { - panic('No Vulkan Video decode-output format supports transfer to the display image') - } - dpb_usage := if d.properties.dpb_and_output_coincide { - vk.ImageUsageFlags(u32(vk.ImageUsageFlagBits.video_decode_dpb) | u32(vk.ImageUsageFlagBits.video_decode_dst) | u32(vk.ImageUsageFlagBits.transfer_src)) - } else { - vk.ImageUsageFlags(vk.ImageUsageFlagBits.video_decode_dpb) - } - d.properties.dpb_format_props = query_video_format(dev_ctx.get_gpu_current(), &d.settings.profile_list_info, dpb_usage) or { - panic('No Vulkan Video DPB format supports the required decode mode') - } - d.properties.usage_dpb = dpb_usage - - num_memory_frames := u64(d.video_data.num_dpb_slots) - mut aligned_frame_size := U64(d.video_data.max_memory_frame_size_bytes).align_to(d.properties.caps.minBitstreamBufferOffsetAlignment) - aligned_frame_size = U64(aligned_frame_size).align_to(d.properties.caps.minBitstreamBufferSizeAlignment) - d.video_data.max_memory_frame_size_bytes = aligned_frame_size - video_decoder_queue_family_index := dev_ctx.get_decoder_queue_family_index() - buffer_size := d.video_data.max_memory_frame_size_bytes * num_memory_frames - buffer_ci := vk.BufferCreateInfo{ - pNext: &d.settings.profile_list_info - flags: 0 - size: buffer_size - usage: vk.BufferUsageFlags(vk.BufferUsageFlagBits.video_decode_src) - sharingMode: vk.SharingMode.exclusive - queueFamilyIndexCount: 0 - pQueueFamilyIndices: unsafe { nil } - } - res = app.device_context.memory_allocator.create_buffer_with_options(&buffer_ci, vkmem.AllocationOptions{ - usage: .upload - }, &d.gpu_bitstream_buffer, mut d.gpu_bitstream_allocation) - if res != vk.Result.success { - panic('Could not create the Vulkan Video bitstream buffer: ${res}') - } - // Host visible - mut p_data := unsafe { nil } - // Use the same live allocator instance that created the allocation. dev_ctx - // was copied before create_buffer() updated the allocator's block table. - res = app.device_context.memory_allocator.map(mut d.gpu_bitstream_allocation, &p_data) - if res != vk.Result.success { - panic('Could not map the Vulkan Video bitstream buffer: ${res}') - } - if d.video_data.num_dpb_slots > d.properties.caps.maxDpbSlots { - panic('Video requires ${d.video_data.num_dpb_slots} DPB slots, but this device supports ${d.properties.caps.maxDpbSlots}') - } - d.video_data.max_reference_pictures = d.video_data.num_dpb_slots - 1 - if d.video_data.max_reference_pictures > d.properties.caps.maxActiveReferencePictures { - panic('Video requires ${d.video_data.max_reference_pictures} active references, but this device supports ${d.properties.caps.maxActiveReferencePictures}') - } - - session_ci := vk.VideoSessionCreateInfoKHR{ - queueFamilyIndex: video_decoder_queue_family_index - pVideoProfile: &d.settings.profile_info - pictureFormat: d.properties.format_props.format - maxCodedExtent: vk.Extent2D{ - width: math.min(d.video_data.width, d.properties.caps.maxCodedExtent.width) - height: math.min(d.video_data.height, d.properties.caps.maxCodedExtent.height) - } - referencePictureFormat: d.properties.dpb_format_props.format - maxDpbSlots: d.video_data.num_dpb_slots - maxActiveReferencePictures: d.video_data.max_reference_pictures - pStdHeaderVersion: &d.properties.caps.stdHeaderVersion - } - res = vk.create_video_session_khr(dev_ctx.vk_device, &session_ci, unsafe { nil }, &d.video_session) - if res != vk.Result.success { - panic('Could not create the Vulkan Video H.264 decode session: ${res}') - } - - mut requirement_count := u32(0) - mut n := unsafe { nil } - vk.get_video_session_memory_requirements_khr(dev_ctx.vk_device, d.video_session, &requirement_count, mut n) - mut requirements := []vk.VideoSessionMemoryRequirementsKHR{len: int(requirement_count), init: vk.VideoSessionMemoryRequirementsKHR{}} - - mut requirements_data := requirements.data - res = vk.get_video_session_memory_requirements_khr(dev_ctx.vk_device, d.video_session, &requirement_count, mut requirements_data) - if res != vk.Result.success { - panic('Could not query Vulkan Video session memory requirements: ${res}') - } - d.session_memory_allocations.ensure_cap(int(requirement_count)) - // V's length-only array initialization zeroes C structs and therefore loses - // the Vulkan binding's required sType default. Construct every element. - mut bind_session_memory_infos := []vk.BindVideoSessionMemoryInfoKHR{len: int(requirement_count), init: vk.BindVideoSessionMemoryInfoKHR{}} - for i in 0 .. requirement_count { - req := requirements[i] - // Video-session memory is opaque driver storage. The requirement's - // memoryTypeBits is authoritative; some drivers expose a dedicated type - // without VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT for this binding. - memory_type_index := dev_ctx.memory_allocator.get_memory_type(req.memoryRequirements.memoryTypeBits, vk.MemoryPropertyFlags(0)) - if memory_type_index == max_u32 { - panic('No compatible Vulkan memory type exists for video-session binding ${req.memoryBindIndex}') - } - allocate_info := vk.MemoryAllocateInfo{ - allocationSize: req.memoryRequirements.size - memoryTypeIndex: memory_type_index - } - mut session_memory := vk.DeviceMemory(unsafe { nil }) - res = vk.allocate_memory(dev_ctx.vk_device, &allocate_info, unsafe { nil }, &session_memory) - if res != vk.Result.success { - panic('Could not allocate ${req.memoryRequirements.size} bytes for Vulkan Video session binding ${req.memoryBindIndex}: ${res}') - } - d.session_memory_allocations << session_memory - - bind_session_memory_infos[i] = vk.BindVideoSessionMemoryInfoKHR{ - memoryBindIndex: req.memoryBindIndex - memory: session_memory - memoryOffset: 0 - memorySize: req.memoryRequirements.size - } - } - - // Resolve this extension command from the device directly. When another - // shared object links libvulkan, ELF symbol interposition can otherwise - // make Volk's same-named global dispatch slot unreliable. - bind_session_memory_fn := vk.PFN_vkBindVideoSessionMemoryKHR(vk.get_device_proc_addr(dev_ctx.vk_device, c'vkBindVideoSessionMemoryKHR')) - if isnil(voidptr(bind_session_memory_fn)) { - panic('vkGetDeviceProcAddr returned null for vkBindVideoSessionMemoryKHR') - } - res = bind_session_memory_fn(dev_ctx.vk_device, d.video_session, u32(bind_session_memory_infos.len), bind_session_memory_infos.data) - if res != vk.Result.success { - panic('Could not bind Vulkan Video session memory: ${res}') - } - - d.create_video_session_parameters(dev_ctx.vk_device) - - $if debug { - if (d.properties.decode_caps.flags & vk.VideoDecodeCapabilityFlagsKHR(vk.VideoDecodeCapabilityFlagBitsKHR.dpb_and_output_coincide)) > 0 { - println('NOTE: video decode: dpb and output coincide') - } else { - println('NOTE: video decode: dpb and output do NOT coincide') - } - if (d.properties.decode_caps.flags & vk.VideoDecodeCapabilityFlagsKHR(vk.VideoDecodeCapabilityFlagBitsKHR.dpb_and_output_distinct)) > 0 { - println('NOTE: video decode: dpb and output distinct') - } else { - println('NOTE: video decode: dpb and output NOT distinct') - } - } - - d.prepare_decoded_picture_buffer(dev_ctx.vk_device, mut app.device_context.memory_allocator) - - d.info.memory_frames = []DecoderVideoMemoryFrameInfo{len: int(num_memory_frames), init: DecoderVideoMemoryFrameInfo{ - data_frame_info: unsafe { nil } - gpu_bitstream_slice_mapped_memory_address: unsafe { nil } - decoding_frame_index: -1 - }} - mut i := u64(0) - for mut frame in d.info.memory_frames { - frame.gpu_bitstream_capacity = d.video_data.max_memory_frame_size_bytes - frame.gpu_bitstream_offset = i * d.video_data.max_memory_frame_size_bytes - frame.gpu_bitstream_size = 0 - frame.gpu_bitstream_slice_mapped_memory_address = unsafe { - byteptr(p_data) + frame.gpu_bitstream_offset - } - i++ - } - // Runtime sample uploads use absolute MP4 offsets. Do not perform an - // unrelated seek here that can turn an otherwise completed initialization - // into a fatal media-I/O panic. -} - -pub fn (mut d Decoder) prepare_decoded_picture_buffer(device vk.Device, mut allocator vkmem.Allocator) { - // Allocate an image array to store decoded pictures in - - // num_dpb_slots already includes one slot for the picture currently decoded. - // - // we know there will be at max 17 images (16+1) as 16 is the max by the standard. - // - // Decoded-picture-buffer images should remain in device-local memory. - dpb_image_count := int(d.video_data.num_dpb_slots) - d.info.images_dpb = []DecoderDpbImage{len: dpb_image_count} - - image_ci := vk.ImageCreateInfo{ - pNext: &d.settings.profile_list_info - flags: 0 - imageType: vk.ImageType._2d - format: d.properties.dpb_format_props.format - extent: vk.Extent3D{ - width: d.video_data.width - height: d.video_data.height - depth: 1 - } - mipLevels: 1 - arrayLayers: 1 - samples: vk.SampleCountFlagBits._1 - tiling: vk.ImageTiling.optimal - usage: d.properties.usage_dpb - sharingMode: vk.SharingMode.exclusive - queueFamilyIndexCount: 0 - pQueueFamilyIndices: unsafe { nil } - initialLayout: vk.ImageLayout.undefined - } - - mut image_view_ci := vk.ImageViewCreateInfo{ - flags: 0 - image: unsafe { nil } - viewType: vk.ImageViewType._2d - format: image_ci.format - components: vk.ComponentMapping{} - subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - baseMipLevel: 0 - levelCount: 1 - baseArrayLayer: 0 - layerCount: 1 - } - } - - mut dpb_index := 0 - mut res := vk.Result.error_unknown - for mut dpb in d.info.images_dpb { - res = allocator.create_image_with_options(&image_ci, vkmem.AllocationOptions{ - usage: .gpu_only - }, &dpb.image, mut dpb.allocation_info) - if res != vk.Result.success { - panic('Could not create decoded-picture-buffer image ${dpb_index}: ${res}') - } - image_view_ci.image = dpb.image - res = vk.create_image_view(device, &image_view_ci, unsafe { nil }, &dpb.view) - if res != vk.Result.success { - panic('Could not create decoded-picture-buffer image view ${dpb_index}: ${res}') - } - // VK_EXT_debug_utils is enabled only for debug builds. Calling its - // device command unconditionally jumps through a null dispatch slot in - // release packages. - $if debug ? { - name := 'myDPBImage${dpb_index}' - name_info := vk.DebugUtilsObjectNameInfoEXT{ - objectType: vk.ObjectType.image - objectHandle: u64(voidptr(dpb.image)) - pObjectName: name.str // C string - } - vk.set_debug_utils_object_name_ext(device, &name_info) - } - dpb_index++ - } // for d.info.images_dpb -} - -pub fn (mut d Decoder) create_video_session_parameters(device vk.Device) { - mut video_picture_parameter_sets := []vk.StdVideoH264PictureParameterSet{len: int(d.video_data.pps_count)} - mut video_scaling_list_pps := []vk.StdVideoH264ScalingLists{len: int(d.video_data.pps_count)} - for i in 0 .. int(d.video_data.pps_count) { - pps_offset := i * int(sizeof(h264.PictureParameterSet)) - pps := unsafe { &h264.PictureParameterSet(&d.video_data.pps_bytes[pps_offset]) } - video_scaling_list_pps[i] = vk.StdVideoH264ScalingLists{} - for j in 0 .. pps.pic_scaling_list_present_flag.len { - video_scaling_list_pps[i].scaling_list_present_mask |= u16(pps.pic_scaling_list_present_flag[j]) << j - } - for j in 0 .. pps.use_default_scaling_matrix_4x4_flag.len { - video_scaling_list_pps[i].use_default_scaling_matrix_mask |= u16(pps.use_default_scaling_matrix_4x4_flag[j]) << j - } - mut list_idx := 0 - mut el_idx := 0 - for list_idx < vk.std_video_h264_scaling_list_4x4_num_lists && list_idx < pps.scaling_list_4x4.len { - for el_idx < vk.std_video_h264_scaling_list_4x4_num_elements && el_idx < pps.scaling_list_4x4[0].len { - unsafe { - C.vv_set_h264_scaling_list_4x4(&video_scaling_list_pps[i], u32(list_idx), u32(el_idx), u8(pps.scaling_list_4x4[list_idx][el_idx])) - } - el_idx++ - } - list_idx++ - } - list_idx = 0 - el_idx = 0 - for list_idx < vk.std_video_h264_scaling_list_8x8_num_lists && list_idx < pps.scaling_list_8x8.len { - for el_idx < vk.std_video_h264_scaling_list_8x8_num_elements && el_idx < pps.scaling_list_8x8[0].len { - unsafe { - C.vv_set_h264_scaling_list_8x8(&video_scaling_list_pps[i], u32(list_idx), u32(el_idx), u8(pps.scaling_list_8x8[list_idx][el_idx])) - } - el_idx++ - } - list_idx++ - } - - video_picture_parameter_sets[i] = vk.StdVideoH264PictureParameterSet{ - flags: vk.StdVideoH264PpsFlags{ - transform_8x8_mode_flag: u32(pps.transform_8x8_mode_flag) - redundant_pic_cnt_present_flag: pps.redundant_pic_cnt_present_flag - constrained_intra_pred_flag: pps.constrained_intra_pred_flag - deblocking_filter_control_present_flag: pps.deblocking_filter_control_present_flag - weighted_pred_flag: pps.weighted_pred_flag - bottom_field_pic_order_in_frame_present_flag: pps.pic_order_present_flag - entropy_coding_mode_flag: pps.entropy_coding_mode_flag - pic_scaling_matrix_present_flag: pps.pic_scaling_matrix_present_flag - } - seq_parameter_set_id: u8(pps.seq_parameter_set_id) - pic_parameter_set_id: u8(pps.pic_parameter_set_id) - num_ref_idx_l0_default_active_minus1: u8(pps.num_ref_idx_l0_active_minus1) - num_ref_idx_l1_default_active_minus1: u8(pps.num_ref_idx_l1_active_minus1) - weighted_bipred_idc: unsafe { vk.StdVideoH264WeightedBipredIdc(pps.weighted_bipred_idc) } - pic_init_qp_minus26: i8(pps.pic_init_qp_minus26) - pic_init_qs_minus26: i8(pps.pic_init_qs_minus26) - chroma_qp_index_offset: i8(pps.chroma_qp_index_offset) - second_chroma_qp_index_offset: i8(pps.second_chroma_qp_index_offset) - pScalingLists: unsafe { &video_scaling_list_pps[i] } - } - } // for d.video_data.pps_count - - mut video_sequence_parameter_set := []vk.StdVideoH264SequenceParameterSet{len: int(d.video_data.sps_count)} - mut video_sequence_parameter_set_vui := []vk.StdVideoH264SequenceParameterSetVui{len: int(d.video_data.sps_count)} - mut video_scaling_list_sps := []vk.StdVideoH264ScalingLists{len: int(d.video_data.sps_count)} - mut video_hrd_parameters := []vk.StdVideoH264HrdParameters{len: int(d.video_data.sps_count)} - get_chroma_format := fn (profile u32, chroma u32) vk.StdVideoH264ChromaFormatIdc { - if profile < unsafe { int(vk.StdVideoH264ProfileIdc.high) } { - // If profile is less than HIGH chroma format will not be explicitly given. (A.2) - // If chroma format is not present, it shall be inferred to be equal to 1 (4:2:0) (7.4.2.1.1) - return vk.StdVideoH264ChromaFormatIdc._420 - } else { - // If profile is greater than HIGH, then we assume chroma to be explicitly specified - return unsafe { vk.StdVideoH264ChromaFormatIdc(chroma) } - } - } - - for i in 0 .. int(d.video_data.sps_count) { - sps_offset := i * int(sizeof(h264.SequenceParameterSet)) - sps := unsafe { &h264.SequenceParameterSet(&d.video_data.sps_bytes[sps_offset]) } - - video_sequence_parameter_set[i] = vk.StdVideoH264SequenceParameterSet{ - flags: vk.StdVideoH264SpsFlags{ - constraint_set0_flag: sps.constraint_set0_flag - constraint_set1_flag: sps.constraint_set1_flag - constraint_set2_flag: sps.constraint_set2_flag - constraint_set3_flag: sps.constraint_set3_flag - constraint_set4_flag: sps.constraint_set4_flag - constraint_set5_flag: sps.constraint_set5_flag - direct_8x8_inference_flag: sps.direct_8x8_inference_flag - mb_adaptive_frame_field_flag: sps.mb_adaptive_frame_field_flag - frame_mbs_only_flag: sps.frame_mbs_only_flag - delta_pic_order_always_zero_flag: sps.delta_pic_order_always_zero_flag - separate_colour_plane_flag: sps.separate_colour_plane_flag - gaps_in_frame_num_value_allowed_flag: sps.gaps_in_frame_num_value_allowed_flag - qpprime_y_zero_transform_bypass_flag: sps.qpprime_y_zero_transform_bypass_flag - frame_cropping_flag: sps.frame_cropping_flag - seq_scaling_matrix_present_flag: sps.seq_scaling_matrix_present_flag - vui_parameters_present_flag: sps.vui_parameters_present_flag - } - // Note: There is no 0 in StdVideoH264ProfileIdc enum - profile_idc: unsafe { vk.StdVideoH264ProfileIdc(sps.profile_idc) } - level_idc: unsafe { vk.StdVideoH264LevelIdc(sps.level_idc) } - chroma_format_idc: get_chroma_format(sps.profile_idc, sps.chroma_format_idc) - seq_parameter_set_id: u8(sps.seq_parameter_set_id) - bit_depth_luma_minus8: u8(sps.bit_depth_luma_minus8) - bit_depth_chroma_minus8: u8(sps.bit_depth_chroma_minus8) - log2_max_frame_num_minus4: u8(sps.log2_max_frame_num_minus4) - pic_order_cnt_type: unsafe { vk.StdVideoH264PocType(sps.pic_order_cnt_type) } - offset_for_non_ref_pic: sps.offset_for_non_ref_pic - log2_max_pic_order_cnt_lsb_minus4: u8(sps.log2_max_pic_order_cnt_lsb_minus4) - num_ref_frames_in_pic_order_cnt_cycle: u8(sps.num_ref_frames_in_pic_order_cnt_cycle) - max_num_ref_frames: u8(sps.num_ref_frames) - reserved1: 0 - pic_width_in_mbs_minus1: sps.pic_width_in_mbs_minus1 - pic_height_in_map_units_minus1: sps.pic_height_in_map_units_minus1 - frame_crop_left_offset: sps.frame_crop_left_offset - frame_crop_right_offset: sps.frame_crop_right_offset - frame_crop_top_offset: sps.frame_crop_top_offset - frame_crop_bottom_offset: sps.frame_crop_bottom_offset - reserved2: 0 - pOffsetForRefFrame: unsafe { nil } - pScalingLists: unsafe { &video_scaling_list_sps[i] } - pSequenceParameterSetVui: unsafe { &video_sequence_parameter_set_vui[i] } - } - - // VUI stands for "Video Usability Information" - vui := &sps.vui - - video_sequence_parameter_set_vui[i] = vk.StdVideoH264SequenceParameterSetVui{ - flags: vk.StdVideoH264SpsVuiFlags{ - aspect_ratio_info_present_flag: vui.aspect_ratio_info_present_flag - overscan_info_present_flag: vui.overscan_info_present_flag - overscan_appropriate_flag: vui.overscan_appropriate_flag - video_signal_type_present_flag: vui.video_signal_type_present_flag - video_full_range_flag: vui.video_full_range_flag - color_description_present_flag: vui.color_description_present_flag - chroma_loc_info_present_flag: vui.chroma_loc_info_present_flag - timing_info_present_flag: vui.timing_info_present_flag - fixed_frame_rate_flag: vui.fixed_frame_rate_flag - bitstream_restriction_flag: vui.bitstream_restriction_flag - nal_hrd_parameters_present_flag: vui.nal_hrd_parameters_present_flag - vcl_hrd_parameters_present_flag: vui.vcl_hrd_parameters_present_flag - } - aspect_ratio_idc: unsafe { vk.StdVideoH264AspectRatioIdc(vui.aspect_ratio_idc) } - sar_width: u16(vui.sar_width) - sar_height: u16(vui.sar_height) - video_format: u8(vui.video_format) - colour_primaries: u8(vui.colour_primaries) - transfer_characteristics: u8(vui.transfer_characteristics) - matrix_coefficients: u8(vui.matrix_coefficients) - num_units_in_tick: vui.num_units_in_tick - time_scale: vui.time_scale - max_num_reorder_frames: u8(vui.num_reorder_frames) - max_dec_frame_buffering: u8(vui.max_dec_frame_buffering) - chroma_sample_loc_type_top_field: u8(vui.chroma_sample_loc_type_top_field) - chroma_sample_loc_type_bottom_field: u8(vui.chroma_sample_loc_type_bottom_field) - reserved1: 0 - pHrdParameters: unsafe { &video_hrd_parameters[i] } - } - - hrd := &sps.hrd - video_hrd_parameters[i] = vk.StdVideoH264HrdParameters{ - cpb_cnt_minus1: u8(hrd.cpb_cnt_minus1) - bit_rate_scale: u8(hrd.bit_rate_scale) - cpb_size_scale: u8(hrd.cpb_size_scale) - // reserved1: u8(0) - // bit_rate_value_minus1: [u32(0)] - // cpb_size_value_minus1: [u32(0)] - // cbr_flag: [u8(0)] - initial_cpb_removal_delay_length_minus1: hrd.initial_cpb_removal_delay_length_minus1 - cpb_removal_delay_length_minus1: hrd.cpb_removal_delay_length_minus1 - dpb_output_delay_length_minus1: hrd.dpb_output_delay_length_minus1 - time_offset_length: hrd.time_offset_length - } - - for j in 0 .. vk.std_video_h264_cpb_cnt_list_size { - video_hrd_parameters[i].bit_rate_value_minus1[j] = hrd.bit_rate_value_minus1[j] - video_hrd_parameters[i].cpb_size_value_minus1[j] = hrd.cpb_size_value_minus1[j] - video_hrd_parameters[i].cbr_flag[j] = u8(hrd.cbr_flag[j]) - } - - // Fill scaling lists - video_scaling_list_sps[i] = vk.StdVideoH264ScalingLists{} - for j in 0 .. sps.seq_scaling_list_present_flag.len { - video_scaling_list_sps[i].scaling_list_present_mask |= u16(sps.seq_scaling_list_present_flag[j]) << j - } - for j in 0 .. sps.use_default_scaling_matrix_4x4_flag.len { - video_scaling_list_sps[i].use_default_scaling_matrix_mask |= u16(sps.use_default_scaling_matrix_4x4_flag[j]) << j - } - - mut list_idx := 0 - mut el_idx := 0 - for list_idx < vk.std_video_h264_scaling_list_4x4_num_lists && list_idx < sps.scaling_list_4x4.len { - for el_idx < vk.std_video_h264_scaling_list_4x4_num_elements && el_idx < sps.scaling_list_4x4[0].len { - unsafe { - C.vv_set_h264_scaling_list_4x4(&video_scaling_list_sps[i], u32(list_idx), u32(el_idx), u8(sps.scaling_list_4x4[list_idx][el_idx])) - } - el_idx++ - } - list_idx++ - } - - list_idx = 0 - el_idx = 0 - for list_idx < vk.std_video_h264_scaling_list_8x8_num_lists && list_idx < sps.scaling_list_8x8.len { - for el_idx < vk.std_video_h264_scaling_list_8x8_num_elements && el_idx < sps.scaling_list_8x8[0].len { - unsafe { - C.vv_set_h264_scaling_list_8x8(&video_scaling_list_sps[i], u32(list_idx), u32(el_idx), u8(sps.scaling_list_8x8[list_idx][el_idx])) - } - el_idx++ - } - list_idx++ - } - } // for d.video_data.sps_count - - mut session_parameters_add_info := vk.VideoDecodeH264SessionParametersAddInfoKHR{ - stdSPSCount: d.video_data.sps_count - pStdSPSs: video_sequence_parameter_set.data - stdPPSCount: d.video_data.pps_count - pStdPPSs: video_picture_parameter_sets.data - } - mut video_decode_session_parameters_ci := vk.VideoDecodeH264SessionParametersCreateInfoKHR{ - maxStdSPSCount: d.video_data.sps_count - maxStdPPSCount: d.video_data.pps_count - pParametersAddInfo: &session_parameters_add_info - } - video_session_parameters_ci := vk.VideoSessionParametersCreateInfoKHR{ - pNext: &video_decode_session_parameters_ci - flags: 0 - videoSessionParametersTemplate: unsafe { nil } - videoSession: d.video_session - } - - res := vk.create_video_session_parameters_khr(device, &video_session_parameters_ci, unsafe { nil }, &d.video_session_parameters) - if res != vk.Result.success { - panic('Could not create H.264 video-session parameters: ${res}') - } -} - -pub struct VKUGPUImage { -pub mut: - image vk.Image = unsafe { nil } - view vk.ImageView = unsafe { nil } - allocation vkmem.Allocator - allocation_info vkmem.AllocationInfo -} - -@[heap] -pub struct CallbackUserData { -pub mut: - file &os.File - last_offset i64 - file_size u64 - read_error string -} - -pub fn read_callback(offset i64, buffer &u8, size usize, user_data voidptr) int { - mut data := unsafe { &CallbackUserData(user_data) } - if offset < 0 || offset >= data.file_size { - return 1 - } - to_copy := math.min(i64(size), data.file_size - offset) - data.file.seek(offset, .start) or { - data.read_error = 'could not seek to MP4 offset ${offset}: ${err}' - return 1 - } - num_bytes_read := data.file.read_into_ptr(buffer, int(to_copy)) or { - data.read_error = 'could not read ${to_copy} bytes at MP4 offset ${offset}: ${err}' - return 1 - } - data.last_offset = offset + num_bytes_read - if num_bytes_read != int(to_copy) { - data.read_error = 'short read at MP4 offset ${offset}: expected ${to_copy} bytes, read ${num_bytes_read}' - return 1 - } - return 0 -} - -// Turn an EBSP (Encapsulated Byte Sequence Payload) into an RBSP (Raw Byte Sequence Payload) -// pub fn remove_emulation_prevention_bytes(ebsp []u8) []u8 { -// mut rbsp := []u8{cap: ebsp.len} -// if ebsp.len == 0 { return rbsp } -// mut i := 0 -// for i < ebsp.len { -// if ((i + 2) < ebsp.len) -// && ebsp[i] == 0 -// && ebsp[i + 1] == 0 -// && ebsp[i + 2] == 3 { -// rbsp << ebsp[i] -// rbsp << ebsp[i + 1] -// i += 2 -// } else { -// rbsp << ebsp[i] -// } -// i++ -// } -// return rbsp -// } -@[unsafe] -pub fn remove_emulation_prevention_bytes(ebsp byteptr, size int) []u8 { - mut rbsp := []u8{cap: size} - if size == 0 { - return rbsp - } - mut i := 0 - for i < size { - if (i + 2) < size && unsafe { ebsp[i] == 0 && ebsp[i + 1] == 0 && ebsp[i + 2] == 3 } { - rbsp << unsafe { ebsp[i] } - rbsp << unsafe { ebsp[i + 1] } - i += 2 - } else { - rbsp << unsafe { ebsp[i] } - } - i++ - } - return rbsp -} - -pub type BytePtr = byteptr - -pub fn (p BytePtr) to_varray[T](len u32) []T { - if isnil(p) && len > 0 { - panic('Nil to_varray for len: ${len}') - } - if len <= 0 { - return []T{} - } - // TODO: doesn't trigger on to_varray[u8] - $if T is u8 { - return p.vbytes(int(len)) - } - mut ret := []T{cap: int(len)} - // vmemcpy(ret.data, p, len * sizeof(T)) - for i in 0 .. len { - ret << *unsafe { &T(p + (i * sizeof(T))) } - } - return ret -} - -pub fn (mut d Decoder) parse_mp4_data(file_path string) ! { - d.video_data.file = os.open_file(file_path, 'rb')! - d.video_data.file_open = true - d.video_data.file.seek(0, .end)! - mp4_file_size := d.video_data.file.tell()! - d.video_data.file.seek(0, .start)! - mut mp4 := minimp4.MP4D_demux_t{} - mut user_data := CallbackUserData{ - file: &d.video_data.file - last_offset: 0 - file_size: os.file_size(file_path) - } - if minimp4.mp4d_open(&mp4, read_callback, &user_data, mp4_file_size) != 1 { - if user_data.read_error != '' { - return error(user_data.read_error) - } - return error('not a readable MP4 file') - } - defer { - minimp4.mp4d_close(&mp4) - } - if mp4.track_count == 0 || isnil(mp4.track) { - return error('MP4 contains no tracks') - } - mut ntrack := u32(0) - mut found_video_track := false - mut found_unsupported_video_track := false - for track_index in u32(0) .. mp4.track_count { - candidate := unsafe { mp4.track[track_index] } - if candidate.handler_type != minimp4.mp4d_handler_type_vide { - continue - } - if candidate.object_type_indication == minimp4.mp4_object_type_avc { - ntrack = track_index - found_video_track = true - break - } - found_unsupported_video_track = true - } - if !found_video_track { - if found_unsupported_video_track { - return error('MP4 video codec is unsupported; this player currently supports H.264/AVC') - } - return error('MP4 contains no H.264/AVC video track') - } - track := unsafe { mp4.track[ntrack] } - if track.sample_count == 0 { - return error('H.264 video track contains no samples') - } - if track.timescale == 0 { - return error('H.264 video track has an invalid zero timescale') - } - d.video_data.metadata.track_matrix = track.track_matrix - d.video_data.metadata.rotation_degrees = rotation_from_track_matrix(track.track_matrix) - - // Read SPS - mut num_bytes_sps := 0 - mut count_sps := 0 - mut sps_array := []h264.SequenceParameterSet{} - mut data_sps := minimp4.mp4d_read_sps(&mp4, ntrack, count_sps, &num_bytes_sps) - for !isnil(data_sps) { - mut nal := h264.NetworkAbstractionLayerHeader{} - mut nal_header_bs := h264.Bitstream{} - nal_header_bs.init(unsafe { data_sps.vbytes(1) }) - nal.read_nal_header(mut &nal_header_bs) - if num_bytes_sps <= 1 { - return error('H.264 track contains an invalid sequence parameter set') - } - mut nal_payload_rbsp_data := unsafe { - remove_emulation_prevention_bytes(byteptr(data_sps) + 1, num_bytes_sps - 1) - } - mut nal_payload_bs := h264.Bitstream{} - nal_payload_bs.init(nal_payload_rbsp_data) - mut sps := h264.SequenceParameterSet{} - sps.read_sps(mut nal_payload_bs) - if sps.profile_idc !in [u32(66), 77, 100] { - return error('H.264 profile_idc ${sps.profile_idc} is unsupported; supported profiles are Baseline, Main, and High 8-bit 4:2:0') - } - if d.video_data.h264_profile_idc == 0 { - d.video_data.h264_profile_idc = sps.profile_idc - } else if d.video_data.h264_profile_idc != sps.profile_idc { - return error('H.264 track changes profile between sequence parameter sets') - } - if sps.bit_depth_luma_minus8 != 0 || sps.bit_depth_chroma_minus8 != 0 { - return error('H.264 bit depths above 8-bit are not supported') - } - if sps.profile_idc >= 100 && sps.chroma_format_idc != 1 { - return error('H.264 chroma formats other than 4:2:0 are not supported') - } - if sps.frame_mbs_only_flag == 0 { - return error('interlaced H.264 video is not currently supported') - } - // Data validation - // https://stackoverflow.com/questions/6394874/fetching-the-dimensions-of-a-h264video-stream - width := ((sps.pic_width_in_mbs_minus1 + 1) * 16) - (sps.frame_crop_left_offset * 2) - (sps.frame_crop_right_offset * 2) - height := ((2 - sps.frame_mbs_only_flag) * (sps.pic_height_in_map_units_minus1 + 1) * 16) - (sps.frame_crop_top_offset * 2) - (sps.frame_crop_bottom_offset * 2) - mp4_width := unsafe { track.sampleDescription.video.width } - mp4_height := unsafe { track.sampleDescription.video.height } - if mp4_width != width || mp4_height != height { - eprintln('Warning: MP4 dimensions ${mp4_width}x${mp4_height} differ from H.264 SPS display dimensions ${width}x${height}') - } - d.video_data.width_padd = (sps.pic_width_in_mbs_minus1 + 1) * 16 - d.video_data.height_padd = (sps.pic_height_in_map_units_minus1 + 1) * 16 - if sps.vui_parameters_present_flag != 0 { - d.video_data.metadata.sar_width, d.video_data.metadata.sar_height = sample_aspect_ratio(sps.vui.aspect_ratio_idc, sps.vui.sar_width, sps.vui.sar_height) - d.video_data.metadata.video_full_range = sps.vui.video_full_range_flag != 0 - d.video_data.metadata.colour_description_present = d.video_data.metadata.colour_description_present - || sps.vui.color_description_present_flag != 0 - d.video_data.metadata.colour_primaries = u8(sps.vui.colour_primaries) - d.video_data.metadata.transfer_function = u8(sps.vui.transfer_characteristics) - d.video_data.metadata.matrix_coefficients = u8(sps.vui.matrix_coefficients) - } - // x ^ ((x ^ y) & -(x < y)) // max(x, y) - // d.video_data.num_dpb_slots = d.video_data.num_dpb_slots ^ ((d.video_data.num_dpb_slots ^ (sps.num_ref_frames * 2 + 1)) & -u32(d.video_data.num_dpb_slots < (sps.num_ref_frames * 2 + 1))) - d.video_data.num_dpb_slots = math.max[u32](d.video_data.num_dpb_slots, sps.num_ref_frames + 1) - d.video_data.sps_bytes << unsafe { byteptr(&sps).vbytes(int(sizeof(sps))) } - sps_array << sps - d.video_data.sps_count++ - count_sps++ - - data_sps = minimp4.mp4d_read_sps(&mp4, ntrack, count_sps, &num_bytes_sps) - } - if sps_array.len == 0 { - return error('H.264 video track contains no sequence parameter set (SPS)') - } - - // Read PPS - mut size_pps := 0 - mut count_pps := 0 - mut pps_array := []h264.PictureParameterSet{} - mut data_pps := minimp4.mp4d_read_pps(&mp4, ntrack, count_pps, &size_pps) - - for !isnil(data_pps) { - mut nal := h264.NetworkAbstractionLayerHeader{} - mut nal_header_bs := h264.Bitstream{} - nal_header_bs.init(unsafe { data_pps.vbytes(1) }) - nal.read_nal_header(mut nal_header_bs) - if size_pps <= 1 { - return error('H.264 track contains an invalid picture parameter set') - } - pps_payload_rbsp_data := unsafe { - remove_emulation_prevention_bytes(byteptr(data_pps) + 1, size_pps - 1) - } - mut pps_payload_bs := h264.Bitstream{} - pps_payload_bs.init(pps_payload_rbsp_data) - - mut pps := h264.PictureParameterSet{} - pps.read_pps(mut pps_payload_bs) - d.video_data.pps_bytes.ensure_cap(d.video_data.pps_bytes.len + int(sizeof(pps))) - d.video_data.pps_bytes << unsafe { byteptr(&pps).vbytes(int(sizeof(pps))) } - pps_array << pps - d.video_data.pps_count++ - count_pps++ - - data_pps = minimp4.mp4d_read_pps(&mp4, ntrack, count_pps, &size_pps) - } - if pps_array.len == 0 { - return error('H.264 video track contains no picture parameter set (PPS)') - } - - d.video_data.width = unsafe { track.sampleDescription.video.width } - d.video_data.height = unsafe { track.sampleDescription.video.height } - d.video_data.metadata.coded_width = d.video_data.width - d.video_data.metadata.coded_height = d.video_data.height - d.video_data.metadata.update_display_dimensions() - println('Video codec: H.264 ${h264_profile_name(d.video_data.h264_profile_idc)} Profile') - println('Display metadata: coded ${d.video_data.metadata.coded_width}x${d.video_data.metadata.coded_height}, display ${d.video_data.metadata.display_width}x${d.video_data.metadata.display_height}, SAR ${d.video_data.metadata.sar_width}:${d.video_data.metadata.sar_height}, rotation ${d.video_data.metadata.rotation_degrees}°') - - timescale_rcp := 1.0 / f64(track.timescale) - mut prev_pic_order_cnt_lsb := u32(0) - mut prev_pic_order_cnt_msb := u32(0) - mut poc_cycle := -1 - mut prev_frame_num := u32(0) - mut prev_frame_offset := u32(0) - - // Read frames - mut track_duration := u32(0) - mut max_frame_size_bytes := u64(0) - mut input_file_position := u64(0) - d.video_data.file.seek(0, .start)! - - d.video_data.frame_infos = []DecoderVideoDataFrameInfo{cap: int(track.sample_count)} - d.video_data.slice_header_bytes.ensure_cap(int(track.sample_count * sizeof(h264.SliceHeader))) - d.video_data.slice_header_count = track.sample_count - - mut file := d.video_data.file - mut sample_index := u32(0) - for { - if sample_index >= track.sample_count { - break - } - mut frame_bytes_num_to_do := u32(0) - mut duration := u32(0) - mut timestamp := u32(0) - // minimp4 returns timestamp before duration. These were previously passed - // in reverse order, causing later frames to use their growing timestamp as - // a duration and making playback progressively slower. - offset := minimp4.mp4d_frame_offset(&mp4, ntrack, sample_index, &frame_bytes_num_to_do, ×tamp, &duration) - // The upload buffer must fit any slice contained in the complete MP4 - // sample, including samples with leading non-slice NAL units. - max_frame_size_bytes = math.max[u64](max_frame_size_bytes, frame_bytes_num_to_do) - track_duration += duration - - mut data_frame := DecoderVideoDataFrameInfo{ - src_offset: offset - frame_bytes_num: frame_bytes_num_to_do - } - - mut src_buffer := []u8{len: int(frame_bytes_num_to_do)} - mut src_buffer_idx := 0 - if offset - input_file_position > 0 { - file.seek(offset - input_file_position, .current) or { - return error('could not seek to MP4 sample ${sample_index}: ${err}') - } - } - if file.eof() { - return error('MP4 sample ${sample_index} points beyond the end of the file') - } - expected_frame_bytes := frame_bytes_num_to_do - frame_bytes_num_to_do = u32(file.read(mut src_buffer) or { - return error('could not read MP4 sample ${sample_index}: ${err}') - }) - if frame_bytes_num_to_do != expected_frame_bytes { - return error('MP4 sample ${sample_index} is truncated: expected ${expected_frame_bytes} bytes, read ${frame_bytes_num_to_do}') - } - input_file_position = offset + frame_bytes_num_to_do - for frame_bytes_num_to_do > 0 { - if frame_bytes_num_to_do < 4 { - return error('MP4 sample ${sample_index} has a truncated H.264 NAL length') - } - // mut size := unsafe{ (u32(*&src_buffer[src_buffer_idx+0]) << 24) | (u32(*&src_buffer[src_buffer_idx+1]) << 16) | (u32(*&src_buffer[src_buffer_idx+2]) << 8) | *&src_buffer[src_buffer_idx+3] } - mut size := unsafe { - (u32(src_buffer[src_buffer_idx + 0]) << 24) | (u32(src_buffer[src_buffer_idx + 1]) << 16) | (u32(src_buffer[src_buffer_idx + 2]) << 8) | src_buffer[src_buffer_idx + 3] - } - size += 4 - if size < 4 || frame_bytes_num_to_do < size { - return error('MP4 sample ${sample_index} has an invalid H.264 NAL size ${size - 4}') - } - - length_prefixed_data_offset := src_buffer_idx + 4 - length_prefixed_data_size := size - 4 - if length_prefixed_data_size <= 1 { - return error('MP4 sample ${sample_index} contains an empty H.264 NAL unit') - } - - mut nal := h264.NetworkAbstractionLayerHeader{} - mut nal_header_bs := h264.Bitstream{} - nal_header_bs.init(src_buffer[length_prefixed_data_offset..length_prefixed_data_offset + 1]) - nal.read_nal_header(mut nal_header_bs) - - slfrom := length_prefixed_data_offset + 1 - nal_payload_rbsp_data := unsafe { - remove_emulation_prevention_bytes(byteptr(src_buffer.data) + slfrom, int(length_prefixed_data_size - 1)) - } - mut nal_payload_bs := h264.Bitstream{} - nal_payload_bs.init(nal_payload_rbsp_data) - - mut is_idr := false - match nal.type { - .coded_slice_idr { - data_frame.frame_type = DecoderFrameType.e_intra - is_idr = true - } - .coded_slice_non_idr { - data_frame.frame_type = DecoderFrameType.e_predictive - } - else { - frame_bytes_num_to_do -= size - src_buffer_idx += int(size) - continue - } - } - - /* - * Decode Picture Order Count - * (tig) see ITU-T H.264 (08/2021) pp.113 - * - */ - // tig: see Rec. ITU-T H.264 (08/2021) p.66 (7-1) - mut slice_header := h264.SliceHeader{} - slice_header.read_slice_header(&nal, pps_array, sps_array, mut nal_payload_bs) - if slice_header.pic_parameter_set_id >= u32(pps_array.len) { - return error('MP4 sample ${sample_index} references missing H.264 PPS ${slice_header.pic_parameter_set_id}') - } - pps := pps_array[slice_header.pic_parameter_set_id] - if pps.seq_parameter_set_id >= u32(sps_array.len) { - return error('MP4 sample ${sample_index} references missing H.264 SPS ${pps.seq_parameter_set_id}') - } - sps := sps_array[pps.seq_parameter_set_id] - - max_frame_num := u32(1) << (sps.log2_max_frame_num_minus4 + 4) - max_pic_order_cnt_lsb := u32(1) << (sps.log2_max_pic_order_cnt_lsb_minus4 + 4) - pic_order_cnt_lsb := u32(slice_header.pic_order_cnt_lsb) - mut pic_order_cnt_msb := u32(0) - mut frame_num_offset := u32(0) - mut tmp_pic_order_cout := u32(0) - - match sps.pic_order_cnt_type { - 0 { - // TYPE 0 - // Rec. ITU-T H.264 (08/2021) page 114 - // Use the NAL unit type, not idr flag - if is_idr { - prev_pic_order_cnt_msb = 0 - prev_pic_order_cnt_lsb = 0 - poc_cycle++ - } - if pic_order_cnt_lsb < prev_pic_order_cnt_lsb && (prev_pic_order_cnt_lsb - pic_order_cnt_lsb) >= max_pic_order_cnt_lsb / 2 { - pic_order_cnt_msb = prev_pic_order_cnt_msb + max_pic_order_cnt_lsb - } else if pic_order_cnt_lsb > prev_pic_order_cnt_lsb && (pic_order_cnt_lsb - prev_pic_order_cnt_lsb) > max_pic_order_cnt_lsb / 2 { - pic_order_cnt_msb = prev_pic_order_cnt_msb - max_pic_order_cnt_lsb - } else { - pic_order_cnt_msb = prev_pic_order_cnt_msb - } - // Top and bottom field order count in case the picture is a field - if slice_header.field_pic_flag == 0 || slice_header.bottom_field_flag == 0 { - data_frame.top_field_order_cnt = pic_order_cnt_msb + pic_order_cnt_lsb - } - if slice_header.field_pic_flag == 0 { - data_frame.bottom_field_order_cnt = data_frame.top_field_order_cnt + u32(slice_header.delta_pic_order_cnt_bottom) - } else if slice_header.bottom_field_flag != 0 { - data_frame.bottom_field_order_cnt = pic_order_cnt_msb + slice_header.pic_order_cnt_lsb - } - - // Same as top field order count - data_frame.poc = int(pic_order_cnt_msb + pic_order_cnt_lsb) - data_frame.gop = poc_cycle - - // TODO: memory_management_control_operation equal to 5 - if nal.idc != h264.NAL_REF_IDC.priority_disposable { - prev_pic_order_cnt_msb = pic_order_cnt_msb - prev_pic_order_cnt_lsb = pic_order_cnt_lsb - } - } - - // match 0 - 2 { - if is_idr { - frame_num_offset = 0 - } else if prev_frame_num > slice_header.frame_num { - frame_num_offset = prev_frame_offset + max_frame_num - } else { - frame_num_offset = prev_frame_offset - } - prev_frame_offset = frame_num_offset - prev_frame_num = slice_header.frame_num - - if is_idr { - tmp_pic_order_cout = 0 - } else if nal.idc == h264.NAL_REF_IDC.priority_disposable { - tmp_pic_order_cout = 2 * (frame_num_offset + slice_header.frame_num) - 1 - } else { - tmp_pic_order_cout = 2 * (frame_num_offset + slice_header.frame_num) - } - - // (tig) Ignore bottom or top fields, as we assume progressive. - // If it were otherwise - for interleaved - either the top or the bottom - // field shall be set, depending on whether the current picture is the - // top or bottom field, as indicated by bottom_field_flag - data_frame.poc = int(tmp_pic_order_cout) - if tmp_pic_order_cout == 0 { - poc_cycle++ - } - data_frame.gop = poc_cycle - } - - // match 2 - else { - return error('H.264 picture-order-count type ${sps.pic_order_cnt_type} is not supported') - } - } // match - - // Accept frame beginning NAL unit - data_frame.nal_ref_idc = u32(nal.idc) - data_frame.nal_unit_type = u8(nal.type) - // TODO: h264.nal_start_code as pub const. error, imported types must start with a capital letter, but const can't be upper case - // data_frame.size = sizeof(h264.nal_start_code) + size - 4 - nal_start_code := h264.NalStartCode{}.value - data_frame.size = u64(nal_start_code.len) + size - 4 - data_frame.reference_priority = u32(nal.idc) - - data_frame.decode_time_ns = i64(f64(timestamp) * timescale_rcp * 1_000_000_000.0) - data_frame.display_time_ns = i64(f64(timestamp) * timescale_rcp * 1_000_000_000.0) - data_frame.duration_ns = math.max[i64](1, i64(f64(duration) * timescale_rcp * 1_000_000_000.0)) - d.video_data.slice_header_bytes << unsafe { - byteptr(&slice_header).vbytes(int(sizeof(slice_header))) - } - - // for frame_bytes_num_to_do > 0 - break - } - - // x ^ ((x ^ y) & -(x < y)) // max(x, y) - // max_frame_size_bytes = max_frame_size_bytes ^ ((max_frame_size_bytes ^ data_frame.size) & -(u64(max_frame_size_bytes < data_frame.size))) - max_frame_size_bytes = math.max[u64](max_frame_size_bytes, data_frame.size) - - d.video_data.frame_infos << data_frame - sample_index++ - } // for sample_index < track.sample_count - - // Fills array values with their index 0..len - d.video_data.frame_display_order = []u64{len: d.video_data.frame_infos.len, init: index} - - // Keep frame_infos in MP4 decode order. Only the display-order index list is - // sorted; slice_header_bytes and current_frame use the original decode order. - for i in 1 .. d.video_data.frame_display_order.len { - index_to_insert := d.video_data.frame_display_order[i] - frame_to_insert := d.video_data.frame_infos[index_to_insert] - mut j := i - for j > 0 { - previous_index := d.video_data.frame_display_order[j - 1] - if compare_frame_display_order(&d.video_data.frame_infos[previous_index], &frame_to_insert) <= 0 { - break - } - d.video_data.frame_display_order[j] = previous_index - j-- - } - d.video_data.frame_display_order[j] = index_to_insert - } - - for i in 0 .. d.video_data.frame_display_order.len { - d.video_data.frame_infos[d.video_data.frame_display_order[i]].display_order = i - } - - d.video_data.max_memory_frame_size_bytes = max_frame_size_bytes - d.video_data.total_duration = i64(f64(track_duration) * timescale_rcp * 1_000_000_000.0) -} - -pub type U64 = u64 - -pub fn (sz U64) align_to(alignment usize) u64 { - return ((sz - 1) / alignment + 1) * alignment -} - -fn (vp &VideoPlayer) ready_output_position(display_order int) ?int { - for position, output_index in vp.output_textures_ready { - if vp.output_textures[output_index].display_order == display_order { - return position - } - } - return none -} - -fn (mut vp VideoPlayer) reclaim_output_textures() { - mut position := 0 - for position < vp.output_textures_retired.len { - retired := vp.output_textures_retired[position] - if retired.reusable_after_serial > vp.render_serial { - position++ - continue - } - vp.output_textures[retired.index].display_order = -1 - vp.output_textures[retired.index].duration_ns = 0 - vp.output_textures_free << retired.index - vp.output_textures_retired.delete(position) - } -} - -fn (mut vp VideoPlayer) retire_current_output() { - if vp.current_output_index < 0 { - return - } - // The application has one fence per swapchain image. Once this many later - // render submissions have begun, the last submission sampling this image is - // guaranteed to have completed on the ordered graphics queue. - vp.output_textures_retired << RetiredOutputImage{ - index: vp.current_output_index - reusable_after_serial: vp.render_serial + u64(vp.command_buffer_infos.len + 1) - } -} - -fn (mut vp VideoPlayer) present_ready_output(display_order int) bool { - position := vp.ready_output_position(display_order) or { return false } - output_index := vp.output_textures_ready[position] - vp.output_textures_ready.delete(position) - vp.retire_current_output() - vp.current_output_index = output_index - vp.next_display_order = display_order + 1 - vp.waiting_for_loop_start = false - vp.playback_timeline.present_frame(vp.output_textures[output_index].duration_ns) - $if debug { - if display_order % 100 == 0 { - eprintln('Presented frame ${display_order}') - } - } - return true -} - -fn (mut vp VideoPlayer) restart_decode_cycle() { - $if debug { - println('Playback loop completed in display order') - } - vp.current_frame = 0 - vp.next_display_order = 0 - vp.decode_finished = false - vp.waiting_for_loop_start = true - vp.dpb.reference_usage.clear() - vp.flags |= u32(VideoPlayerFlags.e_decoder_reset) - vp.playback_timeline.reset() -} - -fn (mut vp VideoPlayer) update_presentation() { - if vp.current_output_index < 0 || vp.waiting_for_loop_start { - // Prime enough codec-order pictures to absorb the stream's maximum - // reordering delay. No-B-frame streams have a target of one and still - // display their first decoded picture immediately. - if !vp.decode_finished && vp.output_textures_ready.len < vp.presentation_buffer_count { - return - } - vp.present_ready_output(vp.next_display_order) - return - } - if !vp.playback_timeline.frame_is_due() { - return - } - frame_count := vp.decoder.video_data.frame_infos.len - if vp.next_display_order < frame_count { - vp.present_ready_output(vp.next_display_order) - return - } - if !vp.decode_finished { - return - } - if vp.is_looping { - // Keep the last picture visible while the first pictures of the next loop - // are decoded and reordered. - vp.restart_decode_cycle() - } else { - vp.is_stopped = true - } -} - -pub fn (vp &VideoPlayer) current_output_view() vk.ImageView { - if vp.current_output_index < 0 || vp.current_output_index >= vp.output_textures.len { - return unsafe { nil } - } - return vp.output_textures[vp.current_output_index].texture.view -} - -pub fn (mut vp VideoPlayer) update(graphics_cmd_buffer vk.CommandBuffer, time_elapsed_ns i64) { - vp.decode_operation = DecoderVideoDecodeOperation{} - vp.render_serial++ - $if debug { - if vp.render_serial <= 5 { - eprintln('Playback update ${vp.render_serial}: ready=${vp.output_textures_ready.len}, free=${vp.output_textures_free.len}') - } - } - vp.reclaim_output_textures() - if vp.current_output_index >= 0 && !vp.waiting_for_loop_start { - vp.playback_timeline.tick(time_elapsed_ns) - } - - if vp.is_stopped { - vp.dpb_slot_used = []int{len: int(vp.decoder.video_data.max_reference_pictures), init: 0} - return - } - should_decode := !vp.decode_finished - && vp.output_textures_ready.len < vp.presentation_buffer_count - && vp.output_textures_free.len > 0 - if !should_decode { - vp.update_presentation() - return - } - - vp.update_decode_video() or { - eprintln('Playback stopped: ${err}') - vp.is_stopped = true - return - } - $if debug { - if vp.render_serial <= 5 { - eprintln('Decoded access unit ${vp.current_frame}: ready=${vp.output_textures_ready.len}') - } - } - vp.update_presentation() - - vk.cmd_wait_events(graphics_cmd_buffer, 1, &vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands), vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands), 0, unsafe { nil }, 0, unsafe { nil }, 0, unsafe { nil }) - vk.cmd_reset_event(graphics_cmd_buffer, vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.bottom_of_pipe)) - - // Finish recording the command buffer and submit - dev_ctx := vp.app.device_context - index_cur_swapchain := dev_ctx.swapchain.get_current_index() - command_buffer_info := vp.command_buffer_infos[index_cur_swapchain] - vk.end_command_buffer(command_buffer_info.graphics_command_buffer) - - mut semaphore := command_buffer_info.sem_video_to_gfx - mut wait_stage := vk.PipelineStageFlags(vk.PipelineStageFlagBits.top_of_pipe) - mut sumbit_info_video := vk.SubmitInfo{ - waitSemaphoreCount: 0 - pWaitSemaphores: unsafe { nil } - pWaitDstStageMask: &wait_stage - commandBufferCount: 1 - pCommandBuffers: &command_buffer_info.video_command_buffer - signalSemaphoreCount: 1 - pSignalSemaphores: &semaphore - } - upload_fence := vp.video_frames[vp.current_upload_index].in_flight_fence - res_video := vk.queue_submit(dev_ctx.get_queue(.video_decode), 1, &sumbit_info_video, upload_fence) - check_vk(res_video, 'Could not submit Vulkan Video decode command') - - mut sumbit_info_graphics := vk.SubmitInfo{ - waitSemaphoreCount: 1 - pWaitSemaphores: &semaphore - pWaitDstStageMask: &wait_stage - commandBufferCount: 1 - pCommandBuffers: &command_buffer_info.graphics_command_buffer - } - res_graphics := vk.queue_submit(dev_ctx.get_queue(.graphics), 1, &sumbit_info_graphics, unsafe { nil }) - check_vk(res_graphics, 'Could not submit decoded frame for graphics use') -} - -pub fn (mut vp VideoPlayer) update_decode_video() ! { - dev_ctx := vp.app.device_context - if vp.output_textures_free.len == 0 { - return error('presentation queue has no reusable output image') - } - output_index := vp.output_textures_free[0] - vp.output_textures_free.delete(0) - mut output_queued := false - defer { - if !output_queued { - vp.output_textures_free << output_index - } - } - command_buffer_info := vp.command_buffer_infos[dev_ctx.swapchain.get_current_index()] - mut begin_command_buffer := vk.CommandBufferBeginInfo{} - vk.reset_command_buffer(command_buffer_info.graphics_command_buffer, 0) - vk.begin_command_buffer(command_buffer_info.graphics_command_buffer, &begin_command_buffer) - if vp.is_stopped { - vk.cmd_set_event(command_buffer_info.graphics_command_buffer, vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands)) - return - } - - video_command_buffer := command_buffer_info.video_command_buffer - vk.reset_command_buffer(video_command_buffer, 0) - vk.begin_command_buffer(video_command_buffer, &begin_command_buffer) - - mut frame_info := vp.decoder.video_data.frame_infos[vp.current_frame] - mut slice_header := &h264.SliceHeader(unsafe { nil }) - mut pps := &h264.PictureParameterSet(unsafe { nil }) - mut sps := &h264.SequenceParameterSet(unsafe { nil }) - lock vp.decoder { - assert !isnil(vp.decoder.get_slice_header()) - assert !isnil(vp.decoder.get_pps()) - assert !isnil(vp.decoder.get_sps()) - - slice_header = unsafe { - &h264.SliceHeader(byteptr(usize(vp.decoder.get_slice_header()) + usize(vp.current_frame) * sizeof(h264.SliceHeader))) - } - pps = unsafe { - &h264.PictureParameterSet(byteptr(usize(vp.decoder.get_pps()) + usize(slice_header.pic_parameter_set_id) * sizeof(h264.PictureParameterSet))) - } - sps = unsafe { - &h264.SequenceParameterSet(byteptr(usize(vp.decoder.get_sps()) + usize(pps.seq_parameter_set_id) * sizeof(h264.SequenceParameterSet))) - } - } - - mut decode_ope := DecoderVideoDecodeOperation{} - if vp.current_frame == 0 || has_flag[VideoPlayerFlags](vp.flags, VideoPlayerFlags.e_decoder_reset) { - decode_ope.flags = u32(DecoderVideoDecodeOperationFlags.e_session_reset) - vp.flags &= ~u32(VideoPlayerFlags.e_decoder_reset) - } - - // An ordinary I-picture can occur in an open GOP and may still depend on the - // existing DPB for pictures decoded after it. Only IDR starts a new reference - // picture sequence here. - if frame_info.nal_unit_type == u8(h264.NAL_UNIT_TYPE.coded_slice_idr) { - vp.dpb.reference_usage.clear() - } - - dpb_slot_num := int(vp.decoder.video_data.num_dpb_slots) - vp.dpb.current_slot = vp.dpb.acquire_decode_slot(dpb_slot_num) - vp.dpb.poc_status[vp.dpb.current_slot] = int(frame_info.poc) - vp.dpb.frame_num_status[vp.dpb.current_slot] = int(slice_header.frame_num) - - // Index variable on initialization comes in handy - dpbs := []vk.Image{len: dpb_slot_num, init: vp.dpb.image[index].image} - dpb_views := []vk.ImageView{len: dpb_slot_num, init: vp.dpb.image[index].view} - - // Only the prefix initialized from decoder.info.memory_frames owns mapped - // bitstream-buffer slices. Using the fixed array length eventually selects an - // uninitialized zero-capacity entry on streams longer than the DPB count. - use_frame_index := vp.current_frame % vp.decoder.info.memory_frames.len - vp.current_upload_index = use_frame_index - upload_fence := vp.video_frames[use_frame_index].in_flight_fence - res_wait := vk.wait_for_fences(dev_ctx.vk_device, 1, &upload_fence, vk._true, max_u64) - check_vk(res_wait, 'Could not wait for decoded-frame fence') - res_reset := vk.reset_fences(dev_ctx.vk_device, 1, &upload_fence) - check_vk(res_reset, 'Could not reset decoded-frame fence') - vp.video_frames[use_frame_index].gpu_bitstream_size = 0 - vp.video_frames[use_frame_index].slice_offsets.clear() - mut use_frame := &vp.video_frames[use_frame_index] - vp.write_video_frame(mut use_frame)! - if use_frame.gpu_bitstream_size == 0 { - // MP4 samples may contain only metadata/non-slice NAL units. They do not - // form a Vulkan decode operation and must not be submitted with range 0. - vk.end_command_buffer(video_command_buffer) - if vp.current_frame + 1 < vp.decoder.video_data.frame_infos.len { - vp.current_frame++ - } else { - vp.decode_finished = true - } - vk.cmd_set_event(command_buffer_info.graphics_command_buffer, vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands)) - return - } - mut flush_result := vk.Result.error_unknown - lock vp.decoder { - flush_result = vp.app.device_context.memory_allocator.flush_range(vp.decoder.gpu_bitstream_allocation, use_frame.gpu_bitstream_offset, use_frame.gpu_bitstream_size) - } - check_vk(flush_result, 'Could not flush the Vulkan Video bitstream buffer') - - decode_ope.stream_offset = use_frame.gpu_bitstream_offset - decode_ope.stream_size = use_frame.gpu_bitstream_size - decode_ope.poc[0] = frame_info.poc - decode_ope.poc[1] = frame_info.poc - decode_ope.frame_type = frame_info.frame_type - decode_ope.reference_priority = frame_info.reference_priority - decode_ope.decoded_frame_index = vp.current_frame - decode_ope.slice_header = slice_header - decode_ope.pps = pps - decode_ope.sps = sps - decode_ope.current_dpb = vp.dpb.current_slot - decode_ope.dpb_reference_count = u32(vp.dpb.reference_usage.len) - decode_ope.dpb_reference_slots = vp.dpb.reference_usage.data - // Pointer to data of fixed size array - decode_ope.dpb_poc = &vp.dpb.poc_status[0] - decode_ope.dpb_frame_num = &vp.dpb.frame_num_status[0] - - decode_ope.dpb_slot_num = u32(dpb_slot_num) - decode_ope.p_dpbs = dpbs.data - decode_ope.p_dpb_views = dpb_views.data - decode_ope.slice_count = u32(use_frame.slice_offsets.len) - decode_ope.slice_offsets = use_frame.slice_offsets.data - - // Copy for display - vp.decode_operation = decode_ope - - vp.video_decode_pre_barrier(video_command_buffer) - vp.video_decode_core(&decode_ope, video_command_buffer) - vp.copy_decoded_frame_to_output(video_command_buffer, output_index) - vp.output_textures[output_index].display_order = frame_info.display_order - vp.output_textures[output_index].duration_ns = frame_info.duration_ns - vp.output_textures_ready << output_index - output_queued = true - - if frame_info.reference_priority > 0 { - vp.dpb.reference_usage << vp.dpb.current_slot - for vp.dpb.reference_usage.len > int(vp.decoder.video_data.max_reference_pictures) { - vp.dpb.reference_usage.delete(0) - } - } - vk.end_command_buffer(video_command_buffer) - mut frame_count := 0 - rlock vp.decoder { - frame_count = vp.decoder.video_data.frame_infos.len - } - if vp.current_frame + 1 < frame_count { - vp.current_frame++ - } else { - // B-frame output may still be waiting in presentation order. Do not reset - // the codec or overwrite those images until the last picture was shown. - vp.decode_finished = true - } - - // The graphics command buffer waits on the decode semaphore before this - // transition, making the copied output safe for fragment sampling. - mut output_barrier := vk.ImageMemoryBarrier2{ - srcStageMask: vk.pipeline_stage_2_transfer_bit - srcAccessMask: vk.access_2_transfer_write_bit - dstStageMask: vk.pipeline_stage_2_fragment_shader_bit - dstAccessMask: vk.access_2_shader_read_bit - oldLayout: .transfer_dst_optimal - newLayout: .shader_read_only_optimal - image: vp.output_textures[output_index].texture.image - subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - levelCount: 1 - layerCount: 1 - } - } - output_dependency := vk.DependencyInfo{ - imageMemoryBarrierCount: 1 - pImageMemoryBarriers: &output_barrier - } - vk.cmd_pipeline_barrier2(command_buffer_info.graphics_command_buffer, &output_dependency) - vp.output_textures[output_index].layout = .shader_read_only_optimal - - // Signal the application command buffer after the decode queue completes. - vk.cmd_set_event(command_buffer_info.graphics_command_buffer, vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands)) -} - -fn (mut vp VideoPlayer) copy_decoded_frame_to_output(command_buffer vk.CommandBuffer, output_index int) { - decode_family := vp.app.device_context.get_decoder_queue_family_index() - mut output := &vp.output_textures[output_index] - coincide := vp.decoder.properties.dpb_and_output_coincide - mut source_image := vp.dpb.image[vp.dpb.current_slot].image - mut source_state := &vp.dpb.resource_state[vp.dpb.current_slot] - mut source_restore_layout := vk.ImageLayout.video_decode_dpb_khr - if !coincide { - source_image = vp.decode_output_image.image - source_state = &vp.decode_output_state - source_restore_layout = .video_decode_dst_khr - } - mut barriers := [ - vk.ImageMemoryBarrier2{ - srcStageMask: vk.pipeline_stage_2_video_decode_bit_khr - srcAccessMask: source_state.flag - dstStageMask: vk.pipeline_stage_2_transfer_bit - dstAccessMask: vk.access_2_transfer_read_bit - oldLayout: source_state.layout - newLayout: .transfer_src_optimal - srcQueueFamilyIndex: decode_family - dstQueueFamilyIndex: decode_family - image: source_image - subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - levelCount: 1 - layerCount: 1 - } - }, - vk.ImageMemoryBarrier2{ - srcStageMask: vk.pipeline_stage_2_transfer_bit - dstStageMask: vk.pipeline_stage_2_transfer_bit - dstAccessMask: vk.access_2_transfer_write_bit - oldLayout: if output.is_new { - vk.ImageLayout.undefined - } else { - output.layout - } - newLayout: .transfer_dst_optimal - srcQueueFamilyIndex: decode_family - dstQueueFamilyIndex: decode_family - image: output.texture.image - subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - levelCount: 1 - layerCount: 1 - } - }, - ] - dependency := vk.DependencyInfo{ - imageMemoryBarrierCount: u32(barriers.len) - pImageMemoryBarriers: barriers.data - } - vk.cmd_pipeline_barrier2(command_buffer, &dependency) - source_state.layout = .transfer_src_optimal - source_state.flag = vk.access_2_transfer_read_bit - - mut regions := [ - vk.ImageCopy2{ - srcSubresource: vk.ImageSubresourceLayers{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.plane0) - layerCount: 1 - } - dstSubresource: vk.ImageSubresourceLayers{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.plane0) - layerCount: 1 - } - extent: vk.Extent3D{ - width: vp.decoder.video_data.width - height: vp.decoder.video_data.height - depth: 1 - } - }, - vk.ImageCopy2{ - srcSubresource: vk.ImageSubresourceLayers{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.plane1) - layerCount: 1 - } - dstSubresource: vk.ImageSubresourceLayers{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.plane1) - layerCount: 1 - } - extent: vk.Extent3D{ - width: vp.decoder.video_data.width / 2 - height: vp.decoder.video_data.height / 2 - depth: 1 - } - }, - ] - copy_info := vk.CopyImageInfo2{ - srcImage: source_image - srcImageLayout: .transfer_src_optimal - dstImage: output.texture.image - dstImageLayout: .transfer_dst_optimal - regionCount: u32(regions.len) - pRegions: regions.data - } - vk.cmd_copy_image2(command_buffer, ©_info) - output.layout = .transfer_dst_optimal - output.is_new = false - - post_barrier := vk.ImageMemoryBarrier2{ - srcStageMask: vk.pipeline_stage_2_transfer_bit - srcAccessMask: vk.access_2_transfer_read_bit - dstStageMask: vk.pipeline_stage_2_video_decode_bit_khr - dstAccessMask: vk.access_2_video_decode_read_bit_khr - oldLayout: .transfer_src_optimal - newLayout: source_restore_layout - srcQueueFamilyIndex: decode_family - dstQueueFamilyIndex: decode_family - image: source_image - subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - levelCount: 1 - layerCount: 1 - } - } - mut post_barriers := [post_barrier] - if !coincide { - post_barriers << vk.ImageMemoryBarrier2{ - srcStageMask: vk.pipeline_stage_2_video_decode_bit_khr - srcAccessMask: vk.access_2_video_decode_write_bit_khr - dstStageMask: vk.pipeline_stage_2_video_decode_bit_khr - dstAccessMask: vk.access_2_video_decode_read_bit_khr - oldLayout: .video_decode_dpb_khr - newLayout: .video_decode_dpb_khr - srcQueueFamilyIndex: decode_family - dstQueueFamilyIndex: decode_family - image: vp.dpb.image[vp.dpb.current_slot].image - subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - levelCount: 1 - layerCount: 1 - } - } - } - post_dependency := vk.DependencyInfo{ - imageMemoryBarrierCount: u32(post_barriers.len) - pImageMemoryBarriers: post_barriers.data - } - vk.cmd_pipeline_barrier2(command_buffer, &post_dependency) - source_state.layout = source_restore_layout - source_state.flag = if coincide { - vk.access_2_video_decode_read_bit_khr - } else { - vk.access_2_video_decode_write_bit_khr - } - if !coincide { - vp.dpb.resource_state[vp.dpb.current_slot].layout = .video_decode_dpb_khr - vp.dpb.resource_state[vp.dpb.current_slot].flag = vk.access_2_video_decode_read_bit_khr - } -} - -pub fn (mut vp VideoPlayer) video_decode_pre_barrier(video_command_buffer vk.CommandBuffer) { - mut image_barriers := []vk.ImageMemoryBarrier2{} - decode_family := vp.app.device_context.get_decoder_queue_family_index() - mut current_state := &vp.dpb.resource_state[vp.dpb.current_slot] - if current_state.layout != .video_decode_dpb_khr || current_state.flag != vk.access_2_video_decode_write_bit_khr { - barrier := vk.ImageMemoryBarrier2{ - srcStageMask: vk.pipeline_stage_2_video_decode_bit_khr - srcAccessMask: current_state.flag - dstStageMask: vk.pipeline_stage_2_video_decode_bit_khr - dstAccessMask: vk.access_2_video_decode_write_bit_khr - oldLayout: current_state.layout - newLayout: .video_decode_dpb_khr - srcQueueFamilyIndex: decode_family - dstQueueFamilyIndex: decode_family - image: vp.dpb.image[vp.dpb.current_slot].image - subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - baseMipLevel: 0 - levelCount: 1 - baseArrayLayer: 0 - layerCount: 1 - } - } - image_barriers << barrier - current_state.layout = barrier.newLayout - current_state.flag = barrier.dstAccessMask - } - if !vp.decoder.properties.dpb_and_output_coincide && (vp.decode_output_state.layout != .video_decode_dst_khr || vp.decode_output_state.flag != vk.access_2_video_decode_write_bit_khr) { - output_barrier := vk.ImageMemoryBarrier2{ - srcStageMask: vk.pipeline_stage_2_all_commands_bit - srcAccessMask: vp.decode_output_state.flag - dstStageMask: vk.pipeline_stage_2_video_decode_bit_khr - dstAccessMask: vk.access_2_video_decode_write_bit_khr - oldLayout: vp.decode_output_state.layout - newLayout: .video_decode_dst_khr - srcQueueFamilyIndex: decode_family - dstQueueFamilyIndex: decode_family - image: vp.decode_output_image.image - subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - levelCount: 1 - layerCount: 1 - } - } - image_barriers << output_barrier - vp.decode_output_state.layout = output_barrier.newLayout - vp.decode_output_state.flag = output_barrier.dstAccessMask - } - for ref_index in vp.dpb.reference_usage { - mut ref_state := &vp.dpb.resource_state[ref_index] - if ref_state.layout != .video_decode_dpb_khr || ref_state.flag != vk.access_2_video_decode_read_bit_khr { - barrier := vk.ImageMemoryBarrier2{ - srcStageMask: vk.pipeline_stage_2_video_decode_bit_khr - srcAccessMask: ref_state.flag - dstStageMask: vk.pipeline_stage_2_video_decode_bit_khr - dstAccessMask: vk.access_2_video_decode_read_bit_khr - oldLayout: ref_state.layout - newLayout: .video_decode_dpb_khr - srcQueueFamilyIndex: decode_family - dstQueueFamilyIndex: decode_family - image: vp.dpb.image[ref_index].image - subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - baseMipLevel: 0 - levelCount: 1 - baseArrayLayer: 0 - layerCount: 1 - } - } - image_barriers << barrier - ref_state.layout = barrier.newLayout - ref_state.flag = barrier.dstAccessMask - } - } - if image_barriers.len > 0 { - dependency := vk.DependencyInfo{ - imageMemoryBarrierCount: u32(image_barriers.len) - pImageMemoryBarriers: image_barriers.data - } - vk.cmd_pipeline_barrier2(video_command_buffer, &dependency) - } -} - -fn (mut vp VideoPlayer) video_decode_core(operation &DecoderVideoDecodeOperation, command_buffer vk.CommandBuffer) { - slice_header := unsafe { &h264.SliceHeader(operation.slice_header) } - pps := unsafe { &h264.PictureParameterSet(operation.pps) } - frame := vp.decoder.video_data.frame_infos[operation.decoded_frame_index] - mut std_picture := vk.StdVideoDecodeH264PictureInfo{ - pic_parameter_set_id: u8(slice_header.pic_parameter_set_id) - seq_parameter_set_id: u8(pps.seq_parameter_set_id) - frame_num: u16(slice_header.frame_num) - idr_pic_id: u16(slice_header.idr_pic_id) - } - C.vv_set_h264_picture_order_count(&std_picture, operation.poc[0], operation.poc[1]) - std_picture.flags.is_intra = u32(operation.frame_type == .e_intra) - std_picture.flags.is_reference = u32(operation.reference_priority > 0) - C.vv_set_h264_idr_picture_flag(&std_picture, u32(frame.nal_unit_type == 5)) - std_picture.flags.field_pic_flag = slice_header.field_pic_flag - std_picture.flags.bottom_field_flag = slice_header.bottom_field_flag - - mut slot_infos := [slot_count]vk.VideoReferenceSlotInfoKHR{} - mut pictures := [slot_count]vk.VideoPictureResourceInfoKHR{} - mut h264_slots := [slot_count]vk.VideoDecodeH264DpbSlotInfoKHR{} - mut reference_infos := [slot_count]vk.StdVideoDecodeH264ReferenceInfo{} - for i in 0 .. int(operation.dpb_slot_num) { - pictures[i] = vk.VideoPictureResourceInfoKHR{ - codedExtent: vk.Extent2D{ - width: vp.decoder.video_data.width - height: vp.decoder.video_data.height - } - baseArrayLayer: 0 - imageViewBinding: vp.dpb.image[i].view - } - C.vv_set_h264_reference_info(&reference_infos[i], u16(unsafe { operation.dpb_frame_num[i] }), unsafe { operation.dpb_poc[i] }, unsafe { operation.dpb_poc[i] }) - h264_slots[i] = vk.VideoDecodeH264DpbSlotInfoKHR{ - pStdReferenceInfo: unsafe { &reference_infos[i] } - } - slot_infos[i] = vk.VideoReferenceSlotInfoKHR{ - pNext: unsafe { &h264_slots[i] } - slotIndex: i - pPictureResource: unsafe { &pictures[i] } - } - } - - mut active_slots := [slot_count]vk.VideoReferenceSlotInfoKHR{} - for i in 0 .. int(operation.dpb_reference_count) { - ref_slot := unsafe { operation.dpb_reference_slots[i] } - active_slots[i] = slot_infos[ref_slot] - } - active_slots[operation.dpb_reference_count] = slot_infos[operation.current_dpb] - active_slots[operation.dpb_reference_count].slotIndex = -1 - begin_info := vk.VideoBeginCodingInfoKHR{ - videoSession: vp.decoder.video_session - videoSessionParameters: vp.decoder.video_session_parameters - referenceSlotCount: operation.dpb_reference_count + 1 - pReferenceSlots: unsafe { &active_slots[0] } - } - vk.cmd_begin_video_coding_khr(command_buffer, &begin_info) - if (operation.flags & u32(DecoderVideoDecodeOperationFlags.e_session_reset)) != 0 { - control_info := vk.VideoCodingControlInfoKHR{ - flags: vk.VideoCodingControlFlagsKHR(vk.VideoCodingControlFlagBitsKHR.reset) - } - vk.cmd_control_video_coding_khr(command_buffer, &control_info) - } - - mut h264_picture := vk.VideoDecodeH264PictureInfoKHR{ - pStdPictureInfo: &std_picture - sliceCount: operation.slice_count - pSliceOffsets: operation.slice_offsets - } - dst_picture := if vp.decoder.properties.dpb_and_output_coincide { - pictures[operation.current_dpb] - } else { - vk.VideoPictureResourceInfoKHR{ - codedExtent: vk.Extent2D{ - width: vp.decoder.video_data.width - height: vp.decoder.video_data.height - } - baseArrayLayer: 0 - imageViewBinding: vp.decode_output_image.view - } - } - mut decode_info := vk.VideoDecodeInfoKHR{ - pNext: &h264_picture - srcBuffer: vp.decoder.gpu_bitstream_buffer - srcBufferOffset: operation.stream_offset - srcBufferRange: operation.stream_size - dstPictureResource: dst_picture - pSetupReferenceSlot: unsafe { &slot_infos[operation.current_dpb] } - referenceSlotCount: operation.dpb_reference_count - pReferenceSlots: if operation.dpb_reference_count == 0 { - unsafe { nil } - } else { - unsafe { &active_slots[0] } - } - } - vk.cmd_decode_video_khr(command_buffer, &decode_info) - end_info := vk.VideoEndCodingInfoKHR{} - vk.cmd_end_video_coding_khr(command_buffer, &end_info) -} - -pub fn (mut vp VideoPlayer) write_video_frame(mut frame VideoPlayerDecodeStreamFrame) ! { - data_frame := vp.decoder.video_data.frame_infos[vp.current_frame] - mut frame_bytes_num_to_do := data_frame.frame_bytes_num - lock vp.decoder { - vp.decoder.video_data.file.seek(data_frame.src_offset, .start) or { - return error('could not seek to MP4 frame ${vp.current_frame}: ${err}') - } - } - for frame_bytes_num_to_do > 0 { - if frame_bytes_num_to_do < 4 { - return error('MP4 frame ${vp.current_frame} has a truncated H.264 NAL length') - } - mut src_buffer := []u8{len: 4} - mut length_bytes_read := 0 - lock vp.decoder { - length_bytes_read = vp.decoder.video_data.file.read(mut src_buffer) or { - return error('could not read H.264 NAL length in MP4 frame ${vp.current_frame}: ${err}') - } - } - if length_bytes_read != src_buffer.len { - return error('short read of H.264 NAL length in MP4 frame ${vp.current_frame}: expected 4 bytes, read ${length_bytes_read}') - } - mut size := u32(src_buffer[0]) << 24 | u32(src_buffer[1]) << 16 | u32(src_buffer[2]) << 8 | src_buffer[3] - size += 4 - if size < 4 || frame_bytes_num_to_do < size { - return error('MP4 frame ${vp.current_frame} has an invalid H.264 NAL size ${size - 4}') - } - mut file := File(os.File{}) - mut nal_header_byte := u8(0) - lock vp.decoder { - file = vp.decoder.video_data.file - nal_header_byte = file.peek()! - } - mut bs := h264.Bitstream{} - bs.init([nal_header_byte]) - mut nal := h264.NetworkAbstractionLayerHeader{} - nal.read_nal_header(mut bs) - // Skip over any frame data that is not idr slice or non-idr slice - if nal.type != h264.NAL_UNIT_TYPE.coded_slice_idr && nal.type != h264.NAL_UNIT_TYPE.coded_slice_non_idr { - frame_bytes_num_to_do -= size - lock vp.decoder { - vp.decoder.video_data.file.seek(size - 4, .current) or { - return error('could not skip non-slice NAL in MP4 frame ${vp.current_frame}: ${err}') - } - } - continue - } - - if frame.gpu_bitstream_size + size <= frame.gpu_bitstream_capacity { - nal_start_code := h264.NalStartCode{}.value - frame.slice_offsets << u32(frame.gpu_bitstream_size) - dst_buffer := unsafe { - frame.gpu_bitstream_slice_mapped_memory_address + frame.gpu_bitstream_size - } - lock vp.decoder { - unsafe { vmemcpy(dst_buffer, nal_start_code.data, nal_start_code.len) } - bytes_read := vp.decoder.video_data.file.read_into_ptr(unsafe { - dst_buffer + nal_start_code.len - }, int(size - 4)) or { - return error('could not read H.264 NAL payload in MP4 frame ${vp.current_frame}: ${err}') - } - if bytes_read != int(size - 4) { - return error('short read of H.264 NAL payload in MP4 frame ${vp.current_frame}: expected ${size - 4} bytes, read ${bytes_read}') - } - } - frame.gpu_bitstream_size += u64(nal_start_code.len) + size - 4 - } else { - return error('encoded access unit ${vp.current_frame} requires more than its ${frame.gpu_bitstream_capacity}-byte aligned bitstream-buffer capacity (written=${frame.gpu_bitstream_size}, next_nal=${size})') - } - frame_bytes_num_to_do -= size - } - lock vp.decoder { - aligned_size := U64(frame.gpu_bitstream_size).align_to(vp.decoder.properties.caps.minBitstreamBufferSizeAlignment) - if aligned_size > frame.gpu_bitstream_capacity { - return error('aligned access unit ${vp.current_frame} exceeds its ${frame.gpu_bitstream_capacity}-byte bitstream-buffer capacity') - } - if aligned_size > frame.gpu_bitstream_size { - unsafe { - vmemset(frame.gpu_bitstream_slice_mapped_memory_address + frame.gpu_bitstream_size, 0, isize(aligned_size - frame.gpu_bitstream_size)) - } - } - frame.gpu_bitstream_size = aligned_size - } -} - -pub type File = os.File - -// Return next byte in file, but don't move the cursor position -pub fn (mut f File) peek() !u8 { - pos_bk := f.tell()! - mut ret := []u8{len: 1} - bytes_read := f.read(mut ret)! - f.seek(pos_bk, .start)! - if bytes_read != 1 { - return error('unexpected end of file while reading H.264 NAL header') - } - return ret[0] -} - -pub fn has_flag[T](lhs u32, rhs T) bool { - return (lhs & u32(rhs)) == u32(rhs) -} - -pub fn (d Decoder) get_slice_header() byteptr { - return d.video_data.slice_header_bytes.data -} - -pub fn (d Decoder) get_pps() byteptr { - return d.video_data.pps_bytes.data -} - -pub fn (d Decoder) get_sps() byteptr { - return d.video_data.sps_bytes.data -} diff --git a/examples/video_decode_app/fragment_shader.v b/fragment_shader.v similarity index 97% rename from examples/video_decode_app/fragment_shader.v rename to fragment_shader.v index 6038d16..12028a1 100644 --- a/examples/video_decode_app/fragment_shader.v +++ b/fragment_shader.v @@ -1,6 +1,6 @@ -module video_decode_app +module main -pub const g_fragment_shader = [ +const g_fragment_shader = [ u32(0x07230203),0x00010000,0x0008000b,0x00000025,0x00000000,0x00020011,0x00000001,0x0006000b, 0x00000001,0x4c534c47,0x6474732e,0x3035342e,0x00000000,0x0003000e,0x00000000,0x00000001, 0x0008000f,0x00000004,0x00000004,0x6e69616d,0x00000000,0x00000009,0x0000000c,0x00000024, diff --git a/main.v b/main.v index 82e8aa3..8a85cd7 100644 --- a/main.v +++ b/main.v @@ -1,13 +1,12 @@ module main -import examples.video_decode_app as vda import os struct CliOptions { mut: video_path string gpu_index int = -1 - decode_output_mode vda.DecodeOutputMode + decode_output_mode DecodeOutputMode list_gpus bool show_help bool } @@ -40,9 +39,9 @@ fn parse_cli(args []string) !CliOptions { return error('--decode-output-mode requires auto, coincident, or distinct') } options.decode_output_mode = match args[i] { - 'auto' { vda.DecodeOutputMode.automatic } - 'coincident' { vda.DecodeOutputMode.coincident } - 'distinct' { vda.DecodeOutputMode.distinct } + 'auto' { DecodeOutputMode.automatic } + 'coincident' { DecodeOutputMode.coincident } + 'distinct' { DecodeOutputMode.distinct } else { return error('invalid decode output mode: ${args[i]} (expected auto, coincident, or distinct)') } @@ -84,12 +83,12 @@ fn main() { exit(2) } - mut app := vda.VideoDecodeApp{ - device_context: vda.DeviceContext{} - video_path: options.video_path + mut app := VideoDecodeApp{ + device_context: DeviceContext{} + video_path: options.video_path preferred_gpu_index: options.gpu_index - decode_output_mode: options.decode_output_mode - list_gpus: options.list_gpus + decode_output_mode: options.decode_output_mode + list_gpus: options.list_gpus } app.device_context.swapchain.app = &app if !app.initialize() { diff --git a/main_test.v b/main_test.v index dcf81f0..45e2a11 100644 --- a/main_test.v +++ b/main_test.v @@ -1,7 +1,5 @@ module main -import examples.video_decode_app as vda - fn test_parse_cli_defaults() { options := parse_cli([]) or { panic(err) } assert options.video_path == '' @@ -19,9 +17,9 @@ fn test_parse_cli_gpu_and_list() { fn test_parse_cli_decode_output_modes() { for name, expected in { - 'auto': vda.DecodeOutputMode.automatic - 'coincident': vda.DecodeOutputMode.coincident - 'distinct': vda.DecodeOutputMode.distinct + 'auto': DecodeOutputMode.automatic + 'coincident': DecodeOutputMode.coincident + 'distinct': DecodeOutputMode.distinct } { options := parse_cli(['--decode-output-mode', name, 'movie.mp4']) or { panic(err) } assert options.decode_output_mode == expected diff --git a/mp4_parser.v b/mp4_parser.v new file mode 100644 index 0000000..c6d8f03 --- /dev/null +++ b/mp4_parser.v @@ -0,0 +1,526 @@ +module main + +import os +import math +import antono2.minimp4 +import antono2.h264 + +@[heap] +struct CallbackUserData { +pub mut: + file &os.File + last_offset i64 + file_size u64 + read_error string +} + +fn read_callback(offset i64, buffer &u8, size usize, user_data voidptr) int { + mut data := unsafe { &CallbackUserData(user_data) } + if offset < 0 || offset >= data.file_size { + return 1 + } + to_copy := math.min(i64(size), data.file_size - offset) + data.file.seek(offset, .start) or { + data.read_error = 'could not seek to MP4 offset ${offset}: ${err}' + return 1 + } + num_bytes_read := data.file.read_into_ptr(buffer, int(to_copy)) or { + data.read_error = 'could not read ${to_copy} bytes at MP4 offset ${offset}: ${err}' + return 1 + } + data.last_offset = offset + num_bytes_read + if num_bytes_read != int(to_copy) { + data.read_error = 'short read at MP4 offset ${offset}: expected ${to_copy} bytes, read ${num_bytes_read}' + return 1 + } + return 0 +} + +// Turn an EBSP (Encapsulated Byte Sequence Payload) into an RBSP (Raw Byte Sequence Payload) +// pub fn remove_emulation_prevention_bytes(ebsp []u8) []u8 { +// mut rbsp := []u8{cap: ebsp.len} +// if ebsp.len == 0 { return rbsp } +// mut i := 0 +// for i < ebsp.len { +// if ((i + 2) < ebsp.len) +// && ebsp[i] == 0 +// && ebsp[i + 1] == 0 +// && ebsp[i + 2] == 3 { +// rbsp << ebsp[i] +// rbsp << ebsp[i + 1] +// i += 2 +// } else { +// rbsp << ebsp[i] +// } +// i++ +// } +// return rbsp +// } +@[unsafe] +fn remove_emulation_prevention_bytes(ebsp byteptr, size int) []u8 { + mut rbsp := []u8{cap: size} + if size == 0 { + return rbsp + } + mut i := 0 + for i < size { + if (i + 2) < size && unsafe { ebsp[i] == 0 && ebsp[i + 1] == 0 && ebsp[i + 2] == 3 } { + rbsp << unsafe { ebsp[i] } + rbsp << unsafe { ebsp[i + 1] } + i += 2 + } else { + rbsp << unsafe { ebsp[i] } + } + i++ + } + return rbsp +} + +type BytePtr = byteptr + +fn (p BytePtr) to_varray[T](len u32) []T { + if isnil(p) && len > 0 { + panic('Nil to_varray for len: ${len}') + } + if len <= 0 { + return []T{} + } + // TODO: doesn't trigger on to_varray[u8] + $if T is u8 { + return p.vbytes(int(len)) + } + mut ret := []T{cap: int(len)} + // vmemcpy(ret.data, p, len * sizeof(T)) + for i in 0 .. len { + ret << *unsafe { &T(p + (i * sizeof(T))) } + } + return ret +} + +fn (mut d Decoder) parse_mp4_data(file_path string) ! { + d.video_data.file = os.open_file(file_path, 'rb')! + d.video_data.file_open = true + d.video_data.file.seek(0, .end)! + mp4_file_size := d.video_data.file.tell()! + d.video_data.file.seek(0, .start)! + mut mp4 := minimp4.MP4D_demux_t{} + mut user_data := CallbackUserData{ + file: &d.video_data.file + last_offset: 0 + file_size: os.file_size(file_path) + } + if minimp4.mp4d_open(&mp4, read_callback, &user_data, mp4_file_size) != 1 { + if user_data.read_error != '' { + return error(user_data.read_error) + } + return error('not a readable MP4 file') + } + defer { + minimp4.mp4d_close(&mp4) + } + if mp4.track_count == 0 || isnil(mp4.track) { + return error('MP4 contains no tracks') + } + mut ntrack := u32(0) + mut found_video_track := false + mut found_unsupported_video_track := false + for track_index in u32(0) .. mp4.track_count { + candidate := unsafe { mp4.track[track_index] } + if candidate.handler_type != minimp4.mp4d_handler_type_vide { + continue + } + if candidate.object_type_indication == minimp4.mp4_object_type_avc { + ntrack = track_index + found_video_track = true + break + } + found_unsupported_video_track = true + } + if !found_video_track { + if found_unsupported_video_track { + return error('MP4 video codec is unsupported; this player currently supports H.264/AVC') + } + return error('MP4 contains no H.264/AVC video track') + } + track := unsafe { mp4.track[ntrack] } + if track.sample_count == 0 { + return error('H.264 video track contains no samples') + } + if track.timescale == 0 { + return error('H.264 video track has an invalid zero timescale') + } + d.video_data.metadata.track_matrix = track.track_matrix + d.video_data.metadata.rotation_degrees = rotation_from_track_matrix(track.track_matrix) + + // Read SPS + mut num_bytes_sps := 0 + mut count_sps := 0 + mut sps_array := []h264.SequenceParameterSet{} + mut data_sps := minimp4.mp4d_read_sps(&mp4, ntrack, count_sps, &num_bytes_sps) + for !isnil(data_sps) { + mut nal := h264.NetworkAbstractionLayerHeader{} + mut nal_header_bs := h264.Bitstream{} + nal_header_bs.init(unsafe { data_sps.vbytes(1) }) + nal.read_nal_header(mut &nal_header_bs) + if num_bytes_sps <= 1 { + return error('H.264 track contains an invalid sequence parameter set') + } + mut nal_payload_rbsp_data := unsafe { + remove_emulation_prevention_bytes(byteptr(data_sps) + 1, num_bytes_sps - 1) + } + mut nal_payload_bs := h264.Bitstream{} + nal_payload_bs.init(nal_payload_rbsp_data) + mut sps := h264.SequenceParameterSet{} + sps.read_sps(mut nal_payload_bs) + if sps.profile_idc !in [u32(66), 77, 100] { + return error('H.264 profile_idc ${sps.profile_idc} is unsupported; supported profiles are Baseline, Main, and High 8-bit 4:2:0') + } + if d.video_data.h264_profile_idc == 0 { + d.video_data.h264_profile_idc = sps.profile_idc + } else if d.video_data.h264_profile_idc != sps.profile_idc { + return error('H.264 track changes profile between sequence parameter sets') + } + if sps.bit_depth_luma_minus8 != 0 || sps.bit_depth_chroma_minus8 != 0 { + return error('H.264 bit depths above 8-bit are not supported') + } + if sps.profile_idc >= 100 && sps.chroma_format_idc != 1 { + return error('H.264 chroma formats other than 4:2:0 are not supported') + } + if sps.frame_mbs_only_flag == 0 { + return error('interlaced H.264 video is not currently supported') + } + // Data validation + // https://stackoverflow.com/questions/6394874/fetching-the-dimensions-of-a-h264video-stream + width := ((sps.pic_width_in_mbs_minus1 + 1) * 16) - (sps.frame_crop_left_offset * 2) - (sps.frame_crop_right_offset * 2) + height := ((2 - sps.frame_mbs_only_flag) * (sps.pic_height_in_map_units_minus1 + 1) * 16) - (sps.frame_crop_top_offset * 2) - (sps.frame_crop_bottom_offset * 2) + mp4_width := unsafe { track.sampleDescription.video.width } + mp4_height := unsafe { track.sampleDescription.video.height } + if mp4_width != width || mp4_height != height { + eprintln('Warning: MP4 dimensions ${mp4_width}x${mp4_height} differ from H.264 SPS display dimensions ${width}x${height}') + } + d.video_data.width_padd = (sps.pic_width_in_mbs_minus1 + 1) * 16 + d.video_data.height_padd = (sps.pic_height_in_map_units_minus1 + 1) * 16 + if sps.vui_parameters_present_flag != 0 { + d.video_data.metadata.sar_width, d.video_data.metadata.sar_height = sample_aspect_ratio(sps.vui.aspect_ratio_idc, sps.vui.sar_width, sps.vui.sar_height) + d.video_data.metadata.video_full_range = sps.vui.video_full_range_flag != 0 + d.video_data.metadata.colour_description_present = d.video_data.metadata.colour_description_present + || sps.vui.color_description_present_flag != 0 + d.video_data.metadata.colour_primaries = u8(sps.vui.colour_primaries) + d.video_data.metadata.transfer_function = u8(sps.vui.transfer_characteristics) + d.video_data.metadata.matrix_coefficients = u8(sps.vui.matrix_coefficients) + } + // x ^ ((x ^ y) & -(x < y)) // max(x, y) + // d.video_data.num_dpb_slots = d.video_data.num_dpb_slots ^ ((d.video_data.num_dpb_slots ^ (sps.num_ref_frames * 2 + 1)) & -u32(d.video_data.num_dpb_slots < (sps.num_ref_frames * 2 + 1))) + d.video_data.num_dpb_slots = math.max[u32](d.video_data.num_dpb_slots, sps.num_ref_frames + 1) + d.video_data.sps_bytes << unsafe { byteptr(&sps).vbytes(int(sizeof(sps))) } + sps_array << sps + d.video_data.sps_count++ + count_sps++ + + data_sps = minimp4.mp4d_read_sps(&mp4, ntrack, count_sps, &num_bytes_sps) + } + if sps_array.len == 0 { + return error('H.264 video track contains no sequence parameter set (SPS)') + } + + // Read PPS + mut size_pps := 0 + mut count_pps := 0 + mut pps_array := []h264.PictureParameterSet{} + mut data_pps := minimp4.mp4d_read_pps(&mp4, ntrack, count_pps, &size_pps) + + for !isnil(data_pps) { + mut nal := h264.NetworkAbstractionLayerHeader{} + mut nal_header_bs := h264.Bitstream{} + nal_header_bs.init(unsafe { data_pps.vbytes(1) }) + nal.read_nal_header(mut nal_header_bs) + if size_pps <= 1 { + return error('H.264 track contains an invalid picture parameter set') + } + pps_payload_rbsp_data := unsafe { + remove_emulation_prevention_bytes(byteptr(data_pps) + 1, size_pps - 1) + } + mut pps_payload_bs := h264.Bitstream{} + pps_payload_bs.init(pps_payload_rbsp_data) + + mut pps := h264.PictureParameterSet{} + pps.read_pps(mut pps_payload_bs) + d.video_data.pps_bytes.ensure_cap(d.video_data.pps_bytes.len + int(sizeof(pps))) + d.video_data.pps_bytes << unsafe { byteptr(&pps).vbytes(int(sizeof(pps))) } + pps_array << pps + d.video_data.pps_count++ + count_pps++ + + data_pps = minimp4.mp4d_read_pps(&mp4, ntrack, count_pps, &size_pps) + } + if pps_array.len == 0 { + return error('H.264 video track contains no picture parameter set (PPS)') + } + + d.video_data.width = unsafe { track.sampleDescription.video.width } + d.video_data.height = unsafe { track.sampleDescription.video.height } + d.video_data.metadata.coded_width = d.video_data.width + d.video_data.metadata.coded_height = d.video_data.height + d.video_data.metadata.update_display_dimensions() + println('Video codec: H.264 ${h264_profile_name(d.video_data.h264_profile_idc)} Profile') + println('Display metadata: coded ${d.video_data.metadata.coded_width}x${d.video_data.metadata.coded_height}, display ${d.video_data.metadata.display_width}x${d.video_data.metadata.display_height}, SAR ${d.video_data.metadata.sar_width}:${d.video_data.metadata.sar_height}, rotation ${d.video_data.metadata.rotation_degrees}°') + + timescale_rcp := 1.0 / f64(track.timescale) + mut prev_pic_order_cnt_lsb := u32(0) + mut prev_pic_order_cnt_msb := u32(0) + mut poc_cycle := -1 + mut prev_frame_num := u32(0) + mut prev_frame_offset := u32(0) + + // Read frames + mut track_duration := u32(0) + mut max_frame_size_bytes := u64(0) + mut input_file_position := u64(0) + d.video_data.file.seek(0, .start)! + + d.video_data.frame_infos = []DecoderVideoDataFrameInfo{cap: int(track.sample_count)} + d.video_data.slice_header_bytes.ensure_cap(int(track.sample_count * sizeof(h264.SliceHeader))) + d.video_data.slice_header_count = track.sample_count + + mut file := d.video_data.file + mut sample_index := u32(0) + for { + if sample_index >= track.sample_count { + break + } + mut frame_bytes_num_to_do := u32(0) + mut duration := u32(0) + mut timestamp := u32(0) + // minimp4 returns timestamp before duration. These were previously passed + // in reverse order, causing later frames to use their growing timestamp as + // a duration and making playback progressively slower. + offset := minimp4.mp4d_frame_offset(&mp4, ntrack, sample_index, &frame_bytes_num_to_do, ×tamp, &duration) + // The upload buffer must fit any slice contained in the complete MP4 + // sample, including samples with leading non-slice NAL units. + max_frame_size_bytes = math.max[u64](max_frame_size_bytes, frame_bytes_num_to_do) + track_duration += duration + + mut data_frame := DecoderVideoDataFrameInfo{ + src_offset: offset + frame_bytes_num: frame_bytes_num_to_do + } + + mut src_buffer := []u8{len: int(frame_bytes_num_to_do)} + mut src_buffer_idx := 0 + if offset - input_file_position > 0 { + file.seek(offset - input_file_position, .current) or { + return error('could not seek to MP4 sample ${sample_index}: ${err}') + } + } + if file.eof() { + return error('MP4 sample ${sample_index} points beyond the end of the file') + } + expected_frame_bytes := frame_bytes_num_to_do + frame_bytes_num_to_do = u32(file.read(mut src_buffer) or { + return error('could not read MP4 sample ${sample_index}: ${err}') + }) + if frame_bytes_num_to_do != expected_frame_bytes { + return error('MP4 sample ${sample_index} is truncated: expected ${expected_frame_bytes} bytes, read ${frame_bytes_num_to_do}') + } + input_file_position = offset + frame_bytes_num_to_do + for frame_bytes_num_to_do > 0 { + if frame_bytes_num_to_do < 4 { + return error('MP4 sample ${sample_index} has a truncated H.264 NAL length') + } + // mut size := unsafe{ (u32(*&src_buffer[src_buffer_idx+0]) << 24) | (u32(*&src_buffer[src_buffer_idx+1]) << 16) | (u32(*&src_buffer[src_buffer_idx+2]) << 8) | *&src_buffer[src_buffer_idx+3] } + mut size := unsafe { + (u32(src_buffer[src_buffer_idx + 0]) << 24) | (u32(src_buffer[src_buffer_idx + 1]) << 16) | (u32(src_buffer[src_buffer_idx + 2]) << 8) | src_buffer[src_buffer_idx + 3] + } + size += 4 + if size < 4 || frame_bytes_num_to_do < size { + return error('MP4 sample ${sample_index} has an invalid H.264 NAL size ${size - 4}') + } + + length_prefixed_data_offset := src_buffer_idx + 4 + length_prefixed_data_size := size - 4 + if length_prefixed_data_size <= 1 { + return error('MP4 sample ${sample_index} contains an empty H.264 NAL unit') + } + + mut nal := h264.NetworkAbstractionLayerHeader{} + mut nal_header_bs := h264.Bitstream{} + nal_header_bs.init(src_buffer[length_prefixed_data_offset..length_prefixed_data_offset + 1]) + nal.read_nal_header(mut nal_header_bs) + + slfrom := length_prefixed_data_offset + 1 + nal_payload_rbsp_data := unsafe { + remove_emulation_prevention_bytes(byteptr(src_buffer.data) + slfrom, int(length_prefixed_data_size - 1)) + } + mut nal_payload_bs := h264.Bitstream{} + nal_payload_bs.init(nal_payload_rbsp_data) + + mut is_idr := false + match nal.type { + .coded_slice_idr { + data_frame.frame_type = DecoderFrameType.e_intra + is_idr = true + } + .coded_slice_non_idr { + data_frame.frame_type = DecoderFrameType.e_predictive + } + else { + frame_bytes_num_to_do -= size + src_buffer_idx += int(size) + continue + } + } + + /* + * Decode Picture Order Count + * (tig) see ITU-T H.264 (08/2021) pp.113 + * + */ + // tig: see Rec. ITU-T H.264 (08/2021) p.66 (7-1) + mut slice_header := h264.SliceHeader{} + slice_header.read_slice_header(&nal, pps_array, sps_array, mut nal_payload_bs) + if slice_header.pic_parameter_set_id >= u32(pps_array.len) { + return error('MP4 sample ${sample_index} references missing H.264 PPS ${slice_header.pic_parameter_set_id}') + } + pps := pps_array[slice_header.pic_parameter_set_id] + if pps.seq_parameter_set_id >= u32(sps_array.len) { + return error('MP4 sample ${sample_index} references missing H.264 SPS ${pps.seq_parameter_set_id}') + } + sps := sps_array[pps.seq_parameter_set_id] + + max_frame_num := u32(1) << (sps.log2_max_frame_num_minus4 + 4) + max_pic_order_cnt_lsb := u32(1) << (sps.log2_max_pic_order_cnt_lsb_minus4 + 4) + pic_order_cnt_lsb := u32(slice_header.pic_order_cnt_lsb) + mut pic_order_cnt_msb := u32(0) + mut frame_num_offset := u32(0) + mut tmp_pic_order_cout := u32(0) + + match sps.pic_order_cnt_type { + 0 { + // TYPE 0 + // Rec. ITU-T H.264 (08/2021) page 114 + // Use the NAL unit type, not idr flag + if is_idr { + prev_pic_order_cnt_msb = 0 + prev_pic_order_cnt_lsb = 0 + poc_cycle++ + } + if pic_order_cnt_lsb < prev_pic_order_cnt_lsb && (prev_pic_order_cnt_lsb - pic_order_cnt_lsb) >= max_pic_order_cnt_lsb / 2 { + pic_order_cnt_msb = prev_pic_order_cnt_msb + max_pic_order_cnt_lsb + } else if pic_order_cnt_lsb > prev_pic_order_cnt_lsb && (pic_order_cnt_lsb - prev_pic_order_cnt_lsb) > max_pic_order_cnt_lsb / 2 { + pic_order_cnt_msb = prev_pic_order_cnt_msb - max_pic_order_cnt_lsb + } else { + pic_order_cnt_msb = prev_pic_order_cnt_msb + } + // Top and bottom field order count in case the picture is a field + if slice_header.field_pic_flag == 0 || slice_header.bottom_field_flag == 0 { + data_frame.top_field_order_cnt = pic_order_cnt_msb + pic_order_cnt_lsb + } + if slice_header.field_pic_flag == 0 { + data_frame.bottom_field_order_cnt = data_frame.top_field_order_cnt + u32(slice_header.delta_pic_order_cnt_bottom) + } else if slice_header.bottom_field_flag != 0 { + data_frame.bottom_field_order_cnt = pic_order_cnt_msb + slice_header.pic_order_cnt_lsb + } + + // Same as top field order count + data_frame.poc = int(pic_order_cnt_msb + pic_order_cnt_lsb) + data_frame.gop = poc_cycle + + // TODO: memory_management_control_operation equal to 5 + if nal.idc != h264.NAL_REF_IDC.priority_disposable { + prev_pic_order_cnt_msb = pic_order_cnt_msb + prev_pic_order_cnt_lsb = pic_order_cnt_lsb + } + } + + // match 0 + 2 { + if is_idr { + frame_num_offset = 0 + } else if prev_frame_num > slice_header.frame_num { + frame_num_offset = prev_frame_offset + max_frame_num + } else { + frame_num_offset = prev_frame_offset + } + prev_frame_offset = frame_num_offset + prev_frame_num = slice_header.frame_num + + if is_idr { + tmp_pic_order_cout = 0 + } else if nal.idc == h264.NAL_REF_IDC.priority_disposable { + tmp_pic_order_cout = 2 * (frame_num_offset + slice_header.frame_num) - 1 + } else { + tmp_pic_order_cout = 2 * (frame_num_offset + slice_header.frame_num) + } + + // (tig) Ignore bottom or top fields, as we assume progressive. + // If it were otherwise - for interleaved - either the top or the bottom + // field shall be set, depending on whether the current picture is the + // top or bottom field, as indicated by bottom_field_flag + data_frame.poc = int(tmp_pic_order_cout) + if tmp_pic_order_cout == 0 { + poc_cycle++ + } + data_frame.gop = poc_cycle + } + + // match 2 + else { + return error('H.264 picture-order-count type ${sps.pic_order_cnt_type} is not supported') + } + } // match + + // Accept frame beginning NAL unit + data_frame.nal_ref_idc = u32(nal.idc) + data_frame.nal_unit_type = u8(nal.type) + // TODO: h264.nal_start_code as pub const. error, imported types must start with a capital letter, but const can't be upper case + // data_frame.size = sizeof(h264.nal_start_code) + size - 4 + nal_start_code := h264.NalStartCode{}.value + data_frame.size = u64(nal_start_code.len) + size - 4 + data_frame.reference_priority = u32(nal.idc) + + data_frame.decode_time_ns = i64(f64(timestamp) * timescale_rcp * 1_000_000_000.0) + data_frame.display_time_ns = i64(f64(timestamp) * timescale_rcp * 1_000_000_000.0) + data_frame.duration_ns = math.max[i64](1, i64(f64(duration) * timescale_rcp * 1_000_000_000.0)) + d.video_data.slice_header_bytes << unsafe { + byteptr(&slice_header).vbytes(int(sizeof(slice_header))) + } + + // for frame_bytes_num_to_do > 0 + break + } + + // x ^ ((x ^ y) & -(x < y)) // max(x, y) + // max_frame_size_bytes = max_frame_size_bytes ^ ((max_frame_size_bytes ^ data_frame.size) & -(u64(max_frame_size_bytes < data_frame.size))) + max_frame_size_bytes = math.max[u64](max_frame_size_bytes, data_frame.size) + + d.video_data.frame_infos << data_frame + sample_index++ + } // for sample_index < track.sample_count + + // Fills array values with their index 0..len + d.video_data.frame_display_order = []u64{len: d.video_data.frame_infos.len, init: index} + + // Keep frame_infos in MP4 decode order. Only the display-order index list is + // sorted; slice_header_bytes and current_frame use the original decode order. + for i in 1 .. d.video_data.frame_display_order.len { + index_to_insert := d.video_data.frame_display_order[i] + frame_to_insert := d.video_data.frame_infos[index_to_insert] + mut j := i + for j > 0 { + previous_index := d.video_data.frame_display_order[j - 1] + if compare_frame_display_order(&d.video_data.frame_infos[previous_index], &frame_to_insert) <= 0 { + break + } + d.video_data.frame_display_order[j] = previous_index + j-- + } + d.video_data.frame_display_order[j] = index_to_insert + } + + for i in 0 .. d.video_data.frame_display_order.len { + d.video_data.frame_infos[d.video_data.frame_display_order[i]].display_order = i + } + + d.video_data.max_memory_frame_size_bytes = max_frame_size_bytes + d.video_data.total_duration = i64(f64(track_duration) * timescale_rcp * 1_000_000_000.0) +} diff --git a/examples/video_decode_app/playback_timeline.v b/playback_timeline.v similarity index 97% rename from examples/video_decode_app/playback_timeline.v rename to playback_timeline.v index e9d3183..c6c8ea5 100644 --- a/examples/video_decode_app/playback_timeline.v +++ b/playback_timeline.v @@ -1,4 +1,4 @@ -module video_decode_app +module main import math diff --git a/examples/video_decode_app/playback_timeline_test.v b/playback_timeline_test.v similarity index 96% rename from examples/video_decode_app/playback_timeline_test.v rename to playback_timeline_test.v index f0a678f..47004f7 100644 --- a/examples/video_decode_app/playback_timeline_test.v +++ b/playback_timeline_test.v @@ -1,4 +1,4 @@ -module video_decode_app +module main fn test_mock_timeline_fixed_rate() { mut timeline := PlaybackTimeline{} @@ -65,7 +65,7 @@ fn test_presentation_buffer_accounts_for_b_frame_reordering() { fn test_ready_outputs_are_presented_by_picture_order() { mut player := VideoPlayer{ - output_textures: [ + output_textures: [ OutputImage{ display_order: 6, duration_ns: 1 }, OutputImage{ display_order: 3, duration_ns: 1 }, OutputImage{ display_order: 0, duration_ns: 1 }, @@ -88,9 +88,9 @@ fn test_ready_outputs_are_presented_by_picture_order() { fn test_loop_restart_keeps_last_picture_until_new_zero_is_ready() { mut player := VideoPlayer{ current_output_index: 4 - current_frame: 9 - next_display_order: 10 - decode_finished: true + current_frame: 9 + next_display_order: 10 + decode_finished: true } player.restart_decode_cycle() assert player.current_output_index == 4 diff --git a/player_decode.v b/player_decode.v new file mode 100644 index 0000000..30f6865 --- /dev/null +++ b/player_decode.v @@ -0,0 +1,634 @@ +module main + +import antono2.vulkan as vk +import os +import antono2.h264 + +fn (mut vp VideoPlayer) update_decode_video() ! { + dev_ctx := vp.app.device_context + if vp.output_textures_free.len == 0 { + return error('presentation queue has no reusable output image') + } + output_index := vp.output_textures_free[0] + vp.output_textures_free.delete(0) + mut output_queued := false + defer { + if !output_queued { + vp.output_textures_free << output_index + } + } + command_buffer_info := vp.command_buffer_infos[dev_ctx.swapchain.get_current_index()] + mut begin_command_buffer := vk.CommandBufferBeginInfo{} + vk.reset_command_buffer(command_buffer_info.graphics_command_buffer, 0) + vk.begin_command_buffer(command_buffer_info.graphics_command_buffer, &begin_command_buffer) + if vp.is_stopped { + vk.cmd_set_event(command_buffer_info.graphics_command_buffer, vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands)) + return + } + + video_command_buffer := command_buffer_info.video_command_buffer + vk.reset_command_buffer(video_command_buffer, 0) + vk.begin_command_buffer(video_command_buffer, &begin_command_buffer) + + mut frame_info := vp.decoder.video_data.frame_infos[vp.current_frame] + mut slice_header := &h264.SliceHeader(unsafe { nil }) + mut pps := &h264.PictureParameterSet(unsafe { nil }) + mut sps := &h264.SequenceParameterSet(unsafe { nil }) + lock vp.decoder { + assert !isnil(vp.decoder.get_slice_header()) + assert !isnil(vp.decoder.get_pps()) + assert !isnil(vp.decoder.get_sps()) + + slice_header = unsafe { + &h264.SliceHeader(byteptr(usize(vp.decoder.get_slice_header()) + usize(vp.current_frame) * sizeof(h264.SliceHeader))) + } + pps = unsafe { + &h264.PictureParameterSet(byteptr(usize(vp.decoder.get_pps()) + usize(slice_header.pic_parameter_set_id) * sizeof(h264.PictureParameterSet))) + } + sps = unsafe { + &h264.SequenceParameterSet(byteptr(usize(vp.decoder.get_sps()) + usize(pps.seq_parameter_set_id) * sizeof(h264.SequenceParameterSet))) + } + } + + mut decode_ope := DecoderVideoDecodeOperation{} + if vp.current_frame == 0 || has_flag[VideoPlayerFlags](vp.flags, VideoPlayerFlags.e_decoder_reset) { + decode_ope.flags = u32(DecoderVideoDecodeOperationFlags.e_session_reset) + vp.flags &= ~u32(VideoPlayerFlags.e_decoder_reset) + } + + // An ordinary I-picture can occur in an open GOP and may still depend on the + // existing DPB for pictures decoded after it. Only IDR starts a new reference + // picture sequence here. + if frame_info.nal_unit_type == u8(h264.NAL_UNIT_TYPE.coded_slice_idr) { + vp.dpb.reference_usage.clear() + } + + dpb_slot_num := int(vp.decoder.video_data.num_dpb_slots) + vp.dpb.current_slot = vp.dpb.acquire_decode_slot(dpb_slot_num) + vp.dpb.poc_status[vp.dpb.current_slot] = int(frame_info.poc) + vp.dpb.frame_num_status[vp.dpb.current_slot] = int(slice_header.frame_num) + + // Index variable on initialization comes in handy + dpbs := []vk.Image{len: dpb_slot_num, init: vp.dpb.image[index].image} + dpb_views := []vk.ImageView{len: dpb_slot_num, init: vp.dpb.image[index].view} + + // Only the prefix initialized from decoder.info.memory_frames owns mapped + // bitstream-buffer slices. Using the fixed array length eventually selects an + // uninitialized zero-capacity entry on streams longer than the DPB count. + use_frame_index := vp.current_frame % vp.decoder.info.memory_frames.len + vp.current_upload_index = use_frame_index + upload_fence := vp.video_frames[use_frame_index].in_flight_fence + res_wait := vk.wait_for_fences(dev_ctx.vk_device, 1, &upload_fence, vk._true, max_u64) + check_vk(res_wait, 'Could not wait for decoded-frame fence') + res_reset := vk.reset_fences(dev_ctx.vk_device, 1, &upload_fence) + check_vk(res_reset, 'Could not reset decoded-frame fence') + vp.video_frames[use_frame_index].gpu_bitstream_size = 0 + vp.video_frames[use_frame_index].slice_offsets.clear() + mut use_frame := &vp.video_frames[use_frame_index] + vp.write_video_frame(mut use_frame)! + if use_frame.gpu_bitstream_size == 0 { + // MP4 samples may contain only metadata/non-slice NAL units. They do not + // form a Vulkan decode operation and must not be submitted with range 0. + vk.end_command_buffer(video_command_buffer) + if vp.current_frame + 1 < vp.decoder.video_data.frame_infos.len { + vp.current_frame++ + } else { + vp.decode_finished = true + } + vk.cmd_set_event(command_buffer_info.graphics_command_buffer, vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands)) + return + } + mut flush_result := vk.Result.error_unknown + lock vp.decoder { + flush_result = vp.app.device_context.memory_allocator.flush_range(vp.decoder.gpu_bitstream_allocation, use_frame.gpu_bitstream_offset, use_frame.gpu_bitstream_size) + } + check_vk(flush_result, 'Could not flush the Vulkan Video bitstream buffer') + + decode_ope.stream_offset = use_frame.gpu_bitstream_offset + decode_ope.stream_size = use_frame.gpu_bitstream_size + decode_ope.poc[0] = frame_info.poc + decode_ope.poc[1] = frame_info.poc + decode_ope.frame_type = frame_info.frame_type + decode_ope.reference_priority = frame_info.reference_priority + decode_ope.decoded_frame_index = vp.current_frame + decode_ope.slice_header = slice_header + decode_ope.pps = pps + decode_ope.sps = sps + decode_ope.current_dpb = vp.dpb.current_slot + decode_ope.dpb_reference_count = u32(vp.dpb.reference_usage.len) + decode_ope.dpb_reference_slots = vp.dpb.reference_usage.data + // Pointer to data of fixed size array + decode_ope.dpb_poc = &vp.dpb.poc_status[0] + decode_ope.dpb_frame_num = &vp.dpb.frame_num_status[0] + + decode_ope.dpb_slot_num = u32(dpb_slot_num) + decode_ope.p_dpbs = dpbs.data + decode_ope.p_dpb_views = dpb_views.data + decode_ope.slice_count = u32(use_frame.slice_offsets.len) + decode_ope.slice_offsets = use_frame.slice_offsets.data + + // Copy for display + vp.decode_operation = decode_ope + + vp.video_decode_pre_barrier(video_command_buffer) + vp.video_decode_core(&decode_ope, video_command_buffer) + vp.copy_decoded_frame_to_output(video_command_buffer, output_index) + vp.output_textures[output_index].display_order = frame_info.display_order + vp.output_textures[output_index].duration_ns = frame_info.duration_ns + vp.output_textures_ready << output_index + output_queued = true + + if frame_info.reference_priority > 0 { + vp.dpb.reference_usage << vp.dpb.current_slot + for vp.dpb.reference_usage.len > int(vp.decoder.video_data.max_reference_pictures) { + vp.dpb.reference_usage.delete(0) + } + } + vk.end_command_buffer(video_command_buffer) + mut frame_count := 0 + rlock vp.decoder { + frame_count = vp.decoder.video_data.frame_infos.len + } + if vp.current_frame + 1 < frame_count { + vp.current_frame++ + } else { + // B-frame output may still be waiting in presentation order. Do not reset + // the codec or overwrite those images until the last picture was shown. + vp.decode_finished = true + } + + // The graphics command buffer waits on the decode semaphore before this + // transition, making the copied output safe for fragment sampling. + mut output_barrier := vk.ImageMemoryBarrier2{ + srcStageMask: vk.pipeline_stage_2_transfer_bit + srcAccessMask: vk.access_2_transfer_write_bit + dstStageMask: vk.pipeline_stage_2_fragment_shader_bit + dstAccessMask: vk.access_2_shader_read_bit + oldLayout: .transfer_dst_optimal + newLayout: .shader_read_only_optimal + image: vp.output_textures[output_index].texture.image + subresourceRange: vk.ImageSubresourceRange{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) + levelCount: 1 + layerCount: 1 + } + } + output_dependency := vk.DependencyInfo{ + imageMemoryBarrierCount: 1 + pImageMemoryBarriers: &output_barrier + } + vk.cmd_pipeline_barrier2(command_buffer_info.graphics_command_buffer, &output_dependency) + vp.output_textures[output_index].layout = .shader_read_only_optimal + + // Signal the application command buffer after the decode queue completes. + vk.cmd_set_event(command_buffer_info.graphics_command_buffer, vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands)) +} + +fn (mut vp VideoPlayer) copy_decoded_frame_to_output(command_buffer vk.CommandBuffer, output_index int) { + decode_family := vp.app.device_context.get_decoder_queue_family_index() + mut output := &vp.output_textures[output_index] + coincide := vp.decoder.properties.dpb_and_output_coincide + mut source_image := vp.dpb.image[vp.dpb.current_slot].image + mut source_state := &vp.dpb.resource_state[vp.dpb.current_slot] + mut source_restore_layout := vk.ImageLayout.video_decode_dpb_khr + if !coincide { + source_image = vp.decode_output_image.image + source_state = &vp.decode_output_state + source_restore_layout = .video_decode_dst_khr + } + mut barriers := [ + vk.ImageMemoryBarrier2{ + srcStageMask: vk.pipeline_stage_2_video_decode_bit_khr + srcAccessMask: source_state.flag + dstStageMask: vk.pipeline_stage_2_transfer_bit + dstAccessMask: vk.access_2_transfer_read_bit + oldLayout: source_state.layout + newLayout: .transfer_src_optimal + srcQueueFamilyIndex: decode_family + dstQueueFamilyIndex: decode_family + image: source_image + subresourceRange: vk.ImageSubresourceRange{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) + levelCount: 1 + layerCount: 1 + } + }, + vk.ImageMemoryBarrier2{ + srcStageMask: vk.pipeline_stage_2_transfer_bit + dstStageMask: vk.pipeline_stage_2_transfer_bit + dstAccessMask: vk.access_2_transfer_write_bit + oldLayout: if output.is_new { + vk.ImageLayout.undefined + } else { + output.layout + } + newLayout: .transfer_dst_optimal + srcQueueFamilyIndex: decode_family + dstQueueFamilyIndex: decode_family + image: output.texture.image + subresourceRange: vk.ImageSubresourceRange{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) + levelCount: 1 + layerCount: 1 + } + }, + ] + dependency := vk.DependencyInfo{ + imageMemoryBarrierCount: u32(barriers.len) + pImageMemoryBarriers: barriers.data + } + vk.cmd_pipeline_barrier2(command_buffer, &dependency) + source_state.layout = .transfer_src_optimal + source_state.flag = vk.access_2_transfer_read_bit + + mut regions := [ + vk.ImageCopy2{ + srcSubresource: vk.ImageSubresourceLayers{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.plane0) + layerCount: 1 + } + dstSubresource: vk.ImageSubresourceLayers{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.plane0) + layerCount: 1 + } + extent: vk.Extent3D{ + width: vp.decoder.video_data.width + height: vp.decoder.video_data.height + depth: 1 + } + }, + vk.ImageCopy2{ + srcSubresource: vk.ImageSubresourceLayers{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.plane1) + layerCount: 1 + } + dstSubresource: vk.ImageSubresourceLayers{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.plane1) + layerCount: 1 + } + extent: vk.Extent3D{ + width: vp.decoder.video_data.width / 2 + height: vp.decoder.video_data.height / 2 + depth: 1 + } + }, + ] + copy_info := vk.CopyImageInfo2{ + srcImage: source_image + srcImageLayout: .transfer_src_optimal + dstImage: output.texture.image + dstImageLayout: .transfer_dst_optimal + regionCount: u32(regions.len) + pRegions: regions.data + } + vk.cmd_copy_image2(command_buffer, ©_info) + output.layout = .transfer_dst_optimal + output.is_new = false + + post_barrier := vk.ImageMemoryBarrier2{ + srcStageMask: vk.pipeline_stage_2_transfer_bit + srcAccessMask: vk.access_2_transfer_read_bit + dstStageMask: vk.pipeline_stage_2_video_decode_bit_khr + dstAccessMask: vk.access_2_video_decode_read_bit_khr + oldLayout: .transfer_src_optimal + newLayout: source_restore_layout + srcQueueFamilyIndex: decode_family + dstQueueFamilyIndex: decode_family + image: source_image + subresourceRange: vk.ImageSubresourceRange{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) + levelCount: 1 + layerCount: 1 + } + } + mut post_barriers := [post_barrier] + if !coincide { + post_barriers << vk.ImageMemoryBarrier2{ + srcStageMask: vk.pipeline_stage_2_video_decode_bit_khr + srcAccessMask: vk.access_2_video_decode_write_bit_khr + dstStageMask: vk.pipeline_stage_2_video_decode_bit_khr + dstAccessMask: vk.access_2_video_decode_read_bit_khr + oldLayout: .video_decode_dpb_khr + newLayout: .video_decode_dpb_khr + srcQueueFamilyIndex: decode_family + dstQueueFamilyIndex: decode_family + image: vp.dpb.image[vp.dpb.current_slot].image + subresourceRange: vk.ImageSubresourceRange{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) + levelCount: 1 + layerCount: 1 + } + } + } + post_dependency := vk.DependencyInfo{ + imageMemoryBarrierCount: u32(post_barriers.len) + pImageMemoryBarriers: post_barriers.data + } + vk.cmd_pipeline_barrier2(command_buffer, &post_dependency) + source_state.layout = source_restore_layout + source_state.flag = if coincide { + vk.access_2_video_decode_read_bit_khr + } else { + vk.access_2_video_decode_write_bit_khr + } + if !coincide { + vp.dpb.resource_state[vp.dpb.current_slot].layout = .video_decode_dpb_khr + vp.dpb.resource_state[vp.dpb.current_slot].flag = vk.access_2_video_decode_read_bit_khr + } +} + +fn (mut vp VideoPlayer) video_decode_pre_barrier(video_command_buffer vk.CommandBuffer) { + mut image_barriers := []vk.ImageMemoryBarrier2{} + decode_family := vp.app.device_context.get_decoder_queue_family_index() + mut current_state := &vp.dpb.resource_state[vp.dpb.current_slot] + if current_state.layout != .video_decode_dpb_khr || current_state.flag != vk.access_2_video_decode_write_bit_khr { + barrier := vk.ImageMemoryBarrier2{ + srcStageMask: vk.pipeline_stage_2_video_decode_bit_khr + srcAccessMask: current_state.flag + dstStageMask: vk.pipeline_stage_2_video_decode_bit_khr + dstAccessMask: vk.access_2_video_decode_write_bit_khr + oldLayout: current_state.layout + newLayout: .video_decode_dpb_khr + srcQueueFamilyIndex: decode_family + dstQueueFamilyIndex: decode_family + image: vp.dpb.image[vp.dpb.current_slot].image + subresourceRange: vk.ImageSubresourceRange{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) + baseMipLevel: 0 + levelCount: 1 + baseArrayLayer: 0 + layerCount: 1 + } + } + image_barriers << barrier + current_state.layout = barrier.newLayout + current_state.flag = barrier.dstAccessMask + } + if !vp.decoder.properties.dpb_and_output_coincide && (vp.decode_output_state.layout != .video_decode_dst_khr || vp.decode_output_state.flag != vk.access_2_video_decode_write_bit_khr) { + output_barrier := vk.ImageMemoryBarrier2{ + srcStageMask: vk.pipeline_stage_2_all_commands_bit + srcAccessMask: vp.decode_output_state.flag + dstStageMask: vk.pipeline_stage_2_video_decode_bit_khr + dstAccessMask: vk.access_2_video_decode_write_bit_khr + oldLayout: vp.decode_output_state.layout + newLayout: .video_decode_dst_khr + srcQueueFamilyIndex: decode_family + dstQueueFamilyIndex: decode_family + image: vp.decode_output_image.image + subresourceRange: vk.ImageSubresourceRange{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) + levelCount: 1 + layerCount: 1 + } + } + image_barriers << output_barrier + vp.decode_output_state.layout = output_barrier.newLayout + vp.decode_output_state.flag = output_barrier.dstAccessMask + } + for ref_index in vp.dpb.reference_usage { + mut ref_state := &vp.dpb.resource_state[ref_index] + if ref_state.layout != .video_decode_dpb_khr || ref_state.flag != vk.access_2_video_decode_read_bit_khr { + barrier := vk.ImageMemoryBarrier2{ + srcStageMask: vk.pipeline_stage_2_video_decode_bit_khr + srcAccessMask: ref_state.flag + dstStageMask: vk.pipeline_stage_2_video_decode_bit_khr + dstAccessMask: vk.access_2_video_decode_read_bit_khr + oldLayout: ref_state.layout + newLayout: .video_decode_dpb_khr + srcQueueFamilyIndex: decode_family + dstQueueFamilyIndex: decode_family + image: vp.dpb.image[ref_index].image + subresourceRange: vk.ImageSubresourceRange{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) + baseMipLevel: 0 + levelCount: 1 + baseArrayLayer: 0 + layerCount: 1 + } + } + image_barriers << barrier + ref_state.layout = barrier.newLayout + ref_state.flag = barrier.dstAccessMask + } + } + if image_barriers.len > 0 { + dependency := vk.DependencyInfo{ + imageMemoryBarrierCount: u32(image_barriers.len) + pImageMemoryBarriers: image_barriers.data + } + vk.cmd_pipeline_barrier2(video_command_buffer, &dependency) + } +} + +fn (mut vp VideoPlayer) video_decode_core(operation &DecoderVideoDecodeOperation, command_buffer vk.CommandBuffer) { + slice_header := unsafe { &h264.SliceHeader(operation.slice_header) } + pps := unsafe { &h264.PictureParameterSet(operation.pps) } + frame := vp.decoder.video_data.frame_infos[operation.decoded_frame_index] + mut std_picture := vk.StdVideoDecodeH264PictureInfo{ + pic_parameter_set_id: u8(slice_header.pic_parameter_set_id) + seq_parameter_set_id: u8(pps.seq_parameter_set_id) + frame_num: u16(slice_header.frame_num) + idr_pic_id: u16(slice_header.idr_pic_id) + } + C.vv_set_h264_picture_order_count(&std_picture, operation.poc[0], operation.poc[1]) + std_picture.flags.is_intra = u32(operation.frame_type == .e_intra) + std_picture.flags.is_reference = u32(operation.reference_priority > 0) + C.vv_set_h264_idr_picture_flag(&std_picture, u32(frame.nal_unit_type == 5)) + std_picture.flags.field_pic_flag = slice_header.field_pic_flag + std_picture.flags.bottom_field_flag = slice_header.bottom_field_flag + + mut slot_infos := [slot_count]vk.VideoReferenceSlotInfoKHR{} + mut pictures := [slot_count]vk.VideoPictureResourceInfoKHR{} + mut h264_slots := [slot_count]vk.VideoDecodeH264DpbSlotInfoKHR{} + mut reference_infos := [slot_count]vk.StdVideoDecodeH264ReferenceInfo{} + for i in 0 .. int(operation.dpb_slot_num) { + pictures[i] = vk.VideoPictureResourceInfoKHR{ + codedExtent: vk.Extent2D{ + width: vp.decoder.video_data.width + height: vp.decoder.video_data.height + } + baseArrayLayer: 0 + imageViewBinding: vp.dpb.image[i].view + } + C.vv_set_h264_reference_info(&reference_infos[i], u16(unsafe { operation.dpb_frame_num[i] }), unsafe { operation.dpb_poc[i] }, unsafe { operation.dpb_poc[i] }) + h264_slots[i] = vk.VideoDecodeH264DpbSlotInfoKHR{ + pStdReferenceInfo: unsafe { &reference_infos[i] } + } + slot_infos[i] = vk.VideoReferenceSlotInfoKHR{ + pNext: unsafe { &h264_slots[i] } + slotIndex: i + pPictureResource: unsafe { &pictures[i] } + } + } + + mut active_slots := [slot_count]vk.VideoReferenceSlotInfoKHR{} + for i in 0 .. int(operation.dpb_reference_count) { + ref_slot := unsafe { operation.dpb_reference_slots[i] } + active_slots[i] = slot_infos[ref_slot] + } + active_slots[operation.dpb_reference_count] = slot_infos[operation.current_dpb] + active_slots[operation.dpb_reference_count].slotIndex = -1 + begin_info := vk.VideoBeginCodingInfoKHR{ + videoSession: vp.decoder.video_session + videoSessionParameters: vp.decoder.video_session_parameters + referenceSlotCount: operation.dpb_reference_count + 1 + pReferenceSlots: unsafe { &active_slots[0] } + } + vk.cmd_begin_video_coding_khr(command_buffer, &begin_info) + if (operation.flags & u32(DecoderVideoDecodeOperationFlags.e_session_reset)) != 0 { + control_info := vk.VideoCodingControlInfoKHR{ + flags: vk.VideoCodingControlFlagsKHR(vk.VideoCodingControlFlagBitsKHR.reset) + } + vk.cmd_control_video_coding_khr(command_buffer, &control_info) + } + + mut h264_picture := vk.VideoDecodeH264PictureInfoKHR{ + pStdPictureInfo: &std_picture + sliceCount: operation.slice_count + pSliceOffsets: operation.slice_offsets + } + dst_picture := if vp.decoder.properties.dpb_and_output_coincide { + pictures[operation.current_dpb] + } else { + vk.VideoPictureResourceInfoKHR{ + codedExtent: vk.Extent2D{ + width: vp.decoder.video_data.width + height: vp.decoder.video_data.height + } + baseArrayLayer: 0 + imageViewBinding: vp.decode_output_image.view + } + } + mut decode_info := vk.VideoDecodeInfoKHR{ + pNext: &h264_picture + srcBuffer: vp.decoder.gpu_bitstream_buffer + srcBufferOffset: operation.stream_offset + srcBufferRange: operation.stream_size + dstPictureResource: dst_picture + pSetupReferenceSlot: unsafe { &slot_infos[operation.current_dpb] } + referenceSlotCount: operation.dpb_reference_count + pReferenceSlots: if operation.dpb_reference_count == 0 { + unsafe { nil } + } else { + unsafe { &active_slots[0] } + } + } + vk.cmd_decode_video_khr(command_buffer, &decode_info) + end_info := vk.VideoEndCodingInfoKHR{} + vk.cmd_end_video_coding_khr(command_buffer, &end_info) +} + +fn (mut vp VideoPlayer) write_video_frame(mut frame VideoPlayerDecodeStreamFrame) ! { + data_frame := vp.decoder.video_data.frame_infos[vp.current_frame] + mut frame_bytes_num_to_do := data_frame.frame_bytes_num + lock vp.decoder { + vp.decoder.video_data.file.seek(data_frame.src_offset, .start) or { + return error('could not seek to MP4 frame ${vp.current_frame}: ${err}') + } + } + for frame_bytes_num_to_do > 0 { + if frame_bytes_num_to_do < 4 { + return error('MP4 frame ${vp.current_frame} has a truncated H.264 NAL length') + } + mut src_buffer := []u8{len: 4} + mut length_bytes_read := 0 + lock vp.decoder { + length_bytes_read = vp.decoder.video_data.file.read(mut src_buffer) or { + return error('could not read H.264 NAL length in MP4 frame ${vp.current_frame}: ${err}') + } + } + if length_bytes_read != src_buffer.len { + return error('short read of H.264 NAL length in MP4 frame ${vp.current_frame}: expected 4 bytes, read ${length_bytes_read}') + } + mut size := u32(src_buffer[0]) << 24 | u32(src_buffer[1]) << 16 | u32(src_buffer[2]) << 8 | src_buffer[3] + size += 4 + if size < 4 || frame_bytes_num_to_do < size { + return error('MP4 frame ${vp.current_frame} has an invalid H.264 NAL size ${size - 4}') + } + mut file := File(os.File{}) + mut nal_header_byte := u8(0) + lock vp.decoder { + file = vp.decoder.video_data.file + nal_header_byte = file.peek()! + } + mut bs := h264.Bitstream{} + bs.init([nal_header_byte]) + mut nal := h264.NetworkAbstractionLayerHeader{} + nal.read_nal_header(mut bs) + // Skip over any frame data that is not idr slice or non-idr slice + if nal.type != h264.NAL_UNIT_TYPE.coded_slice_idr && nal.type != h264.NAL_UNIT_TYPE.coded_slice_non_idr { + frame_bytes_num_to_do -= size + lock vp.decoder { + vp.decoder.video_data.file.seek(size - 4, .current) or { + return error('could not skip non-slice NAL in MP4 frame ${vp.current_frame}: ${err}') + } + } + continue + } + + if frame.gpu_bitstream_size + size <= frame.gpu_bitstream_capacity { + nal_start_code := h264.NalStartCode{}.value + frame.slice_offsets << u32(frame.gpu_bitstream_size) + dst_buffer := unsafe { + frame.gpu_bitstream_slice_mapped_memory_address + frame.gpu_bitstream_size + } + lock vp.decoder { + unsafe { vmemcpy(dst_buffer, nal_start_code.data, nal_start_code.len) } + bytes_read := vp.decoder.video_data.file.read_into_ptr(unsafe { + dst_buffer + nal_start_code.len + }, int(size - 4)) or { + return error('could not read H.264 NAL payload in MP4 frame ${vp.current_frame}: ${err}') + } + if bytes_read != int(size - 4) { + return error('short read of H.264 NAL payload in MP4 frame ${vp.current_frame}: expected ${size - 4} bytes, read ${bytes_read}') + } + } + frame.gpu_bitstream_size += u64(nal_start_code.len) + size - 4 + } else { + return error('encoded access unit ${vp.current_frame} requires more than its ${frame.gpu_bitstream_capacity}-byte aligned bitstream-buffer capacity (written=${frame.gpu_bitstream_size}, next_nal=${size})') + } + frame_bytes_num_to_do -= size + } + lock vp.decoder { + aligned_size := U64(frame.gpu_bitstream_size).align_to(vp.decoder.properties.caps.minBitstreamBufferSizeAlignment) + if aligned_size > frame.gpu_bitstream_capacity { + return error('aligned access unit ${vp.current_frame} exceeds its ${frame.gpu_bitstream_capacity}-byte bitstream-buffer capacity') + } + if aligned_size > frame.gpu_bitstream_size { + unsafe { + vmemset(frame.gpu_bitstream_slice_mapped_memory_address + frame.gpu_bitstream_size, 0, isize(aligned_size - frame.gpu_bitstream_size)) + } + } + frame.gpu_bitstream_size = aligned_size + } +} + +type File = os.File + +// Return next byte in file, but don't move the cursor position +fn (mut f File) peek() !u8 { + pos_bk := f.tell()! + mut ret := []u8{len: 1} + bytes_read := f.read(mut ret)! + f.seek(pos_bk, .start)! + if bytes_read != 1 { + return error('unexpected end of file while reading H.264 NAL header') + } + return ret[0] +} + +fn has_flag[T](lhs u32, rhs T) bool { + return (lhs & u32(rhs)) == u32(rhs) +} + +fn (d Decoder) get_slice_header() byteptr { + return d.video_data.slice_header_bytes.data +} + +fn (d Decoder) get_pps() byteptr { + return d.video_data.pps_bytes.data +} + +fn (d Decoder) get_sps() byteptr { + return d.video_data.sps_bytes.data +} diff --git a/player_presentation.v b/player_presentation.v new file mode 100644 index 0000000..d6e108f --- /dev/null +++ b/player_presentation.v @@ -0,0 +1,180 @@ +module main + +import antono2.vulkan as vk + +fn (vp &VideoPlayer) ready_output_position(display_order int) ?int { + for position, output_index in vp.output_textures_ready { + if vp.output_textures[output_index].display_order == display_order { + return position + } + } + return none +} + +fn (mut vp VideoPlayer) reclaim_output_textures() { + mut position := 0 + for position < vp.output_textures_retired.len { + retired := vp.output_textures_retired[position] + if retired.reusable_after_serial > vp.render_serial { + position++ + continue + } + vp.output_textures[retired.index].display_order = -1 + vp.output_textures[retired.index].duration_ns = 0 + vp.output_textures_free << retired.index + vp.output_textures_retired.delete(position) + } +} + +fn (mut vp VideoPlayer) retire_current_output() { + if vp.current_output_index < 0 { + return + } + // The application has one fence per swapchain image. Once this many later + // render submissions have begun, the last submission sampling this image is + // guaranteed to have completed on the ordered graphics queue. + vp.output_textures_retired << RetiredOutputImage{ + index: vp.current_output_index + reusable_after_serial: vp.render_serial + u64(vp.command_buffer_infos.len + 1) + } +} + +fn (mut vp VideoPlayer) present_ready_output(display_order int) bool { + position := vp.ready_output_position(display_order) or { return false } + output_index := vp.output_textures_ready[position] + vp.output_textures_ready.delete(position) + vp.retire_current_output() + vp.current_output_index = output_index + vp.next_display_order = display_order + 1 + vp.waiting_for_loop_start = false + vp.playback_timeline.present_frame(vp.output_textures[output_index].duration_ns) + $if debug { + if display_order % 100 == 0 { + eprintln('Presented frame ${display_order}') + } + } + return true +} + +fn (mut vp VideoPlayer) restart_decode_cycle() { + $if debug { + println('Playback loop completed in display order') + } + vp.current_frame = 0 + vp.next_display_order = 0 + vp.decode_finished = false + vp.waiting_for_loop_start = true + vp.dpb.reference_usage.clear() + vp.flags |= u32(VideoPlayerFlags.e_decoder_reset) + vp.playback_timeline.reset() +} + +fn (mut vp VideoPlayer) update_presentation() { + if vp.current_output_index < 0 || vp.waiting_for_loop_start { + // Prime enough codec-order pictures to absorb the stream's maximum + // reordering delay. No-B-frame streams have a target of one and still + // display their first decoded picture immediately. + if !vp.decode_finished && vp.output_textures_ready.len < vp.presentation_buffer_count { + return + } + vp.present_ready_output(vp.next_display_order) + return + } + if !vp.playback_timeline.frame_is_due() { + return + } + frame_count := vp.decoder.video_data.frame_infos.len + if vp.next_display_order < frame_count { + vp.present_ready_output(vp.next_display_order) + return + } + if !vp.decode_finished { + return + } + if vp.is_looping { + // Keep the last picture visible while the first pictures of the next loop + // are decoded and reordered. + vp.restart_decode_cycle() + } else { + vp.is_stopped = true + } +} + +fn (vp &VideoPlayer) current_output_view() vk.ImageView { + if vp.current_output_index < 0 || vp.current_output_index >= vp.output_textures.len { + return unsafe { nil } + } + return vp.output_textures[vp.current_output_index].texture.view +} + +fn (mut vp VideoPlayer) update(graphics_cmd_buffer vk.CommandBuffer, time_elapsed_ns i64) { + vp.decode_operation = DecoderVideoDecodeOperation{} + vp.render_serial++ + $if debug { + if vp.render_serial <= 5 { + eprintln('Playback update ${vp.render_serial}: ready=${vp.output_textures_ready.len}, free=${vp.output_textures_free.len}') + } + } + vp.reclaim_output_textures() + if vp.current_output_index >= 0 && !vp.waiting_for_loop_start { + vp.playback_timeline.tick(time_elapsed_ns) + } + + if vp.is_stopped { + vp.dpb_slot_used = []int{len: int(vp.decoder.video_data.max_reference_pictures), init: 0} + return + } + should_decode := !vp.decode_finished + && vp.output_textures_ready.len < vp.presentation_buffer_count + && vp.output_textures_free.len > 0 + if !should_decode { + vp.update_presentation() + return + } + + vp.update_decode_video() or { + eprintln('Playback stopped: ${err}') + vp.is_stopped = true + return + } + $if debug { + if vp.render_serial <= 5 { + eprintln('Decoded access unit ${vp.current_frame}: ready=${vp.output_textures_ready.len}') + } + } + vp.update_presentation() + + vk.cmd_wait_events(graphics_cmd_buffer, 1, &vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands), vk.PipelineStageFlags(vk.PipelineStageFlagBits.all_commands), 0, unsafe { nil }, 0, unsafe { nil }, 0, unsafe { nil }) + vk.cmd_reset_event(graphics_cmd_buffer, vp.event_video_player, vk.PipelineStageFlags(vk.PipelineStageFlagBits.bottom_of_pipe)) + + // Finish recording the command buffer and submit + dev_ctx := vp.app.device_context + index_cur_swapchain := dev_ctx.swapchain.get_current_index() + command_buffer_info := vp.command_buffer_infos[index_cur_swapchain] + vk.end_command_buffer(command_buffer_info.graphics_command_buffer) + + mut semaphore := command_buffer_info.sem_video_to_gfx + mut wait_stage := vk.PipelineStageFlags(vk.PipelineStageFlagBits.top_of_pipe) + mut sumbit_info_video := vk.SubmitInfo{ + waitSemaphoreCount: 0 + pWaitSemaphores: unsafe { nil } + pWaitDstStageMask: &wait_stage + commandBufferCount: 1 + pCommandBuffers: &command_buffer_info.video_command_buffer + signalSemaphoreCount: 1 + pSignalSemaphores: &semaphore + } + upload_fence := vp.video_frames[vp.current_upload_index].in_flight_fence + res_video := vk.queue_submit(dev_ctx.get_queue(.video_decode), 1, &sumbit_info_video, upload_fence) + check_vk(res_video, 'Could not submit Vulkan Video decode command') + + mut sumbit_info_graphics := vk.SubmitInfo{ + waitSemaphoreCount: 1 + pWaitSemaphores: &semaphore + pWaitDstStageMask: &wait_stage + commandBufferCount: 1 + pCommandBuffers: &command_buffer_info.graphics_command_buffer + } + res_graphics := vk.queue_submit(dev_ctx.get_queue(.graphics), 1, &sumbit_info_graphics, unsafe { nil }) + check_vk(res_graphics, 'Could not submit decoded frame for graphics use') +} diff --git a/scripts/build_windows.ps1 b/scripts/build_windows.ps1 index 342e223..1dc08c2 100644 --- a/scripts/build_windows.ps1 +++ b/scripts/build_windows.ps1 @@ -82,7 +82,7 @@ Push-Location $ProjectDirectory try { $VArguments = @() if ($Compiler -eq "v3") { $VArguments += "-new-compiler" } - $VArguments += @("-cc", "msvc", "-cflags", "/MT", "-o", $Executable, ".") + $VArguments += @("-subsystem", "console", "-cc", "msvc", "-cflags", "/MT", "-o", $Executable, ".") & v @VArguments if ($LASTEXITCODE -ne 0) { throw "V application build failed." } } finally { diff --git a/examples/video_decode_app/swapchain.v b/swapchain.v similarity index 78% rename from examples/video_decode_app/swapchain.v rename to swapchain.v index 5836dc4..2e3c6bc 100644 --- a/examples/video_decode_app/swapchain.v +++ b/swapchain.v @@ -1,29 +1,29 @@ -module video_decode_app +module main import antono2.vulkan as vk import math import antono2.glfw -pub struct Swapchain { +struct Swapchain { mut: - swapchain vk.SwapchainKHR = unsafe { nil } + swapchain vk.SwapchainKHR = unsafe { nil } present_mode vk.PresentModeKHR = vk.PresentModeKHR.fifo images []vk.Image current_index u32 = u32(0) pub mut: image_views []vk.ImageView - app &VideoDecodeApp = unsafe { nil } // @[required] - surface vk.SurfaceKHR = unsafe { nil } + app &VideoDecodeApp = unsafe { nil } // @[required] + surface vk.SurfaceKHR = unsafe { nil } sampler_ycbcr_conversion vk.SamplerYcbcrConversion = unsafe { nil } - surface_format vk.SurfaceFormatKHR = vk.SurfaceFormatKHR{ - format: vk.Format.undefined + surface_format vk.SurfaceFormatKHR = vk.SurfaceFormatKHR{ + format: vk.Format.undefined colorSpace: vk.ColorSpaceKHR.srgb_nonlinear } image_count u32 = u32(2) extent_2d vk.Extent2D } -pub fn (mut sc Swapchain) initialize_surface(window_p &glfw.Window) { +fn (mut sc Swapchain) initialize_surface(window_p &glfw.Window) { if !isnil(sc.surface) { return } @@ -34,7 +34,7 @@ pub fn (mut sc Swapchain) initialize_surface(window_p &glfw.Window) { } } -pub fn (mut sc Swapchain) initialize(window_p &glfw.Window, desired_format vk.Format) bool { +fn (mut sc Swapchain) initialize(window_p &glfw.Window, desired_format vk.Format) bool { sc.initialize_surface(window_p) mut n := unsafe { nil } device_context := sc.app.device_context @@ -88,7 +88,7 @@ pub fn (mut sc Swapchain) initialize(window_p &glfw.Window, desired_format vk.Fo return sc.resize(swapchain_size) } -pub fn (mut sc Swapchain) resize(extent vk.Extent2D) bool { +fn (mut sc Swapchain) resize(extent vk.Extent2D) bool { device_context := sc.app.device_context mut surface_caps := vk.SurfaceCapabilitiesKHR{} vk.get_physical_device_surface_capabilities_khr(device_context.get_gpu_current(), sc.surface, mut &surface_caps) @@ -101,19 +101,19 @@ pub fn (mut sc Swapchain) resize(extent vk.Extent2D) bool { } mut old_swapchain := sc.swapchain mut swapchain_ci := vk.SwapchainCreateInfoKHR{ - surface: sc.surface - minImageCount: sc.image_count - imageFormat: sc.surface_format.format - imageColorSpace: sc.surface_format.colorSpace - imageExtent: actual_extent + surface: sc.surface + minImageCount: sc.image_count + imageFormat: sc.surface_format.format + imageColorSpace: sc.surface_format.colorSpace + imageExtent: actual_extent imageArrayLayers: 1 - imageUsage: vk.ImageUsageFlags(vk.ImageUsageFlagBits.color_attachment) + imageUsage: vk.ImageUsageFlags(vk.ImageUsageFlagBits.color_attachment) imageSharingMode: vk.SharingMode.exclusive - preTransform: surface_caps.currentTransform - compositeAlpha: vk.CompositeAlphaFlagBitsKHR.opaque - presentMode: sc.present_mode - clipped: vk._true - oldSwapchain: old_swapchain + preTransform: surface_caps.currentTransform + compositeAlpha: vk.CompositeAlphaFlagBitsKHR.opaque + presentMode: sc.present_mode + clipped: vk._true + oldSwapchain: old_swapchain } vk_device := device_context.vk_device @@ -137,21 +137,21 @@ pub fn (mut sc Swapchain) resize(extent vk.Extent2D) bool { sc.image_count = image_count for i in 0 .. image_count { mut view_ci := vk.ImageViewCreateInfo{ - image: sc.images[i] - viewType: vk.ImageViewType._2d - format: sc.surface_format.format - components: vk.ComponentMapping{ + image: sc.images[i] + viewType: vk.ImageViewType._2d + format: sc.surface_format.format + components: vk.ComponentMapping{ r: vk.ComponentSwizzle.r g: vk.ComponentSwizzle.g b: vk.ComponentSwizzle.b a: vk.ComponentSwizzle.a } subresourceRange: vk.ImageSubresourceRange{ - aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) - baseMipLevel: 0 - levelCount: 1 + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) + baseMipLevel: 0 + levelCount: 1 baseArrayLayer: 0 - layerCount: 1 + layerCount: 1 } } view_result := vk.create_image_view(vk_device, &view_ci, unsafe { nil }, &sc.image_views[i]) @@ -164,11 +164,11 @@ pub fn (mut sc Swapchain) resize(extent vk.Extent2D) bool { return true } -pub fn (sc Swapchain) get_handle() vk.SwapchainKHR { +fn (sc Swapchain) get_handle() vk.SwapchainKHR { return sc.swapchain } -pub fn (mut sc Swapchain) acquire_next_image(mut sem_present_complete vk.Semaphore) vk.Result { +fn (mut sc Swapchain) acquire_next_image(mut sem_present_complete vk.Semaphore) vk.Result { device_context := sc.app.device_context mut vk_device := device_context.get_vk_device() mut index := u32(0) @@ -183,11 +183,11 @@ pub fn (mut sc Swapchain) acquire_next_image(mut sem_present_complete vk.Semapho return res } -pub fn (sc Swapchain) get_current_index() u32 { +fn (sc Swapchain) get_current_index() u32 { return sc.current_index } -pub fn (mut sc Swapchain) shutdown() { +fn (mut sc Swapchain) shutdown() { device_context := sc.app.device_context mut vk_device := device_context.get_vk_device() // Image views must be released before the swapchain images they reference. diff --git a/v.mod b/v.mod index 41e5ffe..f2c5c5c 100644 --- a/v.mod +++ b/v.mod @@ -1,6 +1,6 @@ Module { - name: 'vulkan' - description: 'Video with V and Vulkan' + name: 'v_vulkan_video' + description: 'H.264 MP4 player using Vulkan Video decode' version: '0.0.0' license: 'MIT' dependencies: [ diff --git a/examples/video_decode_app/vertex_shader.v b/vertex_shader.v similarity index 99% rename from examples/video_decode_app/vertex_shader.v rename to vertex_shader.v index acb1467..29c84e2 100644 --- a/examples/video_decode_app/vertex_shader.v +++ b/vertex_shader.v @@ -1,7 +1,7 @@ -module video_decode_app +module main // Generated from video.vert with glslc. Do not edit the words by hand. -pub const g_vertex_shader = [ +const g_vertex_shader = [ u32(0x07230203), 0x00010000, 0x000d000b, 0x00000060, 0x00000000, 0x00020011, 0x00000001, 0x0006000b, 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x00000000, 0x00000001, 0x0009000f, 0x00000000, 0x00000004, 0x6e69616d, 0x00000000, 0x00000016, 0x0000001d, 0x0000003a, diff --git a/examples/video_decode_app/video.vert b/video.vert similarity index 100% rename from examples/video_decode_app/video.vert rename to video.vert diff --git a/examples/video_decode_app/video_player.c.v b/video_player.c.v similarity index 95% rename from examples/video_decode_app/video_player.c.v rename to video_player.c.v index 6e74167..e83c8cf 100644 --- a/examples/video_decode_app/video_player.c.v +++ b/video_player.c.v @@ -1,4 +1,4 @@ -module video_decode_app +module main #flag -I @VMODROOT/include #include "video_bridge.h" diff --git a/video_player.v b/video_player.v new file mode 100644 index 0000000..00efc6c --- /dev/null +++ b/video_player.v @@ -0,0 +1,778 @@ +module main + +import antono2.vulkan as vk +import os +import math +import antono2.vkmemalloc as vkmem + +const max_texture_count = 64 +const slot_count = 17 + +enum DecodeOutputMode { + automatic + coincident + distinct +} + +fn select_decode_output_mode(requested DecodeOutputMode, supports_coincident bool, + supports_distinct bool) !DecodeOutputMode { + return match requested { + .automatic { + if supports_coincident { + DecodeOutputMode.coincident + } else if supports_distinct { + DecodeOutputMode.distinct + } else { + return error('Vulkan Video device supports neither coincident nor distinct DPB/output images') + } + } + .coincident { + if !supports_coincident { + return error('Vulkan Video device does not support forced coincident DPB/output images') + } + DecodeOutputMode.coincident + } + .distinct { + if !supports_distinct { + return error('Vulkan Video device does not support forced distinct DPB/output images') + } + DecodeOutputMode.distinct + } + } +} + +struct VideoPlayer { +mut: + decode_operation DecoderVideoDecodeOperation + output_textures []OutputImage + output_textures_free []int + output_textures_ready []int + output_textures_retired []RetiredOutputImage + current_output_index int = -1 + next_display_order int + presentation_buffer_count int = 1 + decode_finished bool + waiting_for_loop_start bool + render_serial u64 + is_stopped bool + is_looping bool = true + dpb_slot_used []int + dpb DPB + current_frame int + flags u32 + video_frames []VideoPlayerDecodeStreamFrame + decode_output_image Image + decode_output_state DPBResourceState + playback_timeline PlaybackTimeline + current_upload_index int + graphics_command_pool vk.CommandPool = unsafe { nil } + video_command_pool vk.CommandPool = unsafe { nil } +pub mut: + app &VideoDecodeApp = unsafe { nil } + decoder shared Decoder + event_video_player vk.Event + command_buffer_infos []CommandBufferInfo +} + +struct Decoder { +pub mut: + properties DecoderQueryProperties + settings DecoderSettings + video_data DecoderVideoFileProperties + gpu_bitstream_buffer vk.Buffer = unsafe { nil } + gpu_bitstream_allocation vkmem.AllocationInfo + session_memory_allocations []vk.DeviceMemory + video_session vk.VideoSessionKHR = unsafe { nil } + video_session_parameters vk.VideoSessionParametersKHR = unsafe { nil } + info DecoderInfo +} + +struct DecoderDpbState { +pub mut: + slotindex int + frame_num int + reference_info vk.StdVideoDecodeH264ReferenceInfo +} + +struct DecoderInfo { +pub mut: + memory_frames []DecoderVideoMemoryFrameInfo + images_dpb []DecoderDpbImage + dpb_state []DecoderDpbState +} + +struct DecoderVideoMemoryFrameInfo { +pub mut: + data_frame_info &DecoderVideoDataFrameInfo + gpu_bitstream_offset u64 + gpu_bitstream_capacity u64 + gpu_bitstream_size u64 + gpu_bitstream_slice_mapped_memory_address &u8 + decoding_frame_index int = -1 +} + +struct VideoPlayerDecodeStreamFrame { +pub mut: + gpu_bitstream_capacity u64 + gpu_bitstream_offset u64 + gpu_bitstream_size u64 + gpu_bitstream_slice_mapped_memory_address byteptr + slice_offsets []u32 + in_flight_fence vk.Fence = unsafe { nil } +} + +struct CommandBufferInfo { +pub mut: + video_command_buffer vk.CommandBuffer + graphics_command_buffer vk.CommandBuffer + sem_video_to_gfx vk.Semaphore +} + +enum VideoPlayerFlags as u32 { + e_none = 0 + e_playing = 1 << 1 + e_decoder_reset = 1 << 3 + e_need_resolve = 1 << 4 +} + +struct DPB { +pub mut: + image [slot_count]Image + resource_state [slot_count]DPBResourceState + poc_status [slot_count]int + frame_num_status [slot_count]int + reference_usage []u8 + next_ref u8 + next_slot u8 + current_slot u8 +} + +// Choose a slot which is not referenced by the picture being decoded. When all +// slots are references, expire the oldest short-term reference first (the H.264 +// sliding-window default). Explicit MMCO and long-term references are handled +// separately as stream metadata becomes available. +fn (mut dpb DPB) acquire_decode_slot(slot_limit int) u8 { + assert slot_limit > 0 && slot_limit <= slot_count + for slot in 0 .. slot_limit { + if u8(slot) !in dpb.reference_usage { + return u8(slot) + } + } + assert dpb.reference_usage.len > 0 + expired_slot := dpb.reference_usage[0] + dpb.reference_usage.delete(0) + return expired_slot +} + +struct DPBResourceState { +pub mut: + flag vk.AccessFlags2 + layout vk.ImageLayout +} + +struct OutputImage { +pub mut: + display_order int = -1 + texture Image + duration_ns i64 + layout vk.ImageLayout + is_new bool = true +} + +struct RetiredOutputImage { + index int + reusable_after_serial u64 +} + +struct Image { +pub mut: + image vk.Image + view vk.ImageView + allocation vkmem.Allocator + allocation_info vkmem.AllocationInfo +} + +struct DecoderQueryProperties { +pub mut: + decode_h264_caps vk.VideoDecodeH264CapabilitiesKHR + decode_caps vk.VideoDecodeCapabilitiesKHR + caps vk.VideoCapabilitiesKHR + format_props vk.VideoFormatPropertiesKHR + dpb_format_props vk.VideoFormatPropertiesKHR + dpb_and_output_coincide bool + usage_dpb vk.ImageUsageFlags +} + +struct DecoderSettings { +pub mut: + decode_h264_profile_info vk.VideoDecodeH264ProfileInfoKHR + profile_info vk.VideoProfileInfoKHR + profile_list_info vk.VideoProfileListInfoKHR +} + +enum DecoderFrameType as u8 { + e_unknown = 0 + e_intra + e_predictive +} + +struct DecoderVideoDataFrameInfo { +pub mut: + src_offset u64 + frame_bytes_num u64 + size u64 + poc int + bottom_field_order_cnt u32 + top_field_order_cnt u32 + gop int + display_order int + decode_time_ns i64 + display_time_ns i64 + duration_ns i64 + nal_unit_type u8 + frame_type DecoderFrameType + nal_ref_idc u32 + reference_priority u32 +} + +fn compare_frame_display_order(a &DecoderVideoDataFrameInfo, b &DecoderVideoDataFrameInfo) int { + key_a := u64(a.gop) << 32 | u64(a.poc) + key_b := u64(b.gop) << 32 | u64(b.poc) + if key_a < key_b { + return -1 + } + if key_a > key_b { + return 1 + } + return 0 +} + +// presentation_buffer_size returns the number of decoded pictures which may +// have to wait while an earlier display-order picture is still being decoded. +// One additional slot is reserved for that missing picture itself. +fn presentation_buffer_size(display_orders []int) int { + mut waiting := map[int]bool{} + mut next_display_order := 0 + mut max_waiting := 0 + for display_order in display_orders { + waiting[display_order] = true + for waiting[next_display_order] { + waiting.delete(next_display_order) + next_display_order++ + } + max_waiting = math.max(max_waiting, waiting.len) + } + return max_waiting + 1 +} + +struct DecoderVideoFileProperties { +pub mut: + file os.File + file_open bool + h264_profile_idc u32 + width_padd u32 + height_padd u32 + width u32 + height u32 + sps_count u32 + pps_count u32 + slice_header_count u32 + + frame_infos []DecoderVideoDataFrameInfo + max_memory_frame_size_bytes u64 + num_dpb_slots u32 + max_reference_pictures u32 + + sps_bytes []u8 + pps_bytes []u8 + slice_header_bytes []u8 + frame_display_order []u64 + + total_duration i64 + metadata VideoMetadata +} + +struct VideoMetadata { +pub mut: + coded_width u32 + coded_height u32 + display_width u32 + display_height u32 + sar_width u32 = 1 + sar_height u32 = 1 + rotation_degrees int + track_matrix [9]i32 + video_full_range bool + colour_description_present bool + colour_primaries u8 + transfer_function u8 + matrix_coefficients u8 +} + +fn rotation_from_track_matrix(matrix [9]i32) int { + a := matrix[0] + b := matrix[1] + c := matrix[3] + d := matrix[4] + if a < 0 && d < 0 { + return 180 + } + if b > 0 && c < 0 { + return -90 + } + if b < 0 && c > 0 { + return 90 + } + return 0 +} + +fn h264_profile_name(profile_idc u32) string { + return match profile_idc { + 66 { 'Baseline' } + 77 { 'Main' } + 100 { 'High' } + else { 'Unknown (${profile_idc})' } + } +} + +fn sample_aspect_ratio(aspect_ratio_idc u32, extended_width u32, extended_height u32) (u32, u32) { + if aspect_ratio_idc == 255 { + if extended_width > 0 && extended_height > 0 { + return extended_width, extended_height + } + return 1, 1 + } + return match aspect_ratio_idc { + 1 { u32(1), u32(1) } + 2 { u32(12), u32(11) } + 3 { u32(10), u32(11) } + 4 { u32(16), u32(11) } + 5 { u32(40), u32(33) } + 6 { u32(24), u32(11) } + 7 { u32(20), u32(11) } + 8 { u32(32), u32(11) } + 9 { u32(80), u32(33) } + 10 { u32(18), u32(11) } + 11 { u32(15), u32(11) } + 12 { u32(64), u32(33) } + 13 { u32(160), u32(99) } + 14 { u32(4), u32(3) } + 15 { u32(3), u32(2) } + 16 { u32(2), u32(1) } + else { u32(1), u32(1) } + } +} + +fn (mut metadata VideoMetadata) update_display_dimensions() { + mut width := u64(metadata.coded_width) * u64(metadata.sar_width) / u64(metadata.sar_height) + mut height := u64(metadata.coded_height) + if metadata.rotation_degrees in [90, -90] { + width, height = height, width + } + metadata.display_width = u32(width) + metadata.display_height = u32(height) +} + +struct DecoderDpbImage { +pub mut: + image vk.Image + view vk.ImageView + // TODO: Refactor Allocator to Decoder + allocator vkmem.Allocator + allocation_info vkmem.AllocationInfo +} + +struct DecoderVideoDecodeOperation { +pub mut: + flags u32 + stream_offset u64 + stream_size u64 + frame_type DecoderFrameType = DecoderFrameType.e_intra + reference_priority u32 + decoded_frame_index int + slice_header voidptr = unsafe { nil } + pps voidptr = unsafe { nil } + sps voidptr = unsafe { nil } + poc [2]int + current_dpb u32 + dpb_reference_count u32 + dpb_reference_slots &u8 = unsafe { nil } + dpb_poc &int = unsafe { nil } + dpb_frame_num &int = unsafe { nil } + dpb_slot_num u32 + p_dpbs vk.Image + p_dpb_views vk.ImageView + slice_count u32 + slice_offsets &u32 = unsafe { nil } +} + +enum DecoderVideoDecodeOperationFlags { + e_none = 0 + e_session_reset = 1 +} + +// TODO: May be worth to use interface types, but interfaces IApp containg sub interface IDeviceContext "error: `&video_decode_app.VideoDecodeApp` incorrectly implements field `device_context` of interface `examples.video_decode_app.video_player.IApp`, expected `video_player.IDeviceContext`, got `video_decode_app.DeviceContext`", no matter what's in the interface +fn (mut vp VideoPlayer) prepare(path string) ! { + // Do not propagate parser errors from inside the lock: cleanup must be able + // to reacquire it and close a partially opened input file. + mut parse_error := '' + lock vp.decoder { + vp.decoder.parse_mp4_data(path) or { parse_error = err.msg() } + } + if parse_error != '' { + return error(parse_error) + } +} + +fn (mut vp VideoPlayer) close_input() { + lock vp.decoder { + if vp.decoder.video_data.file_open { + vp.decoder.video_data.file.close() + vp.decoder.video_data.file_open = false + } + } +} + +fn (vp &VideoPlayer) h264_profile_idc() u32 { + rlock vp.decoder { + return vp.decoder.video_data.h264_profile_idc + } +} + +fn (vp &VideoPlayer) metadata() VideoMetadata { + rlock vp.decoder { + return vp.decoder.video_data.metadata + } +} + +fn (mut vp VideoPlayer) initialize(mut app VideoDecodeApp) { + vp.app = app + lock vp.decoder { + vp.decoder.initialize(mut app) + vp.video_frames = []VideoPlayerDecodeStreamFrame{len: vp.decoder.info.memory_frames.len} + for i, frame in vp.decoder.info.memory_frames { + vp.video_frames[i] = VideoPlayerDecodeStreamFrame{ + gpu_bitstream_capacity: frame.gpu_bitstream_capacity + gpu_bitstream_offset: frame.gpu_bitstream_offset + gpu_bitstream_size: frame.gpu_bitstream_size + gpu_bitstream_slice_mapped_memory_address: byteptr(frame.gpu_bitstream_slice_mapped_memory_address) + } + } + if vp.decoder.info.images_dpb.len > vp.dpb.image.len { + panic('Video stream requires too many DPB images') + } + for i, image in vp.decoder.info.images_dpb { + vp.dpb.image[i] = Image{ + image: image.image + view: image.view + allocation_info: image.allocation_info + } + } + vp.presentation_buffer_count = presentation_buffer_size(vp.decoder.video_data.frame_infos.map(it.display_order)) + } + + vk_device := app.device_context.vk_device + for i in 0 .. vp.decoder.info.memory_frames.len { + fence_ci := vk.FenceCreateInfo{ + flags: vk.FenceCreateFlags(vk.FenceCreateFlagBits.signaled) + } + res := vk.create_fence(vk_device, &fence_ci, unsafe { nil }, &vp.video_frames[i].in_flight_fence) + check_vk(res, 'Could not create video-frame fence ${i}') + } + mut command_pool_ci := vk.CommandPoolCreateInfo{ + flags: vk.CommandPoolCreateFlags(vk.CommandPoolCreateFlagBits.reset_command_buffer) + } + command_pool_ci.queueFamilyIndex = app.device_context.graphics_family + mut res := vk.create_command_pool(vk_device, &command_pool_ci, unsafe { nil }, &vp.graphics_command_pool) + check_vk(res, 'Could not create video-player graphics command pool') + command_pool_ci.queueFamilyIndex = app.device_context.get_decoder_queue_family_index() + res = vk.create_command_pool(vk_device, &command_pool_ci, unsafe { nil }, &vp.video_command_pool) + check_vk(res, 'Could not create video-decode command pool') + + vp.command_buffer_infos = []CommandBufferInfo{len: app.device_context.swapchain.image_views.len} + for mut info in vp.command_buffer_infos { + mut alloc_info := vk.CommandBufferAllocateInfo{ + level: .primary + commandBufferCount: 1 + } + alloc_info.commandPool = vp.graphics_command_pool + res = vk.allocate_command_buffers(vk_device, &alloc_info, &info.graphics_command_buffer) + check_vk(res, 'Could not allocate video-player graphics command buffer') + alloc_info.commandPool = vp.video_command_pool + res = vk.allocate_command_buffers(vk_device, &alloc_info, &info.video_command_buffer) + check_vk(res, 'Could not allocate video-decode command buffer') + semaphore_ci := vk.SemaphoreCreateInfo{} + res = vk.create_semaphore(vk_device, &semaphore_ci, unsafe { nil }, &info.sem_video_to_gfx) + check_vk(res, 'Could not create video-to-graphics semaphore') + } + event_ci := vk.EventCreateInfo{} + res = vk.create_event(vk_device, &event_ci, unsafe { nil }, &vp.event_video_player) + check_vk(res, 'Could not create video-player event') + output_texture_count := vp.presentation_buffer_count + vp.command_buffer_infos.len + 2 + if output_texture_count > max_texture_count { + panic('Video requires ${output_texture_count} presentation images, but the player supports at most ${max_texture_count}') + } + decode_family := app.device_context.get_decoder_queue_family_index() + if decode_family != app.device_context.graphics_family { + println('Display image queues: decode family ${decode_family}, graphics family ${app.device_context.graphics_family} (concurrent)') + } else { + println('Display image queue family: ${decode_family} (exclusive)') + } + for _ in 0 .. output_texture_count { + vp.create_output_image() + } + println('Presentation queue: ${vp.presentation_buffer_count} reorder images, ${output_texture_count} images total') + if !vp.decoder.properties.dpb_and_output_coincide { + vp.create_decode_output_image() + } +} + +fn (mut vp VideoPlayer) recreate_swapchain_resources() { + vk_device := vp.app.device_context.vk_device + for mut info in vp.command_buffer_infos { + if !isnil(info.graphics_command_buffer) { + vk.free_command_buffers(vk_device, vp.graphics_command_pool, 1, &info.graphics_command_buffer) + } + if !isnil(info.video_command_buffer) { + vk.free_command_buffers(vk_device, vp.video_command_pool, 1, &info.video_command_buffer) + } + if !isnil(info.sem_video_to_gfx) { + vk.destroy_semaphore(vk_device, info.sem_video_to_gfx, unsafe { nil }) + } + } + vp.command_buffer_infos = []CommandBufferInfo{len: vp.app.device_context.swapchain.image_views.len} + for mut info in vp.command_buffer_infos { + mut alloc_info := vk.CommandBufferAllocateInfo{ + level: .primary + commandBufferCount: 1 + commandPool: vp.graphics_command_pool + } + mut result := vk.allocate_command_buffers(vk_device, &alloc_info, &info.graphics_command_buffer) + check_vk(result, 'Could not reallocate video-player graphics command buffer') + alloc_info.commandPool = vp.video_command_pool + result = vk.allocate_command_buffers(vk_device, &alloc_info, &info.video_command_buffer) + check_vk(result, 'Could not reallocate video-decode command buffer') + result = vk.create_semaphore(vk_device, &vk.SemaphoreCreateInfo{}, unsafe { nil }, &info.sem_video_to_gfx) + check_vk(result, 'Could not recreate video-to-graphics semaphore') + } + // The caller waits for device idle before rebuilding the swapchain, so every + // retired presentation image can be reused immediately. Grow the pool if the + // replacement swapchain has more images than the old one. + for retired in vp.output_textures_retired { + vp.output_textures_free << retired.index + } + vp.output_textures_retired.clear() + required_output_count := vp.presentation_buffer_count + vp.command_buffer_infos.len + 2 + if required_output_count > max_texture_count { + panic('Resized swapchain requires ${required_output_count} presentation images, but the player supports at most ${max_texture_count}') + } + for vp.output_textures.len < required_output_count { + vp.create_output_image() + } +} + +fn (mut vp VideoPlayer) shutdown() { + vk_device := vp.app.device_context.vk_device + for mut info in vp.command_buffer_infos { + if !isnil(info.sem_video_to_gfx) { + vk.destroy_semaphore(vk_device, info.sem_video_to_gfx, unsafe { nil }) + info.sem_video_to_gfx = unsafe { nil } + } + } + vp.command_buffer_infos.clear() + if !isnil(vp.graphics_command_pool) { + vk.destroy_command_pool(vk_device, vp.graphics_command_pool, unsafe { nil }) + vp.graphics_command_pool = unsafe { nil } + } + if !isnil(vp.video_command_pool) { + vk.destroy_command_pool(vk_device, vp.video_command_pool, unsafe { nil }) + vp.video_command_pool = unsafe { nil } + } + if !isnil(vp.event_video_player) { + vk.destroy_event(vk_device, vp.event_video_player, unsafe { nil }) + vp.event_video_player = unsafe { nil } + } + for mut frame in vp.video_frames { + if !isnil(frame.in_flight_fence) { + vk.destroy_fence(vk_device, frame.in_flight_fence, unsafe { nil }) + frame.in_flight_fence = unsafe { nil } + } + } + lock vp.decoder { + for mut output in vp.output_textures { + if !isnil(output.texture.view) { + vk.destroy_image_view(vk_device, output.texture.view, unsafe { nil }) + output.texture.view = unsafe { nil } + } + if !isnil(output.texture.image) { + vk.destroy_image(vk_device, output.texture.image, unsafe { nil }) + output.texture.image = unsafe { nil } + } + _ = vp.app.device_context.memory_allocator.release(mut output.texture.allocation_info) + } + vp.output_textures.clear() + vp.output_textures_free.clear() + vp.output_textures_ready.clear() + vp.output_textures_retired.clear() + if !isnil(vp.decode_output_image.view) { + vk.destroy_image_view(vk_device, vp.decode_output_image.view, unsafe { nil }) + vp.decode_output_image.view = unsafe { nil } + } + if !isnil(vp.decode_output_image.image) { + vk.destroy_image(vk_device, vp.decode_output_image.image, unsafe { nil }) + vp.decode_output_image.image = unsafe { nil } + } + _ = vp.app.device_context.memory_allocator.release(mut vp.decode_output_image.allocation_info) + for mut dpb in vp.decoder.info.images_dpb { + if !isnil(dpb.view) { + vk.destroy_image_view(vk_device, dpb.view, unsafe { nil }) + dpb.view = unsafe { nil } + } + if !isnil(dpb.image) { + vk.destroy_image(vk_device, dpb.image, unsafe { nil }) + dpb.image = unsafe { nil } + } + _ = vp.app.device_context.memory_allocator.release(mut dpb.allocation_info) + } + if !isnil(vp.decoder.gpu_bitstream_buffer) { + vp.app.device_context.memory_allocator.unmap(mut vp.decoder.gpu_bitstream_allocation) + vk.destroy_buffer(vk_device, vp.decoder.gpu_bitstream_buffer, unsafe { nil }) + vp.decoder.gpu_bitstream_buffer = unsafe { nil } + } + _ = vp.app.device_context.memory_allocator.release(mut vp.decoder.gpu_bitstream_allocation) + if !isnil(vp.decoder.video_session_parameters) { + vk.destroy_video_session_parameters_khr(vk_device, vp.decoder.video_session_parameters, unsafe { nil }) + vp.decoder.video_session_parameters = unsafe { nil } + } + if !isnil(vp.decoder.video_session) { + vk.destroy_video_session_khr(vk_device, vp.decoder.video_session, unsafe { nil }) + vp.decoder.video_session = unsafe { nil } + } + for memory in vp.decoder.session_memory_allocations { + if !isnil(memory) { + vk.free_memory(vk_device, memory, unsafe { nil }) + } + } + vp.decoder.session_memory_allocations.clear() + } +} + +fn (mut vp VideoPlayer) create_output_image() { + mut dev_ctx := vp.app.device_context + mut output := OutputImage{} + queue_families := [dev_ctx.get_decoder_queue_family_index(), dev_ctx.graphics_family] + queues_differ := queue_families[0] != queue_families[1] + queue_family_data := if queues_differ { + queue_families.data + } else { + unsafe { &u32(nil) } + } + mut image_ci := vk.ImageCreateInfo{ + pNext: unsafe { nil } + flags: 0 + imageType: ._2d + format: vp.decoder.properties.format_props.format + extent: vk.Extent3D{ + width: vp.decoder.video_data.width + height: vp.decoder.video_data.height + depth: 1 + } + mipLevels: 1 + arrayLayers: 1 + samples: ._1 + tiling: .optimal + usage: vk.ImageUsageFlags(u32(vk.ImageUsageFlagBits.transfer_dst) | u32(vk.ImageUsageFlagBits.sampled)) + sharingMode: .exclusive + queueFamilyIndexCount: 0 + pQueueFamilyIndices: unsafe { nil } + initialLayout: .undefined + } + if queues_differ { + image_ci.sharingMode = .concurrent + image_ci.queueFamilyIndexCount = u32(queue_families.len) + image_ci.pQueueFamilyIndices = queue_family_data + } + mut res := vp.app.device_context.memory_allocator.create_image_with_options(&image_ci, vkmem.AllocationOptions{ + usage: .gpu_only + }, &output.texture.image, mut output.texture.allocation_info) + check_vk(res, 'Could not create sampled video output image') + mut conversion_info := vk.SamplerYcbcrConversionInfo{ + conversion: dev_ctx.sampler_ycbcr_conversion + } + view_ci := vk.ImageViewCreateInfo{ + pNext: &conversion_info + image: output.texture.image + viewType: ._2d + format: image_ci.format + subresourceRange: vk.ImageSubresourceRange{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) + levelCount: 1 + layerCount: 1 + } + } + res = vk.create_image_view(dev_ctx.vk_device, &view_ci, unsafe { nil }, &output.texture.view) + check_vk(res, 'Could not create sampled video output image view') + vp.output_textures << output + vp.output_textures_free << vp.output_textures.len - 1 +} + +fn (mut vp VideoPlayer) create_decode_output_image() { + mut dev_ctx := vp.app.device_context + image_ci := vk.ImageCreateInfo{ + pNext: &vp.decoder.settings.profile_list_info + flags: 0 + imageType: ._2d + format: vp.decoder.properties.format_props.format + extent: vk.Extent3D{ + width: vp.decoder.video_data.width + height: vp.decoder.video_data.height + depth: 1 + } + mipLevels: 1 + arrayLayers: 1 + samples: ._1 + tiling: .optimal + usage: vk.ImageUsageFlags(u32(vk.ImageUsageFlagBits.video_decode_dst) | u32(vk.ImageUsageFlagBits.transfer_src)) + sharingMode: .exclusive + queueFamilyIndexCount: 0 + pQueueFamilyIndices: unsafe { nil } + initialLayout: .undefined + } + mut result := vp.app.device_context.memory_allocator.create_image_with_options(&image_ci, vkmem.AllocationOptions{ + usage: .gpu_only + }, &vp.decode_output_image.image, mut vp.decode_output_image.allocation_info) + check_vk(result, 'Could not create distinct video decode-output image') + view_ci := vk.ImageViewCreateInfo{ + image: vp.decode_output_image.image + viewType: ._2d + format: image_ci.format + subresourceRange: vk.ImageSubresourceRange{ + aspectMask: vk.ImageAspectFlags(vk.ImageAspectFlagBits.color) + levelCount: 1 + layerCount: 1 + } + } + result = vk.create_image_view(dev_ctx.vk_device, &view_ci, unsafe { nil }, &vp.decode_output_image.view) + check_vk(result, 'Could not create distinct video decode-output image view') +} + +fn query_video_format(gpu vk.PhysicalDevice, profile_list &vk.VideoProfileListInfoKHR, + usage vk.ImageUsageFlags) ?vk.VideoFormatPropertiesKHR { + format_info := vk.PhysicalDeviceVideoFormatInfoKHR{ + pNext: unsafe { profile_list } + imageUsage: usage + } + mut count := u32(0) + mut no_formats := unsafe { nil } + mut result := vk.get_physical_device_video_format_properties_khr(gpu, &format_info, &count, mut no_formats) + if result != .success || count == 0 { + return none + } + mut formats := []vk.VideoFormatPropertiesKHR{len: int(count), init: vk.VideoFormatPropertiesKHR{}} + mut formats_data := formats.data + result = vk.get_physical_device_video_format_properties_khr(gpu, &format_info, &count, mut formats_data) + if result != .success || count == 0 { + return none + } + return formats[0] +} diff --git a/examples/video_decode_app/video_player_test.v b/video_player_test.v similarity index 97% rename from examples/video_decode_app/video_player_test.v rename to video_player_test.v index dd075f9..8057251 100644 --- a/examples/video_decode_app/video_player_test.v +++ b/video_player_test.v @@ -1,4 +1,4 @@ -module video_decode_app +module main import antono2.minimp4 import os @@ -53,10 +53,10 @@ fn test_rotation_from_common_mp4_track_matrices() { fn test_display_dimensions_apply_sar_before_rotation() { mut metadata := VideoMetadata{ - coded_width: 720 - coded_height: 576 - sar_width: 16 - sar_height: 15 + coded_width: 720 + coded_height: 576 + sar_width: 16 + sar_height: 15 rotation_degrees: -90 } metadata.update_display_dimensions() @@ -83,7 +83,7 @@ fn test_h264_profile_names() { fn test_vui_matrix_coefficients_map_to_vulkan_ycbcr_models() { mut metadata := VideoMetadata{ - coded_height: 1080 + coded_height: 1080 colour_description_present: true } metadata.matrix_coefficients = 0 @@ -114,7 +114,7 @@ fn test_minimp4_retains_track_rotation_matrix() { mut file := os.open(path) or { panic(err) } defer { file.close() } mut user_data := CallbackUserData{ - file: &file + file: &file file_size: os.file_size(path) } mut mp4 := minimp4.MP4D_demux_t{} @@ -134,7 +134,7 @@ fn test_minimp4_phone_sample_timing_is_approximately_30_fps() { mut file := os.open(path) or { panic(err) } defer { file.close() } mut user_data := CallbackUserData{ - file: &file + file: &file file_size: os.file_size(path) } mut mp4 := minimp4.MP4D_demux_t{} @@ -241,7 +241,7 @@ fn test_mp4_callback_reports_a_short_read() { file.close() } mut callback_data := CallbackUserData{ - file: &file + file: &file file_size: 8 } mut destination := []u8{len: 8} @@ -299,7 +299,7 @@ fn test_runtime_frame_read_reports_file_truncation() { player.decoder.properties.caps.minBitstreamBufferSizeAlignment = 1 } mut frame := VideoPlayerDecodeStreamFrame{ - gpu_bitstream_capacity: u64(upload_buffer.len) + gpu_bitstream_capacity: u64(upload_buffer.len) gpu_bitstream_slice_mapped_memory_address: upload_buffer.data } player.write_video_frame(mut frame) or { @@ -313,12 +313,12 @@ fn test_runtime_frame_read_reports_file_truncation() { fn test_render_transform_rotates_minus_90_and_letterboxes_portrait_video() { metadata := VideoMetadata{ - display_width: 1080 - display_height: 1920 + display_width: 1080 + display_height: 1920 rotation_degrees: -90 } transform := video_render_transform(metadata, vk.Extent2D{ - width: 1280 + width: 1280 height: 720 }) assert transform.values[0..7] == [f32(0), 1, 0, 0, -1, 0, 1]