Replace fences with timeline semaphores

Replaced fences with timeline semaphores. I have no idea, if I did this
correctly, but it seems to be working fine. This might come back later
to bite me in my posterior.

Signed-off-by: Piotr Krygier <piotrkrygier@everyonecancode.xyz>

graphics/src/graphics_context.h

@@ -229,7 +229,7 @@ struct vulkan_handles
     VkRenderPass render_pass;
     VkSemaphore image_available_semaphores[SWAP_BUFFER_COUNT];
     VkSemaphore render_finished_semaphores[SWAP_BUFFER_COUNT];
-    VkFence in_flight_fences[SWAP_BUFFER_COUNT];
+    VkSemaphore timeline_semaphore;
 #ifndef NDEBUG
     VkDebugUtilsMessengerEXT debug_messenger;
 #endif // !NDEBUG
@@ -280,6 +280,7 @@ struct graphics_context_t
     uint32_t queue_family_indices[QUEUE_FAMILY_INDEX_LAST_INDEX];
     uint32_t current_frame;
     uint32_t swapchain_images_count;
+    uint64_t frame_finished_idx;
     SDL_Window* window_handle;
     struct mvp_data_t mvp_data;
     struct renderer_data_t renderer_data;

graphics/src/vulkan_base.c

@@ -16,6 +16,7 @@
 #include <SDL3/SDL_main.h>
 #include <SDL3/SDL_vulkan.h>
 #include <stddef.h>
+#include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 #include <vk_mem_alloc.h>
@@ -357,28 +358,50 @@ static rse_err_t create_device(struct graphics_context_t* context)
     uint32_t presentation_familiy_idx = 0;
     uint32_t queue_families_property_count = 0U;
     uint32_t enabled_device_extensions_count = 2;
-    VkBool32 bindless_supported = VK_FALSE;
+    VkBool32 device_feature_supported = VK_FALSE;
     VkDeviceCreateInfo device_create_info = {0};
     VkQueueFamilyProperties* queue_family_properties = NULL;
     VkDeviceQueueCreateInfo device_queue_createinfos[2] = {0};
     VkPhysicalDeviceFeatures2 physical_features2 = {0};
+    VkPhysicalDeviceSynchronization2Features synchronization2_features = {0};
+    VkPhysicalDeviceTimelineSemaphoreFeatures timeline_semaphore_features = {0};
     VkPhysicalDeviceDescriptorIndexingFeatures indexing_features = {0};
     const char* enabled_device_extensions_names[] = {
         "VK_KHR_swapchain",
         "VK_EXT_descriptor_indexing",
     };
 
-    /* Enable descriptori indexing for bindless textures */
+    /* Enable synchronization2 features */
+    synchronization2_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SYNCHRONIZATION_2_FEATURES;
+
+    /* Enable timeline semaphores feature*/
+    timeline_semaphore_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES;
+    timeline_semaphore_features.pNext = &synchronization2_features;
+
+    /* Enable descriptor indexing for bindless textures */
     indexing_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES_EXT;
+    indexing_features.pNext = &timeline_semaphore_features;
 
     physical_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;
     physical_features2.pNext = &indexing_features;
 
     vkGetPhysicalDeviceFeatures2(context->vulkan_handles.physical_device, &physical_features2);
 
-    bindless_supported = indexing_features.descriptorBindingPartiallyBound && indexing_features.runtimeDescriptorArray;
+    device_feature_supported = indexing_features.descriptorBindingPartiallyBound && indexing_features.runtimeDescriptorArray;
-    if (bindless_supported == VK_FALSE) {
+    if (device_feature_supported == VK_FALSE) {
-        SDL_LogError(SDL_LOG_CATEGORY_GPU, "Required features for bindless design not supported");
+        SDL_LogError(SDL_LOG_CATEGORY_GPU, "Required features for bindless design is not supported");
+        return RSE_ERROR_INTERNAL_ERROR;
+    }
+
+    device_feature_supported = synchronization2_features.synchronization2;
+    if (device_feature_supported == VK_FALSE) {
+        SDL_LogError(SDL_LOG_CATEGORY_GPU, "Required features for Synchronization2 is not supported");
+        return RSE_ERROR_INTERNAL_ERROR;
+    }
+
+    device_feature_supported = timeline_semaphore_features.timelineSemaphore;
+    if (device_feature_supported == VK_FALSE) {
+        SDL_LogError(SDL_LOG_CATEGORY_GPU, "Required features for timeline semaphore is not supported");
         return RSE_ERROR_INTERNAL_ERROR;
     }
 
@@ -532,11 +555,14 @@ static rse_err_t create_memory_allocator(struct graphics_context_t* context)
 static rse_err_t create_sync_objects(struct graphics_context_t* context)
 {
     VkSemaphoreCreateInfo semaphore_info = {0};
+    VkSemaphoreTypeCreateInfo semaphore_type = {0};
+
     semaphore_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
 
-    VkFenceCreateInfo fenceInfo = {0};
+    semaphore_type.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO;
-    fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+    semaphore_type.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE;
-    fenceInfo.flags = VK_FENCE_CREATE_SIGNALED_BIT;
+    semaphore_type.initialValue = 0;
+    semaphore_type.pNext = NULL;
 
     for (size_t i = 0; i < SWAP_BUFFER_COUNT; ++i) {
         if (VK_SUCCESS != vkCreateSemaphore(context->vulkan_handles.device,
@@ -546,16 +572,28 @@ static rse_err_t create_sync_objects(struct graphics_context_t* context)
             VK_SUCCESS != vkCreateSemaphore(context->vulkan_handles.device,
                                             &semaphore_info,
                                             NULL,
-                                            &context->vulkan_handles.render_finished_semaphores[i]) ||
+                                            &context->vulkan_handles.render_finished_semaphores[i])) {
-            VK_SUCCESS != vkCreateFence(context->vulkan_handles.device,
-                                        &fenceInfo,
-                                        NULL,
-                                        &context->vulkan_handles.in_flight_fences[i])) {
             SDL_LogCritical(SDL_LOG_CATEGORY_GPU, "Failed to create sync objects.");
             return RSE_ERROR_INTERNAL_ERROR;
         }
     }
 
+    semaphore_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
+
+    semaphore_type.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO;
+    semaphore_type.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE;
+    semaphore_type.initialValue = 0;
+    semaphore_type.pNext = NULL;
+
+    semaphore_info.pNext = &semaphore_type;
+
+    if (VK_SUCCESS != vkCreateSemaphore(context->vulkan_handles.device,
+                                        &semaphore_info,
+                                        NULL,
+                                        &context->vulkan_handles.timeline_semaphore)) {
+        SDL_LogCritical(SDL_LOG_CATEGORY_GPU, "Failed to create timeline semaphore.");
+        return RSE_ERROR_INTERNAL_ERROR;
+    }
     return RSE_ERROR_NO_ERROR;
 }
 
@@ -593,18 +631,23 @@ rse_err_t vulkan_draw_frame(struct graphics_context_t* context)
     rse_err_t status = RSE_ERROR_NO_ERROR;
     uint32_t image_index = 0U;
     VkResult result = VK_FALSE;
-    VkSubmitInfo submit_info = {0};
+    VkSubmitInfo2 submit_info = {0};
-    VkSemaphore wait_semaphores[1];
-    VkPipelineStageFlags wait_stages[1];
-    VkSemaphore signal_semaphores[1];
     VkPresentInfoKHR present_info = {0};
     VkSwapchainKHR swap_chains[1];
+    VkSemaphoreWaitInfo timeline_wait_info = {0};
+    VkSemaphoreSubmitInfo timeline_signal_info = {0};
+    VkSemaphoreSubmitInfo binary_signal_info = {0};
+    VkSemaphoreSubmitInfo binary_wait_info = {0};
+    VkSemaphoreSubmitInfo signal_infos[2] = {0};
+    VkCommandBufferSubmitInfo cmd_buffer_submit_info = {0};
 
-    vkWaitForFences(context->vulkan_handles.device,
+    timeline_wait_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO;
-                    1,
+    timeline_wait_info.pNext = NULL;
-                    &context->vulkan_handles.in_flight_fences[context->current_frame],
+    timeline_wait_info.pSemaphores = &context->vulkan_handles.timeline_semaphore;
-                    VK_TRUE,
+    timeline_wait_info.semaphoreCount = 1;
-                    UINT64_MAX);
+    timeline_wait_info.pValues = &context->frame_finished_idx;
+
+    vkWaitSemaphores(context->vulkan_handles.device, &timeline_wait_info, UINT64_MAX);
 
     STATUS_CHECK(buffers_update(context));
 
@@ -623,33 +666,47 @@ rse_err_t vulkan_draw_frame(struct graphics_context_t* context)
         return RSE_ERROR_INTERNAL_ERROR;
     }
 
-    // update_uniform_buffers(context);
-
-    /* Only reset the fence if we are submitting work */
-    vkResetFences(context->vulkan_handles.device, 1, &context->vulkan_handles.in_flight_fences[context->current_frame]);
     reset_command_buffer(context);
 
     record_command_buffer(context, image_index);
 
-    submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
+    context->frame_finished_idx++;
     
-    wait_semaphores[0] = context->vulkan_handles.image_available_semaphores[context->current_frame];
+    binary_signal_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO;
-    wait_stages[0] = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
+    binary_signal_info.pNext = NULL;
-    submit_info.waitSemaphoreCount = 1;
+    binary_signal_info.semaphore = context->vulkan_handles.render_finished_semaphores[context->current_frame];
-    submit_info.pWaitSemaphores = wait_semaphores;
+    binary_signal_info.value = 0;
-    submit_info.pWaitDstStageMask = wait_stages;
+    binary_signal_info.stageMask = VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT;
 
-    submit_info.commandBufferCount = 1;
+    timeline_signal_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO;
-    submit_info.pCommandBuffers = &context->vulkan_handles.command_buffers[context->current_frame];
+    timeline_signal_info.pNext = NULL;
+    timeline_signal_info.semaphore = context->vulkan_handles.timeline_semaphore;
+    timeline_signal_info.value = context->frame_finished_idx;
+    timeline_signal_info.stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
 
-    signal_semaphores[0] = context->vulkan_handles.render_finished_semaphores[image_index];
+    signal_infos[0] = binary_signal_info;
-    submit_info.signalSemaphoreCount = 1;
+    signal_infos[1] = timeline_signal_info;
-    submit_info.pSignalSemaphores = signal_semaphores;
 
-    if (vkQueueSubmit(context->vulkan_handles.graphics_queue,
+    binary_wait_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO;
-                      1,
+    binary_wait_info.pNext = NULL;
-                      &submit_info,
+    binary_wait_info.semaphore = context->vulkan_handles.image_available_semaphores[context->current_frame];
-                      context->vulkan_handles.in_flight_fences[context->current_frame]) != VK_SUCCESS) {
+    binary_wait_info.value = 0;
+    binary_wait_info.stageMask = VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT;
+
+    cmd_buffer_submit_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO;
+    cmd_buffer_submit_info.pNext = NULL;
+    cmd_buffer_submit_info.commandBuffer = context->vulkan_handles.command_buffers[context->current_frame];
+
+    submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2;
+    submit_info.pNext = NULL;
+    submit_info.commandBufferInfoCount = 1;
+    submit_info.pCommandBufferInfos = &cmd_buffer_submit_info;
+    submit_info.signalSemaphoreInfoCount = 2;
+    submit_info.pSignalSemaphoreInfos = signal_infos;
+    submit_info.waitSemaphoreInfoCount = 1;
+    submit_info.pWaitSemaphoreInfos = &binary_wait_info;
+
+    if (vkQueueSubmit2(context->vulkan_handles.graphics_queue, 1, &submit_info, NULL)) {
         SDL_LogCritical(SDL_LOG_CATEGORY_GPU, "Failed to submit draw command buffer!");
         return RSE_ERROR_INTERNAL_ERROR;
     }
@@ -687,8 +744,8 @@ void vulkan_deinit(struct graphics_context_t* context)
     for (size_t i = 0; i < SWAP_BUFFER_COUNT; ++i) {
         vkDestroySemaphore(context->vulkan_handles.device, context->vulkan_handles.image_available_semaphores[i], NULL);
         vkDestroySemaphore(context->vulkan_handles.device, context->vulkan_handles.render_finished_semaphores[i], NULL);
-        vkDestroyFence(context->vulkan_handles.device, context->vulkan_handles.in_flight_fences[i], NULL);
     }
+    vkDestroySemaphore(context->vulkan_handles.device, context->vulkan_handles.timeline_semaphore, NULL);
 
     destroy_textures(context);