Improvements to thread safety and handling of out of GPU memory conditions when paging.

include/vsg/state/BufferInfo.h

@@ -59,7 +59,13 @@ namespace vsg
         ref_ptr<Data> data;
         ref_ptr<BufferInfo> parent;
 
-        /// return true if the BufferInfo's data has been modified and should be copied to the buffer
+        /// return true if the BufferInfo's has not been assigned a buffer or that buffer doesn't yet have device memory allocated for it.
+        bool requiresAllocation(uint32_t deviceID) const
+        {
+            return !buffer || buffer->getDeviceMemory(deviceID)==nullptr;
+        }
+
+        /// returns true if the data and associated buffers are not in sync and the data needs to be transferred to synchronize them
         bool requiresCopy(uint32_t deviceID) const
         {
             return data && data->differentModifiedCount(copiedModifiedCounts[deviceID]);

include/vsg/state/ImageView.h

@@ -50,8 +50,8 @@ namespace vsg
         {
             VkImageView imageView = VK_NULL_HANDLE;
             ref_ptr<Device> device;
+            ref_ptr<Image> image;
 
-            ~VulkanData() { release(); }
             void release();
         };
 

include/vsg/utils/SharedObjects.h

@@ -72,10 +72,11 @@ namespace vsg
         /// write out stats of objects held, types of objects and their reference counts
         void report(vsg::LogOutput& output);
 
+        mutable std::recursive_mutex mutex;
+
     protected:
         ~SharedObjects() override;
 
-        mutable std::recursive_mutex _mutex;
         std::map<std::type_index, ref_ptr<Object>> _defaults;
         std::map<std::type_index, std::set<ref_ptr<Object>, DereferenceLess>> _sharedObjects;
     };
@@ -127,7 +128,7 @@ namespace vsg
     template<class T>
     ref_ptr<T> SharedObjects::shared_default()
     {
-        std::scoped_lock<std::recursive_mutex> lock(_mutex);
+        std::scoped_lock<std::recursive_mutex> lock(mutex);
 
         auto id = std::type_index(typeid(T));
         auto& def = _defaults[id];
@@ -155,7 +156,7 @@ namespace vsg
     template<class T>
     void SharedObjects::share(ref_ptr<T>& object)
     {
-        std::scoped_lock<std::recursive_mutex> lock(_mutex);
+        std::scoped_lock<std::recursive_mutex> lock(mutex);
 
         if (suitableForSharing && !suitableForSharing->suitable(object.get())) return;
 
@@ -175,7 +176,7 @@ namespace vsg
     void SharedObjects::share(ref_ptr<T>& object, Func init)
     {
         {
-            std::scoped_lock<std::recursive_mutex> lock(_mutex);
+            std::scoped_lock<std::recursive_mutex> lock(mutex);
 
             auto id = std::type_index(typeid(T));
             auto& shared_objects = _sharedObjects[id];
@@ -189,7 +190,7 @@ namespace vsg
         init(object);
 
         {
-            std::scoped_lock<std::recursive_mutex> lock(_mutex);
+            std::scoped_lock<std::recursive_mutex> lock(mutex);
             auto id = std::type_index(typeid(T));
             auto& shared_objects = _sharedObjects[id];
             if (suitableForSharing && suitableForSharing->suitable(object.get()))

include/vsg/vk/CommandBuffer.h

@@ -15,6 +15,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #include <vsg/core/ScratchMemory.h>
 #include <vsg/state/PipelineLayout.h>
 #include <vsg/vk/CommandPool.h>
+#include <vsg/ui/FrameStamp.h>
 
 namespace vsg
 {
@@ -42,6 +43,7 @@ namespace vsg
         ViewDependentState* viewDependentState = nullptr;
         State* state = nullptr;
         const InstanceNode* instanceNode = nullptr;
+        ref_ptr<FrameStamp> frameStamp;
         ref_ptr<GPUStatsCollection> gpuStats;
 
         VkCommandBufferLevel level() const { return _level; }

include/vsg/vk/Context.h

@@ -72,6 +72,9 @@ namespace vsg
 
         ~Context() override;
 
+        // clear any recorded commands for next frame to avoid accumulating work to next compile traversal.
+        void reset();
+
         const uint32_t deviceID = 0;
         ref_ptr<Device> device;
         ResourceRequirements resourceRequirements;
@@ -126,6 +129,9 @@ namespace vsg
 
         std::vector<ref_ptr<Command>> commands;
 
+        BufferInfoList bufferInfosToCopy;
+        ImageInfoList imageInfosToCopy;
+
         ref_ptr<CopyAndReleaseImage> copyImageCmd;
         void copy(ref_ptr<Data> data, ref_ptr<ImageInfo> dest);
         void copy(ref_ptr<Data> data, ref_ptr<ImageInfo> dest, uint32_t numMipMapLevels);
@@ -146,6 +152,10 @@ namespace vsg
         std::vector<ref_ptr<BuildAccelerationStructureCommand>> buildAccelerationStructureCommands;
 
         ref_ptr<TransferTask> transferTask;
+
+        bool createBufferAndTransferData(const BufferInfoList& bufferInfoList, VkBufferUsageFlags usage, VkSharingMode sharingMode);
+        bool copy(const ImageInfoList& imageInfoList);
+
     };
     VSG_type_name(vsg::Context);
 

include/vsg/vk/MemoryBufferPools.h

@@ -55,6 +55,7 @@ namespace vsg
         using DeviceMemoryOffset = std::pair<ref_ptr<DeviceMemory>, VkDeviceSize>;
         DeviceMemoryOffset reserveMemory(VkMemoryRequirements memRequirements, VkMemoryPropertyFlags memoryProperties, void* pNextAllocInfo = nullptr);
 
+        VkResult reserve(const BufferInfoList& bufferInfoList, VkDeviceSize alignment, VkBufferUsageFlags bufferUsageFlags, VkSharingMode sharingMode, VkMemoryPropertyFlags memoryProperties);
         VkResult reserve(ResourceRequirements& requirements);
 
         void report(LogOutput& out) const;

src/vsg/app/CommandGraph.cpp

@@ -112,6 +112,7 @@ void CommandGraph::record(ref_ptr<RecordedCommandBuffers> recordedCommandBuffers
         commandBuffer->reset();
     }
 
+    commandBuffer->frameStamp = frameStamp;
     commandBuffer->numDependentSubmissions().fetch_add(1);
 
     recordTraversal->getState()->connect(commandBuffer);

src/vsg/app/CompileManager.cpp

@@ -253,6 +253,11 @@ CompileResult CompileManager::compile(ref_ptr<Object> object, ContextSelectionFu
     auto run_compile_traversal = [&]() -> void {
         try
         {
+            for (auto& context : compileTraversal->contexts)
+            {
+                context->reset();
+            }
+
             for (auto& context : compileTraversal->contexts)
             {
                 ref_ptr<View> view = context->view;
@@ -345,6 +350,11 @@ CompileResult CompileManager::compile(ref_ptr<Object> object, ContextSelectionFu
     else
     {
         ++failedCompileCount;
+
+        for (auto& context : compileTraversal->contexts)
+        {
+            context->reset();
+        }
     }
 
     return result;
@@ -362,6 +372,11 @@ CompileResult CompileManager::compileTask(ref_ptr<RecordAndSubmitTask> task, Res
         auto compileTraversal = CompileTraversal::create(task->device, resourceRequirements);
         auto deviceMemoryBufferPools = task->device->deviceMemoryBufferPools.ref_ptr();
 
+        for (auto& context : compileTraversal->contexts)
+        {
+            context->reset();
+        }
+
         for (const auto& context : compileTraversal->contexts)
         {
             if (resourceRequirements.dataTransferHint == COMPILE_TRAVERSAL_USE_TRANSFER_TASK)

src/vsg/app/SecondaryCommandGraph.cpp

@@ -109,6 +109,7 @@ void SecondaryCommandGraph::record(ref_ptr<RecordedCommandBuffers> recordedComma
         commandBuffer->reset();
     }
 
+    commandBuffer->frameStamp = frameStamp;
     commandBuffer->numDependentSubmissions().fetch_add(1);
 
     recordTraversal->getState()->connect(commandBuffer);

src/vsg/nodes/VertexDraw.cpp

@@ -128,8 +128,7 @@ void VertexDraw::compile(Context& context)
 
     if (requiresCreateAndCopy)
     {
-        BufferInfoList combinedBufferInfos(arrays);
-        createBufferAndTransferData(context, combinedBufferInfos, VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, VK_SHARING_MODE_EXCLUSIVE);
+        createBufferAndTransferData(context, arrays, VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, VK_SHARING_MODE_EXCLUSIVE);
 
         // info("VertexDraw::compile() create and copy ", this);
     }

src/vsg/state/BufferInfo.cpp

@@ -207,186 +207,18 @@ ref_ptr<BufferInfo> vsg::copyDataToStagingBuffer(Context& context, const Data* d
     return stagingBufferInfo;
 }
 
+VkDeviceSize BufferInfo::computeDataSize() const
+{
+    return data ? data->dataSize() : 0;
+}
+
 /////////////////////////////////////////////////////////////////////////////////////////
 //
 // vsg::createBufferAndTransferData
 //
 bool vsg::createBufferAndTransferData(Context& context, const BufferInfoList& bufferInfoList, VkBufferUsageFlags usage, VkSharingMode sharingMode)
 {
-    if (bufferInfoList.empty()) return false;
-
-    Device* device = context.device;
-    auto deviceID = context.deviceID;
-    auto transferTask = context.transferTask.get();
-    VkDeviceSize alignment = 4;
-    if (usage == VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT)
-        alignment = device->getPhysicalDevice()->getProperties().limits.minUniformBufferOffsetAlignment;
-    else if (usage == VK_BUFFER_USAGE_STORAGE_BUFFER_BIT)
-        alignment = device->getPhysicalDevice()->getProperties().limits.minStorageBufferOffsetAlignment;
-
-    debug("vsg::createBufferAndTransferData(Context& context, const BufferInfoList& bufferInfoList, VkBufferUsageFlags usage, VkSharingMode sharingMode) usage = ", usage, ", alignment = ", alignment);
-
-    //transferTask = nullptr;
-
-    ref_ptr<BufferInfo> deviceBufferInfo;
-    size_t numBuffersRequired = 0;
-    bool containsMultipleParents = false;
-    for (auto& bufferInfo : bufferInfoList)
-    {
-        if (bufferInfo->data)
-        {
-            if (bufferInfo->data->getModifiedCount(bufferInfo->copiedModifiedCounts[deviceID]))
-            {
-                ++numBuffersRequired;
-            }
-        }
-
-        if (bufferInfo->parent)
-        {
-            if (deviceBufferInfo && bufferInfo->parent != deviceBufferInfo) containsMultipleParents = true;
-            deviceBufferInfo = bufferInfo->parent;
-        }
-    }
-
-    if (numBuffersRequired == 0)
-    {
-        debug("\nvsg::createBufferAndTransferData(...) already all compiled. deviceID = ", deviceID);
-        return false;
-    }
-
-    if (containsMultipleParents)
-    {
-        warn("vsg::createBufferAndTransferData(...) does not support multiple parent BufferInfo.");
-        return false;
-    }
-
-    VkDeviceSize totalSize = 0;
-    VkDeviceSize offset = 0;
-    for (const auto& bufferInfo : bufferInfoList)
-    {
-        if (bufferInfo->data)
-        {
-            bufferInfo->offset = offset;
-            bufferInfo->range = bufferInfo->data->dataSize();
-            VkDeviceSize endOfEntry = offset + bufferInfo->range;
-            offset = (alignment == 1 || (endOfEntry % alignment) == 0) ? endOfEntry : ((endOfEntry / alignment) + 1) * alignment;
-            //info("  BufferInfo.data = ", bufferInfo->data, ", dynamic = ", bufferInfo->data->getLayout().dynamic);
-        }
-    }
-
-    totalSize = offset;
-    if (totalSize == 0) return false;
-
-    if (deviceBufferInfo && deviceBufferInfo->buffer)
-    {
-        if (totalSize != deviceBufferInfo->range)
-        {
-            warn("Existing deviceBufferInfo, ", deviceBufferInfo, ", deviceBufferInfo->range  = ", deviceBufferInfo->range, ", ", totalSize, " NOT compatible");
-            return false;
-        }
-        else
-        {
-            debug("Existing deviceBufferInfo, ", deviceBufferInfo, ", deviceBufferInfo->range  = ", deviceBufferInfo->range, ", ", totalSize, " with compatible size");
-
-            // make sure the VkBuffer is created
-            deviceBufferInfo->buffer->compile(context);
-
-            if (!deviceBufferInfo->buffer->getDeviceMemory(deviceID))
-            {
-                VkMemoryRequirements memRequirements;
-                vkGetBufferMemoryRequirements(*device, deviceBufferInfo->buffer->vk(device->deviceID), &memRequirements);
-
-                auto deviceMemoryOffset = context.deviceMemoryBufferPools->reserveMemory(memRequirements, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
-                if (deviceMemoryOffset.first)
-                    deviceBufferInfo->buffer->bind(deviceMemoryOffset.first, deviceMemoryOffset.second);
-                else
-                {
-                    debug("vsg::createBufferAndTransferData() Failure to assign memory to existing BufferInfo");
-                    return false;
-                }
-            }
-        }
-    }
-
-    if (!deviceBufferInfo)
-    {
-        VkBufferUsageFlags bufferUsageFlags = VK_BUFFER_USAGE_TRANSFER_DST_BIT | usage;
-        deviceBufferInfo = context.deviceMemoryBufferPools->reserveBuffer(totalSize, alignment, bufferUsageFlags, sharingMode, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
-    }
-
-    if (!deviceBufferInfo)
-    {
-        debug("vsg::createBufferAndTransferData() Failure to assign Buffer");
-        return false;
-    }
-
-    debug("deviceBufferInfo->buffer ", deviceBufferInfo->buffer, ", ", deviceBufferInfo->offset, ", ", deviceBufferInfo->range, ")");
-
-    // assign the buffer to the bufferData entries and shift the offsets to offset within the buffer
-    for (const auto& bufferInfo : bufferInfoList)
-    {
-        bufferInfo->buffer = deviceBufferInfo->buffer;
-        bufferInfo->offset += deviceBufferInfo->offset;
-    }
-
-    if (transferTask)
-    {
-        vsg::debug("vsg::createBufferAndTransferData(..)");
-
-        for (auto& bufferInfo : bufferInfoList)
-        {
-            vsg::debug("    ", bufferInfo, ", ", bufferInfo->data, ", ", bufferInfo->buffer, ", ", bufferInfo->offset);
-            bufferInfo->data->dirty();
-            bufferInfo->parent = deviceBufferInfo;
-        }
-
-        transferTask->assign(bufferInfoList);
-
-        return true;
-    }
-
-    auto stagingBufferInfo = context.stagingMemoryBufferPools->reserveBuffer(totalSize, alignment, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, sharingMode, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
-
-    debug("stagingBufferInfo->buffer ", stagingBufferInfo->buffer.get(), ", ", stagingBufferInfo->offset, ", ", stagingBufferInfo->range, ")");
-
-    ref_ptr<Buffer> stagingBuffer(stagingBufferInfo->buffer);
-    ref_ptr<DeviceMemory> stagingMemory(stagingBuffer->getDeviceMemory(context.deviceID));
-
-    if (!stagingMemory)
-    {
-        return false;
-    }
-
-    void* buffer_data;
-    stagingMemory->map(stagingBuffer->getMemoryOffset(context.deviceID) + stagingBufferInfo->offset, stagingBufferInfo->range, 0, &buffer_data);
-    char* ptr = reinterpret_cast<char*>(buffer_data);
-
-    debug("    buffer_data ", buffer_data, ", stagingBufferInfo->offset=", stagingBufferInfo->offset, ", ", totalSize);
-
-    for (const auto& bufferInfo : bufferInfoList)
-    {
-        const Data* data = bufferInfo->data;
-        if (data)
-        {
-            std::memcpy(ptr + bufferInfo->offset - deviceBufferInfo->offset, data->dataPointer(), data->dataSize());
-            if (data->properties.dataVariance == STATIC_DATA_UNREF_AFTER_TRANSFER)
-            {
-                bufferInfo->data.reset();
-            }
-        }
-        bufferInfo->parent = deviceBufferInfo;
-    }
-
-    stagingMemory->unmap();
-
-    context.copy(stagingBufferInfo, deviceBufferInfo);
-
-    return true;
-}
-
-VkDeviceSize BufferInfo::computeDataSize() const
-{
-    return data ? data->dataSize() : 0;
+    return context.createBufferAndTransferData(bufferInfoList, usage, sharingMode);
 }
 
 /////////////////////////////////////////////////////////////////////////////////////////