1 /*-------------------------------------------------------------------------
2 * Vulkan CTS Framework
3 * --------------------
4 *
5 * Copyright (c) 2019 Google Inc.
6 * Copyright (c) 2019 The Khronos Group Inc.
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 *
20 *//*!
21 * \file
22 * \brief Memory management utilities.
23 *//*--------------------------------------------------------------------*/
24
25 #include "vkMemUtil.hpp"
26 #include "vkStrUtil.hpp"
27 #include "vkQueryUtil.hpp"
28 #include "vkRef.hpp"
29 #include "vkRefUtil.hpp"
30 #include "vkImageUtil.hpp"
31 #include "deInt32.h"
32
33 #include <sstream>
34
35 namespace vk
36 {
37
38 using de::UniquePtr;
39 using de::MovePtr;
40 using std::vector;
41
42 typedef de::SharedPtr<Allocation> AllocationSp;
43
44 namespace
45 {
46
47 class HostPtr
48 {
49 public:
50 HostPtr (const DeviceInterface& vkd, VkDevice device, VkDeviceMemory memory, VkDeviceSize offset, VkDeviceSize size, VkMemoryMapFlags flags);
51 ~HostPtr (void);
52
get(void) const53 void* get (void) const { return m_ptr; }
54
55 private:
56 const DeviceInterface& m_vkd;
57 const VkDevice m_device;
58 const VkDeviceMemory m_memory;
59 void* const m_ptr;
60 };
61
HostPtr(const DeviceInterface & vkd,VkDevice device,VkDeviceMemory memory,VkDeviceSize offset,VkDeviceSize size,VkMemoryMapFlags flags)62 HostPtr::HostPtr (const DeviceInterface& vkd, VkDevice device, VkDeviceMemory memory, VkDeviceSize offset, VkDeviceSize size, VkMemoryMapFlags flags)
63 : m_vkd (vkd)
64 , m_device (device)
65 , m_memory (memory)
66 , m_ptr (mapMemory(vkd, device, memory, offset, size, flags))
67 {
68 }
69
~HostPtr(void)70 HostPtr::~HostPtr (void)
71 {
72 m_vkd.unmapMemory(m_device, m_memory);
73 }
74
selectMatchingMemoryType(const VkPhysicalDeviceMemoryProperties & deviceMemProps,deUint32 allowedMemTypeBits,MemoryRequirement requirement)75 deUint32 selectMatchingMemoryType (const VkPhysicalDeviceMemoryProperties& deviceMemProps, deUint32 allowedMemTypeBits, MemoryRequirement requirement)
76 {
77 const deUint32 compatibleTypes = getCompatibleMemoryTypes(deviceMemProps, requirement);
78 const deUint32 candidates = allowedMemTypeBits & compatibleTypes;
79
80 if (candidates == 0)
81 TCU_THROW(NotSupportedError, "No compatible memory type found");
82
83 return (deUint32)deCtz32(candidates);
84 }
85
isHostVisibleMemory(const VkPhysicalDeviceMemoryProperties & deviceMemProps,deUint32 memoryTypeNdx)86 bool isHostVisibleMemory (const VkPhysicalDeviceMemoryProperties& deviceMemProps, deUint32 memoryTypeNdx)
87 {
88 DE_ASSERT(memoryTypeNdx < deviceMemProps.memoryTypeCount);
89 return (deviceMemProps.memoryTypes[memoryTypeNdx].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0u;
90 }
91
92 } // anonymous
93
94 // Allocation
95
Allocation(VkDeviceMemory memory,VkDeviceSize offset,void * hostPtr)96 Allocation::Allocation (VkDeviceMemory memory, VkDeviceSize offset, void* hostPtr)
97 : m_memory (memory)
98 , m_offset (offset)
99 , m_hostPtr (hostPtr)
100 {
101 }
102
~Allocation(void)103 Allocation::~Allocation (void)
104 {
105 }
106
flushAlloc(const DeviceInterface & vkd,VkDevice device,const Allocation & alloc)107 void flushAlloc (const DeviceInterface& vkd, VkDevice device, const Allocation& alloc)
108 {
109 flushMappedMemoryRange(vkd, device, alloc.getMemory(), alloc.getOffset(), VK_WHOLE_SIZE);
110 }
111
invalidateAlloc(const DeviceInterface & vkd,VkDevice device,const Allocation & alloc)112 void invalidateAlloc (const DeviceInterface& vkd, VkDevice device, const Allocation& alloc)
113 {
114 invalidateMappedMemoryRange(vkd, device, alloc.getMemory(), alloc.getOffset(), VK_WHOLE_SIZE);
115 }
116
117 // MemoryRequirement
118
119 const MemoryRequirement MemoryRequirement::Any = MemoryRequirement(0x0u);
120 const MemoryRequirement MemoryRequirement::HostVisible = MemoryRequirement(MemoryRequirement::FLAG_HOST_VISIBLE);
121 const MemoryRequirement MemoryRequirement::Coherent = MemoryRequirement(MemoryRequirement::FLAG_COHERENT);
122 const MemoryRequirement MemoryRequirement::LazilyAllocated = MemoryRequirement(MemoryRequirement::FLAG_LAZY_ALLOCATION);
123 const MemoryRequirement MemoryRequirement::Protected = MemoryRequirement(MemoryRequirement::FLAG_PROTECTED);
124 const MemoryRequirement MemoryRequirement::Local = MemoryRequirement(MemoryRequirement::FLAG_LOCAL);
125 const MemoryRequirement MemoryRequirement::Cached = MemoryRequirement(MemoryRequirement::FLAG_CACHED);
126 const MemoryRequirement MemoryRequirement::NonLocal = MemoryRequirement(MemoryRequirement::FLAG_NON_LOCAL);
127 const MemoryRequirement MemoryRequirement::DeviceAddress = MemoryRequirement(MemoryRequirement::FLAG_DEVICE_ADDRESS);
128
matchesHeap(VkMemoryPropertyFlags heapFlags) const129 bool MemoryRequirement::matchesHeap (VkMemoryPropertyFlags heapFlags) const
130 {
131 // sanity check
132 if ((m_flags & FLAG_COHERENT) && !(m_flags & FLAG_HOST_VISIBLE))
133 DE_FATAL("Coherent memory must be host-visible");
134 if ((m_flags & FLAG_HOST_VISIBLE) && (m_flags & FLAG_LAZY_ALLOCATION))
135 DE_FATAL("Lazily allocated memory cannot be mappable");
136 if ((m_flags & FLAG_PROTECTED) && (m_flags & FLAG_HOST_VISIBLE))
137 DE_FATAL("Protected memory cannot be mappable");
138
139 // host-visible
140 if ((m_flags & FLAG_HOST_VISIBLE) && !(heapFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT))
141 return false;
142
143 // coherent
144 if ((m_flags & FLAG_COHERENT) && !(heapFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT))
145 return false;
146
147 // lazy
148 if ((m_flags & FLAG_LAZY_ALLOCATION) && !(heapFlags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT))
149 return false;
150
151 // protected
152 if ((m_flags & FLAG_PROTECTED) && !(heapFlags & VK_MEMORY_PROPERTY_PROTECTED_BIT))
153 return false;
154
155 // local
156 if ((m_flags & FLAG_LOCAL) && !(heapFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT))
157 return false;
158
159 // cached
160 if ((m_flags & FLAG_CACHED) && !(heapFlags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT))
161 return false;
162
163 // non-local
164 if ((m_flags & FLAG_NON_LOCAL) && (heapFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT))
165 return false;
166
167 return true;
168 }
169
MemoryRequirement(deUint32 flags)170 MemoryRequirement::MemoryRequirement (deUint32 flags)
171 : m_flags(flags)
172 {
173 }
174
175 // SimpleAllocator
176
177 class SimpleAllocation : public Allocation
178 {
179 public:
180 SimpleAllocation (Move<VkDeviceMemory> mem, MovePtr<HostPtr> hostPtr);
181 virtual ~SimpleAllocation (void);
182
183 private:
184 const Unique<VkDeviceMemory> m_memHolder;
185 const UniquePtr<HostPtr> m_hostPtr;
186 };
187
SimpleAllocation(Move<VkDeviceMemory> mem,MovePtr<HostPtr> hostPtr)188 SimpleAllocation::SimpleAllocation (Move<VkDeviceMemory> mem, MovePtr<HostPtr> hostPtr)
189 : Allocation (*mem, (VkDeviceSize)0, hostPtr ? hostPtr->get() : DE_NULL)
190 , m_memHolder (mem)
191 , m_hostPtr (hostPtr)
192 {
193 }
194
~SimpleAllocation(void)195 SimpleAllocation::~SimpleAllocation (void)
196 {
197 }
198
SimpleAllocator(const DeviceInterface & vk,VkDevice device,const VkPhysicalDeviceMemoryProperties & deviceMemProps)199 SimpleAllocator::SimpleAllocator (const DeviceInterface& vk, VkDevice device, const VkPhysicalDeviceMemoryProperties& deviceMemProps)
200 : m_vk (vk)
201 , m_device (device)
202 , m_memProps(deviceMemProps)
203 {
204 }
205
allocate(const VkMemoryAllocateInfo & allocInfo,VkDeviceSize alignment)206 MovePtr<Allocation> SimpleAllocator::allocate (const VkMemoryAllocateInfo& allocInfo, VkDeviceSize alignment)
207 {
208 DE_UNREF(alignment);
209
210 Move<VkDeviceMemory> mem = allocateMemory(m_vk, m_device, &allocInfo);
211 MovePtr<HostPtr> hostPtr;
212
213 if (isHostVisibleMemory(m_memProps, allocInfo.memoryTypeIndex))
214 hostPtr = MovePtr<HostPtr>(new HostPtr(m_vk, m_device, *mem, 0u, allocInfo.allocationSize, 0u));
215
216 return MovePtr<Allocation>(new SimpleAllocation(mem, hostPtr));
217 }
218
allocate(const VkMemoryRequirements & memReqs,MemoryRequirement requirement)219 MovePtr<Allocation> SimpleAllocator::allocate (const VkMemoryRequirements& memReqs, MemoryRequirement requirement)
220 {
221 const deUint32 memoryTypeNdx = selectMatchingMemoryType(m_memProps, memReqs.memoryTypeBits, requirement);
222 VkMemoryAllocateInfo allocInfo =
223 {
224 VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, // VkStructureType sType;
225 DE_NULL, // const void* pNext;
226 memReqs.size, // VkDeviceSize allocationSize;
227 memoryTypeNdx, // deUint32 memoryTypeIndex;
228 };
229
230 VkMemoryAllocateFlagsInfo allocFlagsInfo =
231 {
232 VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, // VkStructureType sType
233 DE_NULL, // const void* pNext
234 0, // VkMemoryAllocateFlags flags
235 0, // uint32_t deviceMask
236 };
237
238 if (requirement & MemoryRequirement::DeviceAddress)
239 {
240 allocFlagsInfo.flags |= VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT;
241 allocInfo.pNext = &allocFlagsInfo;
242 }
243
244 Move<VkDeviceMemory> mem = allocateMemory(m_vk, m_device, &allocInfo);
245 MovePtr<HostPtr> hostPtr;
246
247 if (requirement & MemoryRequirement::HostVisible)
248 {
249 DE_ASSERT(isHostVisibleMemory(m_memProps, allocInfo.memoryTypeIndex));
250 hostPtr = MovePtr<HostPtr>(new HostPtr(m_vk, m_device, *mem, 0u, allocInfo.allocationSize, 0u));
251 }
252
253 return MovePtr<Allocation>(new SimpleAllocation(mem, hostPtr));
254 }
255
allocateExtended(const InstanceInterface & vki,const DeviceInterface & vkd,const VkPhysicalDevice & physDevice,const VkDevice device,const VkMemoryRequirements & memReqs,const MemoryRequirement requirement,const void * pNext)256 MovePtr<Allocation> allocateExtended (const InstanceInterface& vki,
257 const DeviceInterface& vkd,
258 const VkPhysicalDevice& physDevice,
259 const VkDevice device,
260 const VkMemoryRequirements& memReqs,
261 const MemoryRequirement requirement,
262 const void* pNext)
263 {
264 const VkPhysicalDeviceMemoryProperties memoryProperties = getPhysicalDeviceMemoryProperties(vki, physDevice);
265 const deUint32 memoryTypeNdx = selectMatchingMemoryType(memoryProperties, memReqs.memoryTypeBits, requirement);
266 const VkMemoryAllocateInfo allocInfo =
267 {
268 VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, // VkStructureType sType
269 pNext, // const void* pNext
270 memReqs.size, // VkDeviceSize allocationSize
271 memoryTypeNdx, // deUint32 memoryTypeIndex
272 };
273 Move<VkDeviceMemory> mem = allocateMemory(vkd, device, &allocInfo);
274 MovePtr<HostPtr> hostPtr;
275
276 if (requirement & MemoryRequirement::HostVisible)
277 {
278 DE_ASSERT(isHostVisibleMemory(memoryProperties, allocInfo.memoryTypeIndex));
279 hostPtr = MovePtr<HostPtr>(new HostPtr(vkd, device, *mem, 0u, allocInfo.allocationSize, 0u));
280 }
281
282 return MovePtr<Allocation>(new SimpleAllocation(mem, hostPtr));
283 }
284
allocateDedicated(const InstanceInterface & vki,const DeviceInterface & vkd,const VkPhysicalDevice & physDevice,const VkDevice device,const VkBuffer buffer,MemoryRequirement requirement)285 de::MovePtr<Allocation> allocateDedicated (const InstanceInterface& vki,
286 const DeviceInterface& vkd,
287 const VkPhysicalDevice& physDevice,
288 const VkDevice device,
289 const VkBuffer buffer,
290 MemoryRequirement requirement)
291 {
292 const VkMemoryRequirements memoryRequirements = getBufferMemoryRequirements(vkd, device, buffer);
293 const VkMemoryDedicatedAllocateInfo dedicatedAllocationInfo =
294 {
295 VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO, // VkStructureType sType
296 DE_NULL, // const void* pNext
297 DE_NULL, // VkImage image
298 buffer // VkBuffer buffer
299 };
300
301 return allocateExtended(vki, vkd, physDevice, device, memoryRequirements, requirement, &dedicatedAllocationInfo);
302 }
303
allocateDedicated(const InstanceInterface & vki,const DeviceInterface & vkd,const VkPhysicalDevice & physDevice,const VkDevice device,const VkImage image,MemoryRequirement requirement)304 de::MovePtr<Allocation> allocateDedicated (const InstanceInterface& vki,
305 const DeviceInterface& vkd,
306 const VkPhysicalDevice& physDevice,
307 const VkDevice device,
308 const VkImage image,
309 MemoryRequirement requirement)
310 {
311 const VkMemoryRequirements memoryRequirements = getImageMemoryRequirements(vkd, device, image);
312 const VkMemoryDedicatedAllocateInfo dedicatedAllocationInfo =
313 {
314 VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO, // VkStructureType sType
315 DE_NULL, // const void* pNext
316 image, // VkImage image
317 DE_NULL // VkBuffer buffer
318 };
319
320 return allocateExtended(vki, vkd, physDevice, device, memoryRequirements, requirement, &dedicatedAllocationInfo);
321 }
322
mapMemory(const DeviceInterface & vkd,VkDevice device,VkDeviceMemory mem,VkDeviceSize offset,VkDeviceSize size,VkMemoryMapFlags flags)323 void* mapMemory (const DeviceInterface& vkd, VkDevice device, VkDeviceMemory mem, VkDeviceSize offset, VkDeviceSize size, VkMemoryMapFlags flags)
324 {
325 void* hostPtr = DE_NULL;
326 VK_CHECK(vkd.mapMemory(device, mem, offset, size, flags, &hostPtr));
327 TCU_CHECK(hostPtr);
328 return hostPtr;
329 }
330
flushMappedMemoryRange(const DeviceInterface & vkd,VkDevice device,VkDeviceMemory memory,VkDeviceSize offset,VkDeviceSize size)331 void flushMappedMemoryRange (const DeviceInterface& vkd, VkDevice device, VkDeviceMemory memory, VkDeviceSize offset, VkDeviceSize size)
332 {
333 const VkMappedMemoryRange range =
334 {
335 VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
336 DE_NULL,
337 memory,
338 offset,
339 size
340 };
341
342 VK_CHECK(vkd.flushMappedMemoryRanges(device, 1u, &range));
343 }
344
invalidateMappedMemoryRange(const DeviceInterface & vkd,VkDevice device,VkDeviceMemory memory,VkDeviceSize offset,VkDeviceSize size)345 void invalidateMappedMemoryRange (const DeviceInterface& vkd, VkDevice device, VkDeviceMemory memory, VkDeviceSize offset, VkDeviceSize size)
346 {
347 const VkMappedMemoryRange range =
348 {
349 VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
350 DE_NULL,
351 memory,
352 offset,
353 size
354 };
355
356 VK_CHECK(vkd.invalidateMappedMemoryRanges(device, 1u, &range));
357 }
358
getCompatibleMemoryTypes(const VkPhysicalDeviceMemoryProperties & deviceMemProps,MemoryRequirement requirement)359 deUint32 getCompatibleMemoryTypes (const VkPhysicalDeviceMemoryProperties& deviceMemProps, MemoryRequirement requirement)
360 {
361 deUint32 compatibleTypes = 0u;
362
363 for (deUint32 memoryTypeNdx = 0; memoryTypeNdx < deviceMemProps.memoryTypeCount; memoryTypeNdx++)
364 {
365 if (requirement.matchesHeap(deviceMemProps.memoryTypes[memoryTypeNdx].propertyFlags))
366 compatibleTypes |= (1u << memoryTypeNdx);
367 }
368
369 return compatibleTypes;
370 }
371
bindImagePlanesMemory(const DeviceInterface & vkd,const VkDevice device,const VkImage image,const deUint32 numPlanes,vector<AllocationSp> & allocations,vk::Allocator & allocator,const vk::MemoryRequirement requirement)372 void bindImagePlanesMemory (const DeviceInterface& vkd,
373 const VkDevice device,
374 const VkImage image,
375 const deUint32 numPlanes,
376 vector<AllocationSp>& allocations,
377 vk::Allocator& allocator,
378 const vk::MemoryRequirement requirement)
379 {
380 vector<VkBindImageMemoryInfo> coreInfos;
381 vector<VkBindImagePlaneMemoryInfo> planeInfos;
382 coreInfos.reserve(numPlanes);
383 planeInfos.reserve(numPlanes);
384
385 for (deUint32 planeNdx = 0; planeNdx < numPlanes; ++planeNdx)
386 {
387 const VkImageAspectFlagBits planeAspect = getPlaneAspect(planeNdx);
388 const VkMemoryRequirements reqs = getImagePlaneMemoryRequirements(vkd, device, image, planeAspect);
389
390 allocations.push_back(AllocationSp(allocator.allocate(reqs, requirement).release()));
391
392 VkBindImagePlaneMemoryInfo planeInfo =
393 {
394 VK_STRUCTURE_TYPE_BIND_IMAGE_PLANE_MEMORY_INFO,
395 DE_NULL,
396 planeAspect
397 };
398 planeInfos.push_back(planeInfo);
399
400 VkBindImageMemoryInfo coreInfo =
401 {
402 VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO,
403 &planeInfos.back(),
404 image,
405 allocations.back()->getMemory(),
406 allocations.back()->getOffset(),
407 };
408 coreInfos.push_back(coreInfo);
409 }
410
411 VK_CHECK(vkd.bindImageMemory2(device, numPlanes, coreInfos.data()));
412 }
413
bindImage(const DeviceInterface & vk,const VkDevice device,Allocator & allocator,const VkImage image,const MemoryRequirement requirement)414 MovePtr<Allocation> bindImage (const DeviceInterface& vk,
415 const VkDevice device,
416 Allocator& allocator,
417 const VkImage image,
418 const MemoryRequirement requirement)
419 {
420 MovePtr<Allocation> alloc = allocator.allocate(getImageMemoryRequirements(vk, device, image), requirement);
421 VK_CHECK(vk.bindImageMemory(device, image, alloc->getMemory(), alloc->getOffset()));
422 return alloc;
423 }
424
bindBuffer(const DeviceInterface & vk,const VkDevice device,Allocator & allocator,const VkBuffer buffer,const MemoryRequirement requirement)425 MovePtr<Allocation> bindBuffer (const DeviceInterface& vk,
426 const VkDevice device,
427 Allocator& allocator,
428 const VkBuffer buffer,
429 const MemoryRequirement requirement)
430 {
431 MovePtr<Allocation> alloc(allocator.allocate(getBufferMemoryRequirements(vk, device, buffer), requirement));
432 VK_CHECK(vk.bindBufferMemory(device, buffer, alloc->getMemory(), alloc->getOffset()));
433 return alloc;
434 }
435
zeroBuffer(const DeviceInterface & vk,const VkDevice device,const Allocation & alloc,const VkDeviceSize size)436 void zeroBuffer (const DeviceInterface& vk,
437 const VkDevice device,
438 const Allocation& alloc,
439 const VkDeviceSize size)
440 {
441 deMemset(alloc.getHostPtr(), 0, static_cast<std::size_t>(size));
442 flushAlloc(vk, device, alloc);
443 }
444
445 } // vk
446