1 // Copyright 2016 The SwiftShader Authors. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "Blitter.hpp"
16
17 #include "Pipeline/ShaderCore.hpp"
18 #include "Reactor/Reactor.hpp"
19 #include "System/CPUID.hpp"
20 #include "System/Debug.hpp"
21 #include "System/Half.hpp"
22 #include "System/Memory.hpp"
23 #include "Vulkan/VkBuffer.hpp"
24 #include "Vulkan/VkImage.hpp"
25 #include "Vulkan/VkImageView.hpp"
26
27 #include <utility>
28
29 #if defined(__i386__) || defined(__x86_64__)
30 # include <xmmintrin.h>
31 # include <emmintrin.h>
32 #endif
33
34 namespace {
PackFields(rr::Int4 const & ints,const sw::int4 shifts)35 rr::RValue<rr::Int> PackFields(rr::Int4 const &ints, const sw::int4 shifts)
36 {
37 return (rr::Int(ints.x) << shifts[0]) |
38 (rr::Int(ints.y) << shifts[1]) |
39 (rr::Int(ints.z) << shifts[2]) |
40 (rr::Int(ints.w) << shifts[3]);
41 }
42 } // namespace
43
44 namespace sw {
45
Blitter()46 Blitter::Blitter()
47 : blitMutex()
48 , blitCache(1024)
49 , cornerUpdateMutex()
50 , cornerUpdateCache(64) // We only need one of these per format
51 {
52 }
53
~Blitter()54 Blitter::~Blitter()
55 {
56 }
57
clear(void * pixel,vk::Format format,vk::Image * dest,const vk::Format & viewFormat,const VkImageSubresourceRange & subresourceRange,const VkRect2D * renderArea)58 void Blitter::clear(void *pixel, vk::Format format, vk::Image *dest, const vk::Format &viewFormat, const VkImageSubresourceRange &subresourceRange, const VkRect2D *renderArea)
59 {
60 VkImageAspectFlagBits aspect = static_cast<VkImageAspectFlagBits>(subresourceRange.aspectMask);
61 vk::Format dstFormat = viewFormat.getAspectFormat(aspect);
62 if(dstFormat == VK_FORMAT_UNDEFINED)
63 {
64 return;
65 }
66
67 float *pPixel = static_cast<float *>(pixel);
68 if(viewFormat.isUnsignedNormalized() || viewFormat.isSRGBformat())
69 {
70 pPixel[0] = sw::clamp(pPixel[0], 0.0f, 1.0f);
71 pPixel[1] = sw::clamp(pPixel[1], 0.0f, 1.0f);
72 pPixel[2] = sw::clamp(pPixel[2], 0.0f, 1.0f);
73 pPixel[3] = sw::clamp(pPixel[3], 0.0f, 1.0f);
74 }
75 else if(viewFormat.isSignedNormalized())
76 {
77 pPixel[0] = sw::clamp(pPixel[0], -1.0f, 1.0f);
78 pPixel[1] = sw::clamp(pPixel[1], -1.0f, 1.0f);
79 pPixel[2] = sw::clamp(pPixel[2], -1.0f, 1.0f);
80 pPixel[3] = sw::clamp(pPixel[3], -1.0f, 1.0f);
81 }
82
83 if(fastClear(pixel, format, dest, dstFormat, subresourceRange, renderArea))
84 {
85 return;
86 }
87
88 State state(format, dstFormat, 1, dest->getSampleCountFlagBits(), Options{ 0xF });
89 auto blitRoutine = getBlitRoutine(state);
90 if(!blitRoutine)
91 {
92 return;
93 }
94
95 VkImageSubresource subres = {
96 subresourceRange.aspectMask,
97 subresourceRange.baseMipLevel,
98 subresourceRange.baseArrayLayer
99 };
100
101 uint32_t lastMipLevel = dest->getLastMipLevel(subresourceRange);
102 uint32_t lastLayer = dest->getLastLayerIndex(subresourceRange);
103
104 VkRect2D area = { { 0, 0 }, { 0, 0 } };
105 if(renderArea)
106 {
107 ASSERT(subresourceRange.levelCount == 1);
108 area = *renderArea;
109 }
110
111 for(; subres.mipLevel <= lastMipLevel; subres.mipLevel++)
112 {
113 VkExtent3D extent = dest->getMipLevelExtent(aspect, subres.mipLevel);
114 if(!renderArea)
115 {
116 area.extent.width = extent.width;
117 area.extent.height = extent.height;
118 }
119
120 BlitData data = {
121 pixel, nullptr, // source, dest
122
123 format.bytes(), // sPitchB
124 dest->rowPitchBytes(aspect, subres.mipLevel), // dPitchB
125 0, // sSliceB (unused in clear operations)
126 dest->slicePitchBytes(aspect, subres.mipLevel), // dSliceB
127
128 0.5f, 0.5f, 0.5f, 0.0f, 0.0f, 0.0f, // x0, y0, z0, w, h, d
129
130 area.offset.x, static_cast<int>(area.offset.x + area.extent.width), // x0d, x1d
131 area.offset.y, static_cast<int>(area.offset.y + area.extent.height), // y0d, y1d
132 0, 1, // z0d, z1d
133
134 0, 0, 0, // sWidth, sHeight, sDepth
135
136 false, // filter3D
137 };
138
139 if(renderArea && dest->is3DSlice())
140 {
141 // Reinterpret layers as depth slices
142 subres.arrayLayer = 0;
143 for(uint32_t depth = subresourceRange.baseArrayLayer; depth <= lastLayer; depth++)
144 {
145 data.dest = dest->getTexelPointer({ 0, 0, static_cast<int32_t>(depth) }, subres);
146 blitRoutine(&data);
147 }
148 }
149 else
150 {
151 for(subres.arrayLayer = subresourceRange.baseArrayLayer; subres.arrayLayer <= lastLayer; subres.arrayLayer++)
152 {
153 for(uint32_t depth = 0; depth < extent.depth; depth++)
154 {
155 data.dest = dest->getTexelPointer({ 0, 0, static_cast<int32_t>(depth) }, subres);
156
157 blitRoutine(&data);
158 }
159 }
160 }
161 }
162 dest->contentsChanged(subresourceRange);
163 }
164
fastClear(void * clearValue,vk::Format clearFormat,vk::Image * dest,const vk::Format & viewFormat,const VkImageSubresourceRange & subresourceRange,const VkRect2D * renderArea)165 bool Blitter::fastClear(void *clearValue, vk::Format clearFormat, vk::Image *dest, const vk::Format &viewFormat, const VkImageSubresourceRange &subresourceRange, const VkRect2D *renderArea)
166 {
167 if(clearFormat != VK_FORMAT_R32G32B32A32_SFLOAT &&
168 clearFormat != VK_FORMAT_D32_SFLOAT &&
169 clearFormat != VK_FORMAT_S8_UINT)
170 {
171 return false;
172 }
173
174 union ClearValue
175 {
176 struct
177 {
178 float r;
179 float g;
180 float b;
181 float a;
182 };
183
184 float rgb[3];
185
186 float d;
187 uint32_t d_as_u32;
188
189 uint32_t s;
190 };
191
192 ClearValue &c = *reinterpret_cast<ClearValue *>(clearValue);
193
194 uint32_t packed = 0;
195
196 VkImageAspectFlagBits aspect = static_cast<VkImageAspectFlagBits>(subresourceRange.aspectMask);
197 switch(viewFormat)
198 {
199 case VK_FORMAT_R5G6B5_UNORM_PACK16:
200 packed = ((uint16_t)(31 * c.b + 0.5f) << 0) |
201 ((uint16_t)(63 * c.g + 0.5f) << 5) |
202 ((uint16_t)(31 * c.r + 0.5f) << 11);
203 break;
204 case VK_FORMAT_B5G6R5_UNORM_PACK16:
205 packed = ((uint16_t)(31 * c.r + 0.5f) << 0) |
206 ((uint16_t)(63 * c.g + 0.5f) << 5) |
207 ((uint16_t)(31 * c.b + 0.5f) << 11);
208 break;
209 case VK_FORMAT_A8B8G8R8_UINT_PACK32:
210 case VK_FORMAT_A8B8G8R8_UNORM_PACK32:
211 case VK_FORMAT_R8G8B8A8_UNORM:
212 packed = ((uint32_t)(255 * c.a + 0.5f) << 24) |
213 ((uint32_t)(255 * c.b + 0.5f) << 16) |
214 ((uint32_t)(255 * c.g + 0.5f) << 8) |
215 ((uint32_t)(255 * c.r + 0.5f) << 0);
216 break;
217 case VK_FORMAT_B8G8R8A8_UNORM:
218 packed = ((uint32_t)(255 * c.a + 0.5f) << 24) |
219 ((uint32_t)(255 * c.r + 0.5f) << 16) |
220 ((uint32_t)(255 * c.g + 0.5f) << 8) |
221 ((uint32_t)(255 * c.b + 0.5f) << 0);
222 break;
223 case VK_FORMAT_B10G11R11_UFLOAT_PACK32:
224 packed = R11G11B10F(c.rgb);
225 break;
226 case VK_FORMAT_E5B9G9R9_UFLOAT_PACK32:
227 packed = RGB9E5(c.rgb);
228 break;
229 case VK_FORMAT_D32_SFLOAT:
230 ASSERT(clearFormat == VK_FORMAT_D32_SFLOAT);
231 packed = c.d_as_u32; // float reinterpreted as uint32
232 break;
233 case VK_FORMAT_S8_UINT:
234 ASSERT(clearFormat == VK_FORMAT_S8_UINT);
235 packed = static_cast<uint8_t>(c.s);
236 break;
237 default:
238 return false;
239 }
240
241 VkImageSubresource subres = {
242 subresourceRange.aspectMask,
243 subresourceRange.baseMipLevel,
244 subresourceRange.baseArrayLayer
245 };
246 uint32_t lastMipLevel = dest->getLastMipLevel(subresourceRange);
247 uint32_t lastLayer = dest->getLastLayerIndex(subresourceRange);
248
249 VkRect2D area = { { 0, 0 }, { 0, 0 } };
250 if(renderArea)
251 {
252 ASSERT(subresourceRange.levelCount == 1);
253 area = *renderArea;
254 }
255
256 for(; subres.mipLevel <= lastMipLevel; subres.mipLevel++)
257 {
258 int rowPitchBytes = dest->rowPitchBytes(aspect, subres.mipLevel);
259 int slicePitchBytes = dest->slicePitchBytes(aspect, subres.mipLevel);
260 VkExtent3D extent = dest->getMipLevelExtent(aspect, subres.mipLevel);
261 if(!renderArea)
262 {
263 area.extent.width = extent.width;
264 area.extent.height = extent.height;
265 }
266 if(dest->is3DSlice())
267 {
268 extent.depth = 1; // The 3D image is instead interpreted as a 2D image with layers
269 }
270
271 for(subres.arrayLayer = subresourceRange.baseArrayLayer; subres.arrayLayer <= lastLayer; subres.arrayLayer++)
272 {
273 for(uint32_t depth = 0; depth < extent.depth; depth++)
274 {
275 uint8_t *slice = (uint8_t *)dest->getTexelPointer(
276 { area.offset.x, area.offset.y, static_cast<int32_t>(depth) }, subres);
277
278 for(int j = 0; j < dest->getSampleCountFlagBits(); j++)
279 {
280 uint8_t *d = slice;
281
282 switch(viewFormat.bytes())
283 {
284 case 4:
285 for(uint32_t i = 0; i < area.extent.height; i++)
286 {
287 ASSERT(d < dest->end());
288 sw::clear((uint32_t *)d, packed, area.extent.width);
289 d += rowPitchBytes;
290 }
291 break;
292 case 2:
293 for(uint32_t i = 0; i < area.extent.height; i++)
294 {
295 ASSERT(d < dest->end());
296 sw::clear((uint16_t *)d, static_cast<uint16_t>(packed), area.extent.width);
297 d += rowPitchBytes;
298 }
299 break;
300 case 1:
301 for(uint32_t i = 0; i < area.extent.height; i++)
302 {
303 ASSERT(d < dest->end());
304 memset(d, packed, area.extent.width);
305 d += rowPitchBytes;
306 }
307 break;
308 default:
309 assert(false);
310 }
311
312 slice += slicePitchBytes;
313 }
314 }
315 }
316 }
317 dest->contentsChanged(subresourceRange);
318
319 return true;
320 }
321
readFloat4(Pointer<Byte> element,const State & state)322 Float4 Blitter::readFloat4(Pointer<Byte> element, const State &state)
323 {
324 Float4 c(0.0f, 0.0f, 0.0f, 1.0f);
325
326 switch(state.sourceFormat)
327 {
328 case VK_FORMAT_B4G4R4A4_UNORM_PACK16:
329 c.w = Float(Int(*Pointer<Byte>(element)) & Int(0xF));
330 c.x = Float((Int(*Pointer<Byte>(element)) >> 4) & Int(0xF));
331 c.y = Float(Int(*Pointer<Byte>(element + 1)) & Int(0xF));
332 c.z = Float((Int(*Pointer<Byte>(element + 1)) >> 4) & Int(0xF));
333 break;
334 case VK_FORMAT_R8_SINT:
335 case VK_FORMAT_R8_SNORM:
336 c.x = Float(Int(*Pointer<SByte>(element)));
337 c.w = float(0x7F);
338 break;
339 case VK_FORMAT_R8_UNORM:
340 case VK_FORMAT_R8_UINT:
341 case VK_FORMAT_R8_SRGB:
342 c.x = Float(Int(*Pointer<Byte>(element)));
343 c.w = float(0xFF);
344 break;
345 case VK_FORMAT_R16_SINT:
346 case VK_FORMAT_R16_SNORM:
347 c.x = Float(Int(*Pointer<Short>(element)));
348 c.w = float(0x7FFF);
349 break;
350 case VK_FORMAT_R16_UNORM:
351 case VK_FORMAT_R16_UINT:
352 c.x = Float(Int(*Pointer<UShort>(element)));
353 c.w = float(0xFFFF);
354 break;
355 case VK_FORMAT_R32_SINT:
356 c.x = Float(*Pointer<Int>(element));
357 c.w = float(0x7FFFFFFF);
358 break;
359 case VK_FORMAT_R32_UINT:
360 c.x = Float(*Pointer<UInt>(element));
361 c.w = float(0xFFFFFFFF);
362 break;
363 case VK_FORMAT_B8G8R8A8_SRGB:
364 case VK_FORMAT_B8G8R8A8_UNORM:
365 c = Float4(*Pointer<Byte4>(element)).zyxw;
366 break;
367 case VK_FORMAT_A8B8G8R8_SINT_PACK32:
368 case VK_FORMAT_R8G8B8A8_SINT:
369 case VK_FORMAT_A8B8G8R8_SNORM_PACK32:
370 case VK_FORMAT_R8G8B8A8_SNORM:
371 c = Float4(*Pointer<SByte4>(element));
372 break;
373 case VK_FORMAT_A8B8G8R8_UINT_PACK32:
374 case VK_FORMAT_A8B8G8R8_UNORM_PACK32:
375 case VK_FORMAT_R8G8B8A8_UNORM:
376 case VK_FORMAT_R8G8B8A8_UINT:
377 case VK_FORMAT_A8B8G8R8_SRGB_PACK32:
378 case VK_FORMAT_R8G8B8A8_SRGB:
379 c = Float4(*Pointer<Byte4>(element));
380 break;
381 case VK_FORMAT_R16G16B16A16_SINT:
382 case VK_FORMAT_R16G16B16A16_SNORM:
383 c = Float4(*Pointer<Short4>(element));
384 break;
385 case VK_FORMAT_R16G16B16A16_UNORM:
386 case VK_FORMAT_R16G16B16A16_UINT:
387 c = Float4(*Pointer<UShort4>(element));
388 break;
389 case VK_FORMAT_R32G32B32A32_SINT:
390 c = Float4(*Pointer<Int4>(element));
391 break;
392 case VK_FORMAT_R32G32B32A32_UINT:
393 c = Float4(*Pointer<UInt4>(element));
394 break;
395 case VK_FORMAT_R8G8_SINT:
396 case VK_FORMAT_R8G8_SNORM:
397 c.x = Float(Int(*Pointer<SByte>(element + 0)));
398 c.y = Float(Int(*Pointer<SByte>(element + 1)));
399 c.w = float(0x7F);
400 break;
401 case VK_FORMAT_R8G8_UNORM:
402 case VK_FORMAT_R8G8_UINT:
403 case VK_FORMAT_R8G8_SRGB:
404 c.x = Float(Int(*Pointer<Byte>(element + 0)));
405 c.y = Float(Int(*Pointer<Byte>(element + 1)));
406 c.w = float(0xFF);
407 break;
408 case VK_FORMAT_R16G16_SINT:
409 case VK_FORMAT_R16G16_SNORM:
410 c.x = Float(Int(*Pointer<Short>(element + 0)));
411 c.y = Float(Int(*Pointer<Short>(element + 2)));
412 c.w = float(0x7FFF);
413 break;
414 case VK_FORMAT_R16G16_UNORM:
415 case VK_FORMAT_R16G16_UINT:
416 c.x = Float(Int(*Pointer<UShort>(element + 0)));
417 c.y = Float(Int(*Pointer<UShort>(element + 2)));
418 c.w = float(0xFFFF);
419 break;
420 case VK_FORMAT_R32G32_SINT:
421 c.x = Float(*Pointer<Int>(element + 0));
422 c.y = Float(*Pointer<Int>(element + 4));
423 c.w = float(0x7FFFFFFF);
424 break;
425 case VK_FORMAT_R32G32_UINT:
426 c.x = Float(*Pointer<UInt>(element + 0));
427 c.y = Float(*Pointer<UInt>(element + 4));
428 c.w = float(0xFFFFFFFF);
429 break;
430 case VK_FORMAT_R32G32B32A32_SFLOAT:
431 c = *Pointer<Float4>(element);
432 break;
433 case VK_FORMAT_R32G32_SFLOAT:
434 c.x = *Pointer<Float>(element + 0);
435 c.y = *Pointer<Float>(element + 4);
436 break;
437 case VK_FORMAT_R32_SFLOAT:
438 c.x = *Pointer<Float>(element);
439 break;
440 case VK_FORMAT_R16G16B16A16_SFLOAT:
441 c.w = Float(*Pointer<Half>(element + 6));
442 case VK_FORMAT_R16G16B16_SFLOAT:
443 c.z = Float(*Pointer<Half>(element + 4));
444 case VK_FORMAT_R16G16_SFLOAT:
445 c.y = Float(*Pointer<Half>(element + 2));
446 case VK_FORMAT_R16_SFLOAT:
447 c.x = Float(*Pointer<Half>(element));
448 break;
449 case VK_FORMAT_B10G11R11_UFLOAT_PACK32:
450 c = r11g11b10Unpack(*Pointer<UInt>(element));
451 break;
452 case VK_FORMAT_E5B9G9R9_UFLOAT_PACK32:
453 // This type contains a common 5 bit exponent (E) and a 9 bit the mantissa for R, G and B.
454 c.x = Float(*Pointer<UInt>(element) & UInt(0x000001FF)); // R's mantissa (bits 0-8)
455 c.y = Float((*Pointer<UInt>(element) & UInt(0x0003FE00)) >> 9); // G's mantissa (bits 9-17)
456 c.z = Float((*Pointer<UInt>(element) & UInt(0x07FC0000)) >> 18); // B's mantissa (bits 18-26)
457 c *= Float4(
458 // 2^E, using the exponent (bits 27-31) and treating it as an unsigned integer value
459 Float(UInt(1) << ((*Pointer<UInt>(element) & UInt(0xF8000000)) >> 27)) *
460 // Since the 9 bit mantissa values currently stored in RGB were converted straight
461 // from int to float (in the [0, 1<<9] range instead of the [0, 1] range), they
462 // are (1 << 9) times too high.
463 // Also, the exponent has 5 bits and we compute the exponent bias of floating point
464 // formats using "2^(k-1) - 1", so, in this case, the exponent bias is 2^(5-1)-1 = 15
465 // Exponent bias (15) + number of mantissa bits per component (9) = 24
466 Float(1.0f / (1 << 24)));
467 c.w = 1.0f;
468 break;
469 case VK_FORMAT_R5G6B5_UNORM_PACK16:
470 c.x = Float(Int((*Pointer<UShort>(element) & UShort(0xF800)) >> UShort(11)));
471 c.y = Float(Int((*Pointer<UShort>(element) & UShort(0x07E0)) >> UShort(5)));
472 c.z = Float(Int(*Pointer<UShort>(element) & UShort(0x001F)));
473 break;
474 case VK_FORMAT_A1R5G5B5_UNORM_PACK16:
475 c.w = Float(Int((*Pointer<UShort>(element) & UShort(0x8000)) >> UShort(15)));
476 c.x = Float(Int((*Pointer<UShort>(element) & UShort(0x7C00)) >> UShort(10)));
477 c.y = Float(Int((*Pointer<UShort>(element) & UShort(0x03E0)) >> UShort(5)));
478 c.z = Float(Int(*Pointer<UShort>(element) & UShort(0x001F)));
479 break;
480 case VK_FORMAT_A2B10G10R10_UNORM_PACK32:
481 case VK_FORMAT_A2B10G10R10_UINT_PACK32:
482 c.x = Float(Int((*Pointer<UInt>(element) & UInt(0x000003FF))));
483 c.y = Float(Int((*Pointer<UInt>(element) & UInt(0x000FFC00)) >> 10));
484 c.z = Float(Int((*Pointer<UInt>(element) & UInt(0x3FF00000)) >> 20));
485 c.w = Float(Int((*Pointer<UInt>(element) & UInt(0xC0000000)) >> 30));
486 break;
487 case VK_FORMAT_A2R10G10B10_UNORM_PACK32:
488 case VK_FORMAT_A2R10G10B10_UINT_PACK32:
489 c.z = Float(Int((*Pointer<UInt>(element) & UInt(0x000003FF))));
490 c.y = Float(Int((*Pointer<UInt>(element) & UInt(0x000FFC00)) >> 10));
491 c.x = Float(Int((*Pointer<UInt>(element) & UInt(0x3FF00000)) >> 20));
492 c.w = Float(Int((*Pointer<UInt>(element) & UInt(0xC0000000)) >> 30));
493 break;
494 case VK_FORMAT_D16_UNORM:
495 c.x = Float(Int((*Pointer<UShort>(element))));
496 break;
497 case VK_FORMAT_X8_D24_UNORM_PACK32:
498 c.x = Float(Int((*Pointer<UInt>(element) & UInt(0xFFFFFF00)) >> 8));
499 break;
500 case VK_FORMAT_D32_SFLOAT:
501 c.x = *Pointer<Float>(element);
502 break;
503 case VK_FORMAT_S8_UINT:
504 c.x = Float(Int(*Pointer<Byte>(element)));
505 break;
506 default:
507 UNSUPPORTED("Blitter source format %d", (int)state.sourceFormat);
508 }
509
510 return c;
511 }
512
write(Float4 & c,Pointer<Byte> element,const State & state)513 void Blitter::write(Float4 &c, Pointer<Byte> element, const State &state)
514 {
515 bool writeR = state.writeRed;
516 bool writeG = state.writeGreen;
517 bool writeB = state.writeBlue;
518 bool writeA = state.writeAlpha;
519 bool writeRGBA = writeR && writeG && writeB && writeA;
520
521 switch(state.destFormat)
522 {
523 case VK_FORMAT_R4G4_UNORM_PACK8:
524 if(writeR | writeG)
525 {
526 if(!writeR)
527 {
528 *Pointer<Byte>(element) = (Byte(RoundInt(Float(c.y))) & Byte(0xF)) |
529 (*Pointer<Byte>(element) & Byte(0xF0));
530 }
531 else if(!writeG)
532 {
533 *Pointer<Byte>(element) = (*Pointer<Byte>(element) & Byte(0xF)) |
534 (Byte(RoundInt(Float(c.x))) << Byte(4));
535 }
536 else
537 {
538 *Pointer<Byte>(element) = (Byte(RoundInt(Float(c.y))) & Byte(0xF)) |
539 (Byte(RoundInt(Float(c.x))) << Byte(4));
540 }
541 }
542 break;
543 case VK_FORMAT_R4G4B4A4_UNORM_PACK16:
544 if(writeR || writeG || writeB || writeA)
545 {
546 *Pointer<UShort>(element) = (writeR ? ((UShort(RoundInt(Float(c.x))) & UShort(0xF)) << UShort(12)) : (*Pointer<UShort>(element) & UShort(0x000F))) |
547 (writeG ? ((UShort(RoundInt(Float(c.y))) & UShort(0xF)) << UShort(8)) : (*Pointer<UShort>(element) & UShort(0x00F0))) |
548 (writeB ? ((UShort(RoundInt(Float(c.z))) & UShort(0xF)) << UShort(4)) : (*Pointer<UShort>(element) & UShort(0x0F00))) |
549 (writeA ? (UShort(RoundInt(Float(c.w))) & UShort(0xF)) : (*Pointer<UShort>(element) & UShort(0xF000)));
550 }
551 break;
552 case VK_FORMAT_B4G4R4A4_UNORM_PACK16:
553 if(writeRGBA)
554 {
555 *Pointer<UShort>(element) = UShort(RoundInt(Float(c.w)) & Int(0xF)) |
556 UShort((RoundInt(Float(c.x)) & Int(0xF)) << 4) |
557 UShort((RoundInt(Float(c.y)) & Int(0xF)) << 8) |
558 UShort((RoundInt(Float(c.z)) & Int(0xF)) << 12);
559 }
560 else
561 {
562 unsigned short mask = (writeA ? 0x000F : 0x0000) |
563 (writeR ? 0x00F0 : 0x0000) |
564 (writeG ? 0x0F00 : 0x0000) |
565 (writeB ? 0xF000 : 0x0000);
566 unsigned short unmask = ~mask;
567 *Pointer<UShort>(element) = (*Pointer<UShort>(element) & UShort(unmask)) |
568 ((UShort(RoundInt(Float(c.w)) & Int(0xF)) |
569 UShort((RoundInt(Float(c.x)) & Int(0xF)) << 4) |
570 UShort((RoundInt(Float(c.y)) & Int(0xF)) << 8) |
571 UShort((RoundInt(Float(c.z)) & Int(0xF)) << 12)) &
572 UShort(mask));
573 }
574 break;
575 case VK_FORMAT_B8G8R8A8_SRGB:
576 case VK_FORMAT_B8G8R8A8_UNORM:
577 if(writeRGBA)
578 {
579 Short4 c0 = RoundShort4(c.zyxw);
580 *Pointer<Byte4>(element) = Byte4(PackUnsigned(c0, c0));
581 }
582 else
583 {
584 if(writeB) { *Pointer<Byte>(element + 0) = Byte(RoundInt(Float(c.z))); }
585 if(writeG) { *Pointer<Byte>(element + 1) = Byte(RoundInt(Float(c.y))); }
586 if(writeR) { *Pointer<Byte>(element + 2) = Byte(RoundInt(Float(c.x))); }
587 if(writeA) { *Pointer<Byte>(element + 3) = Byte(RoundInt(Float(c.w))); }
588 }
589 break;
590 case VK_FORMAT_B8G8R8_SNORM:
591 if(writeB) { *Pointer<SByte>(element + 0) = SByte(RoundInt(Float(c.z))); }
592 if(writeG) { *Pointer<SByte>(element + 1) = SByte(RoundInt(Float(c.y))); }
593 if(writeR) { *Pointer<SByte>(element + 2) = SByte(RoundInt(Float(c.x))); }
594 break;
595 case VK_FORMAT_B8G8R8_UNORM:
596 case VK_FORMAT_B8G8R8_SRGB:
597 if(writeB) { *Pointer<Byte>(element + 0) = Byte(RoundInt(Float(c.z))); }
598 if(writeG) { *Pointer<Byte>(element + 1) = Byte(RoundInt(Float(c.y))); }
599 if(writeR) { *Pointer<Byte>(element + 2) = Byte(RoundInt(Float(c.x))); }
600 break;
601 case VK_FORMAT_A8B8G8R8_UNORM_PACK32:
602 case VK_FORMAT_R8G8B8A8_UNORM:
603 case VK_FORMAT_A8B8G8R8_SRGB_PACK32:
604 case VK_FORMAT_R8G8B8A8_SRGB:
605 case VK_FORMAT_A8B8G8R8_UINT_PACK32:
606 case VK_FORMAT_R8G8B8A8_UINT:
607 case VK_FORMAT_R8G8B8A8_USCALED:
608 case VK_FORMAT_A8B8G8R8_USCALED_PACK32:
609 if(writeRGBA)
610 {
611 Short4 c0 = RoundShort4(c);
612 *Pointer<Byte4>(element) = Byte4(PackUnsigned(c0, c0));
613 }
614 else
615 {
616 if(writeR) { *Pointer<Byte>(element + 0) = Byte(RoundInt(Float(c.x))); }
617 if(writeG) { *Pointer<Byte>(element + 1) = Byte(RoundInt(Float(c.y))); }
618 if(writeB) { *Pointer<Byte>(element + 2) = Byte(RoundInt(Float(c.z))); }
619 if(writeA) { *Pointer<Byte>(element + 3) = Byte(RoundInt(Float(c.w))); }
620 }
621 break;
622 case VK_FORMAT_R32G32B32A32_SFLOAT:
623 if(writeRGBA)
624 {
625 *Pointer<Float4>(element) = c;
626 }
627 else
628 {
629 if(writeR) { *Pointer<Float>(element) = c.x; }
630 if(writeG) { *Pointer<Float>(element + 4) = c.y; }
631 if(writeB) { *Pointer<Float>(element + 8) = c.z; }
632 if(writeA) { *Pointer<Float>(element + 12) = c.w; }
633 }
634 break;
635 case VK_FORMAT_R32G32B32_SFLOAT:
636 if(writeR) { *Pointer<Float>(element) = c.x; }
637 if(writeG) { *Pointer<Float>(element + 4) = c.y; }
638 if(writeB) { *Pointer<Float>(element + 8) = c.z; }
639 break;
640 case VK_FORMAT_R32G32_SFLOAT:
641 if(writeR && writeG)
642 {
643 *Pointer<Float2>(element) = Float2(c);
644 }
645 else
646 {
647 if(writeR) { *Pointer<Float>(element) = c.x; }
648 if(writeG) { *Pointer<Float>(element + 4) = c.y; }
649 }
650 break;
651 case VK_FORMAT_R32_SFLOAT:
652 if(writeR) { *Pointer<Float>(element) = c.x; }
653 break;
654 case VK_FORMAT_R16G16B16A16_SFLOAT:
655 if(writeA) { *Pointer<Half>(element + 6) = Half(c.w); }
656 // [[fallthrough]]
657 case VK_FORMAT_R16G16B16_SFLOAT:
658 if(writeB) { *Pointer<Half>(element + 4) = Half(c.z); }
659 // [[fallthrough]]
660 case VK_FORMAT_R16G16_SFLOAT:
661 if(writeG) { *Pointer<Half>(element + 2) = Half(c.y); }
662 // [[fallthrough]]
663 case VK_FORMAT_R16_SFLOAT:
664 if(writeR) { *Pointer<Half>(element) = Half(c.x); }
665 break;
666 case VK_FORMAT_B10G11R11_UFLOAT_PACK32:
667 {
668 UInt rgb = r11g11b10Pack(c);
669
670 UInt old = *Pointer<UInt>(element);
671
672 unsigned int mask = (writeR ? 0x000007FF : 0) |
673 (writeG ? 0x003FF800 : 0) |
674 (writeB ? 0xFFC00000 : 0);
675
676 *Pointer<UInt>(element) = (rgb & mask) | (old & ~mask);
677 }
678 break;
679 case VK_FORMAT_E5B9G9R9_UFLOAT_PACK32:
680 {
681 ASSERT(writeRGBA); // Can't sensibly write just part of this format.
682
683 // Vulkan 1.1.117 section 15.2.1 RGB to Shared Exponent Conversion
684
685 constexpr int N = 9; // number of mantissa bits per component
686 constexpr int B = 15; // exponent bias
687 constexpr int E_max = 31; // maximum possible biased exponent value
688
689 // Maximum representable value.
690 constexpr float sharedexp_max = ((static_cast<float>(1 << N) - 1) / static_cast<float>(1 << N)) * static_cast<float>(1 << (E_max - B));
691
692 // Clamp components to valid range. NaN becomes 0.
693 Float red_c = Min(IfThenElse(!(c.x > 0), Float(0), Float(c.x)), sharedexp_max);
694 Float green_c = Min(IfThenElse(!(c.y > 0), Float(0), Float(c.y)), sharedexp_max);
695 Float blue_c = Min(IfThenElse(!(c.z > 0), Float(0), Float(c.z)), sharedexp_max);
696
697 // We're reducing the mantissa to 9 bits, so we must round up if the next
698 // bit is 1. In other words add 0.5 to the new mantissa's position and
699 // allow overflow into the exponent so we can scale correctly.
700 constexpr int half = 1 << (23 - N);
701 Float red_r = As<Float>(As<Int>(red_c) + half);
702 Float green_r = As<Float>(As<Int>(green_c) + half);
703 Float blue_r = As<Float>(As<Int>(blue_c) + half);
704
705 // The largest component determines the shared exponent. It can't be lower
706 // than 0 (after bias subtraction) so also limit to the mimimum representable.
707 constexpr float min_s = 0.5f / (1 << B);
708 Float max_s = Max(Max(red_r, green_r), Max(blue_r, min_s));
709
710 // Obtain the reciprocal of the shared exponent by inverting the bits,
711 // and scale by the new mantissa's size. Note that the IEEE-754 single-precision
712 // format has an implicit leading 1, but this shared component format does not.
713 Float scale = As<Float>((As<Int>(max_s) & 0x7F800000) ^ 0x7F800000) * (1 << (N - 2));
714
715 UInt R9 = RoundInt(red_c * scale);
716 UInt G9 = UInt(RoundInt(green_c * scale));
717 UInt B9 = UInt(RoundInt(blue_c * scale));
718 UInt E5 = (As<UInt>(max_s) >> 23) - 127 + 15 + 1;
719
720 UInt E5B9G9R9 = (E5 << 27) | (B9 << 18) | (G9 << 9) | R9;
721
722 *Pointer<UInt>(element) = E5B9G9R9;
723 }
724 break;
725 case VK_FORMAT_B8G8R8A8_SNORM:
726 if(writeB) { *Pointer<SByte>(element) = SByte(RoundInt(Float(c.z))); }
727 if(writeG) { *Pointer<SByte>(element + 1) = SByte(RoundInt(Float(c.y))); }
728 if(writeR) { *Pointer<SByte>(element + 2) = SByte(RoundInt(Float(c.x))); }
729 if(writeA) { *Pointer<SByte>(element + 3) = SByte(RoundInt(Float(c.w))); }
730 break;
731 case VK_FORMAT_A8B8G8R8_SINT_PACK32:
732 case VK_FORMAT_R8G8B8A8_SINT:
733 case VK_FORMAT_A8B8G8R8_SNORM_PACK32:
734 case VK_FORMAT_R8G8B8A8_SNORM:
735 case VK_FORMAT_R8G8B8A8_SSCALED:
736 case VK_FORMAT_A8B8G8R8_SSCALED_PACK32:
737 if(writeA) { *Pointer<SByte>(element + 3) = SByte(RoundInt(Float(c.w))); }
738 // [[fallthrough]]
739 case VK_FORMAT_R8G8B8_SINT:
740 case VK_FORMAT_R8G8B8_SNORM:
741 case VK_FORMAT_R8G8B8_SSCALED:
742 if(writeB) { *Pointer<SByte>(element + 2) = SByte(RoundInt(Float(c.z))); }
743 // [[fallthrough]]
744 case VK_FORMAT_R8G8_SINT:
745 case VK_FORMAT_R8G8_SNORM:
746 case VK_FORMAT_R8G8_SSCALED:
747 if(writeG) { *Pointer<SByte>(element + 1) = SByte(RoundInt(Float(c.y))); }
748 // [[fallthrough]]
749 case VK_FORMAT_R8_SINT:
750 case VK_FORMAT_R8_SNORM:
751 case VK_FORMAT_R8_SSCALED:
752 if(writeR) { *Pointer<SByte>(element) = SByte(RoundInt(Float(c.x))); }
753 break;
754 case VK_FORMAT_R8G8B8_UINT:
755 case VK_FORMAT_R8G8B8_UNORM:
756 case VK_FORMAT_R8G8B8_USCALED:
757 case VK_FORMAT_R8G8B8_SRGB:
758 if(writeB) { *Pointer<Byte>(element + 2) = Byte(RoundInt(Float(c.z))); }
759 // [[fallthrough]]
760 case VK_FORMAT_R8G8_UINT:
761 case VK_FORMAT_R8G8_UNORM:
762 case VK_FORMAT_R8G8_USCALED:
763 case VK_FORMAT_R8G8_SRGB:
764 if(writeG) { *Pointer<Byte>(element + 1) = Byte(RoundInt(Float(c.y))); }
765 // [[fallthrough]]
766 case VK_FORMAT_R8_UINT:
767 case VK_FORMAT_R8_UNORM:
768 case VK_FORMAT_R8_USCALED:
769 case VK_FORMAT_R8_SRGB:
770 if(writeR) { *Pointer<Byte>(element) = Byte(RoundInt(Float(c.x))); }
771 break;
772 case VK_FORMAT_R16G16B16A16_SINT:
773 case VK_FORMAT_R16G16B16A16_SNORM:
774 case VK_FORMAT_R16G16B16A16_SSCALED:
775 if(writeRGBA)
776 {
777 *Pointer<Short4>(element) = Short4(RoundInt(c));
778 }
779 else
780 {
781 if(writeR) { *Pointer<Short>(element) = Short(RoundInt(Float(c.x))); }
782 if(writeG) { *Pointer<Short>(element + 2) = Short(RoundInt(Float(c.y))); }
783 if(writeB) { *Pointer<Short>(element + 4) = Short(RoundInt(Float(c.z))); }
784 if(writeA) { *Pointer<Short>(element + 6) = Short(RoundInt(Float(c.w))); }
785 }
786 break;
787 case VK_FORMAT_R16G16B16_SINT:
788 case VK_FORMAT_R16G16B16_SNORM:
789 case VK_FORMAT_R16G16B16_SSCALED:
790 if(writeR) { *Pointer<Short>(element) = Short(RoundInt(Float(c.x))); }
791 if(writeG) { *Pointer<Short>(element + 2) = Short(RoundInt(Float(c.y))); }
792 if(writeB) { *Pointer<Short>(element + 4) = Short(RoundInt(Float(c.z))); }
793 break;
794 case VK_FORMAT_R16G16_SINT:
795 case VK_FORMAT_R16G16_SNORM:
796 case VK_FORMAT_R16G16_SSCALED:
797 if(writeR && writeG)
798 {
799 *Pointer<Short2>(element) = Short2(Short4(RoundInt(c)));
800 }
801 else
802 {
803 if(writeR) { *Pointer<Short>(element) = Short(RoundInt(Float(c.x))); }
804 if(writeG) { *Pointer<Short>(element + 2) = Short(RoundInt(Float(c.y))); }
805 }
806 break;
807 case VK_FORMAT_R16_SINT:
808 case VK_FORMAT_R16_SNORM:
809 case VK_FORMAT_R16_SSCALED:
810 if(writeR) { *Pointer<Short>(element) = Short(RoundInt(Float(c.x))); }
811 break;
812 case VK_FORMAT_R16G16B16A16_UINT:
813 case VK_FORMAT_R16G16B16A16_UNORM:
814 case VK_FORMAT_R16G16B16A16_USCALED:
815 if(writeRGBA)
816 {
817 *Pointer<UShort4>(element) = UShort4(RoundInt(c));
818 }
819 else
820 {
821 if(writeR) { *Pointer<UShort>(element) = UShort(RoundInt(Float(c.x))); }
822 if(writeG) { *Pointer<UShort>(element + 2) = UShort(RoundInt(Float(c.y))); }
823 if(writeB) { *Pointer<UShort>(element + 4) = UShort(RoundInt(Float(c.z))); }
824 if(writeA) { *Pointer<UShort>(element + 6) = UShort(RoundInt(Float(c.w))); }
825 }
826 break;
827 case VK_FORMAT_R16G16B16_UINT:
828 case VK_FORMAT_R16G16B16_UNORM:
829 case VK_FORMAT_R16G16B16_USCALED:
830 if(writeR) { *Pointer<UShort>(element) = UShort(RoundInt(Float(c.x))); }
831 if(writeG) { *Pointer<UShort>(element + 2) = UShort(RoundInt(Float(c.y))); }
832 if(writeB) { *Pointer<UShort>(element + 4) = UShort(RoundInt(Float(c.z))); }
833 break;
834 case VK_FORMAT_R16G16_UINT:
835 case VK_FORMAT_R16G16_UNORM:
836 case VK_FORMAT_R16G16_USCALED:
837 if(writeR && writeG)
838 {
839 *Pointer<UShort2>(element) = UShort2(UShort4(RoundInt(c)));
840 }
841 else
842 {
843 if(writeR) { *Pointer<UShort>(element) = UShort(RoundInt(Float(c.x))); }
844 if(writeG) { *Pointer<UShort>(element + 2) = UShort(RoundInt(Float(c.y))); }
845 }
846 break;
847 case VK_FORMAT_R16_UINT:
848 case VK_FORMAT_R16_UNORM:
849 case VK_FORMAT_R16_USCALED:
850 if(writeR) { *Pointer<UShort>(element) = UShort(RoundInt(Float(c.x))); }
851 break;
852 case VK_FORMAT_R32G32B32A32_SINT:
853 if(writeRGBA)
854 {
855 *Pointer<Int4>(element) = RoundInt(c);
856 }
857 else
858 {
859 if(writeR) { *Pointer<Int>(element) = RoundInt(Float(c.x)); }
860 if(writeG) { *Pointer<Int>(element + 4) = RoundInt(Float(c.y)); }
861 if(writeB) { *Pointer<Int>(element + 8) = RoundInt(Float(c.z)); }
862 if(writeA) { *Pointer<Int>(element + 12) = RoundInt(Float(c.w)); }
863 }
864 break;
865 case VK_FORMAT_R32G32B32_SINT:
866 if(writeB) { *Pointer<Int>(element + 8) = RoundInt(Float(c.z)); }
867 // [[fallthrough]]
868 case VK_FORMAT_R32G32_SINT:
869 if(writeG) { *Pointer<Int>(element + 4) = RoundInt(Float(c.y)); }
870 // [[fallthrough]]
871 case VK_FORMAT_R32_SINT:
872 if(writeR) { *Pointer<Int>(element) = RoundInt(Float(c.x)); }
873 break;
874 case VK_FORMAT_R32G32B32A32_UINT:
875 if(writeRGBA)
876 {
877 *Pointer<UInt4>(element) = UInt4(RoundInt(c));
878 }
879 else
880 {
881 if(writeR) { *Pointer<UInt>(element) = As<UInt>(RoundInt(Float(c.x))); }
882 if(writeG) { *Pointer<UInt>(element + 4) = As<UInt>(RoundInt(Float(c.y))); }
883 if(writeB) { *Pointer<UInt>(element + 8) = As<UInt>(RoundInt(Float(c.z))); }
884 if(writeA) { *Pointer<UInt>(element + 12) = As<UInt>(RoundInt(Float(c.w))); }
885 }
886 break;
887 case VK_FORMAT_R32G32B32_UINT:
888 if(writeB) { *Pointer<UInt>(element + 8) = As<UInt>(RoundInt(Float(c.z))); }
889 // [[fallthrough]]
890 case VK_FORMAT_R32G32_UINT:
891 if(writeG) { *Pointer<UInt>(element + 4) = As<UInt>(RoundInt(Float(c.y))); }
892 // [[fallthrough]]
893 case VK_FORMAT_R32_UINT:
894 if(writeR) { *Pointer<UInt>(element) = As<UInt>(RoundInt(Float(c.x))); }
895 break;
896 case VK_FORMAT_R5G6B5_UNORM_PACK16:
897 if(writeR && writeG && writeB)
898 {
899 *Pointer<UShort>(element) = UShort(PackFields(RoundInt(c.xyzz), { 11, 5, 0, 0 }));
900 }
901 else
902 {
903 unsigned short mask = (writeB ? 0x001F : 0x0000) | (writeG ? 0x07E0 : 0x0000) | (writeR ? 0xF800 : 0x0000);
904 unsigned short unmask = ~mask;
905 *Pointer<UShort>(element) = (*Pointer<UShort>(element) & UShort(unmask)) |
906 (UShort(PackFields(RoundInt(c.xyzz), { 11, 5, 0, 0 })) &
907 UShort(mask));
908 }
909 break;
910 case VK_FORMAT_R5G5B5A1_UNORM_PACK16:
911 if(writeRGBA)
912 {
913 *Pointer<UShort>(element) = UShort(PackFields(RoundInt(c), { 11, 6, 1, 0 }));
914 }
915 else
916 {
917 unsigned short mask = (writeA ? 0x8000 : 0x0000) |
918 (writeR ? 0x7C00 : 0x0000) |
919 (writeG ? 0x03E0 : 0x0000) |
920 (writeB ? 0x001F : 0x0000);
921 unsigned short unmask = ~mask;
922 *Pointer<UShort>(element) = (*Pointer<UShort>(element) & UShort(unmask)) |
923 (UShort(PackFields(RoundInt(c), { 11, 6, 1, 0 })) &
924 UShort(mask));
925 }
926 break;
927 case VK_FORMAT_B5G5R5A1_UNORM_PACK16:
928 if(writeRGBA)
929 {
930 *Pointer<UShort>(element) = UShort(PackFields(RoundInt(c), { 1, 6, 11, 0 }));
931 }
932 else
933 {
934 unsigned short mask = (writeA ? 0x8000 : 0x0000) |
935 (writeR ? 0x7C00 : 0x0000) |
936 (writeG ? 0x03E0 : 0x0000) |
937 (writeB ? 0x001F : 0x0000);
938 unsigned short unmask = ~mask;
939 *Pointer<UShort>(element) = (*Pointer<UShort>(element) & UShort(unmask)) |
940 (UShort(PackFields(RoundInt(c), { 1, 6, 11, 0 })) &
941 UShort(mask));
942 }
943 break;
944 case VK_FORMAT_A1R5G5B5_UNORM_PACK16:
945 if(writeRGBA)
946 {
947 *Pointer<UShort>(element) = UShort(PackFields(RoundInt(c), { 10, 5, 0, 15 }));
948 }
949 else
950 {
951 unsigned short mask = (writeA ? 0x8000 : 0x0000) |
952 (writeR ? 0x7C00 : 0x0000) |
953 (writeG ? 0x03E0 : 0x0000) |
954 (writeB ? 0x001F : 0x0000);
955 unsigned short unmask = ~mask;
956 *Pointer<UShort>(element) = (*Pointer<UShort>(element) & UShort(unmask)) |
957 (UShort(PackFields(RoundInt(c), { 10, 5, 0, 15 })) &
958 UShort(mask));
959 }
960 break;
961 case VK_FORMAT_A2B10G10R10_UNORM_PACK32:
962 case VK_FORMAT_A2B10G10R10_UINT_PACK32:
963 case VK_FORMAT_A2B10G10R10_SNORM_PACK32:
964 if(writeRGBA)
965 {
966 *Pointer<UInt>(element) = As<UInt>(PackFields(RoundInt(c), { 0, 10, 20, 30 }));
967 }
968 else
969 {
970 unsigned int mask = (writeA ? 0xC0000000 : 0x0000) |
971 (writeB ? 0x3FF00000 : 0x0000) |
972 (writeG ? 0x000FFC00 : 0x0000) |
973 (writeR ? 0x000003FF : 0x0000);
974 unsigned int unmask = ~mask;
975 *Pointer<UInt>(element) = (*Pointer<UInt>(element) & UInt(unmask)) |
976 (As<UInt>(PackFields(RoundInt(c), { 0, 10, 20, 30 })) &
977 UInt(mask));
978 }
979 break;
980 case VK_FORMAT_A2R10G10B10_UNORM_PACK32:
981 case VK_FORMAT_A2R10G10B10_UINT_PACK32:
982 case VK_FORMAT_A2R10G10B10_SNORM_PACK32:
983 if(writeRGBA)
984 {
985 *Pointer<UInt>(element) = As<UInt>(PackFields(RoundInt(c), { 20, 10, 0, 30 }));
986 }
987 else
988 {
989 unsigned int mask = (writeA ? 0xC0000000 : 0x0000) |
990 (writeR ? 0x3FF00000 : 0x0000) |
991 (writeG ? 0x000FFC00 : 0x0000) |
992 (writeB ? 0x000003FF : 0x0000);
993 unsigned int unmask = ~mask;
994 *Pointer<UInt>(element) = (*Pointer<UInt>(element) & UInt(unmask)) |
995 (As<UInt>(PackFields(RoundInt(c), { 20, 10, 0, 30 })) &
996 UInt(mask));
997 }
998 break;
999 case VK_FORMAT_D16_UNORM:
1000 *Pointer<UShort>(element) = UShort(RoundInt(Float(c.x)));
1001 break;
1002 case VK_FORMAT_X8_D24_UNORM_PACK32:
1003 *Pointer<UInt>(element) = UInt(RoundInt(Float(c.x)) << 8);
1004 break;
1005 case VK_FORMAT_D32_SFLOAT:
1006 *Pointer<Float>(element) = c.x;
1007 break;
1008 case VK_FORMAT_S8_UINT:
1009 *Pointer<Byte>(element) = Byte(RoundInt(Float(c.x)));
1010 break;
1011 default:
1012 UNSUPPORTED("Blitter destination format %d", (int)state.destFormat);
1013 break;
1014 }
1015 }
1016
readInt4(Pointer<Byte> element,const State & state)1017 Int4 Blitter::readInt4(Pointer<Byte> element, const State &state)
1018 {
1019 Int4 c(0, 0, 0, 1);
1020
1021 switch(state.sourceFormat)
1022 {
1023 case VK_FORMAT_A8B8G8R8_SINT_PACK32:
1024 case VK_FORMAT_R8G8B8A8_SINT:
1025 c = Insert(c, Int(*Pointer<SByte>(element + 3)), 3);
1026 c = Insert(c, Int(*Pointer<SByte>(element + 2)), 2);
1027 // [[fallthrough]]
1028 case VK_FORMAT_R8G8_SINT:
1029 c = Insert(c, Int(*Pointer<SByte>(element + 1)), 1);
1030 // [[fallthrough]]
1031 case VK_FORMAT_R8_SINT:
1032 c = Insert(c, Int(*Pointer<SByte>(element)), 0);
1033 break;
1034 case VK_FORMAT_A2B10G10R10_UINT_PACK32:
1035 c = Insert(c, Int((*Pointer<UInt>(element) & UInt(0x000003FF))), 0);
1036 c = Insert(c, Int((*Pointer<UInt>(element) & UInt(0x000FFC00)) >> 10), 1);
1037 c = Insert(c, Int((*Pointer<UInt>(element) & UInt(0x3FF00000)) >> 20), 2);
1038 c = Insert(c, Int((*Pointer<UInt>(element) & UInt(0xC0000000)) >> 30), 3);
1039 break;
1040 case VK_FORMAT_A2R10G10B10_UINT_PACK32:
1041 c = Insert(c, Int((*Pointer<UInt>(element) & UInt(0x000003FF))), 2);
1042 c = Insert(c, Int((*Pointer<UInt>(element) & UInt(0x000FFC00)) >> 10), 1);
1043 c = Insert(c, Int((*Pointer<UInt>(element) & UInt(0x3FF00000)) >> 20), 0);
1044 c = Insert(c, Int((*Pointer<UInt>(element) & UInt(0xC0000000)) >> 30), 3);
1045 break;
1046 case VK_FORMAT_A8B8G8R8_UINT_PACK32:
1047 case VK_FORMAT_R8G8B8A8_UINT:
1048 c = Insert(c, Int(*Pointer<Byte>(element + 3)), 3);
1049 c = Insert(c, Int(*Pointer<Byte>(element + 2)), 2);
1050 // [[fallthrough]]
1051 case VK_FORMAT_R8G8_UINT:
1052 c = Insert(c, Int(*Pointer<Byte>(element + 1)), 1);
1053 // [[fallthrough]]
1054 case VK_FORMAT_R8_UINT:
1055 case VK_FORMAT_S8_UINT:
1056 c = Insert(c, Int(*Pointer<Byte>(element)), 0);
1057 break;
1058 case VK_FORMAT_R16G16B16A16_SINT:
1059 c = Insert(c, Int(*Pointer<Short>(element + 6)), 3);
1060 c = Insert(c, Int(*Pointer<Short>(element + 4)), 2);
1061 // [[fallthrough]]
1062 case VK_FORMAT_R16G16_SINT:
1063 c = Insert(c, Int(*Pointer<Short>(element + 2)), 1);
1064 // [[fallthrough]]
1065 case VK_FORMAT_R16_SINT:
1066 c = Insert(c, Int(*Pointer<Short>(element)), 0);
1067 break;
1068 case VK_FORMAT_R16G16B16A16_UINT:
1069 c = Insert(c, Int(*Pointer<UShort>(element + 6)), 3);
1070 c = Insert(c, Int(*Pointer<UShort>(element + 4)), 2);
1071 // [[fallthrough]]
1072 case VK_FORMAT_R16G16_UINT:
1073 c = Insert(c, Int(*Pointer<UShort>(element + 2)), 1);
1074 // [[fallthrough]]
1075 case VK_FORMAT_R16_UINT:
1076 c = Insert(c, Int(*Pointer<UShort>(element)), 0);
1077 break;
1078 case VK_FORMAT_R32G32B32A32_SINT:
1079 case VK_FORMAT_R32G32B32A32_UINT:
1080 c = *Pointer<Int4>(element);
1081 break;
1082 case VK_FORMAT_R32G32_SINT:
1083 case VK_FORMAT_R32G32_UINT:
1084 c = Insert(c, *Pointer<Int>(element + 4), 1);
1085 // [[fallthrough]]
1086 case VK_FORMAT_R32_SINT:
1087 case VK_FORMAT_R32_UINT:
1088 c = Insert(c, *Pointer<Int>(element), 0);
1089 break;
1090 default:
1091 UNSUPPORTED("Blitter source format %d", (int)state.sourceFormat);
1092 }
1093
1094 return c;
1095 }
1096
write(Int4 & c,Pointer<Byte> element,const State & state)1097 void Blitter::write(Int4 &c, Pointer<Byte> element, const State &state)
1098 {
1099 bool writeR = state.writeRed;
1100 bool writeG = state.writeGreen;
1101 bool writeB = state.writeBlue;
1102 bool writeA = state.writeAlpha;
1103 bool writeRGBA = writeR && writeG && writeB && writeA;
1104
1105 switch(state.destFormat)
1106 {
1107 case VK_FORMAT_A2B10G10R10_UINT_PACK32:
1108 case VK_FORMAT_A2R10G10B10_UINT_PACK32:
1109 c = Min(As<UInt4>(c), UInt4(0x03FF, 0x03FF, 0x03FF, 0x0003));
1110 break;
1111 case VK_FORMAT_A8B8G8R8_UINT_PACK32:
1112 case VK_FORMAT_R8G8B8A8_UINT:
1113 case VK_FORMAT_R8G8B8_UINT:
1114 case VK_FORMAT_R8G8_UINT:
1115 case VK_FORMAT_R8_UINT:
1116 case VK_FORMAT_R8G8B8A8_USCALED:
1117 case VK_FORMAT_R8G8B8_USCALED:
1118 case VK_FORMAT_R8G8_USCALED:
1119 case VK_FORMAT_R8_USCALED:
1120 case VK_FORMAT_S8_UINT:
1121 c = Min(As<UInt4>(c), UInt4(0xFF));
1122 break;
1123 case VK_FORMAT_R16G16B16A16_UINT:
1124 case VK_FORMAT_R16G16B16_UINT:
1125 case VK_FORMAT_R16G16_UINT:
1126 case VK_FORMAT_R16_UINT:
1127 case VK_FORMAT_R16G16B16A16_USCALED:
1128 case VK_FORMAT_R16G16B16_USCALED:
1129 case VK_FORMAT_R16G16_USCALED:
1130 case VK_FORMAT_R16_USCALED:
1131 c = Min(As<UInt4>(c), UInt4(0xFFFF));
1132 break;
1133 case VK_FORMAT_A8B8G8R8_SINT_PACK32:
1134 case VK_FORMAT_R8G8B8A8_SINT:
1135 case VK_FORMAT_R8G8_SINT:
1136 case VK_FORMAT_R8_SINT:
1137 case VK_FORMAT_R8G8B8A8_SSCALED:
1138 case VK_FORMAT_R8G8B8_SSCALED:
1139 case VK_FORMAT_R8G8_SSCALED:
1140 case VK_FORMAT_R8_SSCALED:
1141 c = Min(Max(c, Int4(-0x80)), Int4(0x7F));
1142 break;
1143 case VK_FORMAT_R16G16B16A16_SINT:
1144 case VK_FORMAT_R16G16B16_SINT:
1145 case VK_FORMAT_R16G16_SINT:
1146 case VK_FORMAT_R16_SINT:
1147 case VK_FORMAT_R16G16B16A16_SSCALED:
1148 case VK_FORMAT_R16G16B16_SSCALED:
1149 case VK_FORMAT_R16G16_SSCALED:
1150 case VK_FORMAT_R16_SSCALED:
1151 c = Min(Max(c, Int4(-0x8000)), Int4(0x7FFF));
1152 break;
1153 default:
1154 break;
1155 }
1156
1157 switch(state.destFormat)
1158 {
1159 case VK_FORMAT_B8G8R8A8_SINT:
1160 case VK_FORMAT_B8G8R8A8_SSCALED:
1161 if(writeA) { *Pointer<SByte>(element + 3) = SByte(Extract(c, 3)); }
1162 // [[fallthrough]]
1163 case VK_FORMAT_B8G8R8_SINT:
1164 case VK_FORMAT_B8G8R8_SSCALED:
1165 if(writeB) { *Pointer<SByte>(element) = SByte(Extract(c, 2)); }
1166 if(writeG) { *Pointer<SByte>(element + 1) = SByte(Extract(c, 1)); }
1167 if(writeR) { *Pointer<SByte>(element + 2) = SByte(Extract(c, 0)); }
1168 break;
1169 case VK_FORMAT_A8B8G8R8_SINT_PACK32:
1170 case VK_FORMAT_R8G8B8A8_SINT:
1171 case VK_FORMAT_R8G8B8A8_SSCALED:
1172 case VK_FORMAT_A8B8G8R8_SSCALED_PACK32:
1173 if(writeA) { *Pointer<SByte>(element + 3) = SByte(Extract(c, 3)); }
1174 // [[fallthrough]]
1175 case VK_FORMAT_R8G8B8_SINT:
1176 case VK_FORMAT_R8G8B8_SSCALED:
1177 if(writeB) { *Pointer<SByte>(element + 2) = SByte(Extract(c, 2)); }
1178 // [[fallthrough]]
1179 case VK_FORMAT_R8G8_SINT:
1180 case VK_FORMAT_R8G8_SSCALED:
1181 if(writeG) { *Pointer<SByte>(element + 1) = SByte(Extract(c, 1)); }
1182 // [[fallthrough]]
1183 case VK_FORMAT_R8_SINT:
1184 case VK_FORMAT_R8_SSCALED:
1185 if(writeR) { *Pointer<SByte>(element) = SByte(Extract(c, 0)); }
1186 break;
1187 case VK_FORMAT_A2B10G10R10_UINT_PACK32:
1188 case VK_FORMAT_A2B10G10R10_SINT_PACK32:
1189 case VK_FORMAT_A2B10G10R10_USCALED_PACK32:
1190 case VK_FORMAT_A2B10G10R10_SSCALED_PACK32:
1191 if(writeRGBA)
1192 {
1193 *Pointer<UInt>(element) = As<UInt>(PackFields(c, { 0, 10, 20, 30 }));
1194 }
1195 else
1196 {
1197 unsigned int mask = (writeA ? 0xC0000000 : 0x0000) |
1198 (writeB ? 0x3FF00000 : 0x0000) |
1199 (writeG ? 0x000FFC00 : 0x0000) |
1200 (writeR ? 0x000003FF : 0x0000);
1201 unsigned int unmask = ~mask;
1202 *Pointer<UInt>(element) = (*Pointer<UInt>(element) & UInt(unmask)) |
1203 (As<UInt>(PackFields(c, { 0, 10, 20, 30 })) & UInt(mask));
1204 }
1205 break;
1206 case VK_FORMAT_A2R10G10B10_UINT_PACK32:
1207 case VK_FORMAT_A2R10G10B10_SINT_PACK32:
1208 case VK_FORMAT_A2R10G10B10_USCALED_PACK32:
1209 case VK_FORMAT_A2R10G10B10_SSCALED_PACK32:
1210 if(writeRGBA)
1211 {
1212 *Pointer<UInt>(element) = As<UInt>(PackFields(c, { 20, 10, 0, 30 }));
1213 }
1214 else
1215 {
1216 unsigned int mask = (writeA ? 0xC0000000 : 0x0000) |
1217 (writeR ? 0x3FF00000 : 0x0000) |
1218 (writeG ? 0x000FFC00 : 0x0000) |
1219 (writeB ? 0x000003FF : 0x0000);
1220 unsigned int unmask = ~mask;
1221 *Pointer<UInt>(element) = (*Pointer<UInt>(element) & UInt(unmask)) |
1222 (As<UInt>(PackFields(c, { 20, 10, 0, 30 })) & UInt(mask));
1223 }
1224 break;
1225 case VK_FORMAT_B8G8R8A8_UINT:
1226 case VK_FORMAT_B8G8R8A8_USCALED:
1227 if(writeA) { *Pointer<Byte>(element + 3) = Byte(Extract(c, 3)); }
1228 // [[fallthrough]]
1229 case VK_FORMAT_B8G8R8_UINT:
1230 case VK_FORMAT_B8G8R8_USCALED:
1231 case VK_FORMAT_B8G8R8_SRGB:
1232 if(writeB) { *Pointer<Byte>(element) = Byte(Extract(c, 2)); }
1233 if(writeG) { *Pointer<Byte>(element + 1) = Byte(Extract(c, 1)); }
1234 if(writeR) { *Pointer<Byte>(element + 2) = Byte(Extract(c, 0)); }
1235 break;
1236 case VK_FORMAT_A8B8G8R8_UINT_PACK32:
1237 case VK_FORMAT_R8G8B8A8_UINT:
1238 case VK_FORMAT_R8G8B8A8_USCALED:
1239 case VK_FORMAT_A8B8G8R8_USCALED_PACK32:
1240 if(writeA) { *Pointer<Byte>(element + 3) = Byte(Extract(c, 3)); }
1241 // [[fallthrough]]
1242 case VK_FORMAT_R8G8B8_UINT:
1243 case VK_FORMAT_R8G8B8_USCALED:
1244 if(writeB) { *Pointer<Byte>(element + 2) = Byte(Extract(c, 2)); }
1245 // [[fallthrough]]
1246 case VK_FORMAT_R8G8_UINT:
1247 case VK_FORMAT_R8G8_USCALED:
1248 if(writeG) { *Pointer<Byte>(element + 1) = Byte(Extract(c, 1)); }
1249 // [[fallthrough]]
1250 case VK_FORMAT_R8_UINT:
1251 case VK_FORMAT_R8_USCALED:
1252 case VK_FORMAT_S8_UINT:
1253 if(writeR) { *Pointer<Byte>(element) = Byte(Extract(c, 0)); }
1254 break;
1255 case VK_FORMAT_R16G16B16A16_SINT:
1256 case VK_FORMAT_R16G16B16A16_SSCALED:
1257 if(writeA) { *Pointer<Short>(element + 6) = Short(Extract(c, 3)); }
1258 // [[fallthrough]]
1259 case VK_FORMAT_R16G16B16_SINT:
1260 case VK_FORMAT_R16G16B16_SSCALED:
1261 if(writeB) { *Pointer<Short>(element + 4) = Short(Extract(c, 2)); }
1262 // [[fallthrough]]
1263 case VK_FORMAT_R16G16_SINT:
1264 case VK_FORMAT_R16G16_SSCALED:
1265 if(writeG) { *Pointer<Short>(element + 2) = Short(Extract(c, 1)); }
1266 // [[fallthrough]]
1267 case VK_FORMAT_R16_SINT:
1268 case VK_FORMAT_R16_SSCALED:
1269 if(writeR) { *Pointer<Short>(element) = Short(Extract(c, 0)); }
1270 break;
1271 case VK_FORMAT_R16G16B16A16_UINT:
1272 case VK_FORMAT_R16G16B16A16_USCALED:
1273 if(writeA) { *Pointer<UShort>(element + 6) = UShort(Extract(c, 3)); }
1274 // [[fallthrough]]
1275 case VK_FORMAT_R16G16B16_UINT:
1276 case VK_FORMAT_R16G16B16_USCALED:
1277 if(writeB) { *Pointer<UShort>(element + 4) = UShort(Extract(c, 2)); }
1278 // [[fallthrough]]
1279 case VK_FORMAT_R16G16_UINT:
1280 case VK_FORMAT_R16G16_USCALED:
1281 if(writeG) { *Pointer<UShort>(element + 2) = UShort(Extract(c, 1)); }
1282 // [[fallthrough]]
1283 case VK_FORMAT_R16_UINT:
1284 case VK_FORMAT_R16_USCALED:
1285 if(writeR) { *Pointer<UShort>(element) = UShort(Extract(c, 0)); }
1286 break;
1287 case VK_FORMAT_R32G32B32A32_SINT:
1288 if(writeRGBA)
1289 {
1290 *Pointer<Int4>(element) = c;
1291 }
1292 else
1293 {
1294 if(writeR) { *Pointer<Int>(element) = Extract(c, 0); }
1295 if(writeG) { *Pointer<Int>(element + 4) = Extract(c, 1); }
1296 if(writeB) { *Pointer<Int>(element + 8) = Extract(c, 2); }
1297 if(writeA) { *Pointer<Int>(element + 12) = Extract(c, 3); }
1298 }
1299 break;
1300 case VK_FORMAT_R32G32B32_SINT:
1301 if(writeR) { *Pointer<Int>(element) = Extract(c, 0); }
1302 if(writeG) { *Pointer<Int>(element + 4) = Extract(c, 1); }
1303 if(writeB) { *Pointer<Int>(element + 8) = Extract(c, 2); }
1304 break;
1305 case VK_FORMAT_R32G32_SINT:
1306 if(writeR) { *Pointer<Int>(element) = Extract(c, 0); }
1307 if(writeG) { *Pointer<Int>(element + 4) = Extract(c, 1); }
1308 break;
1309 case VK_FORMAT_R32_SINT:
1310 if(writeR) { *Pointer<Int>(element) = Extract(c, 0); }
1311 break;
1312 case VK_FORMAT_R32G32B32A32_UINT:
1313 if(writeRGBA)
1314 {
1315 *Pointer<UInt4>(element) = As<UInt4>(c);
1316 }
1317 else
1318 {
1319 if(writeR) { *Pointer<UInt>(element) = As<UInt>(Extract(c, 0)); }
1320 if(writeG) { *Pointer<UInt>(element + 4) = As<UInt>(Extract(c, 1)); }
1321 if(writeB) { *Pointer<UInt>(element + 8) = As<UInt>(Extract(c, 2)); }
1322 if(writeA) { *Pointer<UInt>(element + 12) = As<UInt>(Extract(c, 3)); }
1323 }
1324 break;
1325 case VK_FORMAT_R32G32B32_UINT:
1326 if(writeB) { *Pointer<UInt>(element + 8) = As<UInt>(Extract(c, 2)); }
1327 // [[fallthrough]]
1328 case VK_FORMAT_R32G32_UINT:
1329 if(writeG) { *Pointer<UInt>(element + 4) = As<UInt>(Extract(c, 1)); }
1330 // [[fallthrough]]
1331 case VK_FORMAT_R32_UINT:
1332 if(writeR) { *Pointer<UInt>(element) = As<UInt>(Extract(c, 0)); }
1333 break;
1334 default:
1335 UNSUPPORTED("Blitter destination format %d", (int)state.destFormat);
1336 }
1337 }
1338
ApplyScaleAndClamp(Float4 & value,const State & state,bool preScaled)1339 void Blitter::ApplyScaleAndClamp(Float4 &value, const State &state, bool preScaled)
1340 {
1341 float4 scale{}, unscale{};
1342
1343 if(state.clearOperation &&
1344 state.sourceFormat.isUnnormalizedInteger() &&
1345 !state.destFormat.isUnnormalizedInteger())
1346 {
1347 // If we're clearing a buffer from an int or uint color into a normalized color,
1348 // then the whole range of the int or uint color must be scaled between 0 and 1.
1349 switch(state.sourceFormat)
1350 {
1351 case VK_FORMAT_R32G32B32A32_SINT:
1352 unscale = float4(static_cast<float>(0x7FFFFFFF));
1353 break;
1354 case VK_FORMAT_R32G32B32A32_UINT:
1355 unscale = float4(static_cast<float>(0xFFFFFFFF));
1356 break;
1357 default:
1358 UNSUPPORTED("Blitter source format %d", (int)state.sourceFormat);
1359 }
1360 }
1361 else
1362 {
1363 unscale = state.sourceFormat.getScale();
1364 }
1365
1366 scale = state.destFormat.getScale();
1367
1368 bool srcSRGB = state.sourceFormat.isSRGBformat();
1369 bool dstSRGB = state.destFormat.isSRGBformat();
1370
1371 if(state.allowSRGBConversion && ((srcSRGB && !preScaled) || dstSRGB)) // One of the formats is sRGB encoded.
1372 {
1373 value *= preScaled ? Float4(1.0f / scale.x, 1.0f / scale.y, 1.0f / scale.z, 1.0f / scale.w) : // Unapply scale
1374 Float4(1.0f / unscale.x, 1.0f / unscale.y, 1.0f / unscale.z, 1.0f / unscale.w); // Apply unscale
1375 value = (srcSRGB && !preScaled) ? sRGBtoLinear(value) : LinearToSRGB(value);
1376 value *= Float4(scale.x, scale.y, scale.z, scale.w); // Apply scale
1377 }
1378 else if(unscale != scale)
1379 {
1380 value *= Float4(scale.x / unscale.x, scale.y / unscale.y, scale.z / unscale.z, scale.w / unscale.w);
1381 }
1382
1383 if(state.sourceFormat.isFloatFormat() && !state.destFormat.isFloatFormat())
1384 {
1385 value = Min(value, Float4(scale.x, scale.y, scale.z, scale.w));
1386
1387 value = Max(value, Float4(state.destFormat.isUnsignedComponent(0) ? 0.0f : -scale.x,
1388 state.destFormat.isUnsignedComponent(1) ? 0.0f : -scale.y,
1389 state.destFormat.isUnsignedComponent(2) ? 0.0f : -scale.z,
1390 state.destFormat.isUnsignedComponent(3) ? 0.0f : -scale.w));
1391 }
1392 }
1393
ComputeOffset(Int & x,Int & y,Int & pitchB,int bytes)1394 Int Blitter::ComputeOffset(Int &x, Int &y, Int &pitchB, int bytes)
1395 {
1396 return y * pitchB + x * bytes;
1397 }
1398
ComputeOffset(Int & x,Int & y,Int & z,Int & sliceB,Int & pitchB,int bytes)1399 Int Blitter::ComputeOffset(Int &x, Int &y, Int &z, Int &sliceB, Int &pitchB, int bytes)
1400 {
1401 return z * sliceB + y * pitchB + x * bytes;
1402 }
1403
LinearToSRGB(const Float4 & c)1404 Float4 Blitter::LinearToSRGB(const Float4 &c)
1405 {
1406 Float4 lc = Min(c, Float4(0.0031308f)) * Float4(12.92f);
1407 Float4 ec = Float4(1.055f) * power(c, Float4(1.0f / 2.4f)) - Float4(0.055f);
1408
1409 Float4 s = c;
1410 s.xyz = Max(lc, ec);
1411
1412 return s;
1413 }
1414
sRGBtoLinear(const Float4 & c)1415 Float4 Blitter::sRGBtoLinear(const Float4 &c)
1416 {
1417 Float4 lc = c * Float4(1.0f / 12.92f);
1418 Float4 ec = power((c + Float4(0.055f)) * Float4(1.0f / 1.055f), Float4(2.4f));
1419
1420 Int4 linear = CmpLT(c, Float4(0.04045f));
1421
1422 Float4 s = c;
1423 s.xyz = As<Float4>((linear & As<Int4>(lc)) | (~linear & As<Int4>(ec))); // TODO: IfThenElse()
1424
1425 return s;
1426 }
1427
sample(Pointer<Byte> & source,Float & x,Float & y,Float & z,Int & sWidth,Int & sHeight,Int & sDepth,Int & sSliceB,Int & sPitchB,const State & state)1428 Float4 Blitter::sample(Pointer<Byte> &source, Float &x, Float &y, Float &z,
1429 Int &sWidth, Int &sHeight, Int &sDepth,
1430 Int &sSliceB, Int &sPitchB, const State &state)
1431 {
1432 bool intSrc = state.sourceFormat.isUnnormalizedInteger();
1433 int srcBytes = state.sourceFormat.bytes();
1434
1435 Float4 color;
1436
1437 bool preScaled = false;
1438 if(!state.filter || intSrc)
1439 {
1440 Int X = Int(x);
1441 Int Y = Int(y);
1442 Int Z = Int(z);
1443
1444 if(state.clampToEdge)
1445 {
1446 X = Clamp(X, 0, sWidth - 1);
1447 Y = Clamp(Y, 0, sHeight - 1);
1448 Z = Clamp(Z, 0, sDepth - 1);
1449 }
1450
1451 Pointer<Byte> s = source + ComputeOffset(X, Y, Z, sSliceB, sPitchB, srcBytes);
1452
1453 color = readFloat4(s, state);
1454
1455 if(state.srcSamples > 1) // Resolve multisampled source
1456 {
1457 if(state.allowSRGBConversion && state.sourceFormat.isSRGBformat()) // sRGB -> RGB
1458 {
1459 ApplyScaleAndClamp(color, state);
1460 preScaled = true;
1461 }
1462 Float4 accum = color;
1463 for(int sample = 1; sample < state.srcSamples; sample++)
1464 {
1465 s += sSliceB;
1466 color = readFloat4(s, state);
1467
1468 if(state.allowSRGBConversion && state.sourceFormat.isSRGBformat()) // sRGB -> RGB
1469 {
1470 ApplyScaleAndClamp(color, state);
1471 preScaled = true;
1472 }
1473 accum += color;
1474 }
1475 color = accum * Float4(1.0f / static_cast<float>(state.srcSamples));
1476 }
1477 }
1478 else // Bilinear filtering
1479 {
1480 Float X = x;
1481 Float Y = y;
1482 Float Z = z;
1483
1484 if(state.clampToEdge)
1485 {
1486 X = Min(Max(x, 0.5f), Float(sWidth) - 0.5f);
1487 Y = Min(Max(y, 0.5f), Float(sHeight) - 0.5f);
1488 Z = Min(Max(z, 0.5f), Float(sDepth) - 0.5f);
1489 }
1490
1491 Float x0 = X - 0.5f;
1492 Float y0 = Y - 0.5f;
1493 Float z0 = Z - 0.5f;
1494
1495 Int X0 = Max(Int(x0), 0);
1496 Int Y0 = Max(Int(y0), 0);
1497 Int Z0 = Max(Int(z0), 0);
1498
1499 Int X1 = X0 + 1;
1500 Int Y1 = Y0 + 1;
1501 X1 = IfThenElse(X1 >= sWidth, X0, X1);
1502 Y1 = IfThenElse(Y1 >= sHeight, Y0, Y1);
1503
1504 if(state.filter3D)
1505 {
1506 Int Z1 = Z0 + 1;
1507 Z1 = IfThenElse(Z1 >= sHeight, Z0, Z1);
1508
1509 Pointer<Byte> s000 = source + ComputeOffset(X0, Y0, Z0, sSliceB, sPitchB, srcBytes);
1510 Pointer<Byte> s010 = source + ComputeOffset(X1, Y0, Z0, sSliceB, sPitchB, srcBytes);
1511 Pointer<Byte> s100 = source + ComputeOffset(X0, Y1, Z0, sSliceB, sPitchB, srcBytes);
1512 Pointer<Byte> s110 = source + ComputeOffset(X1, Y1, Z0, sSliceB, sPitchB, srcBytes);
1513 Pointer<Byte> s001 = source + ComputeOffset(X0, Y0, Z1, sSliceB, sPitchB, srcBytes);
1514 Pointer<Byte> s011 = source + ComputeOffset(X1, Y0, Z1, sSliceB, sPitchB, srcBytes);
1515 Pointer<Byte> s101 = source + ComputeOffset(X0, Y1, Z1, sSliceB, sPitchB, srcBytes);
1516 Pointer<Byte> s111 = source + ComputeOffset(X1, Y1, Z1, sSliceB, sPitchB, srcBytes);
1517
1518 Float4 c000 = readFloat4(s000, state);
1519 Float4 c010 = readFloat4(s010, state);
1520 Float4 c100 = readFloat4(s100, state);
1521 Float4 c110 = readFloat4(s110, state);
1522 Float4 c001 = readFloat4(s001, state);
1523 Float4 c011 = readFloat4(s011, state);
1524 Float4 c101 = readFloat4(s101, state);
1525 Float4 c111 = readFloat4(s111, state);
1526
1527 if(state.allowSRGBConversion && state.sourceFormat.isSRGBformat()) // sRGB -> RGB
1528 {
1529 ApplyScaleAndClamp(c000, state);
1530 ApplyScaleAndClamp(c010, state);
1531 ApplyScaleAndClamp(c100, state);
1532 ApplyScaleAndClamp(c110, state);
1533 ApplyScaleAndClamp(c001, state);
1534 ApplyScaleAndClamp(c011, state);
1535 ApplyScaleAndClamp(c101, state);
1536 ApplyScaleAndClamp(c111, state);
1537 preScaled = true;
1538 }
1539
1540 Float4 fx = Float4(x0 - Float(X0));
1541 Float4 fy = Float4(y0 - Float(Y0));
1542 Float4 fz = Float4(z0 - Float(Z0));
1543 Float4 ix = Float4(1.0f) - fx;
1544 Float4 iy = Float4(1.0f) - fy;
1545 Float4 iz = Float4(1.0f) - fz;
1546
1547 color = ((c000 * ix + c010 * fx) * iy +
1548 (c100 * ix + c110 * fx) * fy) *
1549 iz +
1550 ((c001 * ix + c011 * fx) * iy +
1551 (c101 * ix + c111 * fx) * fy) *
1552 fz;
1553 }
1554 else
1555 {
1556 Pointer<Byte> s00 = source + ComputeOffset(X0, Y0, Z0, sSliceB, sPitchB, srcBytes);
1557 Pointer<Byte> s01 = source + ComputeOffset(X1, Y0, Z0, sSliceB, sPitchB, srcBytes);
1558 Pointer<Byte> s10 = source + ComputeOffset(X0, Y1, Z0, sSliceB, sPitchB, srcBytes);
1559 Pointer<Byte> s11 = source + ComputeOffset(X1, Y1, Z0, sSliceB, sPitchB, srcBytes);
1560
1561 Float4 c00 = readFloat4(s00, state);
1562 Float4 c01 = readFloat4(s01, state);
1563 Float4 c10 = readFloat4(s10, state);
1564 Float4 c11 = readFloat4(s11, state);
1565
1566 if(state.allowSRGBConversion && state.sourceFormat.isSRGBformat()) // sRGB -> RGB
1567 {
1568 ApplyScaleAndClamp(c00, state);
1569 ApplyScaleAndClamp(c01, state);
1570 ApplyScaleAndClamp(c10, state);
1571 ApplyScaleAndClamp(c11, state);
1572 preScaled = true;
1573 }
1574
1575 Float4 fx = Float4(x0 - Float(X0));
1576 Float4 fy = Float4(y0 - Float(Y0));
1577 Float4 ix = Float4(1.0f) - fx;
1578 Float4 iy = Float4(1.0f) - fy;
1579
1580 color = (c00 * ix + c01 * fx) * iy +
1581 (c10 * ix + c11 * fx) * fy;
1582 }
1583 }
1584
1585 ApplyScaleAndClamp(color, state, preScaled);
1586
1587 return color;
1588 }
1589
generate(const State & state)1590 Blitter::BlitRoutineType Blitter::generate(const State &state)
1591 {
1592 BlitFunction function;
1593 {
1594 Pointer<Byte> blit(function.Arg<0>());
1595
1596 Pointer<Byte> source = *Pointer<Pointer<Byte>>(blit + OFFSET(BlitData, source));
1597 Pointer<Byte> dest = *Pointer<Pointer<Byte>>(blit + OFFSET(BlitData, dest));
1598 Int sPitchB = *Pointer<Int>(blit + OFFSET(BlitData, sPitchB));
1599 Int dPitchB = *Pointer<Int>(blit + OFFSET(BlitData, dPitchB));
1600 Int sSliceB = *Pointer<Int>(blit + OFFSET(BlitData, sSliceB));
1601 Int dSliceB = *Pointer<Int>(blit + OFFSET(BlitData, dSliceB));
1602
1603 Float x0 = *Pointer<Float>(blit + OFFSET(BlitData, x0));
1604 Float y0 = *Pointer<Float>(blit + OFFSET(BlitData, y0));
1605 Float z0 = *Pointer<Float>(blit + OFFSET(BlitData, z0));
1606 Float w = *Pointer<Float>(blit + OFFSET(BlitData, w));
1607 Float h = *Pointer<Float>(blit + OFFSET(BlitData, h));
1608 Float d = *Pointer<Float>(blit + OFFSET(BlitData, d));
1609
1610 Int x0d = *Pointer<Int>(blit + OFFSET(BlitData, x0d));
1611 Int x1d = *Pointer<Int>(blit + OFFSET(BlitData, x1d));
1612 Int y0d = *Pointer<Int>(blit + OFFSET(BlitData, y0d));
1613 Int y1d = *Pointer<Int>(blit + OFFSET(BlitData, y1d));
1614 Int z0d = *Pointer<Int>(blit + OFFSET(BlitData, z0d));
1615 Int z1d = *Pointer<Int>(blit + OFFSET(BlitData, z1d));
1616
1617 Int sWidth = *Pointer<Int>(blit + OFFSET(BlitData, sWidth));
1618 Int sHeight = *Pointer<Int>(blit + OFFSET(BlitData, sHeight));
1619 Int sDepth = *Pointer<Int>(blit + OFFSET(BlitData, sDepth));
1620
1621 bool intSrc = state.sourceFormat.isUnnormalizedInteger();
1622 bool intDst = state.destFormat.isUnnormalizedInteger();
1623 bool intBoth = intSrc && intDst;
1624 int srcBytes = state.sourceFormat.bytes();
1625 int dstBytes = state.destFormat.bytes();
1626
1627 bool hasConstantColorI = false;
1628 Int4 constantColorI;
1629 bool hasConstantColorF = false;
1630 Float4 constantColorF;
1631 if(state.clearOperation)
1632 {
1633 if(intBoth) // Integer types
1634 {
1635 constantColorI = readInt4(source, state);
1636 hasConstantColorI = true;
1637 }
1638 else
1639 {
1640 constantColorF = readFloat4(source, state);
1641 hasConstantColorF = true;
1642
1643 ApplyScaleAndClamp(constantColorF, state);
1644 }
1645 }
1646
1647 For(Int k = z0d, k < z1d, k++)
1648 {
1649 Float z = state.clearOperation ? RValue<Float>(z0) : z0 + Float(k) * d;
1650 Pointer<Byte> destSlice = dest + k * dSliceB;
1651
1652 For(Int j = y0d, j < y1d, j++)
1653 {
1654 Float y = state.clearOperation ? RValue<Float>(y0) : y0 + Float(j) * h;
1655 Pointer<Byte> destLine = destSlice + j * dPitchB;
1656
1657 For(Int i = x0d, i < x1d, i++)
1658 {
1659 Float x = state.clearOperation ? RValue<Float>(x0) : x0 + Float(i) * w;
1660 Pointer<Byte> d = destLine + i * dstBytes;
1661
1662 if(hasConstantColorI)
1663 {
1664 for(int s = 0; s < state.destSamples; s++)
1665 {
1666 write(constantColorI, d, state);
1667
1668 d += dSliceB;
1669 }
1670 }
1671 else if(hasConstantColorF)
1672 {
1673 for(int s = 0; s < state.destSamples; s++)
1674 {
1675 write(constantColorF, d, state);
1676
1677 d += dSliceB;
1678 }
1679 }
1680 else if(intBoth) // Integer types do not support filtering
1681 {
1682 Int X = Int(x);
1683 Int Y = Int(y);
1684 Int Z = Int(z);
1685
1686 if(state.clampToEdge)
1687 {
1688 X = Clamp(X, 0, sWidth - 1);
1689 Y = Clamp(Y, 0, sHeight - 1);
1690 Z = Clamp(Z, 0, sDepth - 1);
1691 }
1692
1693 Pointer<Byte> s = source + ComputeOffset(X, Y, Z, sSliceB, sPitchB, srcBytes);
1694
1695 // When both formats are true integer types, we don't go to float to avoid losing precision
1696 Int4 color = readInt4(s, state);
1697 for(int s = 0; s < state.destSamples; s++)
1698 {
1699 write(color, d, state);
1700
1701 d += dSliceB;
1702 }
1703 }
1704 else
1705 {
1706 Float4 color = sample(source, x, y, z, sWidth, sHeight, sDepth, sSliceB, sPitchB, state);
1707
1708 for(int s = 0; s < state.destSamples; s++)
1709 {
1710 write(color, d, state);
1711
1712 d += dSliceB;
1713 }
1714 }
1715 }
1716 }
1717 }
1718 }
1719
1720 return function("BlitRoutine");
1721 }
1722
getBlitRoutine(const State & state)1723 Blitter::BlitRoutineType Blitter::getBlitRoutine(const State &state)
1724 {
1725 marl::lock lock(blitMutex);
1726 auto blitRoutine = blitCache.lookup(state);
1727
1728 if(!blitRoutine)
1729 {
1730 blitRoutine = generate(state);
1731 blitCache.add(state, blitRoutine);
1732 }
1733
1734 return blitRoutine;
1735 }
1736
getCornerUpdateRoutine(const State & state)1737 Blitter::CornerUpdateRoutineType Blitter::getCornerUpdateRoutine(const State &state)
1738 {
1739 marl::lock lock(cornerUpdateMutex);
1740 auto cornerUpdateRoutine = cornerUpdateCache.lookup(state);
1741
1742 if(!cornerUpdateRoutine)
1743 {
1744 cornerUpdateRoutine = generateCornerUpdate(state);
1745 cornerUpdateCache.add(state, cornerUpdateRoutine);
1746 }
1747
1748 return cornerUpdateRoutine;
1749 }
1750
blit(const vk::Image * src,vk::Image * dst,VkImageBlit region,VkFilter filter)1751 void Blitter::blit(const vk::Image *src, vk::Image *dst, VkImageBlit region, VkFilter filter)
1752 {
1753 ASSERT(src->getFormat() != VK_FORMAT_UNDEFINED);
1754 ASSERT(dst->getFormat() != VK_FORMAT_UNDEFINED);
1755
1756 // Vulkan 1.2 section 18.5. Image Copies with Scaling:
1757 // "The layerCount member of srcSubresource and dstSubresource must match"
1758 // "The aspectMask member of srcSubresource and dstSubresource must match"
1759 ASSERT(region.srcSubresource.layerCount == region.dstSubresource.layerCount);
1760 ASSERT(region.srcSubresource.aspectMask == region.dstSubresource.aspectMask);
1761
1762 if(region.dstOffsets[0].x > region.dstOffsets[1].x)
1763 {
1764 std::swap(region.srcOffsets[0].x, region.srcOffsets[1].x);
1765 std::swap(region.dstOffsets[0].x, region.dstOffsets[1].x);
1766 }
1767
1768 if(region.dstOffsets[0].y > region.dstOffsets[1].y)
1769 {
1770 std::swap(region.srcOffsets[0].y, region.srcOffsets[1].y);
1771 std::swap(region.dstOffsets[0].y, region.dstOffsets[1].y);
1772 }
1773
1774 if(region.dstOffsets[0].z > region.dstOffsets[1].z)
1775 {
1776 std::swap(region.srcOffsets[0].z, region.srcOffsets[1].z);
1777 std::swap(region.dstOffsets[0].z, region.dstOffsets[1].z);
1778 }
1779
1780 VkImageAspectFlagBits srcAspect = static_cast<VkImageAspectFlagBits>(region.srcSubresource.aspectMask);
1781 VkImageAspectFlagBits dstAspect = static_cast<VkImageAspectFlagBits>(region.dstSubresource.aspectMask);
1782 VkExtent3D srcExtent = src->getMipLevelExtent(srcAspect, region.srcSubresource.mipLevel);
1783
1784 float widthRatio = static_cast<float>(region.srcOffsets[1].x - region.srcOffsets[0].x) /
1785 static_cast<float>(region.dstOffsets[1].x - region.dstOffsets[0].x);
1786 float heightRatio = static_cast<float>(region.srcOffsets[1].y - region.srcOffsets[0].y) /
1787 static_cast<float>(region.dstOffsets[1].y - region.dstOffsets[0].y);
1788 float depthRatio = static_cast<float>(region.srcOffsets[1].z - region.srcOffsets[0].z) /
1789 static_cast<float>(region.dstOffsets[1].z - region.dstOffsets[0].z);
1790 float x0 = region.srcOffsets[0].x + (0.5f - region.dstOffsets[0].x) * widthRatio;
1791 float y0 = region.srcOffsets[0].y + (0.5f - region.dstOffsets[0].y) * heightRatio;
1792 float z0 = region.srcOffsets[0].z + (0.5f - region.dstOffsets[0].z) * depthRatio;
1793
1794 auto srcFormat = src->getFormat(srcAspect);
1795 auto dstFormat = dst->getFormat(dstAspect);
1796
1797 bool doFilter = (filter != VK_FILTER_NEAREST);
1798 bool allowSRGBConversion =
1799 doFilter ||
1800 (src->getSampleCountFlagBits() > 1) ||
1801 (srcFormat.isSRGBformat() != dstFormat.isSRGBformat());
1802
1803 State state(src->getFormat(srcAspect), dst->getFormat(dstAspect), src->getSampleCountFlagBits(), dst->getSampleCountFlagBits(),
1804 Options{ doFilter, allowSRGBConversion });
1805 state.clampToEdge = (region.srcOffsets[0].x < 0) ||
1806 (region.srcOffsets[0].y < 0) ||
1807 (static_cast<uint32_t>(region.srcOffsets[1].x) > srcExtent.width) ||
1808 (static_cast<uint32_t>(region.srcOffsets[1].y) > srcExtent.height) ||
1809 (doFilter && ((x0 < 0.5f) || (y0 < 0.5f)));
1810 state.filter3D = (region.srcOffsets[1].z - region.srcOffsets[0].z) !=
1811 (region.dstOffsets[1].z - region.dstOffsets[0].z);
1812
1813 auto blitRoutine = getBlitRoutine(state);
1814 if(!blitRoutine)
1815 {
1816 return;
1817 }
1818
1819 BlitData data = {
1820 nullptr, // source
1821 nullptr, // dest
1822 src->rowPitchBytes(srcAspect, region.srcSubresource.mipLevel), // sPitchB
1823 dst->rowPitchBytes(dstAspect, region.dstSubresource.mipLevel), // dPitchB
1824 src->slicePitchBytes(srcAspect, region.srcSubresource.mipLevel), // sSliceB
1825 dst->slicePitchBytes(dstAspect, region.dstSubresource.mipLevel), // dSliceB
1826
1827 x0,
1828 y0,
1829 z0,
1830 widthRatio,
1831 heightRatio,
1832 depthRatio,
1833
1834 region.dstOffsets[0].x, // x0d
1835 region.dstOffsets[1].x, // x1d
1836 region.dstOffsets[0].y, // y0d
1837 region.dstOffsets[1].y, // y1d
1838 region.dstOffsets[0].z, // z0d
1839 region.dstOffsets[1].z, // z1d
1840
1841 static_cast<int>(srcExtent.width), // sWidth
1842 static_cast<int>(srcExtent.height), // sHeight
1843 static_cast<int>(srcExtent.depth), // sDepth
1844
1845 false, // filter3D
1846 };
1847
1848 VkImageSubresource srcSubres = {
1849 region.srcSubresource.aspectMask,
1850 region.srcSubresource.mipLevel,
1851 region.srcSubresource.baseArrayLayer
1852 };
1853
1854 VkImageSubresource dstSubres = {
1855 region.dstSubresource.aspectMask,
1856 region.dstSubresource.mipLevel,
1857 region.dstSubresource.baseArrayLayer
1858 };
1859
1860 VkImageSubresourceRange dstSubresRange = {
1861 region.dstSubresource.aspectMask,
1862 region.dstSubresource.mipLevel,
1863 1, // levelCount
1864 region.dstSubresource.baseArrayLayer,
1865 region.dstSubresource.layerCount
1866 };
1867
1868 uint32_t lastLayer = src->getLastLayerIndex(dstSubresRange);
1869
1870 for(; dstSubres.arrayLayer <= lastLayer; srcSubres.arrayLayer++, dstSubres.arrayLayer++)
1871 {
1872 data.source = src->getTexelPointer({ 0, 0, 0 }, srcSubres);
1873 data.dest = dst->getTexelPointer({ 0, 0, 0 }, dstSubres);
1874
1875 ASSERT(data.source < src->end());
1876 ASSERT(data.dest < dst->end());
1877
1878 blitRoutine(&data);
1879 }
1880
1881 dst->contentsChanged(dstSubresRange);
1882 }
1883
resolveDepth(const vk::ImageView * src,vk::ImageView * dst,const VkSubpassDescriptionDepthStencilResolve & dsrDesc)1884 static void resolveDepth(const vk::ImageView *src, vk::ImageView *dst, const VkSubpassDescriptionDepthStencilResolve &dsrDesc)
1885 {
1886 if(dsrDesc.depthResolveMode == VK_RESOLVE_MODE_NONE)
1887 {
1888 return;
1889 }
1890
1891 vk::Format format = src->getFormat(VK_IMAGE_ASPECT_DEPTH_BIT);
1892 VkExtent2D extent = src->getMipLevelExtent(0, VK_IMAGE_ASPECT_DEPTH_BIT);
1893 int width = extent.width;
1894 int height = extent.height;
1895 int pitch = src->rowPitchBytes(VK_IMAGE_ASPECT_DEPTH_BIT, 0);
1896
1897 // To support other resolve modes, get the slice bytes and get a pointer to each sample plane.
1898 // Then modify the loop below to include logic for handling each new mode.
1899 uint8_t *source = (uint8_t *)src->getOffsetPointer({ 0, 0, 0 }, VK_IMAGE_ASPECT_DEPTH_BIT, 0, 0);
1900 uint8_t *dest = (uint8_t *)dst->getOffsetPointer({ 0, 0, 0 }, VK_IMAGE_ASPECT_DEPTH_BIT, 0, 0);
1901
1902 size_t formatSize = format.bytes();
1903 // TODO(b/167558951) support other resolve modes.
1904 ASSERT(dsrDesc.depthResolveMode == VK_RESOLVE_MODE_SAMPLE_ZERO_BIT);
1905 for(int y = 0; y < height; y++)
1906 {
1907 memcpy(dest, source, formatSize * width);
1908
1909 source += pitch;
1910 dest += pitch;
1911 }
1912
1913 dst->contentsChanged();
1914 }
1915
resolveStencil(const vk::ImageView * src,vk::ImageView * dst,const VkSubpassDescriptionDepthStencilResolve & dsrDesc)1916 static void resolveStencil(const vk::ImageView *src, vk::ImageView *dst, const VkSubpassDescriptionDepthStencilResolve &dsrDesc)
1917 {
1918 if(dsrDesc.stencilResolveMode == VK_RESOLVE_MODE_NONE)
1919 {
1920 return;
1921 }
1922
1923 VkExtent2D extent = src->getMipLevelExtent(0, VK_IMAGE_ASPECT_STENCIL_BIT);
1924 int width = extent.width;
1925 int height = extent.height;
1926 int pitch = src->rowPitchBytes(VK_IMAGE_ASPECT_STENCIL_BIT, 0);
1927
1928 // To support other resolve modes, use src->slicePitchBytes() and get a pointer to each sample's slice.
1929 // Then modify the loop below to include logic for handling each new mode.
1930 uint8_t *source = reinterpret_cast<uint8_t *>(src->getOffsetPointer({ 0, 0, 0 }, VK_IMAGE_ASPECT_STENCIL_BIT, 0, 0));
1931 uint8_t *dest = reinterpret_cast<uint8_t *>(dst->getOffsetPointer({ 0, 0, 0 }, VK_IMAGE_ASPECT_STENCIL_BIT, 0, 0));
1932
1933 // TODO(b/167558951) support other resolve modes.
1934 ASSERT(dsrDesc.stencilResolveMode == VK_RESOLVE_MODE_SAMPLE_ZERO_BIT);
1935 for(int y = 0; y < height; y++)
1936 {
1937 // Stencil is always 8 bits, so the width of the resource we're resolving is
1938 // the number of bytes in each row we need to copy during for SAMPLE_ZERO
1939 memcpy(dest, source, width);
1940
1941 source += pitch;
1942 dest += pitch;
1943 }
1944
1945 dst->contentsChanged();
1946 }
1947
resolveDepthStencil(const vk::ImageView * src,vk::ImageView * dst,const VkSubpassDescriptionDepthStencilResolve & dsrDesc)1948 void Blitter::resolveDepthStencil(const vk::ImageView *src, vk::ImageView *dst, const VkSubpassDescriptionDepthStencilResolve &dsrDesc)
1949 {
1950 VkImageSubresourceRange srcRange = src->getSubresourceRange();
1951 VkImageSubresourceRange dstRange = src->getSubresourceRange();
1952 ASSERT(src->getFormat() == dst->getFormat());
1953 ASSERT(srcRange.layerCount == 1 && dstRange.layerCount == 1);
1954 ASSERT(srcRange.aspectMask == dstRange.aspectMask);
1955
1956 if(srcRange.aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT)
1957 {
1958 resolveDepth(src, dst, dsrDesc);
1959 }
1960 if(srcRange.aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT)
1961 {
1962 resolveStencil(src, dst, dsrDesc);
1963 }
1964 }
1965
resolve(const vk::Image * src,vk::Image * dst,VkImageResolve region)1966 void Blitter::resolve(const vk::Image *src, vk::Image *dst, VkImageResolve region)
1967 {
1968 // "The aspectMask member of srcSubresource and dstSubresource must only contain VK_IMAGE_ASPECT_COLOR_BIT"
1969 ASSERT(region.srcSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1970 ASSERT(region.dstSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1971 // "The layerCount member of srcSubresource and dstSubresource must match"
1972 ASSERT(region.srcSubresource.layerCount == region.dstSubresource.layerCount);
1973
1974 // We use this method both for explicit resolves from vkCmdResolveImage, and implicit ones for resolve attachments.
1975 // - vkCmdResolveImage: "srcImage and dstImage must have been created with the same image format."
1976 // - VkSubpassDescription: "each resolve attachment that is not VK_ATTACHMENT_UNUSED must have the same VkFormat as its corresponding color attachment."
1977 ASSERT(src->getFormat() == dst->getFormat());
1978
1979 if(fastResolve(src, dst, region))
1980 {
1981 return;
1982 }
1983
1984 // Fall back to a generic blit which performs the resolve.
1985 VkImageBlit blitRegion;
1986
1987 blitRegion.srcOffsets[0] = blitRegion.srcOffsets[1] = region.srcOffset;
1988 blitRegion.srcOffsets[1].x += region.extent.width;
1989 blitRegion.srcOffsets[1].y += region.extent.height;
1990 blitRegion.srcOffsets[1].z += region.extent.depth;
1991
1992 blitRegion.dstOffsets[0] = blitRegion.dstOffsets[1] = region.dstOffset;
1993 blitRegion.dstOffsets[1].x += region.extent.width;
1994 blitRegion.dstOffsets[1].y += region.extent.height;
1995 blitRegion.dstOffsets[1].z += region.extent.depth;
1996
1997 blitRegion.srcSubresource = region.srcSubresource;
1998 blitRegion.dstSubresource = region.dstSubresource;
1999
2000 blit(src, dst, blitRegion, VK_FILTER_NEAREST);
2001 }
2002
averageByte4(uint32_t x,uint32_t y)2003 static inline uint32_t averageByte4(uint32_t x, uint32_t y)
2004 {
2005 return (x & y) + (((x ^ y) >> 1) & 0x7F7F7F7F) + ((x ^ y) & 0x01010101);
2006 }
2007
fastResolve(const vk::Image * src,vk::Image * dst,VkImageResolve region)2008 bool Blitter::fastResolve(const vk::Image *src, vk::Image *dst, VkImageResolve region)
2009 {
2010 if(region.dstOffset != VkOffset3D{ 0, 0, 0 })
2011 {
2012 return false;
2013 }
2014
2015 if(region.srcOffset != VkOffset3D{ 0, 0, 0 })
2016 {
2017 return false;
2018 }
2019
2020 if(region.srcSubresource.layerCount != 1)
2021 {
2022 return false;
2023 }
2024
2025 if(region.extent != src->getExtent() ||
2026 region.extent != dst->getExtent() ||
2027 region.extent.depth != 1)
2028 {
2029 return false;
2030 }
2031
2032 VkImageSubresource srcSubresource = {
2033 region.srcSubresource.aspectMask,
2034 region.srcSubresource.mipLevel,
2035 region.srcSubresource.baseArrayLayer
2036 };
2037
2038 VkImageSubresource dstSubresource = {
2039 region.dstSubresource.aspectMask,
2040 region.dstSubresource.mipLevel,
2041 region.dstSubresource.baseArrayLayer
2042 };
2043
2044 VkImageSubresourceRange dstSubresourceRange = {
2045 region.dstSubresource.aspectMask,
2046 region.dstSubresource.mipLevel,
2047 1, // levelCount
2048 region.dstSubresource.baseArrayLayer,
2049 region.dstSubresource.layerCount
2050 };
2051
2052 void *source = src->getTexelPointer({ 0, 0, 0 }, srcSubresource);
2053 uint8_t *dest = reinterpret_cast<uint8_t *>(dst->getTexelPointer({ 0, 0, 0 }, dstSubresource));
2054
2055 auto format = src->getFormat();
2056 auto samples = src->getSampleCountFlagBits();
2057 auto extent = src->getExtent();
2058
2059 int width = extent.width;
2060 int height = extent.height;
2061 int pitch = src->rowPitchBytes(VK_IMAGE_ASPECT_COLOR_BIT, region.srcSubresource.mipLevel);
2062 int slice = src->slicePitchBytes(VK_IMAGE_ASPECT_COLOR_BIT, region.srcSubresource.mipLevel);
2063
2064 uint8_t *source0 = (uint8_t *)source;
2065 uint8_t *source1 = source0 + slice;
2066 uint8_t *source2 = source1 + slice;
2067 uint8_t *source3 = source2 + slice;
2068
2069 [[maybe_unused]] const bool SSE2 = CPUID::supportsSSE2();
2070
2071 if(format == VK_FORMAT_R8G8B8A8_UNORM || format == VK_FORMAT_B8G8R8A8_UNORM || format == VK_FORMAT_A8B8G8R8_UNORM_PACK32)
2072 {
2073 if(samples == 4)
2074 {
2075 for(int y = 0; y < height; y++)
2076 {
2077 int x = 0;
2078
2079 #if defined(__i386__) || defined(__x86_64__)
2080 if(SSE2)
2081 {
2082 for(; (x + 3) < width; x += 4)
2083 {
2084 __m128i c0 = _mm_loadu_si128((__m128i *)(source0 + 4 * x));
2085 __m128i c1 = _mm_loadu_si128((__m128i *)(source1 + 4 * x));
2086 __m128i c2 = _mm_loadu_si128((__m128i *)(source2 + 4 * x));
2087 __m128i c3 = _mm_loadu_si128((__m128i *)(source3 + 4 * x));
2088
2089 c0 = _mm_avg_epu8(c0, c1);
2090 c2 = _mm_avg_epu8(c2, c3);
2091 c0 = _mm_avg_epu8(c0, c2);
2092
2093 _mm_storeu_si128((__m128i *)(dest + 4 * x), c0);
2094 }
2095 }
2096 #endif
2097
2098 for(; x < width; x++)
2099 {
2100 uint32_t c0 = *(uint32_t *)(source0 + 4 * x);
2101 uint32_t c1 = *(uint32_t *)(source1 + 4 * x);
2102 uint32_t c2 = *(uint32_t *)(source2 + 4 * x);
2103 uint32_t c3 = *(uint32_t *)(source3 + 4 * x);
2104
2105 uint32_t c01 = averageByte4(c0, c1);
2106 uint32_t c23 = averageByte4(c2, c3);
2107 uint32_t c03 = averageByte4(c01, c23);
2108
2109 *(uint32_t *)(dest + 4 * x) = c03;
2110 }
2111
2112 source0 += pitch;
2113 source1 += pitch;
2114 source2 += pitch;
2115 source3 += pitch;
2116 dest += pitch;
2117
2118 ASSERT(source0 < src->end());
2119 ASSERT(source3 < src->end());
2120 ASSERT(dest < dst->end());
2121 }
2122 }
2123 else
2124 UNSUPPORTED("Samples: %d", samples);
2125 }
2126 else
2127 {
2128 return false;
2129 }
2130
2131 dst->contentsChanged(dstSubresourceRange);
2132
2133 return true;
2134 }
2135
copy(const vk::Image * src,uint8_t * dst,unsigned int dstPitch)2136 void Blitter::copy(const vk::Image *src, uint8_t *dst, unsigned int dstPitch)
2137 {
2138 VkExtent3D extent = src->getExtent();
2139 size_t rowBytes = src->getFormat(VK_IMAGE_ASPECT_COLOR_BIT).bytes() * extent.width;
2140 unsigned int srcPitch = src->rowPitchBytes(VK_IMAGE_ASPECT_COLOR_BIT, 0);
2141 ASSERT(dstPitch >= rowBytes && srcPitch >= rowBytes && src->getMipLevelExtent(VK_IMAGE_ASPECT_COLOR_BIT, 0).height >= extent.height);
2142
2143 const uint8_t *s = (uint8_t *)src->getTexelPointer({ 0, 0, 0 }, { VK_IMAGE_ASPECT_COLOR_BIT, 0, 0 });
2144 uint8_t *d = dst;
2145
2146 for(uint32_t y = 0; y < extent.height; y++)
2147 {
2148 memcpy(d, s, rowBytes);
2149
2150 s += srcPitch;
2151 d += dstPitch;
2152 }
2153 }
2154
computeCubeCorner(Pointer<Byte> & layer,Int & x0,Int & x1,Int & y0,Int & y1,Int & pitchB,const State & state)2155 void Blitter::computeCubeCorner(Pointer<Byte> &layer, Int &x0, Int &x1, Int &y0, Int &y1, Int &pitchB, const State &state)
2156 {
2157 int bytes = state.sourceFormat.bytes();
2158
2159 Float4 c = readFloat4(layer + ComputeOffset(x0, y1, pitchB, bytes), state) +
2160 readFloat4(layer + ComputeOffset(x1, y0, pitchB, bytes), state) +
2161 readFloat4(layer + ComputeOffset(x1, y1, pitchB, bytes), state);
2162
2163 c *= Float4(1.0f / 3.0f);
2164
2165 write(c, layer + ComputeOffset(x0, y0, pitchB, bytes), state);
2166 }
2167
generateCornerUpdate(const State & state)2168 Blitter::CornerUpdateRoutineType Blitter::generateCornerUpdate(const State &state)
2169 {
2170 // Reading and writing from/to the same image
2171 ASSERT(state.sourceFormat == state.destFormat);
2172 ASSERT(state.srcSamples == state.destSamples);
2173
2174 // Vulkan 1.2: "If samples is not VK_SAMPLE_COUNT_1_BIT, then imageType must be
2175 // VK_IMAGE_TYPE_2D, flags must not contain VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT"
2176 ASSERT(state.srcSamples == 1);
2177
2178 CornerUpdateFunction function;
2179 {
2180 Pointer<Byte> blit(function.Arg<0>());
2181
2182 Pointer<Byte> layers = *Pointer<Pointer<Byte>>(blit + OFFSET(CubeBorderData, layers));
2183 Int pitchB = *Pointer<Int>(blit + OFFSET(CubeBorderData, pitchB));
2184 UInt layerSize = *Pointer<Int>(blit + OFFSET(CubeBorderData, layerSize));
2185 UInt dim = *Pointer<Int>(blit + OFFSET(CubeBorderData, dim));
2186
2187 // Low Border, Low Pixel, High Border, High Pixel
2188 Int LB(-1), LP(0), HB(dim), HP(dim - 1);
2189
2190 for(int face = 0; face < 6; face++)
2191 {
2192 computeCubeCorner(layers, LB, LP, LB, LP, pitchB, state);
2193 computeCubeCorner(layers, LB, LP, HB, HP, pitchB, state);
2194 computeCubeCorner(layers, HB, HP, LB, LP, pitchB, state);
2195 computeCubeCorner(layers, HB, HP, HB, HP, pitchB, state);
2196 layers = layers + layerSize;
2197 }
2198 }
2199
2200 return function("BlitRoutine");
2201 }
2202
updateBorders(vk::Image * image,const VkImageSubresource & subresource)2203 void Blitter::updateBorders(vk::Image *image, const VkImageSubresource &subresource)
2204 {
2205 ASSERT(image->getArrayLayers() >= (subresource.arrayLayer + 6));
2206
2207 // From Vulkan 1.1 spec, section 11.5. Image Views:
2208 // "For cube and cube array image views, the layers of the image view starting
2209 // at baseArrayLayer correspond to faces in the order +X, -X, +Y, -Y, +Z, -Z."
2210 VkImageSubresource posX = subresource;
2211 VkImageSubresource negX = posX;
2212 negX.arrayLayer++;
2213 VkImageSubresource posY = negX;
2214 posY.arrayLayer++;
2215 VkImageSubresource negY = posY;
2216 negY.arrayLayer++;
2217 VkImageSubresource posZ = negY;
2218 posZ.arrayLayer++;
2219 VkImageSubresource negZ = posZ;
2220 negZ.arrayLayer++;
2221
2222 // Copy top / bottom
2223 copyCubeEdge(image, posX, BOTTOM, negY, RIGHT);
2224 copyCubeEdge(image, posY, BOTTOM, posZ, TOP);
2225 copyCubeEdge(image, posZ, BOTTOM, negY, TOP);
2226 copyCubeEdge(image, negX, BOTTOM, negY, LEFT);
2227 copyCubeEdge(image, negY, BOTTOM, negZ, BOTTOM);
2228 copyCubeEdge(image, negZ, BOTTOM, negY, BOTTOM);
2229
2230 copyCubeEdge(image, posX, TOP, posY, RIGHT);
2231 copyCubeEdge(image, posY, TOP, negZ, TOP);
2232 copyCubeEdge(image, posZ, TOP, posY, BOTTOM);
2233 copyCubeEdge(image, negX, TOP, posY, LEFT);
2234 copyCubeEdge(image, negY, TOP, posZ, BOTTOM);
2235 copyCubeEdge(image, negZ, TOP, posY, TOP);
2236
2237 // Copy left / right
2238 copyCubeEdge(image, posX, RIGHT, negZ, LEFT);
2239 copyCubeEdge(image, posY, RIGHT, posX, TOP);
2240 copyCubeEdge(image, posZ, RIGHT, posX, LEFT);
2241 copyCubeEdge(image, negX, RIGHT, posZ, LEFT);
2242 copyCubeEdge(image, negY, RIGHT, posX, BOTTOM);
2243 copyCubeEdge(image, negZ, RIGHT, negX, LEFT);
2244
2245 copyCubeEdge(image, posX, LEFT, posZ, RIGHT);
2246 copyCubeEdge(image, posY, LEFT, negX, TOP);
2247 copyCubeEdge(image, posZ, LEFT, negX, RIGHT);
2248 copyCubeEdge(image, negX, LEFT, negZ, RIGHT);
2249 copyCubeEdge(image, negY, LEFT, negX, BOTTOM);
2250 copyCubeEdge(image, negZ, LEFT, posX, RIGHT);
2251
2252 // Compute corner colors
2253 VkImageAspectFlagBits aspect = static_cast<VkImageAspectFlagBits>(subresource.aspectMask);
2254 vk::Format format = image->getFormat(aspect);
2255 VkSampleCountFlagBits samples = image->getSampleCountFlagBits();
2256 State state(format, format, samples, samples, Options{ 0xF });
2257
2258 // Vulkan 1.2: "If samples is not VK_SAMPLE_COUNT_1_BIT, then imageType must be
2259 // VK_IMAGE_TYPE_2D, flags must not contain VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT"
2260 ASSERT(samples == VK_SAMPLE_COUNT_1_BIT);
2261
2262 auto cornerUpdateRoutine = getCornerUpdateRoutine(state);
2263 if(!cornerUpdateRoutine)
2264 {
2265 return;
2266 }
2267
2268 VkExtent3D extent = image->getMipLevelExtent(aspect, subresource.mipLevel);
2269 CubeBorderData data = {
2270 image->getTexelPointer({ 0, 0, 0 }, posX),
2271 image->rowPitchBytes(aspect, subresource.mipLevel),
2272 static_cast<uint32_t>(image->getLayerSize(aspect)),
2273 extent.width
2274 };
2275 cornerUpdateRoutine(&data);
2276 }
2277
copyCubeEdge(vk::Image * image,const VkImageSubresource & dstSubresource,Edge dstEdge,const VkImageSubresource & srcSubresource,Edge srcEdge)2278 void Blitter::copyCubeEdge(vk::Image *image,
2279 const VkImageSubresource &dstSubresource, Edge dstEdge,
2280 const VkImageSubresource &srcSubresource, Edge srcEdge)
2281 {
2282 ASSERT(srcSubresource.aspectMask == dstSubresource.aspectMask);
2283 ASSERT(srcSubresource.mipLevel == dstSubresource.mipLevel);
2284 ASSERT(srcSubresource.arrayLayer != dstSubresource.arrayLayer);
2285
2286 // Figure out if the edges to be copied in reverse order respectively from one another
2287 // The copy should be reversed whenever the same edges are contiguous or if we're
2288 // copying top <-> right or bottom <-> left. This is explained by the layout, which is:
2289 //
2290 // | +y |
2291 // | -x | +z | +x | -z |
2292 // | -y |
2293
2294 bool reverse = (srcEdge == dstEdge) ||
2295 ((srcEdge == TOP) && (dstEdge == RIGHT)) ||
2296 ((srcEdge == RIGHT) && (dstEdge == TOP)) ||
2297 ((srcEdge == BOTTOM) && (dstEdge == LEFT)) ||
2298 ((srcEdge == LEFT) && (dstEdge == BOTTOM));
2299
2300 VkImageAspectFlagBits aspect = static_cast<VkImageAspectFlagBits>(srcSubresource.aspectMask);
2301 int bytes = image->getFormat(aspect).bytes();
2302 int pitchB = image->rowPitchBytes(aspect, srcSubresource.mipLevel);
2303
2304 VkExtent3D extent = image->getMipLevelExtent(aspect, srcSubresource.mipLevel);
2305 int w = extent.width;
2306 int h = extent.height;
2307 if(w != h)
2308 {
2309 UNSUPPORTED("Cube doesn't have square faces : (%d, %d)", w, h);
2310 }
2311
2312 // Src is expressed in the regular [0, width-1], [0, height-1] space
2313 bool srcHorizontal = ((srcEdge == TOP) || (srcEdge == BOTTOM));
2314 int srcDelta = srcHorizontal ? bytes : pitchB;
2315 VkOffset3D srcOffset = { (srcEdge == RIGHT) ? (w - 1) : 0, (srcEdge == BOTTOM) ? (h - 1) : 0, 0 };
2316
2317 // Dst contains borders, so it is expressed in the [-1, width], [-1, height] space
2318 bool dstHorizontal = ((dstEdge == TOP) || (dstEdge == BOTTOM));
2319 int dstDelta = (dstHorizontal ? bytes : pitchB) * (reverse ? -1 : 1);
2320 VkOffset3D dstOffset = { (dstEdge == RIGHT) ? w : -1, (dstEdge == BOTTOM) ? h : -1, 0 };
2321
2322 // Don't write in the corners
2323 if(dstHorizontal)
2324 {
2325 dstOffset.x += reverse ? w : 1;
2326 }
2327 else
2328 {
2329 dstOffset.y += reverse ? h : 1;
2330 }
2331
2332 const uint8_t *src = static_cast<const uint8_t *>(image->getTexelPointer(srcOffset, srcSubresource));
2333 uint8_t *dst = static_cast<uint8_t *>(image->getTexelPointer(dstOffset, dstSubresource));
2334 ASSERT((src < image->end()) && ((src + (w * srcDelta)) < image->end()));
2335 ASSERT((dst < image->end()) && ((dst + (w * dstDelta)) < image->end()));
2336
2337 for(int i = 0; i < w; ++i, dst += dstDelta, src += srcDelta)
2338 {
2339 memcpy(dst, src, bytes);
2340 }
2341 }
2342
2343 } // namespace sw
2344