Vulkan: Add more data to blob cache header
This is to potentially increase blob cache robustness against
collisions and partially modified data. This CL adds 7 bytes
per chunk (8 bytes in total). During decompression, the data
will be checked in order to determine if the cache data can
be used.
* Added chunk index to the header. (1 byte)
* Added CRC16 of the compressed data to the header. (2 bytes)
* Added the original (uncompressed) data size to the header. (4 bytes)
* Moved kBlobHeaderSize to the anonymous namespace.
* Added the following GN flag to enable CRC calculation for the cache.
* angle_enable_crc_for_pipeline_cache (only available in debug mode)
* Corresponds to kEnableCRCForPipelineCache.
* When it is false, the CRC is set to 0.
Bug: b/246683126
Change-Id: I9aaf9cda52e0af07a2e1c2d0c39aca407e515701
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/4378717
Commit-Queue: Amirali Abdolrashidi <abdolrashidi@google.com>
Reviewed-by: Charlie Lao <cclao@google.com>
Reviewed-by: Shahbaz Youssefi <syoussefi@chromium.org>
diff --git a/src/libANGLE/renderer/vulkan/BUILD.gn b/src/libANGLE/renderer/vulkan/BUILD.gn
index f6f571a..06092f4 100644
--- a/src/libANGLE/renderer/vulkan/BUILD.gn
+++ b/src/libANGLE/renderer/vulkan/BUILD.gn
@@ -31,6 +31,9 @@
# Enable shared ring buffer command buffer allocator
angle_enable_vulkan_shared_ring_buffer_cmd_alloc = false
+
+ # Enable using CRC for pipeline cache data (for debug mode)
+ angle_enable_crc_for_pipeline_cache = false
}
angle_source_set("angle_vk_mem_alloc_wrapper") {
@@ -75,6 +78,12 @@
if (angle_enable_memory_alloc_logging) {
defines += [ "ANGLE_ENABLE_MEMORY_ALLOC_LOGGING=1" ]
}
+
+ # Ensures that CRC is used only in debug mode.
+ assert(is_debug || !angle_enable_crc_for_pipeline_cache)
+ if (angle_enable_crc_for_pipeline_cache) {
+ defines += [ "ANGLE_ENABLE_CRC_FOR_PIPELINE_CACHE" ]
+ }
}
template("angle_vulkan_backend_template") {
diff --git a/src/libANGLE/renderer/vulkan/RendererVk.cpp b/src/libANGLE/renderer/vulkan/RendererVk.cpp
index 42c3bd3..5c32922 100644
--- a/src/libANGLE/renderer/vulkan/RendererVk.cpp
+++ b/src/libANGLE/renderer/vulkan/RendererVk.cpp
@@ -46,6 +46,12 @@
#else
constexpr bool kExposeNonConformantExtensionsAndVersions = false;
#endif
+
+#if defined(ANGLE_ENABLE_CRC_FOR_PIPELINE_CACHE)
+constexpr bool kEnableCRCForPipelineCache = true;
+#else
+constexpr bool kEnableCRCForPipelineCache = false;
+#endif
} // anonymous namespace
namespace rx
@@ -73,6 +79,13 @@
// initialization logic simpler.
constexpr uint32_t kPreferredVulkanAPIVersion = VK_API_VERSION_1_1;
+// For pipeline cache, the values stored in key data has the following format: {originalCacheSize,
+// compressedDataCRC, numChunks, chunkIndex; chunkCompressedData}. The header values are used to
+// validate the data. For example, if the original and compressed sizes are 70000 bytes (68k) and
+// 68841 bytes (67k), the compressed data will be divided into two chunks: {70000,crc0,2,0;34421
+// bytes} and {70000,crc1,2,1;34420 bytes}.
+constexpr size_t kBlobHeaderSize = 8 * sizeof(uint8_t);
+
bool IsVulkan11(uint32_t apiVersion)
{
return apiVersion >= VK_API_VERSION_1_1;
@@ -838,6 +851,54 @@
return true;
}
+// CRC16-CCITT is used for header before the pipeline cache key data.
+uint16_t ComputeCRC16(const uint8_t *data, const size_t size)
+{
+ constexpr uint16_t kPolynomialCRC16 = 0x8408;
+ uint16_t rem = 0;
+
+ for (size_t i = 0; i < size; i++)
+ {
+ rem ^= data[i];
+ for (int j = 0; j < 8; j++)
+ {
+ rem = (rem & 1) ? kPolynomialCRC16 ^ (rem >> 1) : rem >> 1;
+ }
+ }
+ return rem;
+}
+
+// Pack header data for the pipeline cache key data.
+void PackHeaderDataForPipelineCache(uint32_t cacheDataSize,
+ uint16_t compressedDataCRC,
+ uint8_t numChunks,
+ uint8_t chunkIndex,
+ uint64_t *dataOut)
+{
+ uint64_t concatenatedData = cacheDataSize;
+ concatenatedData = (concatenatedData << 16) | compressedDataCRC;
+ concatenatedData = (concatenatedData << 8) | numChunks;
+ concatenatedData = (concatenatedData << 8) | chunkIndex;
+
+ *dataOut = concatenatedData;
+}
+
+// Unpack header data from the pipeline cache key data.
+void UnpackHeaderDataForPipelineCache(uint64_t data,
+ uint32_t *cacheDataSizeOut,
+ uint16_t *compressedDataCRCOut,
+ size_t *numChunksOut,
+ size_t *chunkIndexOut)
+{
+ *chunkIndexOut = data & 0xFF;
+ data >>= 8;
+ *numChunksOut = data & 0xFF;
+ data >>= 8;
+ *compressedDataCRCOut = data & 0xFFFF;
+ data >>= 16;
+ *cacheDataSizeOut = static_cast<uint32_t>(data);
+}
+
void ComputePipelineCacheVkChunkKey(VkPhysicalDeviceProperties physicalDeviceProperties,
const uint8_t chunkIndex,
egl::BlobCache::Key *hashOut)
@@ -894,17 +955,17 @@
// pipelineCache into several parts to store seperately. There is no function to
// query the limit size in android.
constexpr size_t kMaxBlobCacheSize = 64 * 1024;
-
- // Store {numChunks, chunkCompressedData} in keyData, numChunks is used to validate the data.
- // For example, if the compressed size is 68841 bytes(67k), divide into {2,34421 bytes} and
- // {2,34420 bytes}.
- constexpr size_t kBlobHeaderSize = sizeof(uint8_t);
- size_t compressedOffset = 0;
+ size_t compressedOffset = 0;
const size_t numChunks = UnsignedCeilDivide(static_cast<unsigned int>(compressedData.size()),
kMaxBlobCacheSize - kBlobHeaderSize);
size_t chunkSize = UnsignedCeilDivide(static_cast<unsigned int>(compressedData.size()),
static_cast<unsigned int>(numChunks));
+ uint16_t compressedDataCRC = 0;
+ if (kEnableCRCForPipelineCache)
+ {
+ compressedDataCRC = ComputeCRC16(compressedData.data(), compressedData.size());
+ }
for (size_t chunkIndex = 0; chunkIndex < numChunks; ++chunkIndex)
{
@@ -921,8 +982,14 @@
return;
}
- ASSERT(numChunks <= UINT8_MAX);
- keyData.data()[0] = static_cast<uint8_t>(numChunks);
+ // Add the header data, followed by the compressed data.
+ ASSERT(numChunks <= UINT8_MAX && chunkIndex <= UINT8_MAX && cacheData.size() <= UINT32_MAX);
+ uint64_t headerData;
+ PackHeaderDataForPipelineCache(static_cast<uint32_t>(cacheData.size()), compressedDataCRC,
+ static_cast<uint8_t>(numChunks),
+ static_cast<uint8_t>(chunkIndex), &headerData);
+ *reinterpret_cast<uint64_t *>(keyData.data()) = headerData;
+
memcpy(keyData.data() + kBlobHeaderSize, compressedData.data() + compressedOffset,
chunkSize);
compressedOffset += chunkSize;
@@ -979,12 +1046,14 @@
angle::MemoryBuffer *uncompressedData,
bool *success)
{
+ // Make sure that the bool output is initialized to false.
+ *success = false;
+
// Compute the hash key of chunkIndex 0 and find the first cache data in blob cache.
egl::BlobCache::Key chunkCacheHash;
ComputePipelineCacheVkChunkKey(physicalDeviceProperties, 0, &chunkCacheHash);
egl::BlobCache::Value keyData;
- size_t keySize = 0;
- constexpr size_t kBlobHeaderSize = sizeof(uint8_t);
+ size_t keySize = 0;
if (!displayVk->getBlobCache()->get(displayVk->getScratchBuffer(), chunkCacheHash, &keyData,
&keySize) ||
@@ -994,8 +1063,18 @@
return angle::Result::Continue;
}
- // Get the number of chunks.
- size_t numChunks = keyData.data()[0];
+ // Get the number of chunks and other values from the header for data validation.
+ uint32_t uncompressedCacheDataSize;
+ uint16_t compressedDataCRC;
+ size_t numChunks;
+ size_t chunkIndex0;
+
+ uint64_t headerData = *reinterpret_cast<const uint64_t *>(keyData.data());
+ UnpackHeaderDataForPipelineCache(headerData, &uncompressedCacheDataSize, &compressedDataCRC,
+ &numChunks, &chunkIndex0);
+ ASSERT(chunkIndex0 == 0);
+ ASSERT(kEnableCRCForPipelineCache || compressedDataCRC == 0);
+
size_t chunkSize = keySize - kBlobHeaderSize;
size_t compressedSize = 0;
@@ -1019,18 +1098,38 @@
return angle::Result::Continue;
}
- size_t checkNumber = keyData.data()[0];
- chunkSize = keySize - kBlobHeaderSize;
+ // Validate the header values and ensure there is enough space to store.
+ uint32_t checkUncompressedCacheDataSize;
+ uint16_t checkCompressedDataCRC;
+ size_t checkNumChunks;
+ size_t checkChunkIndex;
- if (checkNumber != numChunks || compressedData.size() < (compressedSize + chunkSize))
+ headerData = *reinterpret_cast<const uint64_t *>(keyData.data());
+ UnpackHeaderDataForPipelineCache(headerData, &checkUncompressedCacheDataSize,
+ &checkCompressedDataCRC, &checkNumChunks,
+ &checkChunkIndex);
+ ASSERT(kEnableCRCForPipelineCache || checkCompressedDataCRC == 0);
+
+ chunkSize = keySize - kBlobHeaderSize;
+ bool isHeaderDataCorrupted =
+ (checkNumChunks != numChunks) ||
+ (checkUncompressedCacheDataSize != uncompressedCacheDataSize) ||
+ (checkCompressedDataCRC != compressedDataCRC) || (checkChunkIndex != chunkIndex) ||
+ (compressedData.size() < compressedSize + chunkSize);
+ if (isHeaderDataCorrupted)
{
- // Validate the number value and enough space to store.
- WARN() << "Pipeline cache chunk header corrupted: checkNumber = " << checkNumber
- << ", numChunks = " << numChunks
+ WARN() << "Pipeline cache chunk header corrupted: "
+ << "checkNumChunks = " << checkNumChunks << ", numChunks = " << numChunks
+ << ", checkUncompressedCacheDataSize = " << checkUncompressedCacheDataSize
+ << ", uncompressedCacheDataSize = " << uncompressedCacheDataSize
+ << ", checkCompressedDataCRC = " << checkCompressedDataCRC
+ << ", compressedDataCRC = " << compressedDataCRC
+ << ", checkChunkIndex = " << checkChunkIndex << ", chunkIndex = " << chunkIndex
<< ", compressedData.size() = " << compressedData.size()
<< ", (compressedSize + chunkSize) = " << (compressedSize + chunkSize);
return angle::Result::Continue;
}
+
memcpy(compressedData.data() + compressedSize, keyData.data() + kBlobHeaderSize, chunkSize);
compressedSize += chunkSize;
}
@@ -1040,6 +1139,25 @@
egl::DecompressBlobCacheData(compressedData.data(), compressedSize, uncompressedData),
VK_ERROR_INITIALIZATION_FAILED);
+ // CRC for compressed data and size for decompressed data should match the values in the header.
+ if (kEnableCRCForPipelineCache)
+ {
+ uint16_t computedCompressedDataCRC = ComputeCRC16(compressedData.data(), compressedSize);
+ if (computedCompressedDataCRC != compressedDataCRC)
+ {
+ FATAL() << "Expected CRC = " << compressedDataCRC
+ << ", Actual CRC = " << computedCompressedDataCRC;
+ return angle::Result::Stop;
+ }
+ }
+
+ if (uncompressedData->size() != uncompressedCacheDataSize)
+ {
+ WARN() << "Expected uncompressed size = " << uncompressedCacheDataSize
+ << ", Actual uncompressed size = " << uncompressedData->size();
+ return angle::Result::Continue;
+ }
+
*success = true;
return angle::Result::Continue;
}
