Profile format change for symbolized data access profiles
Next: 1) write llvm-unit unit test 2) how memprof de-serialization finds
the offset
diff --git a/llvm/include/llvm/ProfileData/DataAccessProf.h b/llvm/include/llvm/ProfileData/DataAccessProf.h
new file mode 100644
index 0000000..850ef54
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/DataAccessProf.h
@@ -0,0 +1,57 @@
+#ifndef LLVM_PROFILEDATA_DATAACCESSPROF_H_
+#define LLVM_PROFILEDATA_DATAACCESSPROF_H_
+
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+
+#include <cstdint>
+
+namespace llvm {
+
+struct DataLocation {
+ // Filenames are duplicated.
+ uint64_t FileNameIndex;
+ uint32_t Line;
+};
+
+struct DataAccessProfRecord {
+ uint64_t SymbolNameIndex;
+ uint64_t StringContentHash;
+ uint64_t AccessCount;
+ llvm::SmallVector<DataLocation> Locations;
+};
+
+class DataAccessProfData {
+ uint64_t addStringRef(StringRef Str,
+ llvm::MapVector<StringRef, uint64_t> &Map);
+
+ Error deserializeSymbolNames(const unsigned char *&Ptr);
+
+ Error deserializeFileNames(const unsigned char *&Ptr);
+
+ Error deserializeRecords(const unsigned char *&Ptr);
+
+public:
+ // Canonicalized symbol names.
+ llvm::MapVector<StringRef, uint64_t> SymbolNames;
+ llvm::MapVector<StringRef, uint64_t> FileNames;
+ llvm::SmallVector<DataAccessProfRecord> Records;
+
+ uint64_t addSymbolName(StringRef SymbolName);
+ uint64_t addFileName(StringRef FileName);
+
+ Error deserialize(const unsigned char *&Ptr);
+
+ DataAccessProfRecord &addRecord(uint64_t SymbolNameIndex,
+ uint64_t StringContentHash,
+ uint64_t AccessCount);
+
+ SmallVector<StringRef> getSymbolNames() const;
+
+ SmallVector<StringRef> getFileNames() const;
+};
+} // namespace llvm
+
+#endif // LLVM_PROFILEDATA_DATAACCESSPROF_H_
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 7133c0c..346913b 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -325,6 +325,9 @@
/// the duplicated profile variables for Comdat functions.
bool needsComdatForCounter(const GlobalObject &GV, const Module &M);
+Error readAndDecodeStrings(StringRef NameStrings,
+ std::function<Error(StringRef)> NameCallback);
+
/// An enum describing the attributes of an instrumented profile.
enum class InstrProfKind {
Unknown = 0x0,
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index f1010b3..99ca020 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -18,6 +18,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/IR/ProfileSummary.h"
#include "llvm/Object/BuildID.h"
+#include "llvm/ProfileData/DataAccessProf.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfCorrelator.h"
#include "llvm/ProfileData/MemProf.h"
@@ -697,6 +698,8 @@
std::unique_ptr<MemProfFrameHashTable> MemProfFrameTable;
/// MemProf call stack data on-disk indexed via call stack id.
std::unique_ptr<MemProfCallStackHashTable> MemProfCallStackTable;
+
+ std::unique_ptr<DataAccessProfData> DataAccessProfileData;
/// The starting address of the frame array.
const unsigned char *FrameBase = nullptr;
/// The starting address of the call stack array.
@@ -705,7 +708,11 @@
unsigned RadixTreeSize = 0;
Error deserializeV2(const unsigned char *Start, const unsigned char *Ptr);
- Error deserializeV3(const unsigned char *Start, const unsigned char *Ptr);
+ Error deserializeMemProf(const unsigned char *Start, const unsigned char *Ptr,
+ uint64_t MemProfVersion);
+
+ Error deserializeSymbolizedDataAccessProfiles(const unsigned char *Start,
+ const unsigned char *Ptr);
public:
IndexedMemProfReader() = default;
diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index 67d85da..2cdf2ee 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -19,6 +19,7 @@
#include "llvm/ADT/StringMap.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/Object/BuildID.h"
+#include "llvm/ProfileData/DataAccessProf.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/MemProf.h"
#include "llvm/Support/Error.h"
@@ -54,6 +55,9 @@
// The MemProf data.
memprof::IndexedMemProfData MemProfData;
+ // The symbolized data access profiles.
+ DataAccessProfData DataAccessProfileData;
+
// List of binary ids.
std::vector<llvm::object::BuildID> BinaryIds;
@@ -119,6 +123,10 @@
bool addMemProfData(memprof::IndexedMemProfData Incoming,
function_ref<void(Error)> Warn);
+ bool addSymbolizedDataAccessProfile(
+ StringRef SymbolName, uint64_t StringContentHash, uint64_t AccessCount,
+ llvm::SmallVector<std::pair<StringRef, uint32_t>> &Locations);
+
// Add a binary id to the binary ids list.
void addBinaryIds(ArrayRef<llvm::object::BuildID> BIs);
diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h
index e07a318..a28551d 100644
--- a/llvm/include/llvm/ProfileData/MemProf.h
+++ b/llvm/include/llvm/ProfileData/MemProf.h
@@ -35,6 +35,9 @@
// Version 3: Added a radix tree for call stacks. Switched to linear IDs for
// frames and call stacks.
Version3 = 3,
+
+ // Version 4: Added symbolized data access profiles.
+ Version4 = 4,
};
constexpr uint64_t MinimumSupportedVersion = Version2;
diff --git a/llvm/lib/ProfileData/CMakeLists.txt b/llvm/lib/ProfileData/CMakeLists.txt
index 4fa1b76..4cb8d37 100644
--- a/llvm/lib/ProfileData/CMakeLists.txt
+++ b/llvm/lib/ProfileData/CMakeLists.txt
@@ -1,4 +1,5 @@
add_llvm_component_library(LLVMProfileData
+ DataAccessProf.cpp
GCOV.cpp
InstrProf.cpp
InstrProfCorrelator.cpp
diff --git a/llvm/lib/ProfileData/DataAccessProf.cpp b/llvm/lib/ProfileData/DataAccessProf.cpp
new file mode 100644
index 0000000..1fa8845
--- /dev/null
+++ b/llvm/lib/ProfileData/DataAccessProf.cpp
@@ -0,0 +1,111 @@
+#include "llvm/ProfileData/DataAccessProf.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Support/Endian.h"
+
+namespace llvm {
+
+uint64_t
+DataAccessProfData::addStringRef(StringRef Str,
+ llvm::MapVector<StringRef, uint64_t> &Map) {
+ auto [Iter, Inserted] = Map.insert({Str, Map.size()});
+ return Iter->second;
+}
+
+uint64_t DataAccessProfData::addSymbolName(StringRef SymbolName) {
+ return addStringRef(SymbolName, SymbolNames);
+}
+uint64_t DataAccessProfData::addFileName(StringRef FileName) {
+ return addStringRef(FileName, FileNames);
+}
+
+DataAccessProfRecord &DataAccessProfData::addRecord(uint64_t SymbolNameIndex,
+ uint64_t StringContentHash,
+ uint64_t AccessCount) {
+ Records.push_back({SymbolNameIndex, StringContentHash, AccessCount});
+ return Records.back();
+}
+
+SmallVector<StringRef> DataAccessProfData::getSymbolNames() const {
+ return llvm::to_vector(llvm::map_range(
+ SymbolNames, [](const auto &Pair) { return Pair.first; }));
+}
+
+SmallVector<StringRef> DataAccessProfData::getFileNames() const {
+ return llvm::to_vector(
+ llvm::map_range(FileNames, [](const auto &Pair) { return Pair.first; }));
+}
+
+Error DataAccessProfData::deserialize(const unsigned char *&Ptr) {
+ if (Error E = deserializeSymbolNames(Ptr))
+ return E;
+ if (Error E = deserializeFileNames(Ptr))
+ return E;
+ if (Error E = deserializeRecords(Ptr))
+ return E;
+ return Error::success();
+}
+
+Error DataAccessProfData::deserializeSymbolNames(const unsigned char *&Ptr) {
+ uint64_t SymbolNameLen =
+ support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
+
+ StringRef MaybeCompressedSymbolNames((const char *)Ptr, SymbolNameLen);
+
+ std::function<Error(StringRef)> addSymbolName = [this](StringRef SymbolName) {
+ this->addSymbolName(SymbolName);
+ return Error::success();
+ };
+ if (Error E = readAndDecodeStrings(MaybeCompressedSymbolNames, addSymbolName))
+ return E;
+
+ Ptr += alignTo(SymbolNameLen, 8);
+ return Error::success();
+}
+
+Error DataAccessProfData::deserializeFileNames(const unsigned char *&Ptr) {
+ uint64_t FileNameLen =
+ support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
+
+ StringRef MaybeCompressedFileNames((const char *)Ptr, FileNameLen);
+
+ std::function<Error(StringRef)> addFileName = [this](StringRef FileName) {
+ this->addFileName(FileName);
+ return Error::success();
+ };
+ if (Error E = readAndDecodeStrings(MaybeCompressedFileNames, addFileName))
+ return E;
+
+ Ptr += alignTo(FileNameLen, 8);
+ return Error::success();
+}
+
+Error DataAccessProfData::deserializeRecords(const unsigned char *&Ptr) {
+ uint64_t NumRecords =
+ support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
+ Records.reserve(NumRecords);
+ for (uint64_t I = 0; I < NumRecords; ++I) {
+ uint64_t SymbolNameIndex =
+ support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
+ uint64_t StringContentHash =
+ support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
+ uint64_t AccessCount =
+ support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
+
+ DataAccessProfRecord &Record =
+ addRecord(SymbolNameIndex, StringContentHash, AccessCount);
+
+ uint64_t NumLocations =
+ support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
+ Record.Locations.reserve(NumLocations);
+ for (uint64_t J = 0; J < NumLocations; ++J) {
+ uint64_t FileNameIndex =
+ support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
+ uint32_t Line =
+ support::endian::readNext<uint32_t, llvm::endianness::little>(Ptr);
+ Record.Locations.push_back({FileNameIndex, Line});
+ }
+ }
+ return Error::success();
+}
+} // namespace llvm
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 1e427ca..94272871 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -535,9 +535,8 @@
/// \c NameStrings is a string composed of one of more possibly encoded
/// sub-strings. The substrings are separated by 0 or more zero bytes. This
/// method decodes the string and calls `NameCallback` for each substring.
-static Error
-readAndDecodeStrings(StringRef NameStrings,
- std::function<Error(StringRef)> NameCallback) {
+Error readAndDecodeStrings(StringRef NameStrings,
+ std::function<Error(StringRef)> NameCallback) {
const uint8_t *P = NameStrings.bytes_begin();
const uint8_t *EndP = NameStrings.bytes_end();
while (P < EndP) {
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 4075b51..968548d 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -1282,8 +1282,17 @@
return Error::success();
}
-Error IndexedMemProfReader::deserializeV3(const unsigned char *Start,
- const unsigned char *Ptr) {
+Error IndexedMemProfReader::deserializeSymbolizedDataAccessProfiles(
+ const unsigned char *Start, const unsigned char *Ptr) {
+ DataAccessProfileData = std::make_unique<DataAccessProfData>();
+ return DataAccessProfileData->deserialize(Ptr);
+}
+
+Error IndexedMemProfReader::deserializeMemProf(const unsigned char *Start,
+ const unsigned char *Ptr,
+ uint64_t Version) {
+ assert((Version == memprof::Version3 || Version == memprof::Version4) &&
+ "Only V3 and V4 are supported");
// The offset in the stream right before invoking
// CallStackTableGenerator.Emit.
const uint64_t CallStackPayloadOffset =
@@ -1295,6 +1304,11 @@
const uint64_t RecordTableOffset =
support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
+ uint64_t DataAccessProfileOffset = 0;
+ if (Version == memprof::Version4)
+ DataAccessProfileOffset =
+ support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
+
// Read the schema.
auto SchemaOr = memprof::readMemProfSchema(Ptr);
if (!SchemaOr)
@@ -1316,6 +1330,10 @@
/*Payload=*/Start + RecordPayloadOffset,
/*Base=*/Start, memprof::RecordLookupTrait(memprof::Version3, Schema)));
+ if (Version == memprof::Version4)
+ return deserializeSymbolizedDataAccessProfiles(
+ Start, Start + DataAccessProfileOffset);
+
return Error::success();
}
@@ -1345,7 +1363,8 @@
return E;
break;
case memprof::Version3:
- if (Error E = deserializeV3(Start, Ptr))
+ case memprof::Version4:
+ if (Error E = deserializeMemProf(Start, Ptr, Version))
return E;
break;
}
@@ -1609,6 +1628,7 @@
return getMemProfRecordV2(IndexedRecord, *MemProfFrameTable,
*MemProfCallStackTable);
case memprof::Version3:
+ case memprof::Version4:
assert(!MemProfFrameTable && "MemProfFrameTable must not be available");
assert(!MemProfCallStackTable &&
"MemProfCallStackTable must not be available");
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 18aa76c..9db591e 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -16,6 +16,7 @@
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/IR/ProfileSummary.h"
+#include "llvm/ProfileData/DataAccessProf.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/MemProf.h"
#include "llvm/ProfileData/ProfileCommon.h"
@@ -212,9 +213,7 @@
InfoObj->ValueProfDataEndianness = Endianness;
}
-void InstrProfWriter::setOutputSparse(bool Sparse) {
- this->Sparse = Sparse;
-}
+void InstrProfWriter::setOutputSparse(bool Sparse) { this->Sparse = Sparse; }
void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
function_ref<void(Error)> Warn) {
@@ -385,6 +384,27 @@
return true;
}
+bool InstrProfWriter::addSymbolizedDataAccessProfile(
+ StringRef SymbolName, uint64_t StringContentHash, uint64_t AccessCount,
+ llvm::SmallVector<std::pair<StringRef, uint32_t>> &Locations) {
+ // TODO: Validate that symbol names are not duplicated.
+ if (SymbolName.empty())
+ return false;
+ const uint64_t SymbolNameIndex =
+ DataAccessProfileData.addSymbolName(SymbolName);
+
+ DataAccessProfRecord &Record = DataAccessProfileData.addRecord(
+ SymbolNameIndex, StringContentHash, AccessCount);
+
+ for (const auto &Location : Locations) {
+ const uint64_t FileNameIndex =
+ DataAccessProfileData.addFileName(Location.first);
+ Record.Locations.push_back({FileNameIndex, Location.second});
+ }
+
+ return true;
+}
+
void InstrProfWriter::addBinaryIds(ArrayRef<llvm::object::BuildID> BIs) {
llvm::append_range(BinaryIds, BIs);
}
@@ -705,6 +725,53 @@
return Error::success();
}
+static Error writeStrings(ProfOStream &OS,
+ const llvm::SmallVector<llvm::StringRef> &Strings) {
+ std::vector<std::string> StringStrs;
+ for (StringRef String : Strings)
+ StringStrs.push_back(String.str());
+ std::string CompressedStrings;
+ if (!Strings.empty())
+ if (Error E = collectGlobalObjectNameStrings(
+ StringStrs, compression::zlib::isAvailable(), CompressedStrings))
+ return E;
+ const uint64_t CompressedStringLen = CompressedStrings.length();
+ // Record the length of compressed string.
+ OS.write(CompressedStringLen);
+ // Write the chars in compressed strings.
+ for (auto &c : CompressedStrings)
+ OS.writeByte(static_cast<uint8_t>(c));
+ // Pad up to a multiple of 8.
+ // InstrProfReader could read bytes according to 'CompressedStringLen'.
+ const uint64_t PaddedLength = alignTo(CompressedStringLen, 8);
+ for (uint64_t K = CompressedStringLen; K < PaddedLength; K++)
+ OS.writeByte(0);
+ return Error::success();
+}
+
+static Error writeDataAccessProf(ProfOStream &OS,
+ const DataAccessProfData &DataAccessProfData) {
+ if (Error E = writeStrings(OS, DataAccessProfData.getSymbolNames()))
+ return E;
+
+ if (Error E = writeStrings(OS, DataAccessProfData.getFileNames()))
+ return E;
+
+ OS.write(DataAccessProfData.Records.size());
+ for (const auto &Rec : DataAccessProfData.Records) {
+ OS.write(Rec.SymbolNameIndex);
+ OS.write(Rec.StringContentHash);
+ OS.write(Rec.AccessCount);
+ OS.write(Rec.Locations.size());
+ for (const auto &Loc : Rec.Locations) {
+ OS.write(Loc.FileNameIndex);
+ OS.write(Loc.Line);
+ }
+ }
+
+ return Error::success();
+}
+
// Write out MemProf Version3 as follows:
// uint64_t Version
// uint64_t CallStackPayloadOffset = Offset for the call stack payload
@@ -718,14 +785,19 @@
// Frames serialized one after another
// Call stacks encoded as a radix tree
// OnDiskChainedHashTable MemProfRecordData
-static Error writeMemProfV3(ProfOStream &OS,
- memprof::IndexedMemProfData &MemProfData,
- bool MemProfFullSchema) {
- OS.write(memprof::Version3);
+static Error
+writeMemProfData(ProfOStream &OS, memprof::IndexedMemProfData &MemProfData,
+ bool MemProfFullSchema, uint64_t Version,
+ const DataAccessProfData *DataAccessProfileDataPtr) {
+ assert((Version == memprof::Version3 || Version == memprof::Version4) &&
+ "Only V3 and V4 are supported");
+ OS.write(Version);
uint64_t HeaderUpdatePos = OS.tell();
OS.write(0ULL); // Reserve space for the memprof call stack payload offset.
OS.write(0ULL); // Reserve space for the memprof record payload offset.
OS.write(0ULL); // Reserve space for the memprof record table offset.
+ if (Version == memprof::Version4)
+ OS.write(0ULL); // Reserve space for the data access profile payload offset.
auto Schema = memprof::getHotColdSchema();
if (MemProfFullSchema)
@@ -760,12 +832,21 @@
NumElements * sizeof(memprof::LinearFrameId) ==
RecordPayloadOffset);
- uint64_t Header[] = {
+ const DataAccessProfData &DataAccessProfileData = *DataAccessProfileDataPtr;
+ uint64_t DataAccessProfPayloadOffset = OS.tell();
+ if (Version == memprof::Version4) {
+ if (Error E = writeDataAccessProf(OS, DataAccessProfileData))
+ return E;
+ }
+
+ SmallVector<uint64_t> PatchItemPayload = {
CallStackPayloadOffset,
RecordPayloadOffset,
RecordTableOffset,
};
- OS.patch({{HeaderUpdatePos, Header}});
+ if (Version == memprof::Version4)
+ PatchItemPayload.push_back(DataAccessProfPayloadOffset);
+ OS.patch({{HeaderUpdatePos, PatchItemPayload}});
return Error::success();
}
@@ -773,13 +854,18 @@
// Write out the MemProf data in a requested version.
static Error writeMemProf(ProfOStream &OS,
memprof::IndexedMemProfData &MemProfData,
+ const DataAccessProfData &DataAccessProfileData,
memprof::IndexedVersion MemProfVersionRequested,
bool MemProfFullSchema) {
switch (MemProfVersionRequested) {
case memprof::Version2:
return writeMemProfV2(OS, MemProfData, MemProfFullSchema);
case memprof::Version3:
- return writeMemProfV3(OS, MemProfData, MemProfFullSchema);
+ return writeMemProfData(OS, MemProfData, MemProfFullSchema,
+ memprof::Version3, nullptr);
+ case memprof::Version4:
+ return writeMemProfData(OS, MemProfData, MemProfFullSchema,
+ memprof::Version4, &DataAccessProfileData);
}
return make_error<InstrProfError>(
@@ -955,8 +1041,8 @@
uint64_t MemProfSectionStart = 0;
if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
MemProfSectionStart = OS.tell();
- if (auto E = writeMemProf(OS, MemProfData, MemProfVersionRequested,
- MemProfFullSchema))
+ if (auto E = writeMemProf(OS, MemProfData, DataAccessProfileData,
+ MemProfVersionRequested, MemProfFullSchema))
return E;
}
diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp
index 0af08ca..5e4d7e0 100644
--- a/llvm/lib/ProfileData/MemProf.cpp
+++ b/llvm/lib/ProfileData/MemProf.cpp
@@ -49,6 +49,7 @@
case Version2:
return serializedSizeV2(*this, Schema);
case Version3:
+ case Version4:
return serializedSizeV3(*this, Schema);
}
llvm_unreachable("unsupported MemProf version");
@@ -88,6 +89,7 @@
case Version2:
return serializedSizeV2(*this, Schema);
case Version3:
+ case Version4:
return serializedSizeV3(*this, Schema);
}
llvm_unreachable("unsupported MemProf version");
@@ -143,6 +145,7 @@
serializeV2(*this, Schema, OS);
return;
case Version3:
+ case Version4:
serializeV3(*this, Schema, OS, *MemProfCallStackIndexes);
return;
}
@@ -224,6 +227,7 @@
case Version2:
return deserializeV2(Schema, Ptr);
case Version3:
+ case Version4:
return deserializeV3(Schema, Ptr);
}
llvm_unreachable("unsupported MemProf version");
