diff --git a/llvm/include/llvm/CodeGen/MachineOutliner.h b/llvm/include/llvm/CodeGen/MachineOutliner.h index eaba6c9b18f2bb..84937a8b563ac0 100644 --- a/llvm/include/llvm/CodeGen/MachineOutliner.h +++ b/llvm/include/llvm/CodeGen/MachineOutliner.h @@ -234,11 +234,11 @@ struct OutlinedFunction { unsigned FrameConstructionID = 0; /// Return the number of candidates for this \p OutlinedFunction. - unsigned getOccurrenceCount() const { return Candidates.size(); } + virtual unsigned getOccurrenceCount() const { return Candidates.size(); } /// Return the number of bytes it would take to outline this /// function. - unsigned getOutliningCost() const { + virtual unsigned getOutliningCost() const { unsigned CallOverhead = 0; for (const Candidate &C : Candidates) CallOverhead += C.getCallOverhead(); @@ -272,6 +272,7 @@ struct OutlinedFunction { } OutlinedFunction() = delete; + virtual ~OutlinedFunction() = default; }; } // namespace outliner } // namespace llvm diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h index d4a83e3753d980..1e7be312851929 100644 --- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -2053,13 +2053,20 @@ class TargetInstrInfo : public MCInstrInfo { /// Returns a \p outliner::OutlinedFunction struct containing target-specific /// information for a set of outlining candidates. Returns std::nullopt if the - /// candidates are not suitable for outlining. + /// candidates are not suitable for outlining. \p MinRep is the minimum + /// number of times the instruction sequence must be repeated. virtual std::optional getOutliningCandidateInfo( - std::vector &RepeatedSequenceLocs) const { + std::vector &RepeatedSequenceLocs, + unsigned MipRep) const { llvm_unreachable( "Target didn't implement TargetInstrInfo::getOutliningCandidateInfo!"); } + virtual std::optional getOutliningCandidateInfo( + std::vector &RepeatedSequenceLocs) const { + return getOutliningCandidateInfo(RepeatedSequenceLocs, /*MipRep=*/2); + } + /// Optional target hook to create the LLVM IR attributes for the outlined /// function. If overridden, the overriding function must call the default /// implementation. diff --git a/llvm/include/llvm/CodeGenData/CodeGenData.h b/llvm/include/llvm/CodeGenData/CodeGenData.h new file mode 100644 index 00000000000000..118fb9841d27e8 --- /dev/null +++ b/llvm/include/llvm/CodeGenData/CodeGenData.h @@ -0,0 +1,202 @@ +//===- CodeGenData.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains support for codegen data that has stable summary which +// can be used to optimize the code in the subsequent codegen. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGENDATA_CODEGENDATA_H +#define LLVM_CODEGENDATA_CODEGENDATA_H + +#include "llvm/ADT/BitmaskEnum.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/CodeGenData/OutlinedHashTree.h" +#include "llvm/CodeGenData/OutlinedHashTreeRecord.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/TargetParser/Triple.h" +#include + +namespace llvm { + +enum CGDataSectKind { +#define CG_DATA_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind, +#include "llvm/CodeGenData/CodeGenData.inc" +}; + +std::string getCodeGenDataSectionName(CGDataSectKind CGSK, + Triple::ObjectFormatType OF, + bool AddSegmentInfo = true); + +enum class CGDataKind { + Unknown = 0x0, + // A function outlining info. + FunctionOutlinedHashTree = 0x1, + LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/FunctionOutlinedHashTree) +}; + +const std::error_category &cgdata_category(); + +enum class cgdata_error { + success = 0, + eof, + bad_magic, + bad_header, + empty_cgdata, + malformed, + unsupported_version, +}; + +inline std::error_code make_error_code(cgdata_error E) { + return std::error_code(static_cast(E), cgdata_category()); +} + +class CGDataError : public ErrorInfo { +public: + CGDataError(cgdata_error Err, const Twine &ErrStr = Twine()) + : Err(Err), Msg(ErrStr.str()) { + assert(Err != cgdata_error::success && "Not an error"); + } + + std::string message() const override; + + void log(raw_ostream &OS) const override { OS << message(); } + + std::error_code convertToErrorCode() const override { + return make_error_code(Err); + } + + cgdata_error get() const { return Err; } + const std::string &getMessage() const { return Msg; } + + /// Consume an Error and return the raw enum value contained within it, and + /// the optional error message. The Error must either be a success value, or + /// contain a single CGDataError. + static std::pair take(Error E) { + auto Err = cgdata_error::success; + std::string Msg = ""; + handleAllErrors(std::move(E), [&Err, &Msg](const CGDataError &IPE) { + assert(Err == cgdata_error::success && "Multiple errors encountered"); + Err = IPE.get(); + Msg = IPE.getMessage(); + }); + return {Err, Msg}; + } + + static char ID; + +private: + cgdata_error Err; + std::string Msg; +}; + +enum CGDataMode { + None, + Read, + Write, +}; + +class CodeGenData { + /// Global outlined hash tree that has oulined hash sequences across modules. + std::unique_ptr PublishedHashTree; + + /// This flag is set when -fcgdata-generate is passed. + /// Or, it can be mutated with -ftwo-codegen-rounds during two codegen runs. + bool EmitCGData; + + /// This is a singleton instance which is thread-safe. Unlike profile data + /// which is largely function-based, codegen data describes the whole module. + /// Therefore, this can be initialized once, and can be used across modules + /// instead of constructing the same one for each codegen backend. + static std::unique_ptr Instance; + static std::once_flag OnceFlag; + + CodeGenData() = default; + +public: + ~CodeGenData() = default; + + static CodeGenData &getInstance(); + + /// Returns true if we have a valid outlined hash tree. + bool hasOutlinedHashTree() { + return PublishedHashTree && !PublishedHashTree->empty(); + } + + /// Returns the outlined hash tree. This can be globally used in a read-only + /// manner. + const OutlinedHashTree *getOutlinedHashTree() { + return PublishedHashTree.get(); + } + + /// Returns true if we should write codegen data. + bool emitCGData() { return EmitCGData; } + + /// Publish the (globally) merged or read outlined hash tree. + void publishOutlinedHashTree(std::unique_ptr HashTree) { + PublishedHashTree = std::move(HashTree); + // Ensure we disable emitCGData as we do not want to read and write both. + EmitCGData = false; + } +}; + +namespace cgdata { + +inline bool hasOutlinedHashTree() { + return CodeGenData::getInstance().hasOutlinedHashTree(); +} + +inline const OutlinedHashTree *getOutlinedHashTree() { + return CodeGenData::getInstance().getOutlinedHashTree(); +} + +inline bool emitCGData() { return CodeGenData::getInstance().emitCGData(); } + +inline void +publishOutlinedHashTree(std::unique_ptr HashTree) { + CodeGenData::getInstance().publishOutlinedHashTree(std::move(HashTree)); +} + +void warn(Error E, StringRef Whence = ""); +void warn(Twine Message, std::string Whence = "", std::string Hint = ""); + +} // end namespace cgdata + +namespace IndexedCGData { + +const uint64_t Magic = 0x81617461646763ff; // "\xffcgdata\x81" + +enum CGDataVersion { + // Version 1 is the first version. This version support the outlined + // hash tree. + Version1 = 1, + CurrentVersion = CG_DATA_INDEX_VERSION +}; +const uint64_t Version = CGDataVersion::CurrentVersion; + +struct Header { + uint64_t Magic; + uint32_t Version; + uint32_t DataKind; + uint64_t OutlinedHashTreeOffset; + + // New fields should only be added at the end to ensure that the size + // computation is correct. The methods below need to be updated to ensure that + // the new field is read correctly. + + // Reads a header struct from the buffer. + static Expected
readFromBuffer(const unsigned char *Curr); +}; + +} // end namespace IndexedCGData + +} // end namespace llvm + +#endif // LLVM_CODEGEN_PREPARE_H diff --git a/llvm/include/llvm/CodeGenData/CodeGenData.inc b/llvm/include/llvm/CodeGenData/CodeGenData.inc new file mode 100644 index 00000000000000..5f6df5c0bf1065 --- /dev/null +++ b/llvm/include/llvm/CodeGenData/CodeGenData.inc @@ -0,0 +1,46 @@ +/*===-- CodeGenData.inc ----------------------------------------*- C++ -*-=== *\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ +/* + * This is the main file that defines all the data structure, signature, + * constant literals that are shared across compiler, host tools (reader/writer) + * to support codegen data. + * +\*===----------------------------------------------------------------------===*/ + +#ifdef CG_DATA_SECT_ENTRY +#define CG_DATA_DEFINED +CG_DATA_SECT_ENTRY(CG_outline, CG_DATA_QUOTE(CG_DATA_OUTLINE_COMMON), + CG_DATA_OUTLINE_COFF, "__DATA,") + +#undef CG_DATA_SECT_ENTRY +#endif + +/* section name strings common to all targets other + than WIN32 */ +#define CG_DATA_OUTLINE_COMMON __llvm_outline +/* Since cg data sections are not allocated, we don't need to + * access them at runtime. + */ +#define CG_DATA_OUTLINE_COFF ".loutline" + +#ifdef _WIN32 +/* Runtime section names and name strings. */ +#define CG_DATA_SECT_NAME CG_DATA_OUTLINE_COFF + +#else +/* Runtime section names and name strings. */ +#define CG_DATA_SECT_NAME INSTR_PROF_QUOTE(CG_DATA_OUTLINE_COMMON) + +#endif + +/* Indexed codegen data format version (start from 1). */ +#define CG_DATA_INDEX_VERSION 1 + +/* Helper macros. */ +#define CG_DATA_SIMPLE_QUOTE(x) #x +#define CG_DATA_QUOTE(x) CG_DATA_SIMPLE_QUOTE(x) diff --git a/llvm/include/llvm/CodeGenData/CodeGenDataReader.h b/llvm/include/llvm/CodeGenData/CodeGenDataReader.h new file mode 100644 index 00000000000000..df4ae3ed24e79a --- /dev/null +++ b/llvm/include/llvm/CodeGenData/CodeGenDataReader.h @@ -0,0 +1,154 @@ +//===- CodeGenDataReader.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains support for reading codegen data. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGENDATA_CODEGENDATAREADER_H +#define LLVM_CODEGENDATA_CODEGENDATAREADER_H + +#include "llvm/CodeGenData/CodeGenData.h" +#include "llvm/CodeGenData/OutlinedHashTreeRecord.h" +#include "llvm/Support/LineIterator.h" +#include "llvm/Support/VirtualFileSystem.h" + +namespace llvm { + +class CodeGenDataReader { + cgdata_error LastError = cgdata_error::success; + std::string LastErrorMsg; + +public: + CodeGenDataReader() = default; + virtual ~CodeGenDataReader() = default; + + /// Read the header. Required before reading first record. + virtual Error read() = 0; + /// Return the codegen data version. + virtual uint32_t getVersion() const = 0; + /// Return the codegen data kind. + virtual CGDataKind getDataKind() const = 0; + /// Return true if the data has an outlined hash tree. + virtual bool hasOutlinedHashTree() const = 0; + /// Return the outlined hash tree that is released from the reader. + std::unique_ptr releaseOutlinedHashTree() { + return std::move(HashTreeRecord.HashTree); + } + + /// Factory method to create an appropriately typed reader for the given + /// codegen data file path and file system. + static Expected> + create(const Twine &Path, vfs::FileSystem &FS); + + /// Factory method to create an appropriately typed reader for the given + /// memory buffer. + static Expected> + create(std::unique_ptr Buffer); + + /// Extract the cgdata embedded in sections from the given object file and + /// merge them into the GlobalOutlineRecord. This is a static helper that + /// is used by `llvm-cgdata merge` or ThinLTO's two-codegen rounds. + static Error mergeFromObjectFile(const object::ObjectFile *Obj, + OutlinedHashTreeRecord &GlobalOutlineRecord); + +protected: + /// The outlined hash tree that has been read. When it's released by + /// releaseOutlinedHashTree(), it's no longer valid. + OutlinedHashTreeRecord HashTreeRecord; + + /// Set the current error and return same. + Error error(cgdata_error Err, const std::string &ErrMsg = "") { + LastError = Err; + LastErrorMsg = ErrMsg; + if (Err == cgdata_error::success) + return Error::success(); + return make_error(Err, ErrMsg); + } + + Error error(Error &&E) { + handleAllErrors(std::move(E), [&](const CGDataError &IPE) { + LastError = IPE.get(); + LastErrorMsg = IPE.getMessage(); + }); + return make_error(LastError, LastErrorMsg); + } + + /// Clear the current error and return a successful one. + Error success() { return error(cgdata_error::success); } +}; + +class IndexedCodeGenDataReader : public CodeGenDataReader { + /// The codegen data file contents. + std::unique_ptr DataBuffer; + /// The header + IndexedCGData::Header Header; + +public: + IndexedCodeGenDataReader(std::unique_ptr DataBuffer) + : DataBuffer(std::move(DataBuffer)) {} + IndexedCodeGenDataReader(const IndexedCodeGenDataReader &) = delete; + IndexedCodeGenDataReader & + operator=(const IndexedCodeGenDataReader &) = delete; + + /// Return true if the given buffer is in binary codegen data format. + static bool hasFormat(const MemoryBuffer &Buffer); + /// Read the contents including the header. + Error read() override; + /// Return the codegen data version. + uint32_t getVersion() const override { return Header.Version; } + /// Return the codegen data kind. + CGDataKind getDataKind() const override { + return static_cast(Header.DataKind); + } + /// Return true if the header indicates the data has an outlined hash tree. + /// This does not mean that the data is still available. + bool hasOutlinedHashTree() const override { + return Header.DataKind & + static_cast(CGDataKind::FunctionOutlinedHashTree); + } +}; + +/// This format is a simple text format that's suitable for test data. +/// The header is a custom format starting with `:` per line to indicate which +/// codegen data is recorded. `#` is used to indicate a comment. +/// The subsequent data is a YAML format per each codegen data in order. +/// Currently, it only has a function outlined hash tree. +class TextCodeGenDataReader : public CodeGenDataReader { + /// The codegen data file contents. + std::unique_ptr DataBuffer; + /// Iterator over the profile data. + line_iterator Line; + /// Describe the kind of the codegen data. + CGDataKind DataKind = CGDataKind::Unknown; + +public: + TextCodeGenDataReader(std::unique_ptr DataBuffer_) + : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#') {} + TextCodeGenDataReader(const TextCodeGenDataReader &) = delete; + TextCodeGenDataReader &operator=(const TextCodeGenDataReader &) = delete; + + /// Return true if the given buffer is in text codegen data format. + static bool hasFormat(const MemoryBuffer &Buffer); + /// Read the contents including the header. + Error read() override; + /// Text format does not have version, so return 0. + uint32_t getVersion() const override { return 0; } + /// Return the codegen data kind. + CGDataKind getDataKind() const override { return DataKind; } + /// Return true if the header indicates the data has an outlined hash tree. + /// This does not mean that the data is still available. + bool hasOutlinedHashTree() const override { + return static_cast(DataKind) & + static_cast(CGDataKind::FunctionOutlinedHashTree); + } +}; + +} // end namespace llvm + +#endif // LLVM_CODEGENDATA_CODEGENDATAREADER_H diff --git a/llvm/include/llvm/CodeGenData/CodeGenDataWriter.h b/llvm/include/llvm/CodeGenData/CodeGenDataWriter.h new file mode 100644 index 00000000000000..e17ffc3482ec91 --- /dev/null +++ b/llvm/include/llvm/CodeGenData/CodeGenDataWriter.h @@ -0,0 +1,68 @@ +//===- CodeGenDataWriter.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains support for writing codegen data. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGENDATA_CODEGENDATAWRITER_H +#define LLVM_CODEGENDATA_CODEGENDATAWRITER_H + +#include "llvm/CodeGenData/CodeGenData.h" +#include "llvm/CodeGenData/OutlinedHashTreeRecord.h" +#include "llvm/Support/Error.h" + +namespace llvm { + +class CGDataOStream; + +class CodeGenDataWriter { + /// The outlined hash tree to be written. + OutlinedHashTreeRecord HashTreeRecord; + + /// A bit mask describing the kind of the codegen data. + CGDataKind DataKind = CGDataKind::Unknown; + +public: + CodeGenDataWriter() = default; + ~CodeGenDataWriter() = default; + + /// Add the outlined hash tree record. The input Record is released. + void addRecord(OutlinedHashTreeRecord &Record); + + /// Write the codegen data to \c OS + Error write(raw_fd_ostream &OS); + + /// Write the codegen data in text format to \c OS + Error writeText(raw_fd_ostream &OS); + + /// Return the attributes of the current CGData. + CGDataKind getCGDataKind() const { return DataKind; } + + /// Return true if the header indicates the data has an outlined hash tree. + bool hasOutlinedHashTree() const { + return static_cast(DataKind) & + static_cast(CGDataKind::FunctionOutlinedHashTree); + } + +private: + /// The offset of the outlined hash tree in the file. + uint64_t OutlinedHashTreeOffset; + + /// Write the codegen data header to \c COS + Error writeHeader(CGDataOStream &COS); + + /// Write the codegen data header in text to \c OS + Error writeHeaderText(raw_fd_ostream &OS); + + Error writeImpl(CGDataOStream &COS); +}; + +} // end namespace llvm + +#endif // LLVM_CODEGENDATA_CODEGENDATAWRITER_H diff --git a/llvm/include/llvm/CodeGenData/OutlinedHashTree.h b/llvm/include/llvm/CodeGenData/OutlinedHashTree.h new file mode 100644 index 00000000000000..875e1a78bb4010 --- /dev/null +++ b/llvm/include/llvm/CodeGenData/OutlinedHashTree.h @@ -0,0 +1,107 @@ +//===- OutlinedHashTree.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// +// +// This defines the OutlinedHashTree class. It contains sequences of stable +// hash values of instructions that have been outlined. This OutlinedHashTree +// can be used to track the outlined instruction sequences across modules. +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_CODEGENDATA_OUTLINEDHASHTREE_H +#define LLVM_CODEGENDATA_OUTLINEDHASHTREE_H + +#include "llvm/ADT/StableHashing.h" +#include "llvm/ObjectYAML/YAML.h" +#include "llvm/Support/raw_ostream.h" + +#include +#include + +namespace llvm { + +/// A HashNode is an entry in an OutlinedHashTree, holding a hash value +/// and a collection of Successors (other HashNodes). If a HashNode has +/// a positive terminal value (Terminals > 0), it signifies the end of +/// a hash sequence with that occurrence count. +struct HashNode { + /// The hash value of the node. + stable_hash Hash; + /// The number of terminals in the sequence ending at this node. + unsigned Terminals; + /// The successors of this node. + std::unordered_map> Successors; +}; + +/// HashNodeStable is the serialized, stable, and compact representation +/// of a HashNode. +struct HashNodeStable { + llvm::yaml::Hex64 Hash; + unsigned Terminals; + std::vector SuccessorIds; +}; + +class OutlinedHashTree { + + using EdgeCallbackFn = + std::function; + using NodeCallbackFn = std::function; + + using HashSequence = std::vector; + using HashSequencePair = std::pair, unsigned>; + +public: + /// Walks every edge and node in the OutlinedHashTree and calls CallbackEdge + /// for the edges and CallbackNode for the nodes with the stable_hash for + /// the source and the stable_hash of the sink for an edge. These generic + /// callbacks can be used to traverse a OutlinedHashTree for the purpose of + /// print debugging or serializing it. + void walkGraph(NodeCallbackFn CallbackNode, + EdgeCallbackFn CallbackEdge = nullptr, + bool SortedWalk = false) const; + + /// Release all hash nodes except the root hash node. + void clear() { + assert(getRoot()->Hash == 0 && getRoot()->Terminals == 0); + getRoot()->Successors.clear(); + } + + /// \returns true if the hash tree has only the root node. + bool empty() { return size() == 1; } + + /// \returns the size of a OutlinedHashTree by traversing it. If + /// \p GetTerminalCountOnly is true, it only counts the terminal nodes + /// (meaning it returns the the number of hash sequences in the + /// OutlinedHashTree). + size_t size(bool GetTerminalCountOnly = false) const; + + /// \returns the depth of a OutlinedHashTree by traversing it. + size_t depth() const; + + /// \returns the root hash node of a OutlinedHashTree. + const HashNode *getRoot() const { return Root.get(); } + HashNode *getRoot() { return Root.get(); } + + /// Inserts a \p Sequence into the this tree. The last node in the sequence + /// will increase Terminals. + void insert(const HashSequencePair &SequencePair); + + /// Merge a \p OtherTree into this Tree. + void merge(const OutlinedHashTree *OtherTree); + + /// \returns the matching count if \p Sequence exists in the OutlinedHashTree. + unsigned find(const HashSequence &Sequence) const; + + OutlinedHashTree() { Root = std::make_unique(); } + +private: + std::unique_ptr Root; +}; + +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/CodeGenData/OutlinedHashTreeRecord.h b/llvm/include/llvm/CodeGenData/OutlinedHashTreeRecord.h new file mode 100644 index 00000000000000..ccd2ad26dd0871 --- /dev/null +++ b/llvm/include/llvm/CodeGenData/OutlinedHashTreeRecord.h @@ -0,0 +1,67 @@ +//===- OutlinedHashTreeRecord.h --------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// +// +// This defines the OutlinedHashTreeRecord class. This class holds the outlined +// hash tree for both serialization and deserialization processes. It utilizes +// two data formats for serialization: raw binary data and YAML. +// These two formats can be used interchangeably. +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_CODEGENDATA_OUTLINEDHASHTREERECORD_H +#define LLVM_CODEGENDATA_OUTLINEDHASHTREERECORD_H + +#include "llvm/CodeGenData/OutlinedHashTree.h" + +namespace llvm { + +using IdHashNodeStableMapTy = std::map; +using IdHashNodeMapTy = std::map; +using HashNodeIdMapTy = std::unordered_map; + +struct OutlinedHashTreeRecord { + std::unique_ptr HashTree; + + OutlinedHashTreeRecord() { HashTree = std::make_unique(); } + OutlinedHashTreeRecord(std::unique_ptr HashTree) + : HashTree(std::move(HashTree)){}; + + /// Serialize the outlined hash tree to a raw_ostream. + void serialize(raw_ostream &OS) const; + /// Deserialize the outlined hash tree from a raw_ostream. + void deserialize(const unsigned char *&Ptr); + /// Serialize the outlined hash tree to a YAML stream. + void serializeYAML(yaml::Output &YOS) const; + /// Deserialize the outlined hash tree from a YAML stream. + void deserializeYAML(yaml::Input &YIS); + + /// Merge the other outlined hash tree into this one. + void merge(const OutlinedHashTreeRecord &Other) { + HashTree->merge(Other.HashTree.get()); + } + + /// \returns true if the outlined hash tree is empty. + bool empty() const { return HashTree->empty(); } + + /// Print the outlined hash tree in a YAML format. + void print(raw_ostream &OS = llvm::errs()) const { + yaml::Output YOS(OS); + serializeYAML(YOS); + } + +private: + /// Convert the outlined hash tree to stable data. + void convertToStableData(IdHashNodeStableMapTy &IdNodeStableMap) const; + + /// Convert the stable data back to the outlined hash tree. + void convertFromStableData(const IdHashNodeStableMapTy &IdNodeStableMap); +}; + +} // end namespace llvm + +#endif // LLVM_CODEGENDATA_OUTLINEDHASHTREERECORD_H diff --git a/llvm/include/llvm/Support/SuffixTree.h b/llvm/include/llvm/Support/SuffixTree.h index 4940fbbf308d8b..37b73666404300 100644 --- a/llvm/include/llvm/Support/SuffixTree.h +++ b/llvm/include/llvm/Support/SuffixTree.h @@ -42,6 +42,9 @@ class SuffixTree { /// Each element is an integer representing an instruction in the module. ArrayRef Str; + /// Whether to consider leaf descendants or only leaf children. + bool OutlinerLeafDescendants; + /// A repeated substring in the tree. struct RepeatedSubstring { /// The length of the string. @@ -130,11 +133,27 @@ class SuffixTree { /// this step. unsigned extend(unsigned EndIdx, unsigned SuffixesToAdd); + /// This vector contains all leaf nodes of this suffix tree. These leaf nodes + /// are identified using post-order depth-first traversal, so that the order + /// of these leaf nodes in the vector matches the order of the leaves in the + /// tree from left to right if one were to draw the tree on paper. + std::vector LeafNodes; + + /// Perform a post-order depth-first traversal of the tree and perform two + /// tasks during the traversal. The first is to populate LeafNodes, adding + /// nodes in order of the traversal. The second is to keep track of the leaf + /// descendants of every internal node by assigning values to LeftLeafIndex + /// and RightLefIndex fields of SuffixTreeNode for all internal nodes. + void setLeafNodes(); + public: /// Construct a suffix tree from a sequence of unsigned integers. /// /// \param Str The string to construct the suffix tree for. - SuffixTree(const ArrayRef &Str); + /// \param OutlinerLeafDescendants Whether to consider leaf descendants or + /// only leaf children (used by Machine Outliner). + SuffixTree(const ArrayRef &Str, + bool OutlinerLeafDescendants = false); /// Iterator for finding all repeated substrings in the suffix tree. struct RepeatedSubstringIterator { @@ -154,6 +173,12 @@ class SuffixTree { /// instruction lengths. const unsigned MinLength = 2; + /// Vector of leaf nodes of the suffix tree. + const std::vector &LeafNodes; + + /// Whether to consider leaf descendants or only leaf children. + bool OutlinerLeafDescendants = !LeafNodes.empty(); + /// Move the iterator to the next repeated substring. void advance(); @@ -179,7 +204,10 @@ class SuffixTree { return !(*this == Other); } - RepeatedSubstringIterator(SuffixTreeInternalNode *N) : N(N) { + RepeatedSubstringIterator( + SuffixTreeInternalNode *N, + const std::vector &LeafNodes = {}) + : N(N), LeafNodes(LeafNodes) { // Do we have a non-null node? if (!N) return; @@ -191,7 +219,7 @@ class SuffixTree { }; typedef RepeatedSubstringIterator iterator; - iterator begin() { return iterator(Root); } + iterator begin() { return iterator(Root, LeafNodes); } iterator end() { return iterator(nullptr); } }; diff --git a/llvm/include/llvm/Support/SuffixTreeNode.h b/llvm/include/llvm/Support/SuffixTreeNode.h index 7d0d1cf0c58b95..84b590f2deb0cd 100644 --- a/llvm/include/llvm/Support/SuffixTreeNode.h +++ b/llvm/include/llvm/Support/SuffixTreeNode.h @@ -46,6 +46,17 @@ struct SuffixTreeNode { /// the root to this node. unsigned ConcatLen = 0; + /// These two indices give a range of indices for its leaf descendants. + /// Imagine drawing a tree on paper and assigning a unique index to each leaf + /// node in monotonically increasing order from left to right. This way of + /// numbering the leaf nodes allows us to associate a continuous range of + /// indices with each internal node. For example, if a node has leaf + /// descendants with indices i, i+1, ..., j, then its LeftLeafIdx is i and + /// its RightLeafIdx is j. These indices are for LeafNodes in the SuffixTree + /// class, which is constructed using post-order depth-first traversal. + unsigned LeftLeafIdx = EmptyIdx; + unsigned RightLeafIdx = EmptyIdx; + public: // LLVM RTTI boilerplate. NodeKind getKind() const { return Kind; } @@ -56,6 +67,18 @@ struct SuffixTreeNode { /// \returns the end index of this node. virtual unsigned getEndIdx() const = 0; + /// \return the index of this node's left most leaf node. + unsigned getLeftLeafIdx() const; + + /// \return the index of this node's right most leaf node. + unsigned getRightLeafIdx() const; + + /// Set the index of the left most leaf node of this node to \p Idx. + void setLeftLeafIdx(unsigned Idx); + + /// Set the index of the right most leaf node of this node to \p Idx. + void setRightLeafIdx(unsigned Idx); + /// Advance this node's StartIdx by \p Inc. void incrementStartIdx(unsigned Inc); @@ -168,4 +191,4 @@ struct SuffixTreeLeafNode : SuffixTreeNode { virtual ~SuffixTreeLeafNode() = default; }; } // namespace llvm -#endif // LLVM_SUPPORT_SUFFIXTREE_NODE_H \ No newline at end of file +#endif // LLVM_SUPPORT_SUFFIXTREE_NODE_H diff --git a/llvm/lib/CMakeLists.txt b/llvm/lib/CMakeLists.txt index 74e2d03c07953d..2ac0b0dc026e16 100644 --- a/llvm/lib/CMakeLists.txt +++ b/llvm/lib/CMakeLists.txt @@ -10,6 +10,7 @@ add_subdirectory(InterfaceStub) add_subdirectory(IRPrinter) add_subdirectory(IRReader) add_subdirectory(CodeGen) +add_subdirectory(CodeGenData) add_subdirectory(CodeGenTypes) add_subdirectory(BinaryFormat) add_subdirectory(Bitcode) diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp index dc2f5ef15206e8..a581997ec6c4cb 100644 --- a/llvm/lib/CodeGen/MachineOutliner.cpp +++ b/llvm/lib/CodeGen/MachineOutliner.cpp @@ -121,6 +121,12 @@ static cl::opt OutlinerBenefitThreshold( cl::desc( "The minimum size in bytes before an outlining candidate is accepted")); +static cl::opt OutlinerLeafDescendants( + "outliner-leaf-descendants", cl::init(true), cl::Hidden, + cl::desc("Consider all leaf descendants of internal nodes of the suffix " + "tree as candidates for outlining (if false, only leaf children " + "are considered)")); + namespace { /// Maps \p MachineInstrs to unsigned integers and stores the mappings. @@ -446,8 +452,9 @@ struct MachineOutliner : public ModulePass { /// \param Mapper Contains outlining mapping information. /// \param[out] FunctionList Filled with a list of \p OutlinedFunctions /// each type of candidate. - void findCandidates(InstructionMapper &Mapper, - std::vector &FunctionList); + void + findCandidates(InstructionMapper &Mapper, + std::vector> &FunctionList); /// Replace the sequences of instructions represented by \p OutlinedFunctions /// with calls to functions. @@ -455,7 +462,9 @@ struct MachineOutliner : public ModulePass { /// \param M The module we are outlining from. /// \param FunctionList A list of functions to be inserted into the module. /// \param Mapper Contains the instruction mappings for the module. - bool outline(Module &M, std::vector &FunctionList, + /// \param[out] OutlinedFunctionNum The outlined function number. + bool outline(Module &M, + std::vector> &FunctionList, InstructionMapper &Mapper, unsigned &OutlinedFunctionNum); /// Creates a function for \p OF and inserts it into the module. @@ -574,9 +583,10 @@ void MachineOutliner::emitOutlinedFunctionRemark(OutlinedFunction &OF) { } void MachineOutliner::findCandidates( - InstructionMapper &Mapper, std::vector &FunctionList) { + InstructionMapper &Mapper, + std::vector> &FunctionList) { FunctionList.clear(); - SuffixTree ST(Mapper.UnsignedVec); + SuffixTree ST(Mapper.UnsignedVec, OutlinerLeafDescendants); // First, find all of the repeated substrings in the tree of minimum length // 2. @@ -593,7 +603,11 @@ void MachineOutliner::findCandidates( unsigned NumDiscarded = 0; unsigned NumKept = 0; #endif - for (const unsigned &StartIdx : RS.StartIndices) { + // Sort the start indices so that we can efficiently check if candidates + // overlap with each other in MachineOutliner::findCandidates(). + SmallVector SortedStartIndices(RS.StartIndices); + llvm::sort(SortedStartIndices); + for (const unsigned &StartIdx : SortedStartIndices) { // Trick: Discard some candidates that would be incompatible with the // ones we've already found for this sequence. This will save us some // work in candidate selection. @@ -616,17 +630,15 @@ void MachineOutliner::findCandidates( // * End before the other starts // * Start after the other ends unsigned EndIdx = StartIdx + StringLen - 1; - auto FirstOverlap = find_if( - CandidatesForRepeatedSeq, [StartIdx, EndIdx](const Candidate &C) { - return EndIdx >= C.getStartIdx() && StartIdx <= C.getEndIdx(); - }); - if (FirstOverlap != CandidatesForRepeatedSeq.end()) { + if (CandidatesForRepeatedSeq.size() > 0 && + StartIdx <= CandidatesForRepeatedSeq.back().getEndIdx()) { #ifndef NDEBUG ++NumDiscarded; - LLVM_DEBUG(dbgs() << " .. DISCARD candidate @ [" << StartIdx - << ", " << EndIdx << "]; overlaps with candidate @ [" - << FirstOverlap->getStartIdx() << ", " - << FirstOverlap->getEndIdx() << "]\n"); + LLVM_DEBUG(dbgs() << " .. DISCARD candidate @ [" << StartIdx << ", " + << EndIdx << "]; overlaps with candidate @ [" + << CandidatesForRepeatedSeq.back().getStartIdx() + << ", " << CandidatesForRepeatedSeq.back().getEndIdx() + << "]\n"); #endif continue; } @@ -674,7 +686,7 @@ void MachineOutliner::findCandidates( continue; } - FunctionList.push_back(*OF); + FunctionList.push_back(std::make_unique(*OF)); } } @@ -819,32 +831,33 @@ MachineFunction *MachineOutliner::createOutlinedFunction( return &MF; } -bool MachineOutliner::outline(Module &M, - std::vector &FunctionList, - InstructionMapper &Mapper, - unsigned &OutlinedFunctionNum) { +bool MachineOutliner::outline( + Module &M, std::vector> &FunctionList, + InstructionMapper &Mapper, unsigned &OutlinedFunctionNum) { LLVM_DEBUG(dbgs() << "*** Outlining ***\n"); LLVM_DEBUG(dbgs() << "NUMBER OF POTENTIAL FUNCTIONS: " << FunctionList.size() << "\n"); bool OutlinedSomething = false; - // Sort by benefit. The most beneficial functions should be outlined first. - stable_sort(FunctionList, - [](const OutlinedFunction &LHS, const OutlinedFunction &RHS) { - return LHS.getBenefit() > RHS.getBenefit(); - }); + // Sort by priority where priority := getNotOutlinedCost / getOutliningCost. + // The function with highest priority should be outlined first. + stable_sort(FunctionList, [](const std::unique_ptr &LHS, + const std::unique_ptr &RHS) { + return LHS->getNotOutlinedCost() * RHS->getOutliningCost() > + RHS->getNotOutlinedCost() * LHS->getOutliningCost(); + }); // Walk over each function, outlining them as we go along. Functions are // outlined greedily, based off the sort above. auto *UnsignedVecBegin = Mapper.UnsignedVec.begin(); LLVM_DEBUG(dbgs() << "WALKING FUNCTION LIST\n"); - for (OutlinedFunction &OF : FunctionList) { + for (auto &OF : FunctionList) { #ifndef NDEBUG - auto NumCandidatesBefore = OF.Candidates.size(); + auto NumCandidatesBefore = OF->Candidates.size(); #endif // If we outlined something that overlapped with a candidate in a previous // step, then we can't outline from it. - erase_if(OF.Candidates, [&UnsignedVecBegin](Candidate &C) { + erase_if(OF->Candidates, [&UnsignedVecBegin](Candidate &C) { return std::any_of(UnsignedVecBegin + C.getStartIdx(), UnsignedVecBegin + C.getEndIdx() + 1, [](unsigned I) { return I == static_cast(-1); @@ -852,36 +865,36 @@ bool MachineOutliner::outline(Module &M, }); #ifndef NDEBUG - auto NumCandidatesAfter = OF.Candidates.size(); + auto NumCandidatesAfter = OF->Candidates.size(); LLVM_DEBUG(dbgs() << "PRUNED: " << NumCandidatesBefore - NumCandidatesAfter << "/" << NumCandidatesBefore << " candidates\n"); #endif // If we made it unbeneficial to outline this function, skip it. - if (OF.getBenefit() < OutlinerBenefitThreshold) { - LLVM_DEBUG(dbgs() << "SKIP: Expected benefit (" << OF.getBenefit() + if (OF->getBenefit() < OutlinerBenefitThreshold) { + LLVM_DEBUG(dbgs() << "SKIP: Expected benefit (" << OF->getBenefit() << " B) < threshold (" << OutlinerBenefitThreshold << " B)\n"); continue; } - LLVM_DEBUG(dbgs() << "OUTLINE: Expected benefit (" << OF.getBenefit() + LLVM_DEBUG(dbgs() << "OUTLINE: Expected benefit (" << OF->getBenefit() << " B) > threshold (" << OutlinerBenefitThreshold << " B)\n"); // It's beneficial. Create the function and outline its sequence's // occurrences. - OF.MF = createOutlinedFunction(M, OF, Mapper, OutlinedFunctionNum); - emitOutlinedFunctionRemark(OF); + OF->MF = createOutlinedFunction(M, *OF, Mapper, OutlinedFunctionNum); + emitOutlinedFunctionRemark(*OF); FunctionsCreated++; OutlinedFunctionNum++; // Created a function, move to the next name. - MachineFunction *MF = OF.MF; + MachineFunction *MF = OF->MF; const TargetSubtargetInfo &STI = MF->getSubtarget(); const TargetInstrInfo &TII = *STI.getInstrInfo(); // Replace occurrences of the sequence with calls to the new function. LLVM_DEBUG(dbgs() << "CREATE OUTLINED CALLS\n"); - for (Candidate &C : OF.Candidates) { + for (Candidate &C : OF->Candidates) { MachineBasicBlock &MBB = *C.getMBB(); MachineBasicBlock::iterator StartIt = C.begin(); MachineBasicBlock::iterator EndIt = std::prev(C.end()); @@ -1173,7 +1186,7 @@ bool MachineOutliner::doOutline(Module &M, unsigned &OutlinedFunctionNum) { // Prepare instruction mappings for the suffix tree. populateMapper(Mapper, M, MMI); - std::vector FunctionList; + std::vector> FunctionList; // Find all of the outlining candidates. findCandidates(Mapper, FunctionList); diff --git a/llvm/lib/CodeGenData/CMakeLists.txt b/llvm/lib/CodeGenData/CMakeLists.txt new file mode 100644 index 00000000000000..1156d53afb2e0f --- /dev/null +++ b/llvm/lib/CodeGenData/CMakeLists.txt @@ -0,0 +1,17 @@ +add_llvm_component_library(LLVMCodeGenData + CodeGenData.cpp + CodeGenDataReader.cpp + CodeGenDataWriter.cpp + OutlinedHashTree.cpp + OutlinedHashTreeRecord.cpp + + ADDITIONAL_HEADER_DIRS + ${LLVM_MAIN_INCLUDE_DIR}/llvm/CodeGenData + + DEPENDS + intrinsics_gen + + LINK_COMPONENTS + Core + Support + ) diff --git a/llvm/lib/CodeGenData/CodeGenData.cpp b/llvm/lib/CodeGenData/CodeGenData.cpp new file mode 100644 index 00000000000000..3bd21c97c7de7a --- /dev/null +++ b/llvm/lib/CodeGenData/CodeGenData.cpp @@ -0,0 +1,197 @@ +//===-- CodeGenData.cpp ---------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains support for codegen data that has stable summary which +// can be used to optimize the code in the subsequent codegen. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/CodeGenData/CodeGenDataReader.h" +#include "llvm/CodeGenData/OutlinedHashTreeRecord.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/WithColor.h" + +#define DEBUG_TYPE "cg-data" + +using namespace llvm; +using namespace cgdata; + +static std::string getCGDataErrString(cgdata_error Err, + const std::string &ErrMsg = "") { + std::string Msg; + raw_string_ostream OS(Msg); + + switch (Err) { + case cgdata_error::success: + OS << "success"; + break; + case cgdata_error::eof: + OS << "end of File"; + break; + case cgdata_error::bad_magic: + OS << "invalid codegen data (bad magic)"; + break; + case cgdata_error::bad_header: + OS << "invalid codegen data (file header is corrupt)"; + break; + case cgdata_error::empty_cgdata: + OS << "empty codegen data"; + break; + case cgdata_error::malformed: + OS << "malformed codegen data"; + break; + case cgdata_error::unsupported_version: + OS << "unsupported codegen data version"; + break; + } + + // If optional error message is not empty, append it to the message. + if (!ErrMsg.empty()) + OS << ": " << ErrMsg; + + return OS.str(); +} + +namespace { + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. +class CGDataErrorCategoryType : public std::error_category { + const char *name() const noexcept override { return "llvm.cgdata"; } + + std::string message(int IE) const override { + return getCGDataErrString(static_cast(IE)); + } +}; + +} // end anonymous namespace + +const std::error_category &llvm::cgdata_category() { + static CGDataErrorCategoryType ErrorCategory; + return ErrorCategory; +} + +std::string CGDataError::message() const { + return getCGDataErrString(Err, Msg); +} + +char CGDataError::ID = 0; + +namespace { + +const char *CodeGenDataSectNameCommon[] = { +#define CG_DATA_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ + SectNameCommon, +#include "llvm/CodeGenData/CodeGenData.inc" +}; + +const char *CodeGenDataSectNameCoff[] = { +#define CG_DATA_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ + SectNameCoff, +#include "llvm/CodeGenData/CodeGenData.inc" +}; + +const char *CodeGenDataSectNamePrefix[] = { +#define CG_DATA_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Prefix, +#include "llvm/CodeGenData/CodeGenData.inc" +}; + +} // namespace + +namespace llvm { + +std::string getCodeGenDataSectionName(CGDataSectKind CGSK, + Triple::ObjectFormatType OF, + bool AddSegmentInfo) { + std::string SectName; + + if (OF == Triple::MachO && AddSegmentInfo) + SectName = CodeGenDataSectNamePrefix[CGSK]; + + if (OF == Triple::COFF) + SectName += CodeGenDataSectNameCoff[CGSK]; + else + SectName += CodeGenDataSectNameCommon[CGSK]; + + return SectName; +} + +std::unique_ptr CodeGenData::Instance = nullptr; +std::once_flag CodeGenData::OnceFlag; + +CodeGenData &CodeGenData::getInstance() { + std::call_once(CodeGenData::OnceFlag, []() { + auto *CGD = new CodeGenData(); + Instance.reset(CGD); + + // TODO: Initialize writer or reader mode for the client optimization. + }); + return *(Instance.get()); +} + +namespace IndexedCGData { + +Expected
Header::readFromBuffer(const unsigned char *Curr) { + using namespace support; + + static_assert(std::is_standard_layout_v, + "The header should be standard layout type since we use offset " + "of fields to read."); + Header H; + H.Magic = endian::readNext(Curr); + if (H.Magic != IndexedCGData::Magic) + return make_error(cgdata_error::bad_magic); + H.Version = endian::readNext(Curr); + if (H.Version > IndexedCGData::CGDataVersion::CurrentVersion) + return make_error(cgdata_error::unsupported_version); + H.DataKind = endian::readNext(Curr); + + switch (H.Version) { + // When a new field is added to the header add a case statement here to + // compute the size as offset of the new field + size of the new field. This + // relies on the field being added to the end of the list. + static_assert(IndexedCGData::CGDataVersion::CurrentVersion == Version1, + "Please update the size computation below if a new field has " + "been added to the header, if not add a case statement to " + "fall through to the latest version."); + case 1ull: + H.OutlinedHashTreeOffset = + endian::readNext(Curr); + } + + return H; +} + +} // end namespace IndexedCGData + +namespace cgdata { + +void warn(Twine Message, std::string Whence, std::string Hint) { + WithColor::warning(); + if (!Whence.empty()) + errs() << Whence << ": "; + errs() << Message << "\n"; + if (!Hint.empty()) + WithColor::note() << Hint << "\n"; +} + +void warn(Error E, StringRef Whence) { + if (E.isA()) { + handleAllErrors(std::move(E), [&](const CGDataError &IPE) { + warn(IPE.message(), std::string(Whence), std::string("")); + }); + } +} + +} // end namespace cgdata + +} // end namespace llvm diff --git a/llvm/lib/CodeGenData/CodeGenDataReader.cpp b/llvm/lib/CodeGenData/CodeGenDataReader.cpp new file mode 100644 index 00000000000000..1b08085dec2f25 --- /dev/null +++ b/llvm/lib/CodeGenData/CodeGenDataReader.cpp @@ -0,0 +1,174 @@ +//===- CodeGenDataReader.cpp ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains support for reading codegen data. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGenData/CodeGenDataReader.h" +#include "llvm/CodeGenData/OutlinedHashTreeRecord.h" +#include "llvm/Support/MemoryBuffer.h" + +#define DEBUG_TYPE "cg-data-reader" + +using namespace llvm; + +namespace llvm { + +static Expected> +setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) { + auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN() + : FS.getBufferForFile(Filename); + if (std::error_code EC = BufferOrErr.getError()) + return errorCodeToError(EC); + return std::move(BufferOrErr.get()); +} + +Error CodeGenDataReader::mergeFromObjectFile( + const object::ObjectFile *Obj, + OutlinedHashTreeRecord &GlobalOutlineRecord) { + Triple TT = Obj->makeTriple(); + auto CGOutLineName = + getCodeGenDataSectionName(CG_outline, TT.getObjectFormat(), false); + + for (auto &Section : Obj->sections()) { + Expected NameOrErr = Section.getName(); + if (!NameOrErr) + return NameOrErr.takeError(); + Expected ContentsOrErr = Section.getContents(); + if (!ContentsOrErr) + return ContentsOrErr.takeError(); + auto *Data = reinterpret_cast(ContentsOrErr->data()); + auto *EndData = Data + ContentsOrErr->size(); + + if (*NameOrErr == CGOutLineName) { + // In case dealing with an executable that has concatenaed cgdata, + // we want to merge them into a single cgdata. + // Although it's not a typical workflow, we support this scenario. + while (Data != EndData) { + OutlinedHashTreeRecord LocalOutlineRecord; + LocalOutlineRecord.deserialize(Data); + GlobalOutlineRecord.merge(LocalOutlineRecord); + } + } + // TODO: Add support for other cgdata sections. + } + + return Error::success(); +} + +Error IndexedCodeGenDataReader::read() { + using namespace support; + + // The smallest header with the version 1 is 24 bytes + const unsigned MinHeaderSize = 24; + if (DataBuffer->getBufferSize() < MinHeaderSize) + return error(cgdata_error::bad_header); + + auto *Start = + reinterpret_cast(DataBuffer->getBufferStart()); + auto *End = + reinterpret_cast(DataBuffer->getBufferEnd()); + auto HeaderOr = IndexedCGData::Header::readFromBuffer(Start); + if (!HeaderOr) + return HeaderOr.takeError(); + Header = HeaderOr.get(); + + if (hasOutlinedHashTree()) { + const unsigned char *Ptr = Start + Header.OutlinedHashTreeOffset; + if (Ptr >= End) + return error(cgdata_error::eof); + HashTreeRecord.deserialize(Ptr); + } + + return success(); +} + +Expected> +CodeGenDataReader::create(const Twine &Path, vfs::FileSystem &FS) { + // Set up the buffer to read. + auto BufferOrError = setupMemoryBuffer(Path, FS); + if (Error E = BufferOrError.takeError()) + return std::move(E); + return CodeGenDataReader::create(std::move(BufferOrError.get())); +} + +Expected> +CodeGenDataReader::create(std::unique_ptr Buffer) { + if (Buffer->getBufferSize() == 0) + return make_error(cgdata_error::empty_cgdata); + + std::unique_ptr Reader; + // Create the reader. + if (IndexedCodeGenDataReader::hasFormat(*Buffer)) + Reader.reset(new IndexedCodeGenDataReader(std::move(Buffer))); + else if (TextCodeGenDataReader::hasFormat(*Buffer)) + Reader.reset(new TextCodeGenDataReader(std::move(Buffer))); + else + return make_error(cgdata_error::malformed); + + // Initialize the reader and return the result. + if (Error E = Reader->read()) + return std::move(E); + + return std::move(Reader); +} + +bool IndexedCodeGenDataReader::hasFormat(const MemoryBuffer &DataBuffer) { + using namespace support; + if (DataBuffer.getBufferSize() < 8) + return false; + + uint64_t Magic = endian::read( + DataBuffer.getBufferStart()); + // Verify that it's magical. + return Magic == IndexedCGData::Magic; +} + +bool TextCodeGenDataReader::hasFormat(const MemoryBuffer &Buffer) { + // Verify that this really looks like plain ASCII text by checking a + // 'reasonable' number of characters (up to profile magic size). + size_t count = std::min(Buffer.getBufferSize(), sizeof(uint64_t)); + StringRef buffer = Buffer.getBufferStart(); + return count == 0 || + std::all_of(buffer.begin(), buffer.begin() + count, + [](char c) { return isPrint(c) || isSpace(c); }); +} +Error TextCodeGenDataReader::read() { + using namespace support; + + // Parse the custom header line by line. + while (Line->starts_with(":")) { + StringRef Str = Line->substr(1); + if (Str.equals_insensitive("outlined_hash_tree")) + DataKind |= CGDataKind::FunctionOutlinedHashTree; + else + return error(cgdata_error::bad_header); + ++Line; + } + + // We treat an empty header (that as a comment # only) as a valid header. + if (Line.is_at_eof()) { + if (DataKind != CGDataKind::Unknown) + return error(cgdata_error::bad_header); + return Error::success(); + } + + // The YAML docs follow after the header. + const char *Pos = (*Line).data(); + size_t Size = reinterpret_cast(DataBuffer->getBufferEnd()) - + reinterpret_cast(Pos); + yaml::Input YOS(StringRef(Pos, Size)); + if (hasOutlinedHashTree()) + HashTreeRecord.deserializeYAML(YOS); + + // TODO: Add more yaml cgdata in order + + return Error::success(); +} +} // end namespace llvm diff --git a/llvm/lib/CodeGenData/CodeGenDataWriter.cpp b/llvm/lib/CodeGenData/CodeGenDataWriter.cpp new file mode 100644 index 00000000000000..9aa0d86223f714 --- /dev/null +++ b/llvm/lib/CodeGenData/CodeGenDataWriter.cpp @@ -0,0 +1,162 @@ +//===- CodeGenDataWriter.cpp ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains support for writing codegen data. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGenData/CodeGenDataWriter.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/EndianStream.h" + +#define DEBUG_TYPE "cg-data-writer" + +using namespace llvm; + +namespace llvm { + +/// A struct to define how the data stream should be patched. +struct CGDataPatchItem { + uint64_t Pos; // Where to patch. + uint64_t *D; // Pointer to an array of source data. + int N; // Number of elements in \c D array. +}; + +// A wrapper class to abstract writer stream with support of bytes +// back patching. +class CGDataOStream { +public: + CGDataOStream(raw_fd_ostream &FD) + : IsFDOStream(true), OS(FD), LE(FD, llvm::endianness::little) {} + CGDataOStream(raw_string_ostream &STR) + : IsFDOStream(false), OS(STR), LE(STR, llvm::endianness::little) {} + + uint64_t tell() { return OS.tell(); } + void write(uint64_t V) { LE.write(V); } + void write32(uint32_t V) { LE.write(V); } + void write8(uint8_t V) { LE.write(V); } + + // \c patch can only be called when all data is written and flushed. + // For raw_string_ostream, the patch is done on the target string + // directly and it won't be reflected in the stream's internal buffer. + void patch(ArrayRef P) { + using namespace support; + + if (IsFDOStream) { + raw_fd_ostream &FDOStream = static_cast(OS); + const uint64_t LastPos = FDOStream.tell(); + for (const auto &K : P) { + FDOStream.seek(K.Pos); + for (int I = 0; I < K.N; I++) + write(K.D[I]); + } + // Reset the stream to the last position after patching so that users + // don't accidentally overwrite data. This makes it consistent with + // the string stream below which replaces the data directly. + FDOStream.seek(LastPos); + } else { + raw_string_ostream &SOStream = static_cast(OS); + std::string &Data = SOStream.str(); // with flush + for (const auto &K : P) { + for (int I = 0; I < K.N; I++) { + uint64_t Bytes = + endian::byte_swap(K.D[I]); + Data.replace(K.Pos + I * sizeof(uint64_t), sizeof(uint64_t), + (const char *)&Bytes, sizeof(uint64_t)); + } + } + } + } + + // If \c OS is an instance of \c raw_fd_ostream, this field will be + // true. Otherwise, \c OS will be an raw_string_ostream. + bool IsFDOStream; + raw_ostream &OS; + support::endian::Writer LE; +}; + +} // end namespace llvm + +void CodeGenDataWriter::addRecord(OutlinedHashTreeRecord &Record) { + assert(Record.HashTree && "empty hash tree in the record"); + HashTreeRecord.HashTree = std::move(Record.HashTree); + + DataKind |= CGDataKind::FunctionOutlinedHashTree; +} + +Error CodeGenDataWriter::write(raw_fd_ostream &OS) { + CGDataOStream COS(OS); + return writeImpl(COS); +} + +Error CodeGenDataWriter::writeHeader(CGDataOStream &COS) { + using namespace support; + IndexedCGData::Header Header; + Header.Magic = IndexedCGData::Magic; + Header.Version = IndexedCGData::Version; + + // Set the CGDataKind depending on the kind. + Header.DataKind = 0; + if (static_cast(DataKind & CGDataKind::FunctionOutlinedHashTree)) + Header.DataKind |= + static_cast(CGDataKind::FunctionOutlinedHashTree); + + Header.OutlinedHashTreeOffset = 0; + + // Only write out up to the CGDataKind. We need to remember the offest of the + // remaing fields to allow back patching later. + COS.write(Header.Magic); + COS.write32(Header.Version); + COS.write32(Header.DataKind); + + // Save the location of Header.OutlinedHashTreeOffset field in \c COS. + OutlinedHashTreeOffset = COS.tell(); + + // Reserve the space for OutlinedHashTreeOffset field. + COS.write(0); + + return Error::success(); +} + +Error CodeGenDataWriter::writeImpl(CGDataOStream &COS) { + if (Error E = writeHeader(COS)) + return E; + + uint64_t OutlinedHashTreeFieldStart = COS.tell(); + if (hasOutlinedHashTree()) + HashTreeRecord.serialize(COS.OS); + + // Back patch the offsets. + CGDataPatchItem PatchItems[] = { + {OutlinedHashTreeOffset, &OutlinedHashTreeFieldStart, 1}}; + COS.patch(PatchItems); + + return Error::success(); +} + +Error CodeGenDataWriter::writeHeaderText(raw_fd_ostream &OS) { + if (hasOutlinedHashTree()) + OS << "# Outlined stable hash tree\n:outlined_hash_tree\n"; + + // TODO: Add more data types in this header + + return Error::success(); +} + +Error CodeGenDataWriter::writeText(raw_fd_ostream &OS) { + if (Error E = writeHeaderText(OS)) + return E; + + yaml::Output YOS(OS); + if (hasOutlinedHashTree()) + HashTreeRecord.serializeYAML(YOS); + + // TODO: Write more yaml cgdata in order + + return Error::success(); +} diff --git a/llvm/lib/CodeGenData/OutlinedHashTree.cpp b/llvm/lib/CodeGenData/OutlinedHashTree.cpp new file mode 100644 index 00000000000000..032993ded60ead --- /dev/null +++ b/llvm/lib/CodeGenData/OutlinedHashTree.cpp @@ -0,0 +1,131 @@ +//===-- OutlinedHashTree.cpp ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// An OutlinedHashTree is a Trie that contains sequences of stable hash values +// of instructions that have been outlined. This OutlinedHashTree can be used +// to understand the outlined instruction sequences collected across modules. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGenData/OutlinedHashTree.h" + +#include +#include + +#define DEBUG_TYPE "outlined-hash-tree" + +using namespace llvm; + +void OutlinedHashTree::walkGraph(NodeCallbackFn CallbackNode, + EdgeCallbackFn CallbackEdge, + bool SortedWalk) const { + std::stack Stack; + Stack.push(getRoot()); + + while (!Stack.empty()) { + const auto *Current = Stack.top(); + Stack.pop(); + if (CallbackNode) + CallbackNode(Current); + + auto HandleNext = [&](const HashNode *Next) { + if (CallbackEdge) + CallbackEdge(Current, Next); + Stack.push(Next); + }; + if (SortedWalk) { + std::map SortedSuccessors; + for (const auto &P : Current->Successors) + SortedSuccessors[P.first] = P.second.get(); + for (const auto &P : SortedSuccessors) + HandleNext(P.second); + } else { + for (const auto &P : Current->Successors) + HandleNext(P.second.get()); + } + } +} + +size_t OutlinedHashTree::size(bool GetTerminalCountOnly) const { + size_t Size = 0; + walkGraph([&Size, GetTerminalCountOnly](const HashNode *N) { + Size += (N && (!GetTerminalCountOnly || N->Terminals)); + }); + return Size; +} + +size_t OutlinedHashTree::depth() const { + size_t Size = 0; + std::unordered_map DepthMap; + walkGraph([&Size, &DepthMap]( + const HashNode *N) { Size = std::max(Size, DepthMap[N]); }, + [&DepthMap](const HashNode *Src, const HashNode *Dst) { + size_t Depth = DepthMap[Src]; + DepthMap[Dst] = Depth + 1; + }); + return Size; +} + +void OutlinedHashTree::insert(const HashSequencePair &SequencePair) { + const auto &Sequence = SequencePair.first; + unsigned Count = SequencePair.second; + HashNode *Current = getRoot(); + + for (stable_hash StableHash : Sequence) { + auto I = Current->Successors.find(StableHash); + if (I == Current->Successors.end()) { + std::unique_ptr Next = std::make_unique(); + HashNode *NextPtr = Next.get(); + NextPtr->Hash = StableHash; + Current->Successors.emplace(StableHash, std::move(Next)); + Current = NextPtr; + } else + Current = I->second.get(); + } + Current->Terminals += Count; +} + +void OutlinedHashTree::merge(const OutlinedHashTree *Tree) { + HashNode *Dst = getRoot(); + const HashNode *Src = Tree->getRoot(); + std::stack> Stack; + Stack.push({Dst, Src}); + + while (!Stack.empty()) { + auto [DstNode, SrcNode] = Stack.top(); + Stack.pop(); + if (!SrcNode) + continue; + DstNode->Terminals += SrcNode->Terminals; + + for (auto &[Hash, NextSrcNode] : SrcNode->Successors) { + HashNode *NextDstNode; + auto I = DstNode->Successors.find(Hash); + if (I == DstNode->Successors.end()) { + auto NextDst = std::make_unique(); + NextDstNode = NextDst.get(); + NextDstNode->Hash = Hash; + DstNode->Successors.emplace(Hash, std::move(NextDst)); + } else + NextDstNode = I->second.get(); + + Stack.push({NextDstNode, NextSrcNode.get()}); + } + } +} + +unsigned OutlinedHashTree::find(const HashSequence &Sequence) const { + const HashNode *Current = getRoot(); + for (stable_hash StableHash : Sequence) { + const auto I = Current->Successors.find(StableHash); + if (I == Current->Successors.end()) + return 0; + Current = I->second.get(); + } + return Current->Terminals; +} diff --git a/llvm/lib/CodeGenData/OutlinedHashTreeRecord.cpp b/llvm/lib/CodeGenData/OutlinedHashTreeRecord.cpp new file mode 100644 index 00000000000000..0d5dd864c89c55 --- /dev/null +++ b/llvm/lib/CodeGenData/OutlinedHashTreeRecord.cpp @@ -0,0 +1,168 @@ +//===-- OutlinedHashTreeRecord.cpp ----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This defines the OutlinedHashTreeRecord class. This class holds the outlined +// hash tree for both serialization and deserialization processes. It utilizes +// two data formats for serialization: raw binary data and YAML. +// These two formats can be used interchangeably. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGenData/OutlinedHashTreeRecord.h" +#include "llvm/CodeGenData/OutlinedHashTree.h" +#include "llvm/ObjectYAML/YAML.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/EndianStream.h" + +#define DEBUG_TYPE "outlined-hash-tree" + +using namespace llvm; +using namespace llvm::support; + +namespace llvm { +namespace yaml { + +template <> struct MappingTraits { + static void mapping(IO &io, HashNodeStable &res) { + io.mapRequired("Hash", res.Hash); + io.mapRequired("Terminals", res.Terminals); + io.mapRequired("SuccessorIds", res.SuccessorIds); + } +}; + +template <> struct CustomMappingTraits { + static void inputOne(IO &io, StringRef Key, IdHashNodeStableMapTy &V) { + HashNodeStable NodeStable; + io.mapRequired(Key.str().c_str(), NodeStable); + unsigned Id; + if (Key.getAsInteger(0, Id)) { + io.setError("Id not an integer"); + return; + } + V.insert({Id, NodeStable}); + } + + static void output(IO &io, IdHashNodeStableMapTy &V) { + for (auto Iter = V.begin(); Iter != V.end(); ++Iter) + io.mapRequired(utostr(Iter->first).c_str(), Iter->second); + } +}; + +} // namespace yaml +} // namespace llvm + +void OutlinedHashTreeRecord::serialize(raw_ostream &OS) const { + IdHashNodeStableMapTy IdNodeStableMap; + convertToStableData(IdNodeStableMap); + support::endian::Writer Writer(OS, endianness::little); + Writer.write(IdNodeStableMap.size()); + + for (const auto &[Id, NodeStable] : IdNodeStableMap) { + Writer.write(Id); + Writer.write(NodeStable.Hash); + Writer.write(NodeStable.Terminals); + Writer.write(NodeStable.SuccessorIds.size()); + for (auto SuccessorId : NodeStable.SuccessorIds) + Writer.write(SuccessorId); + } +} + +void OutlinedHashTreeRecord::deserialize(const unsigned char *&Ptr) { + IdHashNodeStableMapTy IdNodeStableMap; + auto NumIdNodeStableMap = + endian::readNext(Ptr); + + for (unsigned I = 0; I < NumIdNodeStableMap; ++I) { + auto Id = endian::readNext(Ptr); + HashNodeStable NodeStable; + NodeStable.Hash = + endian::readNext(Ptr); + NodeStable.Terminals = + endian::readNext(Ptr); + auto NumSuccessorIds = + endian::readNext(Ptr); + for (unsigned J = 0; J < NumSuccessorIds; ++J) + NodeStable.SuccessorIds.push_back( + endian::readNext(Ptr)); + + IdNodeStableMap[Id] = std::move(NodeStable); + } + + convertFromStableData(IdNodeStableMap); +} + +void OutlinedHashTreeRecord::serializeYAML(yaml::Output &YOS) const { + IdHashNodeStableMapTy IdNodeStableMap; + convertToStableData(IdNodeStableMap); + + YOS << IdNodeStableMap; +} + +void OutlinedHashTreeRecord::deserializeYAML(yaml::Input &YIS) { + IdHashNodeStableMapTy IdNodeStableMap; + + YIS >> IdNodeStableMap; + YIS.nextDocument(); + + convertFromStableData(IdNodeStableMap); +} + +void OutlinedHashTreeRecord::convertToStableData( + IdHashNodeStableMapTy &IdNodeStableMap) const { + // Build NodeIdMap + HashNodeIdMapTy NodeIdMap; + HashTree->walkGraph( + [&NodeIdMap](const HashNode *Current) { + size_t Index = NodeIdMap.size(); + NodeIdMap[Current] = Index; + assert(Index = NodeIdMap.size() + 1 && + "Expected size of NodeMap to increment by 1"); + }, + /*EdgeCallbackFn=*/nullptr, /*SortedWork=*/true); + + // Convert NodeIdMap to NodeStableMap + for (auto &P : NodeIdMap) { + auto *Node = P.first; + auto Id = P.second; + HashNodeStable NodeStable; + NodeStable.Hash = Node->Hash; + NodeStable.Terminals = Node->Terminals; + for (auto &P : Node->Successors) + NodeStable.SuccessorIds.push_back(NodeIdMap[P.second.get()]); + IdNodeStableMap[Id] = NodeStable; + } + + // Sort the Successors so that they come out in the same order as in the map. + for (auto &P : IdNodeStableMap) + std::sort(P.second.SuccessorIds.begin(), P.second.SuccessorIds.end()); +} + +void OutlinedHashTreeRecord::convertFromStableData( + const IdHashNodeStableMapTy &IdNodeStableMap) { + IdHashNodeMapTy IdNodeMap; + // Initialize the root node at 0. + IdNodeMap[0] = HashTree->getRoot(); + assert(IdNodeMap[0]->Successors.empty()); + + for (auto &P : IdNodeStableMap) { + auto Id = P.first; + const HashNodeStable &NodeStable = P.second; + assert(IdNodeMap.count(Id)); + HashNode *Curr = IdNodeMap[Id]; + Curr->Hash = NodeStable.Hash; + Curr->Terminals = NodeStable.Terminals; + auto &Successors = Curr->Successors; + assert(Successors.empty()); + for (auto SuccessorId : NodeStable.SuccessorIds) { + auto Sucessor = std::make_unique(); + IdNodeMap[SuccessorId] = Sucessor.get(); + auto Hash = IdNodeStableMap.at(SuccessorId).Hash; + Successors[Hash] = std::move(Sucessor); + } + } +} diff --git a/llvm/lib/Support/SuffixTree.cpp b/llvm/lib/Support/SuffixTree.cpp index eaa653078e0900..4bd5cdfe8791dd 100644 --- a/llvm/lib/Support/SuffixTree.cpp +++ b/llvm/lib/Support/SuffixTree.cpp @@ -11,9 +11,11 @@ //===----------------------------------------------------------------------===// #include "llvm/Support/SuffixTree.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Casting.h" #include "llvm/Support/SuffixTreeNode.h" +#include using namespace llvm; @@ -26,7 +28,9 @@ static size_t numElementsInSubstring(const SuffixTreeNode *N) { return N->getEndIdx() - N->getStartIdx() + 1; } -SuffixTree::SuffixTree(const ArrayRef &Str) : Str(Str) { +SuffixTree::SuffixTree(const ArrayRef &Str, + bool OutlinerLeafDescendants) + : Str(Str), OutlinerLeafDescendants(OutlinerLeafDescendants) { Root = insertRoot(); Active.Node = Root; @@ -46,6 +50,11 @@ SuffixTree::SuffixTree(const ArrayRef &Str) : Str(Str) { // Set the suffix indices of each leaf. assert(Root && "Root node can't be nullptr!"); setSuffixIndices(); + + // Collect all leaf nodes of the suffix tree. And for each internal node, + // record the range of leaf nodes that are descendants of it. + if (OutlinerLeafDescendants) + setLeafNodes(); } SuffixTreeNode *SuffixTree::insertLeaf(SuffixTreeInternalNode &Parent, @@ -105,6 +114,68 @@ void SuffixTree::setSuffixIndices() { } } +void SuffixTree::setLeafNodes() { + // A stack that keeps track of nodes to visit for post-order DFS traversal. + std::stack ToVisit; + ToVisit.push(Root); + + // This keeps track of the index of the next leaf node to be added to + // the LeafNodes vector of the suffix tree. + unsigned LeafCounter = 0; + + // This keeps track of nodes whose children have been added to the stack + // during the post-order depth-first traversal of the tree. + llvm::SmallPtrSet ChildrenAddedToStack; + + // Traverse the tree in post-order. + while (!ToVisit.empty()) { + SuffixTreeNode *CurrNode = ToVisit.top(); + ToVisit.pop(); + if (auto *CurrInternalNode = dyn_cast(CurrNode)) { + // The current node is an internal node. + if (ChildrenAddedToStack.find(CurrInternalNode) != + ChildrenAddedToStack.end()) { + // If the children of the current node has been added to the stack, + // then this is the second time we visit this node and at this point, + // all of its children have already been processed. Now, we can + // set its LeftLeafIdx and RightLeafIdx; + auto it = CurrInternalNode->Children.begin(); + if (it != CurrInternalNode->Children.end()) { + // Get the first child to use its RightLeafIdx. The RightLeafIdx is + // used as the first child is the initial one added to the stack, so + // it's the last one to be processed. This implies that the leaf + // descendants of the first child are assigned the largest index + // numbers. + CurrNode->setRightLeafIdx(it->second->getRightLeafIdx()); + // get the last child to use its LeftLeafIdx. + while (std::next(it) != CurrInternalNode->Children.end()) + it = std::next(it); + CurrNode->setLeftLeafIdx(it->second->getLeftLeafIdx()); + assert(CurrNode->getLeftLeafIdx() <= CurrNode->getRightLeafIdx() && + "LeftLeafIdx should not be larger than RightLeafIdx"); + } + } else { + // This is the first time we visit this node. This means that its + // children have not been added to the stack yet. Hence, we will add + // the current node back to the stack and add its children to the + // stack for processing. + ToVisit.push(CurrNode); + for (auto &ChildPair : CurrInternalNode->Children) + ToVisit.push(ChildPair.second); + ChildrenAddedToStack.insert(CurrInternalNode); + } + } else { + // The current node is a leaf node. + // We can simplyset its LeftLeafIdx and RightLeafIdx. + CurrNode->setLeftLeafIdx(LeafCounter); + CurrNode->setRightLeafIdx(LeafCounter); + LeafCounter++; + auto *CurrLeafNode = cast(CurrNode); + LeafNodes.push_back(CurrLeafNode); + } + } +} + unsigned SuffixTree::extend(unsigned EndIdx, unsigned SuffixesToAdd) { SuffixTreeInternalNode *NeedsLink = nullptr; @@ -230,6 +301,7 @@ void SuffixTree::RepeatedSubstringIterator::advance() { // Each leaf node represents a repeat of a string. SmallVector RepeatedSubstringStarts; + SmallVector LeafDescendants; // Continue visiting nodes until we find one which repeats more than once. while (!InternalNodesToVisit.empty()) { @@ -252,7 +324,7 @@ void SuffixTree::RepeatedSubstringIterator::advance() { continue; } - if (Length < MinLength) + if (Length < MinLength || OutlinerLeafDescendants) continue; // Have an occurrence of a potentially repeated string. Save it. @@ -260,6 +332,13 @@ void SuffixTree::RepeatedSubstringIterator::advance() { RepeatedSubstringStarts.push_back(Leaf->getSuffixIdx()); } + if (OutlinerLeafDescendants && Length >= MinLength) { + LeafDescendants.assign(LeafNodes.begin() + Curr->getLeftLeafIdx(), + LeafNodes.begin() + Curr->getRightLeafIdx() + 1); + for (SuffixTreeLeafNode *Leaf : LeafDescendants) + RepeatedSubstringStarts.push_back(Leaf->getSuffixIdx()); + } + // The root never represents a repeated substring. If we're looking at // that, then skip it. if (Curr->isRoot()) diff --git a/llvm/lib/Support/SuffixTreeNode.cpp b/llvm/lib/Support/SuffixTreeNode.cpp index 113b990fd352fc..9f1f94a39895e8 100644 --- a/llvm/lib/Support/SuffixTreeNode.cpp +++ b/llvm/lib/Support/SuffixTreeNode.cpp @@ -38,3 +38,8 @@ unsigned SuffixTreeLeafNode::getEndIdx() const { unsigned SuffixTreeLeafNode::getSuffixIdx() const { return SuffixIdx; } void SuffixTreeLeafNode::setSuffixIdx(unsigned Idx) { SuffixIdx = Idx; } + +unsigned SuffixTreeNode::getLeftLeafIdx() const { return LeftLeafIdx; } +unsigned SuffixTreeNode::getRightLeafIdx() const { return RightLeafIdx; } +void SuffixTreeNode::setLeftLeafIdx(unsigned Idx) { LeftLeafIdx = Idx; } +void SuffixTreeNode::setRightLeafIdx(unsigned Idx) { RightLeafIdx = Idx; } diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 9518d573bccdd1..47783e11099688 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -8240,7 +8240,8 @@ static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a, std::optional AArch64InstrInfo::getOutliningCandidateInfo( - std::vector &RepeatedSequenceLocs) const { + std::vector &RepeatedSequenceLocs, + unsigned MinRep) const { outliner::Candidate &FirstCand = RepeatedSequenceLocs[0]; unsigned SequenceSize = 0; @@ -8354,7 +8355,7 @@ AArch64InstrInfo::getOutliningCandidateInfo( llvm::erase_if(RepeatedSequenceLocs, hasIllegalSPModification); // If the sequence doesn't have enough candidates left, then we're done. - if (RepeatedSequenceLocs.size() < 2) + if (RepeatedSequenceLocs.size() < MinRep) return std::nullopt; } @@ -8598,7 +8599,7 @@ AArch64InstrInfo::getOutliningCandidateInfo( } // If we dropped all of the candidates, bail out here. - if (RepeatedSequenceLocs.size() < 2) { + if (RepeatedSequenceLocs.size() < MinRep) { RepeatedSequenceLocs.clear(); return std::nullopt; } diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h index 9a2914891675c5..f0eccd541c225a 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h @@ -463,7 +463,8 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo { bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override; std::optional getOutliningCandidateInfo( - std::vector &RepeatedSequenceLocs) const override; + std::vector &RepeatedSequenceLocs, + unsigned MinRep) const override; void mergeOutliningCandidateAttributes( Function &F, std::vector &Candidates) const override; outliner::InstrType diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt index 6127b76db06b7f..be777ce650e874 100644 --- a/llvm/test/CMakeLists.txt +++ b/llvm/test/CMakeLists.txt @@ -73,6 +73,7 @@ set(LLVM_TEST_DEPENDS llvm-c-test llvm-cat llvm-cfi-verify + llvm-cgdata llvm-config llvm-cov llvm-cvtres diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-cfi-tail-some.mir b/llvm/test/CodeGen/AArch64/machine-outliner-cfi-tail-some.mir index 67d411962ce4f7..3afa1d5559a585 100644 --- a/llvm/test/CodeGen/AArch64/machine-outliner-cfi-tail-some.mir +++ b/llvm/test/CodeGen/AArch64/machine-outliner-cfi-tail-some.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=aarch64-apple-unknown -run-pass=machine-outliner -verify-machineinstrs %s -o - | FileCheck %s +# RUN: llc -mtriple=aarch64-apple-unknown -run-pass=machine-outliner -verify-machineinstrs -outliner-leaf-descendants=false %s -o - | FileCheck %s # Outlining CFI instructions is unsafe if we cannot outline all of the CFI # instructions from a function. This shows that we choose not to outline the diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-leaf-descendants.ll b/llvm/test/CodeGen/AArch64/machine-outliner-leaf-descendants.ll new file mode 100644 index 00000000000000..bdaf653a79566d --- /dev/null +++ b/llvm/test/CodeGen/AArch64/machine-outliner-leaf-descendants.ll @@ -0,0 +1,124 @@ +; This test is mainly for the -outliner-leaf-descendants flag for MachineOutliner. +; +; ===================== -outliner-leaf-descendants=false ===================== +; MachineOutliner finds THREE key `OutlinedFunction` and outlines them. They are: +; ``` +; mov w0, #1 +; mov w1, #2 +; mov w2, #3 +; mov w3, #4 +; mov w4, #5 +; mov w5, #6 or #7 or #8 +; b +; ``` +; Each has: +; - `SequenceSize=28` and `OccurrenceCount=2` +; - each Candidate has `CallOverhead=4` and `FrameOverhead=0` +; - `NotOutlinedCost=28*2=56` and `OutliningCost=4*2+28+0=36` +; - `Benefit=56-36=20` and `Priority=56/36=1.56` +; +; ===================== -outliner-leaf-descendants=false ===================== +; MachineOutliner finds a FOURTH key `OutlinedFunction`, which is: +; ``` +; mov w0, #1 +; mov w1, #2 +; mov w2, #3 +; mov w3, #4 +; mov w4, #5 +; ``` +; This corresponds to an internal node that has ZERO leaf children, but SIX leaf descendants. +; It has: +; - `SequenceSize=20` and `OccurrenceCount=6` +; - each Candidate has `CallOverhead=12` and `FrameOverhead=4` +; - `NotOutlinedCost=20*6=120` and `OutliningCost=12*6+20+4=96` +; - `Benefit=120-96=24` and `Priority=120/96=1.25` +; +; The FOURTH `OutlinedFunction` has lower _priority_ compared to the first THREE `OutlinedFunction` +; Hence, if we additionally include the `-sort-per-priority` flag, the first THREE `OutlinedFunction` are outlined. + +; RUN: llc %s -enable-machine-outliner=always -outliner-leaf-descendants=false -filetype=obj -o %t +; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-BASELINE + +; RUN: llc %s -enable-machine-outliner=always -outliner-leaf-descendants=false -outliner-benefit-threshold=22 -filetype=obj -o %t +; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-NO-CANDIDATE + +; RUN: llc %s -enable-machine-outliner=always -outliner-leaf-descendants=true -filetype=obj -o %t +; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-BASELINE + +; RUN: llc %s -enable-machine-outliner=always -outliner-leaf-descendants=true -outliner-benefit-threshold=22 -filetype=obj -o %t +; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-LEAF-DESCENDANTS + + +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" +target triple = "arm64-apple-macosx14.0.0" + +declare i32 @_Z3fooiiii(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) + +define i32 @_Z2f1v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6) + ret i32 %1 +} + +define i32 @_Z2f2v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6) + ret i32 %1 +} + +define i32 @_Z2f3v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 7) + ret i32 %1 +} + +define i32 @_Z2f4v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 7) + ret i32 %1 +} + +define i32 @_Z2f5v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 8) + ret i32 %1 +} + +define i32 @_Z2f6v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 8) + ret i32 %1 +} + +; CHECK-BASELINE: <_OUTLINED_FUNCTION_0>: +; CHECK-BASELINE-NEXT: mov w0, #0x1 +; CHECK-BASELINE-NEXT: mov w1, #0x2 +; CHECK-BASELINE-NEXT: mov w2, #0x3 +; CHECK-BASELINE-NEXT: mov w3, #0x4 +; CHECK-BASELINE-NEXT: mov w4, #0x5 +; CHECK-BASELINE-NEXT: mov w5, #0x6 +; CHECK-BASELINE-NEXT: b + +; CHECK-BASELINE: <_OUTLINED_FUNCTION_1>: +; CHECK-BASELINE-NEXT: mov w0, #0x1 +; CHECK-BASELINE-NEXT: mov w1, #0x2 +; CHECK-BASELINE-NEXT: mov w2, #0x3 +; CHECK-BASELINE-NEXT: mov w3, #0x4 +; CHECK-BASELINE-NEXT: mov w4, #0x5 +; CHECK-BASELINE-NEXT: mov w5, #0x8 +; CHECK-BASELINE-NEXT: b + +; CHECK-BASELINE: <_OUTLINED_FUNCTION_2>: +; CHECK-BASELINE-NEXT: mov w0, #0x1 +; CHECK-BASELINE-NEXT: mov w1, #0x2 +; CHECK-BASELINE-NEXT: mov w2, #0x3 +; CHECK-BASELINE-NEXT: mov w3, #0x4 +; CHECK-BASELINE-NEXT: mov w4, #0x5 +; CHECK-BASELINE-NEXT: mov w5, #0x7 +; CHECK-BASELINE-NEXT: b + +; CHECK-LEAF-DESCENDANTS: <_OUTLINED_FUNCTION_0>: +; CHECK-LEAF-DESCENDANTS-NEXT: mov w0, #0x1 +; CHECK-LEAF-DESCENDANTS-NEXT: mov w1, #0x2 +; CHECK-LEAF-DESCENDANTS-NEXT: mov w2, #0x3 +; CHECK-LEAF-DESCENDANTS-NEXT: mov w3, #0x4 +; CHECK-LEAF-DESCENDANTS-NEXT: mov w4, #0x5 +; CHECK-LEAF-DESCENDANTS-NEXT: ret + +; CHECK-LEAF-DESCENDANTS-NOT: <_OUTLINED_FUNCTION_1>: + +; CHECK-NO-CANDIDATE-NOT: <_OUTLINED_FUNCTION_0>: diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-overlap.mir b/llvm/test/CodeGen/AArch64/machine-outliner-overlap.mir index 649bb33828c32c..8b0e03dbee8d8d 100644 --- a/llvm/test/CodeGen/AArch64/machine-outliner-overlap.mir +++ b/llvm/test/CodeGen/AArch64/machine-outliner-overlap.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --include-generated-funcs -# RUN: llc %s -mtriple aarch64 -debug-only=machine-outliner -run-pass=machine-outliner -o - 2>&1 | FileCheck %s +# RUN: llc %s -mtriple aarch64 -outliner-leaf-descendants=false -debug-only=machine-outliner -run-pass=machine-outliner -o - 2>&1 | FileCheck %s +# RUN: llc %s -mtriple aarch64 -debug-only=machine-outliner -run-pass=machine-outliner -o - 2>&1 | FileCheck %s --check-prefix=CHECK-LEAF # REQUIRES: asserts # CHECK: *** Discarding overlapping candidates *** @@ -8,27 +9,27 @@ # CHECK-NEXT: Candidates discarded: 0 # CHECK-NEXT: Candidates kept: 2 # CHECK-DAG: Sequence length: 8 -# CHECK-NEXT: .. DISCARD candidate @ [5, 12]; overlaps with candidate @ [12, 19] +# CHECK-NEXT: .. DISCARD candidate @ [12, 19]; overlaps with candidate @ [5, 12] # CHECK-NEXT: Candidates discarded: 1 # CHECK-NEXT: Candidates kept: 1 # CHECK-DAG: Sequence length: 9 -# CHECK-NEXT: .. DISCARD candidate @ [4, 12]; overlaps with candidate @ [11, 19] +# CHECK-NEXT: .. DISCARD candidate @ [11, 19]; overlaps with candidate @ [4, 12] # CHECK-NEXT: Candidates discarded: 1 # CHECK-NEXT: Candidates kept: 1 # CHECK-DAG: Sequence length: 10 -# CHECK-NEXT: .. DISCARD candidate @ [3, 12]; overlaps with candidate @ [10, 19] +# CHECK-NEXT: .. DISCARD candidate @ [10, 19]; overlaps with candidate @ [3, 12] # CHECK-NEXT: Candidates discarded: 1 # CHECK-NEXT: Candidates kept: 1 # CHECK-DAG: Sequence length: 11 -# CHECK-NEXT: .. DISCARD candidate @ [2, 12]; overlaps with candidate @ [9, 19] +# CHECK-NEXT: .. DISCARD candidate @ [9, 19]; overlaps with candidate @ [2, 12] # CHECK-NEXT: Candidates discarded: 1 # CHECK-NEXT: Candidates kept: 1 # CHECK-DAG: Sequence length: 12 -# CHECK-NEXT: .. DISCARD candidate @ [1, 12]; overlaps with candidate @ [8, 19] +# CHECK-NEXT: .. DISCARD candidate @ [8, 19]; overlaps with candidate @ [1, 12] # CHECK-NEXT: Candidates discarded: 1 # CHECK-NEXT: Candidates kept: 1 # CHECK-DAG: Sequence length: 13 -# CHECK-NEXT: .. DISCARD candidate @ [0, 12]; overlaps with candidate @ [7, 19] +# CHECK-NEXT: .. DISCARD candidate @ [7, 19]; overlaps with candidate @ [0, 12] # CHECK-NEXT: Candidates discarded: 1 # CHECK-NEXT: Candidates kept: 1 @@ -54,6 +55,27 @@ body: | ; CHECK-NEXT: BL @OUTLINED_FUNCTION_0, implicit-def $lr, implicit $sp, implicit-def $lr, implicit-def $x8, implicit-def $x9, implicit $sp, implicit $x0, implicit $x9 ; CHECK-NEXT: RET undef $x9 + ; CHECK-LEAF-LABEL: name: overlap + ; CHECK-LEAF: liveins: $x0, $x9 + ; CHECK-LEAF-NEXT: {{ $}} + ; CHECK-LEAF-NEXT: BL @OUTLINED_FUNCTION_0 + ; CHECK-LEAF-NEXT: BL @OUTLINED_FUNCTION_0 + ; CHECK-LEAF-NEXT: $x8 = ADDXri $x0, 3, 0 + ; CHECK-LEAF-NEXT: BL @OUTLINED_FUNCTION_0 + ; CHECK-LEAF-NEXT: BL @OUTLINED_FUNCTION_0 + ; CHECK-LEAF-NEXT: $x8 = ADDXri $x0, 3, 0 + ; CHECK-LEAF-NEXT: BL @OUTLINED_FUNCTION_0 + ; CHECK-LEAF-NEXT: BL @OUTLINED_FUNCTION_0 + ; CHECK-LEAF-NEXT: RET undef $x9 + + ; CHECK-LEAF-LABEL: name: OUTLINED_FUNCTION_0 + ; CHECK-LEAF: liveins: $x0, $x9, $lr + ; CHECK-LEAF-NEXT: {{ $}} + ; CHECK-LEAF-NEXT: $x9 = ADDXri $x9, 16, 0 + ; CHECK-LEAF-NEXT: $x9 = ADDXri $x9, 16, 0 + ; CHECK-LEAF-NEXT: $x9 = ADDXri $x9, 16, 0 + ; CHECK-LEAF-NEXT: RET $lr + ; fixme: outline! $x9 = ADDXri $x9, 16, 0 $x9 = ADDXri $x9, 16, 0 diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-sp-mod.mir b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-sp-mod.mir index c1c2720dec6ad6..22e5edef2a9395 100644 --- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-sp-mod.mir +++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-sp-mod.mir @@ -1,4 +1,4 @@ -# RUN: llc -verify-machineinstrs -run-pass=machine-outliner -run-pass=aarch64-ptrauth %s -o - | FileCheck %s +# RUN: llc -verify-machineinstrs -run-pass=machine-outliner -run-pass=aarch64-ptrauth -outliner-leaf-descendants=false %s -o - | FileCheck %s --- | target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll index 9250718fc0d585..618973b9368d1d 100644 --- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll +++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll @@ -1,6 +1,6 @@ -; RUN: llc -mtriple aarch64-arm-linux-gnu --enable-machine-outliner \ +; RUN: llc -mtriple aarch64-arm-linux-gnu --enable-machine-outliner -outliner-leaf-descendants=false \ ; RUN: -verify-machineinstrs %s -o - | FileCheck --check-prefixes CHECK,V8A %s -; RUN-V83A: llc -mtriple aarch64 -enable-machine-outliner \ +; RUN-V83A: llc -mtriple aarch64 -enable-machine-outliner -outliner-leaf-descendants=false \ ; RUN-V83A: -verify-machineinstrs -mattr=+v8.3a %s -o - > %t ; RUN-V83A: FileCheck --check-prefixes CHECK,V83A < %t %s diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll b/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll new file mode 100644 index 00000000000000..00efc3c6e71c89 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll @@ -0,0 +1,96 @@ +; This tests the order in which functions are outlined in MachineOutliner +; There are TWO key OutlinedFunction in FunctionList +; +; ===================== First One ===================== +; ``` +; mov w0, #1 +; mov w1, #2 +; mov w2, #3 +; mov w3, #4 +; mov w4, #5 +; ``` +; It has: +; - `SequenceSize=20` and `OccurrenceCount=6` +; - each Candidate has `CallOverhead=12` and `FrameOverhead=4` +; - `NotOutlinedCost=20*6=120` and `OutliningCost=12*6+20+4=96` +; - `Benefit=120-96=24` and `Priority=120/96=1.25` +; +; ===================== Second One ===================== +; ``` +; mov w6, #6 +; mov w7, #7 +; b +; ``` +; It has: +; - `SequenceSize=12` and `OccurrenceCount=4` +; - each Candidate has `CallOverhead=4` and `FrameOverhead=0` +; - `NotOutlinedCost=12*4=48` and `OutliningCost=4*4+12+0=28` +; - `Benefit=120-96=20` and `Priority=48/28=1.71` +; +; Note that the first one has higher benefit, but lower priority. +; Hence, when outlining per priority, the second one will be outlined first. + +; RUN: llc %s -enable-machine-outliner=always -filetype=obj -o %t +; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-SORT-BY-PRIORITY + +; RUN: llc %s -enable-machine-outliner=always -outliner-benefit-threshold=22 -filetype=obj -o %t +; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-THRESHOLD + + +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" +target triple = "arm64-apple-macosx14.0.0" + +declare i32 @_Z3fooiiii(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) + +define i32 @_Z2f1v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 11, i32 noundef 6, i32 noundef 7) + ret i32 %1 +} + +define i32 @_Z2f2v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 12, i32 noundef 6, i32 noundef 7) + ret i32 %1 +} + +define i32 @_Z2f3v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 13, i32 noundef 6, i32 noundef 7) + ret i32 %1 +} + +define i32 @_Z2f4v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 14, i32 noundef 6, i32 noundef 7) + ret i32 %1 +} + +define i32 @_Z2f5v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 15, i32 noundef 8, i32 noundef 9) + ret i32 %1 +} + +define i32 @_Z2f6v() minsize { + %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 16, i32 noundef 9, i32 noundef 8) + ret i32 %1 +} + +; CHECK-SORT-BY-PRIORITY: <_OUTLINED_FUNCTION_0>: +; CHECK-SORT-BY-PRIORITY-NEXT: mov w6, #0x6 +; CHECK-SORT-BY-PRIORITY-NEXT: mov w7, #0x7 +; CHECK-SORT-BY-PRIORITY-NEXT: b + +; CHECK-SORT-BY-PRIORITY: <_OUTLINED_FUNCTION_1>: +; CHECK-SORT-BY-PRIORITY-NEXT: mov w0, #0x1 +; CHECK-SORT-BY-PRIORITY-NEXT: mov w1, #0x2 +; CHECK-SORT-BY-PRIORITY-NEXT: mov w2, #0x3 +; CHECK-SORT-BY-PRIORITY-NEXT: mov w3, #0x4 +; CHECK-SORT-BY-PRIORITY-NEXT: mov w4, #0x5 +; CHECK-SORT-BY-PRIORITY-NEXT: ret + +; CHECK-THRESHOLD: <_OUTLINED_FUNCTION_0>: +; CHECK-THRESHOLD-NEXT: mov w0, #0x1 +; CHECK-THRESHOLD-NEXT: mov w1, #0x2 +; CHECK-THRESHOLD-NEXT: mov w2, #0x3 +; CHECK-THRESHOLD-NEXT: mov w3, #0x4 +; CHECK-THRESHOLD-NEXT: mov w4, #0x5 +; CHECK-THRESHOLD-NEXT: ret + +; CHECK-THRESHOLD-NOT: <_OUTLINED_FUNCTION_1>: diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll b/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll index aa6e31d6ff21d7..538e1165e39c1d 100644 --- a/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll +++ b/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll @@ -1,5 +1,5 @@ -; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple=aarch64 -frame-pointer=non-leaf < %s | FileCheck %s --check-prefix=NOOMIT -; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple=aarch64 -frame-pointer=none < %s | FileCheck %s --check-prefix=OMITFP +; RUN: llc -verify-machineinstrs -enable-machine-outliner -outliner-leaf-descendants=false -mtriple=aarch64 -frame-pointer=non-leaf < %s | FileCheck %s --check-prefix=NOOMIT +; RUN: llc -verify-machineinstrs -enable-machine-outliner -outliner-leaf-descendants=false -mtriple=aarch64 -frame-pointer=none < %s | FileCheck %s --check-prefix=OMITFP define void @_Z1giii(i32 %x, i32 %y, i32 %z) minsize { ; NOOMIT-LABEL: _Z1giii: diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-thunk.ll b/llvm/test/CodeGen/AArch64/machine-outliner-thunk.ll index 8740aac0549eec..7e34adf16d25d3 100644 --- a/llvm/test/CodeGen/AArch64/machine-outliner-thunk.ll +++ b/llvm/test/CodeGen/AArch64/machine-outliner-thunk.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -enable-machine-outliner -verify-machineinstrs | FileCheck %s +; RUN: llc < %s -enable-machine-outliner -outliner-leaf-descendants=false -verify-machineinstrs | FileCheck %s target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64-pc-linux-gnu" diff --git a/llvm/test/CodeGen/AArch64/machine-outliner.mir b/llvm/test/CodeGen/AArch64/machine-outliner.mir index 83eda744d24a9e..66779addaff0a8 100644 --- a/llvm/test/CodeGen/AArch64/machine-outliner.mir +++ b/llvm/test/CodeGen/AArch64/machine-outliner.mir @@ -1,4 +1,4 @@ -# RUN: llc -mtriple=aarch64--- -run-pass=prologepilog -run-pass=machine-outliner -verify-machineinstrs -frame-pointer=non-leaf %s -o - | FileCheck %s +# RUN: llc -mtriple=aarch64--- -run-pass=prologepilog -run-pass=machine-outliner -verify-machineinstrs -frame-pointer=non-leaf -outliner-leaf-descendants=false %s -o - | FileCheck %s --- | @x = common global i32 0, align 4 diff --git a/llvm/test/CodeGen/ARM/machine-outliner-calls.mir b/llvm/test/CodeGen/ARM/machine-outliner-calls.mir index a92c9dd28be5ae..7634ecd6e863ae 100644 --- a/llvm/test/CodeGen/ARM/machine-outliner-calls.mir +++ b/llvm/test/CodeGen/ARM/machine-outliner-calls.mir @@ -26,15 +26,15 @@ body: | ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8 ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -4 ; CHECK: frame-setup CFI_INSTRUCTION offset $r4, -8 - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_2 ; CHECK: bb.1: - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_2 ; CHECK: bb.2: - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_2 ; CHECK: bb.3: - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_2 ; CHECK: bb.4: - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_2 ; CHECK: bb.5: ; CHECK: $sp = frame-destroy LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $lr ; CHECK: BX_RET 14 /* CC::al */, $noreg @@ -139,13 +139,13 @@ body: | ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8 ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -4 ; CHECK: frame-setup CFI_INSTRUCTION offset $r4, -8 - ; CHECK: BL @OUTLINED_FUNCTION_1 + ; CHECK: BL @OUTLINED_FUNCTION_0 ; CHECK: bb.1: - ; CHECK: BL @OUTLINED_FUNCTION_1 + ; CHECK: BL @OUTLINED_FUNCTION_0 ; CHECK: bb.2: - ; CHECK: BL @OUTLINED_FUNCTION_1 + ; CHECK: BL @OUTLINED_FUNCTION_0 ; CHECK: bb.3: - ; CHECK: BL @OUTLINED_FUNCTION_1 + ; CHECK: BL @OUTLINED_FUNCTION_0 ; CHECK: bb.4: ; CHECK: $sp = frame-destroy LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $lr ; CHECK: BX_RET 14 /* CC::al */, $noreg @@ -245,19 +245,19 @@ body: | ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -4 ; CHECK: frame-setup CFI_INSTRUCTION offset $r4, -8 ; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp - ; CHECK: BL @OUTLINED_FUNCTION_2 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: bb.1: ; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp - ; CHECK: BL @OUTLINED_FUNCTION_2 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: bb.2: ; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp - ; CHECK: BL @OUTLINED_FUNCTION_2 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: bb.3: ; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp - ; CHECK: BL @OUTLINED_FUNCTION_2 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: bb.4: ; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp - ; CHECK: BL @OUTLINED_FUNCTION_2 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: bb.5: ; CHECK: $sp = frame-destroy LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $lr ; CHECK: BX_RET 14 /* CC::al */, $noreg @@ -307,38 +307,17 @@ body: | bb.0: BX_RET 14, $noreg - ; CHECK-LABEL: name: OUTLINED_FUNCTION_0 ; CHECK: bb.0: - ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr - ; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -8 - ; CHECK: BL @bar, implicit-def dead $lr, implicit $sp - ; CHECK: $r0 = MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r1 = MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r2 = MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r3 = MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r4 = MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg - ; CHECK: MOVPCLR 14 /* CC::al */, $noreg - - ; CHECK-LABEL: name: OUTLINED_FUNCTION_1 - ; CHECK: bb.0: - ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr - ; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -8 - ; CHECK: BL @bar, implicit-def dead $lr, implicit $sp + ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 ; CHECK: $r0 = MOVi 2, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r1 = MOVi 2, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r2 = MOVi 2, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r3 = MOVi 2, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r4 = MOVi 2, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg ; CHECK: TAILJMPd @bar, implicit $sp - ; CHECK-LABEL: name: OUTLINED_FUNCTION_2 + ; CHECK-LABEL: name: OUTLINED_FUNCTION_1 ; CHECK: bb.0: ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 ; CHECK: $r0 = MOVi 3, 14 /* CC::al */, $noreg, $noreg @@ -348,31 +327,28 @@ body: | ; CHECK: $r4 = MOVi 3, 14 /* CC::al */, $noreg, $noreg ; CHECK: MOVPCLR 14 /* CC::al */, $noreg + ; CHECK-LABEL: name: OUTLINED_FUNCTION_2 + ; CHECK: bb.0: + ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 + ; CHECK: $r0 = MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r1 = MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r2 = MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r3 = MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r4 = MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: MOVPCLR 14 /* CC::al */, $noreg + ; CHECK-LABEL: name: OUTLINED_FUNCTION_3 ; CHECK: bb.0: - ; CHECK: liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr - ; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -8 - ; CHECK: tBL 14 /* CC::al */, $noreg, @bar, implicit-def dead $lr, implicit $sp + ; CHECK: liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 ; CHECK: $r0 = t2MOVi 2, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r1 = t2MOVi 2, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r2 = t2MOVi 2, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg ; CHECK: tTAILJMPdND @bar, 14 /* CC::al */, $noreg, implicit $sp ; CHECK-LABEL: name: OUTLINED_FUNCTION_4 ; CHECK: bb.0: - ; CHECK: liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr - ; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -8 - ; CHECK: tBL 14 /* CC::al */, $noreg, @bar, implicit-def dead $lr, implicit $sp + ; CHECK: liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 ; CHECK: $r0 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r1 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r2 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg ; CHECK: tBX_RET 14 /* CC::al */, $noreg - - - diff --git a/llvm/test/CodeGen/ARM/machine-outliner-default.mir b/llvm/test/CodeGen/ARM/machine-outliner-default.mir index 6d0218dbfe636d..de2b8f55969765 100644 --- a/llvm/test/CodeGen/ARM/machine-outliner-default.mir +++ b/llvm/test/CodeGen/ARM/machine-outliner-default.mir @@ -19,17 +19,17 @@ body: | ; CHECK: bb.0: ; CHECK: liveins: $lr ; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg ; CHECK: bb.1: ; CHECK: liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11 ; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg ; CHECK: bb.2: ; CHECK: liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11 ; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: BL @OUTLINED_FUNCTION_0 + ; CHECK: BL @OUTLINED_FUNCTION_1 ; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg ; CHECK: bb.3: ; CHECK: liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11 @@ -73,17 +73,17 @@ body: | ; CHECK: bb.0: ; CHECK: liveins: $lr ; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_1 + ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0 ; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg ; CHECK: bb.1: ; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11 ; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_1 + ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0 ; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg ; CHECK: bb.2: ; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11 ; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_1 + ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0 ; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg ; CHECK: bb.3: ; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11 @@ -114,6 +114,15 @@ body: | ; CHECK-LABEL: name: OUTLINED_FUNCTION_0 ; CHECK: bb.0: + ; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11 + ; CHECK: $r0 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r1 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r2 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: $r3 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg + ; CHECK: tBX_RET 14 /* CC::al */, $noreg + + ; CHECK-LABEL: name: OUTLINED_FUNCTION_1 + ; CHECK: bb.0: ; CHECK: liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11 ; CHECK: $r0 = MOVi 1, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r1 = MOVi 1, 14 /* CC::al */, $noreg, $noreg @@ -122,15 +131,3 @@ body: | ; CHECK: $r4 = MOVi 1, 14 /* CC::al */, $noreg, $noreg ; CHECK: $r5 = MOVi 1, 14 /* CC::al */, $noreg, $noreg ; CHECK: MOVPCLR 14 /* CC::al */, $noreg - - ; CHECK-LABEL: name: OUTLINED_FUNCTION_1 - ; CHECK: bb.0: - ; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11 - ; CHECK: $r0 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r1 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r2 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: $r3 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg - ; CHECK: tBX_RET 14 /* CC::al */, $noreg - - - diff --git a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir index ae5caa5b7c06db..e71edc8ceb3f60 100644 --- a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir +++ b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir @@ -18,6 +18,7 @@ body: | liveins: $r0 ; CHECK-LABEL: name: CheckAddrMode_i12 ; CHECK: $r1 = MOVr killed $r0, 14 /* CC::al */, $noreg, $noreg + ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp ; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I12:[0-9]+]] ; CHECK-NEXT: $r6 = LDRi12 $sp, 4088, 14 /* CC::al */, $noreg $r1 = MOVr killed $r0, 14, $noreg, $noreg @@ -47,6 +48,7 @@ body: | liveins: $r1 ; CHECK-LABEL: name: CheckAddrMode3 ; CHECK: $r0 = MOVr killed $r1, 14 /* CC::al */, $noreg, $noreg + ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp ; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I3:[0-9]+]] ; CHECK-NEXT: $r6 = LDRSH $sp, $noreg, 248, 14 /* CC::al */, $noreg $r0 = MOVr killed $r1, 14, $noreg, $noreg @@ -76,6 +78,7 @@ body: | liveins: $r2 ; CHECK-LABEL: name: CheckAddrMode5 ; CHECK: $r0 = MOVr killed $r2, 14 /* CC::al */, $noreg, $noreg + ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp ; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I5:[0-9]+]] ; CHECK-NEXT: $d5 = VLDRD $sp, 254, 14 /* CC::al */, $noreg $r0 = MOVr killed $r2, 14, $noreg, $noreg @@ -110,6 +113,7 @@ body: | liveins: $r3 ; CHECK-LABEL: name: CheckAddrMode5FP16 ; CHECK: $r0 = MOVr killed $r3, 14 /* CC::al */, $noreg, $noreg + ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp ; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I5FP16:[0-9]+]] ; CHECK-NEXT: $s6 = VLDRH $sp, 252, 14, $noreg $r0 = MOVr killed $r3, 14, $noreg, $noreg @@ -146,41 +150,29 @@ body: | BX_RET 14, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[I5]] - ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: $d0 = VLDRD $sp, 2, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $d1 = VLDRD $sp, 10, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $d4 = VLDRD $sp, 255, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg + ;CHECK: liveins: $r10, $r9, $r8, $r7, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 + ;CHECK: $d0 = VLDRD $sp, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $d1 = VLDRD $sp, 8, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $d4 = VLDRD $sp, 253, 14 /* CC::al */, $noreg + ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[I5FP16]] - ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: $s1 = VLDRH $sp, 4, 14, $noreg - ;CHECK-NEXT: $s2 = VLDRH $sp, 12, 14, $noreg - ;CHECK-NEXT: $s5 = VLDRH $sp, 244, 14, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg + ;CHECK: liveins: $r10, $r9, $r8, $r7, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8 + ;CHECK: $s1 = VLDRH $sp, 0, 14, $noreg + ;CHECK-NEXT: $s2 = VLDRH $sp, 8, 14, $noreg + ;CHECK-NEXT: $s5 = VLDRH $sp, 240, 14, $noreg + ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[I12]] - ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: $r1 = LDRi12 $sp, 8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r2 = LDRi12 $sp, 16, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r5 = LDRi12 $sp, 4094, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg + ;CHECK: liveins: $r10, $r9, $r8, $r7, $d8, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9 + ;CHECK: $r1 = LDRi12 $sp, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r2 = LDRi12 $sp, 8, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r5 = LDRi12 $sp, 4086, 14 /* CC::al */, $noreg + ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[I3]] - ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: $r1 = LDRSH $sp, $noreg, 8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r2 = LDRSH $sp, $noreg, 16, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r5 = LDRSH $sp, $noreg, 255, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg + ;CHECK: liveins: $r10, $r9, $r8, $r7, $d8, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9 + ;CHECK: $r1 = LDRSH $sp, $noreg, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r2 = LDRSH $sp, $noreg, 8, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r5 = LDRSH $sp, $noreg, 247, 14 /* CC::al */, $noreg + ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg diff --git a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir index 56184448424489..f2cdaebdfa8baa 100644 --- a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir +++ b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir @@ -19,7 +19,7 @@ body: | bb.0: liveins: $r1 ;CHECK-LABEL: name: CheckAddrModeT2_i12 - ;CHECK: $r0 = tMOVr killed $r1, 14 /* CC::al */, $noreg + ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]] ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[I12:[0-9]+]] ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4088, 14 /* CC::al */, $noreg $r0 = tMOVr killed $r1, 14, $noreg @@ -49,7 +49,7 @@ body: | bb.0: liveins: $r1 ;CHECK-LABEL: name: CheckAddrModeT2_i8 - ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg + ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]] ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[I8:[0-9]+]] ;CHECK-NEXT: t2STRHT $r0, $sp, 248, 14 /* CC::al */, $noreg $r0 = tMOVr $r1, 14, $noreg @@ -79,7 +79,7 @@ body: | bb.0: liveins: $r1 ;CHECK-LABEL: name: CheckAddrModeT2_i8s4 - ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg + ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]] ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[I8S4:[0-9]+]] ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 1020, 14 /* CC::al */, $noreg $r0 = tMOVr $r1, 14, $noreg @@ -109,7 +109,7 @@ body: | bb.0: liveins: $r1 ;CHECK-LABEL: name: CheckAddrModeT2_ldrex - ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg + ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]] ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[LDREX:[0-9]+]] ;CHECK-NEXT: $r1 = t2LDREX $sp, 254, 14 /* CC::al */, $noreg $r0 = tMOVr $r1, 14, $noreg @@ -144,8 +144,10 @@ body: | bb.0: liveins: $r0, $r1 ;CHECK-LABEL: name: CheckAddrModeT1_s - ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg - ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[T1_S:[0-9]+]] + ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]] + ;CHECK-NEXT: tSTRspi $r0, $sp, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tSTRspi $r0, $sp, 4, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tSTRspi $r0, $sp, 253, 14 /* CC::al */, $noreg ;CHECK-NEXT: tSTRspi $r0, $sp, 254, 14 /* CC::al */, $noreg $r0 = tMOVr $r1, 14, $noreg tBL 14, $noreg, @foo, implicit-def dead $lr, implicit $sp @@ -181,51 +183,29 @@ body: | BX_RET 14, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[LDREX]] - ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: $r1 = t2LDREX $sp, 2, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r1 = t2LDREX $sp, 10, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r1 = t2LDREX $sp, 255, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg + ;CHECK: $r1 = t2LDREX $sp, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r1 = t2LDREX $sp, 8, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r1 = t2LDREX $sp, 253, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[I8]] - ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: t2STRHT $r0, $sp, 8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: t2STRHT $r0, $sp, 12, 14 /* CC::al */, $noreg - ;CHECK-NEXT: t2STRHT $r0, $sp, 255, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg + ;CHECK: t2STRHT $r0, $sp, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: t2STRHT $r0, $sp, 4, 14 /* CC::al */, $noreg + ;CHECK-NEXT: t2STRHT $r0, $sp, 247, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $noreg ;CHECK: name: OUTLINED_FUNCTION_[[I8S4]] - ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp + ;CHECK: t2STRDi8 $r0, $r1, $sp, 0, 14 /* CC::al */, $noreg ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 16, 14 /* CC::al */, $noreg - ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 1020, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg + ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 1012, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $nore ;CHECK: name: OUTLINED_FUNCTION_[[I12]] - ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 12, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4094, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg - - ;CHECK: name: OUTLINED_FUNCTION_[[T1_S]] - ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8 - ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8 - ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp - ;CHECK-NEXT: tSTRspi $r0, $sp, 2, 14 /* CC::al */, $noreg - ;CHECK-NEXT: tSTRspi $r0, $sp, 6, 14 /* CC::al */, $noreg - ;CHECK-NEXT: tSTRspi $r0, $sp, 255, 14 /* CC::al */, $noreg - ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg + ;CHECK: $r0 = t2LDRi12 $sp, 0, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4, 14 /* CC::al */, $noreg + ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4086, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $noreg + + ;CHECK: name: OUTLINED_FUNCTION_[[SHARED]] + ;CHECK: $r0 = tMOVr killed $r1, 14 /* CC::al */, $noreg + ;CHECK-NEXT: tTAILJMPdND @foo, 14 /* CC::al */, $noreg, implicit $sp diff --git a/llvm/test/CodeGen/RISCV/machineoutliner-pcrel-lo.mir b/llvm/test/CodeGen/RISCV/machineoutliner-pcrel-lo.mir index 34f7a93b6a168c..8a83543b0280fd 100644 --- a/llvm/test/CodeGen/RISCV/machineoutliner-pcrel-lo.mir +++ b/llvm/test/CodeGen/RISCV/machineoutliner-pcrel-lo.mir @@ -1,11 +1,11 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=riscv32 -x mir -run-pass=machine-outliner -simplify-mir -verify-machineinstrs < %s \ +# RUN: llc -mtriple=riscv32 -x mir -run-pass=machine-outliner -simplify-mir -verify-machineinstrs -outliner-leaf-descendants=false < %s \ # RUN: | FileCheck %s -# RUN: llc -mtriple=riscv64 -x mir -run-pass=machine-outliner -simplify-mir -verify-machineinstrs < %s \ +# RUN: llc -mtriple=riscv64 -x mir -run-pass=machine-outliner -simplify-mir -verify-machineinstrs -outliner-leaf-descendants=false < %s \ # RUN: | FileCheck %s -# RUN: llc -mtriple=riscv32 -x mir -run-pass=machine-outliner -simplify-mir --function-sections -verify-machineinstrs < %s \ +# RUN: llc -mtriple=riscv32 -x mir -run-pass=machine-outliner -simplify-mir --function-sections -verify-machineinstrs -outliner-leaf-descendants=false < %s \ # RUN: | FileCheck -check-prefix=CHECK-FS %s -# RUN: llc -mtriple=riscv64 -x mir -run-pass=machine-outliner -simplify-mir --function-sections -verify-machineinstrs < %s \ +# RUN: llc -mtriple=riscv64 -x mir -run-pass=machine-outliner -simplify-mir --function-sections -verify-machineinstrs -outliner-leaf-descendants=false < %s \ # RUN: | FileCheck -check-prefix=CHECK-FS %s --- | diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py index 4c05317036d1a3..cc7e9d535a9c33 100644 --- a/llvm/test/lit.cfg.py +++ b/llvm/test/lit.cfg.py @@ -180,6 +180,7 @@ def get_asan_rtlib(): "llvm-addr2line", "llvm-bcanalyzer", "llvm-bitcode-strip", + "llvm-cgdata", "llvm-config", "llvm-cov", "llvm-cxxdump", diff --git a/llvm/test/tools/llvm-cgdata/dump.test b/llvm/test/tools/llvm-cgdata/dump.test new file mode 100644 index 00000000000000..ce2ad27a5ff81c --- /dev/null +++ b/llvm/test/tools/llvm-cgdata/dump.test @@ -0,0 +1,30 @@ +# Test dump between the binary and text formats. + +RUN: split-file %s %t + +RUN: llvm-cgdata dump -binary %t/dump.cgtext -o %t/dump.cgdata +RUN: llvm-cgdata dump -text %t/dump.cgdata -o %t/dump-round.cgtext +RUN: llvm-cgdata dump -binary %t/dump-round.cgtext -o %t/dump-round.cgdata +RUN: diff %t/dump.cgdata %t/dump-round.cgdata + +;--- dump.cgtext +# Outlined stable hash tree +:outlined_hash_tree +--- +0: + Hash: 0x0 + Terminals: 0 + SuccessorIds: [ 1 ] +1: + Hash: 0x1 + Terminals: 0 + SuccessorIds: [ 2, 3 ] +2: + Hash: 0x3 + Terminals: 5 + SuccessorIds: [ ] +3: + Hash: 0x2 + Terminals: 4 + SuccessorIds: [ ] +... diff --git a/llvm/test/tools/llvm-cgdata/empty.test b/llvm/test/tools/llvm-cgdata/empty.test new file mode 100644 index 00000000000000..d5e201b9eec17f --- /dev/null +++ b/llvm/test/tools/llvm-cgdata/empty.test @@ -0,0 +1,32 @@ +# Test for empty cgdata file, which is invalid. +RUN: touch %t_emptyfile.cgtext +RUN: not llvm-cgdata dump %t_emptyfile.cgtext -text -o - 2>&1 | FileCheck %s --check-prefix ERROR +ERROR: {{.}}emptyfile.cgtext: empty codegen data + +# Test for empty header in the text format. It can be converted to a valid binary file. +RUN: printf '#' > %t_emptyheader.cgtext +RUN: llvm-cgdata dump %t_emptyheader.cgtext -binary -o %t_emptyheader.cgdata + +# Without any cgdata other than the header, no data shows by default. +RUN: llvm-cgdata show %t_emptyheader.cgdata | FileCheck %s --allow-empty --check-prefix EMPTY +EMPTY-NOT: any + +# The version number appears when asked, as it's in the header +RUN: llvm-cgdata show --cgdata-version %t_emptyheader.cgdata | FileCheck %s --check-prefix VERSION +VERSION: Version: {{.}} + +# When converting a binary file (w/ the header only) to a text file, it's an empty file as the text format does not have an explicit header. +RUN: llvm-cgdata dump %t_emptyheader.cgdata -text -o - | FileCheck %s --allow-empty --check-prefix EMPTY + +# Synthesize a header only cgdata. +# struct Header { +# uint64_t Magic; +# uint32_t Version; +# uint32_t DataKind; +# uint64_t OutlinedHashTreeOffset; +# } +RUN: printf '\xffcgdata\x81' > %t_header.cgdata +RUN: printf '\x01\x00\x00\x00' >> %t_header.cgdata +RUN: printf '\x00\x00\x00\x00' >> %t_header.cgdata +RUN: printf '\x18\x00\x00\x00\x00\x00\x00\x00' >> %t_header.cgdata +RUN: diff %t_header.cgdata %t_emptyheader.cgdata diff --git a/llvm/test/tools/llvm-cgdata/error.test b/llvm/test/tools/llvm-cgdata/error.test new file mode 100644 index 00000000000000..5e1b14de5e509d --- /dev/null +++ b/llvm/test/tools/llvm-cgdata/error.test @@ -0,0 +1,38 @@ +# Test various error cases + +# Synthesize a header only cgdata. +# struct Header { +# uint64_t Magic; +# uint32_t Version; +# uint32_t DataKind; +# uint64_t OutlinedHashTreeOffset; +# } +RUN: touch %t_empty.cgdata +RUN: not llvm-cgdata show %t_empty.cgdata 2>&1 | FileCheck %s --check-prefix EMPTY +EMPTY: {{.}}cgdata: empty codegen data + +# Not a magic. +RUN: printf '\xff' > %t_malformed.cgdata +RUN: not llvm-cgdata show %t_malformed.cgdata 2>&1 | FileCheck %s --check-prefix MALFORMED +MALFORMED: {{.}}cgdata: malformed codegen data + +# The minimum header size is 24. +RUN: printf '\xffcgdata\x81' > %t_corrupt.cgdata +RUN: not llvm-cgdata show %t_corrupt.cgdata 2>&1 | FileCheck %s --check-prefix CORRUPT +CORRUPT: {{.}}cgdata: invalid codegen data (file header is corrupt) + +# The current version 1 while the header says 2. +RUN: printf '\xffcgdata\x81' > %t_version.cgdata +RUN: printf '\x02\x00\x00\x00' >> %t_version.cgdata +RUN: printf '\x00\x00\x00\x00' >> %t_version.cgdata +RUN: printf '\x18\x00\x00\x00\x00\x00\x00\x00' >> %t_version.cgdata +RUN: not llvm-cgdata show %t_version.cgdata 2>&1 | FileCheck %s --check-prefix BAD_VERSION +BAD_VERSION: {{.}}cgdata: unsupported codegen data version + +# Header says an outlined hash tree, but the file ends after the header. +RUN: printf '\xffcgdata\x81' > %t_eof.cgdata +RUN: printf '\x01\x00\x00\x00' >> %t_eof.cgdata +RUN: printf '\x01\x00\x00\x00' >> %t_eof.cgdata +RUN: printf '\x18\x00\x00\x00\x00\x00\x00\x00' >> %t_eof.cgdata +RUN: not llvm-cgdata show %t_eof.cgdata 2>&1 | FileCheck %s --check-prefix EOF +EOF: {{.}}cgdata: end of File diff --git a/llvm/test/tools/llvm-cgdata/merge-archive.test b/llvm/test/tools/llvm-cgdata/merge-archive.test new file mode 100644 index 00000000000000..a27d6c2a16f4ab --- /dev/null +++ b/llvm/test/tools/llvm-cgdata/merge-archive.test @@ -0,0 +1,75 @@ +# Merge an archive that has two object files having cgdata (__llvm_outline) + +RUN: split-file %s %t + +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-1.ll -o %t/merge-1.o +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-2.ll -o %t/merge-2.o +RUN: llvm-ar rcs %t/merge-archive.a %t/merge-1.o %t/merge-2.o +RUN: llvm-cgdata merge %t/merge-archive.a -o %t/merge-archive.cgdata +RUN: llvm-cgdata show %t/merge-archive.cgdata | FileCheck %s +CHECK: Outlined hash tree: +CHECK-NEXT: Total Node Count: 4 +CHECK-NEXT: Terminal Node Count: 2 +CHECK-NEXT: Depth: 2 + +RUN: llvm-cgdata dump %t/merge-archive.cgdata | FileCheck %s --check-prefix TREE +TREE: # Outlined stable hash tree +TREE-NEXT: :outlined_hash_tree +TREE-NEXT: --- +TREE-NEXT: 0: +TREE-NEXT: Hash: 0x0 +TREE-NEXT: Terminals: 0 +TREE-NEXT: SuccessorIds: [ 1 ] +TREE-NEXT: 1: +TREE-NEXT: Hash: 0x1 +TREE-NEXT: Terminals: 0 +TREE-NEXT: SuccessorIds: [ 2, 3 ] +TREE-NEXT: 2: +TREE-NEXT: Hash: 0x3 +TREE-NEXT: Terminals: 5 +TREE-NEXT: SuccessorIds: [ ] +TREE-NEXT: 3: +TREE-NEXT: Hash: 0x2 +TREE-NEXT: Terminals: 4 +TREE-NEXT: SuccessorIds: [ ] +TREE-NEXT: ... + +;--- merge-1.ll + +; The .data is encoded in a binary form based on the following yaml form. See serialize() in OutlinedHashTreeRecord.cpp +;--- +;0: +; Hash: 0x0 +; Terminals: 0 +; SuccessorIds: [ 1 ] +;1: +; Hash: 0x1 +; Terminals: 0 +; SuccessorIds: [ 2 ] +;2: +; Hash: 0x2 +; Terminals: 4 +; SuccessorIds: [ ] +;... + +@.data = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\02\00\00\00\00\00\00\00\04\00\00\00\00\00\00\00", section "__DATA,__llvm_outline" + +;--- merge-2.ll + +; The .data is encoded in a binary form based on the following yaml form. See serialize() in OutlinedHashTreeRecord.cpp +;--- +;0: +; Hash: 0x0 +; Terminals: 0 +; SuccessorIds: [ 1 ] +;1: +; Hash: 0x1 +; Terminals: 0 +; SuccessorIds: [ 2 ] +;2: +; Hash: 0x3 +; Terminals: 5 +; SuccessorIds: [ ] +;... + +@.data = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\03\00\00\00\00\00\00\00\05\00\00\00\00\00\00\00", section "__DATA,__llvm_outline" diff --git a/llvm/test/tools/llvm-cgdata/merge-concat.test b/llvm/test/tools/llvm-cgdata/merge-concat.test new file mode 100644 index 00000000000000..3411133cb7aacb --- /dev/null +++ b/llvm/test/tools/llvm-cgdata/merge-concat.test @@ -0,0 +1,68 @@ +# Merge a binary file (e.g., a linked executable) having concatnated cgdata (__llvm_outline) + +RUN: split-file %s %t + +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-concat.ll -o %t/merge-concat.o +RUN: llvm-cgdata merge %t/merge-concat.o -o %t/merge-concat.cgdata +RUN: llvm-cgdata show %t/merge-concat.cgdata | FileCheck %s +CHECK: Outlined hash tree: +CHECK-NEXT: Total Node Count: 4 +CHECK-NEXT: Terminal Node Count: 2 +CHECK-NEXT: Depth: 2 + +RUN: llvm-cgdata dump %t/merge-concat.cgdata | FileCheck %s --check-prefix TREE +TREE: # Outlined stable hash tree +TREE-NEXT: :outlined_hash_tree +TREE-NEXT: --- +TREE-NEXT: 0: +TREE-NEXT: Hash: 0x0 +TREE-NEXT: Terminals: 0 +TREE-NEXT: SuccessorIds: [ 1 ] +TREE-NEXT: 1: +TREE-NEXT: Hash: 0x1 +TREE-NEXT: Terminals: 0 +TREE-NEXT: SuccessorIds: [ 2, 3 ] +TREE-NEXT: 2: +TREE-NEXT: Hash: 0x3 +TREE-NEXT: Terminals: 5 +TREE-NEXT: SuccessorIds: [ ] +TREE-NEXT: 3: +TREE-NEXT: Hash: 0x2 +TREE-NEXT: Terminals: 4 +TREE-NEXT: SuccessorIds: [ ] +TREE-NEXT: ... + +;--- merge-concat.ll + +; In an linked executable (as opposed to an object file), cgdata in __llvm_outline might be concatenated. Although this is not a typical workflow, we simply support this case to parse cgdata that is concatenated. In other word, the following two trees are encoded back-to-back in a binary format. +;--- +;0: +; Hash: 0x0 +; Terminals: 0 +; SuccessorIds: [ 1 ] +;1: +; Hash: 0x1 +; Terminals: 0 +; SuccessorIds: [ 2 ] +;2: +; Hash: 0x2 +; Terminals: 4 +; SuccessorIds: [ ] +;... +;--- +;0: +; Hash: 0x0 +; Terminals: 0 +; SuccessorIds: [ 1 ] +;1: +; Hash: 0x1 +; Terminals: 0 +; SuccessorIds: [ 2 ] +;2: +; Hash: 0x3 +; Terminals: 5 +; SuccessorIds: [ ] +;... + +@.data1 = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\02\00\00\00\00\00\00\00\04\00\00\00\00\00\00\00", section "__DATA,__llvm_outline" +@.data2 = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\03\00\00\00\00\00\00\00\05\00\00\00\00\00\00\00", section "__DATA,__llvm_outline" diff --git a/llvm/test/tools/llvm-cgdata/merge-double.test b/llvm/test/tools/llvm-cgdata/merge-double.test new file mode 100644 index 00000000000000..6ce358cd72325b --- /dev/null +++ b/llvm/test/tools/llvm-cgdata/merge-double.test @@ -0,0 +1,74 @@ +# Merge two object files having cgdata (__llvm_outline) + +RUN: split-file %s %t + +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-1.ll -o %t/merge-1.o +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-2.ll -o %t/merge-2.o +RUN: llvm-cgdata merge %t/merge-1.o %t/merge-2.o -o %t/merge.cgdata +RUN: llvm-cgdata show %t/merge.cgdata | FileCheck %s +CHECK: Outlined hash tree: +CHECK-NEXT: Total Node Count: 4 +CHECK-NEXT: Terminal Node Count: 2 +CHECK-NEXT: Depth: 2 + +RUN: llvm-cgdata dump %t/merge.cgdata | FileCheck %s --check-prefix TREE +TREE: # Outlined stable hash tree +TREE-NEXT: :outlined_hash_tree +TREE-NEXT: --- +TREE-NEXT: 0: +TREE-NEXT: Hash: 0x0 +TREE-NEXT: Terminals: 0 +TREE-NEXT: SuccessorIds: [ 1 ] +TREE-NEXT: 1: +TREE-NEXT: Hash: 0x1 +TREE-NEXT: Terminals: 0 +TREE-NEXT: SuccessorIds: [ 2, 3 ] +TREE-NEXT: 2: +TREE-NEXT: Hash: 0x3 +TREE-NEXT: Terminals: 5 +TREE-NEXT: SuccessorIds: [ ] +TREE-NEXT: 3: +TREE-NEXT: Hash: 0x2 +TREE-NEXT: Terminals: 4 +TREE-NEXT: SuccessorIds: [ ] +TREE-NEXT: ... + +;--- merge-1.ll + +; The .data is encoded in a binary form based on the following yaml form. See serialize() in OutlinedHashTreeRecord.cpp +;--- +;0: +; Hash: 0x0 +; Terminals: 0 +; SuccessorIds: [ 1 ] +;1: +; Hash: 0x1 +; Terminals: 0 +; SuccessorIds: [ 2 ] +;2: +; Hash: 0x2 +; Terminals: 4 +; SuccessorIds: [ ] +;... + +@.data = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\02\00\00\00\00\00\00\00\04\00\00\00\00\00\00\00", section "__DATA,__llvm_outline" + +;--- merge-2.ll + +; The .data is encoded in a binary form based on the following yaml form. See serialize() in OutlinedHashTreeRecord.cpp +;--- +;0: +; Hash: 0x0 +; Terminals: 0 +; SuccessorIds: [ 1 ] +;1: +; Hash: 0x1 +; Terminals: 0 +; SuccessorIds: [ 2 ] +;2: +; Hash: 0x3 +; Terminals: 5 +; SuccessorIds: [ ] +;... + +@.data = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\03\00\00\00\00\00\00\00\05\00\00\00\00\00\00\00", section "__DATA,__llvm_outline" diff --git a/llvm/test/tools/llvm-cgdata/merge-single.test b/llvm/test/tools/llvm-cgdata/merge-single.test new file mode 100644 index 00000000000000..73bdd9800dbe1d --- /dev/null +++ b/llvm/test/tools/llvm-cgdata/merge-single.test @@ -0,0 +1,43 @@ +# Test merge a single object file into a cgdata + +RUN: split-file %s %t + +# Merge an object file that has no cgdata (__llvm_outline). It still produces a header only cgdata. +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-empty.ll -o %t/merge-empty.o +RUN: llvm-cgdata merge %t/merge-empty.o -o %t/merge-empty.cgdata +RUN: llvm-cgdata show %t/merge-empty.cgdata | FileCheck %s --allow-empty --check-prefix EMPTY +EMPTY-NOT: any + + +# Merge an object file having cgdata (__llvm_outline) +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-single.ll -o %t/merge-single.o +RUN: llvm-cgdata merge %t/merge-single.o -o %t/merge-single.cgdata +RUN: llvm-cgdata show %t/merge-single.cgdata | FileCheck %s +CHECK: Outlined hash tree: +CHECK-NEXT: Total Node Count: 3 +CHECK-NEXT: Terminal Node Count: 1 +CHECK-NEXT: Depth: 2 + +;--- merge-empty.ll +@.data = private unnamed_addr constant [1 x i8] c"\01" + +;--- merge-single.ll + +; The .data is encoded in a binary form based on the following yaml form. See serialize() in OutlinedHashTreeRecord.cpp +;--- +;0: +; Hash: 0x0 +; Terminals: 0 +; SuccessorIds: [ 1 ] +;1: +; Hash: 0x1 +; Terminals: 0 +; SuccessorIds: [ 2 ] +;2: +; Hash: 0x2 +; Terminals: 4 +; SuccessorIds: [ ] +;... + +@.data = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\02\00\00\00\00\00\00\00\04\00\00\00\00\00\00\00", section "__DATA,__llvm_outline" + diff --git a/llvm/test/tools/llvm-cgdata/show.test b/llvm/test/tools/llvm-cgdata/show.test new file mode 100644 index 00000000000000..accb4b77ede246 --- /dev/null +++ b/llvm/test/tools/llvm-cgdata/show.test @@ -0,0 +1,30 @@ +# Test show + +RUN: split-file %s %t +RUN: llvm-cgdata show %t/show.cgtext | FileCheck %s + +CHECK: Outlined hash tree: +CHECK-NEXT: Total Node Count: 3 +CHECK-NEXT: Terminal Node Count: 1 +CHECK-NEXT: Depth: 2 + +# Convert the text file to the binary file +RUN: llvm-cgdata dump -binary %t/show.cgtext -o %t/show.cgdata +RUN: llvm-cgdata show %t/show.cgdata | FileCheck %s + +;--- show.cgtext +:outlined_hash_tree +--- +0: + Hash: 0x0 + Terminals: 0 + SuccessorIds: [ 1 ] +1: + Hash: 0x1 + Terminals: 0 + SuccessorIds: [ 2 ] +2: + Hash: 0x2 + Terminals: 3 + SuccessorIds: [ ] +... diff --git a/llvm/tools/llvm-cgdata/CMakeLists.txt b/llvm/tools/llvm-cgdata/CMakeLists.txt new file mode 100644 index 00000000000000..4f1f7ff635bc3c --- /dev/null +++ b/llvm/tools/llvm-cgdata/CMakeLists.txt @@ -0,0 +1,15 @@ +set(LLVM_LINK_COMPONENTS + CodeGen + CodeGenData + Core + Object + Support + ) + +add_llvm_tool(llvm-cgdata + llvm-cgdata.cpp + + DEPENDS + intrinsics_gen + GENERATE_DRIVER + ) diff --git a/llvm/tools/llvm-cgdata/llvm-cgdata.cpp b/llvm/tools/llvm-cgdata/llvm-cgdata.cpp new file mode 100644 index 00000000000000..195f066fd6b872 --- /dev/null +++ b/llvm/tools/llvm-cgdata/llvm-cgdata.cpp @@ -0,0 +1,268 @@ +//===-- llvm-cgdata.cpp - LLVM CodeGen Data Tool --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// llvm-cgdata parses raw codegen data embedded in compiled binary files, and +// merges them into a single .cgdata file. It can also inspect and maninuplate +// a .cgdata file. This .cgdata can contain various codegen data like outlining +// information, and it can be used to optimize the code in the subsequent build. +// +//===----------------------------------------------------------------------===// +#include "llvm/ADT/StringRef.h" +#include "llvm/CodeGenData/CodeGenDataReader.h" +#include "llvm/CodeGenData/CodeGenDataWriter.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/Object/Archive.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/LLVMDriver.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/VirtualFileSystem.h" +#include "llvm/Support/WithColor.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace llvm::object; + +// TODO: https://llvm.org/docs/CommandGuide/llvm-cgdata.html has documentations +// on each subcommand. +cl::SubCommand DumpSubcommand( + "dump", + "Dump the (indexed) codegen data file in either text or binary format."); +cl::SubCommand MergeSubcommand( + "merge", "Takes binary files having raw codegen data in custom sections, " + "and merge them into an index codegen data file."); +cl::SubCommand + ShowSubcommand("show", "Show summary of the (indexed) codegen data file."); + +enum CGDataFormat { + CD_None = 0, + CD_Text, + CD_Binary, +}; + +cl::opt OutputFilename("output", cl::value_desc("output"), + cl::init("-"), cl::desc("Output file"), + cl::sub(DumpSubcommand), + cl::sub(MergeSubcommand)); +cl::alias OutputFilenameA("o", cl::desc("Alias for --output"), + cl::aliasopt(OutputFilename)); + +cl::opt Filename(cl::Positional, cl::desc(""), + cl::sub(DumpSubcommand), cl::sub(ShowSubcommand)); +cl::list InputFilenames(cl::Positional, cl::sub(MergeSubcommand), + cl::desc("")); +cl::opt OutputFormat( + cl::desc("Format of output data"), cl::sub(DumpSubcommand), + cl::init(CD_Text), + cl::values(clEnumValN(CD_Text, "text", "Text encoding"), + clEnumValN(CD_Binary, "binary", "Binary encoding"))); + +cl::opt ShowCGDataVersion("cgdata-version", cl::init(false), + cl::desc("Show cgdata version. "), + cl::sub(ShowSubcommand)); + +static void exitWithError(Twine Message, std::string Whence = "", + std::string Hint = "") { + WithColor::error(); + if (!Whence.empty()) + errs() << Whence << ": "; + errs() << Message << "\n"; + if (!Hint.empty()) + WithColor::note() << Hint << "\n"; + ::exit(1); +} + +static void exitWithError(Error E, StringRef Whence = "") { + if (E.isA()) { + handleAllErrors(std::move(E), [&](const CGDataError &IPE) { + exitWithError(IPE.message(), std::string(Whence)); + }); + return; + } + + exitWithError(toString(std::move(E)), std::string(Whence)); +} + +static void exitWithErrorCode(std::error_code EC, StringRef Whence = "") { + exitWithError(EC.message(), std::string(Whence)); +} + +static int dump_main(int argc, const char *argv[]) { + if (Filename == OutputFilename) { + errs() << sys::path::filename(argv[0]) << " " << argv[1] + << ": Input file name cannot be the same as the output file name!\n"; + return 1; + } + + std::error_code EC; + raw_fd_ostream OS(OutputFilename.data(), EC, + OutputFormat == CD_Text ? sys::fs::OF_TextWithCRLF + : sys::fs::OF_None); + if (EC) + exitWithErrorCode(EC, OutputFilename); + + auto FS = vfs::getRealFileSystem(); + auto ReaderOrErr = CodeGenDataReader::create(Filename, *FS); + if (Error E = ReaderOrErr.takeError()) + exitWithError(std::move(E), Filename); + + CodeGenDataWriter Writer; + auto Reader = ReaderOrErr->get(); + if (Reader->hasOutlinedHashTree()) { + OutlinedHashTreeRecord Record(Reader->releaseOutlinedHashTree()); + Writer.addRecord(Record); + } + + if (OutputFormat == CD_Text) { + if (Error E = Writer.writeText(OS)) + exitWithError(std::move(E)); + } else { + if (Error E = Writer.write(OS)) + exitWithError(std::move(E)); + } + + return 0; +} + +static bool handleBuffer(StringRef Filename, MemoryBufferRef Buffer, + OutlinedHashTreeRecord &GlobalOutlineRecord); + +static bool handleArchive(StringRef Filename, Archive &Arch, + OutlinedHashTreeRecord &GlobalOutlineRecord) { + bool Result = true; + Error Err = Error::success(); + for (const auto &Child : Arch.children(Err)) { + auto BuffOrErr = Child.getMemoryBufferRef(); + if (Error E = BuffOrErr.takeError()) + exitWithError(std::move(E), Filename); + auto NameOrErr = Child.getName(); + if (Error E = NameOrErr.takeError()) + exitWithError(std::move(E), Filename); + std::string Name = (Filename + "(" + NameOrErr.get() + ")").str(); + Result &= handleBuffer(Name, BuffOrErr.get(), GlobalOutlineRecord); + } + if (Err) + exitWithError(std::move(Err), Filename); + return Result; +} + +static bool handleBuffer(StringRef Filename, MemoryBufferRef Buffer, + OutlinedHashTreeRecord &GlobalOutlineRecord) { + Expected> BinOrErr = object::createBinary(Buffer); + if (Error E = BinOrErr.takeError()) + exitWithError(std::move(E), Filename); + + bool Result = true; + if (auto *Obj = dyn_cast(BinOrErr->get())) { + if (Error E = + CodeGenDataReader::mergeFromObjectFile(Obj, GlobalOutlineRecord)) + exitWithError(std::move(E), Filename); + } else if (auto *Arch = dyn_cast(BinOrErr->get())) { + Result &= handleArchive(Filename, *Arch, GlobalOutlineRecord); + } else { + // TODO: Support for the MachO universal binary format. + errs() << "Error: unsupported binary file: " << Filename << "\n"; + Result = false; + } + + return Result; +} + +static bool handleFile(StringRef Filename, + OutlinedHashTreeRecord &GlobalOutlineRecord) { + ErrorOr> BuffOrErr = + MemoryBuffer::getFileOrSTDIN(Filename); + if (std::error_code EC = BuffOrErr.getError()) + exitWithErrorCode(EC, Filename); + return handleBuffer(Filename, *BuffOrErr.get(), GlobalOutlineRecord); +} + +static int merge_main(int argc, const char *argv[]) { + bool Result = true; + OutlinedHashTreeRecord GlobalOutlineRecord; + for (auto &Filename : InputFilenames) + Result &= handleFile(Filename, GlobalOutlineRecord); + + if (!Result) { + errs() << "Error: failed to merge codegen data files.\n"; + return 1; + } + + CodeGenDataWriter Writer; + if (!GlobalOutlineRecord.empty()) + Writer.addRecord(GlobalOutlineRecord); + + std::error_code EC; + raw_fd_ostream Output(OutputFilename, EC, sys::fs::OF_None); + if (EC) + exitWithErrorCode(EC, OutputFilename); + + if (auto E = Writer.write(Output)) + exitWithError(std::move(E)); + + return 0; +} + +static int show_main(int argc, const char *argv[]) { + if (Filename == OutputFilename) { + errs() << sys::path::filename(argv[0]) << " " << argv[1] + << ": Input file name cannot be the same as the output file name!\n"; + return 1; + } + + std::error_code EC; + raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF); + if (EC) + exitWithErrorCode(EC, OutputFilename); + + auto FS = vfs::getRealFileSystem(); + auto ReaderOrErr = CodeGenDataReader::create(Filename, *FS); + if (Error E = ReaderOrErr.takeError()) + exitWithError(std::move(E), Filename); + + auto Reader = ReaderOrErr->get(); + if (ShowCGDataVersion) + OS << "Version: " << Reader->getVersion() << "\n"; + + if (Reader->hasOutlinedHashTree()) { + auto Tree = Reader->releaseOutlinedHashTree(); + OS << "Outlined hash tree:\n"; + OS << " Total Node Count: " << Tree->size() << "\n"; + OS << " Terminal Node Count: " << Tree->size(/*GetTerminalCountOnly=*/true) + << "\n"; + OS << " Depth: " << Tree->depth() << "\n"; + } + + return 0; +} + +int llvm_cgdata_main(int argc, char **argvNonConst, const llvm::ToolContext &) { + const char **argv = const_cast(argvNonConst); + + StringRef ProgName(sys::path::filename(argv[0])); + + if (argc < 2) { + errs() << ProgName + << ": No subcommand specified! Run llvm-cgdata --help for usage.\n"; + return 1; + } + + cl::ParseCommandLineOptions(argc, argv, "LLVM codegen data\n"); + + if (DumpSubcommand) + return dump_main(argc, argv); + + if (MergeSubcommand) + return merge_main(argc, argv); + + if (ShowSubcommand) + return show_main(argc, argv); + + errs() << ProgName + << ": Unknown command. Run llvm-cgdata --help for usage.\n"; + return 1; +} diff --git a/llvm/unittests/CMakeLists.txt b/llvm/unittests/CMakeLists.txt index 46f30ff398e10d..cb4b8513e6d02e 100644 --- a/llvm/unittests/CMakeLists.txt +++ b/llvm/unittests/CMakeLists.txt @@ -21,6 +21,7 @@ add_subdirectory(BinaryFormat) add_subdirectory(Bitcode) add_subdirectory(Bitstream) add_subdirectory(CodeGen) +add_subdirectory(CodeGenData) add_subdirectory(DebugInfo) add_subdirectory(Debuginfod) add_subdirectory(Demangle) diff --git a/llvm/unittests/CodeGenData/CMakeLists.txt b/llvm/unittests/CodeGenData/CMakeLists.txt new file mode 100644 index 00000000000000..3d821b87e29d8c --- /dev/null +++ b/llvm/unittests/CodeGenData/CMakeLists.txt @@ -0,0 +1,14 @@ +set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + CodeGen + CodeGenData + Core + Support + ) + +add_llvm_unittest(CodeGenDataTests + OutlinedHashTreeRecordTest.cpp + OutlinedHashTreeTest.cpp + ) + +target_link_libraries(CodeGenDataTests PRIVATE LLVMTestingSupport) diff --git a/llvm/unittests/CodeGenData/OutlinedHashTreeRecordTest.cpp b/llvm/unittests/CodeGenData/OutlinedHashTreeRecordTest.cpp new file mode 100644 index 00000000000000..aa7ad4a33754ff --- /dev/null +++ b/llvm/unittests/CodeGenData/OutlinedHashTreeRecordTest.cpp @@ -0,0 +1,118 @@ +//===- OutlinedHashTreeRecordTest.cpp -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGenData/OutlinedHashTreeRecord.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace llvm; + +namespace { + +TEST(OutlinedHashTreeRecordTest, Empty) { + OutlinedHashTreeRecord HashTreeRecord; + ASSERT_TRUE(HashTreeRecord.empty()); +} + +TEST(OutlinedHashTreeRecordTest, Print) { + OutlinedHashTreeRecord HashTreeRecord; + HashTreeRecord.HashTree->insert({{1, 2}, 3}); + + const char *ExpectedTreeStr = R"(--- +0: + Hash: 0x0 + Terminals: 0 + SuccessorIds: [ 1 ] +1: + Hash: 0x1 + Terminals: 0 + SuccessorIds: [ 2 ] +2: + Hash: 0x2 + Terminals: 3 + SuccessorIds: [ ] +... +)"; + std::string TreeDump; + raw_string_ostream OS(TreeDump); + HashTreeRecord.print(OS); + EXPECT_EQ(ExpectedTreeStr, TreeDump); +} + +TEST(OutlinedHashTreeRecordTest, Stable) { + OutlinedHashTreeRecord HashTreeRecord1; + HashTreeRecord1.HashTree->insert({{1, 2}, 4}); + HashTreeRecord1.HashTree->insert({{1, 3}, 5}); + + OutlinedHashTreeRecord HashTreeRecord2; + HashTreeRecord2.HashTree->insert({{1, 3}, 5}); + HashTreeRecord2.HashTree->insert({{1, 2}, 4}); + + // Output is stable regardless of insertion order. + std::string TreeDump1; + raw_string_ostream OS1(TreeDump1); + HashTreeRecord1.print(OS1); + std::string TreeDump2; + raw_string_ostream OS2(TreeDump2); + HashTreeRecord2.print(OS2); + + EXPECT_EQ(TreeDump1, TreeDump2); +} + +TEST(OutlinedHashTreeRecordTest, Serialize) { + OutlinedHashTreeRecord HashTreeRecord1; + HashTreeRecord1.HashTree->insert({{1, 2}, 4}); + HashTreeRecord1.HashTree->insert({{1, 3}, 5}); + + // Serialize and deserialize the tree. + SmallVector Out; + raw_svector_ostream OS(Out); + HashTreeRecord1.serialize(OS); + + OutlinedHashTreeRecord HashTreeRecord2; + const uint8_t *Data = reinterpret_cast(Out.data()); + HashTreeRecord2.deserialize(Data); + + // Two trees should be identical. + std::string TreeDump1; + raw_string_ostream OS1(TreeDump1); + HashTreeRecord1.print(OS1); + std::string TreeDump2; + raw_string_ostream OS2(TreeDump2); + HashTreeRecord2.print(OS2); + + EXPECT_EQ(TreeDump1, TreeDump2); +} + +TEST(OutlinedHashTreeRecordTest, SerializeYAML) { + OutlinedHashTreeRecord HashTreeRecord1; + HashTreeRecord1.HashTree->insert({{1, 2}, 4}); + HashTreeRecord1.HashTree->insert({{1, 3}, 5}); + + // Serialize and deserialize the tree in a YAML format. + std::string Out; + raw_string_ostream OS(Out); + yaml::Output YOS(OS); + HashTreeRecord1.serializeYAML(YOS); + + OutlinedHashTreeRecord HashTreeRecord2; + yaml::Input YIS(StringRef(Out.data(), Out.size())); + HashTreeRecord2.deserializeYAML(YIS); + + // Two trees should be identical. + std::string TreeDump1; + raw_string_ostream OS1(TreeDump1); + HashTreeRecord1.print(OS1); + std::string TreeDump2; + raw_string_ostream OS2(TreeDump2); + HashTreeRecord2.print(OS2); + + EXPECT_EQ(TreeDump1, TreeDump2); +} + +} // end namespace diff --git a/llvm/unittests/CodeGenData/OutlinedHashTreeTest.cpp b/llvm/unittests/CodeGenData/OutlinedHashTreeTest.cpp new file mode 100644 index 00000000000000..d11618cf8e4fae --- /dev/null +++ b/llvm/unittests/CodeGenData/OutlinedHashTreeTest.cpp @@ -0,0 +1,81 @@ +//===- OutlinedHashTreeTest.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGenData/OutlinedHashTree.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace llvm; + +namespace { + +TEST(OutlinedHashTreeTest, Empty) { + OutlinedHashTree HashTree; + ASSERT_TRUE(HashTree.empty()); + // The header node is always present. + ASSERT_TRUE(HashTree.size() == 1); + ASSERT_TRUE(HashTree.depth() == 0); +} + +TEST(OutlinedHashTreeTest, Insert) { + OutlinedHashTree HashTree; + HashTree.insert({{1, 2, 3}, 1}); + // The node count is 4 (including the root node). + ASSERT_TRUE(HashTree.size() == 4); + // The terminal count is 1. + ASSERT_TRUE(HashTree.size(/*GetTerminalCountOnly=*/true) == 1); + // The depth is 3. + ASSERT_TRUE(HashTree.depth() == 3); + + HashTree.clear(); + ASSERT_TRUE(HashTree.empty()); + + HashTree.insert({{1, 2, 3}, 1}); + HashTree.insert({{1, 2, 4}, 2}); + // The nodes of 1 and 2 are shared with the same prefix. + // The nodes are root, 1, 2, 3 and 4, whose counts are 5. + ASSERT_TRUE(HashTree.size() == 5); +} + +TEST(OutlinedHashTreeTest, Find) { + OutlinedHashTree HashTree; + HashTree.insert({{1, 2, 3}, 1}); + HashTree.insert({{1, 2, 3}, 2}); + + // The node count does not change as the same sequences are added. + ASSERT_TRUE(HashTree.size() == 4); + // The terminal counts are accumulated from two same sequences. + ASSERT_TRUE(HashTree.find({1, 2, 3}) == 3); + ASSERT_TRUE(HashTree.find({1, 2}) == 0); +} + +TEST(OutlinedHashTreeTest, Merge) { + // Build HashTree1 inserting 2 sequences. + OutlinedHashTree HashTree1; + + HashTree1.insert({{1, 2}, 20}); + HashTree1.insert({{1, 4}, 30}); + + // Build HashTree2 and HashTree3 for each + OutlinedHashTree HashTree2; + HashTree2.insert({{1, 2}, 20}); + OutlinedHashTree HashTree3; + HashTree3.insert({{1, 4}, 30}); + + // Merge HashTree3 into HashTree2. + HashTree2.merge(&HashTree3); + + // Compare HashTree1 and HashTree2. + EXPECT_EQ(HashTree1.size(), HashTree2.size()); + EXPECT_EQ(HashTree1.depth(), HashTree2.depth()); + EXPECT_EQ(HashTree1.find({1, 2}), HashTree2.find({1, 2})); + EXPECT_EQ(HashTree1.find({1, 4}), HashTree2.find({1, 4})); + EXPECT_EQ(HashTree1.find({1, 3}), HashTree2.find({1, 3})); +} + +} // end namespace diff --git a/llvm/unittests/Support/SuffixTreeTest.cpp b/llvm/unittests/Support/SuffixTreeTest.cpp index f5d8112ccf5cec..59f6dc9bf963fa 100644 --- a/llvm/unittests/Support/SuffixTreeTest.cpp +++ b/llvm/unittests/Support/SuffixTreeTest.cpp @@ -140,4 +140,138 @@ TEST(SuffixTreeTest, TestExclusion) { } } +// Tests that the SuffixTree is able to find the following substrings: +// {1, 1} at indices 0, 1, 2, 3, and 4; +// {1, 1, 1} at indices 0, 1, 2, and 3; +// {1, 1, 1, 1} at indices 0, 1, and 2; and +// {1, 1, 1, 1, 1} at indices 0 and 1. +// +// This is a FIX to the Test TestSingleCharacterRepeat +TEST(SuffixTreeTest, TestSingleCharacterRepeatWithLeafDescendants) { + std::vector RepeatedRepetitionData = {1, 1, 1, 1, 1, 1, 2}; + std::vector::iterator RRDIt, RRDIt2; + SuffixTree ST(RepeatedRepetitionData, true); + std::vector SubStrings; + for (auto It = ST.begin(); It != ST.end(); It++) + SubStrings.push_back(*It); + EXPECT_EQ(SubStrings.size(), 4u); + for (SuffixTree::RepeatedSubstring &RS : SubStrings) { + EXPECT_EQ(RS.StartIndices.size(), + RepeatedRepetitionData.size() - RS.Length); + for (unsigned StartIdx : SubStrings[0].StartIndices) { + RRDIt = RRDIt2 = RepeatedRepetitionData.begin(); + std::advance(RRDIt, StartIdx); + std::advance(RRDIt2, StartIdx + SubStrings[0].Length); + ASSERT_TRUE( + all_of(make_range::iterator>(RRDIt, RRDIt2), + [](unsigned Elt) { return Elt == 1; })); + } + } +} + +// Tests that the SuffixTree is able to find three substrings +// {1, 2, 3} at indices 6 and 10; +// {2, 3} at indices 7 and 11; and +// {1, 2} at indicies 0 and 3. +// +// FIXME: {1, 2} has indices 6 and 10 missing as it is a substring of {1, 2, 3} +// See Test TestSubstringRepeatsWithLeafDescendants for the FIX +TEST(SuffixTreeTest, TestSubstringRepeats) { + std::vector RepeatedRepetitionData = {1, 2, 100, 1, 2, 101, 1, + 2, 3, 103, 1, 2, 3, 104}; + SuffixTree ST(RepeatedRepetitionData); + std::vector SubStrings; + for (auto It = ST.begin(); It != ST.end(); It++) + SubStrings.push_back(*It); + EXPECT_EQ(SubStrings.size(), 3u); + unsigned Len; + for (SuffixTree::RepeatedSubstring &RS : SubStrings) { + Len = RS.Length; + bool IsExpectedLen = (Len == 3u || Len == 2u); + ASSERT_TRUE(IsExpectedLen); + bool IsExpectedIndex; + + if (Len == 3u) { // {1, 2, 3} + EXPECT_EQ(RS.StartIndices.size(), 2u); + for (unsigned StartIdx : RS.StartIndices) { + IsExpectedIndex = (StartIdx == 6u || StartIdx == 10u); + EXPECT_TRUE(IsExpectedIndex); + EXPECT_EQ(RepeatedRepetitionData[StartIdx], 1u); + EXPECT_EQ(RepeatedRepetitionData[StartIdx + 1], 2u); + EXPECT_EQ(RepeatedRepetitionData[StartIdx + 2], 3u); + } + } else { + if (RepeatedRepetitionData[RS.StartIndices[0]] == 1u) { // {1, 2} + EXPECT_EQ(RS.StartIndices.size(), 2u); + for (unsigned StartIdx : RS.StartIndices) { + IsExpectedIndex = (StartIdx == 0u || StartIdx == 3u); + EXPECT_TRUE(IsExpectedIndex); + EXPECT_EQ(RepeatedRepetitionData[StartIdx + 1], 2u); + } + } else { // {2, 3} + EXPECT_EQ(RS.StartIndices.size(), 2u); + for (unsigned StartIdx : RS.StartIndices) { + IsExpectedIndex = (StartIdx == 7u || StartIdx == 11u); + EXPECT_TRUE(IsExpectedIndex); + EXPECT_EQ(RepeatedRepetitionData[StartIdx], 2u); + EXPECT_EQ(RepeatedRepetitionData[StartIdx + 1], 3u); + } + } + } + } +} + +// Tests that the SuffixTree is able to find three substrings +// {1, 2, 3} at indices 6 and 10; +// {2, 3} at indices 7 and 11; and +// {1, 2} at indicies 0, 3, 6, and 10. +// +// This is a FIX to the Test TestSubstringRepeats + +TEST(SuffixTreeTest, TestSubstringRepeatsWithLeafDescendants) { + std::vector RepeatedRepetitionData = {1, 2, 100, 1, 2, 101, 1, + 2, 3, 103, 1, 2, 3, 104}; + SuffixTree ST(RepeatedRepetitionData, true); + std::vector SubStrings; + for (auto It = ST.begin(); It != ST.end(); It++) + SubStrings.push_back(*It); + EXPECT_EQ(SubStrings.size(), 3u); + unsigned Len; + for (SuffixTree::RepeatedSubstring &RS : SubStrings) { + Len = RS.Length; + bool IsExpectedLen = (Len == 3u || Len == 2u); + ASSERT_TRUE(IsExpectedLen); + bool IsExpectedIndex; + + if (Len == 3u) { // {1, 2, 3} + EXPECT_EQ(RS.StartIndices.size(), 2u); + for (unsigned StartIdx : RS.StartIndices) { + IsExpectedIndex = (StartIdx == 6u || StartIdx == 10u); + EXPECT_TRUE(IsExpectedIndex); + EXPECT_EQ(RepeatedRepetitionData[StartIdx], 1u); + EXPECT_EQ(RepeatedRepetitionData[StartIdx + 1], 2u); + EXPECT_EQ(RepeatedRepetitionData[StartIdx + 2], 3u); + } + } else { + if (RepeatedRepetitionData[RS.StartIndices[0]] == 1u) { // {1, 2} + EXPECT_EQ(RS.StartIndices.size(), 4u); + for (unsigned StartIdx : RS.StartIndices) { + IsExpectedIndex = (StartIdx == 0u || StartIdx == 3u || + StartIdx == 6u || StartIdx == 10u); + EXPECT_TRUE(IsExpectedIndex); + EXPECT_EQ(RepeatedRepetitionData[StartIdx + 1], 2u); + } + } else { // {2, 3} + EXPECT_EQ(RS.StartIndices.size(), 2u); + for (unsigned StartIdx : RS.StartIndices) { + IsExpectedIndex = (StartIdx == 7u || StartIdx == 11u); + EXPECT_TRUE(IsExpectedIndex); + EXPECT_EQ(RepeatedRepetitionData[StartIdx], 2u); + EXPECT_EQ(RepeatedRepetitionData[StartIdx + 1], 3u); + } + } + } + } +} + } // namespace