diff --git a/ACPO_README.md b/ACPO_README.md new file mode 100644 index 0000000000000000000000000000000000000000..3227af5872802d748fab886c20268e55ca6152e2 --- /dev/null +++ b/ACPO_README.md @@ -0,0 +1,36 @@ +# The AI-Enabled Continuous Program Optimization Infrastructure for LLVM + +Welcome to the ACPO Extension to LLVM! + +LLVM is the most popular open-source compiler framework, with massive community +involvement. With the addition of ACPO framework, we are providing tools +to include ML models for decision making, example models for some passes and +a wide range of feature collectors to make performance-oriented compiler +development easier to perform. + +The changes provided comprise LLVM-side changes to enable the use of ML models +in the LLVM compiler, which are meant to be used in tandem with CPLLab-Huawei/ACPO +repository that comprises models, ML-framework interfaces and data analysis scripts +to make compiler developers more productive. + +## Getting the Source Code and Building LLVM + +Consult the +[Getting Started with LLVM](https://llvm.org/docs/GettingStarted.html#getting-the-source-code-and-building-llvm) +page for information on building and running LLVM. + +For information on how to contribute to the LLVM project, please take a look at +the [Contributing to LLVM](https://llvm.org/docs/Contributing.html) guide. + +## Adding ACPO into your flow + +With the changes provided the LLVM-side of the compiler is already included. To make it work, +you will need to include the CPLLab-Huawei/ACPO added to your build to ensure interface with +ML frameworks and models can be developed, stored and deployed. + +Here are the steps to build and test LLVM with ACPO: +1. Build LLVM with ENABLE_ACPO macro turned on. +2. Include a repo https://github.com/Huawei-CPLLab/ACPO to obtain models and model training infrastructure. +3. Test the flow on existing ACPO-enabled passes, such as inliner to ensure your set up is correct. +4. Begin adding ACPO to passes of your choice. + diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt index 79de9eb2e3e71be225959af472b7b675848258c8..b0afb47a7243067918aaf154612583e1d52051bd 100644 --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -1001,6 +1001,9 @@ endif() # set(TENSORFLOW_AOT_PATH "" CACHE PATH "Path to TensorFlow pip install dir") +set(LLVM_INLINER_MODEL_PATH "" CACHE PATH "Path to the inliner model") +set(ACPO_AOT OFF CACHE BOOL "Whether or not ACPO AOT is enabled") + if (NOT TENSORFLOW_AOT_PATH STREQUAL "") set(LLVM_HAVE_TF_AOT "ON" CACHE BOOL "Tensorflow AOT available") set(TENSORFLOW_AOT_COMPILER @@ -1009,6 +1012,7 @@ if (NOT TENSORFLOW_AOT_PATH STREQUAL "") include_directories(${TENSORFLOW_AOT_PATH}/include) add_subdirectory(${TENSORFLOW_AOT_PATH}/xla_aot_runtime_src ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/tf_runtime) + target_compile_definitions(tf_xla_runtime_objects PUBLIC EIGEN_NEON_GEBP_NR=4) # Fix for issue https://github.com/tensorflow/tensorflow/issues/58481 install(TARGETS tf_xla_runtime EXPORT LLVMExports ARCHIVE DESTINATION lib${LLVM_LIBDIR_SUFFIX} COMPONENT tf_xla_runtime) set_property(GLOBAL APPEND PROPERTY LLVM_EXPORTS tf_xla_runtime) diff --git a/llvm/include/llvm/Analysis/ACPOCollectFeatures.h b/llvm/include/llvm/Analysis/ACPOCollectFeatures.h new file mode 100644 index 0000000000000000000000000000000000000000..8b266b3bc756d53b15ea503462f953a0ac163f1c --- /dev/null +++ b/llvm/include/llvm/Analysis/ACPOCollectFeatures.h @@ -0,0 +1,299 @@ +//===- ACPOCollectFeatures.h - ACPO Class for Feature Collection ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This header file defines the type, scope, and number of features to be +// collected on a given ACPOModel class from all available features. +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#ifndef LLVM_ANALYSIS_ACPOCOLLECTFEATURES_H +#define LLVM_ANALYSIS_ACPOCOLLECTFEATURES_H +#include "llvm/Analysis/InlineAdvisor.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/PassManager.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace llvm { +class ACPOCollectFeatures { +public: + // A feature is related to one of the following scope + enum class Scope { + Module, + Function, + Loop, + Callgraph, + CallSite, + NumOfScope, + }; + + // In the future as more features are added, features can be calculated + // simultanously. + // Suppose feature A and B could be calculated in the same loop, + // then it would make sense to calculate both the features at the same time + // and save it in a cache system + // (which could be implemented similarly like Dumpfeatures.h/cpp). + enum class GroupID { + EdgeNodeCount, + FPIRelated, + HotColdCallSite, + InlineCostFeatureGroup, + ACPOFIExtendedFeatures, + NumOfGroupID + }; + + // List of features we support to be calculated. + // (1) For each feature there should be a corresponding scope on which it + // depends on for calculating. + // (2) A feature may belong in a group for which those features could be + // calculated together. + // (3) Once you decided to add a feature you should register it to all the + // static maps in the .cpp file. Except for some special indicator enum's + // like InlineCostFeatureGroupBegin/End + enum class FeatureIndex { + // Begin: InlineCostFeatureGroup + InlineCostFeatureGroupBegin, + SROASavings, + SROALosses, + LoadElimination, + CallPenalty, + CallArgumentSetup, + LoadRelativeIntrinsic, + LoweredCallArgSetup, + IndirectCallPenalty, + JumpTablePenalty, + CaseClusterPenalty, + SwitchPenalty, + UnsimplifiedCommonInstructions, + NumLoops, + DeadBlocks, + SimplifiedInstructions, + ConstantArgs, + ConstantOffsetPtrArgs, + CallSiteCost, + ColdCcPenalty, + LastCallToStaticBonus, + IsMultipleBlocks, + NestedInlines, + NestedInlineCostEstimate, + Threshold, + InlineCostFeatureGroupEnd, + // End: InlineCostFeatureGroup + + // Begin: FPIRelated + BasicBlockCount, + BlocksReachedFromConditionalInstruction, + Uses, + // End: FPIRelated + + // Begin: EdgeNodeCount + EdgeCount, + NodeCount, + // End: EdgeNodeCount + + // Begin: HotColdCallsite + ColdCallSite, + HotCallSite, + // End: HotColdCallsite + + // Begin: ACPOFIExtendedFeatures + ACPOFIExtendedFeaturesNamedFeatureBegin, + ACPOFIExtendedFeaturesInitialSize, + ACPOFIExtendedFeaturesBlocks, + ACPOFIExtendedFeaturesCalls, + ACPOFIExtendedFeaturesIsLocal, + ACPOFIExtendedFeaturesIsLinkOnceODR, + ACPOFIExtendedFeaturesIsLinkOnce, + ACPOFIExtendedFeaturesLoops, + ACPOFIExtendedFeaturesMaxLoopDepth, + ACPOFIExtendedFeaturesMaxDomTreeLevel, + ACPOFIExtendedFeaturesPtrArgs, + ACPOFIExtendedFeaturesPtrCallee, + ACPOFIExtendedFeaturesCallReturnPtr, + ACPOFIExtendedFeaturesConditionalBranch, + ACPOFIExtendedFeaturesCBwithArg, + ACPOFIExtendedFeaturesCallerHeight, + ACPOFIExtendedFeaturesCallUsage, + ACPOFIExtendedFeaturesIsRecursive, + ACPOFIExtendedFeaturesNumCallsiteInLoop, + ACPOFIExtendedFeaturesNumOfCallUsesInLoop, + ACPOFIExtendedFeaturesEntryBlockFreq, + ACPOFIExtendedFeaturesMaxCallsiteBlockFreq, + ACPOFIExtendedFeaturesNamedFeatureEnd, + ACPOFIExtendedFeaturesFloatFeatureBegin, + ACPOFIExtendedFeaturesInstructionPerBlock, + ACPOFIExtendedFeaturesSuccessorPerBlock, + ACPOFIExtendedFeaturesAvgVecInstr, + ACPOFIExtendedFeaturesAvgNestedLoopLevel, + ACPOFIExtendedFeaturesInstrPerLoop, + ACPOFIExtendedFeaturesBlockWithMultipleSuccecorsPerLoop, + ACPOFIExtendedFeaturesFloatFeatureEnd, + // End: ACPOFIExtendedFeatures + + CallerBlockFreq, + CallSiteHeight, + ConstantParam, + CostEstimate, + LoopLevel, + MandatoryKind, + MandatoryOnly, + OptCode, + IsIndirectCall, + IsInInnerLoop, + IsMustTailCall, + IsTailCall, + NumOfFeatures + }; + + struct AnalysisManagers { + FunctionAnalysisManager *FAM = nullptr; + ModuleAnalysisManager *MAM = nullptr; + }; + + // ScopeInfo is a struct that contains the correpsonding needed information to + // calculate the corresponding feature. + struct ScopeInfo { + Function *F = nullptr; + CallBase *CB = nullptr; + BasicBlock *BB = nullptr; + Module *M = nullptr; + Loop *L = nullptr; + // Can add Instructions or other types later. + }; + + struct OtherInfo { + bool MandatoryOnly = false; + InlineAdvisor *IA = nullptr; + }; + + // FeatureInfo should contain all the relevant information to calculate + // the corresponding FeatureIndex. + struct FeatureInfo { + // When Idx = NumOfFeatures. We assume this is a global FeatureInfo. + FeatureIndex Idx; + // Once we have the Idx we should know the following two attribute. + // Scope ScopeIdx // + // GroupID Group // + AnalysisManagers Managers; + ScopeInfo SI; + OtherInfo OI; + }; + + using FeatureValueMap = std::unordered_map; + using FeatureInfoMap = std::unordered_map; + using FeaturesInfo = std::vector; + using Scopes = std::vector; + using GroupIDs = std::vector; + typedef void (*CalculateFeatureFunction)(ACPOCollectFeatures &, + const FeatureInfo &); + + // Constructors/Destructors + ACPOCollectFeatures(); + ACPOCollectFeatures(FeatureInfo GlobalInfo); + ~ACPOCollectFeatures(); + + // Setters/getters + void setFeatureValue(FeatureIndex Idx, std::string Val); + + void setFeatureInfo(FeatureIndex Idx, FeatureInfo Info); + + void setFeatureValueAndInfo(FeatureIndex Idx, FeatureInfo Info, + std::string Val); + + void setGlobalFeatureInfo(FeatureInfo &Info); + + std::string getFeature(FeatureIndex Idx) const; + + // Check if the feature is alrady calculated. + bool containsFeature(FeatureIndex); + bool containsFeature(GroupID); + + static std::string getFeatureName(FeatureIndex Idx); + static GroupID getFeatureGroup(FeatureIndex Idx); + static Scope getFeatureScope(FeatureIndex Idx); + static std::set getGroupFeatures(GroupID Group); + static std::set getScopeFeatures(Scope S); + + void clearFeatureValueMap(); + bool registeredFeature(FeatureIndex Idx) const; + + // Calculate and Return the feature values specified by FeaturesInfo + FeatureValueMap getFeaturesPair(FeaturesInfo Features); + + // Calculate and Return the feature values specified from [Beg, End) + // TODO: Make a similar method for Scopes and GroupIDs + FeatureValueMap getFeaturesPair(FeatureIndex Beg, FeatureIndex End); + + // Calculate and Return the feature values specified by Scope. + FeatureValueMap getFeaturesPair(Scopes); + + // Calculate and Return the feature values specified by GroupID. + FeatureValueMap getFeaturesPair(GroupIDs); + + static InlineAdvisor::MandatoryInliningKind + getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, + OptimizationRemarkEmitter &ORE); + + static void clearFunctionLevel(); + static void insertFunctionLevel(const Function *, unsigned); + static std::optional getFunctionLevel(const Function *); + +private: + // Global mappings. + // FeatureIndexToName and FeatureIndexToScope should be a one to one mapping. + static const std::unordered_map FeatureIndexToName; + static const std::unordered_map FeatureIndexToScope; + static const std::unordered_map FeatureIndexToGroup; + static const std::multimap GroupToFeatureIndices; + static const std::multimap ScopeToFeatureIndices; + // The CalculateFeatureMap maps each feature to a corresponding function that + // calculates the feature and also sets the feature value inside + // FeatureValues field. + static const std::unordered_map + CalculateFeatureMap; + + // TODO: + // Implement the cache systems here. See similar example in DumpFeature.cpp + // Notice we've only cached the FunctionLevels. + // But in the future this should be generalized for all features. + // One way to do this is to define a map from FeatureIndex -> Mapping. + // Inside this mapping, the key should be the Scope and a set of analysis it + // depends on. + + static std::map FunctionLevels; + + // Saved FeatureValues when we collect the features. + FeatureValueMap FeatureToValue; + FeatureInfoMap FeatureToInfo; + FeatureInfo GlobalFeatureInfo; +}; + +ACPOCollectFeatures::FeatureIndex operator+(ACPOCollectFeatures::FeatureIndex, + int); +ACPOCollectFeatures::FeatureIndex operator-(ACPOCollectFeatures::FeatureIndex, + int); +ACPOCollectFeatures::FeatureIndex & +operator++(ACPOCollectFeatures::FeatureIndex &); +ACPOCollectFeatures::FeatureIndex +operator++(ACPOCollectFeatures::FeatureIndex &, int); + +} // namespace llvm +#endif // LLVM_ANALYSIS_ACPOCOLLECTFEATURES_H +#endif // ENABLE_ACPO diff --git a/llvm/include/llvm/Analysis/ACPOFIModel.h b/llvm/include/llvm/Analysis/ACPOFIModel.h new file mode 100644 index 0000000000000000000000000000000000000000..8753dd3d7c63248c8a39d4df522704ff68e1110a --- /dev/null +++ b/llvm/include/llvm/Analysis/ACPOFIModel.h @@ -0,0 +1,144 @@ +//===- ACPOFIModel.h - AI-Enabled Continuous Program Optimization ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#ifndef LLVM_ANALYSIS_ACPOFIMODEL_H +#define LLVM_ANALYSIS_ACPOFIMODEL_H + +#include "llvm/Analysis/ACPOModel.h" +#include "llvm/Analysis/DumpFeature.h" +#include "llvm/Analysis/FunctionPropertiesAnalysis.h" +#include "llvm/Analysis/InlineAdvisor.h" +#include "llvm/Analysis/InlineSizeEstimatorAnalysis.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" + +#include + +namespace llvm { + +//class ACPOmodel; + +class ACPOFIModel : public ACPOModel { +public: + ACPOFIModel(CallBase *CB, InlineAdvisor *IA, OptimizationRemarkEmitter *ORE, + bool OnlyMandatory, bool UseML = true); + + ~ACPOFIModel(); + + void setMLCustomFeatures( + std::vector> FeatureValues); + + void sendCustomFeatures() override; + + InlineAdvisor *getNotACPOAdvisor(); + + // Recorder's to micmic the behavior for default InlineAdvice. + // If the model is turned off or was decided to fall back to + // default inline advisor then we need to make sure the advice returned + // is properly recorded. Or else there would be an error. + void recordUnattemptedInlining(); + + void recordInlining(); + + void recordUnsuccessfulInlining(InlineResult &IR); + + void recordInliningWithCalleeDeleted(); + + // Interface for IRToPerf Cache system. + struct FunctionFeaturesCache { + using FunctionSizeMap = DenseMap; + using FunctionFloatMap = DenseMap; + + std::array( + ACPOFIExtendedFeatures::NamedFeatureIndex::NumNamedFeatures)> + NamedFeatures; + std::array( + ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: + NumNamedFloatFeatures)> + NamedFloatFeatures; + + FunctionSizeMap &operator[](ACPOFIExtendedFeatures::NamedFeatureIndex Pos) { + return NamedFeatures[static_cast(Pos)]; + } + FunctionFloatMap & + operator[](ACPOFIExtendedFeatures::NamedFloatFeatureIndex Pos) { + return NamedFloatFeatures[static_cast(Pos)]; + } + }; + + struct FunctionAnalysisMap { + DenseMap DomCache; + DenseMap LICache; + DenseMap TTICache; + }; + + // Invalidation mechanisms + static void invalidateCache(CallBase *CB); + + static void invalidateCache(const Function *F); + + static void clearCache(); + + // Getters/setters for the cache system. + static std::optional + getCachedSize(const Function *F, + ACPOFIExtendedFeatures::NamedFeatureIndex idx); + + static std::optional + getCachedFloat(const Function *F, + ACPOFIExtendedFeatures::NamedFloatFeatureIndex idx); + + static void insertSizeCache(const Function *F, + ACPOFIExtendedFeatures::NamedFeatureIndex idx, + size_t val); + + static void + insertFloatCache(const Function *F, + ACPOFIExtendedFeatures::NamedFloatFeatureIndex idx, + float val); + + static const DominatorTree *getDomCachedAnalysis(const Function *F); + + static const LoopInfo *getLICachedAnalysis(const Function *F); + + static const TargetTransformInfo *getTTICachedAnalysis(const Function *F); + + static void insertAnalysisCache(const Function *F, const DominatorTree *Tree); + + static void insertAnalysisCache(const Function *F, const LoopInfo *LI); + + static void insertAnalysisCache(const Function *F, + const TargetTransformInfo *TTI); + +protected: + // Interface to run the MLInference/default advisor and get advice from the + // model/default advisor + virtual std::unique_ptr getAdviceML() override; + + virtual std::unique_ptr getAdviceNoML() override; + +private: + static FunctionFeaturesCache FeatureCache; + static FunctionAnalysisMap FunctionAnalysisCache; + CallBase *CurrentCB = nullptr; + InlineAdvisor *NotACPOAdvisor = nullptr; + bool ShouldInline = false; + bool OnlyMandatory = false; + std::unique_ptr NotACPOAdvice = nullptr; + std::vector> CustomFeatureValues; +}; + +} // end namespace llvm + +#endif // LLVM_ANALYSIS_ACPOFIMODEL_H + +#endif // ENABLE_ACPO diff --git a/llvm/include/llvm/Analysis/ACPOMLInterface.h b/llvm/include/llvm/Analysis/ACPOMLInterface.h new file mode 100644 index 0000000000000000000000000000000000000000..fbc8a46b3d9ae1078e19b42ce254296fa97a1dc6 --- /dev/null +++ b/llvm/include/llvm/Analysis/ACPOMLInterface.h @@ -0,0 +1,482 @@ +//===- ACPOMLInterface.h - AI-Enabled Continuous Program Optimization -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#ifndef LLVM_ANALYSIS_ACPOML_INTERFACE_H +#define LLVM_ANALYSIS_ACPOML_INTERFACE_H + +#include "llvm/Analysis/ACPOModelRunner.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/Support/Program.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace llvm { + +class ACPOModelRunner; + +// This is class for storing information about a model. +class Model { +public: + // Constructors + Model() : NumFeatures{1}, NumOutputs{1} {} + Model(std::size_t NumFeatures) : NumFeatures{NumFeatures}, NumOutputs{1} {} + Model(std::size_t NumFeatures, int NumOutputs) + : NumFeatures{NumFeatures}, NumOutputs{NumOutputs} {} + + // Getters/Setters + std::size_t getNumFeatures() const { return NumFeatures; } + void setNumFeatures(int NumFeatures) { this->NumFeatures = NumFeatures; } + + int getNumOutputs() const { return NumOutputs; } + void setNumOutputs(int NumOutputs) { this->NumOutputs = NumOutputs; } + + std::string getSignature() const { return Signature; } + void setSignature(std::string Signature) { this->Signature = Signature; } + + // Register a feature into the NametoID and IDToIndex maps. + bool registerFeature(std::string FeatureName, uint64_t FeatureID, int Index); + + // Register an input into the map. + bool registerInput(std::string InputName, std::string InputType); + + // Register an output into the map. + bool registerOutput(std::string OutputName, std::string OutputType); + + // Return the index of a feature within the feature list used by inference(). + int getIndex(uint64_t FeatureID) const; + int getIndex(std::string FeatureName) const; + + // Return the name of a feature within the feature list used by inference(). + std::string getName(uint64_t FeatureID) const; + + // Return true if output name exists. + bool checkOutputExists(std::string OutputName) const; + + // Return the type of an input given its name. + std::string getInputType(std::string OutputName) const; + + // Return the type of an output given its name. + std::string getOutputType(std::string OutputName) const; + +private: + std::size_t NumFeatures; + int NumOutputs; + std::string Signature; + std::unordered_map NameToID; + std::unordered_map IDToName; + std::unordered_map IDToIndex; + + // A map from input name to input type + std::unordered_map InputMap; + + // A map from output name to output type + std::unordered_map OutputMap; +}; + +// This is the base class to define an interface with an ML framework. +class ACPOMLInterface { +public: + // Constructor/Destructor. + ACPOMLInterface() {} + virtual ~ACPOMLInterface() {} + + // Getters/Setters + bool isInitialized() const { return Initialized; } + void setInitialized(bool Val) { Initialized = Val; } + + // Interface methods. + // Return the next available ID for a feature. + virtual uint64_t assignID() = 0; + + // Load a model by reading from the specified file. + // Return false if the operation failed. + virtual bool loadModel(std::string ModelSpecFile) = 0; + + // Insert a new model into the model map. + virtual bool registerModel(std::string ModelName, int NumFeatures) = 0; + virtual bool registerModel(std::string ModelName, int NumFeatures, + int NumOutputs) = 0; + + // Register a new feature for a given model. + virtual bool registerFeature(std::string ModelName, std::string FeatureName, + int Index) = 0; + + // Register a new output for a given model. + virtual bool registerOutput(std::string ModelName, std::string OutputName, + std::string OutputType) = 0; + + // Specify how many models are currently live in ML framework memory. + virtual int getNumLoadedModels() = 0; + + // Specify the input file to use as IR to be passed to the model (however + // it is processed afterwards). Return false if the operation failed. + virtual bool defineInputIR(std::string Filename) = 0; + + // Specify a custom feature for a model to use as input at the next model + // invocation. Return false if the operation failed. + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + int FeatureValue) = 0; + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + int64_t FeatureValue) = 0; + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + double FeatureValue) = 0; + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + float FeatureValue) = 0; + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + bool FeatureValue) = 0; + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + int FeatureValue) = 0; + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + int64_t FeatureValue) = 0; + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + double FeatureValue) = 0; + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + float FeatureValue) = 0; + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + bool FeatureValue) = 0; + + // Replace all features with the given feature values. + // Activate the specified model. + virtual bool initializeFeatures( + std::string ModelName, + const std::vector> &FeatureValues) = 0; + + virtual bool + initializeFeatures(std::string ModelName, + const std::vector> + &FeatureValues) = 0; + + // Set features with the specified feature values. + // Not changing the currently active model. + virtual bool setCustomFeatures( + std::string ModelName, + const std::vector> &FeatureValues) = 0; + + virtual bool + setCustomFeatures(std::string ModelName, + const std::vector> + &FeatureValues) = 0; + + // Run a model with specified name. Return false if the execution was not + // possible or an error was encountered. + virtual bool runModel(std::string ModelName) = 0; + + // Return the type of an output within the model specified by the name. + virtual std::string getOutputType(std::string ModelName, + std::string OutputName) = 0; + + // Return model results, based on the output name. + virtual int getModelResultI(std::string OutputName) = 0; + virtual int64_t getModelResultI64(std::string OutputName) = 0; + virtual float getModelResultF(std::string OutputName) = 0; + virtual double getModelResultD(std::string OutputName) = 0; + virtual bool getModelResultB(std::string OutputName) = 0; + + // Get status of the ML interface. Return zero if succeeded. + virtual int getStatus() = 0; + + // Free up memory taken by a model. + virtual bool releaseModel(std::string ModelName) = 0; + + // Close interface when done. Return false if the command was not successful. + // In some cases this just requires a constructor for this class to be called, + // but in others, additional steps that require feedback may be needed. + virtual bool closeMLInterface() = 0; + + // Set a flag to invoke closeMLInterface when the instance of the class is + // destroyed. + void setCloseOnDestruction() { CloseOnDestruction = true; } + +protected: + bool CloseOnDestruction = false; + +private: + bool Initialized = false; +}; + +class ACPOMLPythonInterface : public ACPOMLInterface { +public: + ACPOMLPythonInterface(); + virtual ~ACPOMLPythonInterface(); + + // Interface methods. + // Return the next available ID for a feature. + virtual uint64_t assignID() override; + + // Load a model by reading from the specified file. + // Return false if the operation failed. + virtual bool loadModel(std::string ModelSpecFile) override; + + // Insert a new model into the model map. + virtual bool registerModel(std::string ModelName, int NumFeatures) override; + virtual bool registerModel(std::string ModelName, int NumFeatures, + int NumOutputs) override; + + // Register a new feature for a given model. + virtual bool registerFeature(std::string ModelName, std::string FeatureName, + int Index) override; + + // Register a new output for a given model. + virtual bool registerOutput(std::string ModelName, std::string OutputName, + std::string OutputType) override; + + // Specify how many models are currently live in ML framework memory. + virtual int getNumLoadedModels() override; + + // Specify the input file to use as IR to be passed to the model (however + // it is processed afterwards). Return false if the operation failed. + virtual bool defineInputIR(std::string Filename) override; + + // Specify a custom feature for a model to use as input at the next model + // invocation. Return false if the operation failed. + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + int FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + int64_t FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + double FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + float FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + bool FeatureValue) override; + + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + int FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + int64_t FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + double FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + float FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + bool FeatureValue) override; + + // Replace all features with the given feature values. + // Activate the specified model. + virtual bool + initializeFeatures(std::string ModelName, + const std::vector> + &FeatureValues) override; + + virtual bool + initializeFeatures(std::string ModelName, + const std::vector> + &FeatureValues) override; + + // Set features with the specified feature values. + // Not changing the currently active model. + virtual bool + setCustomFeatures(std::string ModelName, + const std::vector> + &FeatureValues) override; + + virtual bool + setCustomFeatures(std::string ModelName, + const std::vector> + &FeatureValues) override; + + // Run a model with the specified name. Return false if the execution was not + // possible or an error was encountered. + virtual bool runModel(std::string ModelName) override; + + // Return the type of an output within the model specified by the name. + virtual std::string getOutputType(std::string ModelName, + std::string OutputName) override; + + // Return model results, based on the output name. + virtual int getModelResultI(std::string OutputName) override; + virtual int64_t getModelResultI64(std::string OutputName) override; + virtual float getModelResultF(std::string OutputName) override; + virtual double getModelResultD(std::string OutputName) override; + virtual bool getModelResultB(std::string OutputName) override; + + // Get status of the ML interface. Return zero if succeeded. + virtual int getStatus() override; + + // Free up memory taken by a model. + virtual bool releaseModel(std::string ModelName) override; + + // Close interface when done. Return false if the command was not successful. + // In some cases this just requires a constructor for this class to be called, + // but in others, additional steps that require feedback may be needed. + virtual bool closeMLInterface() override; + +protected: + void sendCommand(const std::string &Command); + void sendCommand(const std::vector &Features); + std::string getResponse(); + std::vector tokenize(const std::string &Line); + +private: + llvm::sys::ProcessInfo SubProcess; + FILE *PipeOut = nullptr; + FILE *PipeIn = nullptr; + + uint64_t NextID; + + std::string CurrentlyActiveModel; + + // Mapping model names to their corresponding Model + std::unordered_map> ModelMap; +}; + +std::shared_ptr createPersistentPythonMLIF(); + +class ACPOMLCPPInterface : public ACPOMLInterface { +public: + ACPOMLCPPInterface(); + virtual ~ACPOMLCPPInterface(); + + // Interface methods. + // Return the next available ID for a feature. + virtual uint64_t assignID() override; + + // Load a model by reading from the specified file. + // Return false if the operation failed. + // For ACPOMLCompiledInterface, loadCompiledModel() should be used instead. + virtual bool loadModel(std::string ModelSpecFile) override; + + // Insert a new model into the model map. + virtual bool registerModel(std::string ModelName, int NumFeatures) override; + virtual bool registerModel(std::string ModelName, int NumFeatures, + int NumOutputs) override; + + // Register a new feature for a given model. + virtual bool registerFeature(std::string ModelName, std::string FeatureName, + int Index) override; + + // Register a new output for a given model. + virtual bool registerOutput(std::string ModelName, std::string OutputName, + std::string OutputType) override; + + // Specify how many models are currently live in ML framework memory. + virtual int getNumLoadedModels() override; + + // Specify the input file to use as IR to be passed to the model (however + // it is processed afterwards). Return false if the operation failed. + virtual bool defineInputIR(std::string Filename) override; + + // Specify a custom feature for a model to use as input at the next model + // invocation. Return false if the operation failed. + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + int FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + int64_t FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + double FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + float FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, + bool FeatureValue) override; + + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + int FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + int64_t FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + double FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + float FeatureValue) override; + virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, + bool FeatureValue) override; + + // Replace all features with the given feature values. + // Activate the specified model. + virtual bool + initializeFeatures(std::string ModelName, + const std::vector> + &FeatureValues) override; + + virtual bool + initializeFeatures(std::string ModelName, + const std::vector> + &FeatureValues) override; + + // Set features with the specified feature values. + // Not changing the currently active model. + virtual bool + setCustomFeatures(std::string ModelName, + const std::vector> + &FeatureValues) override; + + virtual bool + setCustomFeatures(std::string ModelName, + const std::vector> + &FeatureValues) override; + + // Run a model with the specified name. Return false if the execution was not + // possible or an error was encountered. + virtual bool runModel(std::string ModelName) override; + + // Return the type of an input within the model specified by the name. + virtual std::string getInputType(std::string ModelName, + std::string InputName); + + // Return the type of an output within the model specified by the name. + virtual std::string getOutputType(std::string ModelName, + std::string OutputName) override; + + // Return model results, based on the output name. + virtual int getModelResultI(std::string OutputName) override; + virtual int64_t getModelResultI64(std::string OutputName) override; + virtual float getModelResultF(std::string OutputName) override; + virtual double getModelResultD(std::string OutputName) override; + virtual bool getModelResultB(std::string OutputName) override; + + // Get status of the ML interface. Return zero if succeeded. + virtual int getStatus() override; + + // Free up memory taken by a model. + virtual bool releaseModel(std::string ModelName) override; + + // Close interface when done. Return false if the command was not successful. + // In some cases this just requires a constructor for this class to be called, + // but in others, additional steps that require feedback may be needed. + virtual bool closeMLInterface() override; + +private: + uint64_t NextID; + + std::string CurrentlyActiveModel; + + // Mapping model names to their corresponding Model + std::unordered_map> ModelMap; + + // Mapping model names to their corresponding Runner + std::unordered_map> RunnerMap; + + std::string readModelParam(std::string FilePath, std::string Param); + + void readFeatures(std::string FilePath, + std::vector> &Features); + void readOutputs(std::string FilePath, + std::vector> &Outputs); + + typedef std::unique_ptr (*CreateModelRunnerFunction)( + std::vector>, + StringRef); // function pointer type + const static std::unordered_map + CreateModelRunnerMap; +}; + +std::shared_ptr createPersistentCompiledMLIF(); + +} // namespace llvm + +#endif // LLVM_ANALYSIS_ACPOML_INTERFACE_H +#endif // ENABLE_ACPO diff --git a/llvm/include/llvm/Analysis/ACPOModel.h b/llvm/include/llvm/Analysis/ACPOModel.h new file mode 100644 index 0000000000000000000000000000000000000000..d61ac00efaaf06fcadee9c539dccc4474b63e76b --- /dev/null +++ b/llvm/include/llvm/Analysis/ACPOModel.h @@ -0,0 +1,124 @@ +//===- ACPOModel.h - AI-Enabled Continuous Program Optimization -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#ifndef LLVM_ANALYSIS_ACPOMODEL_H +#define LLVM_ANALYSIS_ACPOMODEL_H + +#include "llvm/Analysis/ACPOMLInterface.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Type.h" +#include +#include +#include +#include + +namespace llvm { + +class OptimizationRemarkEmitter; +class LLVMContext; + +class ACPOAdvice { +public: + struct FieldType { + Type::TypeID T; + Constant *Val; + }; + + ACPOAdvice() {} + ACPOAdvice(std::unique_ptr &ResultFormat); + virtual ~ACPOAdvice() {}; + + Constant *getField(std::string name) { + auto Search = FieldMap.find(name); + if (Search == FieldMap.end()) { + return nullptr; + } + return Search->second.Val; + } + + void reserveField(std::string name, Type::TypeID &ID) { + FieldMap[name].T = ID; + FieldMap[name].Val = nullptr; + } + + void addField(std::string name, Constant *Val) { + assert(Val != nullptr); + FieldMap[name].T = Val->getType()->getTypeID(); + FieldMap[name].Val = Val; + } + + std::unordered_map &getFieldMap() { + return FieldMap; + } + +private: + std::unordered_map FieldMap; +}; + +class ACPOModel { +public: + ACPOModel(OptimizationRemarkEmitter *OptReEm, bool UseML = true) : + ORE(OptReEm), ShouldUseML(UseML) { + ResultFormat = std::make_unique(); + assert(ResultFormat != nullptr); + } + + ~ACPOModel() {} + + bool isForcedToStop() const { return ForceStop; } + + // This is a interface method to return result of estimation either via an ML + // model or by employing a heuristic. The ML version should be implemented in + // the getAdviceML, which can be overwritten when necessary. The non-ML + // version should be implemented in getAdviceNoML and that should always be + // overwritten (and it will be marked as pure (=0) to force the programmer + // to do so). + std::unique_ptr getAdvice(); + void addRequiredResultField(std::string name, Type::TypeID &ID); + + void setContextPtr(LLVMContext *C) { Context = C; } + LLVMContext *getContextPtr() { return Context; } + + void setMLIF(std::shared_ptr ML) { MLIF = ML; } + std::shared_ptr getMLIF() { return MLIF; } + +protected: + void addFeature(int64_t ID, Constant *Val); + virtual void sendCustomFeatures() {} + virtual void prepareModelInput(); + virtual bool runModel(std::unique_ptr &Result); + + virtual std::unique_ptr getAdviceML(); + virtual std::unique_ptr getAdviceNoML() = 0; + +private: + // Pointer to means of feedback propagation + OptimizationRemarkEmitter *ORE; + + // We may need LLVMContext to set values of a Constant + LLVMContext *Context = nullptr; + + // Specify expected format of the ACPOAdvice result. + std::unique_ptr ResultFormat = nullptr; + + // Custom feature list. + std::unordered_map CustomFeatureMap; + + // Interface to ML framework. + std::shared_ptr MLIF = nullptr; + + // Specify if ML infra is in use + bool ShouldUseML = false; + bool ForceStop = false; +}; + +} // namespace llvm + +#endif // LLVM_ANALYSIS_ACPOMODEL_H +#endif // ENABLE_ACPO diff --git a/llvm/include/llvm/Analysis/ACPOModelRunner.h b/llvm/include/llvm/Analysis/ACPOModelRunner.h new file mode 100644 index 0000000000000000000000000000000000000000..044f3af15bbeb6de4fb183c768338ae6174f0009 --- /dev/null +++ b/llvm/include/llvm/Analysis/ACPOModelRunner.h @@ -0,0 +1,41 @@ +//===- ACPOModelRunner.h - AI-Enabled Continuous Program Optimization -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#ifndef LLVM_ANALYSIS_ACPOMODEL_H +#define LLVM_ANALYSIS_ACPOMODEL_H + +#include "llvm/Analysis/MLModelRunner.h" + +namespace llvm { + +class ACPOModelRunner : public MLModelRunner { +public: + virtual bool setCustomFeature(int FeatureIndex, int FeatureValue) = 0; + virtual bool setCustomFeature(int FeatureIndex, int64_t FeatureValue) = 0; + virtual bool setCustomFeature(int FeatureIndex, double FeatureValue) = 0; + virtual bool setCustomFeature(int FeatureIndex, float FeatureValue) = 0; + virtual bool setCustomFeature(int FeatureIndex, bool FeatureValue) = 0; + + virtual bool runModel() = 0; + + virtual int getModelResultI(std::string OutputName) = 0; + virtual int64_t getModelResultI64(std::string OutputName) = 0; + virtual float getModelResultF(std::string OutputName) = 0; + virtual double getModelResultD(std::string OutputName) = 0; + virtual bool getModelResultB(std::string OutputName) = 0; + +protected: + ACPOModelRunner(LLVMContext &Ctx, size_t NrInputs) + : MLModelRunner(Ctx, MLModelRunner::Kind::Release, NrInputs) {} +}; + +} // namespace llvm + +#endif // LLVM_ANALYSIS_ACPOMODEL_H +#endif // ENABLE_ACPO diff --git a/llvm/include/llvm/Analysis/AOTModelRunner.h b/llvm/include/llvm/Analysis/AOTModelRunner.h new file mode 100644 index 0000000000000000000000000000000000000000..fe19a33a5a08973078de0bb4f2d0f9e73419e7e2 --- /dev/null +++ b/llvm/include/llvm/Analysis/AOTModelRunner.h @@ -0,0 +1,205 @@ +//===- AOTModelRunner.h - AI-Enabled Continuous Program Optimization ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#ifndef LLVM_ANALYSIS_AOTMODEL_H +#define LLVM_ANALYSIS_AOTMODEL_H + +#include "llvm/Analysis/ACPOModelRunner.h" +#include "llvm/Analysis/TensorSpec.h" + +#define DEBUG_TYPE "acpo-aot" + +namespace llvm { + +template class AOTModelRunner : public ACPOModelRunner { +public: + /// FeatureNames' type should be an indexed collection of std::string, like + /// std::array or std::vector, that has a size() method. + /// In the future, this method could be expanded to allow AOT models with + /// multiple outputs, by taking in a vector of string pairs similar to the + /// Features vector. + /// The current implementation does work for AOT models with a single output + /// which is a vector (or higher-dimensional tensor) of multiple values. + AOTModelRunner( + LLVMContext &Ctx, + const std::vector> &Features, + StringRef DecisionName, StringRef FeedPrefix = "feed_", + StringRef FetchPrefix = "fetch_") + : ACPOModelRunner(Ctx, Features.size()), + CompiledModel(std::make_unique()) { + assert(CompiledModel && "The CompiledModel should be valid"); + + for (size_t I = 0; I < Features.size(); ++I) { + const int Index = + CompiledModel->LookupArgIndex(FeedPrefix.str() + Features[I].first); + void *Buffer = nullptr; + if (Index >= 0) { + Buffer = CompiledModel->arg_data(Index); + } else { + LLVM_DEBUG(dbgs() << "Warning: AOTModelRunner was unable to find the " + "feature " + << (FeedPrefix.str() + Features[I].first) + << " in the compiled model\n"); + } + // The order of features passed to the model runner is important, it + // determines their index + TensorSpec Spec = makeSpec(Features[I].first, Features[I].second); + setUpBufferForTensor(I, Spec, Buffer); + } + + ResultIndex = CompiledModel->LookupResultIndex(FetchPrefix.str() + + DecisionName.str()); + assert(ResultIndex >= 0 && "Cannot find DecisionName in inlining model"); + } + + virtual ~AOTModelRunner() = default; + + static bool classof(const ACPOModelRunner *R) { + return R->getKind() == ACPOModelRunner::Kind::Release; + } + + bool setCustomFeature(int FeatureIndex, int FeatureValue) override { + LLVM_DEBUG(dbgs() << "AOTModelRunner: setting int feature " << FeatureIndex + << " to " << FeatureValue << "\n"); + *getTensor(FeatureIndex) = FeatureValue; + return true; + } + bool setCustomFeature(int FeatureIndex, int64_t FeatureValue) override { + LLVM_DEBUG(dbgs() << "AOTModelRunner: setting int64 feature " + << FeatureIndex << " to " << FeatureValue << "\n"); + *getTensor(FeatureIndex) = FeatureValue; + return true; + } + bool setCustomFeature(int FeatureIndex, double FeatureValue) override { + LLVM_DEBUG(dbgs() << "AOTModelRunner: setting double feature " + << FeatureIndex << " to " << FeatureValue << "\n"); + *getTensor(FeatureIndex) = FeatureValue; + return true; + } + bool setCustomFeature(int FeatureIndex, float FeatureValue) override { + LLVM_DEBUG(dbgs() << "AOTModelRunner: setting float feature " + << FeatureIndex << " to " << FeatureValue << "\n"); + *getTensor(FeatureIndex) = FeatureValue; + return true; + } + bool setCustomFeature(int FeatureIndex, bool FeatureValue) override { + // There are no bool tensors, so assume int for now + LLVM_DEBUG(dbgs() << "AOTModelRunner: setting bool feature " << FeatureIndex + << " to " << FeatureValue << "\n"); + *getTensor(FeatureIndex) = FeatureValue; + return true; + } + + bool runModel() override { + CompiledModel->Run(); + return true; + } + + int getModelResultI(std::string OutputName) override { + void *Data = CompiledModel->result_data(ResultIndex); + int Result = *reinterpret_cast(Data); + LLVM_DEBUG(dbgs() << "Returning int model result " << OutputName << " = " + << Result << "\n"); + return Result; + } + + int64_t getModelResultI64(std::string OutputName) override { + void *Data = CompiledModel->result_data(ResultIndex); + int64_t Result = *reinterpret_cast(Data); + LLVM_DEBUG(dbgs() << "Returning int64 model result " << OutputName << " = " + << Result << "\n"); + return Result; + } + + float getModelResultF(std::string OutputName) override { + void *Data = CompiledModel->result_data(ResultIndex); + float Result = *reinterpret_cast(Data); + LLVM_DEBUG(dbgs() << "Returning float model result " << OutputName << " = " + << Result << "\n"); + return Result; + } + + double getModelResultD(std::string OutputName) override { + void *Data = CompiledModel->result_data(ResultIndex); + double Result = *reinterpret_cast(Data); + LLVM_DEBUG(dbgs() << "Returning double model result " << OutputName << " = " + << Result << "\n"); + return Result; + } + + bool getModelResultB(std::string OutputName) override { + // Since there are no bool tensors, use int and return the corresponding + // result + void *Data = CompiledModel->result_data(ResultIndex); + bool Result = (*reinterpret_cast(Data)) > 0; + LLVM_DEBUG(dbgs() << "Returning bool model result " << OutputName << " = " + << Result << "\n"); + return Result; + } + +protected: + std::unique_ptr CompiledModel; + +private: + void *evaluateUntyped() override { + CompiledModel->Run(); + return CompiledModel->result_data(ResultIndex); + } + + llvm::TensorSpec makeSpec(std::string Name, std::string Type) { + std::vector Shape{}; + // If the string is of the form "float32[7][8]", read the value in brackets + // as the shape (read from left to right) + size_t RightBracket = 0; + size_t LeftBracket = 0; + do { + LeftBracket = Type.find("[", RightBracket + 1); + if (LeftBracket == std::string::npos) { + break; + } + RightBracket = Type.find("]", LeftBracket + 1); + size_t Value = std::stol( + Type.substr(LeftBracket + 1, RightBracket - LeftBracket - 1)); + Shape.push_back(Value); + } while (RightBracket != std::string::npos); + + // Remove array indices to just get type + if (Type.find("[") != std::string::npos) { + Type = Type.substr(0, Type.find("[")); + } + + if (Shape.size() == 0) + Shape.push_back(1); // Default shape is {1} + + if (Type == "int64") { + return TensorSpec::createSpec(Name, Shape); + } + if (Type == "int32") { + return TensorSpec::createSpec(Name, Shape); + } + if (Type == "int" || Type == "bool") { + // There are no bool tensors, so assume int for now + return TensorSpec::createSpec(Name, Shape); + } + if (Type == "float64") { + return TensorSpec::createSpec(Name, Shape); + } + if (Type == "float32") { + return TensorSpec::createSpec(Name, Shape); + } + assert(false && "ACPO AOT: received unknown feature type"); + } + + int32_t ResultIndex = -1; +}; + +} // namespace llvm + +#endif // LLVM_ANALYSIS_AOTMODEL_H +#endif // ENABLE_ACPO diff --git a/llvm/include/llvm/Analysis/CallHeight.h b/llvm/include/llvm/Analysis/CallHeight.h new file mode 100644 index 0000000000000000000000000000000000000000..84e94075ea39327e24100952d3a7384828f2d10e --- /dev/null +++ b/llvm/include/llvm/Analysis/CallHeight.h @@ -0,0 +1,74 @@ +//===- CallHeight.h - Call height for function ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This header file defines passes to get the call height of functions. +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#ifndef LLVM_ANALYSIS_CALLHEIGHT +#define LLVM_ANALYSIS_CALLHEIGHT + +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Pass.h" + +#include +#include + +namespace llvm { + +class CallHeight { +private: + /// Map from function to its level (callheight) + std::unique_ptr> Levels; + +public: + CallHeight(Module &M); + + // Change this to getHeight + unsigned getLevel(Function &F); + + bool invalidate(Module &, const PreservedAnalyses &PA, + ModuleAnalysisManager::Invalidator &) { + return false; + } +}; + +/// This analysis computes the mapping from function to level (callheight) +/// for MLInliner +class CallHeightAnalysis : public AnalysisInfoMixin { +public: + static AnalysisKey Key; + using Result = CallHeight; + + Result run(Module &M, ModuleAnalysisManager &MAM); +}; + +/// Legacy wrapper pass to provide the CallHeightAnalysis object. +class CallHeightAnalysisWrapper : public ModulePass { + std::unique_ptr Result; + +public: + static char ID; + + CallHeightAnalysisWrapper() : ModulePass(ID) {} + + bool runOnModule(Module &M) override; + + llvm::CallHeight &getResult() { return *Result; } + const llvm::CallHeight &getResult() const { return *Result; } + void getAnalysisUsage(AnalysisUsage &AU) const override; +}; + +Pass *createCallHeightAnalysisWrapper(); + +} // namespace llvm + +#endif +#endif // ENABLE_ACPO diff --git a/llvm/include/llvm/Analysis/DumpCallsite.h b/llvm/include/llvm/Analysis/DumpCallsite.h new file mode 100644 index 0000000000000000000000000000000000000000..02238521580b94ce2c49cd5524911052b574a8be --- /dev/null +++ b/llvm/include/llvm/Analysis/DumpCallsite.h @@ -0,0 +1,29 @@ +//===- DumpCallSite.h - Dump information about a callsite -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This header file defines the pass used to dump a callsite. +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#ifndef LLVM_ANALYSIS_DUMPCALLSITE +#define LLVM_ANALYSIS_DUMPCALLSITE + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class DumpCallsitePass : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); +}; + +} // namespace llvm + +#endif +#endif // ENABLE_ACPO diff --git a/llvm/include/llvm/Analysis/DumpFeature.h b/llvm/include/llvm/Analysis/DumpFeature.h new file mode 100644 index 0000000000000000000000000000000000000000..67ca36b106cb56d2f4114d0f48661199c4efb555 --- /dev/null +++ b/llvm/include/llvm/Analysis/DumpFeature.h @@ -0,0 +1,196 @@ +//===- DumpFeature.h - Dump features for a function -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This header file defines passes to dump features for functions in an scc. +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#ifndef LLVM_ANALYSIS_DUMPFEATURE +#define LLVM_ANALYSIS_DUMPFEATURE + +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/CGSCCPassManager.h" +#include "llvm/Analysis/CallGraph.h" +#include "llvm/Analysis/CallGraphSCCPass.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Pass.h" + +#include + +// EnableFeatureDump - This boolean is set to true if '-enable-feature-dump' is +// used as command line option. And we dump function features. +extern bool EnableFeatureDump; + +namespace llvm { + +class DumpFeaturePass : public PassInfoMixin { +public: + PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, + LazyCallGraph &CG, CGSCCUpdateResult &UR); + +private: + /// Get the caller height from cache or calculate from scratch + /// for a specific function F + int getCallHeight(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, + LazyCallGraph &CG, Function *F); +}; + +class ACPOFIExtendedFeatures { +public: + enum class NamedFeatureIndex : size_t { + InitialSize, + Blocks, + Calls, + IsLocal, + IsLinkOnceODR, + IsLinkOnce, + Loops, + MaxLoopDepth, + MaxDomTreeLevel, + PtrArgs, + PtrCallee, + CallReturnPtr, + ConditionalBranch, + CBwithArg, + CallerHeight, + CallUsage, + IsRecursive, + NumCallsiteInLoop, + NumOfCallUsesInLoop, + EntryBlockFreq, + MaxCallsiteBlockFreq, + NumNamedFeatures + }; + + enum class NamedFloatFeatureIndex : size_t { + InstructionPerBlock, + SuccessorPerBlock, + AvgVecInstr, + AvgNestedLoopLevel, + InstrPerLoop, + BlockWithMultipleSuccecorsPerLoop, + NumNamedFloatFeatures + }; + + struct FunctionFeatures { + static const size_t FeatureCount; + + std::array(NamedFeatureIndex::NumNamedFeatures)> + NamedFeatures = {{0}}; + std::array( + NamedFloatFeatureIndex::NumNamedFloatFeatures)> + NamedFloatFeatures = {{0}}; + std::vector InstructionHistogram; + std::vector InstructionPairHistogram; + + void fillTensor(int32_t *Ptr) const; + uint64_t &operator[](NamedFeatureIndex Pos) { + return NamedFeatures[static_cast(Pos)]; + } + float &operator[](NamedFloatFeatureIndex Pos) { + return NamedFloatFeatures[static_cast(Pos)]; + } + }; + + ACPOFIExtendedFeatures() = default; + + // Collect a number of features from the function F + static FunctionFeatures getFunctionFeatures( + Function &F, DominatorTree &DomTree, TargetTransformInfo &TTI, + LoopInfo &LI, FunctionAnalysisManager *FAM = nullptr, bool ValidSize = false, + bool ValidLoop = false, bool ValidTree = false); + +private: + // Loop related features, will update FF + static void updateLoopRelatedFeatures(Function &F, LoopInfo &LI, + FunctionFeatures &FF); + // Instruction and BasicBlock related features, will update FF + static void updateInstBBRelatedFeatures(Function &F, FunctionFeatures &FF); + + // This function should mimic the behaviour of updating all features below at + // once: + // getMaxCallsiteBlockFreq + // updateCallsiteRelatedFeatures + // updateInstBBRelatedFeatures + static void + updateBBLoopCallsiteBFFeatures(Function &F, FunctionFeatures &FF, + LoopInfo &LI, + FunctionAnalysisManager *FAM = nullptr); +}; + +const std::map + NamedFeatureIndexToName = { + {ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize, "InitialSize"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::Blocks, "Blocks"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::Calls, "Calls"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::IsLocal, "IsLocal"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::IsLinkOnceODR, + "IsLinkOnceODR"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::IsLinkOnce, "IsLinkOnce"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::Loops, "Loops"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth, + "MaxLoopDepth"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel, + "MaxDomTreeLevel"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::PtrArgs, "PtrArgs"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::PtrCallee, "PtrCallee"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::CallReturnPtr, + "CallReturnPtr"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::ConditionalBranch, + "ConditionalBranch"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::CBwithArg, "CBwithArg"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::CallerHeight, + "CallerHeight"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::CallUsage, "CallUsage"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::IsRecursive, "IsRecursive"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::NumCallsiteInLoop, + "NumCallsiteInLoop"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::NumOfCallUsesInLoop, + "NumOfCallUsesInLoop"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::EntryBlockFreq, + "EntryBlockFreq"}, + {ACPOFIExtendedFeatures::NamedFeatureIndex::MaxCallsiteBlockFreq, + "MaxCallsiteBlockFreq"}}; + +const std::map + FloatFeatureIndexToName = { + {ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstructionPerBlock, + "InstructionPerBlock"}, + {ACPOFIExtendedFeatures::NamedFloatFeatureIndex::SuccessorPerBlock, + "SuccessorPerBlock"}, + {ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgVecInstr, + "AvgVecInstr"}, + {ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgNestedLoopLevel, + "AvgNestedLoopLevel"}, + {ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop, + "InstrPerLoop"}, + {ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: + BlockWithMultipleSuccecorsPerLoop, + "BlockWithMultipleSuccecorsPerLoop"}}; + +ACPOFIExtendedFeatures::NamedFeatureIndex & +operator++(ACPOFIExtendedFeatures::NamedFeatureIndex &n); + +ACPOFIExtendedFeatures::NamedFeatureIndex +operator++(ACPOFIExtendedFeatures::NamedFeatureIndex &n, int); + +ACPOFIExtendedFeatures::NamedFloatFeatureIndex & +operator++(ACPOFIExtendedFeatures::NamedFloatFeatureIndex &n); + +ACPOFIExtendedFeatures::NamedFloatFeatureIndex +operator++(ACPOFIExtendedFeatures::NamedFloatFeatureIndex &n, int); + +} // namespace llvm + +#endif +#endif // ENABLE_ACPO diff --git a/llvm/include/llvm/Analysis/FIModelRunner.h b/llvm/include/llvm/Analysis/FIModelRunner.h new file mode 100644 index 0000000000000000000000000000000000000000..3685220aa07410390afea0d697e4ff58773ddde0 --- /dev/null +++ b/llvm/include/llvm/Analysis/FIModelRunner.h @@ -0,0 +1,277 @@ +//===- FIModelRunner.h - AI-Enabled Continuous Program Optimization -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#ifdef LLVM_HAVE_TF_AOT_FICOMPILEDMODEL + +#ifndef LLVM_ANALYSIS_FIMODELRUNNER_H +#define LLVM_ANALYSIS_FIMODELRUNNER_H + +#include "llvm/Analysis/AOTModelRunner.h" +#include "llvm/Analysis/FICompiledModel.h" + +namespace llvm { + +class FIModelRunner : public AOTModelRunner { + std::vector Means = { + 0.40009943697174110699421589742996729910373687744141, + 0.0, + 47.2218788212687599070704891346395015716552734375, + 0.0, + 0.07675459224122871404460966004990041255950927734375, + 5816.8243862454482950852252542972564697265625, + 1333.68016232413765465025790035724639892578125, + 321.9700210967629345759632997214794158935546875, + 0.94076781467098458122677584469784051179885864257812, + 0.0, + 0.0, + 24.57427538666200916850357316434383392333984375, + 0.72785175828753412297089653293369337916374206542969, + 22.362582136282401990001744707114994525909423828125, + 2.3236404681600126842511144786840304732322692871094, + 219.476437468925951179699040949344635009765625, + 123.872156304169635632206336595118045806884765625, + 759.6211988873809559663641266524791717529296875, + 3.5118047810371009198604497214546427130699157714844, + 0.0, + 14.689125089022963877027905255090445280075073242188, + 0.2720138674263292699606608948670327663421630859375, + 97.33707789677367827607668004930019378662109375, + 5.4576519437240493815011177503038197755813598632812, + 222416123463299168.0, + 697004967939498496.0, + 6.2712796684314486839184610289521515369415283203125, + 1.4856427516360068974421437815180979669094085693359, + 0.0041427067953076499376430241738944459939375519752502, + 0.72785175828753412297089653293369337916374206542969, + 552.7808652140716958456323482096195220947265625, + 62.5524652090595196796130039729177951812744140625, + 385.68509386043888298445381224155426025390625, + 92.9494483935554143272383953444659709930419921875, + 24.2728066757145342080548289231956005096435546875, + 0.90531987798814816947867711860453709959983825683594, + 0.0, + 0.0, + 2.9322753597871509256833633116912096738815307617188, + 0.49584111584407208894731411419343203306198120117188, + 7.9963853317029256473347231803927570581436157226562, + 1.4571144465795025091381376114441081881523132324219, + 15.557169540036818844441768305841833353042602539062, + 9.6481678066085265754736610688269138336181640625, + 50.98738225453177363988288561813533306121826171875, + 1.3425469302194332765765238946187309920787811279297, + 0.0, + 839.271140434566405019722878932952880859375, + 0.16440693908813608370422798543586395680904388427734, + 2.8829196844891762374629706755513325333595275878906, + 132.0555906421747067724936641752719879150390625, + 92791372484119440.0, + 166968642875823456.0, + 5.5557876796248262252220229129306972026824951171875, + 1.1750766644405326033506753446999937295913696289062, + 0.0042161570432282073628282859090177225880324840545654, + 0.49584111584407208894731411419343203306198120117188, + 41.15953665944181949498670292086899280548095703125, + 5.14903426051142787400749512016773223876953125, + 2.0527687821658449074391228350577875971794128417969, + 0.52614251736787642776960183255141600966453552246094, + 0.74523979091361081117383946548216044902801513671875, + 222.345100041656024814074044115841388702392578125, + 7.4997648449992606600744693423621356487274169921875, + 0.0, + 78.5584998454695693226312869228422641754150390625, + 0.0, + 10.409640011287439875786731136031448841094970703125, + 8.4653112780338357623577394406311213970184326171875, + 1.3630927585697201198655648113344796001911163330078, + 566.7381985783200661899172700941562652587890625, + 0.0, + 1.2066945269353257508271326514659449458122253417969, + 55.41075531786237462483768467791378498077392578125, + 0.51243634018194272883306439325679093599319458007812, + 1.1147556403606606600931172579294070601463317871094, + -31.471868743197301654390685143880546092987060546875, + 0.0, + 0.030368588666872708276001091576290491502732038497925, + 0.58478345583789081985059965518303215503692626953125, + 0.00034937314395517275094141251834400918596656993031502, + -0.23764092503258577027125397762574721127748489379883, + -62.20223330063559075142620713450014591217041015625, + 5.8952014942420616350204909394960850477218627929688, + 3339.09353794057960840291343629360198974609375, + 0.71960117711874660439974604742019437253475189208984, + -49.2720273048549444183663581497967243194580078125, + 27818.32155766672440222464501857757568359375, + 91.64824843118020680776680819690227508544921875, + 106.3296335613216996307528461329638957977294921875, + 469.83727273948858282892615534365177154541015625, + 0.30689743210739195422576131022651679813861846923828, + 1071.964175815315911677316762506961822509765625, + 1363.988766309679022015188820660114288330078125, + 14.079536139964256236112305487040430307388305664062, + 63165365211952664.0, + 0.38502264206721403816402471420587971806526184082031, + 0.015573979763232508391479491649533883901312947273254, + 0.13859363872129429329227434664062457159161567687988, + 0.0}; + + std::vector Scales = { + 0.48991823553184549178141082848014775663614273071289, + 1.0, + 19.2517211876445770712962257675826549530029296875, + 1.0, + 0.26620166192402217042456413764739409089088439941406, + 13580.447773648038491955958306789398193359375, + 3192.7079136089387247920967638492584228515625, + 633.0586155859824657454737462103366851806640625, + 0.23605875020885080939336830851971171796321868896484, + 1.0, + 1.0, + 101.565906032925312274528550915420055389404296875, + 0.44506581113952026207414292002795264124870300292969, + 25.4451961539476627649492002092301845550537109375, + 1.8819488669919737233726664271671324968338012695312, + 399.4446922340151786556816659867763519287109375, + 253.61174866766344848656444810330867767333984375, + 1934.51814232197148157865740358829498291015625, + 9.2671206485376131922748754732310771942138671875, + 1.0, + 101.7363052307218964642743230797350406646728515625, + 0.44499699252253444026194983962341211736202239990234, + 241.819662633324895750774885527789592742919921875, + 41.0624051346520815286567085422575473785400390625, + 1810657384453411584.0, + 2590019375355715584.0, + 18.6007475145233769353581010363996028900146484375, + 0.30589376767499054654564361044322140514850616455078, + 0.021661308027730186848147653222440567333251237869263, + 0.44506581113952026207414292002795264124870300292969, + 2210.9835111177717408281750977039337158203125, + 252.28469071093292086516157723963260650634765625, + 1479.28580699818076027440838515758514404296875, + 358.2883493183543350824038498103618621826171875, + 86.4399992258764626740230596624314785003662109375, + 0.29277260204409949473358665272826328873634338378906, + 1.0, + 1.0, + 11.300678128510535103146139590535312891006469726562, + 0.49998270338340455865022704529110342264175415039062, + 9.4889928089799600030573856201954185962677001953125, + 1.0885854822898506366612991769216023385524749755859, + 53.20529981175358358314042561687529087066650390625, + 36.65171139901388386306280153803527355194091796875, + 214.68561782216193023486994206905364990234375, + 2.8728217196022858281878598063485696911811828613281, + 1.0, + 1653.1016242378727838513441383838653564453125, + 0.37064443536603375317639574859640561044216156005859, + 20.0905336391907667348277755081653594970703125, + 288.66579115116110187955200672149658203125, + 967784087203564544.0, + 986920622098821248.0, + 17.499765511468584833210115903057157993316650390625, + 0.57797196338014200645005757905892096459865570068359, + 0.028955889395889600895772630906321865040808916091919, + 0.49998270338340455865022704529110342264175415039062, + 319.19585661999855119574931450188159942626953125, + 38.6813101625874224964718450792133808135986328125, + 39.62777871280881214488545083440840244293212890625, + 5.0871202966110988796799574629403650760650634765625, + 0.69504605038799238680979897253564558923244476318359, + 673.3477042973012203219695948064327239990234375, + 56.94168682747444876213194220326840877532958984375, + 1.0, + 261.01902251155337353338836692273616790771484375, + 1.0, + 85.0611943221388884239786420948803424835205078125, + 53.12927927294536090130350203253328800201416015625, + 21.829518414441992035790462978184223175048828125, + 1898.72146183866834689979441463947296142578125, + 1.0, + 9.7285926829767870316345579340122640132904052734375, + 174.40267892003106453557847999036312103271484375, + 0.98364895900708060327843895720434375107288360595703, + 1.1152676652901183373955973365809768438339233398438, + 18.12268289087599981712628505192697048187255859375, + 1.0, + 0.1715993516574435828747624555035145021975040435791, + 0.49275933843630442821037718204024713486433029174805, + 0.031531692879025040310292382628176710568368434906006, + 23.13033056510358420609918539412319660186767578125, + 210.58233961820729973624111153185367584228515625, + 5.1604155410259560099461850768420845270156860351562, + 2053.87275307550726211047731339931488037109375, + 1.0834136602451556186110792623367160558700561523438, + 3840.080091990574146620929241180419921875, + 13192.047960544839952490292489528656005859375, + 348.088713237990532434196211397647857666015625, + 439.96013885313283253708505071699619293212890625, + 897.3433304220051240918110124766826629638671875, + 0.69288480487588777201324319321429356932640075683594, + 2894.596744865002619917504489421844482421875, + 3788.94162413956064483500085771083831787109375, + 94.549943427633166947998688556253910064697265625, + 649339661894085888.0, + 0.48660066498392295919472871901234611868858337402344, + 0.12382015553845396316212656984134810045361518859863, + 0.50791641118256847242662388453027233481407165527344, + 1.0}; + +public: + FIModelRunner(LLVMContext &Ctx, + std::vector> Features, + StringRef DecisionName) + : AOTModelRunner( + Ctx, + {{"input_1", "float32[" + std::to_string(Features.size()) + "]"}}, + DecisionName) {} + + // Features for this model are only floats so we only need to override the + // float method to handle feature scaling and the input type + bool setCustomFeature(int FeatureIndex, float FeatureValue) override { + // Scale the feature according to the constant mean and scale value + // Feature scaling is done to create a standard normal distribution: + // subtract mean, then divide by standard deviation ("scale") + float ScaledValue = + (FeatureValue - Means[FeatureIndex]) / Scales[FeatureIndex]; + // Assuming the Buffer at index 0 is for feature input of shape: + // (Feature.size()) + float *Location = getTensor(0) + FeatureIndex; + *Location = ScaledValue; + return true; + } + + // Outputs for this model are only int so we only need to override this + // method + int getModelResultI(std::string OutputName) override { + if (OutputName == "FI-ShouldInline") { + int Classes[] = {0, 1}; + void *ResultUntyped = CompiledModel->result_data(0); + float *Result = reinterpret_cast(ResultUntyped); + float Max = Result[0]; + int MaxClass = 0; + for (size_t I = 0; I < sizeof(Classes) / sizeof(int); ++I) { + if (Result[I] > Max) { + Max = Result[I]; + MaxClass = I; + } + } + + return Classes[MaxClass]; + } + assert(false && "ModelRunner received invalid result name"); + } +}; + +} // namespace llvm + +#endif // LLVM_ANALYSIS_FIMODELRUNNER_H + +#endif // LLVM_HAVE_TF_AOT_FICOMPILEDMODEL + +#endif // ENABLE_ACPO diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h index 53c018d15cd71a6211ff3471f5dd455badd540bb..adf36a3857255a6d01c1223c36948b5d7de3f24b 100644 --- a/llvm/include/llvm/Analysis/InlineAdvisor.h +++ b/llvm/include/llvm/Analysis/InlineAdvisor.h @@ -200,6 +200,22 @@ public: return AnnotatedInlinePassName.c_str(); } +#if defined(ENABLE_ACPO) + /// Helper functions used by getFeatures to retrieve certain information + ///{ + CallBase *getInlinableCS(Instruction &I); + int64_t getLocalCalls(Function &F); + unsigned getCallLoopLevel(CallBase &CB) const; + uint64_t getCalleeBlockFreq(CallBase &CB) const; + unsigned getCallSiteHeight(CallBase *CB); + ///} + + // Allow ACPO infrastructure to replicate Advisor behaviour + virtual bool isForcedToStop() const { return false; } + bool neverInline(CallBase &CB) const; + bool isCSInlinable(CallBase &CB) const; +#endif + protected: InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, std::optional IC = std::nullopt); @@ -213,6 +229,15 @@ protected: const std::string AnnotatedInlinePassName; std::unique_ptr ImportedFunctionsStats; +#if defined(ENABLE_ACPO) + /// Map a function to its callheight + std::map FunctionLevels; + // used by getORE() for legacy PM + static std::unique_ptr ORE; + + friend class ACPOCollectFeatures; +#endif + enum class MandatoryInliningKind { NotMandatory, Always, Never }; static MandatoryInliningKind getMandatoryKind(CallBase &CB, @@ -389,6 +414,11 @@ void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, bool ForProfileContext = false, const char *PassName = nullptr); +#if defined(ENABLE_ACPO) +/// get call site location as string. +std::string getCallSiteLocation(DebugLoc DLoc); +#endif + /// Add location info to ORE message. void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc); diff --git a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h index 77ae60059ce9ebd9b0e2a28fd94cb1fc8576b122..e7ece46342fd9bce471a038baf170cca5a0c964a 100644 --- a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h +++ b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h @@ -72,6 +72,36 @@ enum class InlineCostFeatureIndex : size_t { NumberOfFeatures }; + +#if defined(ENABLE_ACPO) +const std::map InlineCostFeatureIndexToName = { + { InlineCostFeatureIndex::sroa_savings, "sroa_savings" }, + { InlineCostFeatureIndex::sroa_losses, "sroa_losses" }, + { InlineCostFeatureIndex::load_elimination, "load_elimination" }, + { InlineCostFeatureIndex::call_penalty, "call_penalty" }, + { InlineCostFeatureIndex::call_argument_setup, "call_argument_setup" }, + { InlineCostFeatureIndex::load_relative_intrinsic, "load_relative_intrinsic" }, + { InlineCostFeatureIndex::lowered_call_arg_setup, "lowered_call_arg_setup" }, + { InlineCostFeatureIndex::indirect_call_penalty, "indirect_call_penalty" }, + { InlineCostFeatureIndex::jump_table_penalty, "jump_table_penalty" }, + { InlineCostFeatureIndex::case_cluster_penalty, "case_cluster_penalty" }, + { InlineCostFeatureIndex::switch_penalty, "switch_penalty" }, + { InlineCostFeatureIndex::unsimplified_common_instructions, "unsimplified_common_instructions" }, + { InlineCostFeatureIndex::num_loops, "num_loops" }, + { InlineCostFeatureIndex::dead_blocks, "dead_blocks" }, + { InlineCostFeatureIndex::simplified_instructions, "simplified_instructions" }, + { InlineCostFeatureIndex::constant_args, "constant_args" }, + { InlineCostFeatureIndex::constant_offset_ptr_args, "constant_offset_ptr_args" }, + { InlineCostFeatureIndex::callsite_cost, "callsite_cost" }, + { InlineCostFeatureIndex::cold_cc_penalty, "cold_cc_penalty" }, + { InlineCostFeatureIndex::last_call_to_static_bonus, "last_call_to_static_bonus" }, + { InlineCostFeatureIndex::is_multiple_blocks, "is_multiple_blocks" }, + { InlineCostFeatureIndex::nested_inlines, "nested_inlines" }, + { InlineCostFeatureIndex::nested_inline_cost_estimate, "nested_inline_cost_estimate" }, + { InlineCostFeatureIndex::threshold, "threshold" } +}; +#endif + // clang-format on using InlineCostFeatures = diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h index 9be3e056cf76061120f746c0b88a8d616b7cf4f7..ea4cb7f7c68415bd1c663483aea443cc795df516 100644 --- a/llvm/include/llvm/Analysis/LoopInfo.h +++ b/llvm/include/llvm/Analysis/LoopInfo.h @@ -386,6 +386,17 @@ public: void dump() const; void dumpVerbose() const; +#if defined(ENABLE_ACPO) + /// Print loop IR wrapped in a dummy function + void printWithFunctionWrapper(raw_ostream &ROS, Function *F, + ArrayRef LoopBlocks, + BasicBlock *Header, + SmallVector ExitBlocks, + AssemblyAnnotationWriter *AAW, + bool ShouldPreserveUseListOrder, + bool IsForDebug) const; +#endif + /// Return the debug location of the start of this loop. /// This looks for a BB terminating instruction with a known debug /// location by looking at the preheader and header blocks. If it diff --git a/llvm/include/llvm/Analysis/MLInlineAdvisor.h b/llvm/include/llvm/Analysis/MLInlineAdvisor.h index f58862e533529c8d5a52800161948ad49066ef79..e302b0a979a50c8ef013ddb00a8387568536d7f4 100644 --- a/llvm/include/llvm/Analysis/MLInlineAdvisor.h +++ b/llvm/include/llvm/Analysis/MLInlineAdvisor.h @@ -41,8 +41,11 @@ public: } void onSuccessfulInlining(const MLInlineAdvice &Advice, bool CalleeWasDeleted); - +#if defined(ENABLE_ACPO) + bool isForcedToStop() const override { return ForceStop; } +#else bool isForcedToStop() const { return ForceStop; } +#endif int64_t getLocalCalls(Function &F); const MLModelRunner &getModelRunner() const { return *ModelRunner.get(); } FunctionPropertiesInfo &getCachedFPI(Function &) const; diff --git a/llvm/include/llvm/Analysis/ModelDataCollector.h b/llvm/include/llvm/Analysis/ModelDataCollector.h new file mode 100644 index 0000000000000000000000000000000000000000..ad3fc476a9b2576959d1133c3615439233725f08 --- /dev/null +++ b/llvm/include/llvm/Analysis/ModelDataCollector.h @@ -0,0 +1,108 @@ +//===- ModelDataCollector.h - Data collector for ML model -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_MODELDATACOLLECTOR_H +#define LLVM_ANALYSIS_MODELDATACOLLECTOR_H + +#if defined(ENABLE_ACPO) +#include "llvm/ADT/StringMap.h" +#include "llvm/Analysis/ACPOCollectFeatures.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/Support/raw_ostream.h" +#include +#include + +namespace llvm { +class ModelDataCollector { +public: + enum DumpOption { function, loop, before, after }; + + ModelDataCollector(formatted_raw_ostream &OS, std::string OutputFileName = "") + : OutputFileName(OutputFileName), Out(OS) {} + + ~ModelDataCollector() {} + + std::string getDumpOptionAsString(DumpOption DO); + std::string getIRFileName(StringRef Key); + std::string getOutputFileName(); + bool isEmptyOutputFile(); + //std::string generateIRFileName(autotuning::CodeRegion CR); + std::string demangleName(const std::string &Name); + std::vector> getFeatures(); + std::unique_ptr + createFile(const Twine &FilePath, const Twine &FileName, std::error_code &EC); + StringMap getIRFileNameMap(); + void + setFeatures(std::vector> NewFeatures); + void setIRFileNameMap(StringMap IRFileNameMap); + void + addFeatures(std::vector> NewFeatures); + + // Print out the features + void printRow(bool printHeader = false); + + // Create the directory structure and store IR files in their corresponding + // directory + void writeIR(Loop *L, Function *F, std::string NewIRFileName, + std::string PassName, DumpOption DumpBeforeOrAfter, + bool PrintLoop, bool PrintFunction, + bool OverwriteIRFile = false); + + // Print the loop IR to a file + void createIRFileForLoop(Loop *L, const Twine &IRFilePath, + const Twine &NewIRFileName, bool OverwriteIRFile); + + // Print the function IR to a file + void createIRFileForFunction(Function *F, const Twine &IRFilePath, + const Twine &NewIRFileName, + bool OverwriteIRFile); + + virtual void collectFeatures(Loop *L, const std::string &ModuleName, + const std::string &FuncName, + const std::string &LoopName); + + virtual void collectFeatures(); + + // FeatureCollectInfo contains the information of registered feature. + struct FeatureCollectInfo { + std::unique_ptr FeaturesInfo; + std::unique_ptr RegisteredScopes; + std::unique_ptr RegisteredGroupIDs; + std::unique_ptr GlobalInfo; + std::unique_ptr FeatureCollector; + std::string Prefix; + std::string Postfix; + }; + + void registerFeature(ACPOCollectFeatures::FeaturesInfo, std::string = "", + std::string = ""); + void registerFeature(ACPOCollectFeatures::Scopes, + ACPOCollectFeatures::FeatureInfo, std::string = "", + std::string = ""); + void registerFeature(ACPOCollectFeatures::GroupIDs, + ACPOCollectFeatures::FeatureInfo, std::string = "", + std::string = ""); + void resetRegisteredFeatures(); + +protected: + // Collected features + std::vector> Features; + // NOTE: OutputFileName being empty (null) is treated as stdout + std::string OutputFileName; + std::vector> FeatureCollectInfos; + +private: + // Stream for dumping training data + formatted_raw_ostream &Out; + StringMap IRFileNames; +}; +} // namespace llvm + +#endif // ENABLE_ACPO +#endif // LLVM_ANALYSIS_MODELDATACOLLECTOR_H diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index 80bec2d82e247a16f1efb5360407560e2134f347..89160cfd17d1c85654cfd28c1499de28e430528b 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -100,6 +100,9 @@ void initializeDomPrinterWrapperPassPass(PassRegistry &); void initializeDomViewerWrapperPassPass(PassRegistry &); void initializeDominanceFrontierWrapperPassPass(PassRegistry&); void initializeDominatorTreeWrapperPassPass(PassRegistry&); +#if defined(ENABLE_ACPO) +void initializeDumpCallsiteLegacyPass(PassRegistry &); +#endif void initializeDwarfEHPrepareLegacyPassPass(PassRegistry &); void initializeEarlyCSELegacyPassPass(PassRegistry&); void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry&); @@ -124,6 +127,7 @@ void initializeFixIrreduciblePass(PassRegistry &); void initializeFixupStatepointCallerSavedPass(PassRegistry&); void initializeFlattenCFGLegacyPassPass(PassRegistry &); void initializeFuncletLayoutPass(PassRegistry&); +void initializeCallHeightAnalysisWrapperPass(PassRegistry &); void initializeGCMachineCodeAnalysisPass(PassRegistry&); void initializeGCModuleInfoPass(PassRegistry&); void initializeGVNLegacyPassPass(PassRegistry&); @@ -132,6 +136,9 @@ void initializeGlobalsAAWrapperPassPass(PassRegistry&); void initializeGuardWideningLegacyPassPass(PassRegistry&); void initializeHardwareLoopsLegacyPass(PassRegistry&); void initializeMIRProfileLoaderPassPass(PassRegistry &); +#if defined(ENABLE_ACPO) +void initializeInlineAdvisorAnalysisWrapperPass(PassRegistry &); +#endif void initializeIRSimilarityIdentifierWrapperPassPass(PassRegistry&); void initializeIRTranslatorPass(PassRegistry&); void initializeIVUsersWrapperPassPass(PassRegistry&); @@ -149,6 +156,13 @@ void initializeInterleavedLoadCombinePass(PassRegistry &); void initializeIntervalPartitionPass(PassRegistry&); void initializeJMCInstrumenterPass(PassRegistry&); void initializeKCFIPass(PassRegistry &); +#if defined(ENABLE_ACPO) +void initializeLegacyFAMPass(PassRegistry &); +void initializeLegacyFunctionPropertiesAnalysisPass(PassRegistry &); +void initializeLegacyInlinerPassPass(PassRegistry &); +void initializeLegacyInlineSizeEstimatorAnalysisPass(PassRegistry &); +void initializeLegacyModuleInlinerWrapperPassPass(PassRegistry &); +#endif void initializeLCSSAVerificationPassPass(PassRegistry&); void initializeLCSSAWrapperPassPass(PassRegistry&); void initializeLazyBlockFrequencyInfoPassPass(PassRegistry&); diff --git a/llvm/include/llvm/Transforms/IPO.h b/llvm/include/llvm/Transforms/IPO.h index 4995b000c454244abfbb6090fa4e9db6c38893f6..6905acb261fe016752a2055f93bc7a568691453a 100644 --- a/llvm/include/llvm/Transforms/IPO.h +++ b/llvm/include/llvm/Transforms/IPO.h @@ -18,6 +18,10 @@ #include #include +#if defined(ENABLE_ACPO) +#include "llvm/Analysis/InlineAdvisor.h" +#endif + namespace llvm { class ModulePass; diff --git a/llvm/include/llvm/Transforms/IPO/Inliner.h b/llvm/include/llvm/Transforms/IPO/Inliner.h index 401aa2d3a0cc6322876339848f21c3fcbd1ba747..46a3468c927ca2ad1ab67f65023f932373be3666 100644 --- a/llvm/include/llvm/Transforms/IPO/Inliner.h +++ b/llvm/include/llvm/Transforms/IPO/Inliner.h @@ -16,6 +16,15 @@ #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" #include "llvm/IR/PassManager.h" +#if defined(ENABLE_ACPO) +#include "llvm/ADT/STLExtras.h" +#include "llvm/Analysis/BasicAliasAnalysis.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" +#include +#endif + namespace llvm { /// The inliner pass for the new pass manager. diff --git a/llvm/lib/Analysis/ACPOCollectFeatures.cpp b/llvm/lib/Analysis/ACPOCollectFeatures.cpp new file mode 100644 index 0000000000000000000000000000000000000000..daa924f2cb3b1a59c7424cbeb6fb0bc4bdcccb11 --- /dev/null +++ b/llvm/lib/Analysis/ACPOCollectFeatures.cpp @@ -0,0 +1,1260 @@ +//===- ACPOCollectFeatures.cpp - ACPO Class for Feature Collection -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements ACPOCollectFeatures class +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#include "llvm/Analysis/ACPOCollectFeatures.h" +#include "llvm/ADT/SCCIterator.h" +// The ACPOFIModel.h currently contains only the cache system for +// ACPOFIExtendedFeatures. +#include "llvm/Analysis/ACPOFIModel.h" +#include "llvm/Analysis/AssumptionCache.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/CallGraph.h" +#include "llvm/Analysis/DumpFeature.h" +#include "llvm/Analysis/FunctionPropertiesAnalysis.h" +#include "llvm/Analysis/InlineAdvisor.h" +#include "llvm/Analysis/InlineCost.h" +#include "llvm/Analysis/OptimizationRemarkEmitter.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/Instructions.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "ACPOCollectFeatures" + +namespace llvm { + +// Helper function that is used to calculate features and each function should +// registered in the CalculateFeatureMap. +static void calculateFPIRelated(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void +calculateCallerBlockFreq(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void +calculateCallSiteHeight(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void +calculateConstantParam(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void calculateCostEstimate(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void +calculateEdgeNodeCount(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void +calculateHotColdCallSite(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void calculateLoopLevel(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void +calculateMandatoryKind(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void +calculateMandatoryOnly(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void +calculateInlineCostFeatures(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void calculateACPOFIExtendedFeaturesFeatures( + ACPOCollectFeatures &ACF, const ACPOCollectFeatures::FeatureInfo &info); +static void +calculateIsIndirectCall(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void +calculateIsInInnerLoop(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void +calculateIsMustTailCall(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void calculateIsTailCall(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); +static void calculateOptCode(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &info); + +// Register FeatureIdx -> Feature name +// FeatureIdx -> Scope, Scope -> FeatureIdx +// FeatureIdx -> Group, Group -> FeatureIdx +// FeatureIdx -> Calculating function +#define REGISTER_NAME(INDEX_NAME, NAME) \ + { ACPOCollectFeatures::FeatureIndex::INDEX_NAME, NAME } +const std::unordered_map + ACPOCollectFeatures::FeatureIndexToName{ + REGISTER_NAME(SROASavings, "sroa_savings"), + REGISTER_NAME(SROALosses, "sroa_losses"), + REGISTER_NAME(LoadElimination, "load_elimination"), + REGISTER_NAME(CallPenalty, "call_penalty"), + REGISTER_NAME(CallArgumentSetup, "call_argument_setup"), + REGISTER_NAME(LoadRelativeIntrinsic, "load_relative_intrinsic"), + REGISTER_NAME(LoweredCallArgSetup, "lowered_call_arg_setup"), + REGISTER_NAME(IndirectCallPenalty, "indirect_call_penalty"), + REGISTER_NAME(JumpTablePenalty, "jump_table_penalty"), + REGISTER_NAME(CaseClusterPenalty, "case_cluster_penalty"), + REGISTER_NAME(SwitchPenalty, "switch_penalty"), + REGISTER_NAME(UnsimplifiedCommonInstructions, + "unsimplified_common_instructions"), + REGISTER_NAME(NumLoops, "num_loops"), + REGISTER_NAME(DeadBlocks, "dead_blocks"), + REGISTER_NAME(SimplifiedInstructions, "simplified_instructions"), + REGISTER_NAME(ConstantArgs, "constant_args"), + REGISTER_NAME(ConstantOffsetPtrArgs, "constant_offset_ptr_args"), + REGISTER_NAME(CallSiteCost, "callsite_cost"), + REGISTER_NAME(ColdCcPenalty, "cold_cc_penalty"), + REGISTER_NAME(LastCallToStaticBonus, "last_call_to_static_bonus"), + REGISTER_NAME(IsMultipleBlocks, "is_multiple_blocks"), + REGISTER_NAME(NestedInlines, "nested_inlines"), + REGISTER_NAME(NestedInlineCostEstimate, "nested_inline_cost_estimate"), + REGISTER_NAME(Threshold, "threshold"), + REGISTER_NAME(BasicBlockCount, "basic_block_count"), + REGISTER_NAME(BlocksReachedFromConditionalInstruction, + "conditionally_executed_blocks"), + REGISTER_NAME(Uses, "users"), + REGISTER_NAME(EdgeCount, "edge_count"), + REGISTER_NAME(NodeCount, "node_count"), + REGISTER_NAME(ColdCallSite, "cold_callsite"), + REGISTER_NAME(HotCallSite, "hot_callsite"), + REGISTER_NAME(ACPOFIExtendedFeaturesInitialSize, "InitialSize"), + REGISTER_NAME(ACPOFIExtendedFeaturesBlocks, "Blocks"), + REGISTER_NAME(ACPOFIExtendedFeaturesCalls, "Calls"), + REGISTER_NAME(ACPOFIExtendedFeaturesIsLocal, "IsLocal"), + REGISTER_NAME(ACPOFIExtendedFeaturesIsLinkOnceODR, "IsLinkOnceODR"), + REGISTER_NAME(ACPOFIExtendedFeaturesIsLinkOnce, "IsLinkOnce"), + REGISTER_NAME(ACPOFIExtendedFeaturesLoops, "Loops"), + REGISTER_NAME(ACPOFIExtendedFeaturesMaxLoopDepth, "MaxLoopDepth"), + REGISTER_NAME(ACPOFIExtendedFeaturesMaxDomTreeLevel, "MaxDomTreeLevel"), + REGISTER_NAME(ACPOFIExtendedFeaturesPtrArgs, "PtrArgs"), + REGISTER_NAME(ACPOFIExtendedFeaturesPtrCallee, "PtrCallee"), + REGISTER_NAME(ACPOFIExtendedFeaturesCallReturnPtr, "CallReturnPtr"), + REGISTER_NAME(ACPOFIExtendedFeaturesConditionalBranch, + "ConditionalBranch"), + REGISTER_NAME(ACPOFIExtendedFeaturesCBwithArg, "CBwithArg"), + REGISTER_NAME(ACPOFIExtendedFeaturesCallerHeight, "CallerHeight"), + REGISTER_NAME(ACPOFIExtendedFeaturesCallUsage, "CallUsage"), + REGISTER_NAME(ACPOFIExtendedFeaturesIsRecursive, "IsRecursive"), + REGISTER_NAME(ACPOFIExtendedFeaturesNumCallsiteInLoop, + "NumCallsiteInLoop"), + REGISTER_NAME(ACPOFIExtendedFeaturesNumOfCallUsesInLoop, + "NumOfCallUsesInLoop"), + REGISTER_NAME(ACPOFIExtendedFeaturesEntryBlockFreq, "EntryBlockFreq"), + REGISTER_NAME(ACPOFIExtendedFeaturesMaxCallsiteBlockFreq, + "MaxCallsiteBlockFreq"), + REGISTER_NAME(ACPOFIExtendedFeaturesInstructionPerBlock, + "InstructionPerBlock"), + REGISTER_NAME(ACPOFIExtendedFeaturesSuccessorPerBlock, + "SuccessorPerBlock"), + REGISTER_NAME(ACPOFIExtendedFeaturesAvgVecInstr, "AvgVecInstr"), + REGISTER_NAME(ACPOFIExtendedFeaturesAvgNestedLoopLevel, + "AvgNestedLoopLevel"), + REGISTER_NAME(ACPOFIExtendedFeaturesInstrPerLoop, "InstrPerLoop"), + REGISTER_NAME(ACPOFIExtendedFeaturesBlockWithMultipleSuccecorsPerLoop, + "BlockWithMultipleSuccecorsPerLoop"), + REGISTER_NAME(CallerBlockFreq, "block_freq"), + REGISTER_NAME(CallSiteHeight, "callsite_height"), + REGISTER_NAME(ConstantParam, "nr_ctant_params"), + REGISTER_NAME(CostEstimate, "cost_estimate"), + REGISTER_NAME(LoopLevel, "loop_level"), + REGISTER_NAME(MandatoryKind, "mandatory_kind"), + REGISTER_NAME(MandatoryOnly, "mandatory_only"), + REGISTER_NAME(OptCode, "opt_code"), + REGISTER_NAME(IsIndirectCall, "is_indirect"), + REGISTER_NAME(IsInInnerLoop, "is_in_inner_loop"), + REGISTER_NAME(IsMustTailCall, "is_must_tail"), + REGISTER_NAME(IsTailCall, "is_tail"), + REGISTER_NAME(NumOfFeatures,"num_features"), + }; +#undef REGISTER_NAME + +#define REGISTER_SCOPE(INDEX_NAME, NAME) \ + { \ + ACPOCollectFeatures::FeatureIndex::INDEX_NAME, \ + ACPOCollectFeatures::Scope::NAME \ + } +const std::unordered_map + ACPOCollectFeatures::FeatureIndexToScope{ + REGISTER_SCOPE(SROASavings, CallSite), + REGISTER_SCOPE(SROALosses, CallSite), + REGISTER_SCOPE(LoadElimination, CallSite), + REGISTER_SCOPE(CallPenalty, CallSite), + REGISTER_SCOPE(CallArgumentSetup, CallSite), + REGISTER_SCOPE(LoadRelativeIntrinsic, CallSite), + REGISTER_SCOPE(LoweredCallArgSetup, CallSite), + REGISTER_SCOPE(IndirectCallPenalty, CallSite), + REGISTER_SCOPE(JumpTablePenalty, CallSite), + REGISTER_SCOPE(CaseClusterPenalty, CallSite), + REGISTER_SCOPE(SwitchPenalty, CallSite), + REGISTER_SCOPE(UnsimplifiedCommonInstructions, CallSite), + REGISTER_SCOPE(NumLoops, CallSite), + REGISTER_SCOPE(DeadBlocks, CallSite), + REGISTER_SCOPE(SimplifiedInstructions, CallSite), + REGISTER_SCOPE(ConstantArgs, CallSite), + REGISTER_SCOPE(ConstantOffsetPtrArgs, CallSite), + REGISTER_SCOPE(CallSiteCost, CallSite), + REGISTER_SCOPE(ColdCcPenalty, CallSite), + REGISTER_SCOPE(LastCallToStaticBonus, CallSite), + REGISTER_SCOPE(IsMultipleBlocks, CallSite), + REGISTER_SCOPE(NestedInlines, CallSite), + REGISTER_SCOPE(NestedInlineCostEstimate, CallSite), + REGISTER_SCOPE(Threshold, CallSite), + REGISTER_SCOPE(BasicBlockCount, Function), + REGISTER_SCOPE(BlocksReachedFromConditionalInstruction, Function), + REGISTER_SCOPE(Uses, Function), + REGISTER_SCOPE(EdgeCount, Module), + REGISTER_SCOPE(NodeCount, Module), + REGISTER_SCOPE(ColdCallSite, CallSite), + REGISTER_SCOPE(HotCallSite, CallSite), + REGISTER_SCOPE(ACPOFIExtendedFeaturesInitialSize, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesBlocks, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesCalls, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesIsLocal, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesIsLinkOnceODR, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesIsLinkOnce, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesLoops, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesMaxLoopDepth, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesMaxDomTreeLevel, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesPtrArgs, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesPtrCallee, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesCallReturnPtr, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesConditionalBranch, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesCBwithArg, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesCallerHeight, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesCallUsage, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesIsRecursive, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesNumCallsiteInLoop, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesNumOfCallUsesInLoop, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesEntryBlockFreq, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesMaxCallsiteBlockFreq, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesInstructionPerBlock, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesSuccessorPerBlock, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesAvgVecInstr, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesAvgNestedLoopLevel, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesInstrPerLoop, Function), + REGISTER_SCOPE(ACPOFIExtendedFeaturesBlockWithMultipleSuccecorsPerLoop, + Function), + REGISTER_SCOPE(CallerBlockFreq, CallSite), + REGISTER_SCOPE(CallSiteHeight, CallSite), + REGISTER_SCOPE(ConstantParam, CallSite), + REGISTER_SCOPE(CostEstimate, CallSite), + REGISTER_SCOPE(LoopLevel, CallSite), + REGISTER_SCOPE(MandatoryKind, CallSite), + REGISTER_SCOPE(MandatoryOnly, CallSite), + REGISTER_SCOPE(OptCode, CallSite), + REGISTER_SCOPE(IsIndirectCall, CallSite), + REGISTER_SCOPE(IsInInnerLoop, CallSite), + REGISTER_SCOPE(IsMustTailCall, CallSite), + REGISTER_SCOPE(IsTailCall, CallSite), + }; +#undef REGISTER_SCOPE + +#define REGISTER_GROUP(INDEX_NAME, NAME) \ + { \ + ACPOCollectFeatures::FeatureIndex::INDEX_NAME, \ + ACPOCollectFeatures::GroupID::NAME \ + } +const std::unordered_map + ACPOCollectFeatures::FeatureIndexToGroup{ + REGISTER_GROUP(SROASavings, InlineCostFeatureGroup), + REGISTER_GROUP(SROALosses, InlineCostFeatureGroup), + REGISTER_GROUP(LoadElimination, InlineCostFeatureGroup), + REGISTER_GROUP(CallPenalty, InlineCostFeatureGroup), + REGISTER_GROUP(CallArgumentSetup, InlineCostFeatureGroup), + REGISTER_GROUP(LoadRelativeIntrinsic, InlineCostFeatureGroup), + REGISTER_GROUP(LoweredCallArgSetup, InlineCostFeatureGroup), + REGISTER_GROUP(IndirectCallPenalty, InlineCostFeatureGroup), + REGISTER_GROUP(JumpTablePenalty, InlineCostFeatureGroup), + REGISTER_GROUP(CaseClusterPenalty, InlineCostFeatureGroup), + REGISTER_GROUP(SwitchPenalty, InlineCostFeatureGroup), + REGISTER_GROUP(UnsimplifiedCommonInstructions, InlineCostFeatureGroup), + REGISTER_GROUP(NumLoops, InlineCostFeatureGroup), + REGISTER_GROUP(DeadBlocks, InlineCostFeatureGroup), + REGISTER_GROUP(SimplifiedInstructions, InlineCostFeatureGroup), + REGISTER_GROUP(ConstantArgs, InlineCostFeatureGroup), + REGISTER_GROUP(ConstantOffsetPtrArgs, InlineCostFeatureGroup), + REGISTER_GROUP(CallSiteCost, InlineCostFeatureGroup), + REGISTER_GROUP(ColdCcPenalty, InlineCostFeatureGroup), + REGISTER_GROUP(LastCallToStaticBonus, InlineCostFeatureGroup), + REGISTER_GROUP(IsMultipleBlocks, InlineCostFeatureGroup), + REGISTER_GROUP(NestedInlines, InlineCostFeatureGroup), + REGISTER_GROUP(NestedInlineCostEstimate, InlineCostFeatureGroup), + REGISTER_GROUP(Threshold, InlineCostFeatureGroup), + REGISTER_GROUP(BasicBlockCount, FPIRelated), + REGISTER_GROUP(BlocksReachedFromConditionalInstruction, FPIRelated), + REGISTER_GROUP(Uses, FPIRelated), + REGISTER_GROUP(EdgeCount, EdgeNodeCount), + REGISTER_GROUP(NodeCount, EdgeNodeCount), + REGISTER_GROUP(ColdCallSite, HotColdCallSite), + REGISTER_GROUP(HotCallSite, HotColdCallSite), + REGISTER_GROUP(ACPOFIExtendedFeaturesInitialSize, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesBlocks, ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesCalls, ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesIsLocal, ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesIsLinkOnceODR, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesIsLinkOnce, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesLoops, ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesMaxLoopDepth, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesMaxDomTreeLevel, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesPtrArgs, ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesPtrCallee, ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesCallReturnPtr, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesConditionalBranch, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesCBwithArg, ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesCallerHeight, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesCallUsage, ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesIsRecursive, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesNumCallsiteInLoop, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesNumOfCallUsesInLoop, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesEntryBlockFreq, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesMaxCallsiteBlockFreq, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesInstructionPerBlock, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesSuccessorPerBlock, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesAvgVecInstr, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesAvgNestedLoopLevel, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesInstrPerLoop, + ACPOFIExtendedFeatures), + REGISTER_GROUP(ACPOFIExtendedFeaturesBlockWithMultipleSuccecorsPerLoop, + ACPOFIExtendedFeatures), + }; +#undef REGISTER_GROUP + +// Given a map that may not be one to one. Returns the inverse mapping. +// EX: Input: A -> 1, B -> 1 +// Output: 1 -> A, 1 -> B +template +static std::multimap inverseMap(std::unordered_map Map) { + std::multimap InverseMap; + for (const auto &It : Map) { + InverseMap.insert(std::pair(It.second, It.first)); + } + return InverseMap; +} + +const std::multimap + ACPOCollectFeatures::GroupToFeatureIndices{ + inverseMap(FeatureIndexToGroup)}; + +const std::multimap + ACPOCollectFeatures::ScopeToFeatureIndices{ + inverseMap(FeatureIndexToScope)}; + +#define REGISTER_FUNCTION(INDEX_NAME, NAME) \ + { ACPOCollectFeatures::FeatureIndex::INDEX_NAME, NAME } +const std::unordered_map + ACPOCollectFeatures::CalculateFeatureMap{ + REGISTER_FUNCTION(SROASavings, calculateInlineCostFeatures), + REGISTER_FUNCTION(SROALosses, calculateInlineCostFeatures), + REGISTER_FUNCTION(LoadElimination, calculateInlineCostFeatures), + REGISTER_FUNCTION(CallPenalty, calculateInlineCostFeatures), + REGISTER_FUNCTION(CallArgumentSetup, calculateInlineCostFeatures), + REGISTER_FUNCTION(LoadRelativeIntrinsic, calculateInlineCostFeatures), + REGISTER_FUNCTION(LoweredCallArgSetup, calculateInlineCostFeatures), + REGISTER_FUNCTION(IndirectCallPenalty, calculateInlineCostFeatures), + REGISTER_FUNCTION(JumpTablePenalty, calculateInlineCostFeatures), + REGISTER_FUNCTION(CaseClusterPenalty, calculateInlineCostFeatures), + REGISTER_FUNCTION(SwitchPenalty, calculateInlineCostFeatures), + REGISTER_FUNCTION(UnsimplifiedCommonInstructions, + calculateInlineCostFeatures), + REGISTER_FUNCTION(NumLoops, calculateInlineCostFeatures), + REGISTER_FUNCTION(DeadBlocks, calculateInlineCostFeatures), + REGISTER_FUNCTION(SimplifiedInstructions, calculateInlineCostFeatures), + REGISTER_FUNCTION(ConstantArgs, calculateInlineCostFeatures), + REGISTER_FUNCTION(ConstantOffsetPtrArgs, calculateInlineCostFeatures), + REGISTER_FUNCTION(CallSiteCost, calculateInlineCostFeatures), + REGISTER_FUNCTION(ColdCcPenalty, calculateInlineCostFeatures), + REGISTER_FUNCTION(LastCallToStaticBonus, calculateInlineCostFeatures), + REGISTER_FUNCTION(IsMultipleBlocks, calculateInlineCostFeatures), + REGISTER_FUNCTION(NestedInlines, calculateInlineCostFeatures), + REGISTER_FUNCTION(NestedInlineCostEstimate, + calculateInlineCostFeatures), + REGISTER_FUNCTION(Threshold, calculateInlineCostFeatures), + REGISTER_FUNCTION(BasicBlockCount, calculateFPIRelated), + REGISTER_FUNCTION(BlocksReachedFromConditionalInstruction, + calculateFPIRelated), + REGISTER_FUNCTION(Uses, calculateFPIRelated), + REGISTER_FUNCTION(EdgeCount, calculateEdgeNodeCount), + REGISTER_FUNCTION(NodeCount, calculateEdgeNodeCount), + REGISTER_FUNCTION(ColdCallSite, calculateHotColdCallSite), + REGISTER_FUNCTION(HotCallSite, calculateHotColdCallSite), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesInitialSize, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesBlocks, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesCalls, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesIsLocal, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesIsLinkOnceODR, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesIsLinkOnce, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesLoops, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesMaxLoopDepth, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesMaxDomTreeLevel, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesPtrArgs, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesPtrCallee, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesCallReturnPtr, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesConditionalBranch, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesCBwithArg, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesCallerHeight, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesCallUsage, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesIsRecursive, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesNumCallsiteInLoop, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesNumOfCallUsesInLoop, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesEntryBlockFreq, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesMaxCallsiteBlockFreq, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesInstructionPerBlock, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesSuccessorPerBlock, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesAvgVecInstr, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesAvgNestedLoopLevel, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(ACPOFIExtendedFeaturesInstrPerLoop, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION( + ACPOFIExtendedFeaturesBlockWithMultipleSuccecorsPerLoop, + calculateACPOFIExtendedFeaturesFeatures), + REGISTER_FUNCTION(CallerBlockFreq, calculateCallerBlockFreq), + REGISTER_FUNCTION(CallSiteHeight, calculateCallSiteHeight), + REGISTER_FUNCTION(ConstantParam, calculateConstantParam), + REGISTER_FUNCTION(CostEstimate, calculateCostEstimate), + REGISTER_FUNCTION(LoopLevel, calculateLoopLevel), + REGISTER_FUNCTION(MandatoryKind, calculateMandatoryKind), + REGISTER_FUNCTION(MandatoryOnly, calculateMandatoryOnly), + REGISTER_FUNCTION(OptCode, calculateOptCode), + REGISTER_FUNCTION(IsIndirectCall, calculateIsIndirectCall), + REGISTER_FUNCTION(IsInInnerLoop, calculateIsInInnerLoop), + REGISTER_FUNCTION(IsMustTailCall, calculateIsMustTailCall), + REGISTER_FUNCTION(IsTailCall, calculateIsTailCall), + }; +#undef REGISTER_FUNCTION + +std::map ACPOCollectFeatures::FunctionLevels{}; + +ACPOCollectFeatures::ACPOCollectFeatures() {} + +ACPOCollectFeatures::ACPOCollectFeatures( + ACPOCollectFeatures::FeatureInfo GlobalInfo) + : GlobalFeatureInfo(GlobalInfo) { + assert(GlobalFeatureInfo.Idx == FeatureIndex::NumOfFeatures && + "When setting glboal FeatureInfo the Idx should always be " + "NumOfFeatures"); +} + +ACPOCollectFeatures::~ACPOCollectFeatures() {} + +void ACPOCollectFeatures::setFeatureValue(ACPOCollectFeatures::FeatureIndex Idx, + std::string Val) { + FeatureToValue[Idx] = Val; +} + +void ACPOCollectFeatures::setFeatureInfo( + ACPOCollectFeatures::FeatureIndex Idx, + ACPOCollectFeatures::FeatureInfo Info) { + assert( + (Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == Idx || getFeatureGroup(Info.Idx) == getFeatureGroup(Idx)) && + "When setting FeatureToInfo map the key and value pair should both refer " + "to the same Feature or the FeatureInfo.Idx should be NumOfFeatures."); + FeatureToInfo[Idx] = Info; +} + +void ACPOCollectFeatures::setFeatureValueAndInfo( + ACPOCollectFeatures::FeatureIndex Idx, + ACPOCollectFeatures::FeatureInfo Info, std::string Val) { + setFeatureValue(Idx, Val); + setFeatureInfo(Idx, Info); +} + +void ACPOCollectFeatures::setGlobalFeatureInfo( + ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == FeatureIndex::NumOfFeatures && + "When setting glboal FeatureInfo the Idx should always be " + "NumOfFeatures"); + GlobalFeatureInfo = Info; +} + +std::string +ACPOCollectFeatures::getFeature(ACPOCollectFeatures::FeatureIndex Idx) const { + assert(registeredFeature(Idx) && "Feature not registered"); + return FeatureToValue.find(Idx)->second; +} + +std::string +ACPOCollectFeatures::getFeatureName(ACPOCollectFeatures::FeatureIndex Idx) { + return FeatureIndexToName.find(Idx)->second; +} + +ACPOCollectFeatures::GroupID +ACPOCollectFeatures::getFeatureGroup(ACPOCollectFeatures::FeatureIndex Idx) { + return FeatureIndexToGroup.find(Idx)->second; +} + +ACPOCollectFeatures::Scope +ACPOCollectFeatures::getFeatureScope(ACPOCollectFeatures::FeatureIndex Idx) { + return FeatureIndexToScope.find(Idx)->second; +} + +std::set +ACPOCollectFeatures::getGroupFeatures(ACPOCollectFeatures::GroupID Group) { + std::set FeatureIndices; + auto Range = GroupToFeatureIndices.equal_range(Group); + for (auto It = Range.first; It != Range.second; ++It) { + FeatureIndices.insert(It->second); + } + return FeatureIndices; +} + +std::set +ACPOCollectFeatures::getScopeFeatures(ACPOCollectFeatures::Scope S) { + std::set FeatureIndices; + auto Range = ScopeToFeatureIndices.equal_range(S); + for (auto It = Range.first; It != Range.second; ++It) { + FeatureIndices.insert(It->second); + } + return FeatureIndices; +} + +bool ACPOCollectFeatures::containsFeature( + ACPOCollectFeatures::FeatureIndex Idx) { + return FeatureToValue.count(Idx) > 0; +} + +bool ACPOCollectFeatures::containsFeature( + ACPOCollectFeatures::GroupID GroupID) { + for (auto FeatureIdx : getGroupFeatures(GroupID)) { + if (!containsFeature(FeatureIdx)) + return false; + } + return true; +} + +void ACPOCollectFeatures::clearFeatureValueMap() { FeatureToValue.clear(); } + +bool ACPOCollectFeatures::registeredFeature( + ACPOCollectFeatures::FeatureIndex Idx) const { + return FeatureToValue.find(Idx) != FeatureToValue.end(); +} + +void calculateFPIRelated(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == ACPOCollectFeatures::FeatureIndex::BasicBlockCount); + + auto *FAM = Info.Managers.FAM; + auto *F = Info.SI.F; + + assert(F && FAM && "Function or FAM is nullptr"); + + auto &FPI = FAM->getResult(*F); + + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::BasicBlockCount, + Info, std::to_string(FPI.BasicBlockCount)); + ACF.setFeatureValueAndInfo( + ACPOCollectFeatures::FeatureIndex:: + BlocksReachedFromConditionalInstruction, + Info, std::to_string(FPI.BlocksReachedFromConditionalInstruction)); + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::Uses, Info, + std::to_string(FPI.Uses)); +} + +void calculateCallerBlockFreq(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == ACPOCollectFeatures::FeatureIndex::CallerBlockFreq); + + auto *CB = Info.SI.CB; + auto *FAM = Info.Managers.FAM; + + assert(CB && FAM && "CallSite or FAM is nullptr"); + + Function *F = CB->getCaller(); + BasicBlock *BB = CB->getParent(); + BlockFrequencyInfo &BFI = FAM->getResult(*F); + + uint64_t CallerBlockFreq = BFI.getBlockFreq(BB).getFrequency(); + // The model uses signed 64-bit thus we need to take care of int overflow. + if (CallerBlockFreq >= std::numeric_limits::max()) { + CallerBlockFreq = std::numeric_limits::max() - 1; + } + + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::CallerBlockFreq, + Info, std::to_string(CallerBlockFreq)); +} + +void calculateCallSiteHeight(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == ACPOCollectFeatures::FeatureIndex::CallSiteHeight); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::CallSiteHeight)) + return; + + auto *CB = Info.SI.CB; + auto *IA = Info.OI.IA; + + assert(CB && IA && "CallSite or IA is nullptr"); + + if (IA) { + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::CallSiteHeight, + Info, std::to_string(IA->getCallSiteHeight(CB))); + return; + } + LLVM_DEBUG(dbgs() << "IA was nullptr & callsite height is not set!" << "\n"); +} + +void calculateConstantParam(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == ACPOCollectFeatures::FeatureIndex::ConstantParam); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::ConstantParam)) + return; + + auto *CB = Info.SI.CB; + assert(CB && "CallSite is nullptr"); + + size_t NrCtantParams = 0; + for (auto I = CB->arg_begin(), E = CB->arg_end(); I != E; ++I) { + NrCtantParams += (isa(*I)); + } + + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::ConstantParam, + Info, std::to_string(NrCtantParams)); +} + +void calculateCostEstimate(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == ACPOCollectFeatures::FeatureIndex::CostEstimate); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::CostEstimate)) + return; + + auto *CB = Info.SI.CB; + auto *FAM = Info.Managers.FAM; + + assert(CB && FAM && "CallBase or FAM is nullptr"); + + auto &Callee = *CB->getCalledFunction(); + auto &TIR = FAM->getResult(Callee); + + auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { + return FAM->getResult(F); + }; + + int CostEstimate = 0; + auto IsCallSiteInlinable = + llvm::getInliningCostEstimate(*CB, TIR, GetAssumptionCache); + if (IsCallSiteInlinable) + CostEstimate = *IsCallSiteInlinable; + + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::CostEstimate, + Info, std::to_string(CostEstimate)); +} + +int64_t getLocalCalls(Function &F, FunctionAnalysisManager &FAM) { + return FAM.getResult(F) + .DirectCallsToDefinedFunctions; +} + +void calculateEdgeNodeCount(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + ACPOCollectFeatures::getFeatureGroup(Info.Idx) == + ACPOCollectFeatures::GroupID::EdgeNodeCount); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::GroupID::EdgeNodeCount)) + return; + + auto *M = Info.SI.M; + auto *FAM = Info.Managers.FAM; + + assert(M && FAM && "Module or FAM is nullptr"); + + int NodeCount = 0; + int EdgeCount = 0; + for (auto &F : *M) + if (!F.isDeclaration()) { + ++NodeCount; + EdgeCount += getLocalCalls(F, *FAM); + } + + std::string EdgeCountStr = std::to_string(EdgeCount); + std::string NodeCountStr = std::to_string(NodeCount); + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::EdgeCount, Info, + EdgeCountStr); + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::NodeCount, Info, + NodeCountStr); +} + +void calculateHotColdCallSite(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + ACPOCollectFeatures::getFeatureGroup(Info.Idx) == + ACPOCollectFeatures::GroupID::HotColdCallSite); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::GroupID::HotColdCallSite)) + return; + + auto *CB = Info.SI.CB; + auto *FAM = Info.Managers.FAM; + + assert(CB && FAM && "Module or FAM is nullptr"); + + auto &Caller = *CB->getCaller(); + auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & { + return FAM->getResult(F); + }; + + BlockFrequencyInfo &CallerBFI = GetBFI(Caller); + const BranchProbability ColdProb(2, 100); + auto *CallSiteBB = CB->getParent(); + auto CallSiteFreq = CallerBFI.getBlockFreq(CallSiteBB); + auto CallerEntryFreq = + CallerBFI.getBlockFreq(&(CB->getCaller()->getEntryBlock())); + bool ColdCallSite = CallSiteFreq < CallerEntryFreq * ColdProb; + auto CallerEntryFreqHot = CallerBFI.getEntryFreq(); + bool HotCallSite = (CallSiteFreq.getFrequency() >= CallerEntryFreqHot * 60); + + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::ColdCallSite, + Info, std::to_string(ColdCallSite)); + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::HotCallSite, + Info, std::to_string(HotCallSite)); +} + +void calculateLoopLevel(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == ACPOCollectFeatures::FeatureIndex::LoopLevel); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::LoopLevel)) + return; + + auto *CB = Info.SI.CB; + auto *FAM = Info.Managers.FAM; + + assert(CB && FAM && "CallBase or FAM is nullptr"); + + Function *F = CB->getCaller(); + BasicBlock *BB = CB->getParent(); + LoopInfo &LI = FAM->getResult(*F); + + std::string OptCode = std::to_string(CB->getOpcode()); + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::LoopLevel, Info, + std::to_string(LI.getLoopDepth(BB))); +} + +InlineAdvisor::MandatoryInliningKind +ACPOCollectFeatures::getMandatoryKind(CallBase &CB, + FunctionAnalysisManager &FAM, + OptimizationRemarkEmitter &ORE) { + return InlineAdvisor::getMandatoryKind(CB, FAM, ORE); +} + +void calculateMandatoryKind(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == ACPOCollectFeatures::FeatureIndex::MandatoryKind); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::MandatoryKind)) + return; + + auto *CB = Info.SI.CB; + auto *FAM = Info.Managers.FAM; + + assert(CB && FAM && "CallBase or FAM is nullptr"); + + auto &Caller = *CB->getCaller(); + auto &ORE = FAM->getResult(Caller); + auto MandatoryKind = ACPOCollectFeatures::getMandatoryKind(*CB, *FAM, ORE); + + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::MandatoryKind, + Info, std::to_string((int)MandatoryKind)); +} + +void calculateMandatoryOnly(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == ACPOCollectFeatures::FeatureIndex::MandatoryOnly); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::MandatoryOnly)) + return; + + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::MandatoryOnly, + Info, std::to_string((int)Info.OI.MandatoryOnly)); +} + +void calculateOptCode(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == ACPOCollectFeatures::FeatureIndex::OptCode); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::OptCode)) + return; + + auto *CB = Info.SI.CB; + + assert(CB && "CallBase is nullptr"); + + std::string OptCode = std::to_string(CB->getOpcode()); + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::OptCode, Info, + OptCode); +} + +void calculateInlineCostFeatures(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + (ACPOCollectFeatures::getFeatureGroup(Info.Idx) == + ACPOCollectFeatures::GroupID::InlineCostFeatureGroup)); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::GroupID::InlineCostFeatureGroup)) + return; + + auto *CB = Info.SI.CB; + auto *FAM = Info.Managers.FAM; + + assert(CB && FAM && "CallBase or FAM is nullptr"); + + auto &Callee = *CB->getCalledFunction(); + auto &TIR = FAM->getResult(Callee); + + auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { + return FAM->getResult(F); + }; + + const auto CostFeaturesOpt = + getInliningCostFeatures(*CB, TIR, GetAssumptionCache); + + for (auto Idx = + ACPOCollectFeatures::FeatureIndex::InlineCostFeatureGroupBegin + 1; + Idx != ACPOCollectFeatures::FeatureIndex::InlineCostFeatureGroupEnd; + ++Idx) { + size_t TmpIdx = + static_cast(Idx) - + static_cast( + ACPOCollectFeatures::FeatureIndex::InlineCostFeatureGroupBegin) - + 1; + ACF.setFeatureValueAndInfo( + Idx, Info, + std::to_string(CostFeaturesOpt ? CostFeaturesOpt.value()[TmpIdx] : 0)); + } +} + +static void +checkValidFFCache(Function &F, + struct ACPOFIExtendedFeatures::FunctionFeatures &FF, + DominatorTree &Tree, TargetTransformInfo &TTI, LoopInfo &LI, + bool &ValidSize, bool &ValidLoop, bool &ValidTree) { + std::optional SizeCache = ACPOFIModel::getCachedSize( + &F, ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize); + auto TTIAnalysisCache = ACPOFIModel::getTTICachedAnalysis(&F); + if (SizeCache && TTIAnalysisCache == &TTI) { + ValidSize = true; + } + + std::optional MaxDomTreeLevelCache = ACPOFIModel::getCachedSize( + &F, ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel); + auto DomCache = ACPOFIModel::getDomCachedAnalysis(&F); + if (MaxDomTreeLevelCache && DomCache == &Tree) { + ValidTree = true; + } + + std::optional LoopNumCache = ACPOFIModel::getCachedSize( + &F, ACPOFIExtendedFeatures::NamedFeatureIndex::Loops); + auto LIAnalysisCache = ACPOFIModel::getLICachedAnalysis(&F); + if (LoopNumCache && LIAnalysisCache == &LI) { + ValidLoop = true; + } +} + +static void getCachedFF(Function &F, + struct ACPOFIExtendedFeatures::FunctionFeatures &FF, + DominatorTree &Tree, TargetTransformInfo &TTI, + LoopInfo &LI) { + std::optional SizeCache = ACPOFIModel::getCachedSize( + &F, ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize); + auto TTIAnalysisCache = ACPOFIModel::getTTICachedAnalysis(&F); + if (SizeCache && TTIAnalysisCache == &TTI) { + FF[ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize] = + SizeCache.value(); + } + + std::optional MaxDomTreeLevelCache = ACPOFIModel::getCachedSize( + &F, ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel); + auto DomCache = ACPOFIModel::getDomCachedAnalysis(&F); + if (MaxDomTreeLevelCache && DomCache == &Tree) { + FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel] = + MaxDomTreeLevelCache.value(); + } + + std::optional LoopNumCache = ACPOFIModel::getCachedSize( + &F, ACPOFIExtendedFeatures::NamedFeatureIndex::Loops); + auto LIAnalysisCache = ACPOFIModel::getLICachedAnalysis(&F); + if (LoopNumCache && LIAnalysisCache == &LI) { + FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Loops] = LoopNumCache.value(); + FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth] = + ACPOFIModel::getCachedSize( + &F, ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth) + .value(); + if (LoopNumCache.value() != 0) { + FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop] = + ACPOFIModel::getCachedFloat( + &F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop) + .value(); + FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: + BlockWithMultipleSuccecorsPerLoop] = + ACPOFIModel::getCachedFloat( + &F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: + BlockWithMultipleSuccecorsPerLoop) + .value(); + FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgNestedLoopLevel] = + ACPOFIModel::getCachedFloat( + &F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: + AvgNestedLoopLevel) + .value(); + } + } +} + +static void updateCachedFF(Function &F, + struct ACPOFIExtendedFeatures::FunctionFeatures &FF, + DominatorTree &Tree, TargetTransformInfo &TTI, + LoopInfo &LI) { + ACPOFIModel::insertSizeCache( + &F, ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize, + FF[ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize]); + ACPOFIModel::insertAnalysisCache(&F, &TTI); + ACPOFIModel::insertSizeCache( + &F, ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel, + FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel]); + ACPOFIModel::insertAnalysisCache(&F, &Tree); + ACPOFIModel::insertSizeCache( + &F, ACPOFIExtendedFeatures::NamedFeatureIndex::Loops, + FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Loops]); + ACPOFIModel::insertSizeCache( + &F, ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth, + FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth]); + ACPOFIModel::insertFloatCache( + &F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop, + FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop]); + ACPOFIModel::insertFloatCache( + &F, + ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: + BlockWithMultipleSuccecorsPerLoop, + FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: + BlockWithMultipleSuccecorsPerLoop]); + ACPOFIModel::insertFloatCache( + &F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgNestedLoopLevel, + FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgNestedLoopLevel]); + ACPOFIModel::insertAnalysisCache(&F, &LI); +} + +void calculateACPOFIExtendedFeaturesFeatures( + ACPOCollectFeatures &ACF, const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + ACPOCollectFeatures::getFeatureGroup(Info.Idx) == + ACPOCollectFeatures::GroupID::ACPOFIExtendedFeatures); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::GroupID::ACPOFIExtendedFeatures)) + return; + + auto F = Info.SI.F; + auto *FAM = Info.Managers.FAM; + + assert(F && FAM && "F or FAM is nullptr"); + + struct ACPOFIExtendedFeatures::FunctionFeatures FF; + auto &DomTree = FAM->getResult(*F); + auto &TTI = FAM->getResult(*F); + auto &LI = FAM->getResult(*F); + bool ValidSize = false; + bool ValidLoop = false; + bool ValidTree = false; + checkValidFFCache(*F, FF, DomTree, TTI, LI, ValidSize, ValidLoop, ValidTree); + FF = ACPOFIExtendedFeatures::getFunctionFeatures( + *F, DomTree, TTI, LI, FAM, ValidSize, ValidLoop, ValidTree); + getCachedFF(*F, FF, DomTree, TTI, LI); + updateCachedFF(*F, FF, DomTree, TTI, LI); + + for (auto Idx = ACPOCollectFeatures::FeatureIndex:: + ACPOFIExtendedFeaturesNamedFeatureBegin + + 1; + Idx != + ACPOCollectFeatures::FeatureIndex::ACPOFIExtendedFeaturesNamedFeatureEnd; + ++Idx) { + size_t TmpIdx = + static_cast(Idx) - + static_cast(ACPOCollectFeatures::FeatureIndex:: + ACPOFIExtendedFeaturesNamedFeatureBegin) - + 1; + ACF.setFeatureValueAndInfo(Idx, Info, + std::to_string(FF.NamedFeatures[TmpIdx])); + } + for (auto Idx = ACPOCollectFeatures::FeatureIndex:: + ACPOFIExtendedFeaturesFloatFeatureBegin + + 1; + Idx != + ACPOCollectFeatures::FeatureIndex::ACPOFIExtendedFeaturesFloatFeatureEnd; + ++Idx) { + size_t TmpIdx = + static_cast(Idx) - + static_cast(ACPOCollectFeatures::FeatureIndex:: + ACPOFIExtendedFeaturesFloatFeatureBegin) - + 1; + ACF.setFeatureValueAndInfo(Idx, Info, + std::to_string(FF.NamedFloatFeatures[TmpIdx])); + } +} + +void calculateIsIndirectCall(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == ACPOCollectFeatures::FeatureIndex::IsIndirectCall); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::IsIndirectCall)) + return; + + auto *CB = Info.SI.CB; + + assert(CB && "CallBase is nullptr"); + + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::IsIndirectCall, + Info, std::to_string(CB->isIndirectCall())); +} + +void calculateIsInInnerLoop(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == ACPOCollectFeatures::FeatureIndex::IsInInnerLoop); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::IsInInnerLoop)) + return; + + auto *CB = Info.SI.CB; + auto *FAM = Info.Managers.FAM; + + assert(CB && FAM && "CallBase or FAM is nullptr"); + + auto &Caller = *CB->getCaller(); + auto &CallerLI = FAM->getResult(Caller); + + // Get loop for CB's BB. And check whether the loop is an inner most loop. + bool CallSiteInInnerLoop = false; + for (auto &L : CallerLI) { + if (L->isInnermost() && L->contains(CB)) + CallSiteInInnerLoop = true; + } + + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::IsInInnerLoop, + Info, std::to_string(CallSiteInInnerLoop)); +} + +void calculateIsMustTailCall(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == ACPOCollectFeatures::FeatureIndex::IsMustTailCall); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::IsMustTailCall)) + return; + + auto *CB = Info.SI.CB; + + assert(CB && "CallBase is nullptr"); + + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::IsMustTailCall, + Info, std::to_string(CB->isMustTailCall())); +} + +void calculateIsTailCall(ACPOCollectFeatures &ACF, + const ACPOCollectFeatures::FeatureInfo &Info) { + assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || + Info.Idx == ACPOCollectFeatures::FeatureIndex::IsTailCall); + + // Check if we already calculated the values. + if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::IsTailCall)) + return; + + auto *CB = Info.SI.CB; + + assert(CB && "CallBase is nullptr"); + + ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::IsTailCall, + Info, std::to_string(CB->isTailCall())); +} + +ACPOCollectFeatures::FeatureValueMap ACPOCollectFeatures::getFeaturesPair( + ACPOCollectFeatures::FeaturesInfo FeatureInfoVec) { + clearFeatureValueMap(); + for (auto &FeatureInfo : FeatureInfoVec) { + auto It = CalculateFeatureMap.find(FeatureInfo.Idx); + if (It == CalculateFeatureMap.end()) { + assert("Could not find the corresponding function to calculate feature"); + } + auto CalculateFunction = It->second; + CalculateFunction(*this, FeatureInfo); + LLVM_DEBUG(dbgs() << "ACPO Feature " << getFeatureName(FeatureInfo.Idx) + << ": " << FeatureToValue[FeatureInfo.Idx] << "\n"); + } + + return FeatureToValue; +} + +ACPOCollectFeatures::FeatureValueMap +ACPOCollectFeatures::getFeaturesPair(ACPOCollectFeatures::Scopes ScopeVec) { + clearFeatureValueMap(); + for (auto Scope : ScopeVec) { + for (auto FeatureIdx : getScopeFeatures(Scope)) { + auto It = CalculateFeatureMap.find(FeatureIdx); + if (It == CalculateFeatureMap.end()) { + assert( + "Could not find the corresponding function to calculate feature"); + } + auto CalculateFunction = It->second; + CalculateFunction(*this, GlobalFeatureInfo); + LLVM_DEBUG(dbgs() << "ACPO Feature " << getFeatureName(FeatureIdx) + << ": " << FeatureToValue[FeatureIdx] << "\n"); + } + } + + return FeatureToValue; +} + +ACPOCollectFeatures::FeatureValueMap +ACPOCollectFeatures::getFeaturesPair(ACPOCollectFeatures::GroupIDs GroupIDVec) { + clearFeatureValueMap(); + for (auto GroupID : GroupIDVec) { + for (auto FeatureIdx : getGroupFeatures(GroupID)) { + auto It = CalculateFeatureMap.find(FeatureIdx); + if (It == CalculateFeatureMap.end()) { + assert( + "Could not find the corresponding function to calculate feature"); + } + auto CalculateFunction = It->second; + CalculateFunction(*this, GlobalFeatureInfo); + LLVM_DEBUG(dbgs() << "ACPO Feature " << getFeatureName(FeatureIdx) + << ": " << FeatureToValue[FeatureIdx] << "\n"); + } + } + + return FeatureToValue; +} + +ACPOCollectFeatures::FeatureValueMap +ACPOCollectFeatures::getFeaturesPair(ACPOCollectFeatures::FeatureIndex Beg, + ACPOCollectFeatures::FeatureIndex End) { + assert(Beg <= End); + for (auto Idx = Beg; Idx != End; ++Idx) { + auto It = CalculateFeatureMap.find(Idx); + if (It == CalculateFeatureMap.end()) { + assert("Could not find the corresponding function to calculate feature"); + } + auto CalculateFunction = It->second; + CalculateFunction(*this, GlobalFeatureInfo); + } + + return FeatureToValue; +} + +void ACPOCollectFeatures::clearFunctionLevel() { FunctionLevels.clear(); } + +void ACPOCollectFeatures::insertFunctionLevel(const Function *F, unsigned FL) { + FunctionLevels[F] = FL; +} + +std::optional +ACPOCollectFeatures::getFunctionLevel(const Function *F) { + auto It = FunctionLevels.find(F); + if (It == FunctionLevels.end()) { + return std::nullopt; + } else { + return It->second; + } +} + +ACPOCollectFeatures::FeatureIndex operator+(ACPOCollectFeatures::FeatureIndex N, + int Counter) { + return static_cast((int)N + Counter); +} + +ACPOCollectFeatures::FeatureIndex operator-(ACPOCollectFeatures::FeatureIndex N, + int Counter) { + return static_cast((int)N - Counter); +} + +ACPOCollectFeatures::FeatureIndex & +operator++(ACPOCollectFeatures::FeatureIndex &N) { + return N = static_cast((int)N + 1); +} + +ACPOCollectFeatures::FeatureIndex +operator++(ACPOCollectFeatures::FeatureIndex &N, int) { + ACPOCollectFeatures::FeatureIndex Res = N; + ++N; + return Res; +} + +} // namespace llvm +#endif // ENABLE_ACPO diff --git a/llvm/lib/Analysis/ACPOFIModel.cpp b/llvm/lib/Analysis/ACPOFIModel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d9a647ec101220b7c065983d5771250c2083a544 --- /dev/null +++ b/llvm/lib/Analysis/ACPOFIModel.cpp @@ -0,0 +1,243 @@ +//===- ACPOFIModel.cpp - AI-Enabled Continuous Program Optimization -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the interface between ACPO and ML-guided optimizations. +// It delegates decision making to inference with a pre-trained model. +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#include "llvm/Analysis/ACPOFIModel.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Process.h" + +using namespace llvm; + +#define DEBUG_TYPE "acpo" +#define ACPO_ENV_VAR_DIR "ACPO_DIR" + +cl::opt + EnableACPOFI("enable-acpo-fi", cl::init(false), cl::Hidden, + cl::desc("Leverage ACPO ML model to decide inlining.")); + +cl::opt + EnableAOTFI("enable-acpo-fi-aot", cl::init(false), cl::Hidden, + cl::desc("Leverage AOT ML model to decide inlining.")); + +ACPOFIModel::ACPOFIModel(CallBase *CB, InlineAdvisor *IA, + OptimizationRemarkEmitter *ORE, bool OnlyMandatory, + bool UseML) + : ACPOModel(ORE, UseML), CurrentCB(CB), NotACPOAdvisor(IA), + OnlyMandatory(OnlyMandatory) { + Function *Caller = CB->getCaller(); + LLVMContext *Context = &(Caller->getContext()); + setContextPtr(Context); + if (EnableACPOFI) + // ACPO Python support + setMLIF(createPersistentPythonMLIF()); + else if (EnableAOTFI) + // ACPO AOT support + setMLIF(createPersistentCompiledMLIF()); +} + +ACPOFIModel::~ACPOFIModel() {} + +void ACPOFIModel::setMLCustomFeatures( + std::vector> FeatureValues) { + CustomFeatureValues = FeatureValues; +} + +void ACPOFIModel::sendCustomFeatures() { + // Get an ACPOMLInterface to communicate with the Python side + std::shared_ptr MLIF = getMLIF(); + MLIF->initializeFeatures("FI", CustomFeatureValues); +} + +void ACPOFIModel::recordUnattemptedInlining() { + if (NotACPOAdvice) + NotACPOAdvice->recordUnattemptedInlining(); +} + +void ACPOFIModel::recordInlining() { + if (NotACPOAdvice) + NotACPOAdvice->recordInlining(); +} + +void ACPOFIModel::recordUnsuccessfulInlining(InlineResult &IR) { + if (NotACPOAdvice) + NotACPOAdvice->recordUnsuccessfulInlining(IR); +} + +void ACPOFIModel::recordInliningWithCalleeDeleted() { + if (NotACPOAdvice) + NotACPOAdvice->recordInliningWithCalleeDeleted(); +} + +void ACPOFIModel::invalidateCache(CallBase *CB) { + if (CB) { + invalidateCache(CB->getCaller()); + } +} + +InlineAdvisor *ACPOFIModel::getNotACPOAdvisor() { return NotACPOAdvisor; } + +void ACPOFIModel::invalidateCache(const Function *F) { + for (ACPOFIExtendedFeatures::NamedFeatureIndex feature = + ACPOFIExtendedFeatures::NamedFeatureIndex(0); + feature != ACPOFIExtendedFeatures::NamedFeatureIndex::NumNamedFeatures; + ++feature) { + FeatureCache[feature].erase(F); + } + for (ACPOFIExtendedFeatures::NamedFloatFeatureIndex feature = + ACPOFIExtendedFeatures::NamedFloatFeatureIndex(0); + feature != + ACPOFIExtendedFeatures::NamedFloatFeatureIndex::NumNamedFloatFeatures; + ++feature) { + FeatureCache[feature].erase(F); + } + FunctionAnalysisCache.DomCache.erase(F); + FunctionAnalysisCache.LICache.erase(F); + FunctionAnalysisCache.TTICache.erase(F); +} + +void ACPOFIModel::clearCache() { + for (ACPOFIExtendedFeatures::NamedFeatureIndex feature = + ACPOFIExtendedFeatures::NamedFeatureIndex(0); + feature != ACPOFIExtendedFeatures::NamedFeatureIndex::NumNamedFeatures; + ++feature) { + FeatureCache[feature].clear(); + } + for (ACPOFIExtendedFeatures::NamedFloatFeatureIndex feature = + ACPOFIExtendedFeatures::NamedFloatFeatureIndex(0); + feature != + ACPOFIExtendedFeatures::NamedFloatFeatureIndex::NumNamedFloatFeatures; + ++feature) { + FeatureCache[feature].clear(); + } + FunctionAnalysisCache.DomCache.clear(); + FunctionAnalysisCache.LICache.clear(); + FunctionAnalysisCache.TTICache.clear(); +} + +std::optional +ACPOFIModel::getCachedSize(const Function *F, + ACPOFIExtendedFeatures::NamedFeatureIndex idx) { + auto it = FeatureCache[idx].find(F); + return it != FeatureCache[idx].end() ? std::optional(it->second) + : std::nullopt; +} + +std::optional ACPOFIModel::getCachedFloat( + const Function *F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex idx) { + auto it = FeatureCache[idx].find(F); + return it != FeatureCache[idx].end() ? std::optional(it->second) + : std::nullopt; +} + +void ACPOFIModel::insertSizeCache(const Function *F, + ACPOFIExtendedFeatures::NamedFeatureIndex idx, + size_t val) { + FeatureCache[idx].insert(std::make_pair(F, val)); +} + +void ACPOFIModel::insertFloatCache( + const Function *F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex idx, + float val) { + FeatureCache[idx].insert(std::make_pair(F, val)); +} + +const DominatorTree *ACPOFIModel::getDomCachedAnalysis(const Function *F) { + auto it = FunctionAnalysisCache.DomCache.find(F); + return it != FunctionAnalysisCache.DomCache.end() ? it->second : nullptr; +} + +const LoopInfo *ACPOFIModel::getLICachedAnalysis(const Function *F) { + auto it = FunctionAnalysisCache.LICache.find(F); + return it != FunctionAnalysisCache.LICache.end() ? it->second : nullptr; +} + +const TargetTransformInfo * +ACPOFIModel::getTTICachedAnalysis(const Function *F) { + auto it = FunctionAnalysisCache.TTICache.find(F); + return it != FunctionAnalysisCache.TTICache.end() ? it->second : nullptr; +} + +void ACPOFIModel::insertAnalysisCache(const Function *F, + const DominatorTree *Tree) { + FunctionAnalysisCache.DomCache.insert(std::make_pair(F, Tree)); +} + +void ACPOFIModel::insertAnalysisCache(const Function *F, const LoopInfo *LI) { + FunctionAnalysisCache.LICache.insert(std::make_pair(F, LI)); +} + +void ACPOFIModel::insertAnalysisCache(const Function *F, + const TargetTransformInfo *TTI) { + FunctionAnalysisCache.TTICache.insert(std::make_pair(F, TTI)); +} + +std::unique_ptr ACPOFIModel::getAdviceML() { + std::shared_ptr MLIF = getMLIF(); + // Generate result. + std::unique_ptr Advice = std::make_unique(); + // handle mandatory case, forcestop, never inline or not inlinable cases + if (OnlyMandatory) + return getAdviceNoML(); + if (NotACPOAdvisor->neverInline(*CurrentCB) || + !NotACPOAdvisor->isCSInlinable(*CurrentCB)) { + Advice->addField("FI-ShouldInline", + ConstantInt::get(Type::getInt64Ty(*(getContextPtr())), + (int64_t) false)); + NotACPOAdvice = nullptr; + return Advice; + } + std::optional Env = llvm::sys::Process::GetEnv(ACPO_ENV_VAR_DIR); + if (!Env || *Env == "") { + std::optional LLVMDIROpt = + llvm::sys::Process::GetEnv("LLVM_DIR"); + if (!LLVMDIROpt) { + outs() << "ACPO_DIR not found. " + << "Did you export ACPO_DIR to $LLVM_DIR/acpo ?\n" + << "Falling back to default advisor. \n"; + return getAdviceNoML(); + } + } + assert(MLIF != nullptr); + if (!MLIF->loadModel("model-fi.acpo")) { + outs() << "Model not loaded correctly. \n"; + return getAdviceNoML(); + } + if (!MLIF->initializeFeatures("FI", CustomFeatureValues)) { + outs() << "Features not initialized correctly. \n"; + return getAdviceNoML(); + } + bool ModelRunOK = MLIF->runModel("FI"); + assert(ModelRunOK); + ShouldInline = MLIF->getModelResultI("FI-ShouldInline"); + assert(getContextPtr() != nullptr); + Advice->addField("FI-ShouldInline", + ConstantInt::get(Type::getInt64Ty(*(getContextPtr())), + (int64_t)ShouldInline)); + return Advice; +} + +std::unique_ptr ACPOFIModel::getAdviceNoML() { + // Use the advisor used by default inlining + std::unique_ptr Advice = std::make_unique(); + assert(getContextPtr() != nullptr); + NotACPOAdvice = NotACPOAdvisor->getAdvice(*CurrentCB, OnlyMandatory); + bool ShouldInline = NotACPOAdvice->isInliningRecommended(); + Advice->addField("FI-ShouldInline", + ConstantInt::get(Type::getInt64Ty(*(getContextPtr())), + (int64_t)ShouldInline)); + return Advice; +} + +ACPOFIModel::FunctionFeaturesCache ACPOFIModel::FeatureCache; +ACPOFIModel::FunctionAnalysisMap ACPOFIModel::FunctionAnalysisCache; +#endif // ENABLE_ACPO diff --git a/llvm/lib/Analysis/ACPOMLInterface.cpp b/llvm/lib/Analysis/ACPOMLInterface.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f48eb46638e3e06ced371635d474cdbd9b3d6734 --- /dev/null +++ b/llvm/lib/Analysis/ACPOMLInterface.cpp @@ -0,0 +1,1407 @@ +//===- ACPOMLInterface.cpp - AI-Enabled Continuous Program Optimization ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements an interface to the ML framework. +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#include "llvm/Analysis/ACPOMLInterface.h" +#include "llvm/Analysis/ACPOModelRunner.h" +#include "llvm/Analysis/FIModelRunner.h" +#include "llvm/Analysis/TensorSpec.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/raw_ostream.h" + +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +using namespace llvm; + +#define DEBUG_TYPE "acpo" + +#define ACPO_ENV_VAR_DIR "ACPO_DIR" +#define ACPO_ML_PYTHON_INTERFACE_PY "MLInterface.py" +#define ACPO_PYTHON_EXECUTABLE "python" +#define ACPO_PIPE_PREFIX "ACPO_Pipe" + +#define RESPONSE_MODEL_LOADED "Model loaded" +#define RESPONSE_ALREADY_IN_DICT "already in dict" +#define RESPONSE_FEATURE_SET "Feature set" +#define RESPONSE_FEATURES_INITIALIZED "Features initialized" +#define RESPONSE_FEATURES_SET "Features set" +#define RESPONSE_COMPLETED "Completed" +#define RESPONSE_ACTIVE "Active" +#define RESPONSE_ERROR "ERROR" + +// Static variables + +static std::shared_ptr PersistentMLIF = nullptr; + +// Class definitions + +bool Model::registerFeature(std::string FeatureName, uint64_t FeatureID, + int Index) { + auto Find1 = NameToID.find(FeatureName); + if (Find1 != NameToID.end()) { + LLVM_DEBUG(dbgs() << "ERROR in registerFeature: Feature " << FeatureName + << " already exists\n"); + return false; + } + NameToID.insert(std::make_pair(FeatureName, FeatureID)); + IDToName.insert(std::make_pair(FeatureID, FeatureName)); + auto Find2 = IDToIndex.find(FeatureID); + if (Find2 != IDToIndex.end()) { + LLVM_DEBUG(dbgs() << "ERROR in registerFeature: Feature with ID " + << FeatureID << " already exists\n"); + return false; + } + IDToIndex.insert(std::make_pair(FeatureID, Index)); + return true; +} + +bool Model::registerInput(std::string InputName, std::string InputType) { + auto Find = InputMap.find(InputName); + if (Find != InputMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in registerInput: Input " << InputName + << " already exists\n"); + return false; + } + InputMap.insert(std::make_pair(InputName, InputType)); + return true; +} + +bool Model::registerOutput(std::string OutputName, std::string OutputType) { + auto Find = OutputMap.find(OutputName); + if (Find != OutputMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in registerOutput: Output " << OutputName + << " already exists\n"); + return false; + } + OutputMap.insert(std::make_pair(OutputName, OutputType)); + return true; +} + +int Model::getIndex(uint64_t FeatureID) const { + auto Find = IDToIndex.find(FeatureID); + assert(Find != IDToIndex.end()); + return Find->second; +} + +int Model::getIndex(std::string FeatureName) const { + auto Find = NameToID.find(FeatureName); + assert(Find != NameToID.end()); + uint64_t ID = Find->second; + return getIndex(ID); +} + +std::string Model::getName(uint64_t FeatureID) const { + auto Find = IDToName.find(FeatureID); + assert(Find != IDToName.end()); + return Find->second; +} + +bool Model::checkOutputExists(std::string OutputName) const { + return (OutputMap.find(OutputName) != OutputMap.end()); +} + +std::string Model::getInputType(std::string InputName) const { + auto Find = InputMap.find(InputName); + assert(Find != InputMap.end()); + return Find->second; +} + +std::string Model::getOutputType(std::string OutputName) const { + auto Find = OutputMap.find(OutputName); + assert(Find != OutputMap.end()); + return Find->second; +} + +ACPOMLPythonInterface::ACPOMLPythonInterface() : NextID{0} { + std::optional Env = llvm::sys::Process::GetEnv(ACPO_ENV_VAR_DIR); + if (!Env || *Env == "") { + std::optional LLVMDIROpt = + llvm::sys::Process::GetEnv("LLVM_DIR"); + if (LLVMDIROpt) { + Env = *LLVMDIROpt + "/acpo/"; + } else { + return; + } + } + + int32_t PID = (int32_t) llvm::sys::Process::getProcessId(); + std::string ExecPython = "/usr/bin/python3"; + std::string + PythonScript = *Env + "/" + std::string(ACPO_ML_PYTHON_INTERFACE_PY); + std::string PIDStr = std::to_string(PID); + std::string TimeStr = std::to_string(time(nullptr)); + std::string NameOut = + *Env + "/" + ACPO_PIPE_PREFIX + "_CMD_" + PIDStr + "_" + TimeStr; + std::string NameIn = + *Env + "/" + ACPO_PIPE_PREFIX + "_RESP_" + PIDStr + "_" + TimeStr; + StringRef Args[] = { ExecPython, PythonScript, NameOut, NameIn }; + + // Start a process and don't wait for it to finish. We want it running in + // tandem. + std::string ErrMsg; + SubProcess = + sys::ExecuteNoWait(ExecPython, Args, std::nullopt, {}, 0, &ErrMsg); + if (!SubProcess.Pid) { + // Print out error message if the process fails to start. + LLVM_DEBUG(dbgs() << ErrMsg << "\n"); + return; + } + // Yield to Python Process to set up pipes. + const int PythonProcessStartupLatency = 100; + usleep(PythonProcessStartupLatency); + + // Now link to named pipes created by the process we just started. Note that + // because the creation of this file as a pipe was done elsewhere, the + // interface here is simple. + + // First check that the response pipe has been created by attempting to open a + // file for reading. If this is not successful, then sleep for 100us to allow + // the ML interface the time to create named pipes and open the response pipe + // for writing. Once that is done, the fopen call will pass here. + + // FIXME: Support library provides robust and portable APIs for opening files + // and creating input/output streams. Use them instead of calling libc + // functions. + PipeIn = fopen(NameIn.c_str(), "r"); + if (PipeIn == nullptr) { + do { + usleep(100); + PipeIn = fopen(NameIn.c_str(), "r"); + } while (PipeIn == nullptr); + } + + // Once the response FIFO is created, then open the command FIFO for writing. + // This will complete the handshake with the MLInterface in Python. + PipeOut = fopen(NameOut.c_str(), "w"); + // Now open named pipes to the new process. + setInitialized(true); +} + +ACPOMLPythonInterface::~ACPOMLPythonInterface() { + if (SubProcess.Pid) + closeMLInterface(); + if (PipeIn) + fclose(PipeIn); + if (PipeOut) + fclose(PipeOut); + if (SubProcess.Pid) { + // Wait for the MLInterface 3 seconds and kill it. + sys::Wait(SubProcess, 3) ; + SubProcess = sys::ProcessInfo{}; + } + setInitialized(false); +} + +uint64_t ACPOMLPythonInterface::assignID() { + NextID++; + return NextID - 1; +} + +bool ACPOMLPythonInterface::loadModel(std::string ModelSpecFile) { + sendCommand("LoadModel " + ModelSpecFile); + std::string Response = getResponse(); + std::vector Tokens = tokenize(Response); + if (Tokens[0] != RESPONSE_MODEL_LOADED) { + return false; + } + if (Tokens[1] == RESPONSE_ALREADY_IN_DICT) { + LLVM_DEBUG(dbgs() << "loadModel: the model specified in " << ModelSpecFile + << " has already been loaded\n"); + return true; + } + std::string ModelName = Tokens[1]; + int NumFeatures = std::stoi(Tokens[2]); + LLVM_DEBUG(dbgs() << "Registering features: " << NumFeatures << "\n"); + registerModel(ModelName, NumFeatures); + auto ModelPtr = ModelMap.find(ModelName)->second; + std::string FeatureName = ""; + for (int I = 0; I < NumFeatures; I++) { + FeatureName = Tokens[I + 3]; + if (!registerFeature(ModelName, FeatureName, I)) { + return false; + } + } + int OutputStart = 3 + NumFeatures; + int NumOutputs = std::stoi(Tokens[OutputStart]); + ModelPtr->setNumOutputs(NumOutputs); + OutputStart++; + std::string OutputName; + std::string OutputType; + for (int I = 0; I < NumOutputs; I++) { + std::istringstream IS(Tokens[OutputStart + I]); + IS >> OutputName >> OutputType; + if (!registerOutput(ModelName, OutputName, OutputType)) { + return false; + } + } + std::string Signature = Tokens[OutputStart + NumOutputs]; + ModelPtr->setSignature(Signature); + return true; +} + +bool ACPOMLPythonInterface::registerModel(std::string ModelName, + int NumFeatures) { + auto Find = ModelMap.find(ModelName); + if (Find != ModelMap.end()) { + LLVM_DEBUG(dbgs() << "registerModel: Model " << ModelName + << " already exists\n"); + return false; + } + std::shared_ptr NewModel = std::make_shared(NumFeatures); + ModelMap.insert(std::make_pair(ModelName, NewModel)); + return true; +} + +bool ACPOMLPythonInterface::registerModel(std::string ModelName, + int NumFeatures, int NumOutputs) { + auto Find = ModelMap.find(ModelName); + if (Find != ModelMap.end()) { + LLVM_DEBUG(dbgs() << "registerModel: Model " << ModelName + << " already exists\n"); + return false; + } + std::shared_ptr NewModel = + std::make_shared(NumFeatures, NumOutputs); + ModelMap.insert(std::make_pair(ModelName, NewModel)); + return true; +} + +bool ACPOMLPythonInterface::registerFeature(std::string ModelName, + std::string FeatureName, + int Index) { + auto Find = ModelMap.find(ModelName); + assert(Find != ModelMap.end()); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in registerFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + uint64_t ID = assignID(); + return Find->second->registerFeature(FeatureName, ID, Index); +} + +bool ACPOMLPythonInterface::registerOutput(std::string ModelName, + std::string OutputName, + std::string OutputType) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in registerOutput: Model " << ModelName + << " has not been loaded\n"); + return false; + } + return Find->second->registerOutput(OutputName, OutputType); +} + +int ACPOMLPythonInterface::getNumLoadedModels() { return ModelMap.size(); } + +bool ACPOMLPythonInterface::defineInputIR(std::string Filename) { + return false; +} + +bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, + uint64_t FeatureID, + int FeatureValue) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureID); + sendCommand("SetCustomFeature " + std::to_string(Index) + " " + + std::to_string(FeatureValue)); + std::string Response = getResponse(); + return (Response.find(RESPONSE_FEATURE_SET) == 0); +} + +bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, + uint64_t FeatureID, + int64_t FeatureValue) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureID); + sendCommand("SetCustomFeature " + std::to_string(Index) + " " + + std::to_string(FeatureValue)); + std::string Response = getResponse(); + return (Response.find(RESPONSE_FEATURE_SET) == 0); +} + +bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, + uint64_t FeatureID, + double FeatureValue) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureID); + sendCommand("SetCustomFeature " + std::to_string(Index) + " " + + std::to_string(FeatureValue)); + std::string Response = getResponse(); + return (Response.find(RESPONSE_FEATURE_SET) == 0); +} + +bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, + uint64_t FeatureID, + float FeatureValue) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureID); + sendCommand("SetCustomFeature " + std::to_string(Index) + " " + + std::to_string(FeatureValue)); + std::string Response = getResponse(); + return (Response.find(RESPONSE_FEATURE_SET) == 0); +} + +bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, + uint64_t FeatureID, + bool FeatureValue) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureID); + std::string Command = "SetCustomFeature " + std::to_string(Index) + " "; + Command += FeatureValue ? "1" : "0"; + sendCommand(Command); + std::string Response = getResponse(); + return (Response.find(RESPONSE_FEATURE_SET) == 0); +} + +bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, + std::string FeatureName, + int FeatureValue) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureName); + sendCommand("SetCustomFeature " + std::to_string(Index) + " " + + std::to_string(FeatureValue)); + std::string Response = getResponse(); + return (Response.find(RESPONSE_FEATURE_SET) == 0); +} + +bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, + std::string FeatureName, + int64_t FeatureValue) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureName); + sendCommand("SetCustomFeature " + std::to_string(Index) + " " + + std::to_string(FeatureValue)); + std::string Response = getResponse(); + return (Response.find(RESPONSE_FEATURE_SET) == 0); +} + +bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, + std::string FeatureName, + double FeatureValue) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureName); + sendCommand("SetCustomFeature " + std::to_string(Index) + " " + + std::to_string(FeatureValue)); + std::string Response = getResponse(); + return (Response.find(RESPONSE_FEATURE_SET) == 0); +} + +bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, + std::string FeatureName, + float FeatureValue) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureName); + sendCommand("SetCustomFeature " + std::to_string(Index) + " " + + std::to_string(FeatureValue)); + std::string Response = getResponse(); + return (Response.find(RESPONSE_FEATURE_SET) == 0); +} + +bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, + std::string FeatureName, + bool FeatureValue) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureName); + std::string Command = "SetCustomFeature " + std::to_string(Index) + " "; + Command += FeatureValue ? "1" : "0"; + sendCommand(Command); + std::string Response = getResponse(); + std::vector Tokens = tokenize(Response); + return (Response.find(RESPONSE_FEATURE_SET) == 0); +} + +bool ACPOMLPythonInterface::initializeFeatures( + std::string ModelName, + const std::vector> &FeatureValues) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Model " << ModelName + << " has not been loaded\n"); + return false; + } + if (FeatureValues.size() > Find->second->getNumFeatures()) { + LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Invalid features\n"); + return false; + } + CurrentlyActiveModel = ModelName; + std::string Command = "InitializeFeatures " + ModelName; + for (const auto &Feature : FeatureValues) { + uint64_t FeatureID = Feature.first; + std::string FeatureValue = Feature.second; + int Index = Find->second->getIndex(FeatureID); + Command += " " + std::to_string(Index) + " " + FeatureValue; + } + sendCommand(Command); + std::string Response = getResponse(); + return (Response.find(RESPONSE_FEATURES_INITIALIZED) == 0); +} + +bool ACPOMLPythonInterface::initializeFeatures( + std::string ModelName, + const std::vector> &FeatureValues) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Model " << ModelName + << " has not been loaded\n"); + return false; + } + if (FeatureValues.size() > Find->second->getNumFeatures()) { + LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Invalid features\n"); + return false; + } + CurrentlyActiveModel = ModelName; + std::string Command = "InitializeFeatures " + ModelName; + for (const auto &Feature : FeatureValues) { + std::string FeatureName = Feature.first; + std::string FeatureValue = Feature.second; + int Index = Find->second->getIndex(FeatureName); + Command += " " + std::to_string(Index) + " " + FeatureValue; + } + sendCommand(Command); + std::string Response = getResponse(); + return (Response.find(RESPONSE_FEATURES_INITIALIZED) == 0); +} + +bool ACPOMLPythonInterface::setCustomFeatures( + std::string ModelName, + const std::vector> &FeatureValues) { + if (ModelName != CurrentlyActiveModel) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Model " << ModelName + << " has not been loaded or is not active\n"); + return false; + } + auto Find = ModelMap.find(ModelName); + if (FeatureValues.size() > Find->second->getNumFeatures()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Invalid features\n"); + return false; + } + std::string Command = "SetCustomFeatures"; + for (const auto &Feature : FeatureValues) { + uint64_t FeatureID = Feature.first; + std::string FeatureValue = Feature.second; + int Index = Find->second->getIndex(FeatureID); + Command += " " + std::to_string(Index) + " " + FeatureValue; + } + sendCommand(Command); + std::string Response = getResponse(); + return (Response.find(RESPONSE_FEATURES_SET) == 0); +} + +bool ACPOMLPythonInterface::setCustomFeatures( + std::string ModelName, + const std::vector> &FeatureValues) { + if (ModelName != CurrentlyActiveModel) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Model " << ModelName + << " has not been loaded or is not active\n"); + return false; + } + auto Find = ModelMap.find(ModelName); + if (FeatureValues.size() > Find->second->getNumFeatures()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Invalid features\n"); + return false; + } + std::string Command = "SetCustomFeatures"; + for (const auto &Feature : FeatureValues) { + std::string FeatureName = Feature.first; + std::string FeatureValue = Feature.second; + int Index = Find->second->getIndex(FeatureName); + Command += " " + std::to_string(Index) + " " + FeatureValue; + } + sendCommand(Command); + std::string Response = getResponse(); + return (Response.find(RESPONSE_FEATURES_SET) == 0); +} + +bool ACPOMLPythonInterface::runModel(std::string ModelName) { + if (ModelName != CurrentlyActiveModel) { + LLVM_DEBUG(dbgs() << "ERROR in runModel: Model " << ModelName + << " is not active\n"); + return false; + } + sendCommand("RunModel"); + std::string Response = getResponse(); + return (Response.find(RESPONSE_COMPLETED) == 0); +} + +std::string ACPOMLPythonInterface::getOutputType(std::string ModelName, + std::string OutputName) { + auto Find = ModelMap.find(ModelName); + assert(Find != ModelMap.end()); + return Find->second->getOutputType(OutputName); +} + +int ACPOMLPythonInterface::getModelResultI(std::string OutputName) { + auto Find = ModelMap.find(CurrentlyActiveModel); + assert(Find->second->checkOutputExists(OutputName)); + sendCommand("GetModelOutput " + OutputName); + std::string Response = getResponse(); + std::vector Tokens = tokenize(Response); + assert(Tokens.size() == 3); + assert(Tokens[0] == OutputName); + int Result = std::stoi(Tokens[2]); + return Result; +} + +int64_t ACPOMLPythonInterface::getModelResultI64(std::string OutputName) { + auto Find = ModelMap.find(CurrentlyActiveModel); + assert(Find->second->checkOutputExists(OutputName)); + sendCommand("GetModelOutput " + OutputName); + std::string Response = getResponse(); + std::vector Tokens = tokenize(Response); + assert(Tokens.size() == 3); + assert(Tokens[0] == OutputName); + int64_t Result = std::stol(Tokens[2]); + return Result; +} + +float ACPOMLPythonInterface::getModelResultF(std::string OutputName) { + auto Find = ModelMap.find(CurrentlyActiveModel); + assert(Find->second->checkOutputExists(OutputName)); + sendCommand("GetModelOutput " + OutputName); + std::string Response = getResponse(); + std::vector Tokens = tokenize(Response); + assert(Tokens.size() == 3); + assert(Tokens[0] == OutputName); + float Result = std::stof(Tokens[2]); + return Result; +} + +double ACPOMLPythonInterface::getModelResultD(std::string OutputName) { + auto Find = ModelMap.find(CurrentlyActiveModel); + assert(Find->second->checkOutputExists(OutputName)); + sendCommand("GetModelOutput " + OutputName); + std::string Response = getResponse(); + std::vector Tokens = tokenize(Response); + assert(Tokens.size() == 3); + assert(Tokens[0] == OutputName); + double Result = std::stod(Tokens[2]); + return Result; +} + +bool ACPOMLPythonInterface::getModelResultB(std::string OutputName) { + auto Find = ModelMap.find(CurrentlyActiveModel); + assert(Find->second->checkOutputExists(OutputName)); + sendCommand("GetModelOutput " + OutputName); + std::string Response = getResponse(); + std::vector Tokens = tokenize(Response); + assert(Tokens.size() == 3); + assert(Tokens[0] == OutputName); + return (Tokens[2] == "1"); +} + +int ACPOMLPythonInterface::getStatus() { + sendCommand("GetStatus"); + std::string Response = getResponse(); + return Response.find(RESPONSE_ACTIVE) == 0; +} + +bool ACPOMLPythonInterface::releaseModel(std::string ModelName) { + sendCommand("ReleaseModel " + ModelName); + std::string Response = getResponse(); + ModelMap.erase(ModelName); + CurrentlyActiveModel = ""; + return true; +} + +bool ACPOMLPythonInterface::closeMLInterface() { + sendCommand("CloseMLInterface"); + std::string Response = getResponse(); + return true; +} + +void ACPOMLPythonInterface::sendCommand(const std::string &Command) { + fprintf(PipeOut,"%s\n", Command.c_str()); + fflush(PipeOut); + usleep(1); +} + +void ACPOMLPythonInterface::sendCommand( + const std::vector &Features) { + for (auto I = Features.begin(); I != Features.end(); I++) { + fprintf(PipeOut,"%s\n", I->c_str()); + fflush(PipeOut); + usleep(1); + } +} + +std::string ACPOMLPythonInterface::getResponse() { + std::string Response = ""; + char Letter = getc(PipeIn); + while (Letter != '\n') { + if (feof(PipeIn)) + assert(false && "ACPO pipeline is closed unexpectively."); + + Response += Letter; + Letter = getc(PipeIn); + } + Response += '\n'; + if (Response.substr(0, 5) == RESPONSE_ERROR) { + LLVM_DEBUG(dbgs() << Response); + assert(false && "MLInterface reutrned error"); + } + return Response; +} + +std::vector +ACPOMLPythonInterface::tokenize(const std::string &Line) { + std::vector Result; + std::string Temp = Line; + auto Loc = Temp.find(","); + while (Loc != std::string::npos) { + std::string Sub = Temp.substr(0, Loc); + Result.push_back(Sub); + Temp = Temp.substr(Loc + 1); + Loc = Temp.find(","); + } + if (Temp.length() > 0) + Result.push_back(Temp); + + return Result; +} + +std::shared_ptr llvm::createPersistentPythonMLIF() { + if (PersistentMLIF == nullptr) { + PersistentMLIF = std::make_shared(); + + if (!PersistentMLIF->isInitialized()) + PersistentMLIF = nullptr; + } + return PersistentMLIF; +} + +ACPOMLCPPInterface::ACPOMLCPPInterface() { setInitialized(true); } + +ACPOMLCPPInterface::~ACPOMLCPPInterface() {} + +uint64_t ACPOMLCPPInterface::assignID() { + NextID++; + return NextID - 1; +} + +bool ACPOMLCPPInterface::loadModel(std::string ModelSpecFile) { + std::string ModelName = readModelParam(ModelSpecFile, "ModelName"); + // Check if the model is already in the dictionary + if (RunnerMap.find(ModelName) != RunnerMap.end()) { + LLVM_DEBUG(dbgs() << "loadModel: the compiled model '" << ModelName + << "' has already been loaded\n"); + return true; + } + std::vector> Features{}; + readFeatures(ModelSpecFile, Features); + std::vector> Outputs{}; + readOutputs(ModelSpecFile, Outputs); + + LLVM_DEBUG(llvm::dbgs() << "Loading compiled model with name " << ModelName + << "\n"); + + auto CreatorFunctionIterator = CreateModelRunnerMap.find(ModelName); + if (CreatorFunctionIterator == CreateModelRunnerMap.end()) { + LLVM_DEBUG(llvm::dbgs() + << ("Could not find compiled model class for model '" + + ModelName + "'\n")); + return false; + } + + auto CreatorFunction = CreatorFunctionIterator->second; + + std::string OutputKey = readModelParam(ModelSpecFile, "OutputKey"); + auto ModelRunner = CreatorFunction(Features, OutputKey); + + registerModel(ModelName, Features.size()); + RunnerMap.insert(std::make_pair(ModelName, std::move(ModelRunner))); + auto ModelPtr = ModelMap.find(ModelName)->second; + for (size_t I = 0; I < Features.size(); I++) { + if (!registerFeature(ModelName, Features[I].first, I)) { + return false; + } + if (!ModelPtr->registerInput(Features[I].first, Features[I].second)) { + return false; + } + } + + ModelPtr->setNumOutputs(Outputs.size()); + for (size_t I = 0; I < Outputs.size(); I++) { + if (!registerOutput(ModelName, Outputs[I].first, Outputs[I].second)) { + return false; + } + } + + LLVM_DEBUG(llvm::dbgs() << "Model " << ModelName + << " was successfully loaded\n"); + + // We do not need to set signature here because it is already given to make + // the precompiled model + return true; +} + +bool ACPOMLCPPInterface::registerModel(std::string ModelName, int NumFeatures) { + auto Find = ModelMap.find(ModelName); + if (Find != ModelMap.end()) { + LLVM_DEBUG(dbgs() << "registerModel: Model " << ModelName + << " already exists\n"); + return false; + } + std::shared_ptr NewModel = std::make_shared(NumFeatures); + ModelMap.insert(std::make_pair(ModelName, NewModel)); + return true; +} + +bool ACPOMLCPPInterface::registerModel(std::string ModelName, int NumFeatures, + int NumOutputs) { + auto Find = ModelMap.find(ModelName); + if (Find != ModelMap.end()) { + LLVM_DEBUG(dbgs() << "registerModel: Model " << ModelName + << " already exists\n"); + return false; + } + std::shared_ptr NewModel = + std::make_shared(NumFeatures, NumOutputs); + ModelMap.insert(std::make_pair(ModelName, NewModel)); + return true; +} + +bool ACPOMLCPPInterface::registerFeature(std::string ModelName, + std::string FeatureName, int Index) { + auto Find = ModelMap.find(ModelName); + assert(Find != ModelMap.end()); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in registerFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + uint64_t ID = assignID(); + return Find->second->registerFeature(FeatureName, ID, Index); +} + +bool ACPOMLCPPInterface::registerOutput(std::string ModelName, + std::string OutputName, + std::string OutputType) { + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in registerOutput: Model " << ModelName + << " has not been loaded\n"); + return false; + } + return Find->second->registerOutput(OutputName, OutputType); +} + +int ACPOMLCPPInterface::getNumLoadedModels() { return ModelMap.size(); } + +bool ACPOMLCPPInterface::defineInputIR(std::string Filename) { return false; } + +bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, + uint64_t FeatureID, + int FeatureValue) { + LLVM_DEBUG( + dbgs() + << "ACPOMLCPPInterface: setting custom feature of type int in model " + << ModelName << "\n"); + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureID); + std::shared_ptr Runner = + RunnerMap.find(ModelName)->second; + return Runner->setCustomFeature(Index, FeatureValue); +} + +bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, + uint64_t FeatureID, + int64_t FeatureValue) { + LLVM_DEBUG( + dbgs() + << "ACPOMLCPPInterface: setting custom feature of type double in model " + << ModelName << "\n"); + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureID); + std::shared_ptr Runner = + RunnerMap.find(ModelName)->second; + return Runner->setCustomFeature(Index, FeatureValue); +} + +bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, + uint64_t FeatureID, + double FeatureValue) { + LLVM_DEBUG( + dbgs() + << "ACPOMLCPPInterface: setting custom feature of type double in model " + << ModelName << "\n"); + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureID); + std::shared_ptr Runner = + RunnerMap.find(ModelName)->second; + return Runner->setCustomFeature(Index, FeatureValue); +} + +bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, + uint64_t FeatureID, + float FeatureValue) { + LLVM_DEBUG( + dbgs() + << "ACPOMLCPPInterface: setting custom feature of type float in model " + << ModelName << "\n"); + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureID); + std::shared_ptr Runner = + RunnerMap.find(ModelName)->second; + return Runner->setCustomFeature(Index, FeatureValue); +} + +bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, + uint64_t FeatureID, + bool FeatureValue) { + LLVM_DEBUG( + dbgs() + << "ACPOMLCPPInterface: setting custom feature of type bool in model " + << ModelName << "\n"); + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureID); + std::shared_ptr Runner = + RunnerMap.find(ModelName)->second; + return Runner->setCustomFeature(Index, FeatureValue); +} + +bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, + std::string FeatureName, + int FeatureValue) { + LLVM_DEBUG( + dbgs() + << "ACPOMLCPPInterface: setting custom feature of type int in model " + << ModelName << "\n"); + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureName); + std::shared_ptr Runner = + RunnerMap.find(ModelName)->second; + return Runner->setCustomFeature(Index, FeatureValue); +} + +bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, + std::string FeatureName, + int64_t FeatureValue) { + LLVM_DEBUG( + dbgs() + << "ACPOMLCPPInterface: setting custom feature of type int64 in model " + << ModelName << "\n"); + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureName); + std::shared_ptr Runner = + RunnerMap.find(ModelName)->second; + return Runner->setCustomFeature(Index, FeatureValue); +} + +bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, + std::string FeatureName, + double FeatureValue) { + LLVM_DEBUG( + dbgs() + << "ACPOMLCPPInterface: setting custom feature of type double in model " + << ModelName << "\n"); + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureName); + std::shared_ptr Runner = + RunnerMap.find(ModelName)->second; + return Runner->setCustomFeature(Index, FeatureValue); +} + +bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, + std::string FeatureName, + float FeatureValue) { + LLVM_DEBUG( + dbgs() + << "ACPOMLCPPInterface: setting custom feature of type float in model " + << ModelName << "\n"); + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureName); + std::shared_ptr Runner = + RunnerMap.find(ModelName)->second; + return Runner->setCustomFeature(Index, FeatureValue); +} + +bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, + std::string FeatureName, + bool FeatureValue) { + LLVM_DEBUG( + dbgs() + << "ACPOMLCPPInterface: setting custom feature of type bool in model " + << ModelName << "\n"); + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName + << " has not been loaded\n"); + return false; + } + int Index = Find->second->getIndex(FeatureName); + std::shared_ptr Runner = + RunnerMap.find(ModelName)->second; + return Runner->setCustomFeature(Index, FeatureValue); +} + +bool ACPOMLCPPInterface::initializeFeatures( + std::string ModelName, + const std::vector> &FeatureValues) { + LLVM_DEBUG(dbgs() << "Initializing features for model " << ModelName + << " using feature IDs\n"); + auto Find = ModelMap.find(ModelName); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Model " << ModelName + << " has not been loaded\n"); + return false; + } + if (FeatureValues.size() > Find->second->getNumFeatures()) { + LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Invalid features\n"); + return false; + } + CurrentlyActiveModel = ModelName; + for (const auto &Feature : FeatureValues) { + uint64_t FeatureID = Feature.first; + std::string FeatureValue = Feature.second; + + std::string FeatureType = + getInputType(ModelName, Find->second->getName(FeatureID)); + if (FeatureType == "int64") { + int64_t Value = std::stoi(FeatureValue); + setCustomFeature(ModelName, FeatureID, Value); + } else if (FeatureType == "int32") { + int32_t Value = std::stoi(FeatureValue); + setCustomFeature(ModelName, FeatureID, Value); + } else if (FeatureType == "int") { + int Value = std::stoi(FeatureValue); + setCustomFeature(ModelName, FeatureID, Value); + } else if (FeatureType == "float64") { + double Value = std::stod(FeatureValue); + setCustomFeature(ModelName, FeatureID, Value); + } else if (FeatureType == "float32") { + float Value = std::stof(FeatureValue); + setCustomFeature(ModelName, FeatureID, Value); + } else { + LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Invalid feature type " + << FeatureType << "\n"); + return false; + } + } + return true; +} + +bool ACPOMLCPPInterface::initializeFeatures( + std::string ModelName, + const std::vector> &FeatureValues) { + auto Find = ModelMap.find(ModelName); + LLVM_DEBUG(dbgs() << "Initializing features for model " << ModelName + << " using feature names\n"); + if (Find == ModelMap.end()) { + LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Model " << ModelName + << " has not been loaded\n"); + return false; + } + if (FeatureValues.size() > Find->second->getNumFeatures()) { + LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Invalid features\n"); + return false; + } + CurrentlyActiveModel = ModelName; + for (const auto &Feature : FeatureValues) { + std::string FeatureName = Feature.first; + std::string FeatureValue = Feature.second; + + std::string FeatureType = getInputType(ModelName, FeatureName); + if (FeatureType == "int64") { + int64_t Value = std::stol(FeatureValue); + setCustomFeature(ModelName, FeatureName, Value); + } else if (FeatureType == "int32") { + int32_t Value = std::stoi(FeatureValue); + setCustomFeature(ModelName, FeatureName, Value); + } else if (FeatureType == "int") { + int Value = std::stoi(FeatureValue); + setCustomFeature(ModelName, FeatureName, Value); + } else if (FeatureType == "float64") { + double Value = std::stod(FeatureValue); + setCustomFeature(ModelName, FeatureName, Value); + } else if (FeatureType == "float32") { + float Value = std::stof(FeatureValue); + setCustomFeature(ModelName, FeatureName, Value); + } else { + LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Invalid feature type " + << FeatureType << "\n"); + return false; + } + } + return true; +} + +bool ACPOMLCPPInterface::setCustomFeatures( + std::string ModelName, + const std::vector> &FeatureValues) { + if (ModelName != CurrentlyActiveModel) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Model " << ModelName + << " has not been loaded or is not active\n"); + return false; + } + auto Find = ModelMap.find(ModelName); + if (FeatureValues.size() > Find->second->getNumFeatures()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Invalid features\n"); + return false; + } + std::string Command = "SetCustomFeatures"; + for (const auto &Feature : FeatureValues) { + uint64_t FeatureID = Feature.first; + std::string FeatureValue = Feature.second; + + std::string FeatureType = + getInputType(ModelName, Find->second->getName(FeatureID)); + if (FeatureType == "int64") { + int64_t Value = std::stol(FeatureValue); + setCustomFeature(ModelName, FeatureID, Value); + } else if (FeatureType == "int32") { + int32_t Value = std::stoi(FeatureValue); + setCustomFeature(ModelName, FeatureID, Value); + } else if (FeatureType == "int") { + int Value = std::stoi(FeatureValue); + setCustomFeature(ModelName, FeatureID, Value); + } else if (FeatureType == "float64") { + double Value = std::stod(FeatureValue); + setCustomFeature(ModelName, FeatureID, Value); + } else if (FeatureType == "float32") { + float Value = std::stof(FeatureValue); + setCustomFeature(ModelName, FeatureID, Value); + } else { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Invalid feature type " + << FeatureType << "\n"); + return false; + } + } + return true; +} + +bool ACPOMLCPPInterface::setCustomFeatures( + std::string ModelName, + const std::vector> &FeatureValues) { + if (ModelName != CurrentlyActiveModel) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Model " << ModelName + << " has not been loaded or is not active\n"); + return false; + } + auto Find = ModelMap.find(ModelName); + if (FeatureValues.size() > Find->second->getNumFeatures()) { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Invalid features\n"); + return false; + } + std::string Command = "SetCustomFeatures"; + for (const auto &Feature : FeatureValues) { + std::string FeatureName = Feature.first; + std::string FeatureValueStr = Feature.second; + + std::string FeatureType = getInputType(ModelName, FeatureName); + if (FeatureType == "int64") { + int64_t FeatureValue = std::stoi(FeatureValueStr); + setCustomFeature(ModelName, FeatureName, FeatureValue); + } else if (FeatureType == "int32") { + int32_t FeatureValue = std::stoi(FeatureValueStr); + setCustomFeature(ModelName, FeatureName, FeatureValue); + } else if (FeatureType == "int") { + int FeatureValue = std::stoi(FeatureValueStr); + setCustomFeature(ModelName, FeatureName, FeatureValue); + } else if (FeatureType == "float64") { + double FeatureValue = std::stod(FeatureValueStr); + setCustomFeature(ModelName, FeatureName, FeatureValue); + } else if (FeatureType == "float32") { + float FeatureValue = std::stof(FeatureValueStr); + setCustomFeature(ModelName, FeatureName, FeatureValue); + } else { + LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Invalid feature type " + << FeatureType << "\n"); + return false; + } + } + return true; +} + +bool ACPOMLCPPInterface::runModel(std::string ModelName) { + if (ModelName != CurrentlyActiveModel) { + LLVM_DEBUG(dbgs() << "ERROR in runModel: Model " << ModelName + << " is not active\n"); + return false; + } + std::shared_ptr Runner = + RunnerMap.find(CurrentlyActiveModel)->second; + return Runner->runModel(); +} + +std::string ACPOMLCPPInterface::getInputType(std::string ModelName, + std::string InputName) { + auto Find = ModelMap.find(ModelName); + assert(Find != ModelMap.end()); + return Find->second->getInputType(InputName); +} + +std::string ACPOMLCPPInterface::getOutputType(std::string ModelName, + std::string OutputName) { + auto Find = ModelMap.find(ModelName); + assert(Find != ModelMap.end()); + return Find->second->getOutputType(OutputName); +} + +int ACPOMLCPPInterface::getModelResultI(std::string OutputName) { + std::shared_ptr Runner = + RunnerMap.find(CurrentlyActiveModel)->second; + return Runner->getModelResultI(OutputName); +} + +int64_t ACPOMLCPPInterface::getModelResultI64(std::string OutputName) { + std::shared_ptr Runner = + RunnerMap.find(CurrentlyActiveModel)->second; + return Runner->getModelResultI64(OutputName); +} + +float ACPOMLCPPInterface::getModelResultF(std::string OutputName) { + std::shared_ptr Runner = + RunnerMap.find(CurrentlyActiveModel)->second; + return Runner->getModelResultF(OutputName); +} + +double ACPOMLCPPInterface::getModelResultD(std::string OutputName) { + std::shared_ptr Runner = + RunnerMap.find(CurrentlyActiveModel)->second; + return Runner->getModelResultD(OutputName); +} + +bool ACPOMLCPPInterface::getModelResultB(std::string OutputName) { + std::shared_ptr Runner = + RunnerMap.find(CurrentlyActiveModel)->second; + return Runner->getModelResultB(OutputName); +} + +int ACPOMLCPPInterface::getStatus() { return 1; } + +bool ACPOMLCPPInterface::releaseModel(std::string ModelName) { + ModelMap.erase(ModelName); + RunnerMap.erase(ModelName); + CurrentlyActiveModel = ""; + return true; +} + +bool ACPOMLCPPInterface::closeMLInterface() { return true; } + +std::string ACPOMLCPPInterface::readModelParam(std::string FilePath, + std::string Param) { + std::optional Env = llvm::sys::Process::GetEnv(ACPO_ENV_VAR_DIR); + if (!Env || *Env == "") { + std::optional LLVMDIROpt = + llvm::sys::Process::GetEnv("LLVM_DIR"); + if (LLVMDIROpt) { + Env = *LLVMDIROpt + "/acpo/"; + } else { + return ""; + } + } + + FilePath = *Env + "/" + FilePath; + + std::ifstream FileStream{FilePath}; + + std::string Line; + while (std::getline(FileStream, Line)) { + if (Line.rfind(Param, 0) == 0) { + return Line.substr(Param.size() + 1); + } + } + return ""; +} + +void ACPOMLCPPInterface::readFeatures( + std::string FilePath, + std::vector> &Features) { + std::string Line = readModelParam(FilePath, "Features"); + while (!Line.empty()) { + // This reads the features, assuming each feature is written as + // {feature_name, feature_type} + size_t LeftBracket = Line.find("{"); + size_t Comma = Line.find(",", LeftBracket); + size_t Space = Line.find(" ", Comma); + size_t RightBracket = Line.find("}", Space); + if (LeftBracket == Line.size() || Comma == Line.size() || + Space == Line.size() || RightBracket == Line.size()) { + break; + } + std::string Feature = Line.substr(LeftBracket + 1, Comma - LeftBracket - 1); + std::string Type = Line.substr(Space + 1, RightBracket - Space - 1); + + Features.emplace_back(std::make_pair(Feature, Type)); + int oldLength = Line.size(); + Line = Line.substr(RightBracket + 1); + int newLength = Line.size(); + if (oldLength == newLength) + break; + } +} + +void ACPOMLCPPInterface::readOutputs( + std::string FilePath, + std::vector> &Outputs) { + std::string Line = readModelParam(FilePath, "Outputs"); + while (!Line.empty()) { + // This reads the features, assuming each feature is written as + // {feature_name, feature_type} + size_t LeftBracket = Line.find("{"); + size_t Comma = Line.find(",", LeftBracket); + size_t Space = Line.find(" ", Comma); + size_t RightBracket = Line.find("}", Space); + if (LeftBracket == Line.size() || Comma == Line.size() || + Space == Line.size() || RightBracket == Line.size()) { + break; + } + std::string Output = Line.substr(LeftBracket + 1, Comma - LeftBracket - 1); + std::string Type = Line.substr(Space + 1, RightBracket - Space - 1); + + Outputs.emplace_back(std::make_pair(Output, Type)); + int oldLength = Line.size(); + Line = Line.substr(RightBracket + 1); + int newLength = Line.size(); + if (oldLength == newLength) + break; + } +} + +std::shared_ptr llvm::createPersistentCompiledMLIF() { + if (PersistentMLIF == nullptr) { + PersistentMLIF = std::make_shared(); + if (!PersistentMLIF->isInitialized()) + PersistentMLIF = nullptr; + } + return PersistentMLIF; +} + +#ifdef LLVM_HAVE_TF_AOT_FICOMPILEDMODEL +std::unique_ptr +createFI(std::vector> Inputs, + StringRef Decision) { + // Context does not ever seem to be used in the model runner, + // so for now just create an empty context object + LLVMContext Ctx; + return std::make_unique(Ctx, Inputs, Decision); +} +#endif + +// Generate map using ifdefs for now, in the future we could have this +// automatically populate using macros +const std::unordered_map + ACPOMLCPPInterface::CreateModelRunnerMap = { +#ifdef LLVM_HAVE_TF_AOT_FICOMPILEDMODEL + {"FI", createFI}, +#endif +}; +#endif diff --git a/llvm/lib/Analysis/ACPOModel.cpp b/llvm/lib/Analysis/ACPOModel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fac760f37ef529ee883443c98bf6dd0cbc52fef4 --- /dev/null +++ b/llvm/lib/Analysis/ACPOModel.cpp @@ -0,0 +1,65 @@ +//===- ACPOModel.cpp - AI-Enabled Continuous Program Optimization ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the interface between ACPO and ML-guided optimizations. +// It delegates decision making to inference with a pre-trained model. +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#include "llvm/Analysis/ACPOModel.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/OptimizationRemarkEmitter.h" +#include "llvm/Support/Debug.h" +#include + +using namespace llvm; + +#define DEBUG_TYPE "acpo" + +ACPOAdvice::ACPOAdvice(std::unique_ptr &ResultFormat) { + assert(ResultFormat != nullptr); + for (auto &Entry : ResultFormat->getFieldMap()) { + reserveField(Entry.first, Entry.second.T); + } +} + +void ACPOModel::prepareModelInput() {} + +bool ACPOModel::runModel(std::unique_ptr &Result) { return true; } + +void ACPOModel::addRequiredResultField(std::string name, Type::TypeID &ID) { + ResultFormat->reserveField(name, ID); +} + +std::unique_ptr ACPOModel::getAdvice() { + if (ShouldUseML) + return getAdviceML(); + else + return getAdviceNoML(); +} + +std::unique_ptr ACPOModel::getAdviceML() { + // This needs to be filled with a mechanism to invoke a model selected + // using the ModelRunner. + sendCustomFeatures(); + prepareModelInput(); + std::unique_ptr Result = + std::make_unique(ResultFormat); + + if (runModel(Result)) + return Result; + else + return nullptr; +} + +void ACPOModel::addFeature(int64_t ID, Constant *Val) { + assert(CustomFeatureMap.find(ID) == CustomFeatureMap.end()); + CustomFeatureMap[ID] = Val; +} +#endif diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt index 9c6a70f0221f769742c7fd3f193cda0f7d3f8985..961b5037dd4899a361e3e9ea8df27eced3a7163f 100644 --- a/llvm/lib/Analysis/CMakeLists.txt +++ b/llvm/lib/Analysis/CMakeLists.txt @@ -4,6 +4,30 @@ if (DEFINED LLVM_HAVE_TF_AOT OR LLVM_HAVE_TFLITE) set(LLVM_INLINER_MODEL_CURRENT_URL "" CACHE STRING "URL to download the LLVM inliner model") + if (ACPO_AOT) + foreach (model_name model_path model_signature IN ZIP_LISTS LLVM_ACPO_MODEL_NAMES LLVM_ACPO_MODEL_PATHS LLVM_ACPO_MODEL_SIGNATURES) + set(fname ${model_name}CompiledModel) + string(TOUPPER ${fname} fname_allcaps) + if (LLVM_ACPO_OVERRIDE) + string(TOUPPER ${LLVM_ACPO_OVERRIDE_ARCH} arch_allcaps) + set(LLVM_OVERRIDE_MODEL_HEADER_${fname_allcaps} + ${LLVM_ACPO_OVERRIDE_PATH}/${fname}-${arch_allcaps}.h) + set(LLVM_OVERRIDE_MODEL_OBJECT_${fname_allcaps} + ${LLVM_ACPO_OVERRIDE_PATH}/${fname}-${arch_allcaps}.o) + endif() + + tf_find_and_compile( + ${model_path} + ${LLVM_INLINER_MODEL_CURRENT_URL} + ${LLVM_INLINER_MODEL_PATH_DEFAULT} + "" + serve + "${model_signature}" + "${fname}" + "llvm::${fname}" + ) + endforeach() + endif() if (DEFINED LLVM_HAVE_TF_AOT) tf_find_and_compile( ${LLVM_INLINER_MODEL_PATH} @@ -24,6 +48,10 @@ if (DEFINED LLVM_HAVE_TF_AOT OR LLVM_HAVE_TFLITE) endif() add_llvm_component_library(LLVMAnalysis + ACPOCollectFeatures.cpp + ACPOFIModel.cpp + ACPOMLInterface.cpp + ACPOModel.cpp AliasAnalysis.cpp AliasAnalysisEvaluator.cpp AliasSetTracker.cpp @@ -41,6 +69,7 @@ add_llvm_component_library(LLVMAnalysis CGSCCPassManager.cpp CallGraph.cpp CallGraphSCCPass.cpp + CallHeight.cpp CallPrinter.cpp CaptureTracking.cpp CmpInstAnalysis.cpp @@ -59,6 +88,8 @@ add_llvm_component_library(LLVMAnalysis DomPrinter.cpp DomTreeUpdater.cpp DominanceFrontier.cpp + DumpCallsite.cpp + DumpFeature.cpp FunctionPropertiesAnalysis.cpp GlobalsModRef.cpp GuardUtils.cpp @@ -100,6 +131,7 @@ add_llvm_component_library(LLVMAnalysis MemoryProfileInfo.cpp MemorySSA.cpp MemorySSAUpdater.cpp + ModelDataCollector.cpp ModelUnderTrainingRunner.cpp ModuleDebugInfoPrinter.cpp ModuleSummaryAnalysis.cpp diff --git a/llvm/lib/Analysis/CallHeight.cpp b/llvm/lib/Analysis/CallHeight.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e42547344796e224872ecaa22b39b5c71d3bef1e --- /dev/null +++ b/llvm/lib/Analysis/CallHeight.cpp @@ -0,0 +1,92 @@ +//===- CallHeight.cpp - CallHeight implementation ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements getting the call height of functions in a module. +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#include "llvm/Analysis/CallHeight.h" +#include "llvm/ADT/SCCIterator.h" +#include "llvm/Analysis/CallGraph.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/InitializePasses.h" + +using namespace llvm; + +#define DEBUG_TYPE "call-height" + +// Adapted from MLInlineAdvisor.cpp +CallBase *getInlinableCallSite(Instruction &I) { + if (auto *CS = dyn_cast(&I)) { + if (Function *Callee = CS->getCalledFunction()) + if (!Callee->isDeclaration()) { + return CS; + } + } + return nullptr; +} + +unsigned CallHeight::getLevel(Function &F) { return (*Levels)[&F]; } + +CallHeight::CallHeight(Module &M) + : Levels(std::make_unique>()) { + // Adapted from MLInlineAdvisor.cpp + CallGraph CG = CallGraph(M); + + for (auto I = scc_begin(&CG); !I.isAtEnd(); ++I) { + const std::vector &CGNodes = *I; + unsigned Level = 0; + for (auto *CGNode : CGNodes) { + Function *F = CGNode->getFunction(); + if (!F || F->isDeclaration()) + continue; + for (auto &I : instructions(F)) { + if (auto *CS = getInlinableCallSite(I)) { + auto *Called = CS->getCalledFunction(); + auto Pos = Levels->find(Called); + // In bottom up traversal, an inlinable callee is either in the + // same SCC, or to a function in a visited SCC. So not finding its + // level means we haven't visited it yet, meaning it's in this SCC. + if (Pos == Levels->end()) + continue; + Level = std::max(Level, Pos->second + 1); + } + } + } + for (auto *CGNode : CGNodes) { + Function *F = CGNode->getFunction(); + if (F && !F->isDeclaration()) + (*Levels)[F] = Level; + } + } +} + +AnalysisKey CallHeightAnalysis::Key; + +CallHeight CallHeightAnalysis::run(Module &M, ModuleAnalysisManager &MAM) { + return CallHeight(M); +} + +bool CallHeightAnalysisWrapper::runOnModule(Module &M) { + Result.reset(new CallHeight(M)); + return false; +} + +void CallHeightAnalysisWrapper::getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesAll(); +} + +char CallHeightAnalysisWrapper::ID = 0; +INITIALIZE_PASS(CallHeightAnalysisWrapper, DEBUG_TYPE, "Call Height Analysis", + false, true) + +Pass *llvm::createCallHeightAnalysisWrapper() { + return new CallHeightAnalysisWrapper(); +} +#endif diff --git a/llvm/lib/Analysis/DumpCallsite.cpp b/llvm/lib/Analysis/DumpCallsite.cpp new file mode 100644 index 0000000000000000000000000000000000000000..64ff0d0db8f7f2f863e8e52aa86770b6ce0f3d80 --- /dev/null +++ b/llvm/lib/Analysis/DumpCallsite.cpp @@ -0,0 +1,84 @@ +//===- DumpCallsite.cpp - DumpCallsite implementation --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the ability to dump all callsites in a given function. +// +//===----------------------------------------------------------------------===// +#if defined(ENABLE_ACPO) +#include "llvm/Analysis/DumpCallsite.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" +#include "llvm/Support/CommandLine.h" + +using namespace llvm; + +static cl::opt + IncludeDeclaration("include-declaration", cl::Hidden, + cl::desc("Also dump declaration in dump-callsite pass")); + +namespace { + +// Implementation of actual DumpCallsite +class DumpCallsite { +public: + void run(Function &F); +}; + +// Wrapper for legacy PM +class DumpCallsiteLegacy : public FunctionPass { +public: + static char ID; + DumpCallsiteLegacy() : FunctionPass(ID) {} + + bool runOnFunction(Function &F) override; +}; + +void DumpCallsite::run(Function &F) { + outs() << F.getName(); + // Get all callees from 'call' inst + for (auto &I : instructions(F)) { + // Is a call inst + if (auto *CS = dyn_cast(&I)) { + // callee is present + if (Function *Callee = CS->getCalledFunction()) { + // Not intrinsic + if (!Callee->isIntrinsic()) { + // decide whether to dump declaration + if (!Callee->isDeclaration() || IncludeDeclaration) { + outs() << " " << Callee->getName(); + } + } + } + } + } + outs() << "\n"; +} + +bool DumpCallsiteLegacy::runOnFunction(Function &F) { + DumpCallsite Impl; + Impl.run(F); + return false; +} + +} // namespace + +char DumpCallsiteLegacy::ID = 0; +INITIALIZE_PASS(DumpCallsiteLegacy, "dump-callsite", "Dump Callsite", false, + false) + +PreservedAnalyses DumpCallsitePass::run(Function &F, + FunctionAnalysisManager &FAM) { + DumpCallsite Impl; + Impl.run(F); + return PreservedAnalyses::all(); +} +#endif diff --git a/llvm/lib/Analysis/DumpFeature.cpp b/llvm/lib/Analysis/DumpFeature.cpp new file mode 100644 index 0000000000000000000000000000000000000000..52acbb2456ad9ac2641177eac01076398d2d9168 --- /dev/null +++ b/llvm/lib/Analysis/DumpFeature.cpp @@ -0,0 +1,578 @@ +//===- DumpFeature.cpp - DumpFeature implementation -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements dumping features for functions in an scc. +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#include "llvm/Analysis/DumpFeature.h" +#include "llvm/ADT/SCCIterator.h" +#include "llvm/Analysis/CallHeight.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/PassManager.h" +#include "llvm/InitializePasses.h" +#include "llvm/MC/MCAsmLayout.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/raw_ostream.h" + +#include +#include +#include + +using namespace llvm; + +bool EnableFeatureDump; +static cl::opt EnableFeatureDumpFlag( + "enable-feature-dump", cl::location(EnableFeatureDump), cl::init(false), + cl::Hidden, cl::ZeroOrMore, cl::desc("Enable Feature Dump")); + +static cl::opt + CheckPairHisto("check-pair-histo", cl::Hidden, + cl::desc("Dump instruction pairs in the histogram")); + +static cl::opt Verbose("dump-verbose", cl::Hidden, + cl::desc("Dump as human readable format")); + +static llvm::cl::opt + OutFile("feature-output", llvm::cl::desc("File for outputting features"), + llvm::cl::init("features.csv")); + +namespace { +unsigned getMaxInstructionID() { +#define LAST_OTHER_INST(NR) return NR; +#include "llvm/IR/Instruction.def" +} + +// This is a point in time - we determined including these pairs of +// consecutive instructions (in the IR layout available at inline time) as +// features improves the model performance. We want to move away from manual +// feature selection. +// The array is given in opcode pairs rather than labels because 1) labels +// weren't readily available, and 2) the successions were hand - extracted. +// +// This array must be sorted. +static const std::array, 137> + ImportantInstructionSuccessions{ + {{1, 1}, {1, 4}, {1, 5}, {1, 7}, {1, 8}, {1, 9}, {1, 11}, + {1, 12}, {1, 13}, {1, 14}, {1, 18}, {1, 20}, {1, 22}, {1, 24}, + {1, 25}, {1, 26}, {1, 27}, {1, 28}, {1, 29}, {1, 30}, {1, 31}, + {1, 32}, {1, 33}, {1, 34}, {1, 39}, {1, 40}, {1, 42}, {1, 45}, + {2, 1}, {2, 2}, {2, 13}, {2, 28}, {2, 29}, {2, 32}, {2, 33}, + {2, 34}, {2, 38}, {2, 48}, {2, 49}, {2, 53}, {2, 55}, {2, 56}, + {13, 2}, {13, 13}, {13, 26}, {13, 33}, {13, 34}, {13, 56}, {15, 27}, + {28, 2}, {28, 48}, {28, 53}, {29, 2}, {29, 33}, {29, 56}, {31, 31}, + {31, 33}, {31, 34}, {31, 49}, {32, 1}, {32, 2}, {32, 13}, {32, 15}, + {32, 28}, {32, 29}, {32, 32}, {32, 33}, {32, 34}, {32, 39}, {32, 40}, + {32, 48}, {32, 49}, {32, 53}, {32, 56}, {33, 1}, {33, 2}, {33, 32}, + {33, 33}, {33, 34}, {33, 49}, {33, 53}, {33, 56}, {34, 1}, {34, 2}, + {34, 32}, {34, 33}, {34, 34}, {34, 49}, {34, 53}, {34, 56}, {38, 34}, + {39, 57}, {40, 34}, {47, 15}, {47, 49}, {48, 2}, {48, 34}, {48, 56}, + {49, 1}, {49, 2}, {49, 28}, {49, 32}, {49, 33}, {49, 34}, {49, 39}, + {49, 49}, {49, 56}, {53, 1}, {53, 2}, {53, 28}, {53, 34}, {53, 53}, + {53, 57}, {55, 1}, {55, 28}, {55, 34}, {55, 53}, {55, 55}, {55, 56}, + {56, 1}, {56, 2}, {56, 7}, {56, 13}, {56, 32}, {56, 33}, {56, 34}, + {56, 49}, {56, 53}, {56, 56}, {56, 64}, {57, 34}, {57, 56}, {57, 57}, + {64, 1}, {64, 64}, {65, 1}, {65, 65}}}; + +size_t getSize(Function &F, TargetTransformInfo &TTI) { + size_t SumOfAllInstCost = 0; + for (const auto &BB : F) + for (const auto &I : BB) { + std::optional cost = + TTI.getInstructionCost( + &I, TargetTransformInfo::TargetCostKind::TCK_CodeSize) + .getValue(); + if (cost.has_value()) + SumOfAllInstCost += cost.value(); + } + return SumOfAllInstCost; +} + +unsigned getMaxDominatorTreeDepth(const Function &F, + const DominatorTree &Tree) { + unsigned MaxBBDepth = 0; + for (const auto &BB : F) + if (const auto *TN = Tree.getNode(&BB)) + MaxBBDepth = std::max(MaxBBDepth, TN->getLevel()); + + return MaxBBDepth; +} + +// get valid call uses and valid call uses in loop counts. +std::pair +getValidCallUsesAndInLoopCounts(Function &F, + FunctionAnalysisManager *FAM = nullptr) { + unsigned CallUses = 0; + unsigned CallUsesInLoop = 0; + + for (User *U : F.users()) { + if (CallBase *CB = dyn_cast(U)) { + ++CallUses; + BasicBlock *BB = CB->getParent(); + Function *FUser = CB->getCaller(); + auto &LI = + FAM->getResult(*FUser); + if (LI.getLoopFor(BB) != nullptr) { + ++CallUsesInLoop; + } + } + } + return std::make_pair(CallUses, CallUsesInLoop); +} +} // namespace + +// We have: 9 calculated features (the features here); 1 feature for each +// instruction opcode; and 1 feature for each manually-identified sequence. +// For the latter 2, we build a histogram: we count the number of +// occurrences of each instruction opcode or succession of instructions, +// respectively. +// Note that instruction opcodes start from 1. For convenience, we also have +// an always 0 feature for the '0' opcode, hence the extra 1. +const size_t ACPOFIExtendedFeatures::FunctionFeatures::FeatureCount = + ImportantInstructionSuccessions.size() + getMaxInstructionID() + 1 + + static_cast( + ACPOFIExtendedFeatures::NamedFloatFeatureIndex::NumNamedFloatFeatures) + + static_cast( + ACPOFIExtendedFeatures::NamedFeatureIndex::NumNamedFeatures); + +void ACPOFIExtendedFeatures::updateLoopRelatedFeatures(Function &F, + LoopInfo &LI, + FunctionFeatures &FF) { + uint64_t LoopNum = std::distance(LI.begin(), LI.end()); + + uint64_t LoopInstrCount = 0; + uint64_t BlockWithMulSuccNum = 0; + uint64_t LoopLevelSum = 0; + for (auto &L : LI) { + LoopLevelSum += static_cast(L->getLoopDepth()); + FF[NamedFeatureIndex::MaxLoopDepth] = + std::max(FF[NamedFeatureIndex::MaxLoopDepth], + static_cast(L->getLoopDepth())); + for (const BasicBlock *BB : L->getBlocks()) { + unsigned SuccCount = std::distance(succ_begin(BB), succ_end(BB)); + if (SuccCount > 1) + BlockWithMulSuccNum++; + LoopInstrCount += std::distance(BB->instructionsWithoutDebug().begin(), + BB->instructionsWithoutDebug().end()); + } + } + + FF[NamedFeatureIndex::Loops] = LoopNum; + if (LoopNum != 0) { + uint64_t q = LoopInstrCount / LoopNum; + FF[NamedFloatFeatureIndex::InstrPerLoop] = + q + ((float)(LoopInstrCount - q * LoopNum)) / LoopNum; + q = BlockWithMulSuccNum / LoopNum; + FF[NamedFloatFeatureIndex::BlockWithMultipleSuccecorsPerLoop] = + q + ((float)(BlockWithMulSuccNum - q * LoopNum)) / LoopNum; + q = LoopLevelSum / LoopNum; + FF[NamedFloatFeatureIndex::AvgNestedLoopLevel] = + q + ((float)(LoopLevelSum - q * LoopNum)) / LoopNum; + } +} + +void ACPOFIExtendedFeatures::updateBBLoopCallsiteBFFeatures( + Function &F, FunctionFeatures &FF, LoopInfo &LI, + FunctionAnalysisManager *FAM) { + // Initializations before looping + unsigned NumCallsiteInLoop = 0; + unsigned NumCallsite = 0; + uint64_t MaxCallsiteBlockFreq = 0; + uint64_t InstrNum = 0; + uint64_t SuccNum = 0; + uint64_t VecNum = 0; + uint64_t BlockNum = F.size(); + auto getPairIndex = [](size_t a, size_t b) { + auto I = llvm::find(ImportantInstructionSuccessions, std::make_pair(a, b)); + if (I == ImportantInstructionSuccessions.end()) + return -1; + return static_cast( + std::distance(ImportantInstructionSuccessions.begin(), I)); + }; + int StartID = 0; + int LastID = StartID; + + // We don't want debug calls, because they'd just add noise. + // Sum number of instructions and successors on the way + for (auto &BB : F) { + SuccNum += std::distance(succ_begin(&BB), succ_end(&BB)); + for (auto &I : BB.instructionsWithoutDebug()) { + if (CallBase *CB = dyn_cast(&I)) { + Function *Callee = CB->getCalledFunction(); + if (Callee && !Callee->isIntrinsic()) { + ++NumCallsite; + if (!Callee->isDeclaration()) { + // Check all the functions that was called and get the max block + // frequency. + uint64_t EntryFreq = + FAM->getResult(*Callee) + .getEntryFreq(); + MaxCallsiteBlockFreq = std::max(EntryFreq, MaxCallsiteBlockFreq); + } + + if (Callee != nullptr) { + // Collect the number of callsites that were invoked with a pointer + // argument. + for (auto arg = Callee->arg_begin(); arg != Callee->arg_end(); + arg++) + if (isa(arg->getType())) { + FF[NamedFeatureIndex::PtrCallee]++; + break; + } + } + + // Collect the number of callsites that returns a pointer type. + if (isa(CB->getType())) { + FF[NamedFeatureIndex::CallReturnPtr]++; + } + + // Check if the given function is recursive. + if (&F == Callee) { + FF[NamedFeatureIndex::IsRecursive] = 1; + } + + BasicBlock *BB = CB->getParent(); + // if we found a loop for the BB that Call is in, we do +1 + if (LI.getLoopFor(BB) != nullptr) { + ++NumCallsiteInLoop; + } + } + } + + auto ID = I.getOpcode(); + ++FF.InstructionHistogram[ID]; + int PairIndex = getPairIndex(LastID, ID); + if (PairIndex >= 0) + ++FF.InstructionPairHistogram[PairIndex]; + LastID = ID; + InstrNum++; + unsigned NumOp = I.getNumOperands(); + + // If instruction contains vector operand, consider it as a vector + // instruction + for (unsigned i = 0; i < NumOp; i++) { + if (isa(I.getOperand(i)->getType())) { + VecNum++; + break; + } + } + + // If this is a conditional branch, check if it uses an argument + if (const auto II = dyn_cast(&I)) + if (II->isConditional()) { + FF[NamedFeatureIndex::ConditionalBranch]++; + // find the instruction where the condition is defined. + if (auto def = dyn_cast(II->getCondition())) { + // For all operands of def check if isa (operand) then + // increment CBwithArg. + bool found = false; + for (unsigned i = 0; i < def->getNumOperands(); i++) { + if (isa(def->getOperand(i))) { + FF[NamedFeatureIndex::CBwithArg]++; + found = true; + break; + } + } + if (found) + break; + } + } + } + } + + FF[NamedFloatFeatureIndex::AvgVecInstr] = (float)VecNum / InstrNum; + FF[NamedFeatureIndex::Blocks] = BlockNum; + if (BlockNum > 0) { + uint64_t q = InstrNum / BlockNum; + FF[NamedFloatFeatureIndex::InstructionPerBlock] = + q + ((float)(InstrNum - q * BlockNum)) / BlockNum; + q = SuccNum / BlockNum; + FF[NamedFloatFeatureIndex::SuccessorPerBlock] = + q + ((float)(SuccNum - q * BlockNum)) / BlockNum; + } + + FF[NamedFeatureIndex::MaxCallsiteBlockFreq] = MaxCallsiteBlockFreq; + FF[NamedFeatureIndex::NumCallsiteInLoop] = NumCallsiteInLoop; + FF[NamedFeatureIndex::Calls] = NumCallsite; +} + +ACPOFIExtendedFeatures::FunctionFeatures +ACPOFIExtendedFeatures::getFunctionFeatures( + Function &F, DominatorTree &DomTree, TargetTransformInfo &TTI, LoopInfo &LI, + FunctionAnalysisManager *FAM, + bool ValidSize, bool ValidLoop, bool ValidTree) { + assert(llvm::is_sorted(ImportantInstructionSuccessions) && + "expected function features are sorted"); + + FunctionFeatures FF; + size_t InstrCount = getMaxInstructionID() + 1; + FF.InstructionHistogram.resize(InstrCount); + FF.InstructionPairHistogram.resize(ImportantInstructionSuccessions.size()); + + // check all the argument to see if there is a pointer type + for (auto arg = F.arg_begin(); arg != F.arg_end(); arg++) { + if (isa(arg->getType())) { + FF[NamedFeatureIndex::PtrArgs]++; + } + } + + std::pair ValidCallAndInLoopCounts = + getValidCallUsesAndInLoopCounts(F, FAM); + if (!ValidSize) + FF[NamedFeatureIndex::InitialSize] = getSize(F, TTI); + FF[NamedFeatureIndex::IsLocal] = F.hasLocalLinkage(); + FF[NamedFeatureIndex::IsLinkOnceODR] = F.hasLinkOnceODRLinkage(); + FF[NamedFeatureIndex::IsLinkOnce] = F.hasLinkOnceLinkage(); + if (!ValidTree) + FF[NamedFeatureIndex::MaxDomTreeLevel] = + getMaxDominatorTreeDepth(F, DomTree); + FF[NamedFeatureIndex::CallUsage] = ValidCallAndInLoopCounts.first; + FF[NamedFeatureIndex::NumOfCallUsesInLoop] = ValidCallAndInLoopCounts.second; + FF[NamedFeatureIndex::EntryBlockFreq] = + FAM->getResult(F) + .getEntryFreq(); + ACPOFIExtendedFeatures::updateBBLoopCallsiteBFFeatures(F, FF, LI, FAM); + if (!ValidLoop) + ACPOFIExtendedFeatures::updateLoopRelatedFeatures(F, LI, FF); + return FF; +} + +static int getCallHeight(Module &M, CallHeight *CH, Function *F) { + if (CH == nullptr) { + // If we don't have cached result (for ex, running with opt) + // We re-calculate the function level + // Or using the old pass manager + CallHeight CH = CallHeight(M); + return CH.getLevel(*F); + } + return CH->getLevel(*F); +} + +void dumpInstructionPairs(raw_fd_ostream &OS) { + for (size_t i = 0; i < ImportantInstructionSuccessions.size(); i++) { + std::pair pair = ImportantInstructionSuccessions[i]; + OS << "{" << Instruction::getOpcodeName(pair.first) << ", " + << Instruction::getOpcodeName(pair.second) << "} "; + } + OS << "\n"; +} + +void dumpFunctionFeatures(raw_fd_ostream &OS, + ACPOFIExtendedFeatures::FunctionFeatures &FF, + Function &F, bool Verbose) { + if (Verbose) { + OS << "Function Name: " << F.getName() << "\n"; + OS << "FeatureCount: " << FF.FeatureCount << "\n"; + OS << "\nAverage instructions per basic block: " + << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: + InstructionPerBlock] + << "\nAverage number of successors per block: " + << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::SuccessorPerBlock] + << "\nAverage number of vector instructions per instruction: " + << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgVecInstr] + << "\nAverage nest level per loop: " + << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgNestedLoopLevel] + << "\nAverage instructions per loop: " + << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop] + << "\nAverage blocks with multiple succssors per loop: " + << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: + BlockWithMultipleSuccecorsPerLoop] + << "\nInitial Size: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize] << "\n" + << "Blocks: " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Blocks] + << "\n" + << "Calls (Number of callsites): " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Calls] << "\n" + << "IsLocal: " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsLocal] + << "\n" + << "IsLinkOnceODR: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsLinkOnceODR] << "\n" + << "IsLinkOnce: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsLinkOnce] << "\n" + << "Loops: " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Loops] + << "\n" + << "MaxLoopDepth: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth] << "\n" + << "MaxDomTreeLevel: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel] + << "\nPointer arguments of this caller: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::PtrArgs] + << "\nCallees with pointer arguments: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::PtrCallee] + << "\nCallees that return a pointer: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallReturnPtr] + << "\nConditional Branches: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::ConditionalBranch] + << "\nConditional Branches that depends on an argument: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CBwithArg] + << "\nCaller Height of the current function: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallerHeight] + << "\nNumber of explict calls to this function: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallUsage] + << "\nIs recursive: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsRecursive] + << "\nNumber of callsites that are inside loop in this function: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::NumCallsiteInLoop] + << "\nNumber of explict calls to this function that are in loop: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::NumOfCallUsesInLoop] + << "\nBlock Frequency for the first block of this function: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::EntryBlockFreq] + << "\nMaximum of all callsites' entry Block Frequency: " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxCallsiteBlockFreq] + << "\n"; + OS << "InstructionHistogram: "; + OS << "Size: " << FF.InstructionHistogram.size() << "\n"; + for (size_t i = 0; i < FF.InstructionHistogram.size(); i++) { + OS << FF.InstructionHistogram[i] << " "; + } + OS << "\n"; + OS << "InstructionPairHistogram: "; + OS << "Size: " << FF.InstructionPairHistogram.size() << "\n"; + for (size_t i = 0; i < FF.InstructionPairHistogram.size(); i++) { + OS << FF.InstructionPairHistogram[i] << " "; + } + OS << "\n\n"; + } else { + OS << F.getName() << " " + << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: + InstructionPerBlock] + << " " + << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::SuccessorPerBlock] + << " " << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgVecInstr] + << " " + << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgNestedLoopLevel] + << " " + << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop] + << " " + << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: + BlockWithMultipleSuccecorsPerLoop] + << " " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize] + << " " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Blocks] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Calls] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsLocal] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsLinkOnceODR] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsLinkOnce] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Loops] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::PtrArgs] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::PtrCallee] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallReturnPtr] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::ConditionalBranch] + << " " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CBwithArg] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallerHeight] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallUsage] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsRecursive] << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::NumCallsiteInLoop] + << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::NumOfCallUsesInLoop] + << " " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::EntryBlockFreq] + << " " + << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxCallsiteBlockFreq] + << " "; + + for (size_t i = 0; i < FF.InstructionHistogram.size(); i++) { + OS << FF.InstructionHistogram[i] << " "; + } + for (size_t i = 0; i < FF.InstructionPairHistogram.size(); i++) { + OS << FF.InstructionPairHistogram[i] << " "; + } + OS << "\n"; + } +} + +void runAndDump(raw_fd_ostream &OS, Function *F, DominatorTree &DomTree, + TargetTransformInfo &TTI, LoopInfo &LI, Module &M, + CallHeight *CH, FunctionAnalysisManager *FAM = nullptr) { + struct ACPOFIExtendedFeatures::FunctionFeatures FF = + ACPOFIExtendedFeatures::getFunctionFeatures(*F, DomTree, TTI, LI, FAM); + // Get the call height feature + FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallerHeight] = + getCallHeight(M, CH, F); + dumpFunctionFeatures(OS, FF, *F, Verbose); +} + +std::unique_ptr setUpOS() { + // Check ACPO/llvm-project issue #112 + std::error_code FileErr; + std::unique_ptr OS( + new raw_fd_ostream(OutFile.c_str(), FileErr, llvm::sys::fs::OF_Append)); + + if (FileErr) { + llvm::errs() << "Error opening info file " << OutFile.c_str() << ": " + << FileErr.message() << "\n"; + return nullptr; + } + + if (CheckPairHisto) { + dumpInstructionPairs(*OS); + return nullptr; + } + + return OS; +} + +PreservedAnalyses DumpFeaturePass::run(LazyCallGraph::SCC &C, + CGSCCAnalysisManager &AM, + LazyCallGraph &CG, + CGSCCUpdateResult &UR) { + std::unique_ptr OS = setUpOS(); + if (!OS) + return PreservedAnalyses::all(); + + FunctionAnalysisManager &FAM = + AM.getResult(C, CG).getManager(); + + const auto &MAMProxy = AM.getResult(C, CG); + Module &M = *C.begin()->getFunction().getParent(); + CallHeight *CH = MAMProxy.getCachedResult(M); + for (LazyCallGraph::Node &N : C) { + Function *F = &N.getFunction(); + if (F->empty()) { + continue; + } + + auto &DomTree = FAM.getResult(*F); + auto &TTI = FAM.getResult(*F); + auto &LI = FAM.getResult(*F); + + runAndDump(*OS, F, DomTree, TTI, LI, M, CH, &FAM); + } + return PreservedAnalyses::all(); +} + +ACPOFIExtendedFeatures::NamedFeatureIndex & +llvm::operator++(ACPOFIExtendedFeatures::NamedFeatureIndex &n) { + return n = static_cast((int)n + 1); +} + +ACPOFIExtendedFeatures::NamedFeatureIndex +operator++(ACPOFIExtendedFeatures::NamedFeatureIndex &n, int) { + ACPOFIExtendedFeatures::NamedFeatureIndex res = n; + ++n; + return res; +} + +ACPOFIExtendedFeatures::NamedFloatFeatureIndex & +llvm::operator++(ACPOFIExtendedFeatures::NamedFloatFeatureIndex &n) { + return n = static_cast((int)n + 1); +} + +ACPOFIExtendedFeatures::NamedFloatFeatureIndex +operator++(ACPOFIExtendedFeatures::NamedFloatFeatureIndex &n, int) { + ACPOFIExtendedFeatures::NamedFloatFeatureIndex res = n; + ++n; + return res; +} +#endif diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp index f6b3c14a03459e72991c9657ee0ee7fcb32d2d7c..4df989ca2b6406c48d713f292f70f92d3414bec8 100644 --- a/llvm/lib/Analysis/InlineAdvisor.cpp +++ b/llvm/lib/Analysis/InlineAdvisor.cpp @@ -27,6 +27,25 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/raw_ostream.h" +#if defined(ENABLE_ACPO) +#include "llvm/ADT/StringRef.h" +#include + +#include "llvm/ADT/SCCIterator.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Analysis/ACPOFIModel.h" +#include "llvm/Analysis/DumpFeature.h" +#include "llvm/Analysis/FunctionPropertiesAnalysis.h" +#include "llvm/Analysis/InlineModelFeatureMaps.h" +#include "llvm/Analysis/InlineSizeEstimatorAnalysis.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/Instructions.h" +#include "llvm/InitializePasses.h" +#include "llvm/Transforms/IPO/Inliner.h" +#include +#endif + using namespace llvm; #define DEBUG_TYPE "inline" #ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL @@ -537,6 +556,46 @@ void llvm::emitInlinedIntoBasedOnCost( PassName); } +#if defined(ENABLE_ACPO) +CallBase *InlineAdvisor::getInlinableCS(Instruction &I) { + if (auto *CS = dyn_cast(&I)) + if (Function *Callee = CS->getCalledFunction()) { + if (!Callee->isDeclaration()) { + return CS; + } + } + return nullptr; +} + +// TODO: We can make this faster on large programs by applying +// this patch from MLGO f46dd19b480496d2ba0a57d12935882e530f2b93. +// This patch incrementally computes FunctionPropertiesInfo +// instead of recomputing. +int64_t InlineAdvisor::getLocalCalls(Function &F) { + return FAM.getResult(F) + .DirectCallsToDefinedFunctions; +} + +unsigned InlineAdvisor::getCallLoopLevel(CallBase &CB) const { + Function *F = CB.getCaller(); + BasicBlock *BB = CB.getParent(); + LoopInfo &LI = FAM.getResult(*F); + return LI.getLoopDepth(BB); +} + +uint64_t InlineAdvisor::getCalleeBlockFreq(CallBase &CB) const { + Function *F = CB.getCaller(); + BasicBlock *BB = CB.getParent(); + BlockFrequencyInfo &BFI = FAM.getResult(*F); + return BFI.getBlockFreq(BB).getFrequency(); +} + +unsigned InlineAdvisor::getCallSiteHeight(CallBase *CB) { + Function *Caller = CB->getCaller(); + return FunctionLevels[Caller]; +} +#endif + InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, std::optional IC) : M(M), FAM(FAM), IC(IC), @@ -548,6 +607,35 @@ InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, std::make_unique(); ImportedFunctionsStats->setModuleInfo(M); } +#if defined(ENABLE_ACPO) + std::unique_ptr CG(std::make_unique(M)); + for (auto I = scc_begin(CG.get()); !I.isAtEnd(); ++I) { + const std::vector &CGNodes = *I; + unsigned Level = 0; + for (auto *CGNode : CGNodes) { + Function *F = CGNode->getFunction(); + if (!F || F->isDeclaration()) + continue; + for (auto &I : instructions(F)) { + if (auto *CS = getInlinableCS(I)) { + auto *Called = CS->getCalledFunction(); + auto Pos = FunctionLevels.find(Called); + // In bottom up traversal, an inlinable callee is either in the + // same SCC, or to a function in a visited SCC. So not finding its + // level means we haven't visited it yet, meaning it's in this SCC. + if (Pos == FunctionLevels.end()) + continue; + Level = std::max(Level, Pos->second + 1); + } + } + } + for (auto *CGNode : CGNodes) { + Function *F = CGNode->getFunction(); + if (F && !F->isDeclaration()) + FunctionLevels[F] = Level; + } + } +#endif } InlineAdvisor::~InlineAdvisor() { @@ -639,6 +727,33 @@ std::unique_ptr InlineAdvisor::getAdvice(CallBase &CB, return getMandatoryAdvice(CB, Advice); } +#if defined(ENABLE_ACPO) +bool InlineAdvisor::neverInline(CallBase &CB) const { + auto &Caller = *CB.getCaller(); + auto &Callee = *CB.getCalledFunction(); + auto &ORE = FAM.getResult(Caller); + auto MandatoryKind = InlineAdvisor::getMandatoryKind(CB, FAM, ORE); + // If this is a "never inline" case, there won't be any changes to internal + // state we need to track, so we can just return the base InlineAdvice, + // which will do nothing interesting. Same thing if this is a recursive + // case. + return MandatoryKind == InlineAdvisor::MandatoryInliningKind::Never || + &Caller == &Callee; +} + +bool InlineAdvisor::isCSInlinable(CallBase &CB) const { + auto &Callee = *CB.getCalledFunction(); + + auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { + return FAM.getResult(F); + }; + auto &TIR = FAM.getResult(Callee); + auto IsCallSiteInlinable = + llvm::getInliningCostEstimate(CB, TIR, GetAssumptionCache); + return !!IsCallSiteInlinable; +} +#endif + OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) { return FAM.getResult(*CB.getCaller()); } diff --git a/llvm/lib/Analysis/MLInlineAdvisor.cpp b/llvm/lib/Analysis/MLInlineAdvisor.cpp index 0660a9993b6dd7b34a31e55387f766ff5c318879..c7ea2eb8ffe94426f57bc682c4dc09a4644b2a92 100644 --- a/llvm/lib/Analysis/MLInlineAdvisor.cpp +++ b/llvm/lib/Analysis/MLInlineAdvisor.cpp @@ -323,6 +323,11 @@ std::unique_ptr MLInlineAdvisor::getAdviceImpl(CallBase &CB) { auto &Caller = *CB.getCaller(); auto &Callee = *CB.getCalledFunction(); +#if defined(ENABLE_ACPO) + LLVM_DEBUG(dbgs() << "Advice on call: " << Caller.getName() << " to " + << Callee.getName() << "\n"); +#endif + auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { return FAM.getResult(F); }; diff --git a/llvm/lib/Analysis/ModelDataCollector.cpp b/llvm/lib/Analysis/ModelDataCollector.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5d599bff25a409e62b3aa9c25ea1d8b3a54099e1 --- /dev/null +++ b/llvm/lib/Analysis/ModelDataCollector.cpp @@ -0,0 +1,350 @@ +//===- ModelDataCollector.cpp - Data collector for ML model --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the collection and dumping of data for the ML models +// +//===----------------------------------------------------------------------===// + +#if defined(ENABLE_ACPO) +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ModelDataCollector.h" +#include "llvm/Demangle/Demangle.h" +#include "llvm/IR/Function.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" + +using namespace llvm; + +#define DEBUG_TYPE "model-data-collector" + +// Defined in 'lib/IR/AsmWriter.cpp' +extern cl::opt UnnamedVariablePrefix; + +static cl::opt IRFileDirectory( + "IR-file-directory", cl::Hidden, + cl::desc("Name of a directory to store IR files.")); + +cl::opt + ACPOModelFile("acpo-dump-file", cl::init("-"), cl::Hidden, + cl::desc("Name of a file to store feature data in.")); + +std::string ModelDataCollector::getDumpOptionAsString(DumpOption DO) { + switch (DO) { + case DumpOption::loop: + return "loop"; + case DumpOption::function: + return "function"; + case DumpOption::before: + return "before"; + case DumpOption::after: + return "after"; + default: + return ""; + } +} + +std::vector> ModelDataCollector::getFeatures() { + return Features; +} + +StringMap ModelDataCollector::getIRFileNameMap() { + return IRFileNames; +} + +std::string ModelDataCollector::getOutputFileName() { return OutputFileName; } + +bool ModelDataCollector::isEmptyOutputFile() { + if (OutputFileName.empty()) + return false; + + if (!sys::fs::exists(OutputFileName)) + return true; + + uint64_t Size; + std::error_code EC = sys::fs::file_size(OutputFileName, Size); + if (EC) { + llvm::errs() << "Cannot get file size: " << EC.message() << "\n"; + assert(false && "Cannot get file size."); + } + + if (Size == 0) + return true; + + return false; +} + +void ModelDataCollector::collectFeatures(Loop *L, const std::string &ModuleName, + const std::string &FuncName, const std::string &LoopName) { +} + +void ModelDataCollector::collectFeatures() { + for (auto &FeatureCollectInfo : FeatureCollectInfos) { + ACPOCollectFeatures::FeatureValueMap FeatureMap; + + if (FeatureCollectInfo->FeaturesInfo.get()) { + FeatureMap = FeatureCollectInfo->FeatureCollector->getFeaturesPair( + *FeatureCollectInfo->FeaturesInfo.get()); + } else if (FeatureCollectInfo->RegisteredScopes.get()) { + FeatureCollectInfo->FeatureCollector->setGlobalFeatureInfo( + *FeatureCollectInfo->GlobalInfo.get()); + FeatureMap = FeatureCollectInfo->FeatureCollector->getFeaturesPair( + *FeatureCollectInfo->RegisteredScopes.get()); + } else if (FeatureCollectInfo->RegisteredGroupIDs.get()) { + FeatureCollectInfo->FeatureCollector->setGlobalFeatureInfo( + *FeatureCollectInfo->GlobalInfo.get()); + FeatureMap = FeatureCollectInfo->FeatureCollector->getFeaturesPair( + *FeatureCollectInfo->RegisteredGroupIDs.get()); + } else { + outs() << "No Features are collected, since the given " + "FeatureCollectInfo is invalid.\n"; + return; + } + for (auto const &[key, val] : FeatureMap) { + std::string FeatureName; + + if (FeatureCollectInfo->Prefix != "") + FeatureName += FeatureCollectInfo->Prefix + "_"; + + FeatureName += ACPOCollectFeatures::getFeatureName(key); + + if (FeatureCollectInfo->Postfix != "") + FeatureName += "_" + FeatureCollectInfo->Postfix; + + Features.insert(Features.end(), {std::make_pair(FeatureName, val)}); + } + } +} + +void ModelDataCollector::registerFeature(ACPOCollectFeatures::FeaturesInfo Info, + std::string Pre, std::string Post) { + std::unique_ptr tmp = + std::make_unique(); + tmp->FeaturesInfo.reset(new ACPOCollectFeatures::FeaturesInfo{Info}); + tmp->FeatureCollector.reset(new ACPOCollectFeatures{}); + tmp->Prefix = Pre; + tmp->Postfix = Post; + + FeatureCollectInfos.push_back(std::move(tmp)); +} + +void ModelDataCollector::registerFeature( + ACPOCollectFeatures::Scopes ScopeVec, + ACPOCollectFeatures::FeatureInfo GlobalInfo, std::string Pre, + std::string Post) { + std::unique_ptr tmp = + std::make_unique(); + tmp->RegisteredScopes.reset(new ACPOCollectFeatures::Scopes{ScopeVec}); + tmp->FeatureCollector.reset(new ACPOCollectFeatures{}); + tmp->GlobalInfo.reset(new ACPOCollectFeatures::FeatureInfo{GlobalInfo}); + tmp->Prefix = Pre; + tmp->Postfix = Post; + + FeatureCollectInfos.push_back(std::move(tmp)); +} + +void ModelDataCollector::registerFeature( + ACPOCollectFeatures::GroupIDs GroupIDVec, + ACPOCollectFeatures::FeatureInfo GlobalInfo, std::string Pre, + std::string Post) { + std::unique_ptr tmp = + std::make_unique(); + tmp->RegisteredGroupIDs.reset(new ACPOCollectFeatures::GroupIDs{GroupIDVec}); + tmp->FeatureCollector.reset(new ACPOCollectFeatures{}); + tmp->GlobalInfo.reset(new ACPOCollectFeatures::FeatureInfo{GlobalInfo}); + tmp->Prefix = Pre; + tmp->Postfix = Post; + + FeatureCollectInfos.push_back(std::move(tmp)); +} + +void ModelDataCollector::resetRegisteredFeatures() { + FeatureCollectInfos.clear(); + Features.clear(); +} + +std::string ModelDataCollector::demangleName(const std::string &Name) { + ItaniumPartialDemangler D; + if (!D.partialDemangle(Name.c_str())) + return D.getFunctionBaseName(nullptr, nullptr); + + return Name; +} + +void ModelDataCollector::setFeatures( + std::vector> NewFeatures) { + Features = NewFeatures; +} + +void ModelDataCollector::addFeatures( + std::vector> NewFeatures) { + Features.insert(Features.end(), NewFeatures.begin(), NewFeatures.end()); +} + +void ModelDataCollector::setIRFileNameMap(StringMap IRFileNameMap) { + IRFileNames = IRFileNameMap; +} + +void ModelDataCollector::printRow(bool printHeader) { + // Print the IR file names first + for (const auto &P : IRFileNames) { + if (printHeader) + Out << P.getKey(); + else + Out << P.getValue(); + + Out << ","; + } + + for (unsigned I = 0, E = Features.size(); I != E; ++I ) { + // First value does not get a comma + if (I) + Out << ","; + + if (printHeader) + Out << Features.at(I).first; + else + Out << Features.at(I).second; + } + + Out << "\n"; +} + +/*std::string ModelDataCollector::generateIRFileName(autotuning::CodeRegion CR) { + // File name = source_location + pass_name + coderegion_type + hash, + // where source_location = file_name + func_name + loop_name + // + line_number + column_number + std::string IRFileName = + sys::path::filename(StringRef(CR.getFileName())).str() + "_" + + demangleName(CR.getFuncName()) + "_" + + CR.getName() + "_" + + std::to_string(CR.getSourceLoc().SourceLine) + "_" + + std::to_string(CR.getSourceLoc().SourceColumn) + "_" + + CR.getPassName() + "_" + + CR.getTypeAsString() + "_" + + std::to_string(CR.getHash()) + ".ll"; + return IRFileName; +}*/ + +std::string ModelDataCollector::getIRFileName(StringRef Key) { + if (IRFileNames.count(Key)) + return IRFileNames.find(Key)->second; + + return "None"; +} + +std::unique_ptr +ModelDataCollector::createFile(const Twine &FilePath, + const Twine &FileName, + std::error_code &EC) { + if (std::error_code EC = sys::fs::create_directories(FilePath)) + errs() << "Error creating directory: " << FilePath << ": " + << EC.message() << "\n"; + + return std::make_unique((FilePath + "/" + FileName).str(), EC); +} + +void ModelDataCollector::createIRFileForLoop(Loop *L, const Twine &IRFilePath, + const Twine &IRFileName, + bool OverwriteIRFile) { + if (!OverwriteIRFile && sys::fs::exists(IRFilePath + "/" + IRFileName)) + return; + + // Write IR to file + std::error_code EC; + auto OS = createFile(IRFilePath, Twine(IRFileName), EC); + if (EC) { + errs() << "Error creating loop IR file: " << IRFileName << ": " + << EC.message() << "\n"; + return; + } + + // Print loop wrapped in function if -unnamed-var-prefix is set by user + if (UnnamedVariablePrefix.getNumOccurrences() > 0) { + SmallVector ExitBlocks; + L->getExitBlocks(ExitBlocks); + // May need to move this code out of Loop data structure in LLVM. Will see. + L->printWithFunctionWrapper(*OS, L->getHeader()->getParent(), + L->getBlocks(), L->getHeader(), ExitBlocks, + /*AAW*/ nullptr, + /*ShouldPreserveUseListOrder*/ false, + /*IsForDebug*/ false); + } else { + L->print(*OS, /*Depth*/ 0, /*Verbose*/ true); + } +} + +void ModelDataCollector::createIRFileForFunction(Function *F, + const Twine &IRFilePath, + const Twine &IRFileName, + bool OverwriteIRFile) { + if (!OverwriteIRFile && sys::fs::exists(IRFilePath + "/" + IRFileName)) + return; + + // Write IR to file + std::error_code EC; + auto OS = createFile(IRFilePath, Twine(IRFileName), EC); + if (EC) { + errs() << "Error creating function IR file: " << IRFileName << ": " + << EC.message() << "\n"; + return; + } + // May need to investigate this print function change. + F->print(*OS, /*AAW*/ nullptr, /*ShouldPreserveUseListOrder*/ false, + /*IsForDebug*/ false); +} + +void ModelDataCollector::writeIR(Loop *L, Function *F, + std::string NewIRFileName, + std::string PassName, + DumpOption DumpBeforeOrAfter, bool PrintLoop, + bool PrintFunction, bool OverwriteIRFile) { + // Create base directory first + SmallString<256> IRFilePath; + if (IRFileDirectory.getNumOccurrences() > 0) { + // Third priority is the directory specified by + // the -IR-file-directory option + Twine BaseDir(IRFileDirectory); + BaseDir.toVector(IRFilePath); + } else { + // No directory specified + return; + } + + if (getDumpOptionAsString(DumpBeforeOrAfter).empty()) + return; + + // Create sub-directories to store corresponding IR files. + // Directory name = before/after + pass_name + coderegion_type + std::string SubDir = getDumpOptionAsString(DumpBeforeOrAfter) + + "_" + PassName; + if (L && PrintLoop) { + createIRFileForLoop(L, + Twine(IRFilePath) + "/" + SubDir + "_" + + getDumpOptionAsString(DumpOption::loop), + Twine(NewIRFileName), OverwriteIRFile); + // Add IR file name for summary data file + IRFileNames.insert(std::pair ( + getDumpOptionAsString(DumpBeforeOrAfter) + + getDumpOptionAsString(DumpOption::loop), + NewIRFileName)); + } + + if (F && PrintFunction) { + createIRFileForFunction(F, + Twine(IRFilePath) + "/" + SubDir + "_" + + getDumpOptionAsString(DumpOption::function), + Twine(NewIRFileName), OverwriteIRFile); + // Add IR file name for summary data file + IRFileNames.insert(std::pair ( + getDumpOptionAsString(DumpBeforeOrAfter) + + getDumpOptionAsString(DumpOption::function), + NewIRFileName)); + } +} +#endif // ENABLE_ACPO diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt index 9029dc7bb3d9af0a8170838b9f67ca631d3c22fe..579074408b554acb4632787a9c0c21672252517e 100644 --- a/llvm/lib/CodeGen/CMakeLists.txt +++ b/llvm/lib/CodeGen/CMakeLists.txt @@ -1,4 +1,4 @@ -if (DEFINED LLVM_HAVE_TF_AOT OR LLVM_HAVE_TFLITE) +if ((DEFINED LLVM_HAVE_TF_AOT OR DEFINED LLVM_HAVE_TF_API) AND (NOT ACPO_AOT)) include(TensorFlowCompile) set(LLVM_RAEVICT_MODEL_PATH_DEFAULT "models/regalloc-eviction") diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index af77e6c2dc4d55fcf387f85a32eea583f75a3773..370b248c2b85edb910dacbc9afc07a7e3a22fcea 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -86,8 +86,20 @@ #include #include +#if defined(ENABLE_ACPO) +#include "llvm/ADT/StringSet.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Support/CommandLine.h" +#endif + using namespace llvm; +#if defined(ENABLE_ACPO) +cl::opt UnnamedVariablePrefix( + "unnamed-var-prefix", cl::Hidden, + cl::desc("Specify the prefix added to unnamed variables"), cl::init("")); +#endif + // Make virtual table appear in this compilation unit. AssemblyAnnotationWriter::~AssemblyAnnotationWriter() = default; @@ -2486,10 +2498,17 @@ static void WriteAsOperandInternal(raw_ostream &Out, const Value *V, } else { Slot = -1; } - +#if defined(ENABLE_ACPO) + if (Slot != -1) { + // By default, UnnamedVariablePrefix is empty so it matches original behaviour + // unless specified. + Out << Prefix << UnnamedVariablePrefix << Slot; + } else +#else if (Slot != -1) Out << Prefix << Slot; else +#endif Out << ""; } @@ -2602,12 +2621,13 @@ public: void writeAllAttributeGroups(); void printTypeIdentities(); -#if defined(ENABLE_AUTOTUNER) +#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) void printGlobal(const GlobalVariable *GV, bool PrintDeclarationOnly = false); void printAlias(const GlobalAlias *GA); void printIFunc(const GlobalIFunc *GI); void printComdat(const Comdat *C); - void printRequisiteDeclarations(const Function *F); + void printRequisiteDeclarations(const Function *F, + std::vector LoopBlocks = {}); void printFunction(const Function *F, bool PrintCompleteIR = false, bool PrintDeclarationOnly = false); #else @@ -2616,9 +2636,19 @@ public: void printIFunc(const GlobalIFunc *GI); void printComdat(const Comdat *C); void printFunction(const Function *F); +#endif +#if defined(ENABLE_ACPO) + void printLoopWithFunctionWrapper(Function *F, + std::vector LoopBlocks, + BasicBlock *Header, + SmallVector ExitBlocks); #endif void printArgument(const Argument *FA, AttributeSet Attrs); +#if defined(ENABLE_ACPO) + void printBasicBlock(const BasicBlock *BB, bool PrintLabelOnly = false); +#else void printBasicBlock(const BasicBlock *BB); +#endif void printInstructionLine(const Instruction &I); void printInstruction(const Instruction &I); @@ -3603,7 +3633,7 @@ static void maybePrintComdat(formatted_raw_ostream &Out, Out << ')'; } -#if defined(ENABLE_AUTOTUNER) +#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) void AssemblyWriter::printGlobal(const GlobalVariable *GV, bool PrintDeclarationOnly) { if (GV->isMaterializable() && !PrintDeclarationOnly) @@ -3617,7 +3647,7 @@ void AssemblyWriter::printGlobal(const GlobalVariable *GV) { WriteAsOperandInternal(Out, GV, WriterCtx); Out << " = "; -#if defined(ENABLE_AUTOTUNER) +#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) if ((!GV->hasInitializer() || PrintDeclarationOnly) && GV->hasExternalLinkage()) #else @@ -3640,7 +3670,7 @@ void AssemblyWriter::printGlobal(const GlobalVariable *GV) { Out << (GV->isConstant() ? "constant " : "global "); TypePrinter.print(GV->getValueType(), Out); -#if defined(ENABLE_AUTOTUNER) +#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) if (GV->hasInitializer() && !PrintDeclarationOnly) { #else if (GV->hasInitializer()) { @@ -3794,21 +3824,34 @@ void AssemblyWriter::printTypeIdentities() { } } -#if defined(ENABLE_AUTOTUNER) +#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) /// printRequisiteDeclarations - Print the declarations of type identities, /// global variables, functions, and function attribute groups of a function. -void AssemblyWriter::printRequisiteDeclarations(const Function *F) { +void AssemblyWriter::printRequisiteDeclarations( + const Function *F, std::vector LoopBlocks) { // walk through instructions and collect global variables & functions SmallPtrSet GVs; SmallPtrSet Functions; - for (const BasicBlock &BB : *F) { - for (const Instruction &I : BB) { + std::vector BasicBlocks; + if (!LoopBlocks.empty()) { + for (BasicBlock *BB : LoopBlocks) + BasicBlocks.push_back(BB); + } else { + for (const BasicBlock &BB : *F) + BasicBlocks.push_back(const_cast(&BB)); + } + + for (const BasicBlock *BB : BasicBlocks) { + for (const Instruction &I : *BB) { // Check for function if (const auto *CI = dyn_cast(&I)) { Function *func = CI->getCalledFunction(); if (func) Functions.insert(func); } + if (const InvokeInst *II = dyn_cast(&I)) + if (Function *func = dyn_cast(II->getCalledOperand())) + Functions.insert(func); // Check for global variables for (const Use &U : I.operands()) { if (GlobalVariable *gv = dyn_cast(U)) @@ -3823,6 +3866,16 @@ void AssemblyWriter::printRequisiteDeclarations(const Function *F) { GVs.insert(gv); } } + // Check for ConstantExpr BitCast + if (const auto *CstExpr = dyn_cast(U)) + if (CstExpr->isCast()) + for (const Use &UU : CstExpr->operands()) { + if (GlobalVariable *gv = dyn_cast(UU)) + GVs.insert(gv); + else if (const Function *func = + dyn_cast(CstExpr->stripPointerCasts())) + Functions.insert(const_cast(func)); + } } } } @@ -3842,7 +3895,7 @@ void AssemblyWriter::printRequisiteDeclarations(const Function *F) { // modify property if needed if (!(*GVit)->hasAvailableExternallyLinkage() && !((*GVit)->getName() == "llvm.global_ctors") && - (*GVit)->hasLocalLinkage()) { + ((*GVit)->hasLocalLinkage() || (*GVit)->hasCommonLinkage())) { (*GVit)->setLinkage(GlobalValue::ExternalLinkage); (*GVit)->setVisibility(GlobalValue::HiddenVisibility); } @@ -3860,8 +3913,14 @@ void AssemblyWriter::printRequisiteDeclarations(const Function *F) { // print functions for (auto FuncIt = Functions.begin(), et = Functions.end(); FuncIt != et; ++FuncIt) { + if (!LoopBlocks.empty() && *FuncIt == F) + continue; Out << '\n'; + GlobalValue::LinkageTypes SavedLinkage = (*FuncIt)->getLinkage(); + // Function declarations can only have external or extern_weak linkage + (*FuncIt)->setLinkage(GlobalValue::ExternalLinkage); printFunction(*FuncIt, false, true); + (*FuncIt)->setLinkage(SavedLinkage); } // Write attribute groups. @@ -3873,7 +3932,8 @@ void AssemblyWriter::printRequisiteDeclarations(const Function *F) { } /// printFunction - Print all aspects of a function. -void AssemblyWriter::printFunction(const Function *F, bool PrintCompleteIR, +void AssemblyWriter::printFunction(const Function *F, + bool PrintCompleteIR, bool PrintDeclarationOnly) { if (PrintCompleteIR && !PrintDeclarationOnly) { printRequisiteDeclarations(F); @@ -3887,6 +3947,9 @@ void AssemblyWriter::printFunction(const Function *F, bool PrintCompleteIR, void AssemblyWriter::printFunction(const Function *F) { if (AnnotationWriter) AnnotationWriter->emitFunctionAnnot(F, Out); + if (AnnotationWriter) + AnnotationWriter->emitFunctionAnnot(F, Out); + if (F->isMaterializable()) Out << "; Materializable\n"; #endif @@ -3907,7 +3970,7 @@ void AssemblyWriter::printFunction(const Function *F) { Out << "; Function Attrs: " << AttrStr << '\n'; } -#if defined(ENABLE_AUTOTUNER) +#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) if (!PrintDeclarationOnly) Machine.incorporateFunction(F); @@ -3952,7 +4015,7 @@ void AssemblyWriter::printFunction(const Function *F) { Out << '('; // Loop over the arguments, printing them... -#if defined(ENABLE_AUTOTUNER) +#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) if ((F->isDeclaration() && !IsForDebug) || PrintDeclarationOnly) { #else if (F->isDeclaration() && !IsForDebug) { @@ -4027,7 +4090,7 @@ void AssemblyWriter::printFunction(const Function *F) { writeOperand(F->getPersonalityFn(), /*PrintType=*/true); } -#if defined(ENABLE_AUTOTUNER) +#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) if (F->isDeclaration() || PrintDeclarationOnly) { #else if (F->isDeclaration()) { @@ -4049,16 +4112,102 @@ void AssemblyWriter::printFunction(const Function *F) { Out << "}\n"; } -#if defined(ENABLE_AUTOTUNER) +#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) // Output metadata if (!Machine.mdn_empty() && PrintCompleteIR && !PrintDeclarationOnly) { Out << '\n'; writeAllMDNodes(); } -#endif + if (!PrintDeclarationOnly) + Machine.purgeFunction(); +#else Machine.purgeFunction(); +#endif } +#if defined(ENABLE_ACPO) +/// printLoopWithFunctionWrapper - print out a loop wrapped in a dummy +/// function. All global/local variables, functions and metadata that +/// are referenced inside the loop are printed out. Loop predecessors +/// and loop exit blocks are also included. +void AssemblyWriter::printLoopWithFunctionWrapper( + Function *F, std::vector LoopBlocks, BasicBlock *Header, + SmallVector ExitBlocks) { + printRequisiteDeclarations(F, LoopBlocks); + + // Output the dummy function + bool IsFirstArgument = true; + Out << "define void "; + std::string FunctionName = "foo"; + PrintLLVMName(Out, FunctionName, GlobalPrefix); + Out << '('; + for (const Argument &Arg : F->args()) { + if (IsFirstArgument) // Add commas if there are more than one + IsFirstArgument = false; + else + Out << ", "; + Out << &Arg; + } + + // All local variables referenced in this loop but are not declared here + // are printed next in the argument list + SmallPtrSet AddedVariables; + for (const BasicBlock *BB : LoopBlocks) + for (const Instruction &I : *BB) + for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) { + Value *Op = I.getOperand(i); + // Print out the operand in the function argument list + // if it is an instruction that is not contained in this loop + if (const Instruction *II = dyn_cast_or_null(Op)) + if (!AddedVariables.contains(II) && + std::find(LoopBlocks.begin(), LoopBlocks.end(), + II->getParent()) != LoopBlocks.end()) { + AddedVariables.insert(II); + if (IsFirstArgument) + IsFirstArgument = false; + else + Out << ", "; + + writeOperand(Op, true); + } + } + + Out << ") {\n"; + + // Output loop predecessors + // Each predecessor only needs to have an unconditional 'br' instruction + // that branches to the loop header + for (const BasicBlock *Pred : children>(Header)) + // If the block is not in the loop + if (std::find(LoopBlocks.begin(), LoopBlocks.end(), Pred) != + LoopBlocks.end()) { + printBasicBlock(Pred, true); + Out << " br label %"; + PrintLLVMName(Out, Header->getName(), LabelPrefix); + Out << "\n"; + } + + // Output all of the loop's basic blocks + for (const BasicBlock *BB : LoopBlocks) + printBasicBlock(BB); + + // Output loop exit blocks + // Each exit block only needs a 'ret' instruction + for (const BasicBlock *Succ : ExitBlocks) { + printBasicBlock(Succ, true); + Out << " ret void\n"; + } + + Out << "}\n"; + + // Output metadata + if (!Machine.mdn_empty()) { + Out << '\n'; + writeAllMDNodes(); + } +} +#endif + /// printArgument - This member is called for every argument that is passed into /// the function. Simply print it out void AssemblyWriter::printArgument(const Argument *Arg, AttributeSet Attrs) { @@ -4078,13 +4227,27 @@ void AssemblyWriter::printArgument(const Argument *Arg, AttributeSet Attrs) { } else { int Slot = Machine.getLocalSlot(Arg); assert(Slot != -1 && "expect argument in function here"); +#if defined(ENABLE_ACPO) + // By default, UnnamedVariablePrefix is empty so it matches original behaviour + // unless specified. + Out << " %" << UnnamedVariablePrefix << Slot; +#else Out << " %" << Slot; +#endif } } + /// printBasicBlock - This member is called for each basic block in a method. +#if defined(ENABLE_ACPO) +void AssemblyWriter::printBasicBlock(const BasicBlock *BB, + bool PrintLabelOnly) { + assert(BB && BB->getParent() && "block without parent!"); + bool IsEntryBlock = BB == &BB->getParent()->getEntryBlock(); +#else void AssemblyWriter::printBasicBlock(const BasicBlock *BB) { - bool IsEntryBlock = BB->getParent() && BB->isEntryBlock(); + bool IsEntryBlock = BB == &BB->getParent()->getEntryBlock(); +#endif if (BB->hasName()) { // Print out the label if it exists... Out << "\n"; PrintLLVMName(Out, BB->getName(), LabelPrefix); @@ -4092,12 +4255,25 @@ void AssemblyWriter::printBasicBlock(const BasicBlock *BB) { } else if (!IsEntryBlock) { Out << "\n"; int Slot = Machine.getLocalSlot(BB); - if (Slot != -1) + if (Slot != -1) { +#if defined(ENABLE_ACPO) + // By default, UnnamedVariablePrefix is empty so it matches original behaviour + // unless specified. + Out << UnnamedVariablePrefix << Slot << ":"; +#else Out << Slot << ":"; - else +#endif + } else Out << ":"; } +#if defined(ENABLE_ACPO) + if (PrintLabelOnly) { + Out << "\n"; + return; + } +#endif + if (!IsEntryBlock) { // Output predecessors for the block. Out.PadToColumn(50); @@ -4191,8 +4367,15 @@ void AssemblyWriter::printInstruction(const Instruction &I) { int SlotNum = Machine.getLocalSlot(&I); if (SlotNum == -1) Out << " = "; - else + else { +#if defined(ENABLE_ACPO) + // By default, UnnamedVariablePrefix is empty so it matches original behaviour + // unless specified. + Out << '%' << UnnamedVariablePrefix << SlotNum << " = "; +#else Out << '%' << SlotNum << " = "; +#endif + } } if (const CallInst *CI = dyn_cast(&I)) { @@ -4762,6 +4945,20 @@ void BasicBlock::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW, W.printBasicBlock(this); } +#if defined(ENABLE_ACPO) +void Loop::printWithFunctionWrapper( + raw_ostream &ROS, Function *F, ArrayRef LoopBlocks, + BasicBlock *Header, SmallVector ExitBlocks, + AssemblyAnnotationWriter *AAW, bool ShouldPreserveUseListOrder, + bool IsForDebug) const { + SlotTracker SlotTable(F); + formatted_raw_ostream OS(ROS); + AssemblyWriter W(OS, SlotTable, F->getParent(), AAW, IsForDebug, + ShouldPreserveUseListOrder); + W.printLoopWithFunctionWrapper(F, LoopBlocks, Header, ExitBlocks); +} +#endif + void Module::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW, bool ShouldPreserveUseListOrder, bool IsForDebug) const { SlotTracker SlotTable(this); diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index a3ccbc6d258f464c2531c53d904fae9335853e11..e2fe3322aef4fa1f24a95ddeed1874e485da1905 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -267,6 +267,12 @@ #include "llvm/Transforms/Scalar/AutoTuningCompile.h" #endif +#if defined(ENABLE_ACPO) +#include "llvm/Analysis/CallHeight.h" +#include "llvm/Analysis/DumpCallsite.h" +#include "llvm/Analysis/DumpFeature.h" +#endif + using namespace llvm; static const Regex DefaultAliasRegex( diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index 8009e011833cc41a3418e75a59e20c4dd110336c..de89f5393ba26e64701d03d51d70c1ae109c77d2 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -138,6 +138,12 @@ #include "llvm/Transforms/Scalar/AutoTuningCompile.h" #endif +#if defined(ENABLE_ACPO) +#include "llvm/Analysis/CallHeight.h" +#include "llvm/Analysis/DumpCallsite.h" +#include "llvm/Analysis/DumpFeature.h" +#endif + using namespace llvm; static cl::opt UseInlineAdvisor( @@ -894,6 +900,14 @@ PassBuilder::buildInlinerPipeline(OptimizationLevel Level, // make a lot of sense and we should revisit the core CGSCC structure. CGSCCPassManager &MainCGPipeline = MIWP.getPM(); +#if defined(ENABLE_ACPO) + if (EnableFeatureDump) { + // Add CallHeight analysis for dump feature + MIWP.addModulePass(RequireAnalysisPass()); + MainCGPipeline.addPass(DumpFeaturePass()); + } +#endif + // Note: historically, the PruneEH pass was run first to deduce nounwind and // generally clean up exception handling overhead. It isn't clear this is // valuable as the inliner doesn't currently care whether it is inlining an diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 45a539f14b93e210be4fa34e94107a86b80e0ba2..6ef0d6791ff29630400616407c741a574dc57b5c 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -33,6 +33,10 @@ MODULE_ANALYSIS("ir-similarity", IRSimilarityAnalysis()) MODULE_ANALYSIS("autotuning-dump", AutotuningDumpAnalysis()) #endif +#if defined(ENABLE_ACPO) +MODULE_ANALYSIS("call-height", CallHeightAnalysis()) +#endif + #ifndef MODULE_ALIAS_ANALYSIS #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ MODULE_ANALYSIS(NAME, CREATE_PASS) @@ -215,6 +219,9 @@ CGSCC_PASS("invalidate", InvalidateAllAnalysesPass()) CGSCC_PASS("attributor-cgscc", AttributorCGSCCPass()) CGSCC_PASS("openmp-opt-cgscc", OpenMPOptCGSCCPass()) CGSCC_PASS("no-op-cgscc", NoOpCGSCCPass()) +#if defined(ENABLE_ACPO) +CGSCC_PASS("dump-feature", DumpFeaturePass()) +#endif #undef CGSCC_PASS #ifndef CGSCC_PASS_WITH_PARAMS @@ -325,6 +332,9 @@ FUNCTION_PASS("view-dom", DomViewer()) FUNCTION_PASS("view-dom-only", DomOnlyViewer()) FUNCTION_PASS("view-post-dom", PostDomViewer()) FUNCTION_PASS("view-post-dom-only", PostDomOnlyViewer()) +#if defined(ENABLE_ACPO) +FUNCTION_PASS("dump-callsite", DumpCallsitePass()) +#endif FUNCTION_PASS("fix-irreducible", FixIrreduciblePass()) FUNCTION_PASS("flattencfg", FlattenCFGPass()) FUNCTION_PASS("make-guards-explicit", MakeGuardsExplicitPass()) diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp index 802667819c44b6e04d4d38096de8838d40964a46..3a0a2494c36a0ff16e04e1936b82c01edfaf68ba 100644 --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -64,10 +64,21 @@ #include #include #include -#if defined(ENABLE_AUTOTUNER) +#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) #include "llvm/AutoTuner/AutoTuning.h" #endif +#if defined(ENABLE_ACPO) +#include "llvm/Analysis/ACPOFIModel.h" +#include "llvm/Analysis/ModelDataCollector.h" +#include "llvm/IR/IRPrintingPasses.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/Utils/CallGraphUpdater.h" +#include +#endif + using namespace llvm; #define DEBUG_TYPE "inline" @@ -149,6 +160,124 @@ static cl::opt CGSCCInlineReplayFormat( ":. (default)")), cl::desc("How cgscc inline replay file is formatted"), cl::Hidden); +#if defined(ENABLE_ACPO) +static cl::opt + ACPOVerboseFI("acpo-verbose-fi", cl::init(false), cl::Hidden, + cl::desc("Print ACPO invocation messages for FI.")); + +static cl::opt FeatureDump("enable-fi-feature-dump", cl::init(false)); + +// Defined in 'lib/Analysis/ACPOFIModel.cpp' +extern cl::opt EnableACPOFI; +extern cl::opt EnableAOTFI; +// In "llvm/lib/Analysis/ModelDataCollector.cpp" +extern cl::opt ACPOModelFile; + +namespace { +/// Class for collecting inlining model data +class ModelDataFICollector : public ModelDataCollector { +public: + ModelDataFICollector(formatted_raw_ostream &OS, bool OnlyMandatory, + std::string OutputFileName) + : ModelDataCollector(OS, OutputFileName), OnlyMandatory(OnlyMandatory) {} + + void collectFeatures(CallBase *CB, InlineAdvisor *IA, + FunctionAnalysisManager *FAM) { + bool MandatoryOnly = getOnlyMandatory(); + resetRegisteredFeatures(); + ACPOCollectFeatures::FeaturesInfo CallerFeatures{ + {ACPOCollectFeatures::FeatureIndex::BasicBlockCount, + /* ACPOCollectFeatures::Scope::Function, */ + /* ACPOCollectFeatures::GroupID::FPIRelated, */ + {FAM, nullptr}, + {CB->getCaller(), nullptr, nullptr, nullptr, nullptr}, + {MandatoryOnly, IA}}}; + ACPOCollectFeatures::FeaturesInfo CalleeFeatures{ + {ACPOCollectFeatures::FeatureIndex::BasicBlockCount, + /* ACPOCollectFeatures::Scope::Function, */ + /* ACPOCollectFeatures::GroupID::FPIRelated, */ + {FAM, nullptr}, + {CB->getCalledFunction(), nullptr, nullptr, nullptr, nullptr}, + {MandatoryOnly, IA}}}; + BasicBlock *GlobalBB = CB->getParent(); + Function *GlobalF = GlobalBB->getParent(); + Module *GlobalM = GlobalF->getParent(); + ACPOCollectFeatures::FeatureInfo GlobalFeatureInfo{ + ACPOCollectFeatures::FeatureIndex::NumOfFeatures, + {FAM, nullptr}, + {GlobalF, CB, GlobalBB, GlobalM, nullptr}, + {MandatoryOnly, IA}}; + ACPOCollectFeatures::FeatureInfo CallerInfo{ + ACPOCollectFeatures::FeatureIndex::NumOfFeatures, + {FAM, nullptr}, + {CB->getCaller(), CB, GlobalBB, GlobalM, nullptr}, + {MandatoryOnly, IA}}; + ACPOCollectFeatures::FeatureInfo CalleeInfo{ + ACPOCollectFeatures::FeatureIndex::NumOfFeatures, + {FAM, nullptr}, + {CB->getCalledFunction(), CB, GlobalBB, GlobalM, nullptr}, + {MandatoryOnly, IA}}; + + registerFeature({ACPOCollectFeatures::Scope::Function}, CalleeInfo, + "callee"); + registerFeature({ACPOCollectFeatures::Scope::Function}, CallerInfo, + "caller"); + registerFeature({ACPOCollectFeatures::Scope::CallSite}, GlobalFeatureInfo); + registerFeature({ACPOCollectFeatures::Scope::Module}, GlobalFeatureInfo); + ModelDataCollector::collectFeatures(); + } + bool getOnlyMandatory() { return OnlyMandatory; } + +private: + bool OnlyMandatory = false; +}; + +llvm::SmallDenseSet, 4> + InlinedInternalEdges = + llvm::SmallDenseSet, 4>(); +} // end anonymous namespace + +/// helper function for getting advice with acpo infrastructure +bool getACPOAdvice(CallBase *CB, std::unique_ptr &FI, + ModelDataFICollector *MDC, InlineAdvisor *Advisor, + FunctionAnalysisManager *FAM) { + bool ShouldInline = false; + // ------------------------------------------------------------------------ + // Begin ACPO invocation + if ((EnableACPOFI || EnableAOTFI) && !MDC->getOnlyMandatory() && + !Advisor->neverInline(*CB) && Advisor->isCSInlinable(*CB)) { + if (ACPOVerboseFI) { + errs() << "--- ACPOModel is activated\n"; + } + MDC->collectFeatures(CB, Advisor, FAM); + std::vector> Features = + MDC->getFeatures(); + FI->setMLCustomFeatures(Features); + } + std::unique_ptr Advice = FI->getAdvice(); + Constant *Val = Advice->getField("FI-ShouldInline"); + assert(Val != nullptr); + assert(isa(Val)); + ConstantInt *ACPOInline = dyn_cast(Val); + ShouldInline = ACPOInline->getSExtValue(); + if ((EnableACPOFI || EnableAOTFI) && ACPOVerboseFI) { + errs() << "ACPOModel's inline prediction: " << ShouldInline << "\n"; + } + if (FeatureDump) { + MDC->collectFeatures(CB, Advisor, FAM); + std::vector> Features = + MDC->getFeatures(); + if (MDC->isEmptyOutputFile()) { + MDC->printRow(true); + } + MDC->printRow(); + } + return ShouldInline; + // End ACPO Invocation + // --------------------------------------------------------------------- +} +#endif + /// Return true if the specified inline history ID /// indicates an inline history that includes the specified function. static bool inlineHistoryIncludes( @@ -205,6 +334,14 @@ InlinerPass::getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM, PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR) { +#if defined(ENABLE_ACPO) + if (EnableACPOFI || EnableAOTFI) { + // Need to clear the cache at the beggining of the inliner pass, since during + // optimization we may have transofrmed the code which invalidated the cache. + ACPOFIModel::clearCache(); + } +#endif + const auto &MAMProxy = AM.getResult(InitialC, CG); bool Changed = false; @@ -221,6 +358,10 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, Advisor.onPassEntry(&InitialC); auto AdvisorOnExit = make_scope_exit([&] { Advisor.onPassExit(&InitialC); }); +#if defined(ENABLE_ACPO) + if (EnableACPOFI || EnableAOTFI) + ACPOCollectFeatures::clearFunctionLevel(); +#endif // We use a single common worklist for calls across the entire SCC. We // process these in-order and append new calls introduced during inlining to @@ -377,8 +518,51 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, continue; } - std::unique_ptr Advice = - Advisor.getAdvice(*CB, OnlyMandatory); + std::unique_ptr Advice = nullptr; + #if defined(ENABLE_ACPO) + std::unique_ptr FI = nullptr; + if (EnableACPOFI || EnableAOTFI) { + auto &ORE = + FAM.getResult(*CB->getCaller()); + FI = std::make_unique( + CB, &Advisor, &ORE, OnlyMandatory, EnableACPOFI || EnableAOTFI); + std::error_code EC; + raw_fd_ostream RawOS(ACPOModelFile.getValue(), EC, sys::fs::CD_OpenAlways, + sys::fs::FA_Write, sys::fs::OF_Append); + if (EC) + errs() << "Could not create/open training data file (Falling back to " + "debug mode): " + << EC.message() << "\n"; + + formatted_raw_ostream OS(RawOS); + ModelDataFICollector MDC(OS, OnlyMandatory, ACPOModelFile); + if (EnableACPOFI) + LLVM_DEBUG(dbgs() << "ACPO Python ML infra is activated" << "\n"); + else if (EnableAOTFI) + LLVM_DEBUG(dbgs() << "ACPO AOT C++ ML infra is activated" << "\n"); + bool ShouldInline = getACPOAdvice(CB, FI, &MDC, &Advisor, &FAM); + + // Check whether we want to inline this callsite. + if (!ShouldInline) { + FI->recordUnattemptedInlining(); + continue; + } else { + ACPOFIModel::invalidateCache(CB); + } + } else { + Advice = Advisor.getAdvice(*CB, OnlyMandatory); + + // Check whether we want to inline this callsite. + if (!Advice) + continue; + + if (!Advice->isInliningRecommended()) { + Advice->recordUnattemptedInlining(); + continue; + } + } +#else + Advice = Advisor.getAdvice(*CB, OnlyMandatory); // Check whether we want to inline this callsite. if (!Advice) @@ -388,6 +572,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, Advice->recordUnattemptedInlining(); continue; } +#endif int CBCostMult = getStringFnAttrAsInt( @@ -396,6 +581,13 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, // Setup the data structure used to plumb customization into the // `InlineFunction` routine. +#if defined(ENABLE_ACPO) + if ((EnableACPOFI || EnableAOTFI) && ACPOVerboseFI) { + Function &F2 = *CB->getCaller(); + LLVM_DEBUG(dbgs() << "check: " << F2.getName() << ", " + << Callee.getName() << "\n"); + } +#endif InlineFunctionInfo IFI( GetAssumptionCache, PSI, &FAM.getResult(*(CB->getCaller())), @@ -405,7 +597,14 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, InlineFunction(*CB, IFI, /*MergeAttributes=*/true, &FAM.getResult(*CB->getCaller())); if (!IR.isSuccess()) { +#if defined(ENABLE_ACPO) + if (EnableACPOFI || EnableAOTFI) + FI->recordUnsuccessfulInlining(IR); + else + Advice->recordUnsuccessfulInlining(IR); +#else Advice->recordUnsuccessfulInlining(IR); +#endif continue; } @@ -494,10 +693,24 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, DeadFunctionsInComdats.push_back(&Callee); } } +#if defined(ENABLE_ACPO) + if (EnableACPOFI || EnableAOTFI) { + if (CalleeWasDeleted) + FI->recordInliningWithCalleeDeleted(); + else + FI->recordInlining(); + } else { + if (CalleeWasDeleted) + Advice->recordInliningWithCalleeDeleted(); + else + Advice->recordInlining(); + } +#else if (CalleeWasDeleted) Advice->recordInliningWithCalleeDeleted(); else Advice->recordInlining(); +#endif } // Back the call index up by one to put us in a good position to go around diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp index 1401352647cd6ccea5e999df61e9d967cc37786d..671a33309a1b50e739f3ec2cc3fbac899b413716 100644 --- a/llvm/tools/opt/opt.cpp +++ b/llvm/tools/opt/opt.cpp @@ -430,6 +430,9 @@ int main(int argc, char **argv) { initializeTransformUtils(Registry); initializeInstCombine(Registry); initializeTarget(Registry); +#if defined(ENABLE_ACPO) + initializeDumpCallsiteLegacyPass(Registry); +#endif // For codegen passes, only passes that do IR to IR transformation are // supported. initializeExpandLargeDivRemLegacyPassPass(Registry);