diff --git a/src/pipeline.cpp b/src/pipeline.cpp index 4403653a9d8e4f..7049f4f814e009 100644 --- a/src/pipeline.cpp +++ b/src/pipeline.cpp @@ -202,7 +202,7 @@ namespace { // TODO(vchuravy/maleadt): // Since we are not using the PassBuilder fully and instead rolling our own, we are missing out on - // TargetMachine::registerPassBuilderCallbacks. We need to find a solution either in working with upstream + // TargetMachine::registerPassBuilderEPCallbacks. We need to find a solution either in working with upstream // or adapting PassBuilder (or subclassing it) to suite our needs. This is in particular important for // BPF, NVPTX, and AMDGPU. //TODO implement these once LLVM exposes @@ -210,16 +210,16 @@ namespace { //For now we'll maintain the insertion points even though they don't do anything //for the sake of documentation //If PB is a nullptr, don't invoke anything (this happens when running julia from opt) - void invokePipelineStartCallbacks(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} + void invokePipelineStartEPCallbacks(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} void invokePeepholeEPCallbacks(FunctionPassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} - void invokeEarlySimplificationCallbacks(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} - void invokeCGSCCCallbacks(CGSCCPassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} - void invokeOptimizerEarlyCallbacks(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} - void invokeLateLoopOptimizationCallbacks(LoopPassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} - void invokeLoopOptimizerEndCallbacks(LoopPassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} - void invokeScalarOptimizerCallbacks(FunctionPassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} - void invokeVectorizerCallbacks(FunctionPassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} - void invokeOptimizerLastCallbacks(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} + void invokeEarlySimplificationEPCallbacks(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} + void invokeCGSCCEPCallbacks(CGSCCPassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} + void invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} + void invokeLateLoopOptimizationEPCallbacks(LoopPassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} + void invokeLoopOptimizerEndEPCallbacks(LoopPassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} + void invokeScalarOptimizerEPCallbacks(FunctionPassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} + void invokeVectorizerEPCallbacks(FunctionPassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} + void invokeOptimizerLastEPCallbacks(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O) JL_NOTSAFEPOINT {} } //The actual pipelines @@ -249,7 +249,7 @@ static void buildBasicPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimiza #ifdef JL_DEBUG_BUILD addVerificationPasses(MPM, options.llvm_only); #endif - invokePipelineStartCallbacks(MPM, PB, O); + invokePipelineStartEPCallbacks(MPM, PB, O); MPM.addPass(ConstantMergePass()); if (!options.dump_native) { JULIA_PASS(MPM.addPass(CPUFeatures())); @@ -268,25 +268,25 @@ static void buildBasicPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimiza FPM.addPass(MemCpyOptPass()); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } - invokeEarlySimplificationCallbacks(MPM, PB, O); + invokeEarlySimplificationEPCallbacks(MPM, PB, O); MPM.addPass(AlwaysInlinerPass()); { CGSCCPassManager CGPM; - invokeCGSCCCallbacks(CGPM, PB, O); + invokeCGSCCEPCallbacks(CGPM, PB, O); MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); } - invokeOptimizerEarlyCallbacks(MPM, PB, O); + invokeOptimizerEarlyEPCallbacks(MPM, PB, O); JULIA_PASS(MPM.addPass(LowerSIMDLoop())); { FunctionPassManager FPM; { LoopPassManager LPM; - invokeLateLoopOptimizationCallbacks(LPM, PB, O); - invokeLoopOptimizerEndCallbacks(LPM, PB, O); + invokeLateLoopOptimizationEPCallbacks(LPM, PB, O); + invokeLoopOptimizerEndEPCallbacks(LPM, PB, O); FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM))); } - invokeScalarOptimizerCallbacks(FPM, PB, O); - invokeVectorizerCallbacks(FPM, PB, O); + invokeScalarOptimizerEPCallbacks(FPM, PB, O); + invokeVectorizerEPCallbacks(FPM, PB, O); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } if (options.lower_intrinsics) { @@ -315,7 +315,7 @@ static void buildBasicPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimiza MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } } - invokeOptimizerLastCallbacks(MPM, PB, O); + invokeOptimizerLastEPCallbacks(MPM, PB, O); addSanitizerPasses(MPM, O); JULIA_PASS(MPM.addPass(createModuleToFunctionPassAdaptor(DemoteFloat16()))); } @@ -325,7 +325,7 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat #ifdef JL_DEBUG_BUILD addVerificationPasses(MPM, options.llvm_only); #endif - invokePipelineStartCallbacks(MPM, PB, O); + invokePipelineStartEPCallbacks(MPM, PB, O); MPM.addPass(ConstantMergePass()); { FunctionPassManager FPM; @@ -339,12 +339,12 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat FPM.addPass(SROAPass()); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } - invokeEarlySimplificationCallbacks(MPM, PB, O); + invokeEarlySimplificationEPCallbacks(MPM, PB, O); MPM.addPass(AlwaysInlinerPass()); - invokeOptimizerEarlyCallbacks(MPM, PB, O); + invokeOptimizerEarlyEPCallbacks(MPM, PB, O); { CGSCCPassManager CGPM; - invokeCGSCCCallbacks(CGPM, PB, O); + invokeCGSCCEPCallbacks(CGPM, PB, O); { FunctionPassManager FPM; JULIA_PASS(FPM.addPass(AllocOptPass())); @@ -367,7 +367,7 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat FPM.addPass(ReassociatePass()); FPM.addPass(EarlyCSEPass()); JULIA_PASS(FPM.addPass(AllocOptPass())); - invokePeepholeEPCallbacks(FPM, PB, O); + invokePeepholeEPEPCallbacks(FPM, PB, O); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } MPM.addPass(LowerSIMDLoop()); @@ -376,7 +376,7 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat { LoopPassManager LPM1, LPM2; LPM1.addPass(LoopRotatePass()); - invokeLateLoopOptimizationCallbacks(LPM1, PB, O); + invokeLateLoopOptimizationEPCallbacks(LPM1, PB, O); //We don't know if the loop callbacks support MSSA FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1), /*UseMemorySSA = */false)); #if JL_LLVM_VERSION < 150000 @@ -397,11 +397,11 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat LPM.addPass(LoopIdiomRecognizePass()); LPM.addPass(IndVarSimplifyPass()); LPM.addPass(LoopDeletionPass()); - invokeLoopOptimizerEndCallbacks(LPM, PB, O); + invokeLoopOptimizerEndEPCallbacks(LPM, PB, O); //We don't know if the loop end callbacks support MSSA FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA = */false)); } - FPM.addPass(LoopUnrollPass(LoopUnrollOptions().setRuntime(false))); + FPM.addPass(LoopFullUnrollPass(O.getSpeedupLevel())); JULIA_PASS(FPM.addPass(AllocOptPass())); FPM.addPass(SROAPass()); FPM.addPass(InstSimplifyPass()); @@ -417,7 +417,7 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat FPM.addPass(GVNPass()); } FPM.addPass(DSEPass()); - invokePeepholeEPCallbacks(FPM, PB, O); + invokePeepholeEPEPCallbacks(FPM, PB, O); FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions())); JULIA_PASS(FPM.addPass(AllocOptPass())); { @@ -426,14 +426,14 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat LPM.addPass(LoopInstSimplifyPass()); FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM))); } - invokeScalarOptimizerCallbacks(FPM, PB, O); + invokeScalarOptimizerEPCallbacks(FPM, PB, O); //TODO look into loop vectorize options FPM.addPass(LoopVectorizePass()); FPM.addPass(LoopLoadEliminationPass()); FPM.addPass(InstCombinePass()); FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions())); FPM.addPass(SLPVectorizerPass()); - invokeVectorizerCallbacks(FPM, PB, O); + invokeVectorizerEPCallbacks(FPM, PB, O); FPM.addPass(ADCEPass()); //TODO add BDCEPass here? MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); @@ -474,7 +474,7 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat FPM.addPass(DivRemPairsPass()); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } - invokeOptimizerLastCallbacks(MPM, PB, O); + invokeOptimizerLastEPCallbacks(MPM, PB, O); addSanitizerPasses(MPM, O); { FunctionPassManager FPM; @@ -488,7 +488,7 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat namespace { auto createPIC(StandardInstrumentations &SI) JL_NOTSAFEPOINT { - auto PIC = std::make_unique(); + auto PIC = std::make_unique(); //Borrowed from LLVM PassBuilder.cpp:386 #define MODULE_PASS(NAME, CREATE_PASS) \ PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME); @@ -533,7 +533,7 @@ PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME); #undef CGSCC_PASS_WITH_PARAMS #undef CGSCC_ANALYSIS - SI.registerCallbacks(*PIC); + SI.registerEPCallbacks(*PIC); return PIC; } @@ -679,7 +679,7 @@ static llvm::Optional> parseJu // NOTE: Instead of exporting all the constructors in passes.h we could // forward the callbacks to the respective passes. LLVM seems to prefer this, // and when we add the full pass builder having them directly will be helpful. -void registerCallbacks(PassBuilder &PB) JL_NOTSAFEPOINT { +void registerEPCallbacks(PassBuilder &PB) JL_NOTSAFEPOINT { PB.registerPipelineParsingCallback( [](StringRef Name, FunctionPassManager &PM, ArrayRef InnerPipeline) { @@ -722,5 +722,5 @@ void registerCallbacks(PassBuilder &PB) JL_NOTSAFEPOINT { extern "C" JL_DLLEXPORT ::llvm::PassPluginLibraryInfo llvmGetPassPluginInfo() JL_NOTSAFEPOINT { - return {LLVM_PLUGIN_API_VERSION, "Julia", "1", registerCallbacks}; + return {LLVM_PLUGIN_API_VERSION, "Julia", "1", registerEPCallbacks}; }