[RISCV] Add software pipeliner support for RISC-V

This patch adds basic support of `MachinePipeliner` and disable
it by default.

The functionality should be OK and all llvm-test-suite tests have
passed.

llvm/lib/Target/RISCV/RISCVInstrInfo.cpp

@@ -23,6 +23,7 @@
 #include "llvm/CodeGen/LiveVariables.h"
 #include "llvm/CodeGen/MachineCombinerPattern.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/MachineTraceMetrics.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
@@ -4136,3 +4137,84 @@ bool RISCV::isVLKnownLE(const MachineOperand &LHS, const MachineOperand &RHS) {
     return false;
   return LHS.getImm() <= RHS.getImm();
 }
+
+namespace {
+class RISCVPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
+  const MachineInstr *LHS;
+  const MachineInstr *RHS;
+  SmallVector<MachineOperand, 4> Cond;
+
+public:
+  RISCVPipelinerLoopInfo(const MachineInstr *LHS, const MachineInstr *RHS,
+                         const SmallVectorImpl<MachineOperand> &Cond)
+      : LHS(LHS), RHS(RHS), Cond(Cond.begin(), Cond.end()) {}
+
+  bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
+    // Make the instructions for loop control be placed in stage 0.
+    // The predecessors of PredBranch are considered by the caller.
+    if (LHS && MI == LHS)
+      return true;
+    if (RHS && MI == RHS)
+      return true;
+    return false;
+  }
+
+  std::optional<bool> createTripCountGreaterCondition(
+      int TC, MachineBasicBlock &MBB,
+      SmallVectorImpl<MachineOperand> &CondParam) override {
+    // A branch instruction will be inserted as "if (Cond) goto epilogue".
+    // Cond is normalized for such use.
+    // The predecessors of the branch are assumed to have already been inserted.
+    CondParam = Cond;
+    return {};
+  }
+
+  void setPreheader(MachineBasicBlock *NewPreheader) override {}
+
+  void adjustTripCount(int TripCountAdjust) override {}
+
+  void disposed() override {}
+};
+} // namespace
+
+std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+RISCVInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
+  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
+  SmallVector<MachineOperand, 4> Cond;
+  if (analyzeBranch(*LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
+    return nullptr;
+
+  // Infinite loops are not supported
+  if (TBB == LoopBB && FBB == LoopBB)
+    return nullptr;
+
+  // Must be conditional branch
+  if (FBB == nullptr)
+    return nullptr;
+
+  assert((TBB == LoopBB || FBB == LoopBB) &&
+         "The Loop must be a single-basic-block loop");
+
+  // Normalization for createTripCountGreaterCondition()
+  if (TBB == LoopBB)
+    reverseBranchCondition(Cond);
+
+  const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
+  auto FindRegDef = [&MRI](MachineOperand &Op) -> const MachineInstr * {
+    if (!Op.isReg())
+      return nullptr;
+    Register Reg = Op.getReg();
+    if (!Reg.isVirtual())
+      return nullptr;
+    return MRI.getVRegDef(Reg);
+  };
+
+  const MachineInstr *LHS = FindRegDef(Cond[1]);
+  const MachineInstr *RHS = FindRegDef(Cond[2]);
+  if (LHS && LHS->isPHI())
+    return nullptr;
+  if (RHS && RHS->isPHI())
+    return nullptr;
+
+  return std::make_unique<RISCVPipelinerLoopInfo>(LHS, RHS, Cond);
+}

llvm/lib/Target/RISCV/RISCVInstrInfo.h

@@ -293,6 +293,9 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
 
   unsigned getTailDuplicateSize(CodeGenOptLevel OptLevel) const override;
 
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
+
 protected:
   const RISCVSubtarget &STI;
 

llvm/lib/Target/RISCV/RISCVSubtarget.cpp

@@ -186,6 +186,10 @@ bool RISCVSubtarget::useRVVForFixedLengthVectors() const {
 
 bool RISCVSubtarget::enableSubRegLiveness() const { return true; }
 
+bool RISCVSubtarget::enableMachinePipeliner() const {
+  return getSchedModel().hasInstrSchedModel();
+}
+
 void RISCVSubtarget::getPostRAMutations(
     std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
   Mutations.push_back(createMacroFusionDAGMutation(getMacroFusions()));

llvm/lib/Target/RISCV/RISCVSubtarget.h

@@ -304,6 +304,10 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
   void getPostRAMutations(std::vector<std::unique_ptr<ScheduleDAGMutation>>
                               &Mutations) const override;
 
+  bool enableMachinePipeliner() const override;
+
+  bool useDFAforSMS() const override { return false; }
+
   bool useAA() const override;
 
   unsigned getCacheLineSize() const override {

llvm/lib/Target/RISCV/RISCVTargetMachine.cpp

@@ -114,6 +114,11 @@ static cl::opt<bool>
                       cl::desc("Enable the RISC-V VL Optimizer pass"),
                       cl::init(false), cl::Hidden);
 
+static cl::opt<bool>
+    EnableMachinePipeliner("riscv-enable-pipeliner",
+                           cl::desc("Enable Machine Pipeliner for RISC-V"),
+                           cl::init(false), cl::Hidden);
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
   RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
   RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
@@ -600,6 +605,9 @@ void RISCVPassConfig::addPreRegAlloc() {
   addPass(createRISCVInsertReadWriteCSRPass());
   addPass(createRISCVInsertWriteVXRMPass());
   addPass(createRISCVLandingPadSetupPass());
+
+  if (TM->getOptLevel() != CodeGenOptLevel::None && EnableMachinePipeliner)
+    addPass(&MachinePipelinerID);
 }
 
 void RISCVPassConfig::addFastRegAlloc() {

llvm/test/CodeGen/RISCV/machine-pipeliner.ll

@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv64 -mcpu=sifive-p670 -O3 -verify-machineinstrs -riscv-enable-pipeliner=false < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK-NOT-PIPELINED
+; RUN: llc -mtriple=riscv64 -mcpu=sifive-p670 -O3 -verify-machineinstrs -riscv-enable-pipeliner=true < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK-PIPELINED
+
+define void @test_1(ptr noalias %in, ptr noalias %out, i32 signext %cnt) "no-builtins" {
+; CHECK-NOT-PIPELINED-LABEL: test_1:
+; CHECK-NOT-PIPELINED:       # %bb.0: # %entry
+; CHECK-NOT-PIPELINED-NEXT:    blez a2, .LBB0_3
+; CHECK-NOT-PIPELINED-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NOT-PIPELINED-NEXT:    addi a2, a2, -1
+; CHECK-NOT-PIPELINED-NEXT:    sh2add.uw a2, a2, a1
+; CHECK-NOT-PIPELINED-NEXT:    addi a2, a2, 4
+; CHECK-NOT-PIPELINED-NEXT:  .LBB0_2: # %for.body
+; CHECK-NOT-PIPELINED-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NOT-PIPELINED-NEXT:    lw a3, 0(a1)
+; CHECK-NOT-PIPELINED-NEXT:    addi a1, a1, 4
+; CHECK-NOT-PIPELINED-NEXT:    sw a3, 0(a0)
+; CHECK-NOT-PIPELINED-NEXT:    addi a0, a0, 4
+; CHECK-NOT-PIPELINED-NEXT:    bne a1, a2, .LBB0_2
+; CHECK-NOT-PIPELINED-NEXT:  .LBB0_3: # %for.end
+; CHECK-NOT-PIPELINED-NEXT:    ret
+;
+; CHECK-PIPELINED-LABEL: test_1:
+; CHECK-PIPELINED:       # %bb.0: # %entry
+; CHECK-PIPELINED-NEXT:    blez a2, .LBB0_6
+; CHECK-PIPELINED-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-PIPELINED-NEXT:    lw a3, 0(a1)
+; CHECK-PIPELINED-NEXT:    addi a2, a2, -1
+; CHECK-PIPELINED-NEXT:    addi a4, a0, 4
+; CHECK-PIPELINED-NEXT:    sh2add.uw a6, a2, a1
+; CHECK-PIPELINED-NEXT:    addi a1, a1, 4
+; CHECK-PIPELINED-NEXT:    addi a6, a6, 4
+; CHECK-PIPELINED-NEXT:    beq a1, a6, .LBB0_5
+; CHECK-PIPELINED-NEXT:  # %bb.2: # %for.body
+; CHECK-PIPELINED-NEXT:    lw a5, 0(a1)
+; CHECK-PIPELINED-NEXT:    addi a2, a4, 4
+; CHECK-PIPELINED-NEXT:    addi a1, a1, 4
+; CHECK-PIPELINED-NEXT:    beq a1, a6, .LBB0_4
+; CHECK-PIPELINED-NEXT:  .LBB0_3: # %for.body
+; CHECK-PIPELINED-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-PIPELINED-NEXT:    sw a3, 0(a0)
+; CHECK-PIPELINED-NEXT:    mv a3, a5
+; CHECK-PIPELINED-NEXT:    lw a5, 0(a1)
+; CHECK-PIPELINED-NEXT:    mv a0, a4
+; CHECK-PIPELINED-NEXT:    mv a4, a2
+; CHECK-PIPELINED-NEXT:    addi a2, a2, 4
+; CHECK-PIPELINED-NEXT:    addi a1, a1, 4
+; CHECK-PIPELINED-NEXT:    bne a1, a6, .LBB0_3
+; CHECK-PIPELINED-NEXT:  .LBB0_4:
+; CHECK-PIPELINED-NEXT:    sw a3, 0(a0)
+; CHECK-PIPELINED-NEXT:    mv a0, a4
+; CHECK-PIPELINED-NEXT:    mv a3, a5
+; CHECK-PIPELINED-NEXT:  .LBB0_5:
+; CHECK-PIPELINED-NEXT:    sw a3, 0(a0)
+; CHECK-PIPELINED-NEXT:  .LBB0_6: # %for.end
+; CHECK-PIPELINED-NEXT:    ret
+entry:
+  %cmp5 = icmp sgt i32 %cnt, 0
+  br i1 %cmp5, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %in.addr.07 = phi ptr [ %incdec.ptr, %for.body ], [ %in, %entry ]
+  %out.addr.06 = phi ptr [ %incdec.ptr1, %for.body ], [ %out, %entry ]
+  %0 = load i32, ptr %out.addr.06, align 4
+  store i32 %0, ptr %in.addr.07, align 4
+  %incdec.ptr = getelementptr inbounds i8, ptr %in.addr.07, i64 4
+  %incdec.ptr1 = getelementptr inbounds i8, ptr %out.addr.06, i64 4
+  %inc = add nuw nsw i32 %i.08, 1
+  %exitcond.not = icmp eq i32 %inc, %cnt
+  br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+