AArch64: use CAS instead of LDXR/STXR if available

This covers 128-bit loads, and atomicrmw operations without a single native
instruction. Using CAS saves has a better chance of succeeding with high
contention on some systems.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

@@ -21903,7 +21903,10 @@ AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
  if (getTargetMachine().getOptLevel() == CodeGenOpt::None)
  return AtomicExpansionKind::CmpXChg;
 
- return AtomicExpansionKind::LLSC;
+ // Using CAS for an atomic load has a better chance of succeeding under high
+ // contention situations. So use it if available.
+ return Subtarget->hasLSE() ? AtomicExpansionKind::CmpXChg
+ : AtomicExpansionKind::LLSC;
 }
 
 // For the real atomic operations, we have ldxr/stxr up to 128 bits,
@@ -21940,8 +21943,10 @@ AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
  // implement atomicrmw without spilling. If the target address is also on the
  // stack and close enough to the spill slot, this can lead to a situation
  // where the monitor always gets cleared and the atomic operation can never
- // succeed. So at -O0 lower this operation to a CAS loop.
- if (getTargetMachine().getOptLevel() == CodeGenOpt::None)
+ // succeed. So at -O0 lower this operation to a CAS loop. Also worthwhile if
+ // we have a single CAS instruction that can replace the loop.
+ if (getTargetMachine().getOptLevel() == CodeGenOpt::None ||
+ Subtarget->hasLSE())
  return AtomicExpansionKind::CmpXChg;
 
  return AtomicExpansionKind::LLSC;

llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll

@@ -360,14 +360,11 @@ define void @atomic_load_relaxed(i64, i64, ptr %p, ptr %p2) {
 ;
 ; CHECK-CAS-O1-LABEL: atomic_load_relaxed:
 ; CHECK-CAS-O1: // %bb.0:
-; CHECK-CAS-O1-NEXT: .LBB4_1: // %atomicrmw.start
-; CHECK-CAS-O1-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-CAS-O1-NEXT: ldxp x9, x8, [x2]
-; CHECK-CAS-O1-NEXT: stxp w10, x9, x8, [x2]
-; CHECK-CAS-O1-NEXT: cbnz w10, .LBB4_1
-; CHECK-CAS-O1-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-CAS-O1-NEXT: mov v0.d[0], x9
-; CHECK-CAS-O1-NEXT: mov v0.d[1], x8
+; CHECK-CAS-O1-NEXT: mov x0, xzr
+; CHECK-CAS-O1-NEXT: mov x1, xzr
+; CHECK-CAS-O1-NEXT: casp x0, x1, x0, x1, [x2]
+; CHECK-CAS-O1-NEXT: mov v0.d[0], x0
+; CHECK-CAS-O1-NEXT: mov v0.d[1], x1
 ; CHECK-CAS-O1-NEXT: str q0, [x3]
 ; CHECK-CAS-O1-NEXT: ret
 ;

llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll

@@ -362,15 +362,17 @@ define i32 @fetch_and_nand(ptr %p) #0 {
 ;
 ; CHECK-LSE-O1-LABEL: fetch_and_nand:
 ; CHECK-LSE-O1: ; %bb.0:
+; CHECK-LSE-O1-NEXT: mov x8, x0
+; CHECK-LSE-O1-NEXT: ldr w0, [x0]
 ; CHECK-LSE-O1-NEXT: LBB6_1: ; %atomicrmw.start
 ; CHECK-LSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-LSE-O1-NEXT: ldxr w8, [x0]
-; CHECK-LSE-O1-NEXT: and w9, w8, #0x7
-; CHECK-LSE-O1-NEXT: mvn w9, w9
-; CHECK-LSE-O1-NEXT: stlxr w10, w9, [x0]
-; CHECK-LSE-O1-NEXT: cbnz w10, LBB6_1
+; CHECK-LSE-O1-NEXT: mov x9, x0
+; CHECK-LSE-O1-NEXT: and w10, w0, #0x7
+; CHECK-LSE-O1-NEXT: mvn w10, w10
+; CHECK-LSE-O1-NEXT: casl w0, w10, [x8]
+; CHECK-LSE-O1-NEXT: cmp w0, w9
+; CHECK-LSE-O1-NEXT: b.ne LBB6_1
 ; CHECK-LSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end
-; CHECK-LSE-O1-NEXT: mov x0, x8
 ; CHECK-LSE-O1-NEXT: ret
 ;
 ; CHECK-LSE-O0-LABEL: fetch_and_nand:
@@ -455,15 +457,17 @@ define i64 @fetch_and_nand_64(ptr %p) #0 {
 ;
 ; CHECK-LSE-O1-LABEL: fetch_and_nand_64:
 ; CHECK-LSE-O1: ; %bb.0:
+; CHECK-LSE-O1-NEXT: mov x8, x0
+; CHECK-LSE-O1-NEXT: ldr x0, [x0]
 ; CHECK-LSE-O1-NEXT: LBB7_1: ; %atomicrmw.start
 ; CHECK-LSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-LSE-O1-NEXT: ldaxr x8, [x0]
-; CHECK-LSE-O1-NEXT: and x9, x8, #0x7
-; CHECK-LSE-O1-NEXT: mvn x9, x9
-; CHECK-LSE-O1-NEXT: stlxr w10, x9, [x0]
-; CHECK-LSE-O1-NEXT: cbnz w10, LBB7_1
+; CHECK-LSE-O1-NEXT: mov x9, x0
+; CHECK-LSE-O1-NEXT: and x10, x0, #0x7
+; CHECK-LSE-O1-NEXT: mvn x10, x10
+; CHECK-LSE-O1-NEXT: casal x0, x10, [x8]
+; CHECK-LSE-O1-NEXT: cmp x0, x9
+; CHECK-LSE-O1-NEXT: b.ne LBB7_1
 ; CHECK-LSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end
-; CHECK-LSE-O1-NEXT: mov x0, x8
 ; CHECK-LSE-O1-NEXT: ret
 ;
 ; CHECK-LSE-O0-LABEL: fetch_and_nand_64: