From c4fdd3e518f2b053cbf46f0d985c80ebf7a604d1 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Thu, 29 Aug 2024 19:36:24 +0400 Subject: [PATCH 1/3] AArch64: Add tests for atomicrmw fp operations (#103701) There were only codegen tests for the fadd vector case, so round out the test coverage for the scalar cases and all the other operations. --- .../AArch64/atomicrmw-fadd-fp-vector.ll | 115 -- llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll | 1209 ++++++++++++++++ llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll | 1272 +++++++++++++++++ llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll | 1272 +++++++++++++++++ llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll | 1209 ++++++++++++++++ 5 files changed, 4962 insertions(+), 115 deletions(-) delete mode 100644 llvm/test/CodeGen/AArch64/atomicrmw-fadd-fp-vector.ll create mode 100644 llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll create mode 100644 llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll create mode 100644 llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll create mode 100644 llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fadd-fp-vector.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fadd-fp-vector.ll deleted file mode 100644 index a7539ac3cce8..000000000000 --- a/llvm/test/CodeGen/AArch64/atomicrmw-fadd-fp-vector.ll +++ /dev/null @@ -1,115 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc -mtriple=aarch64-- -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefixes=CHECK,NOLSE %s -; RUN: llc -mtriple=aarch64-- -mattr=+lse -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefixes=CHECK,LSE %s - -define <2 x half> @test_atomicrmw_fadd_v2f16_align4(ptr addrspace(1) %ptr, <2 x half> %value) #0 { -; NOLSE-LABEL: test_atomicrmw_fadd_v2f16_align4: -; NOLSE: // %bb.0: -; NOLSE-NEXT: fcvtl v1.4s, v0.4h -; NOLSE-NEXT: ldr s0, [x0] -; NOLSE-NEXT: b .LBB0_2 -; NOLSE-NEXT: .LBB0_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.eq .LBB0_5 -; NOLSE-NEXT: .LBB0_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB0_3 Depth 2 -; NOLSE-NEXT: fcvtl v2.4s, v0.4h -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: fadd v2.4s, v2.4s, v1.4s -; NOLSE-NEXT: fcvtn v2.4h, v2.4s -; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB0_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB0_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr w10, [x0] -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.ne .LBB0_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB0_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB0_3 -; NOLSE-NEXT: b .LBB0_1 -; NOLSE-NEXT: .LBB0_5: // %atomicrmw.end -; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 -; NOLSE-NEXT: ret -; -; LSE-LABEL: test_atomicrmw_fadd_v2f16_align4: -; LSE: // %bb.0: -; LSE-NEXT: fcvtl v1.4s, v0.4h -; LSE-NEXT: ldr s0, [x0] -; LSE-NEXT: .LBB0_1: // %atomicrmw.start -; LSE-NEXT: // =>This Inner Loop Header: Depth=1 -; LSE-NEXT: fcvtl v2.4s, v0.4h -; LSE-NEXT: fmov w8, s0 -; LSE-NEXT: mov w10, w8 -; LSE-NEXT: fadd v2.4s, v2.4s, v1.4s -; LSE-NEXT: fcvtn v2.4h, v2.4s -; LSE-NEXT: fmov w9, s2 -; LSE-NEXT: casal w10, w9, [x0] -; LSE-NEXT: fmov s0, w10 -; LSE-NEXT: cmp w10, w8 -; LSE-NEXT: b.ne .LBB0_1 -; LSE-NEXT: // %bb.2: // %atomicrmw.end -; LSE-NEXT: // kill: def $d0 killed $d0 killed $q0 -; LSE-NEXT: ret - %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x half> %value seq_cst, align 4 - ret <2 x half> %res -} - -define <2 x float> @test_atomicrmw_fadd_v2f32_align8(ptr addrspace(1) %ptr, <2 x float> %value) #0 { -; NOLSE-LABEL: test_atomicrmw_fadd_v2f32_align8: -; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr d1, [x0] -; NOLSE-NEXT: b .LBB1_2 -; NOLSE-NEXT: .LBB1_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=1 -; NOLSE-NEXT: fmov d1, x10 -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.eq .LBB1_5 -; NOLSE-NEXT: .LBB1_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB1_3 Depth 2 -; NOLSE-NEXT: fadd v2.2s, v1.2s, v0.2s -; NOLSE-NEXT: fmov x9, d1 -; NOLSE-NEXT: fmov x8, d2 -; NOLSE-NEXT: .LBB1_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB1_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr x10, [x0] -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.ne .LBB1_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB1_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, x8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB1_3 -; NOLSE-NEXT: b .LBB1_1 -; NOLSE-NEXT: .LBB1_5: // %atomicrmw.end -; NOLSE-NEXT: fmov d0, d1 -; NOLSE-NEXT: ret -; -; LSE-LABEL: test_atomicrmw_fadd_v2f32_align8: -; LSE: // %bb.0: -; LSE-NEXT: ldr d1, [x0] -; LSE-NEXT: .LBB1_1: // %atomicrmw.start -; LSE-NEXT: // =>This Inner Loop Header: Depth=1 -; LSE-NEXT: fadd v2.2s, v1.2s, v0.2s -; LSE-NEXT: fmov x8, d1 -; LSE-NEXT: mov x10, x8 -; LSE-NEXT: fmov x9, d2 -; LSE-NEXT: casal x10, x9, [x0] -; LSE-NEXT: fmov d1, x10 -; LSE-NEXT: cmp x10, x8 -; LSE-NEXT: b.ne .LBB1_1 -; LSE-NEXT: // %bb.2: // %atomicrmw.end -; LSE-NEXT: fmov d0, d1 -; LSE-NEXT: ret - %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x float> %value seq_cst, align 8 - ret <2 x float> %res -} - -attributes #0 = { nounwind } -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; CHECK: {{.*}} diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll new file mode 100644 index 000000000000..89c9880ffc78 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll @@ -0,0 +1,1209 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=aarch64-linux-gnu -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=NOLSE %s +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+lse -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=LSE %s +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=-lse,-fp-armv8 -O1 < %s | FileCheck -check-prefix=SOFTFP-NOLSE %s + +define half @test_atomicrmw_fadd_f16_seq_cst_align2(ptr %ptr, half %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align2: +; NOLSE: // %bb.0: +; NOLSE-NEXT: fcvt s1, h0 +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: b .LBB0_2 +; NOLSE-NEXT: .LBB0_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.eq .LBB0_5 +; NOLSE-NEXT: .LBB0_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB0_3 Depth 2 +; NOLSE-NEXT: fcvt s2, h0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fadd s2, s2, s1 +; NOLSE-NEXT: fcvt h2, s2 +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB0_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB0_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w10, [x0] +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.ne .LBB0_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB0_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB0_3 +; NOLSE-NEXT: b .LBB0_1 +; NOLSE-NEXT: .LBB0_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align2: +; LSE: // %bb.0: +; LSE-NEXT: fcvt s1, h0 +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: .LBB0_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fcvt s2, h0 +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fadd s2, s2, s1 +; LSE-NEXT: fcvt h2, s2 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casalh w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8, uxth +; LSE-NEXT: b.ne .LBB0_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align2: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: b .LBB0_2 +; SOFTFP-NOLSE-NEXT: .LBB0_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w23 +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB0_5 +; SOFTFP-NOLSE-NEXT: .LBB0_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB0_3 Depth 2 +; SOFTFP-NOLSE-NEXT: and w0, w21, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: and w23, w20, #0xffff +; SOFTFP-NOLSE-NEXT: mov w22, w0 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: bl __addsf3 +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: .LBB0_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB0_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB0_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB0_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB0_3 +; SOFTFP-NOLSE-NEXT: b .LBB0_1 +; SOFTFP-NOLSE-NEXT: .LBB0_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2 + ret half %res +} + +define half @test_atomicrmw_fadd_f16_seq_cst_align4(ptr %ptr, half %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: fcvt s1, h0 +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: b .LBB1_2 +; NOLSE-NEXT: .LBB1_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.eq .LBB1_5 +; NOLSE-NEXT: .LBB1_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB1_3 Depth 2 +; NOLSE-NEXT: fcvt s2, h0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fadd s2, s2, s1 +; NOLSE-NEXT: fcvt h2, s2 +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB1_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB1_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w10, [x0] +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.ne .LBB1_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB1_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB1_3 +; NOLSE-NEXT: b .LBB1_1 +; NOLSE-NEXT: .LBB1_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: fcvt s1, h0 +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: .LBB1_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fcvt s2, h0 +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fadd s2, s2, s1 +; LSE-NEXT: fcvt h2, s2 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casalh w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8, uxth +; LSE-NEXT: b.ne .LBB1_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: b .LBB1_2 +; SOFTFP-NOLSE-NEXT: .LBB1_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w23 +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB1_5 +; SOFTFP-NOLSE-NEXT: .LBB1_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB1_3 Depth 2 +; SOFTFP-NOLSE-NEXT: and w0, w21, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: and w23, w20, #0xffff +; SOFTFP-NOLSE-NEXT: mov w22, w0 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: bl __addsf3 +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: .LBB1_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB1_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB1_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB1_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB1_3 +; SOFTFP-NOLSE-NEXT: b .LBB1_1 +; SOFTFP-NOLSE-NEXT: .LBB1_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fadd ptr %ptr, half %value seq_cst, align 4 + ret half %res +} + +define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align2(ptr %ptr, bfloat %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fadd_bf16_seq_cst_align2: +; NOLSE: // %bb.0: +; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: mov w8, #32767 // =0x7fff +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s1, w9 +; NOLSE-NEXT: b .LBB2_2 +; NOLSE-NEXT: .LBB2_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB2_2 Depth=1 +; NOLSE-NEXT: fmov s0, w11 +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.eq .LBB2_5 +; NOLSE-NEXT: .LBB2_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB2_3 Depth 2 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fadd s2, s2, s1 +; NOLSE-NEXT: fmov w9, s2 +; NOLSE-NEXT: ubfx w10, w9, #16, #1 +; NOLSE-NEXT: add w9, w9, w8 +; NOLSE-NEXT: add w9, w10, w9 +; NOLSE-NEXT: lsr w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fmov w10, s2 +; NOLSE-NEXT: .LBB2_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB2_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w11, [x0] +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.ne .LBB2_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB2_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w10, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB2_3 +; NOLSE-NEXT: b .LBB2_1 +; NOLSE-NEXT: .LBB2_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fadd_bf16_seq_cst_align2: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: mov w8, #32767 // =0x7fff +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s1, w9 +; LSE-NEXT: .LBB2_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s2, w9 +; LSE-NEXT: fadd s2, s2, s1 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: ubfx w10, w9, #16, #1 +; LSE-NEXT: add w9, w9, w8 +; LSE-NEXT: add w9, w10, w9 +; LSE-NEXT: fmov w10, s0 +; LSE-NEXT: lsr w9, w9, #16 +; LSE-NEXT: mov w11, w10 +; LSE-NEXT: casalh w11, w9, [x0] +; LSE-NEXT: fmov s0, w11 +; LSE-NEXT: cmp w11, w10, uxth +; LSE-NEXT: b.ne .LBB2_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fadd_bf16_seq_cst_align2: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: lsl w21, w1, #16 +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: b .LBB2_2 +; SOFTFP-NOLSE-NEXT: .LBB2_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB2_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB2_5 +; SOFTFP-NOLSE-NEXT: .LBB2_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB2_3 Depth 2 +; SOFTFP-NOLSE-NEXT: lsl w0, w20, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl __addsf3 +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: .LBB2_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB2_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB2_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB2_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB2_3 +; SOFTFP-NOLSE-NEXT: b .LBB2_1 +; SOFTFP-NOLSE-NEXT: .LBB2_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 2 + ret bfloat %res +} + +define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align4(ptr %ptr, bfloat %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fadd_bf16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: mov w8, #32767 // =0x7fff +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s1, w9 +; NOLSE-NEXT: b .LBB3_2 +; NOLSE-NEXT: .LBB3_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB3_2 Depth=1 +; NOLSE-NEXT: fmov s0, w11 +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.eq .LBB3_5 +; NOLSE-NEXT: .LBB3_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB3_3 Depth 2 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fadd s2, s2, s1 +; NOLSE-NEXT: fmov w9, s2 +; NOLSE-NEXT: ubfx w10, w9, #16, #1 +; NOLSE-NEXT: add w9, w9, w8 +; NOLSE-NEXT: add w9, w10, w9 +; NOLSE-NEXT: lsr w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fmov w10, s2 +; NOLSE-NEXT: .LBB3_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB3_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w11, [x0] +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.ne .LBB3_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB3_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w10, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB3_3 +; NOLSE-NEXT: b .LBB3_1 +; NOLSE-NEXT: .LBB3_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fadd_bf16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: mov w8, #32767 // =0x7fff +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s1, w9 +; LSE-NEXT: .LBB3_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s2, w9 +; LSE-NEXT: fadd s2, s2, s1 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: ubfx w10, w9, #16, #1 +; LSE-NEXT: add w9, w9, w8 +; LSE-NEXT: add w9, w10, w9 +; LSE-NEXT: fmov w10, s0 +; LSE-NEXT: lsr w9, w9, #16 +; LSE-NEXT: mov w11, w10 +; LSE-NEXT: casalh w11, w9, [x0] +; LSE-NEXT: fmov s0, w11 +; LSE-NEXT: cmp w11, w10, uxth +; LSE-NEXT: b.ne .LBB3_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fadd_bf16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: lsl w21, w1, #16 +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: b .LBB3_2 +; SOFTFP-NOLSE-NEXT: .LBB3_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB3_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB3_5 +; SOFTFP-NOLSE-NEXT: .LBB3_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB3_3 Depth 2 +; SOFTFP-NOLSE-NEXT: lsl w0, w20, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl __addsf3 +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: .LBB3_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB3_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB3_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB3_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB3_3 +; SOFTFP-NOLSE-NEXT: b .LBB3_1 +; SOFTFP-NOLSE-NEXT: .LBB3_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 4 + ret bfloat %res +} + +define float @test_atomicrmw_fadd_f32_seq_cst_align4(ptr %ptr, float %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr s1, [x0] +; NOLSE-NEXT: b .LBB4_2 +; NOLSE-NEXT: .LBB4_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=1 +; NOLSE-NEXT: fmov s1, w10 +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.eq .LBB4_5 +; NOLSE-NEXT: .LBB4_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB4_3 Depth 2 +; NOLSE-NEXT: fadd s2, s1, s0 +; NOLSE-NEXT: fmov w9, s1 +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB4_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB4_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr w10, [x0] +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.ne .LBB4_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB4_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB4_3 +; NOLSE-NEXT: b .LBB4_1 +; NOLSE-NEXT: .LBB4_5: // %atomicrmw.end +; NOLSE-NEXT: fmov s0, s1 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: ldr s1, [x0] +; LSE-NEXT: .LBB4_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fadd s2, s1, s0 +; LSE-NEXT: fmov w8, s1 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casal w10, w9, [x0] +; LSE-NEXT: fmov s1, w10 +; LSE-NEXT: cmp w10, w8 +; LSE-NEXT: b.ne .LBB4_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: fmov s0, s1 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldr w20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: b .LBB4_2 +; SOFTFP-NOLSE-NEXT: .LBB4_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w20 +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB4_5 +; SOFTFP-NOLSE-NEXT: .LBB4_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB4_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl __addsf3 +; SOFTFP-NOLSE-NEXT: .LBB4_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB4_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20 +; SOFTFP-NOLSE-NEXT: b.ne .LBB4_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB4_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB4_3 +; SOFTFP-NOLSE-NEXT: b .LBB4_1 +; SOFTFP-NOLSE-NEXT: .LBB4_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4 + ret float %res +} + +define double @test_atomicrmw_fadd_f32_seq_cst_align8(ptr %ptr, double %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr d1, [x0] +; NOLSE-NEXT: b .LBB5_2 +; NOLSE-NEXT: .LBB5_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB5_2 Depth=1 +; NOLSE-NEXT: fmov d1, x10 +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.eq .LBB5_5 +; NOLSE-NEXT: .LBB5_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB5_3 Depth 2 +; NOLSE-NEXT: fadd d2, d1, d0 +; NOLSE-NEXT: fmov x9, d1 +; NOLSE-NEXT: fmov x8, d2 +; NOLSE-NEXT: .LBB5_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB5_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr x10, [x0] +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.ne .LBB5_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB5_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, x8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB5_3 +; NOLSE-NEXT: b .LBB5_1 +; NOLSE-NEXT: .LBB5_5: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, d1 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align8: +; LSE: // %bb.0: +; LSE-NEXT: ldr d1, [x0] +; LSE-NEXT: .LBB5_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fadd d2, d1, d0 +; LSE-NEXT: fmov x8, d1 +; LSE-NEXT: mov x10, x8 +; LSE-NEXT: fmov x9, d2 +; LSE-NEXT: casal x10, x9, [x0] +; LSE-NEXT: fmov d1, x10 +; LSE-NEXT: cmp x10, x8 +; LSE-NEXT: b.ne .LBB5_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: fmov d0, d1 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align8: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldr x20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: mov x21, x1 +; SOFTFP-NOLSE-NEXT: b .LBB5_2 +; SOFTFP-NOLSE-NEXT: .LBB5_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB5_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp x8, x20 +; SOFTFP-NOLSE-NEXT: mov x20, x8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB5_5 +; SOFTFP-NOLSE-NEXT: .LBB5_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB5_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov x0, x20 +; SOFTFP-NOLSE-NEXT: mov x1, x21 +; SOFTFP-NOLSE-NEXT: bl __adddf3 +; SOFTFP-NOLSE-NEXT: .LBB5_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB5_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr x8, [x19] +; SOFTFP-NOLSE-NEXT: cmp x8, x20 +; SOFTFP-NOLSE-NEXT: b.ne .LBB5_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB5_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, x0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB5_3 +; SOFTFP-NOLSE-NEXT: b .LBB5_1 +; SOFTFP-NOLSE-NEXT: .LBB5_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov x0, x20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8 + ret double %res +} + +define fp128 @test_atomicrmw_fadd_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fadd_fp128_seq_cst_align16: +; NOLSE: // %bb.0: +; NOLSE-NEXT: sub sp, sp, #96 +; NOLSE-NEXT: ldr q1, [x0] +; NOLSE-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill +; NOLSE-NEXT: mov x19, x0 +; NOLSE-NEXT: str q0, [sp] // 16-byte Folded Spill +; NOLSE-NEXT: b .LBB6_2 +; NOLSE-NEXT: .LBB6_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 +; NOLSE-NEXT: stp x12, x13, [sp, #32] +; NOLSE-NEXT: cmp x13, x10 +; NOLSE-NEXT: ldr q1, [sp, #32] +; NOLSE-NEXT: ccmp x12, x11, #0, eq +; NOLSE-NEXT: b.eq .LBB6_6 +; NOLSE-NEXT: .LBB6_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB6_3 Depth 2 +; NOLSE-NEXT: mov v0.16b, v1.16b +; NOLSE-NEXT: str q1, [sp, #16] // 16-byte Folded Spill +; NOLSE-NEXT: ldr q1, [sp] // 16-byte Folded Reload +; NOLSE-NEXT: bl __addtf3 +; NOLSE-NEXT: str q0, [sp, #48] +; NOLSE-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload +; NOLSE-NEXT: ldp x9, x8, [sp, #48] +; NOLSE-NEXT: str q0, [sp, #64] +; NOLSE-NEXT: ldp x11, x10, [sp, #64] +; NOLSE-NEXT: .LBB6_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxp x12, x13, [x19] +; NOLSE-NEXT: cmp x12, x11 +; NOLSE-NEXT: cset w14, ne +; NOLSE-NEXT: cmp x13, x10 +; NOLSE-NEXT: cinc w14, w14, ne +; NOLSE-NEXT: cbz w14, .LBB6_5 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x12, x13, [x19] +; NOLSE-NEXT: cbnz w14, .LBB6_3 +; NOLSE-NEXT: b .LBB6_1 +; NOLSE-NEXT: .LBB6_5: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x9, x8, [x19] +; NOLSE-NEXT: cbnz w14, .LBB6_3 +; NOLSE-NEXT: b .LBB6_1 +; NOLSE-NEXT: .LBB6_6: // %atomicrmw.end +; NOLSE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload +; NOLSE-NEXT: mov v0.16b, v1.16b +; NOLSE-NEXT: add sp, sp, #96 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fadd_fp128_seq_cst_align16: +; LSE: // %bb.0: +; LSE-NEXT: sub sp, sp, #96 +; LSE-NEXT: ldr q1, [x0] +; LSE-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill +; LSE-NEXT: mov x19, x0 +; LSE-NEXT: str q0, [sp] // 16-byte Folded Spill +; LSE-NEXT: .LBB6_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: mov v0.16b, v1.16b +; LSE-NEXT: str q1, [sp, #16] // 16-byte Folded Spill +; LSE-NEXT: ldr q1, [sp] // 16-byte Folded Reload +; LSE-NEXT: bl __addtf3 +; LSE-NEXT: str q0, [sp, #48] +; LSE-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload +; LSE-NEXT: ldp x0, x1, [sp, #48] +; LSE-NEXT: str q0, [sp, #64] +; LSE-NEXT: ldp x2, x3, [sp, #64] +; LSE-NEXT: mov x4, x2 +; LSE-NEXT: mov x5, x3 +; LSE-NEXT: caspal x4, x5, x0, x1, [x19] +; LSE-NEXT: stp x4, x5, [sp, #32] +; LSE-NEXT: cmp x5, x3 +; LSE-NEXT: ldr q1, [sp, #32] +; LSE-NEXT: ccmp x4, x2, #0, eq +; LSE-NEXT: b.ne .LBB6_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload +; LSE-NEXT: mov v0.16b, v1.16b +; LSE-NEXT: add sp, sp, #96 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fadd_fp128_seq_cst_align16: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: mov x19, x3 +; SOFTFP-NOLSE-NEXT: ldp x0, x1, [x0] +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x21, x2 +; SOFTFP-NOLSE-NEXT: b .LBB6_2 +; SOFTFP-NOLSE-NEXT: .LBB6_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: ccmp x0, x23, #0, eq +; SOFTFP-NOLSE-NEXT: b.eq .LBB6_6 +; SOFTFP-NOLSE-NEXT: .LBB6_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB6_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov x2, x21 +; SOFTFP-NOLSE-NEXT: mov x3, x19 +; SOFTFP-NOLSE-NEXT: mov x22, x1 +; SOFTFP-NOLSE-NEXT: mov x23, x0 +; SOFTFP-NOLSE-NEXT: bl __addtf3 +; SOFTFP-NOLSE-NEXT: mov x8, x0 +; SOFTFP-NOLSE-NEXT: mov x9, x1 +; SOFTFP-NOLSE-NEXT: .LBB6_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxp x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cmp x0, x23 +; SOFTFP-NOLSE-NEXT: cset w10, ne +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: cinc w10, w10, ne +; SOFTFP-NOLSE-NEXT: cbz w10, .LBB6_5 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w10, x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w10, .LBB6_3 +; SOFTFP-NOLSE-NEXT: b .LBB6_1 +; SOFTFP-NOLSE-NEXT: .LBB6_5: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w10, x8, x9, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w10, .LBB6_3 +; SOFTFP-NOLSE-NEXT: b .LBB6_1 +; SOFTFP-NOLSE-NEXT: .LBB6_6: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fadd ptr %ptr, fp128 %value seq_cst, align 16 + ret fp128 %res +} + +define <2 x half> @test_atomicrmw_fadd_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fadd_v2f16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: fcvtl v1.4s, v0.4h +; NOLSE-NEXT: ldr s0, [x0] +; NOLSE-NEXT: b .LBB7_2 +; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.eq .LBB7_5 +; NOLSE-NEXT: .LBB7_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB7_3 Depth 2 +; NOLSE-NEXT: fcvtl v2.4s, v0.4h +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fadd v2.4s, v2.4s, v1.4s +; NOLSE-NEXT: fcvtn v2.4h, v2.4s +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB7_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr w10, [x0] +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.ne .LBB7_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB7_3 +; NOLSE-NEXT: b .LBB7_1 +; NOLSE-NEXT: .LBB7_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fadd_v2f16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: fcvtl v1.4s, v0.4h +; LSE-NEXT: ldr s0, [x0] +; LSE-NEXT: .LBB7_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fcvtl v2.4s, v0.4h +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fadd v2.4s, v2.4s, v1.4s +; LSE-NEXT: fcvtn v2.4h, v2.4s +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casal w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8 +; LSE-NEXT: b.ne .LBB7_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fadd_v2f16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x25, [sp, #-64]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w23, [x0, #2] +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w21, [x0] +; SOFTFP-NOLSE-NEXT: mov w22, w1 +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w19, w2 +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: b .LBB7_2 +; SOFTFP-NOLSE-NEXT: .LBB7_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 +; SOFTFP-NOLSE-NEXT: lsr w23, w8, #16 +; SOFTFP-NOLSE-NEXT: cmp w8, w21 +; SOFTFP-NOLSE-NEXT: mov w21, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB7_5 +; SOFTFP-NOLSE-NEXT: .LBB7_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB7_3 Depth 2 +; SOFTFP-NOLSE-NEXT: and w0, w19, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: and w0, w23, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w24 +; SOFTFP-NOLSE-NEXT: bl __addsf3 +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: and w0, w22, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w25, w0 +; SOFTFP-NOLSE-NEXT: and w0, w21, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w25 +; SOFTFP-NOLSE-NEXT: bl __addsf3 +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: bfi w21, w23, #16, #16 +; SOFTFP-NOLSE-NEXT: bfi w0, w24, #16, #16 +; SOFTFP-NOLSE-NEXT: .LBB7_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr w8, [x20] +; SOFTFP-NOLSE-NEXT: cmp w8, w21 +; SOFTFP-NOLSE-NEXT: b.ne .LBB7_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x20] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB7_3 +; SOFTFP-NOLSE-NEXT: b .LBB7_1 +; SOFTFP-NOLSE-NEXT: .LBB7_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w21 +; SOFTFP-NOLSE-NEXT: mov w1, w23 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x25, [sp], #64 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fadd ptr %ptr, <2 x half> %value seq_cst, align 4 + ret <2 x half> %res +} + +define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_seq_cst_align4(ptr %ptr, <2 x bfloat> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fadd_v2bf16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: movi v1.4s, #1 +; NOLSE-NEXT: movi v2.4s, #127, msl #8 +; NOLSE-NEXT: shll v3.4s, v0.4h, #16 +; NOLSE-NEXT: ldr s0, [x0] +; NOLSE-NEXT: b .LBB8_2 +; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.eq .LBB8_5 +; NOLSE-NEXT: .LBB8_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB8_3 Depth 2 +; NOLSE-NEXT: shll v4.4s, v0.4h, #16 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fadd v4.4s, v4.4s, v3.4s +; NOLSE-NEXT: ushr v5.4s, v4.4s, #16 +; NOLSE-NEXT: and v5.16b, v5.16b, v1.16b +; NOLSE-NEXT: add v4.4s, v5.4s, v4.4s +; NOLSE-NEXT: addhn v4.4h, v4.4s, v2.4s +; NOLSE-NEXT: fmov w8, s4 +; NOLSE-NEXT: .LBB8_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr w10, [x0] +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.ne .LBB8_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB8_3 +; NOLSE-NEXT: b .LBB8_1 +; NOLSE-NEXT: .LBB8_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fadd_v2bf16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: movi v1.4s, #1 +; LSE-NEXT: movi v2.4s, #127, msl #8 +; LSE-NEXT: shll v3.4s, v0.4h, #16 +; LSE-NEXT: ldr s0, [x0] +; LSE-NEXT: .LBB8_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: shll v4.4s, v0.4h, #16 +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: fadd v4.4s, v4.4s, v3.4s +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: ushr v5.4s, v4.4s, #16 +; LSE-NEXT: and v5.16b, v5.16b, v1.16b +; LSE-NEXT: add v4.4s, v5.4s, v4.4s +; LSE-NEXT: addhn v4.4h, v4.4s, v2.4s +; LSE-NEXT: fmov w9, s4 +; LSE-NEXT: casal w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8 +; LSE-NEXT: b.ne .LBB8_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fadd_v2bf16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: str x30, [sp, #-64]! // 8-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w8, w1 +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w1, [x0, #2] +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w21, [x0] +; SOFTFP-NOLSE-NEXT: lsl w20, w2, #16 +; SOFTFP-NOLSE-NEXT: lsl w22, w8, #16 +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: b .LBB8_2 +; SOFTFP-NOLSE-NEXT: .LBB8_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 +; SOFTFP-NOLSE-NEXT: lsr w1, w21, #16 +; SOFTFP-NOLSE-NEXT: cmp w21, w23 +; SOFTFP-NOLSE-NEXT: b.eq .LBB8_5 +; SOFTFP-NOLSE-NEXT: .LBB8_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB8_3 Depth 2 +; SOFTFP-NOLSE-NEXT: lsl w23, w1, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w20 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: bl __addsf3 +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: lsl w0, w21, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: bl __addsf3 +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: bfxil w23, w21, #0, #16 +; SOFTFP-NOLSE-NEXT: bfi w0, w24, #16, #16 +; SOFTFP-NOLSE-NEXT: .LBB8_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr w21, [x19] +; SOFTFP-NOLSE-NEXT: cmp w21, w23 +; SOFTFP-NOLSE-NEXT: b.ne .LBB8_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB8_3 +; SOFTFP-NOLSE-NEXT: b .LBB8_1 +; SOFTFP-NOLSE-NEXT: .LBB8_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w21 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldr x30, [sp], #64 // 8-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fadd ptr %ptr, <2 x bfloat> %value seq_cst, align 4 + ret <2 x bfloat> %res +} + +define <2 x float> @test_atomicrmw_fadd_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fadd_v2f32_seq_cst_align8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr d1, [x0] +; NOLSE-NEXT: b .LBB9_2 +; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 +; NOLSE-NEXT: fmov d1, x10 +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.eq .LBB9_5 +; NOLSE-NEXT: .LBB9_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; NOLSE-NEXT: fadd v2.2s, v1.2s, v0.2s +; NOLSE-NEXT: fmov x9, d1 +; NOLSE-NEXT: fmov x8, d2 +; NOLSE-NEXT: .LBB9_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr x10, [x0] +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.ne .LBB9_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, x8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB9_3 +; NOLSE-NEXT: b .LBB9_1 +; NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, d1 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fadd_v2f32_seq_cst_align8: +; LSE: // %bb.0: +; LSE-NEXT: ldr d1, [x0] +; LSE-NEXT: .LBB9_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fadd v2.2s, v1.2s, v0.2s +; LSE-NEXT: fmov x8, d1 +; LSE-NEXT: mov x10, x8 +; LSE-NEXT: fmov x9, d2 +; LSE-NEXT: casal x10, x9, [x0] +; LSE-NEXT: fmov d1, x10 +; LSE-NEXT: cmp x10, x8 +; LSE-NEXT: b.ne .LBB9_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: fmov d0, d1 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fadd_v2f32_seq_cst_align8: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: str x30, [sp, #-64]! // 8-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: ldp w23, w22, [x0] +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w19, w2 +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: b .LBB9_2 +; SOFTFP-NOLSE-NEXT: .LBB9_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 +; SOFTFP-NOLSE-NEXT: lsr x22, x23, #32 +; SOFTFP-NOLSE-NEXT: cmp x23, x8 +; SOFTFP-NOLSE-NEXT: // kill: def $w22 killed $w22 killed $x22 def $x22 +; SOFTFP-NOLSE-NEXT: b.eq .LBB9_5 +; SOFTFP-NOLSE-NEXT: .LBB9_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov w0, w22 +; SOFTFP-NOLSE-NEXT: mov w1, w19 +; SOFTFP-NOLSE-NEXT: bl __addsf3 +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl __addsf3 +; SOFTFP-NOLSE-NEXT: mov w8, w23 +; SOFTFP-NOLSE-NEXT: mov w9, w0 +; SOFTFP-NOLSE-NEXT: orr x9, x9, x24, lsl #32 +; SOFTFP-NOLSE-NEXT: orr x8, x8, x22, lsl #32 +; SOFTFP-NOLSE-NEXT: .LBB9_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr x23, [x20] +; SOFTFP-NOLSE-NEXT: cmp x23, x8 +; SOFTFP-NOLSE-NEXT: b.ne .LBB9_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, x9, [x20] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB9_3 +; SOFTFP-NOLSE-NEXT: b .LBB9_1 +; SOFTFP-NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldr x30, [sp], #64 // 8-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fadd ptr %ptr, <2 x float> %value seq_cst, align 8 + ret <2 x float> %res +} + +define <2 x double> @test_atomicrmw_fadd_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fadd_v2f64_seq_cst_align8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr q1, [x0] +; NOLSE-NEXT: b .LBB10_2 +; NOLSE-NEXT: .LBB10_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 +; NOLSE-NEXT: fmov d1, x12 +; NOLSE-NEXT: cmp x13, x9 +; NOLSE-NEXT: ccmp x12, x11, #0, eq +; NOLSE-NEXT: mov v1.d[1], x13 +; NOLSE-NEXT: b.eq .LBB10_6 +; NOLSE-NEXT: .LBB10_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; NOLSE-NEXT: fadd v2.2d, v1.2d, v0.2d +; NOLSE-NEXT: mov x9, v1.d[1] +; NOLSE-NEXT: fmov x11, d1 +; NOLSE-NEXT: mov x8, v2.d[1] +; NOLSE-NEXT: fmov x10, d2 +; NOLSE-NEXT: .LBB10_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxp x12, x13, [x0] +; NOLSE-NEXT: cmp x12, x11 +; NOLSE-NEXT: cset w14, ne +; NOLSE-NEXT: cmp x13, x9 +; NOLSE-NEXT: cinc w14, w14, ne +; NOLSE-NEXT: cbz w14, .LBB10_5 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x12, x13, [x0] +; NOLSE-NEXT: cbnz w14, .LBB10_3 +; NOLSE-NEXT: b .LBB10_1 +; NOLSE-NEXT: .LBB10_5: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x10, x8, [x0] +; NOLSE-NEXT: cbnz w14, .LBB10_3 +; NOLSE-NEXT: b .LBB10_1 +; NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; NOLSE-NEXT: mov v0.16b, v1.16b +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fadd_v2f64_seq_cst_align8: +; LSE: // %bb.0: +; LSE-NEXT: ldr q1, [x0] +; LSE-NEXT: .LBB10_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fadd v2.2d, v1.2d, v0.2d +; LSE-NEXT: mov x3, v1.d[1] +; LSE-NEXT: fmov x2, d1 +; LSE-NEXT: mov x7, x3 +; LSE-NEXT: mov x5, v2.d[1] +; LSE-NEXT: mov x6, x2 +; LSE-NEXT: fmov x4, d2 +; LSE-NEXT: caspal x6, x7, x4, x5, [x0] +; LSE-NEXT: fmov d1, x6 +; LSE-NEXT: cmp x7, x3 +; LSE-NEXT: ccmp x6, x2, #0, eq +; LSE-NEXT: mov v1.d[1], x7 +; LSE-NEXT: b.ne .LBB10_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: mov v0.16b, v1.16b +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fadd_v2f64_seq_cst_align8: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: str x30, [sp, #-64]! // 8-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: mov x19, x3 +; SOFTFP-NOLSE-NEXT: ldp x0, x1, [x0] +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x21, x2 +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: b .LBB10_2 +; SOFTFP-NOLSE-NEXT: .LBB10_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: ccmp x0, x23, #0, eq +; SOFTFP-NOLSE-NEXT: b.eq .LBB10_6 +; SOFTFP-NOLSE-NEXT: .LBB10_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov x22, x1 +; SOFTFP-NOLSE-NEXT: mov x23, x0 +; SOFTFP-NOLSE-NEXT: mov x0, x1 +; SOFTFP-NOLSE-NEXT: mov x1, x19 +; SOFTFP-NOLSE-NEXT: bl __adddf3 +; SOFTFP-NOLSE-NEXT: mov x24, x0 +; SOFTFP-NOLSE-NEXT: mov x0, x23 +; SOFTFP-NOLSE-NEXT: mov x1, x21 +; SOFTFP-NOLSE-NEXT: bl __adddf3 +; SOFTFP-NOLSE-NEXT: mov x8, x0 +; SOFTFP-NOLSE-NEXT: .LBB10_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxp x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cmp x0, x23 +; SOFTFP-NOLSE-NEXT: cset w9, ne +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: cinc w9, w9, ne +; SOFTFP-NOLSE-NEXT: cbz w9, .LBB10_5 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w9, x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB10_3 +; SOFTFP-NOLSE-NEXT: b .LBB10_1 +; SOFTFP-NOLSE-NEXT: .LBB10_5: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w9, x8, x24, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB10_3 +; SOFTFP-NOLSE-NEXT: b .LBB10_1 +; SOFTFP-NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldr x30, [sp], #64 // 8-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fadd ptr %ptr, <2 x double> %value seq_cst, align 16 + ret <2 x double> %res +} + +attributes #0 = { nounwind } diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll new file mode 100644 index 000000000000..998d8ae0c1de --- /dev/null +++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll @@ -0,0 +1,1272 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=aarch64-linux-gnu -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=NOLSE %s +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+lse -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=LSE %s +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=-lse,-fp-armv8 -O1 < %s | FileCheck -check-prefix=SOFTFP-NOLSE %s + +; FIXME: Windows hosts assigns stack slots to different offsets for some reason. +; UNSUPPORTED: system-windows + +define half @test_atomicrmw_fmax_f16_seq_cst_align2(ptr %ptr, half %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align2: +; NOLSE: // %bb.0: +; NOLSE-NEXT: fcvt s1, h0 +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: b .LBB0_2 +; NOLSE-NEXT: .LBB0_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.eq .LBB0_5 +; NOLSE-NEXT: .LBB0_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB0_3 Depth 2 +; NOLSE-NEXT: fcvt s2, h0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fmaxnm s2, s2, s1 +; NOLSE-NEXT: fcvt h2, s2 +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB0_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB0_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w10, [x0] +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.ne .LBB0_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB0_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB0_3 +; NOLSE-NEXT: b .LBB0_1 +; NOLSE-NEXT: .LBB0_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align2: +; LSE: // %bb.0: +; LSE-NEXT: fcvt s1, h0 +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: .LBB0_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fcvt s2, h0 +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fmaxnm s2, s2, s1 +; LSE-NEXT: fcvt h2, s2 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casalh w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8, uxth +; LSE-NEXT: b.ne .LBB0_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align2: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: b .LBB0_2 +; SOFTFP-NOLSE-NEXT: .LBB0_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w23 +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB0_5 +; SOFTFP-NOLSE-NEXT: .LBB0_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB0_3 Depth 2 +; SOFTFP-NOLSE-NEXT: and w0, w21, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: and w23, w20, #0xffff +; SOFTFP-NOLSE-NEXT: mov w22, w0 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: bl fmaxf +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: .LBB0_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB0_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB0_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB0_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB0_3 +; SOFTFP-NOLSE-NEXT: b .LBB0_1 +; SOFTFP-NOLSE-NEXT: .LBB0_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2 + ret half %res +} + +define half @test_atomicrmw_fmax_f16_seq_cst_align4(ptr %ptr, half %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: fcvt s1, h0 +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: b .LBB1_2 +; NOLSE-NEXT: .LBB1_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.eq .LBB1_5 +; NOLSE-NEXT: .LBB1_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB1_3 Depth 2 +; NOLSE-NEXT: fcvt s2, h0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fmaxnm s2, s2, s1 +; NOLSE-NEXT: fcvt h2, s2 +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB1_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB1_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w10, [x0] +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.ne .LBB1_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB1_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB1_3 +; NOLSE-NEXT: b .LBB1_1 +; NOLSE-NEXT: .LBB1_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: fcvt s1, h0 +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: .LBB1_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fcvt s2, h0 +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fmaxnm s2, s2, s1 +; LSE-NEXT: fcvt h2, s2 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casalh w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8, uxth +; LSE-NEXT: b.ne .LBB1_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: b .LBB1_2 +; SOFTFP-NOLSE-NEXT: .LBB1_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w23 +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB1_5 +; SOFTFP-NOLSE-NEXT: .LBB1_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB1_3 Depth 2 +; SOFTFP-NOLSE-NEXT: and w0, w21, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: and w23, w20, #0xffff +; SOFTFP-NOLSE-NEXT: mov w22, w0 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: bl fmaxf +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: .LBB1_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB1_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB1_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB1_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB1_3 +; SOFTFP-NOLSE-NEXT: b .LBB1_1 +; SOFTFP-NOLSE-NEXT: .LBB1_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmax ptr %ptr, half %value seq_cst, align 4 + ret half %res +} + +define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align2(ptr %ptr, bfloat %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmax_bf16_seq_cst_align2: +; NOLSE: // %bb.0: +; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: mov w8, #32767 // =0x7fff +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s1, w9 +; NOLSE-NEXT: b .LBB2_2 +; NOLSE-NEXT: .LBB2_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB2_2 Depth=1 +; NOLSE-NEXT: fmov s0, w11 +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.eq .LBB2_5 +; NOLSE-NEXT: .LBB2_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB2_3 Depth 2 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fmaxnm s2, s2, s1 +; NOLSE-NEXT: fmov w9, s2 +; NOLSE-NEXT: ubfx w10, w9, #16, #1 +; NOLSE-NEXT: add w9, w9, w8 +; NOLSE-NEXT: add w9, w10, w9 +; NOLSE-NEXT: lsr w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fmov w10, s2 +; NOLSE-NEXT: .LBB2_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB2_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w11, [x0] +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.ne .LBB2_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB2_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w10, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB2_3 +; NOLSE-NEXT: b .LBB2_1 +; NOLSE-NEXT: .LBB2_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmax_bf16_seq_cst_align2: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: mov w8, #32767 // =0x7fff +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s1, w9 +; LSE-NEXT: .LBB2_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s2, w9 +; LSE-NEXT: fmaxnm s2, s2, s1 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: ubfx w10, w9, #16, #1 +; LSE-NEXT: add w9, w9, w8 +; LSE-NEXT: add w9, w10, w9 +; LSE-NEXT: fmov w10, s0 +; LSE-NEXT: lsr w9, w9, #16 +; LSE-NEXT: mov w11, w10 +; LSE-NEXT: casalh w11, w9, [x0] +; LSE-NEXT: fmov s0, w11 +; LSE-NEXT: cmp w11, w10, uxth +; LSE-NEXT: b.ne .LBB2_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmax_bf16_seq_cst_align2: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: lsl w21, w1, #16 +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: b .LBB2_2 +; SOFTFP-NOLSE-NEXT: .LBB2_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB2_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB2_5 +; SOFTFP-NOLSE-NEXT: .LBB2_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB2_3 Depth 2 +; SOFTFP-NOLSE-NEXT: lsl w0, w20, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl fmaxf +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: .LBB2_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB2_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB2_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB2_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB2_3 +; SOFTFP-NOLSE-NEXT: b .LBB2_1 +; SOFTFP-NOLSE-NEXT: .LBB2_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 2 + ret bfloat %res +} + +define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align4(ptr %ptr, bfloat %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmax_bf16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: mov w8, #32767 // =0x7fff +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s1, w9 +; NOLSE-NEXT: b .LBB3_2 +; NOLSE-NEXT: .LBB3_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB3_2 Depth=1 +; NOLSE-NEXT: fmov s0, w11 +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.eq .LBB3_5 +; NOLSE-NEXT: .LBB3_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB3_3 Depth 2 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fmaxnm s2, s2, s1 +; NOLSE-NEXT: fmov w9, s2 +; NOLSE-NEXT: ubfx w10, w9, #16, #1 +; NOLSE-NEXT: add w9, w9, w8 +; NOLSE-NEXT: add w9, w10, w9 +; NOLSE-NEXT: lsr w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fmov w10, s2 +; NOLSE-NEXT: .LBB3_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB3_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w11, [x0] +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.ne .LBB3_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB3_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w10, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB3_3 +; NOLSE-NEXT: b .LBB3_1 +; NOLSE-NEXT: .LBB3_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmax_bf16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: mov w8, #32767 // =0x7fff +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s1, w9 +; LSE-NEXT: .LBB3_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s2, w9 +; LSE-NEXT: fmaxnm s2, s2, s1 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: ubfx w10, w9, #16, #1 +; LSE-NEXT: add w9, w9, w8 +; LSE-NEXT: add w9, w10, w9 +; LSE-NEXT: fmov w10, s0 +; LSE-NEXT: lsr w9, w9, #16 +; LSE-NEXT: mov w11, w10 +; LSE-NEXT: casalh w11, w9, [x0] +; LSE-NEXT: fmov s0, w11 +; LSE-NEXT: cmp w11, w10, uxth +; LSE-NEXT: b.ne .LBB3_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmax_bf16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: lsl w21, w1, #16 +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: b .LBB3_2 +; SOFTFP-NOLSE-NEXT: .LBB3_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB3_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB3_5 +; SOFTFP-NOLSE-NEXT: .LBB3_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB3_3 Depth 2 +; SOFTFP-NOLSE-NEXT: lsl w0, w20, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl fmaxf +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: .LBB3_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB3_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB3_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB3_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB3_3 +; SOFTFP-NOLSE-NEXT: b .LBB3_1 +; SOFTFP-NOLSE-NEXT: .LBB3_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 4 + ret bfloat %res +} + +define float @test_atomicrmw_fmax_f32_seq_cst_align4(ptr %ptr, float %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr s1, [x0] +; NOLSE-NEXT: b .LBB4_2 +; NOLSE-NEXT: .LBB4_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=1 +; NOLSE-NEXT: fmov s1, w10 +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.eq .LBB4_5 +; NOLSE-NEXT: .LBB4_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB4_3 Depth 2 +; NOLSE-NEXT: fmaxnm s2, s1, s0 +; NOLSE-NEXT: fmov w9, s1 +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB4_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB4_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr w10, [x0] +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.ne .LBB4_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB4_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB4_3 +; NOLSE-NEXT: b .LBB4_1 +; NOLSE-NEXT: .LBB4_5: // %atomicrmw.end +; NOLSE-NEXT: fmov s0, s1 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: ldr s1, [x0] +; LSE-NEXT: .LBB4_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fmaxnm s2, s1, s0 +; LSE-NEXT: fmov w8, s1 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casal w10, w9, [x0] +; LSE-NEXT: fmov s1, w10 +; LSE-NEXT: cmp w10, w8 +; LSE-NEXT: b.ne .LBB4_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: fmov s0, s1 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldr w20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: b .LBB4_2 +; SOFTFP-NOLSE-NEXT: .LBB4_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w20 +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB4_5 +; SOFTFP-NOLSE-NEXT: .LBB4_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB4_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl fmaxf +; SOFTFP-NOLSE-NEXT: .LBB4_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB4_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20 +; SOFTFP-NOLSE-NEXT: b.ne .LBB4_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB4_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB4_3 +; SOFTFP-NOLSE-NEXT: b .LBB4_1 +; SOFTFP-NOLSE-NEXT: .LBB4_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4 + ret float %res +} + +define double @test_atomicrmw_fmax_f32_seq_cst_align8(ptr %ptr, double %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr d1, [x0] +; NOLSE-NEXT: b .LBB5_2 +; NOLSE-NEXT: .LBB5_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB5_2 Depth=1 +; NOLSE-NEXT: fmov d1, x10 +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.eq .LBB5_5 +; NOLSE-NEXT: .LBB5_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB5_3 Depth 2 +; NOLSE-NEXT: fmaxnm d2, d1, d0 +; NOLSE-NEXT: fmov x9, d1 +; NOLSE-NEXT: fmov x8, d2 +; NOLSE-NEXT: .LBB5_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB5_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr x10, [x0] +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.ne .LBB5_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB5_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, x8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB5_3 +; NOLSE-NEXT: b .LBB5_1 +; NOLSE-NEXT: .LBB5_5: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, d1 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align8: +; LSE: // %bb.0: +; LSE-NEXT: ldr d1, [x0] +; LSE-NEXT: .LBB5_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fmaxnm d2, d1, d0 +; LSE-NEXT: fmov x8, d1 +; LSE-NEXT: mov x10, x8 +; LSE-NEXT: fmov x9, d2 +; LSE-NEXT: casal x10, x9, [x0] +; LSE-NEXT: fmov d1, x10 +; LSE-NEXT: cmp x10, x8 +; LSE-NEXT: b.ne .LBB5_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: fmov d0, d1 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align8: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldr x20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: mov x21, x1 +; SOFTFP-NOLSE-NEXT: b .LBB5_2 +; SOFTFP-NOLSE-NEXT: .LBB5_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB5_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp x8, x20 +; SOFTFP-NOLSE-NEXT: mov x20, x8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB5_5 +; SOFTFP-NOLSE-NEXT: .LBB5_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB5_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov x0, x20 +; SOFTFP-NOLSE-NEXT: mov x1, x21 +; SOFTFP-NOLSE-NEXT: bl fmax +; SOFTFP-NOLSE-NEXT: .LBB5_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB5_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr x8, [x19] +; SOFTFP-NOLSE-NEXT: cmp x8, x20 +; SOFTFP-NOLSE-NEXT: b.ne .LBB5_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB5_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, x0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB5_3 +; SOFTFP-NOLSE-NEXT: b .LBB5_1 +; SOFTFP-NOLSE-NEXT: .LBB5_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov x0, x20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8 + ret double %res +} + +define fp128 @test_atomicrmw_fmax_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmax_fp128_seq_cst_align16: +; NOLSE: // %bb.0: +; NOLSE-NEXT: sub sp, sp, #96 +; NOLSE-NEXT: ldr q1, [x0] +; NOLSE-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill +; NOLSE-NEXT: mov x19, x0 +; NOLSE-NEXT: str q0, [sp] // 16-byte Folded Spill +; NOLSE-NEXT: b .LBB6_2 +; NOLSE-NEXT: .LBB6_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 +; NOLSE-NEXT: stp x12, x13, [sp, #32] +; NOLSE-NEXT: cmp x13, x10 +; NOLSE-NEXT: ldr q1, [sp, #32] +; NOLSE-NEXT: ccmp x12, x11, #0, eq +; NOLSE-NEXT: b.eq .LBB6_6 +; NOLSE-NEXT: .LBB6_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB6_3 Depth 2 +; NOLSE-NEXT: mov v0.16b, v1.16b +; NOLSE-NEXT: str q1, [sp, #16] // 16-byte Folded Spill +; NOLSE-NEXT: ldr q1, [sp] // 16-byte Folded Reload +; NOLSE-NEXT: bl fmaxl +; NOLSE-NEXT: str q0, [sp, #48] +; NOLSE-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload +; NOLSE-NEXT: ldp x9, x8, [sp, #48] +; NOLSE-NEXT: str q0, [sp, #64] +; NOLSE-NEXT: ldp x11, x10, [sp, #64] +; NOLSE-NEXT: .LBB6_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxp x12, x13, [x19] +; NOLSE-NEXT: cmp x12, x11 +; NOLSE-NEXT: cset w14, ne +; NOLSE-NEXT: cmp x13, x10 +; NOLSE-NEXT: cinc w14, w14, ne +; NOLSE-NEXT: cbz w14, .LBB6_5 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x12, x13, [x19] +; NOLSE-NEXT: cbnz w14, .LBB6_3 +; NOLSE-NEXT: b .LBB6_1 +; NOLSE-NEXT: .LBB6_5: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x9, x8, [x19] +; NOLSE-NEXT: cbnz w14, .LBB6_3 +; NOLSE-NEXT: b .LBB6_1 +; NOLSE-NEXT: .LBB6_6: // %atomicrmw.end +; NOLSE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload +; NOLSE-NEXT: mov v0.16b, v1.16b +; NOLSE-NEXT: add sp, sp, #96 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmax_fp128_seq_cst_align16: +; LSE: // %bb.0: +; LSE-NEXT: sub sp, sp, #96 +; LSE-NEXT: ldr q1, [x0] +; LSE-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill +; LSE-NEXT: mov x19, x0 +; LSE-NEXT: str q0, [sp] // 16-byte Folded Spill +; LSE-NEXT: .LBB6_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: mov v0.16b, v1.16b +; LSE-NEXT: str q1, [sp, #16] // 16-byte Folded Spill +; LSE-NEXT: ldr q1, [sp] // 16-byte Folded Reload +; LSE-NEXT: bl fmaxl +; LSE-NEXT: str q0, [sp, #48] +; LSE-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload +; LSE-NEXT: ldp x0, x1, [sp, #48] +; LSE-NEXT: str q0, [sp, #64] +; LSE-NEXT: ldp x2, x3, [sp, #64] +; LSE-NEXT: mov x4, x2 +; LSE-NEXT: mov x5, x3 +; LSE-NEXT: caspal x4, x5, x0, x1, [x19] +; LSE-NEXT: stp x4, x5, [sp, #32] +; LSE-NEXT: cmp x5, x3 +; LSE-NEXT: ldr q1, [sp, #32] +; LSE-NEXT: ccmp x4, x2, #0, eq +; LSE-NEXT: b.ne .LBB6_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload +; LSE-NEXT: mov v0.16b, v1.16b +; LSE-NEXT: add sp, sp, #96 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmax_fp128_seq_cst_align16: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: mov x19, x3 +; SOFTFP-NOLSE-NEXT: ldp x0, x1, [x0] +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x21, x2 +; SOFTFP-NOLSE-NEXT: b .LBB6_2 +; SOFTFP-NOLSE-NEXT: .LBB6_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: ccmp x0, x23, #0, eq +; SOFTFP-NOLSE-NEXT: b.eq .LBB6_6 +; SOFTFP-NOLSE-NEXT: .LBB6_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB6_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov x2, x21 +; SOFTFP-NOLSE-NEXT: mov x3, x19 +; SOFTFP-NOLSE-NEXT: mov x22, x1 +; SOFTFP-NOLSE-NEXT: mov x23, x0 +; SOFTFP-NOLSE-NEXT: bl fmaxl +; SOFTFP-NOLSE-NEXT: mov x8, x0 +; SOFTFP-NOLSE-NEXT: mov x9, x1 +; SOFTFP-NOLSE-NEXT: .LBB6_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxp x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cmp x0, x23 +; SOFTFP-NOLSE-NEXT: cset w10, ne +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: cinc w10, w10, ne +; SOFTFP-NOLSE-NEXT: cbz w10, .LBB6_5 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w10, x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w10, .LBB6_3 +; SOFTFP-NOLSE-NEXT: b .LBB6_1 +; SOFTFP-NOLSE-NEXT: .LBB6_5: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w10, x8, x9, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w10, .LBB6_3 +; SOFTFP-NOLSE-NEXT: b .LBB6_1 +; SOFTFP-NOLSE-NEXT: .LBB6_6: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmax ptr %ptr, fp128 %value seq_cst, align 16 + ret fp128 %res +} + +define <2 x half> @test_atomicrmw_fmax_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmax_v2f16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: // kill: def $d0 killed $d0 def $q0 +; NOLSE-NEXT: mov h1, v0.h[1] +; NOLSE-NEXT: fcvt s2, h0 +; NOLSE-NEXT: ldr s0, [x0] +; NOLSE-NEXT: fcvt s1, h1 +; NOLSE-NEXT: b .LBB7_2 +; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.eq .LBB7_5 +; NOLSE-NEXT: .LBB7_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB7_3 Depth 2 +; NOLSE-NEXT: mov h3, v0.h[1] +; NOLSE-NEXT: fcvt s4, h0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fcvt s3, h3 +; NOLSE-NEXT: fmaxnm s4, s4, s2 +; NOLSE-NEXT: fmaxnm s3, s3, s1 +; NOLSE-NEXT: fcvt h4, s4 +; NOLSE-NEXT: fcvt h3, s3 +; NOLSE-NEXT: mov v4.h[1], v3.h[0] +; NOLSE-NEXT: fmov w8, s4 +; NOLSE-NEXT: .LBB7_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr w10, [x0] +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.ne .LBB7_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB7_3 +; NOLSE-NEXT: b .LBB7_1 +; NOLSE-NEXT: .LBB7_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmax_v2f16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $d0 killed $d0 def $q0 +; LSE-NEXT: mov h1, v0.h[1] +; LSE-NEXT: fcvt s2, h0 +; LSE-NEXT: ldr s0, [x0] +; LSE-NEXT: fcvt s1, h1 +; LSE-NEXT: .LBB7_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: mov h3, v0.h[1] +; LSE-NEXT: fcvt s4, h0 +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fcvt s3, h3 +; LSE-NEXT: fmaxnm s4, s4, s2 +; LSE-NEXT: fmaxnm s3, s3, s1 +; LSE-NEXT: fcvt h4, s4 +; LSE-NEXT: fcvt h3, s3 +; LSE-NEXT: mov v4.h[1], v3.h[0] +; LSE-NEXT: fmov w9, s4 +; LSE-NEXT: casal w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8 +; LSE-NEXT: b.ne .LBB7_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmax_v2f16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x25, [sp, #-64]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w23, [x0, #2] +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w21, [x0] +; SOFTFP-NOLSE-NEXT: mov w22, w1 +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w19, w2 +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: b .LBB7_2 +; SOFTFP-NOLSE-NEXT: .LBB7_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 +; SOFTFP-NOLSE-NEXT: lsr w23, w8, #16 +; SOFTFP-NOLSE-NEXT: cmp w8, w21 +; SOFTFP-NOLSE-NEXT: mov w21, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB7_5 +; SOFTFP-NOLSE-NEXT: .LBB7_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB7_3 Depth 2 +; SOFTFP-NOLSE-NEXT: and w0, w19, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: and w0, w23, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w24 +; SOFTFP-NOLSE-NEXT: bl fmaxf +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: and w0, w22, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w25, w0 +; SOFTFP-NOLSE-NEXT: and w0, w21, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w25 +; SOFTFP-NOLSE-NEXT: bl fmaxf +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: bfi w21, w23, #16, #16 +; SOFTFP-NOLSE-NEXT: bfi w0, w24, #16, #16 +; SOFTFP-NOLSE-NEXT: .LBB7_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr w8, [x20] +; SOFTFP-NOLSE-NEXT: cmp w8, w21 +; SOFTFP-NOLSE-NEXT: b.ne .LBB7_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x20] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB7_3 +; SOFTFP-NOLSE-NEXT: b .LBB7_1 +; SOFTFP-NOLSE-NEXT: .LBB7_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w21 +; SOFTFP-NOLSE-NEXT: mov w1, w23 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x25, [sp], #64 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmax ptr %ptr, <2 x half> %value seq_cst, align 4 + ret <2 x half> %res +} + +define <2 x bfloat> @test_atomicrmw_fmax_v2bf16_seq_cst_align4(ptr %ptr, <2 x bfloat> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmax_v2bf16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: // kill: def $d0 killed $d0 def $q0 +; NOLSE-NEXT: mov h1, v0.h[1] +; NOLSE-NEXT: fmov w10, s0 +; NOLSE-NEXT: mov w8, #32767 // =0x7fff +; NOLSE-NEXT: ldr s0, [x0] +; NOLSE-NEXT: lsl w10, w10, #16 +; NOLSE-NEXT: fmov w9, s1 +; NOLSE-NEXT: fmov s2, w10 +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s1, w9 +; NOLSE-NEXT: b .LBB8_2 +; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 +; NOLSE-NEXT: fmov s0, w11 +; NOLSE-NEXT: cmp w11, w9 +; NOLSE-NEXT: b.eq .LBB8_5 +; NOLSE-NEXT: .LBB8_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB8_3 Depth 2 +; NOLSE-NEXT: mov h3, v0.h[1] +; NOLSE-NEXT: fmov w10, s0 +; NOLSE-NEXT: lsl w10, w10, #16 +; NOLSE-NEXT: fmov w9, s3 +; NOLSE-NEXT: fmov s4, w10 +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmaxnm s4, s4, s2 +; NOLSE-NEXT: fmov s3, w9 +; NOLSE-NEXT: fmaxnm s3, s3, s1 +; NOLSE-NEXT: fmov w10, s4 +; NOLSE-NEXT: ubfx w12, w10, #16, #1 +; NOLSE-NEXT: add w10, w10, w8 +; NOLSE-NEXT: fmov w9, s3 +; NOLSE-NEXT: add w10, w12, w10 +; NOLSE-NEXT: lsr w10, w10, #16 +; NOLSE-NEXT: ubfx w11, w9, #16, #1 +; NOLSE-NEXT: add w9, w9, w8 +; NOLSE-NEXT: fmov s4, w10 +; NOLSE-NEXT: add w9, w11, w9 +; NOLSE-NEXT: lsr w9, w9, #16 +; NOLSE-NEXT: fmov s3, w9 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: mov v4.h[1], v3.h[0] +; NOLSE-NEXT: fmov w10, s4 +; NOLSE-NEXT: .LBB8_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr w11, [x0] +; NOLSE-NEXT: cmp w11, w9 +; NOLSE-NEXT: b.ne .LBB8_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, w10, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB8_3 +; NOLSE-NEXT: b .LBB8_1 +; NOLSE-NEXT: .LBB8_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmax_v2bf16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $d0 killed $d0 def $q0 +; LSE-NEXT: mov h1, v0.h[1] +; LSE-NEXT: fmov w10, s0 +; LSE-NEXT: mov w8, #32767 // =0x7fff +; LSE-NEXT: ldr s0, [x0] +; LSE-NEXT: lsl w10, w10, #16 +; LSE-NEXT: fmov w9, s1 +; LSE-NEXT: fmov s2, w10 +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s1, w9 +; LSE-NEXT: .LBB8_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: mov h3, v0.h[1] +; LSE-NEXT: fmov w10, s0 +; LSE-NEXT: lsl w10, w10, #16 +; LSE-NEXT: fmov w9, s3 +; LSE-NEXT: fmov s4, w10 +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmaxnm s4, s4, s2 +; LSE-NEXT: fmov s3, w9 +; LSE-NEXT: fmaxnm s3, s3, s1 +; LSE-NEXT: fmov w10, s4 +; LSE-NEXT: ubfx w12, w10, #16, #1 +; LSE-NEXT: add w10, w10, w8 +; LSE-NEXT: fmov w9, s3 +; LSE-NEXT: add w10, w12, w10 +; LSE-NEXT: lsr w10, w10, #16 +; LSE-NEXT: ubfx w11, w9, #16, #1 +; LSE-NEXT: add w9, w9, w8 +; LSE-NEXT: fmov s4, w10 +; LSE-NEXT: add w9, w11, w9 +; LSE-NEXT: lsr w9, w9, #16 +; LSE-NEXT: fmov s3, w9 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: mov v4.h[1], v3.h[0] +; LSE-NEXT: mov w11, w9 +; LSE-NEXT: fmov w10, s4 +; LSE-NEXT: casal w11, w10, [x0] +; LSE-NEXT: fmov s0, w11 +; LSE-NEXT: cmp w11, w9 +; LSE-NEXT: b.ne .LBB8_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmax_v2bf16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: str x30, [sp, #-64]! // 8-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w8, w1 +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w1, [x0, #2] +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w21, [x0] +; SOFTFP-NOLSE-NEXT: lsl w20, w2, #16 +; SOFTFP-NOLSE-NEXT: lsl w22, w8, #16 +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: b .LBB8_2 +; SOFTFP-NOLSE-NEXT: .LBB8_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 +; SOFTFP-NOLSE-NEXT: lsr w1, w21, #16 +; SOFTFP-NOLSE-NEXT: cmp w21, w23 +; SOFTFP-NOLSE-NEXT: b.eq .LBB8_5 +; SOFTFP-NOLSE-NEXT: .LBB8_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB8_3 Depth 2 +; SOFTFP-NOLSE-NEXT: lsl w23, w1, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w20 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: bl fmaxf +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: lsl w0, w21, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: bl fmaxf +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: bfxil w23, w21, #0, #16 +; SOFTFP-NOLSE-NEXT: bfi w0, w24, #16, #16 +; SOFTFP-NOLSE-NEXT: .LBB8_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr w21, [x19] +; SOFTFP-NOLSE-NEXT: cmp w21, w23 +; SOFTFP-NOLSE-NEXT: b.ne .LBB8_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB8_3 +; SOFTFP-NOLSE-NEXT: b .LBB8_1 +; SOFTFP-NOLSE-NEXT: .LBB8_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w21 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldr x30, [sp], #64 // 8-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmax ptr %ptr, <2 x bfloat> %value seq_cst, align 4 + ret <2 x bfloat> %res +} + +define <2 x float> @test_atomicrmw_fmax_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmax_v2f32_seq_cst_align8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr d1, [x0] +; NOLSE-NEXT: b .LBB9_2 +; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 +; NOLSE-NEXT: fmov d1, x10 +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.eq .LBB9_5 +; NOLSE-NEXT: .LBB9_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; NOLSE-NEXT: fmaxnm v2.2s, v1.2s, v0.2s +; NOLSE-NEXT: fmov x9, d1 +; NOLSE-NEXT: fmov x8, d2 +; NOLSE-NEXT: .LBB9_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr x10, [x0] +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.ne .LBB9_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, x8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB9_3 +; NOLSE-NEXT: b .LBB9_1 +; NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, d1 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmax_v2f32_seq_cst_align8: +; LSE: // %bb.0: +; LSE-NEXT: ldr d1, [x0] +; LSE-NEXT: .LBB9_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fmaxnm v2.2s, v1.2s, v0.2s +; LSE-NEXT: fmov x8, d1 +; LSE-NEXT: mov x10, x8 +; LSE-NEXT: fmov x9, d2 +; LSE-NEXT: casal x10, x9, [x0] +; LSE-NEXT: fmov d1, x10 +; LSE-NEXT: cmp x10, x8 +; LSE-NEXT: b.ne .LBB9_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: fmov d0, d1 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmax_v2f32_seq_cst_align8: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: str x30, [sp, #-64]! // 8-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: ldp w23, w22, [x0] +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w19, w2 +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: b .LBB9_2 +; SOFTFP-NOLSE-NEXT: .LBB9_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 +; SOFTFP-NOLSE-NEXT: lsr x22, x23, #32 +; SOFTFP-NOLSE-NEXT: cmp x23, x8 +; SOFTFP-NOLSE-NEXT: // kill: def $w22 killed $w22 killed $x22 def $x22 +; SOFTFP-NOLSE-NEXT: b.eq .LBB9_5 +; SOFTFP-NOLSE-NEXT: .LBB9_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov w0, w22 +; SOFTFP-NOLSE-NEXT: mov w1, w19 +; SOFTFP-NOLSE-NEXT: bl fmaxf +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl fmaxf +; SOFTFP-NOLSE-NEXT: mov w8, w23 +; SOFTFP-NOLSE-NEXT: mov w9, w0 +; SOFTFP-NOLSE-NEXT: orr x9, x9, x24, lsl #32 +; SOFTFP-NOLSE-NEXT: orr x8, x8, x22, lsl #32 +; SOFTFP-NOLSE-NEXT: .LBB9_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr x23, [x20] +; SOFTFP-NOLSE-NEXT: cmp x23, x8 +; SOFTFP-NOLSE-NEXT: b.ne .LBB9_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, x9, [x20] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB9_3 +; SOFTFP-NOLSE-NEXT: b .LBB9_1 +; SOFTFP-NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldr x30, [sp], #64 // 8-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmax ptr %ptr, <2 x float> %value seq_cst, align 8 + ret <2 x float> %res +} + +define <2 x double> @test_atomicrmw_fmax_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmax_v2f64_seq_cst_align8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr q1, [x0] +; NOLSE-NEXT: b .LBB10_2 +; NOLSE-NEXT: .LBB10_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 +; NOLSE-NEXT: fmov d1, x12 +; NOLSE-NEXT: cmp x13, x9 +; NOLSE-NEXT: ccmp x12, x11, #0, eq +; NOLSE-NEXT: mov v1.d[1], x13 +; NOLSE-NEXT: b.eq .LBB10_6 +; NOLSE-NEXT: .LBB10_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; NOLSE-NEXT: fmaxnm v2.2d, v1.2d, v0.2d +; NOLSE-NEXT: mov x9, v1.d[1] +; NOLSE-NEXT: fmov x11, d1 +; NOLSE-NEXT: mov x8, v2.d[1] +; NOLSE-NEXT: fmov x10, d2 +; NOLSE-NEXT: .LBB10_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxp x12, x13, [x0] +; NOLSE-NEXT: cmp x12, x11 +; NOLSE-NEXT: cset w14, ne +; NOLSE-NEXT: cmp x13, x9 +; NOLSE-NEXT: cinc w14, w14, ne +; NOLSE-NEXT: cbz w14, .LBB10_5 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x12, x13, [x0] +; NOLSE-NEXT: cbnz w14, .LBB10_3 +; NOLSE-NEXT: b .LBB10_1 +; NOLSE-NEXT: .LBB10_5: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x10, x8, [x0] +; NOLSE-NEXT: cbnz w14, .LBB10_3 +; NOLSE-NEXT: b .LBB10_1 +; NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; NOLSE-NEXT: mov v0.16b, v1.16b +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmax_v2f64_seq_cst_align8: +; LSE: // %bb.0: +; LSE-NEXT: ldr q1, [x0] +; LSE-NEXT: .LBB10_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fmaxnm v2.2d, v1.2d, v0.2d +; LSE-NEXT: mov x3, v1.d[1] +; LSE-NEXT: fmov x2, d1 +; LSE-NEXT: mov x7, x3 +; LSE-NEXT: mov x5, v2.d[1] +; LSE-NEXT: mov x6, x2 +; LSE-NEXT: fmov x4, d2 +; LSE-NEXT: caspal x6, x7, x4, x5, [x0] +; LSE-NEXT: fmov d1, x6 +; LSE-NEXT: cmp x7, x3 +; LSE-NEXT: ccmp x6, x2, #0, eq +; LSE-NEXT: mov v1.d[1], x7 +; LSE-NEXT: b.ne .LBB10_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: mov v0.16b, v1.16b +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmax_v2f64_seq_cst_align8: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: str x30, [sp, #-64]! // 8-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: mov x19, x3 +; SOFTFP-NOLSE-NEXT: ldp x0, x1, [x0] +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x21, x2 +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: b .LBB10_2 +; SOFTFP-NOLSE-NEXT: .LBB10_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: ccmp x0, x23, #0, eq +; SOFTFP-NOLSE-NEXT: b.eq .LBB10_6 +; SOFTFP-NOLSE-NEXT: .LBB10_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov x22, x1 +; SOFTFP-NOLSE-NEXT: mov x23, x0 +; SOFTFP-NOLSE-NEXT: mov x0, x1 +; SOFTFP-NOLSE-NEXT: mov x1, x19 +; SOFTFP-NOLSE-NEXT: bl fmax +; SOFTFP-NOLSE-NEXT: mov x24, x0 +; SOFTFP-NOLSE-NEXT: mov x0, x23 +; SOFTFP-NOLSE-NEXT: mov x1, x21 +; SOFTFP-NOLSE-NEXT: bl fmax +; SOFTFP-NOLSE-NEXT: mov x8, x0 +; SOFTFP-NOLSE-NEXT: .LBB10_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxp x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cmp x0, x23 +; SOFTFP-NOLSE-NEXT: cset w9, ne +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: cinc w9, w9, ne +; SOFTFP-NOLSE-NEXT: cbz w9, .LBB10_5 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w9, x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB10_3 +; SOFTFP-NOLSE-NEXT: b .LBB10_1 +; SOFTFP-NOLSE-NEXT: .LBB10_5: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w9, x8, x24, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB10_3 +; SOFTFP-NOLSE-NEXT: b .LBB10_1 +; SOFTFP-NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldr x30, [sp], #64 // 8-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmax ptr %ptr, <2 x double> %value seq_cst, align 16 + ret <2 x double> %res +} + +attributes #0 = { nounwind } diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll new file mode 100644 index 000000000000..2697dbf5b219 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll @@ -0,0 +1,1272 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=aarch64-linux-gnu -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=NOLSE %s +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+lse -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=LSE %s +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=-lse,-fp-armv8 -O1 < %s | FileCheck -check-prefix=SOFTFP-NOLSE %s + +; FIXME: Windows hosts assigns stack slots to different offsets for some reason. +; UNSUPPORTED: system-windows + +define half @test_atomicrmw_fmin_f16_seq_cst_align2(ptr %ptr, half %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align2: +; NOLSE: // %bb.0: +; NOLSE-NEXT: fcvt s1, h0 +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: b .LBB0_2 +; NOLSE-NEXT: .LBB0_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.eq .LBB0_5 +; NOLSE-NEXT: .LBB0_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB0_3 Depth 2 +; NOLSE-NEXT: fcvt s2, h0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fminnm s2, s2, s1 +; NOLSE-NEXT: fcvt h2, s2 +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB0_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB0_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w10, [x0] +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.ne .LBB0_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB0_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB0_3 +; NOLSE-NEXT: b .LBB0_1 +; NOLSE-NEXT: .LBB0_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align2: +; LSE: // %bb.0: +; LSE-NEXT: fcvt s1, h0 +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: .LBB0_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fcvt s2, h0 +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fminnm s2, s2, s1 +; LSE-NEXT: fcvt h2, s2 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casalh w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8, uxth +; LSE-NEXT: b.ne .LBB0_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align2: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: b .LBB0_2 +; SOFTFP-NOLSE-NEXT: .LBB0_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w23 +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB0_5 +; SOFTFP-NOLSE-NEXT: .LBB0_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB0_3 Depth 2 +; SOFTFP-NOLSE-NEXT: and w0, w21, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: and w23, w20, #0xffff +; SOFTFP-NOLSE-NEXT: mov w22, w0 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: bl fminf +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: .LBB0_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB0_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB0_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB0_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB0_3 +; SOFTFP-NOLSE-NEXT: b .LBB0_1 +; SOFTFP-NOLSE-NEXT: .LBB0_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2 + ret half %res +} + +define half @test_atomicrmw_fmin_f16_seq_cst_align4(ptr %ptr, half %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: fcvt s1, h0 +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: b .LBB1_2 +; NOLSE-NEXT: .LBB1_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.eq .LBB1_5 +; NOLSE-NEXT: .LBB1_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB1_3 Depth 2 +; NOLSE-NEXT: fcvt s2, h0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fminnm s2, s2, s1 +; NOLSE-NEXT: fcvt h2, s2 +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB1_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB1_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w10, [x0] +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.ne .LBB1_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB1_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB1_3 +; NOLSE-NEXT: b .LBB1_1 +; NOLSE-NEXT: .LBB1_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: fcvt s1, h0 +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: .LBB1_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fcvt s2, h0 +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fminnm s2, s2, s1 +; LSE-NEXT: fcvt h2, s2 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casalh w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8, uxth +; LSE-NEXT: b.ne .LBB1_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: b .LBB1_2 +; SOFTFP-NOLSE-NEXT: .LBB1_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w23 +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB1_5 +; SOFTFP-NOLSE-NEXT: .LBB1_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB1_3 Depth 2 +; SOFTFP-NOLSE-NEXT: and w0, w21, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: and w23, w20, #0xffff +; SOFTFP-NOLSE-NEXT: mov w22, w0 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: bl fminf +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: .LBB1_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB1_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB1_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB1_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB1_3 +; SOFTFP-NOLSE-NEXT: b .LBB1_1 +; SOFTFP-NOLSE-NEXT: .LBB1_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmin ptr %ptr, half %value seq_cst, align 4 + ret half %res +} + +define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align2(ptr %ptr, bfloat %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmin_bf16_seq_cst_align2: +; NOLSE: // %bb.0: +; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: mov w8, #32767 // =0x7fff +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s1, w9 +; NOLSE-NEXT: b .LBB2_2 +; NOLSE-NEXT: .LBB2_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB2_2 Depth=1 +; NOLSE-NEXT: fmov s0, w11 +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.eq .LBB2_5 +; NOLSE-NEXT: .LBB2_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB2_3 Depth 2 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fminnm s2, s2, s1 +; NOLSE-NEXT: fmov w9, s2 +; NOLSE-NEXT: ubfx w10, w9, #16, #1 +; NOLSE-NEXT: add w9, w9, w8 +; NOLSE-NEXT: add w9, w10, w9 +; NOLSE-NEXT: lsr w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fmov w10, s2 +; NOLSE-NEXT: .LBB2_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB2_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w11, [x0] +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.ne .LBB2_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB2_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w10, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB2_3 +; NOLSE-NEXT: b .LBB2_1 +; NOLSE-NEXT: .LBB2_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmin_bf16_seq_cst_align2: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: mov w8, #32767 // =0x7fff +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s1, w9 +; LSE-NEXT: .LBB2_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s2, w9 +; LSE-NEXT: fminnm s2, s2, s1 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: ubfx w10, w9, #16, #1 +; LSE-NEXT: add w9, w9, w8 +; LSE-NEXT: add w9, w10, w9 +; LSE-NEXT: fmov w10, s0 +; LSE-NEXT: lsr w9, w9, #16 +; LSE-NEXT: mov w11, w10 +; LSE-NEXT: casalh w11, w9, [x0] +; LSE-NEXT: fmov s0, w11 +; LSE-NEXT: cmp w11, w10, uxth +; LSE-NEXT: b.ne .LBB2_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmin_bf16_seq_cst_align2: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: lsl w21, w1, #16 +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: b .LBB2_2 +; SOFTFP-NOLSE-NEXT: .LBB2_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB2_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB2_5 +; SOFTFP-NOLSE-NEXT: .LBB2_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB2_3 Depth 2 +; SOFTFP-NOLSE-NEXT: lsl w0, w20, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl fminf +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: .LBB2_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB2_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB2_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB2_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB2_3 +; SOFTFP-NOLSE-NEXT: b .LBB2_1 +; SOFTFP-NOLSE-NEXT: .LBB2_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 2 + ret bfloat %res +} + +define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align4(ptr %ptr, bfloat %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmin_bf16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: mov w8, #32767 // =0x7fff +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s1, w9 +; NOLSE-NEXT: b .LBB3_2 +; NOLSE-NEXT: .LBB3_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB3_2 Depth=1 +; NOLSE-NEXT: fmov s0, w11 +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.eq .LBB3_5 +; NOLSE-NEXT: .LBB3_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB3_3 Depth 2 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fminnm s2, s2, s1 +; NOLSE-NEXT: fmov w9, s2 +; NOLSE-NEXT: ubfx w10, w9, #16, #1 +; NOLSE-NEXT: add w9, w9, w8 +; NOLSE-NEXT: add w9, w10, w9 +; NOLSE-NEXT: lsr w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fmov w10, s2 +; NOLSE-NEXT: .LBB3_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB3_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w11, [x0] +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.ne .LBB3_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB3_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w10, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB3_3 +; NOLSE-NEXT: b .LBB3_1 +; NOLSE-NEXT: .LBB3_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmin_bf16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: mov w8, #32767 // =0x7fff +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s1, w9 +; LSE-NEXT: .LBB3_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s2, w9 +; LSE-NEXT: fminnm s2, s2, s1 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: ubfx w10, w9, #16, #1 +; LSE-NEXT: add w9, w9, w8 +; LSE-NEXT: add w9, w10, w9 +; LSE-NEXT: fmov w10, s0 +; LSE-NEXT: lsr w9, w9, #16 +; LSE-NEXT: mov w11, w10 +; LSE-NEXT: casalh w11, w9, [x0] +; LSE-NEXT: fmov s0, w11 +; LSE-NEXT: cmp w11, w10, uxth +; LSE-NEXT: b.ne .LBB3_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmin_bf16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: lsl w21, w1, #16 +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: b .LBB3_2 +; SOFTFP-NOLSE-NEXT: .LBB3_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB3_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB3_5 +; SOFTFP-NOLSE-NEXT: .LBB3_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB3_3 Depth 2 +; SOFTFP-NOLSE-NEXT: lsl w0, w20, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl fminf +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: .LBB3_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB3_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB3_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB3_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB3_3 +; SOFTFP-NOLSE-NEXT: b .LBB3_1 +; SOFTFP-NOLSE-NEXT: .LBB3_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 4 + ret bfloat %res +} + +define float @test_atomicrmw_fmin_f32_seq_cst_align4(ptr %ptr, float %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr s1, [x0] +; NOLSE-NEXT: b .LBB4_2 +; NOLSE-NEXT: .LBB4_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=1 +; NOLSE-NEXT: fmov s1, w10 +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.eq .LBB4_5 +; NOLSE-NEXT: .LBB4_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB4_3 Depth 2 +; NOLSE-NEXT: fminnm s2, s1, s0 +; NOLSE-NEXT: fmov w9, s1 +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB4_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB4_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr w10, [x0] +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.ne .LBB4_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB4_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB4_3 +; NOLSE-NEXT: b .LBB4_1 +; NOLSE-NEXT: .LBB4_5: // %atomicrmw.end +; NOLSE-NEXT: fmov s0, s1 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: ldr s1, [x0] +; LSE-NEXT: .LBB4_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fminnm s2, s1, s0 +; LSE-NEXT: fmov w8, s1 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casal w10, w9, [x0] +; LSE-NEXT: fmov s1, w10 +; LSE-NEXT: cmp w10, w8 +; LSE-NEXT: b.ne .LBB4_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: fmov s0, s1 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldr w20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: b .LBB4_2 +; SOFTFP-NOLSE-NEXT: .LBB4_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w20 +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB4_5 +; SOFTFP-NOLSE-NEXT: .LBB4_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB4_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl fminf +; SOFTFP-NOLSE-NEXT: .LBB4_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB4_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20 +; SOFTFP-NOLSE-NEXT: b.ne .LBB4_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB4_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB4_3 +; SOFTFP-NOLSE-NEXT: b .LBB4_1 +; SOFTFP-NOLSE-NEXT: .LBB4_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4 + ret float %res +} + +define double @test_atomicrmw_fmin_f32_seq_cst_align8(ptr %ptr, double %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr d1, [x0] +; NOLSE-NEXT: b .LBB5_2 +; NOLSE-NEXT: .LBB5_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB5_2 Depth=1 +; NOLSE-NEXT: fmov d1, x10 +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.eq .LBB5_5 +; NOLSE-NEXT: .LBB5_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB5_3 Depth 2 +; NOLSE-NEXT: fminnm d2, d1, d0 +; NOLSE-NEXT: fmov x9, d1 +; NOLSE-NEXT: fmov x8, d2 +; NOLSE-NEXT: .LBB5_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB5_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr x10, [x0] +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.ne .LBB5_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB5_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, x8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB5_3 +; NOLSE-NEXT: b .LBB5_1 +; NOLSE-NEXT: .LBB5_5: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, d1 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align8: +; LSE: // %bb.0: +; LSE-NEXT: ldr d1, [x0] +; LSE-NEXT: .LBB5_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fminnm d2, d1, d0 +; LSE-NEXT: fmov x8, d1 +; LSE-NEXT: mov x10, x8 +; LSE-NEXT: fmov x9, d2 +; LSE-NEXT: casal x10, x9, [x0] +; LSE-NEXT: fmov d1, x10 +; LSE-NEXT: cmp x10, x8 +; LSE-NEXT: b.ne .LBB5_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: fmov d0, d1 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align8: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldr x20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: mov x21, x1 +; SOFTFP-NOLSE-NEXT: b .LBB5_2 +; SOFTFP-NOLSE-NEXT: .LBB5_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB5_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp x8, x20 +; SOFTFP-NOLSE-NEXT: mov x20, x8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB5_5 +; SOFTFP-NOLSE-NEXT: .LBB5_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB5_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov x0, x20 +; SOFTFP-NOLSE-NEXT: mov x1, x21 +; SOFTFP-NOLSE-NEXT: bl fmin +; SOFTFP-NOLSE-NEXT: .LBB5_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB5_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr x8, [x19] +; SOFTFP-NOLSE-NEXT: cmp x8, x20 +; SOFTFP-NOLSE-NEXT: b.ne .LBB5_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB5_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, x0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB5_3 +; SOFTFP-NOLSE-NEXT: b .LBB5_1 +; SOFTFP-NOLSE-NEXT: .LBB5_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov x0, x20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8 + ret double %res +} + +define fp128 @test_atomicrmw_fmin_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmin_fp128_seq_cst_align16: +; NOLSE: // %bb.0: +; NOLSE-NEXT: sub sp, sp, #96 +; NOLSE-NEXT: ldr q1, [x0] +; NOLSE-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill +; NOLSE-NEXT: mov x19, x0 +; NOLSE-NEXT: str q0, [sp] // 16-byte Folded Spill +; NOLSE-NEXT: b .LBB6_2 +; NOLSE-NEXT: .LBB6_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 +; NOLSE-NEXT: stp x12, x13, [sp, #32] +; NOLSE-NEXT: cmp x13, x10 +; NOLSE-NEXT: ldr q1, [sp, #32] +; NOLSE-NEXT: ccmp x12, x11, #0, eq +; NOLSE-NEXT: b.eq .LBB6_6 +; NOLSE-NEXT: .LBB6_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB6_3 Depth 2 +; NOLSE-NEXT: mov v0.16b, v1.16b +; NOLSE-NEXT: str q1, [sp, #16] // 16-byte Folded Spill +; NOLSE-NEXT: ldr q1, [sp] // 16-byte Folded Reload +; NOLSE-NEXT: bl fminl +; NOLSE-NEXT: str q0, [sp, #48] +; NOLSE-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload +; NOLSE-NEXT: ldp x9, x8, [sp, #48] +; NOLSE-NEXT: str q0, [sp, #64] +; NOLSE-NEXT: ldp x11, x10, [sp, #64] +; NOLSE-NEXT: .LBB6_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxp x12, x13, [x19] +; NOLSE-NEXT: cmp x12, x11 +; NOLSE-NEXT: cset w14, ne +; NOLSE-NEXT: cmp x13, x10 +; NOLSE-NEXT: cinc w14, w14, ne +; NOLSE-NEXT: cbz w14, .LBB6_5 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x12, x13, [x19] +; NOLSE-NEXT: cbnz w14, .LBB6_3 +; NOLSE-NEXT: b .LBB6_1 +; NOLSE-NEXT: .LBB6_5: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x9, x8, [x19] +; NOLSE-NEXT: cbnz w14, .LBB6_3 +; NOLSE-NEXT: b .LBB6_1 +; NOLSE-NEXT: .LBB6_6: // %atomicrmw.end +; NOLSE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload +; NOLSE-NEXT: mov v0.16b, v1.16b +; NOLSE-NEXT: add sp, sp, #96 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmin_fp128_seq_cst_align16: +; LSE: // %bb.0: +; LSE-NEXT: sub sp, sp, #96 +; LSE-NEXT: ldr q1, [x0] +; LSE-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill +; LSE-NEXT: mov x19, x0 +; LSE-NEXT: str q0, [sp] // 16-byte Folded Spill +; LSE-NEXT: .LBB6_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: mov v0.16b, v1.16b +; LSE-NEXT: str q1, [sp, #16] // 16-byte Folded Spill +; LSE-NEXT: ldr q1, [sp] // 16-byte Folded Reload +; LSE-NEXT: bl fminl +; LSE-NEXT: str q0, [sp, #48] +; LSE-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload +; LSE-NEXT: ldp x0, x1, [sp, #48] +; LSE-NEXT: str q0, [sp, #64] +; LSE-NEXT: ldp x2, x3, [sp, #64] +; LSE-NEXT: mov x4, x2 +; LSE-NEXT: mov x5, x3 +; LSE-NEXT: caspal x4, x5, x0, x1, [x19] +; LSE-NEXT: stp x4, x5, [sp, #32] +; LSE-NEXT: cmp x5, x3 +; LSE-NEXT: ldr q1, [sp, #32] +; LSE-NEXT: ccmp x4, x2, #0, eq +; LSE-NEXT: b.ne .LBB6_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload +; LSE-NEXT: mov v0.16b, v1.16b +; LSE-NEXT: add sp, sp, #96 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmin_fp128_seq_cst_align16: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: mov x19, x3 +; SOFTFP-NOLSE-NEXT: ldp x0, x1, [x0] +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x21, x2 +; SOFTFP-NOLSE-NEXT: b .LBB6_2 +; SOFTFP-NOLSE-NEXT: .LBB6_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: ccmp x0, x23, #0, eq +; SOFTFP-NOLSE-NEXT: b.eq .LBB6_6 +; SOFTFP-NOLSE-NEXT: .LBB6_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB6_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov x2, x21 +; SOFTFP-NOLSE-NEXT: mov x3, x19 +; SOFTFP-NOLSE-NEXT: mov x22, x1 +; SOFTFP-NOLSE-NEXT: mov x23, x0 +; SOFTFP-NOLSE-NEXT: bl fminl +; SOFTFP-NOLSE-NEXT: mov x8, x0 +; SOFTFP-NOLSE-NEXT: mov x9, x1 +; SOFTFP-NOLSE-NEXT: .LBB6_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxp x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cmp x0, x23 +; SOFTFP-NOLSE-NEXT: cset w10, ne +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: cinc w10, w10, ne +; SOFTFP-NOLSE-NEXT: cbz w10, .LBB6_5 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w10, x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w10, .LBB6_3 +; SOFTFP-NOLSE-NEXT: b .LBB6_1 +; SOFTFP-NOLSE-NEXT: .LBB6_5: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w10, x8, x9, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w10, .LBB6_3 +; SOFTFP-NOLSE-NEXT: b .LBB6_1 +; SOFTFP-NOLSE-NEXT: .LBB6_6: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmin ptr %ptr, fp128 %value seq_cst, align 16 + ret fp128 %res +} + +define <2 x half> @test_atomicrmw_fmin_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmin_v2f16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: // kill: def $d0 killed $d0 def $q0 +; NOLSE-NEXT: mov h1, v0.h[1] +; NOLSE-NEXT: fcvt s2, h0 +; NOLSE-NEXT: ldr s0, [x0] +; NOLSE-NEXT: fcvt s1, h1 +; NOLSE-NEXT: b .LBB7_2 +; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.eq .LBB7_5 +; NOLSE-NEXT: .LBB7_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB7_3 Depth 2 +; NOLSE-NEXT: mov h3, v0.h[1] +; NOLSE-NEXT: fcvt s4, h0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fcvt s3, h3 +; NOLSE-NEXT: fminnm s4, s4, s2 +; NOLSE-NEXT: fminnm s3, s3, s1 +; NOLSE-NEXT: fcvt h4, s4 +; NOLSE-NEXT: fcvt h3, s3 +; NOLSE-NEXT: mov v4.h[1], v3.h[0] +; NOLSE-NEXT: fmov w8, s4 +; NOLSE-NEXT: .LBB7_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr w10, [x0] +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.ne .LBB7_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB7_3 +; NOLSE-NEXT: b .LBB7_1 +; NOLSE-NEXT: .LBB7_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmin_v2f16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $d0 killed $d0 def $q0 +; LSE-NEXT: mov h1, v0.h[1] +; LSE-NEXT: fcvt s2, h0 +; LSE-NEXT: ldr s0, [x0] +; LSE-NEXT: fcvt s1, h1 +; LSE-NEXT: .LBB7_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: mov h3, v0.h[1] +; LSE-NEXT: fcvt s4, h0 +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fcvt s3, h3 +; LSE-NEXT: fminnm s4, s4, s2 +; LSE-NEXT: fminnm s3, s3, s1 +; LSE-NEXT: fcvt h4, s4 +; LSE-NEXT: fcvt h3, s3 +; LSE-NEXT: mov v4.h[1], v3.h[0] +; LSE-NEXT: fmov w9, s4 +; LSE-NEXT: casal w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8 +; LSE-NEXT: b.ne .LBB7_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmin_v2f16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x25, [sp, #-64]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w23, [x0, #2] +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w21, [x0] +; SOFTFP-NOLSE-NEXT: mov w22, w1 +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w19, w2 +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: b .LBB7_2 +; SOFTFP-NOLSE-NEXT: .LBB7_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 +; SOFTFP-NOLSE-NEXT: lsr w23, w8, #16 +; SOFTFP-NOLSE-NEXT: cmp w8, w21 +; SOFTFP-NOLSE-NEXT: mov w21, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB7_5 +; SOFTFP-NOLSE-NEXT: .LBB7_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB7_3 Depth 2 +; SOFTFP-NOLSE-NEXT: and w0, w19, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: and w0, w23, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w24 +; SOFTFP-NOLSE-NEXT: bl fminf +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: and w0, w22, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w25, w0 +; SOFTFP-NOLSE-NEXT: and w0, w21, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w25 +; SOFTFP-NOLSE-NEXT: bl fminf +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: bfi w21, w23, #16, #16 +; SOFTFP-NOLSE-NEXT: bfi w0, w24, #16, #16 +; SOFTFP-NOLSE-NEXT: .LBB7_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr w8, [x20] +; SOFTFP-NOLSE-NEXT: cmp w8, w21 +; SOFTFP-NOLSE-NEXT: b.ne .LBB7_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x20] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB7_3 +; SOFTFP-NOLSE-NEXT: b .LBB7_1 +; SOFTFP-NOLSE-NEXT: .LBB7_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w21 +; SOFTFP-NOLSE-NEXT: mov w1, w23 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x25, [sp], #64 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmin ptr %ptr, <2 x half> %value seq_cst, align 4 + ret <2 x half> %res +} + +define <2 x bfloat> @test_atomicrmw_fmin_v2bf16_seq_cst_align4(ptr %ptr, <2 x bfloat> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmin_v2bf16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: // kill: def $d0 killed $d0 def $q0 +; NOLSE-NEXT: mov h1, v0.h[1] +; NOLSE-NEXT: fmov w10, s0 +; NOLSE-NEXT: mov w8, #32767 // =0x7fff +; NOLSE-NEXT: ldr s0, [x0] +; NOLSE-NEXT: lsl w10, w10, #16 +; NOLSE-NEXT: fmov w9, s1 +; NOLSE-NEXT: fmov s2, w10 +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s1, w9 +; NOLSE-NEXT: b .LBB8_2 +; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 +; NOLSE-NEXT: fmov s0, w11 +; NOLSE-NEXT: cmp w11, w9 +; NOLSE-NEXT: b.eq .LBB8_5 +; NOLSE-NEXT: .LBB8_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB8_3 Depth 2 +; NOLSE-NEXT: mov h3, v0.h[1] +; NOLSE-NEXT: fmov w10, s0 +; NOLSE-NEXT: lsl w10, w10, #16 +; NOLSE-NEXT: fmov w9, s3 +; NOLSE-NEXT: fmov s4, w10 +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fminnm s4, s4, s2 +; NOLSE-NEXT: fmov s3, w9 +; NOLSE-NEXT: fminnm s3, s3, s1 +; NOLSE-NEXT: fmov w10, s4 +; NOLSE-NEXT: ubfx w12, w10, #16, #1 +; NOLSE-NEXT: add w10, w10, w8 +; NOLSE-NEXT: fmov w9, s3 +; NOLSE-NEXT: add w10, w12, w10 +; NOLSE-NEXT: lsr w10, w10, #16 +; NOLSE-NEXT: ubfx w11, w9, #16, #1 +; NOLSE-NEXT: add w9, w9, w8 +; NOLSE-NEXT: fmov s4, w10 +; NOLSE-NEXT: add w9, w11, w9 +; NOLSE-NEXT: lsr w9, w9, #16 +; NOLSE-NEXT: fmov s3, w9 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: mov v4.h[1], v3.h[0] +; NOLSE-NEXT: fmov w10, s4 +; NOLSE-NEXT: .LBB8_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr w11, [x0] +; NOLSE-NEXT: cmp w11, w9 +; NOLSE-NEXT: b.ne .LBB8_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, w10, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB8_3 +; NOLSE-NEXT: b .LBB8_1 +; NOLSE-NEXT: .LBB8_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmin_v2bf16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $d0 killed $d0 def $q0 +; LSE-NEXT: mov h1, v0.h[1] +; LSE-NEXT: fmov w10, s0 +; LSE-NEXT: mov w8, #32767 // =0x7fff +; LSE-NEXT: ldr s0, [x0] +; LSE-NEXT: lsl w10, w10, #16 +; LSE-NEXT: fmov w9, s1 +; LSE-NEXT: fmov s2, w10 +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s1, w9 +; LSE-NEXT: .LBB8_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: mov h3, v0.h[1] +; LSE-NEXT: fmov w10, s0 +; LSE-NEXT: lsl w10, w10, #16 +; LSE-NEXT: fmov w9, s3 +; LSE-NEXT: fmov s4, w10 +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fminnm s4, s4, s2 +; LSE-NEXT: fmov s3, w9 +; LSE-NEXT: fminnm s3, s3, s1 +; LSE-NEXT: fmov w10, s4 +; LSE-NEXT: ubfx w12, w10, #16, #1 +; LSE-NEXT: add w10, w10, w8 +; LSE-NEXT: fmov w9, s3 +; LSE-NEXT: add w10, w12, w10 +; LSE-NEXT: lsr w10, w10, #16 +; LSE-NEXT: ubfx w11, w9, #16, #1 +; LSE-NEXT: add w9, w9, w8 +; LSE-NEXT: fmov s4, w10 +; LSE-NEXT: add w9, w11, w9 +; LSE-NEXT: lsr w9, w9, #16 +; LSE-NEXT: fmov s3, w9 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: mov v4.h[1], v3.h[0] +; LSE-NEXT: mov w11, w9 +; LSE-NEXT: fmov w10, s4 +; LSE-NEXT: casal w11, w10, [x0] +; LSE-NEXT: fmov s0, w11 +; LSE-NEXT: cmp w11, w9 +; LSE-NEXT: b.ne .LBB8_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmin_v2bf16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: str x30, [sp, #-64]! // 8-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w8, w1 +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w1, [x0, #2] +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w21, [x0] +; SOFTFP-NOLSE-NEXT: lsl w20, w2, #16 +; SOFTFP-NOLSE-NEXT: lsl w22, w8, #16 +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: b .LBB8_2 +; SOFTFP-NOLSE-NEXT: .LBB8_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 +; SOFTFP-NOLSE-NEXT: lsr w1, w21, #16 +; SOFTFP-NOLSE-NEXT: cmp w21, w23 +; SOFTFP-NOLSE-NEXT: b.eq .LBB8_5 +; SOFTFP-NOLSE-NEXT: .LBB8_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB8_3 Depth 2 +; SOFTFP-NOLSE-NEXT: lsl w23, w1, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w20 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: bl fminf +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: lsl w0, w21, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: bl fminf +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: bfxil w23, w21, #0, #16 +; SOFTFP-NOLSE-NEXT: bfi w0, w24, #16, #16 +; SOFTFP-NOLSE-NEXT: .LBB8_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr w21, [x19] +; SOFTFP-NOLSE-NEXT: cmp w21, w23 +; SOFTFP-NOLSE-NEXT: b.ne .LBB8_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB8_3 +; SOFTFP-NOLSE-NEXT: b .LBB8_1 +; SOFTFP-NOLSE-NEXT: .LBB8_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w21 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldr x30, [sp], #64 // 8-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmin ptr %ptr, <2 x bfloat> %value seq_cst, align 4 + ret <2 x bfloat> %res +} + +define <2 x float> @test_atomicrmw_fmin_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmin_v2f32_seq_cst_align8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr d1, [x0] +; NOLSE-NEXT: b .LBB9_2 +; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 +; NOLSE-NEXT: fmov d1, x10 +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.eq .LBB9_5 +; NOLSE-NEXT: .LBB9_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; NOLSE-NEXT: fminnm v2.2s, v1.2s, v0.2s +; NOLSE-NEXT: fmov x9, d1 +; NOLSE-NEXT: fmov x8, d2 +; NOLSE-NEXT: .LBB9_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr x10, [x0] +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.ne .LBB9_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, x8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB9_3 +; NOLSE-NEXT: b .LBB9_1 +; NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, d1 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmin_v2f32_seq_cst_align8: +; LSE: // %bb.0: +; LSE-NEXT: ldr d1, [x0] +; LSE-NEXT: .LBB9_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fminnm v2.2s, v1.2s, v0.2s +; LSE-NEXT: fmov x8, d1 +; LSE-NEXT: mov x10, x8 +; LSE-NEXT: fmov x9, d2 +; LSE-NEXT: casal x10, x9, [x0] +; LSE-NEXT: fmov d1, x10 +; LSE-NEXT: cmp x10, x8 +; LSE-NEXT: b.ne .LBB9_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: fmov d0, d1 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmin_v2f32_seq_cst_align8: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: str x30, [sp, #-64]! // 8-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: ldp w23, w22, [x0] +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w19, w2 +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: b .LBB9_2 +; SOFTFP-NOLSE-NEXT: .LBB9_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 +; SOFTFP-NOLSE-NEXT: lsr x22, x23, #32 +; SOFTFP-NOLSE-NEXT: cmp x23, x8 +; SOFTFP-NOLSE-NEXT: // kill: def $w22 killed $w22 killed $x22 def $x22 +; SOFTFP-NOLSE-NEXT: b.eq .LBB9_5 +; SOFTFP-NOLSE-NEXT: .LBB9_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov w0, w22 +; SOFTFP-NOLSE-NEXT: mov w1, w19 +; SOFTFP-NOLSE-NEXT: bl fminf +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl fminf +; SOFTFP-NOLSE-NEXT: mov w8, w23 +; SOFTFP-NOLSE-NEXT: mov w9, w0 +; SOFTFP-NOLSE-NEXT: orr x9, x9, x24, lsl #32 +; SOFTFP-NOLSE-NEXT: orr x8, x8, x22, lsl #32 +; SOFTFP-NOLSE-NEXT: .LBB9_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr x23, [x20] +; SOFTFP-NOLSE-NEXT: cmp x23, x8 +; SOFTFP-NOLSE-NEXT: b.ne .LBB9_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, x9, [x20] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB9_3 +; SOFTFP-NOLSE-NEXT: b .LBB9_1 +; SOFTFP-NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldr x30, [sp], #64 // 8-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmin ptr %ptr, <2 x float> %value seq_cst, align 8 + ret <2 x float> %res +} + +define <2 x double> @test_atomicrmw_fmin_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fmin_v2f64_seq_cst_align8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr q1, [x0] +; NOLSE-NEXT: b .LBB10_2 +; NOLSE-NEXT: .LBB10_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 +; NOLSE-NEXT: fmov d1, x12 +; NOLSE-NEXT: cmp x13, x9 +; NOLSE-NEXT: ccmp x12, x11, #0, eq +; NOLSE-NEXT: mov v1.d[1], x13 +; NOLSE-NEXT: b.eq .LBB10_6 +; NOLSE-NEXT: .LBB10_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; NOLSE-NEXT: fminnm v2.2d, v1.2d, v0.2d +; NOLSE-NEXT: mov x9, v1.d[1] +; NOLSE-NEXT: fmov x11, d1 +; NOLSE-NEXT: mov x8, v2.d[1] +; NOLSE-NEXT: fmov x10, d2 +; NOLSE-NEXT: .LBB10_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxp x12, x13, [x0] +; NOLSE-NEXT: cmp x12, x11 +; NOLSE-NEXT: cset w14, ne +; NOLSE-NEXT: cmp x13, x9 +; NOLSE-NEXT: cinc w14, w14, ne +; NOLSE-NEXT: cbz w14, .LBB10_5 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x12, x13, [x0] +; NOLSE-NEXT: cbnz w14, .LBB10_3 +; NOLSE-NEXT: b .LBB10_1 +; NOLSE-NEXT: .LBB10_5: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x10, x8, [x0] +; NOLSE-NEXT: cbnz w14, .LBB10_3 +; NOLSE-NEXT: b .LBB10_1 +; NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; NOLSE-NEXT: mov v0.16b, v1.16b +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fmin_v2f64_seq_cst_align8: +; LSE: // %bb.0: +; LSE-NEXT: ldr q1, [x0] +; LSE-NEXT: .LBB10_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fminnm v2.2d, v1.2d, v0.2d +; LSE-NEXT: mov x3, v1.d[1] +; LSE-NEXT: fmov x2, d1 +; LSE-NEXT: mov x7, x3 +; LSE-NEXT: mov x5, v2.d[1] +; LSE-NEXT: mov x6, x2 +; LSE-NEXT: fmov x4, d2 +; LSE-NEXT: caspal x6, x7, x4, x5, [x0] +; LSE-NEXT: fmov d1, x6 +; LSE-NEXT: cmp x7, x3 +; LSE-NEXT: ccmp x6, x2, #0, eq +; LSE-NEXT: mov v1.d[1], x7 +; LSE-NEXT: b.ne .LBB10_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: mov v0.16b, v1.16b +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmin_v2f64_seq_cst_align8: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: str x30, [sp, #-64]! // 8-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: mov x19, x3 +; SOFTFP-NOLSE-NEXT: ldp x0, x1, [x0] +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x21, x2 +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: b .LBB10_2 +; SOFTFP-NOLSE-NEXT: .LBB10_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: ccmp x0, x23, #0, eq +; SOFTFP-NOLSE-NEXT: b.eq .LBB10_6 +; SOFTFP-NOLSE-NEXT: .LBB10_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov x22, x1 +; SOFTFP-NOLSE-NEXT: mov x23, x0 +; SOFTFP-NOLSE-NEXT: mov x0, x1 +; SOFTFP-NOLSE-NEXT: mov x1, x19 +; SOFTFP-NOLSE-NEXT: bl fmin +; SOFTFP-NOLSE-NEXT: mov x24, x0 +; SOFTFP-NOLSE-NEXT: mov x0, x23 +; SOFTFP-NOLSE-NEXT: mov x1, x21 +; SOFTFP-NOLSE-NEXT: bl fmin +; SOFTFP-NOLSE-NEXT: mov x8, x0 +; SOFTFP-NOLSE-NEXT: .LBB10_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxp x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cmp x0, x23 +; SOFTFP-NOLSE-NEXT: cset w9, ne +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: cinc w9, w9, ne +; SOFTFP-NOLSE-NEXT: cbz w9, .LBB10_5 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w9, x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB10_3 +; SOFTFP-NOLSE-NEXT: b .LBB10_1 +; SOFTFP-NOLSE-NEXT: .LBB10_5: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w9, x8, x24, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB10_3 +; SOFTFP-NOLSE-NEXT: b .LBB10_1 +; SOFTFP-NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldr x30, [sp], #64 // 8-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fmin ptr %ptr, <2 x double> %value seq_cst, align 16 + ret <2 x double> %res +} + +attributes #0 = { nounwind } diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll new file mode 100644 index 000000000000..f41ddcb81d5c --- /dev/null +++ b/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll @@ -0,0 +1,1209 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=aarch64-linux-gnu -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=NOLSE %s +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+lse -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=LSE %s +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=-lse,-fp-armv8 -O1 < %s | FileCheck -check-prefix=SOFTFP-NOLSE %s + +define half @test_atomicrmw_fsub_f16_seq_cst_align2(ptr %ptr, half %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align2: +; NOLSE: // %bb.0: +; NOLSE-NEXT: fcvt s1, h0 +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: b .LBB0_2 +; NOLSE-NEXT: .LBB0_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.eq .LBB0_5 +; NOLSE-NEXT: .LBB0_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB0_3 Depth 2 +; NOLSE-NEXT: fcvt s2, h0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fsub s2, s2, s1 +; NOLSE-NEXT: fcvt h2, s2 +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB0_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB0_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w10, [x0] +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.ne .LBB0_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB0_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB0_3 +; NOLSE-NEXT: b .LBB0_1 +; NOLSE-NEXT: .LBB0_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align2: +; LSE: // %bb.0: +; LSE-NEXT: fcvt s1, h0 +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: .LBB0_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fcvt s2, h0 +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fsub s2, s2, s1 +; LSE-NEXT: fcvt h2, s2 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casalh w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8, uxth +; LSE-NEXT: b.ne .LBB0_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align2: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: b .LBB0_2 +; SOFTFP-NOLSE-NEXT: .LBB0_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w23 +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB0_5 +; SOFTFP-NOLSE-NEXT: .LBB0_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB0_3 Depth 2 +; SOFTFP-NOLSE-NEXT: and w0, w21, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: and w23, w20, #0xffff +; SOFTFP-NOLSE-NEXT: mov w22, w0 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: bl __subsf3 +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: .LBB0_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB0_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB0_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB0_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB0_3 +; SOFTFP-NOLSE-NEXT: b .LBB0_1 +; SOFTFP-NOLSE-NEXT: .LBB0_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fsub ptr %ptr, half %value seq_cst, align 2 + ret half %res +} + +define half @test_atomicrmw_fsub_f16_seq_cst_align4(ptr %ptr, half %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: fcvt s1, h0 +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: b .LBB1_2 +; NOLSE-NEXT: .LBB1_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.eq .LBB1_5 +; NOLSE-NEXT: .LBB1_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB1_3 Depth 2 +; NOLSE-NEXT: fcvt s2, h0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fsub s2, s2, s1 +; NOLSE-NEXT: fcvt h2, s2 +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB1_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB1_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w10, [x0] +; NOLSE-NEXT: cmp w10, w9, uxth +; NOLSE-NEXT: b.ne .LBB1_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB1_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB1_3 +; NOLSE-NEXT: b .LBB1_1 +; NOLSE-NEXT: .LBB1_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: fcvt s1, h0 +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: .LBB1_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fcvt s2, h0 +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fsub s2, s2, s1 +; LSE-NEXT: fcvt h2, s2 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casalh w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8, uxth +; LSE-NEXT: b.ne .LBB1_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: b .LBB1_2 +; SOFTFP-NOLSE-NEXT: .LBB1_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w23 +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB1_5 +; SOFTFP-NOLSE-NEXT: .LBB1_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB1_3 Depth 2 +; SOFTFP-NOLSE-NEXT: and w0, w21, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: and w23, w20, #0xffff +; SOFTFP-NOLSE-NEXT: mov w22, w0 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: bl __subsf3 +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: .LBB1_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB1_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB1_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB1_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB1_3 +; SOFTFP-NOLSE-NEXT: b .LBB1_1 +; SOFTFP-NOLSE-NEXT: .LBB1_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fsub ptr %ptr, half %value seq_cst, align 4 + ret half %res +} + +define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align2(ptr %ptr, bfloat %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fsub_bf16_seq_cst_align2: +; NOLSE: // %bb.0: +; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: mov w8, #32767 // =0x7fff +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s1, w9 +; NOLSE-NEXT: b .LBB2_2 +; NOLSE-NEXT: .LBB2_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB2_2 Depth=1 +; NOLSE-NEXT: fmov s0, w11 +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.eq .LBB2_5 +; NOLSE-NEXT: .LBB2_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB2_3 Depth 2 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fsub s2, s2, s1 +; NOLSE-NEXT: fmov w9, s2 +; NOLSE-NEXT: ubfx w10, w9, #16, #1 +; NOLSE-NEXT: add w9, w9, w8 +; NOLSE-NEXT: add w9, w10, w9 +; NOLSE-NEXT: lsr w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fmov w10, s2 +; NOLSE-NEXT: .LBB2_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB2_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w11, [x0] +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.ne .LBB2_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB2_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w10, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB2_3 +; NOLSE-NEXT: b .LBB2_1 +; NOLSE-NEXT: .LBB2_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fsub_bf16_seq_cst_align2: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: mov w8, #32767 // =0x7fff +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s1, w9 +; LSE-NEXT: .LBB2_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s2, w9 +; LSE-NEXT: fsub s2, s2, s1 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: ubfx w10, w9, #16, #1 +; LSE-NEXT: add w9, w9, w8 +; LSE-NEXT: add w9, w10, w9 +; LSE-NEXT: fmov w10, s0 +; LSE-NEXT: lsr w9, w9, #16 +; LSE-NEXT: mov w11, w10 +; LSE-NEXT: casalh w11, w9, [x0] +; LSE-NEXT: fmov s0, w11 +; LSE-NEXT: cmp w11, w10, uxth +; LSE-NEXT: b.ne .LBB2_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fsub_bf16_seq_cst_align2: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: lsl w21, w1, #16 +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: b .LBB2_2 +; SOFTFP-NOLSE-NEXT: .LBB2_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB2_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB2_5 +; SOFTFP-NOLSE-NEXT: .LBB2_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB2_3 Depth 2 +; SOFTFP-NOLSE-NEXT: lsl w0, w20, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl __subsf3 +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: .LBB2_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB2_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB2_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB2_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB2_3 +; SOFTFP-NOLSE-NEXT: b .LBB2_1 +; SOFTFP-NOLSE-NEXT: .LBB2_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 2 + ret bfloat %res +} + +define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align4(ptr %ptr, bfloat %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fsub_bf16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: mov w8, #32767 // =0x7fff +; NOLSE-NEXT: ldr h0, [x0] +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s1, w9 +; NOLSE-NEXT: b .LBB3_2 +; NOLSE-NEXT: .LBB3_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB3_2 Depth=1 +; NOLSE-NEXT: fmov s0, w11 +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.eq .LBB3_5 +; NOLSE-NEXT: .LBB3_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB3_3 Depth 2 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: lsl w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fsub s2, s2, s1 +; NOLSE-NEXT: fmov w9, s2 +; NOLSE-NEXT: ubfx w10, w9, #16, #1 +; NOLSE-NEXT: add w9, w9, w8 +; NOLSE-NEXT: add w9, w10, w9 +; NOLSE-NEXT: lsr w9, w9, #16 +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fmov w10, s2 +; NOLSE-NEXT: .LBB3_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB3_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxrh w11, [x0] +; NOLSE-NEXT: cmp w11, w9, uxth +; NOLSE-NEXT: b.ne .LBB3_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB3_3 Depth=2 +; NOLSE-NEXT: stlxrh wzr, w10, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB3_3 +; NOLSE-NEXT: b .LBB3_1 +; NOLSE-NEXT: .LBB3_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fsub_bf16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $h0 killed $h0 def $s0 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: mov w8, #32767 // =0x7fff +; LSE-NEXT: ldr h0, [x0] +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s1, w9 +; LSE-NEXT: .LBB3_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fmov w9, s0 +; LSE-NEXT: lsl w9, w9, #16 +; LSE-NEXT: fmov s2, w9 +; LSE-NEXT: fsub s2, s2, s1 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: ubfx w10, w9, #16, #1 +; LSE-NEXT: add w9, w9, w8 +; LSE-NEXT: add w9, w10, w9 +; LSE-NEXT: fmov w10, s0 +; LSE-NEXT: lsr w9, w9, #16 +; LSE-NEXT: mov w11, w10 +; LSE-NEXT: casalh w11, w9, [x0] +; LSE-NEXT: fmov s0, w11 +; LSE-NEXT: cmp w11, w10, uxth +; LSE-NEXT: b.ne .LBB3_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $h0 killed $h0 killed $s0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fsub_bf16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w20, [x0] +; SOFTFP-NOLSE-NEXT: lsl w21, w1, #16 +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: b .LBB3_2 +; SOFTFP-NOLSE-NEXT: .LBB3_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB3_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB3_5 +; SOFTFP-NOLSE-NEXT: .LBB3_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB3_3 Depth 2 +; SOFTFP-NOLSE-NEXT: lsl w0, w20, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl __subsf3 +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: .LBB3_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB3_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxrh w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20, uxth +; SOFTFP-NOLSE-NEXT: b.ne .LBB3_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB3_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxrh wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB3_3 +; SOFTFP-NOLSE-NEXT: b .LBB3_1 +; SOFTFP-NOLSE-NEXT: .LBB3_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 4 + ret bfloat %res +} + +define float @test_atomicrmw_fsub_f32_seq_cst_align4(ptr %ptr, float %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr s1, [x0] +; NOLSE-NEXT: b .LBB4_2 +; NOLSE-NEXT: .LBB4_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=1 +; NOLSE-NEXT: fmov s1, w10 +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.eq .LBB4_5 +; NOLSE-NEXT: .LBB4_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB4_3 Depth 2 +; NOLSE-NEXT: fsub s2, s1, s0 +; NOLSE-NEXT: fmov w9, s1 +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB4_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB4_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr w10, [x0] +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.ne .LBB4_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB4_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB4_3 +; NOLSE-NEXT: b .LBB4_1 +; NOLSE-NEXT: .LBB4_5: // %atomicrmw.end +; NOLSE-NEXT: fmov s0, s1 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: ldr s1, [x0] +; LSE-NEXT: .LBB4_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fsub s2, s1, s0 +; LSE-NEXT: fmov w8, s1 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casal w10, w9, [x0] +; LSE-NEXT: fmov s1, w10 +; LSE-NEXT: cmp w10, w8 +; LSE-NEXT: b.ne .LBB4_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: fmov s0, s1 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldr w20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: b .LBB4_2 +; SOFTFP-NOLSE-NEXT: .LBB4_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp w8, w20 +; SOFTFP-NOLSE-NEXT: mov w20, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB4_5 +; SOFTFP-NOLSE-NEXT: .LBB4_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB4_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl __subsf3 +; SOFTFP-NOLSE-NEXT: .LBB4_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB4_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr w8, [x19] +; SOFTFP-NOLSE-NEXT: cmp w8, w20 +; SOFTFP-NOLSE-NEXT: b.ne .LBB4_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB4_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB4_3 +; SOFTFP-NOLSE-NEXT: b .LBB4_1 +; SOFTFP-NOLSE-NEXT: .LBB4_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fsub ptr %ptr, float %value seq_cst, align 4 + ret float %res +} + +define double @test_atomicrmw_fsub_f32_seq_cst_align8(ptr %ptr, double %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr d1, [x0] +; NOLSE-NEXT: b .LBB5_2 +; NOLSE-NEXT: .LBB5_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB5_2 Depth=1 +; NOLSE-NEXT: fmov d1, x10 +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.eq .LBB5_5 +; NOLSE-NEXT: .LBB5_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB5_3 Depth 2 +; NOLSE-NEXT: fsub d2, d1, d0 +; NOLSE-NEXT: fmov x9, d1 +; NOLSE-NEXT: fmov x8, d2 +; NOLSE-NEXT: .LBB5_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB5_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr x10, [x0] +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.ne .LBB5_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB5_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, x8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB5_3 +; NOLSE-NEXT: b .LBB5_1 +; NOLSE-NEXT: .LBB5_5: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, d1 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align8: +; LSE: // %bb.0: +; LSE-NEXT: ldr d1, [x0] +; LSE-NEXT: .LBB5_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fsub d2, d1, d0 +; LSE-NEXT: fmov x8, d1 +; LSE-NEXT: mov x10, x8 +; LSE-NEXT: fmov x9, d2 +; LSE-NEXT: casal x10, x9, [x0] +; LSE-NEXT: fmov d1, x10 +; LSE-NEXT: cmp x10, x8 +; LSE-NEXT: b.ne .LBB5_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: fmov d0, d1 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align8: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldr x20, [x0] +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: mov x21, x1 +; SOFTFP-NOLSE-NEXT: b .LBB5_2 +; SOFTFP-NOLSE-NEXT: .LBB5_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB5_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp x8, x20 +; SOFTFP-NOLSE-NEXT: mov x20, x8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB5_5 +; SOFTFP-NOLSE-NEXT: .LBB5_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB5_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov x0, x20 +; SOFTFP-NOLSE-NEXT: mov x1, x21 +; SOFTFP-NOLSE-NEXT: bl __subdf3 +; SOFTFP-NOLSE-NEXT: .LBB5_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB5_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr x8, [x19] +; SOFTFP-NOLSE-NEXT: cmp x8, x20 +; SOFTFP-NOLSE-NEXT: b.ne .LBB5_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB5_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, x0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB5_3 +; SOFTFP-NOLSE-NEXT: b .LBB5_1 +; SOFTFP-NOLSE-NEXT: .LBB5_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov x0, x20 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fsub ptr %ptr, double %value seq_cst, align 8 + ret double %res +} + +define fp128 @test_atomicrmw_fsub_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fsub_fp128_seq_cst_align16: +; NOLSE: // %bb.0: +; NOLSE-NEXT: sub sp, sp, #96 +; NOLSE-NEXT: ldr q1, [x0] +; NOLSE-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill +; NOLSE-NEXT: mov x19, x0 +; NOLSE-NEXT: str q0, [sp] // 16-byte Folded Spill +; NOLSE-NEXT: b .LBB6_2 +; NOLSE-NEXT: .LBB6_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 +; NOLSE-NEXT: stp x12, x13, [sp, #32] +; NOLSE-NEXT: cmp x13, x10 +; NOLSE-NEXT: ldr q1, [sp, #32] +; NOLSE-NEXT: ccmp x12, x11, #0, eq +; NOLSE-NEXT: b.eq .LBB6_6 +; NOLSE-NEXT: .LBB6_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB6_3 Depth 2 +; NOLSE-NEXT: mov v0.16b, v1.16b +; NOLSE-NEXT: str q1, [sp, #16] // 16-byte Folded Spill +; NOLSE-NEXT: ldr q1, [sp] // 16-byte Folded Reload +; NOLSE-NEXT: bl __subtf3 +; NOLSE-NEXT: str q0, [sp, #48] +; NOLSE-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload +; NOLSE-NEXT: ldp x9, x8, [sp, #48] +; NOLSE-NEXT: str q0, [sp, #64] +; NOLSE-NEXT: ldp x11, x10, [sp, #64] +; NOLSE-NEXT: .LBB6_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxp x12, x13, [x19] +; NOLSE-NEXT: cmp x12, x11 +; NOLSE-NEXT: cset w14, ne +; NOLSE-NEXT: cmp x13, x10 +; NOLSE-NEXT: cinc w14, w14, ne +; NOLSE-NEXT: cbz w14, .LBB6_5 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x12, x13, [x19] +; NOLSE-NEXT: cbnz w14, .LBB6_3 +; NOLSE-NEXT: b .LBB6_1 +; NOLSE-NEXT: .LBB6_5: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x9, x8, [x19] +; NOLSE-NEXT: cbnz w14, .LBB6_3 +; NOLSE-NEXT: b .LBB6_1 +; NOLSE-NEXT: .LBB6_6: // %atomicrmw.end +; NOLSE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload +; NOLSE-NEXT: mov v0.16b, v1.16b +; NOLSE-NEXT: add sp, sp, #96 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fsub_fp128_seq_cst_align16: +; LSE: // %bb.0: +; LSE-NEXT: sub sp, sp, #96 +; LSE-NEXT: ldr q1, [x0] +; LSE-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill +; LSE-NEXT: mov x19, x0 +; LSE-NEXT: str q0, [sp] // 16-byte Folded Spill +; LSE-NEXT: .LBB6_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: mov v0.16b, v1.16b +; LSE-NEXT: str q1, [sp, #16] // 16-byte Folded Spill +; LSE-NEXT: ldr q1, [sp] // 16-byte Folded Reload +; LSE-NEXT: bl __subtf3 +; LSE-NEXT: str q0, [sp, #48] +; LSE-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload +; LSE-NEXT: ldp x0, x1, [sp, #48] +; LSE-NEXT: str q0, [sp, #64] +; LSE-NEXT: ldp x2, x3, [sp, #64] +; LSE-NEXT: mov x4, x2 +; LSE-NEXT: mov x5, x3 +; LSE-NEXT: caspal x4, x5, x0, x1, [x19] +; LSE-NEXT: stp x4, x5, [sp, #32] +; LSE-NEXT: cmp x5, x3 +; LSE-NEXT: ldr q1, [sp, #32] +; LSE-NEXT: ccmp x4, x2, #0, eq +; LSE-NEXT: b.ne .LBB6_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload +; LSE-NEXT: mov v0.16b, v1.16b +; LSE-NEXT: add sp, sp, #96 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fsub_fp128_seq_cst_align16: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: mov x19, x3 +; SOFTFP-NOLSE-NEXT: ldp x0, x1, [x0] +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x21, x2 +; SOFTFP-NOLSE-NEXT: b .LBB6_2 +; SOFTFP-NOLSE-NEXT: .LBB6_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: ccmp x0, x23, #0, eq +; SOFTFP-NOLSE-NEXT: b.eq .LBB6_6 +; SOFTFP-NOLSE-NEXT: .LBB6_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB6_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov x2, x21 +; SOFTFP-NOLSE-NEXT: mov x3, x19 +; SOFTFP-NOLSE-NEXT: mov x22, x1 +; SOFTFP-NOLSE-NEXT: mov x23, x0 +; SOFTFP-NOLSE-NEXT: bl __subtf3 +; SOFTFP-NOLSE-NEXT: mov x8, x0 +; SOFTFP-NOLSE-NEXT: mov x9, x1 +; SOFTFP-NOLSE-NEXT: .LBB6_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxp x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cmp x0, x23 +; SOFTFP-NOLSE-NEXT: cset w10, ne +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: cinc w10, w10, ne +; SOFTFP-NOLSE-NEXT: cbz w10, .LBB6_5 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w10, x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w10, .LBB6_3 +; SOFTFP-NOLSE-NEXT: b .LBB6_1 +; SOFTFP-NOLSE-NEXT: .LBB6_5: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w10, x8, x9, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w10, .LBB6_3 +; SOFTFP-NOLSE-NEXT: b .LBB6_1 +; SOFTFP-NOLSE-NEXT: .LBB6_6: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fsub ptr %ptr, fp128 %value seq_cst, align 16 + ret fp128 %res +} + +define <2 x half> @test_atomicrmw_fsub_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fsub_v2f16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: fcvtl v1.4s, v0.4h +; NOLSE-NEXT: ldr s0, [x0] +; NOLSE-NEXT: b .LBB7_2 +; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.eq .LBB7_5 +; NOLSE-NEXT: .LBB7_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB7_3 Depth 2 +; NOLSE-NEXT: fcvtl v2.4s, v0.4h +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fsub v2.4s, v2.4s, v1.4s +; NOLSE-NEXT: fcvtn v2.4h, v2.4s +; NOLSE-NEXT: fmov w8, s2 +; NOLSE-NEXT: .LBB7_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr w10, [x0] +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.ne .LBB7_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB7_3 +; NOLSE-NEXT: b .LBB7_1 +; NOLSE-NEXT: .LBB7_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fsub_v2f16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: fcvtl v1.4s, v0.4h +; LSE-NEXT: ldr s0, [x0] +; LSE-NEXT: .LBB7_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fcvtl v2.4s, v0.4h +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: fsub v2.4s, v2.4s, v1.4s +; LSE-NEXT: fcvtn v2.4h, v2.4s +; LSE-NEXT: fmov w9, s2 +; LSE-NEXT: casal w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8 +; LSE-NEXT: b.ne .LBB7_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fsub_v2f16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: stp x30, x25, [sp, #-64]! // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w23, [x0, #2] +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w21, [x0] +; SOFTFP-NOLSE-NEXT: mov w22, w1 +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w19, w2 +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: b .LBB7_2 +; SOFTFP-NOLSE-NEXT: .LBB7_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 +; SOFTFP-NOLSE-NEXT: lsr w23, w8, #16 +; SOFTFP-NOLSE-NEXT: cmp w8, w21 +; SOFTFP-NOLSE-NEXT: mov w21, w8 +; SOFTFP-NOLSE-NEXT: b.eq .LBB7_5 +; SOFTFP-NOLSE-NEXT: .LBB7_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB7_3 Depth 2 +; SOFTFP-NOLSE-NEXT: and w0, w19, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: and w0, w23, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w24 +; SOFTFP-NOLSE-NEXT: bl __subsf3 +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: and w0, w22, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w25, w0 +; SOFTFP-NOLSE-NEXT: and w0, w21, #0xffff +; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee +; SOFTFP-NOLSE-NEXT: mov w1, w25 +; SOFTFP-NOLSE-NEXT: bl __subsf3 +; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee +; SOFTFP-NOLSE-NEXT: bfi w21, w23, #16, #16 +; SOFTFP-NOLSE-NEXT: bfi w0, w24, #16, #16 +; SOFTFP-NOLSE-NEXT: .LBB7_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr w8, [x20] +; SOFTFP-NOLSE-NEXT: cmp w8, w21 +; SOFTFP-NOLSE-NEXT: b.ne .LBB7_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x20] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB7_3 +; SOFTFP-NOLSE-NEXT: b .LBB7_1 +; SOFTFP-NOLSE-NEXT: .LBB7_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w21 +; SOFTFP-NOLSE-NEXT: mov w1, w23 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x30, x25, [sp], #64 // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fsub ptr %ptr, <2 x half> %value seq_cst, align 4 + ret <2 x half> %res +} + +define <2 x bfloat> @test_atomicrmw_fsub_v2bf16_seq_cst_align4(ptr %ptr, <2 x bfloat> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fsub_v2bf16_seq_cst_align4: +; NOLSE: // %bb.0: +; NOLSE-NEXT: movi v1.4s, #1 +; NOLSE-NEXT: movi v2.4s, #127, msl #8 +; NOLSE-NEXT: shll v3.4s, v0.4h, #16 +; NOLSE-NEXT: ldr s0, [x0] +; NOLSE-NEXT: b .LBB8_2 +; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 +; NOLSE-NEXT: fmov s0, w10 +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.eq .LBB8_5 +; NOLSE-NEXT: .LBB8_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB8_3 Depth 2 +; NOLSE-NEXT: shll v4.4s, v0.4h, #16 +; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: fsub v4.4s, v4.4s, v3.4s +; NOLSE-NEXT: ushr v5.4s, v4.4s, #16 +; NOLSE-NEXT: and v5.16b, v5.16b, v1.16b +; NOLSE-NEXT: add v4.4s, v5.4s, v4.4s +; NOLSE-NEXT: addhn v4.4h, v4.4s, v2.4s +; NOLSE-NEXT: fmov w8, s4 +; NOLSE-NEXT: .LBB8_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr w10, [x0] +; NOLSE-NEXT: cmp w10, w9 +; NOLSE-NEXT: b.ne .LBB8_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, w8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB8_3 +; NOLSE-NEXT: b .LBB8_1 +; NOLSE-NEXT: .LBB8_5: // %atomicrmw.end +; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fsub_v2bf16_seq_cst_align4: +; LSE: // %bb.0: +; LSE-NEXT: movi v1.4s, #1 +; LSE-NEXT: movi v2.4s, #127, msl #8 +; LSE-NEXT: shll v3.4s, v0.4h, #16 +; LSE-NEXT: ldr s0, [x0] +; LSE-NEXT: .LBB8_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: shll v4.4s, v0.4h, #16 +; LSE-NEXT: fmov w8, s0 +; LSE-NEXT: fsub v4.4s, v4.4s, v3.4s +; LSE-NEXT: mov w10, w8 +; LSE-NEXT: ushr v5.4s, v4.4s, #16 +; LSE-NEXT: and v5.16b, v5.16b, v1.16b +; LSE-NEXT: add v4.4s, v5.4s, v4.4s +; LSE-NEXT: addhn v4.4h, v4.4s, v2.4s +; LSE-NEXT: fmov w9, s4 +; LSE-NEXT: casal w10, w9, [x0] +; LSE-NEXT: fmov s0, w10 +; LSE-NEXT: cmp w10, w8 +; LSE-NEXT: b.ne .LBB8_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fsub_v2bf16_seq_cst_align4: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: str x30, [sp, #-64]! // 8-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w8, w1 +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w1, [x0, #2] +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: ldrh w21, [x0] +; SOFTFP-NOLSE-NEXT: lsl w20, w2, #16 +; SOFTFP-NOLSE-NEXT: lsl w22, w8, #16 +; SOFTFP-NOLSE-NEXT: mov x19, x0 +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: b .LBB8_2 +; SOFTFP-NOLSE-NEXT: .LBB8_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 +; SOFTFP-NOLSE-NEXT: lsr w1, w21, #16 +; SOFTFP-NOLSE-NEXT: cmp w21, w23 +; SOFTFP-NOLSE-NEXT: b.eq .LBB8_5 +; SOFTFP-NOLSE-NEXT: .LBB8_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB8_3 Depth 2 +; SOFTFP-NOLSE-NEXT: lsl w23, w1, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w20 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: bl __subsf3 +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: lsl w0, w21, #16 +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: bl __subsf3 +; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 +; SOFTFP-NOLSE-NEXT: bfxil w23, w21, #0, #16 +; SOFTFP-NOLSE-NEXT: bfi w0, w24, #16, #16 +; SOFTFP-NOLSE-NEXT: .LBB8_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr w21, [x19] +; SOFTFP-NOLSE-NEXT: cmp w21, w23 +; SOFTFP-NOLSE-NEXT: b.ne .LBB8_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x19] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB8_3 +; SOFTFP-NOLSE-NEXT: b .LBB8_1 +; SOFTFP-NOLSE-NEXT: .LBB8_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w21 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldr x30, [sp], #64 // 8-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fsub ptr %ptr, <2 x bfloat> %value seq_cst, align 4 + ret <2 x bfloat> %res +} + +define <2 x float> @test_atomicrmw_fsub_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fsub_v2f32_seq_cst_align8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr d1, [x0] +; NOLSE-NEXT: b .LBB9_2 +; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 +; NOLSE-NEXT: fmov d1, x10 +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.eq .LBB9_5 +; NOLSE-NEXT: .LBB9_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; NOLSE-NEXT: fsub v2.2s, v1.2s, v0.2s +; NOLSE-NEXT: fmov x9, d1 +; NOLSE-NEXT: fmov x8, d2 +; NOLSE-NEXT: .LBB9_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxr x10, [x0] +; NOLSE-NEXT: cmp x10, x9 +; NOLSE-NEXT: b.ne .LBB9_1 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 +; NOLSE-NEXT: stlxr wzr, x8, [x0] +; NOLSE-NEXT: cbnz wzr, .LBB9_3 +; NOLSE-NEXT: b .LBB9_1 +; NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, d1 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fsub_v2f32_seq_cst_align8: +; LSE: // %bb.0: +; LSE-NEXT: ldr d1, [x0] +; LSE-NEXT: .LBB9_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fsub v2.2s, v1.2s, v0.2s +; LSE-NEXT: fmov x8, d1 +; LSE-NEXT: mov x10, x8 +; LSE-NEXT: fmov x9, d2 +; LSE-NEXT: casal x10, x9, [x0] +; LSE-NEXT: fmov d1, x10 +; LSE-NEXT: cmp x10, x8 +; LSE-NEXT: b.ne .LBB9_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: fmov d0, d1 +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fsub_v2f32_seq_cst_align8: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: str x30, [sp, #-64]! // 8-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w21, w1 +; SOFTFP-NOLSE-NEXT: ldp w23, w22, [x0] +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov w19, w2 +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: b .LBB9_2 +; SOFTFP-NOLSE-NEXT: .LBB9_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 +; SOFTFP-NOLSE-NEXT: lsr x22, x23, #32 +; SOFTFP-NOLSE-NEXT: cmp x23, x8 +; SOFTFP-NOLSE-NEXT: // kill: def $w22 killed $w22 killed $x22 def $x22 +; SOFTFP-NOLSE-NEXT: b.eq .LBB9_5 +; SOFTFP-NOLSE-NEXT: .LBB9_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov w0, w22 +; SOFTFP-NOLSE-NEXT: mov w1, w19 +; SOFTFP-NOLSE-NEXT: bl __subsf3 +; SOFTFP-NOLSE-NEXT: mov w24, w0 +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: mov w1, w21 +; SOFTFP-NOLSE-NEXT: bl __subsf3 +; SOFTFP-NOLSE-NEXT: mov w8, w23 +; SOFTFP-NOLSE-NEXT: mov w9, w0 +; SOFTFP-NOLSE-NEXT: orr x9, x9, x24, lsl #32 +; SOFTFP-NOLSE-NEXT: orr x8, x8, x22, lsl #32 +; SOFTFP-NOLSE-NEXT: .LBB9_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxr x23, [x20] +; SOFTFP-NOLSE-NEXT: cmp x23, x8 +; SOFTFP-NOLSE-NEXT: b.ne .LBB9_1 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxr wzr, x9, [x20] +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB9_3 +; SOFTFP-NOLSE-NEXT: b .LBB9_1 +; SOFTFP-NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: mov w0, w23 +; SOFTFP-NOLSE-NEXT: mov w1, w22 +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldr x30, [sp], #64 // 8-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fsub ptr %ptr, <2 x float> %value seq_cst, align 8 + ret <2 x float> %res +} + +define <2 x double> @test_atomicrmw_fsub_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 { +; NOLSE-LABEL: test_atomicrmw_fsub_v2f64_seq_cst_align8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: ldr q1, [x0] +; NOLSE-NEXT: b .LBB10_2 +; NOLSE-NEXT: .LBB10_1: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 +; NOLSE-NEXT: fmov d1, x12 +; NOLSE-NEXT: cmp x13, x9 +; NOLSE-NEXT: ccmp x12, x11, #0, eq +; NOLSE-NEXT: mov v1.d[1], x13 +; NOLSE-NEXT: b.eq .LBB10_6 +; NOLSE-NEXT: .LBB10_2: // %atomicrmw.start +; NOLSE-NEXT: // =>This Loop Header: Depth=1 +; NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; NOLSE-NEXT: fsub v2.2d, v1.2d, v0.2d +; NOLSE-NEXT: mov x9, v1.d[1] +; NOLSE-NEXT: fmov x11, d1 +; NOLSE-NEXT: mov x8, v2.d[1] +; NOLSE-NEXT: fmov x10, d2 +; NOLSE-NEXT: .LBB10_3: // %atomicrmw.start +; NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 +; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; NOLSE-NEXT: ldaxp x12, x13, [x0] +; NOLSE-NEXT: cmp x12, x11 +; NOLSE-NEXT: cset w14, ne +; NOLSE-NEXT: cmp x13, x9 +; NOLSE-NEXT: cinc w14, w14, ne +; NOLSE-NEXT: cbz w14, .LBB10_5 +; NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x12, x13, [x0] +; NOLSE-NEXT: cbnz w14, .LBB10_3 +; NOLSE-NEXT: b .LBB10_1 +; NOLSE-NEXT: .LBB10_5: // %atomicrmw.start +; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; NOLSE-NEXT: stlxp w14, x10, x8, [x0] +; NOLSE-NEXT: cbnz w14, .LBB10_3 +; NOLSE-NEXT: b .LBB10_1 +; NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; NOLSE-NEXT: mov v0.16b, v1.16b +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomicrmw_fsub_v2f64_seq_cst_align8: +; LSE: // %bb.0: +; LSE-NEXT: ldr q1, [x0] +; LSE-NEXT: .LBB10_1: // %atomicrmw.start +; LSE-NEXT: // =>This Inner Loop Header: Depth=1 +; LSE-NEXT: fsub v2.2d, v1.2d, v0.2d +; LSE-NEXT: mov x3, v1.d[1] +; LSE-NEXT: fmov x2, d1 +; LSE-NEXT: mov x7, x3 +; LSE-NEXT: mov x5, v2.d[1] +; LSE-NEXT: mov x6, x2 +; LSE-NEXT: fmov x4, d2 +; LSE-NEXT: caspal x6, x7, x4, x5, [x0] +; LSE-NEXT: fmov d1, x6 +; LSE-NEXT: cmp x7, x3 +; LSE-NEXT: ccmp x6, x2, #0, eq +; LSE-NEXT: mov v1.d[1], x7 +; LSE-NEXT: b.ne .LBB10_1 +; LSE-NEXT: // %bb.2: // %atomicrmw.end +; LSE-NEXT: mov v0.16b, v1.16b +; LSE-NEXT: ret +; +; SOFTFP-NOLSE-LABEL: test_atomicrmw_fsub_v2f64_seq_cst_align8: +; SOFTFP-NOLSE: // %bb.0: +; SOFTFP-NOLSE-NEXT: str x30, [sp, #-64]! // 8-byte Folded Spill +; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x20, x0 +; SOFTFP-NOLSE-NEXT: mov x19, x3 +; SOFTFP-NOLSE-NEXT: ldp x0, x1, [x0] +; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: mov x21, x2 +; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill +; SOFTFP-NOLSE-NEXT: b .LBB10_2 +; SOFTFP-NOLSE-NEXT: .LBB10_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: ccmp x0, x23, #0, eq +; SOFTFP-NOLSE-NEXT: b.eq .LBB10_6 +; SOFTFP-NOLSE-NEXT: .LBB10_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 +; SOFTFP-NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; SOFTFP-NOLSE-NEXT: mov x22, x1 +; SOFTFP-NOLSE-NEXT: mov x23, x0 +; SOFTFP-NOLSE-NEXT: mov x0, x1 +; SOFTFP-NOLSE-NEXT: mov x1, x19 +; SOFTFP-NOLSE-NEXT: bl __subdf3 +; SOFTFP-NOLSE-NEXT: mov x24, x0 +; SOFTFP-NOLSE-NEXT: mov x0, x23 +; SOFTFP-NOLSE-NEXT: mov x1, x21 +; SOFTFP-NOLSE-NEXT: bl __subdf3 +; SOFTFP-NOLSE-NEXT: mov x8, x0 +; SOFTFP-NOLSE-NEXT: .LBB10_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 +; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 +; SOFTFP-NOLSE-NEXT: ldaxp x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cmp x0, x23 +; SOFTFP-NOLSE-NEXT: cset w9, ne +; SOFTFP-NOLSE-NEXT: cmp x1, x22 +; SOFTFP-NOLSE-NEXT: cinc w9, w9, ne +; SOFTFP-NOLSE-NEXT: cbz w9, .LBB10_5 +; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w9, x0, x1, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB10_3 +; SOFTFP-NOLSE-NEXT: b .LBB10_1 +; SOFTFP-NOLSE-NEXT: .LBB10_5: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; SOFTFP-NOLSE-NEXT: stlxp w9, x8, x24, [x20] +; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB10_3 +; SOFTFP-NOLSE-NEXT: b .LBB10_1 +; SOFTFP-NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ldr x30, [sp], #64 // 8-byte Folded Reload +; SOFTFP-NOLSE-NEXT: ret + %res = atomicrmw fsub ptr %ptr, <2 x double> %value seq_cst, align 16 + ret <2 x double> %res +} + +attributes #0 = { nounwind } -- Gitee From 1c8a1158220c3dbd3604c7ada25293824c4eb98c Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Thu, 29 Aug 2024 19:48:51 +0400 Subject: [PATCH 2/3] AArch64: Use consistent atomicrmw expansion for FP operations (#103702) Use LLSC or cmpxchg in the same cases as for the unsupported integer operations. This required some fixups to the LLSC implementatation to deal with the fp128 case. The comment about floating-point exceptions was wrong, because floating-point exceptions are not really exceptions at all. --- .../Target/AArch64/AArch64ISelLowering.cpp | 59 ++- llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll | 330 ++++------------ llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll | 356 +++++------------- llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll | 356 +++++------------- llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll | 330 ++++------------ 5 files changed, 376 insertions(+), 1055 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index ef2789e96213..77a86d03dcec 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -26565,21 +26565,37 @@ AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const { : AtomicExpansionKind::LLSC; } +// Return true if the atomic operation expansion will lower to use a library +// call, and is thus ineligible to use an LLSC expansion. +static bool rmwOpMayLowerToLibcall(const AArch64Subtarget &Subtarget, + const AtomicRMWInst *RMW) { + if (!RMW->isFloatingPointOperation()) + return false; + switch (RMW->getType()->getScalarType()->getTypeID()) { + case Type::FloatTyID: + case Type::DoubleTyID: + case Type::HalfTyID: + case Type::BFloatTyID: + // Will use soft float + return !Subtarget.hasFPARMv8(); + default: + // fp128 will emit library calls. + return true; + } + + llvm_unreachable("covered type switch"); +} + // The "default" for integer RMW operations is to expand to an LL/SC loop. // However, with the LSE instructions (or outline-atomics mode, which provides // library routines in place of the LSE-instructions), we can directly emit many // operations instead. -// -// Floating-point operations are always emitted to a cmpxchg loop, because they -// may trigger a trap which aborts an LLSC sequence. TargetLowering::AtomicExpansionKind AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { - unsigned Size = AI->getType()->getPrimitiveSizeInBits(); + Type *Ty = AI->getType(); + unsigned Size = Ty->getPrimitiveSizeInBits(); assert(Size <= 128 && "AtomicExpandPass should've handled larger sizes."); - if (AI->isFloatingPointOperation()) - return AtomicExpansionKind::CmpXChg; - bool CanUseLSE128 = Subtarget->hasLSE128() && Size == 128 && (AI->getOperation() == AtomicRMWInst::Xchg || AI->getOperation() == AtomicRMWInst::Or || @@ -26589,7 +26605,8 @@ AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { // Nand is not supported in LSE. // Leave 128 bits to LLSC or CmpXChg. - if (AI->getOperation() != AtomicRMWInst::Nand && Size < 128) { + if (AI->getOperation() != AtomicRMWInst::Nand && Size < 128 && + !AI->isFloatingPointOperation()) { if (Subtarget->hasLSE()) return AtomicExpansionKind::None; if (Subtarget->outlineAtomics()) { @@ -26615,7 +26632,7 @@ AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { // succeed. So at -O0 lower this operation to a CAS loop. Also worthwhile if // we have a single CAS instruction that can replace the loop. if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None || - Subtarget->hasLSE()) + Subtarget->hasLSE() || rmwOpMayLowerToLibcall(*Subtarget, AI)) return AtomicExpansionKind::CmpXChg; return AtomicExpansionKind::LLSC; @@ -26662,10 +26679,14 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilderBase &Builder, Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo"); Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi"); - Lo = Builder.CreateZExt(Lo, ValueTy, "lo64"); - Hi = Builder.CreateZExt(Hi, ValueTy, "hi64"); - return Builder.CreateOr( - Lo, Builder.CreateShl(Hi, ConstantInt::get(ValueTy, 64)), "val64"); + + auto *Int128Ty = Type::getInt128Ty(Builder.getContext()); + Lo = Builder.CreateZExt(Lo, Int128Ty, "lo64"); + Hi = Builder.CreateZExt(Hi, Int128Ty, "hi64"); + + Value *Or = Builder.CreateOr( + Lo, Builder.CreateShl(Hi, ConstantInt::get(Int128Ty, 64)), "val64"); + return Builder.CreateBitCast(Or, ValueTy); } Type *Tys[] = { Addr->getType() }; @@ -26676,8 +26697,8 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilderBase &Builder, const DataLayout &DL = M->getDataLayout(); IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(ValueTy)); CallInst *CI = Builder.CreateCall(Ldxr, Addr); - CI->addParamAttr( - 0, Attribute::get(Builder.getContext(), Attribute::ElementType, ValueTy)); + CI->addParamAttr(0, Attribute::get(Builder.getContext(), + Attribute::ElementType, IntEltTy)); Value *Trunc = Builder.CreateTrunc(CI, IntEltTy); return Builder.CreateBitCast(Trunc, ValueTy); @@ -26703,9 +26724,13 @@ Value *AArch64TargetLowering::emitStoreConditional(IRBuilderBase &Builder, IsRelease ? Intrinsic::aarch64_stlxp : Intrinsic::aarch64_stxp; Function *Stxr = Intrinsic::getDeclaration(M, Int); Type *Int64Ty = Type::getInt64Ty(M->getContext()); + Type *Int128Ty = Type::getInt128Ty(M->getContext()); + + Value *CastVal = Builder.CreateBitCast(Val, Int128Ty); - Value *Lo = Builder.CreateTrunc(Val, Int64Ty, "lo"); - Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 64), Int64Ty, "hi"); + Value *Lo = Builder.CreateTrunc(CastVal, Int64Ty, "lo"); + Value *Hi = + Builder.CreateTrunc(Builder.CreateLShr(CastVal, 64), Int64Ty, "hi"); return Builder.CreateCall(Stxr, {Lo, Hi, Addr}); } diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll index 89c9880ffc78..0d230bb9dcc6 100644 --- a/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll +++ b/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll @@ -7,33 +7,17 @@ define half @test_atomicrmw_fadd_f16_seq_cst_align2(ptr %ptr, half %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align2: ; NOLSE: // %bb.0: ; NOLSE-NEXT: fcvt s1, h0 -; NOLSE-NEXT: ldr h0, [x0] -; NOLSE-NEXT: b .LBB0_2 ; NOLSE-NEXT: .LBB0_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.eq .LBB0_5 -; NOLSE-NEXT: .LBB0_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB0_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w8, [x0] +; NOLSE-NEXT: fmov s0, w8 ; NOLSE-NEXT: fcvt s2, h0 -; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: fadd s2, s2, s1 ; NOLSE-NEXT: fcvt h2, s2 ; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB0_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB0_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w10, [x0] -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.ne .LBB0_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB0_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB0_3 -; NOLSE-NEXT: b .LBB0_1 -; NOLSE-NEXT: .LBB0_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w9, w8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB0_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -108,33 +92,17 @@ define half @test_atomicrmw_fadd_f16_seq_cst_align4(ptr %ptr, half %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align4: ; NOLSE: // %bb.0: ; NOLSE-NEXT: fcvt s1, h0 -; NOLSE-NEXT: ldr h0, [x0] -; NOLSE-NEXT: b .LBB1_2 ; NOLSE-NEXT: .LBB1_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.eq .LBB1_5 -; NOLSE-NEXT: .LBB1_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB1_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w8, [x0] +; NOLSE-NEXT: fmov s0, w8 ; NOLSE-NEXT: fcvt s2, h0 -; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: fadd s2, s2, s1 ; NOLSE-NEXT: fcvt h2, s2 ; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB1_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB1_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w10, [x0] -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.ne .LBB1_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB1_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB1_3 -; NOLSE-NEXT: b .LBB1_1 -; NOLSE-NEXT: .LBB1_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w9, w8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB1_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -211,19 +179,12 @@ define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align2(ptr %ptr, bfloat %value) ; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 ; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: mov w8, #32767 // =0x7fff -; NOLSE-NEXT: ldr h0, [x0] ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s1, w9 -; NOLSE-NEXT: b .LBB2_2 ; NOLSE-NEXT: .LBB2_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB2_2 Depth=1 -; NOLSE-NEXT: fmov s0, w11 -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.eq .LBB2_5 -; NOLSE-NEXT: .LBB2_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB2_3 Depth 2 -; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w9, [x0] +; NOLSE-NEXT: fmov s0, w9 ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s2, w9 ; NOLSE-NEXT: fadd s2, s2, s1 @@ -232,21 +193,9 @@ define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align2(ptr %ptr, bfloat %value) ; NOLSE-NEXT: add w9, w9, w8 ; NOLSE-NEXT: add w9, w10, w9 ; NOLSE-NEXT: lsr w9, w9, #16 -; NOLSE-NEXT: fmov s2, w9 -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: fmov w10, s2 -; NOLSE-NEXT: .LBB2_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB2_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w11, [x0] -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.ne .LBB2_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB2_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w10, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB2_3 -; NOLSE-NEXT: b .LBB2_1 -; NOLSE-NEXT: .LBB2_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB2_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -325,19 +274,12 @@ define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align4(ptr %ptr, bfloat %value) ; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 ; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: mov w8, #32767 // =0x7fff -; NOLSE-NEXT: ldr h0, [x0] ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s1, w9 -; NOLSE-NEXT: b .LBB3_2 ; NOLSE-NEXT: .LBB3_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB3_2 Depth=1 -; NOLSE-NEXT: fmov s0, w11 -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.eq .LBB3_5 -; NOLSE-NEXT: .LBB3_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB3_3 Depth 2 -; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w9, [x0] +; NOLSE-NEXT: fmov s0, w9 ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s2, w9 ; NOLSE-NEXT: fadd s2, s2, s1 @@ -346,21 +288,9 @@ define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align4(ptr %ptr, bfloat %value) ; NOLSE-NEXT: add w9, w9, w8 ; NOLSE-NEXT: add w9, w10, w9 ; NOLSE-NEXT: lsr w9, w9, #16 -; NOLSE-NEXT: fmov s2, w9 -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: fmov w10, s2 -; NOLSE-NEXT: .LBB3_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB3_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w11, [x0] -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.ne .LBB3_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB3_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w10, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB3_3 -; NOLSE-NEXT: b .LBB3_1 -; NOLSE-NEXT: .LBB3_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB3_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -436,31 +366,15 @@ define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align4(ptr %ptr, bfloat %value) define float @test_atomicrmw_fadd_f32_seq_cst_align4(ptr %ptr, float %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align4: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr s1, [x0] -; NOLSE-NEXT: b .LBB4_2 ; NOLSE-NEXT: .LBB4_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=1 -; NOLSE-NEXT: fmov s1, w10 -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.eq .LBB4_5 -; NOLSE-NEXT: .LBB4_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB4_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr w8, [x0] +; NOLSE-NEXT: fmov s1, w8 ; NOLSE-NEXT: fadd s2, s1, s0 -; NOLSE-NEXT: fmov w9, s1 ; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB4_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB4_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr w10, [x0] -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.ne .LBB4_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB4_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB4_3 -; NOLSE-NEXT: b .LBB4_1 -; NOLSE-NEXT: .LBB4_5: // %atomicrmw.end +; NOLSE-NEXT: stlxr w9, w8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB4_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov s0, s1 ; NOLSE-NEXT: ret ; @@ -523,31 +437,15 @@ define float @test_atomicrmw_fadd_f32_seq_cst_align4(ptr %ptr, float %value) #0 define double @test_atomicrmw_fadd_f32_seq_cst_align8(ptr %ptr, double %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr d1, [x0] -; NOLSE-NEXT: b .LBB5_2 ; NOLSE-NEXT: .LBB5_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB5_2 Depth=1 -; NOLSE-NEXT: fmov d1, x10 -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.eq .LBB5_5 -; NOLSE-NEXT: .LBB5_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB5_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr x8, [x0] +; NOLSE-NEXT: fmov d1, x8 ; NOLSE-NEXT: fadd d2, d1, d0 -; NOLSE-NEXT: fmov x9, d1 ; NOLSE-NEXT: fmov x8, d2 -; NOLSE-NEXT: .LBB5_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB5_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr x10, [x0] -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.ne .LBB5_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB5_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, x8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB5_3 -; NOLSE-NEXT: b .LBB5_1 -; NOLSE-NEXT: .LBB5_5: // %atomicrmw.end +; NOLSE-NEXT: stlxr w9, x8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB5_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, d1 ; NOLSE-NEXT: ret ; @@ -748,35 +646,19 @@ define fp128 @test_atomicrmw_fadd_fp128_seq_cst_align16(ptr %ptr, fp128 %value) define <2 x half> @test_atomicrmw_fadd_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fadd_v2f16_seq_cst_align4: ; NOLSE: // %bb.0: -; NOLSE-NEXT: fcvtl v1.4s, v0.4h -; NOLSE-NEXT: ldr s0, [x0] -; NOLSE-NEXT: b .LBB7_2 +; NOLSE-NEXT: fcvtl v0.4s, v0.4h ; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.eq .LBB7_5 -; NOLSE-NEXT: .LBB7_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB7_3 Depth 2 -; NOLSE-NEXT: fcvtl v2.4s, v0.4h -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: fadd v2.4s, v2.4s, v1.4s -; NOLSE-NEXT: fcvtn v2.4h, v2.4s -; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB7_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr w10, [x0] -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.ne .LBB7_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB7_3 -; NOLSE-NEXT: b .LBB7_1 -; NOLSE-NEXT: .LBB7_5: // %atomicrmw.end -; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr w8, [x0] +; NOLSE-NEXT: fmov s1, w8 +; NOLSE-NEXT: fcvtl v1.4s, v1.4h +; NOLSE-NEXT: fadd v1.4s, v1.4s, v0.4s +; NOLSE-NEXT: fcvtn v1.4h, v1.4s +; NOLSE-NEXT: fmov w9, s1 +; NOLSE-NEXT: stlxr w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB7_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, x8 ; NOLSE-NEXT: ret ; ; LSE-LABEL: test_atomicrmw_fadd_v2f16_seq_cst_align4: @@ -867,38 +749,22 @@ define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; NOLSE: // %bb.0: ; NOLSE-NEXT: movi v1.4s, #1 ; NOLSE-NEXT: movi v2.4s, #127, msl #8 -; NOLSE-NEXT: shll v3.4s, v0.4h, #16 -; NOLSE-NEXT: ldr s0, [x0] -; NOLSE-NEXT: b .LBB8_2 +; NOLSE-NEXT: shll v0.4s, v0.4h, #16 ; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.eq .LBB8_5 -; NOLSE-NEXT: .LBB8_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB8_3 Depth 2 -; NOLSE-NEXT: shll v4.4s, v0.4h, #16 -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: fadd v4.4s, v4.4s, v3.4s -; NOLSE-NEXT: ushr v5.4s, v4.4s, #16 -; NOLSE-NEXT: and v5.16b, v5.16b, v1.16b -; NOLSE-NEXT: add v4.4s, v5.4s, v4.4s -; NOLSE-NEXT: addhn v4.4h, v4.4s, v2.4s -; NOLSE-NEXT: fmov w8, s4 -; NOLSE-NEXT: .LBB8_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr w10, [x0] -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.ne .LBB8_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB8_3 -; NOLSE-NEXT: b .LBB8_1 -; NOLSE-NEXT: .LBB8_5: // %atomicrmw.end -; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr w8, [x0] +; NOLSE-NEXT: fmov s3, w8 +; NOLSE-NEXT: shll v3.4s, v3.4h, #16 +; NOLSE-NEXT: fadd v3.4s, v3.4s, v0.4s +; NOLSE-NEXT: ushr v4.4s, v3.4s, #16 +; NOLSE-NEXT: and v4.16b, v4.16b, v1.16b +; NOLSE-NEXT: add v3.4s, v4.4s, v3.4s +; NOLSE-NEXT: addhn v3.4h, v3.4s, v2.4s +; NOLSE-NEXT: fmov w9, s3 +; NOLSE-NEXT: stlxr w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB8_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, x8 ; NOLSE-NEXT: ret ; ; LSE-LABEL: test_atomicrmw_fadd_v2bf16_seq_cst_align4: @@ -984,31 +850,15 @@ define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf define <2 x float> @test_atomicrmw_fadd_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fadd_v2f32_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr d1, [x0] -; NOLSE-NEXT: b .LBB9_2 ; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 -; NOLSE-NEXT: fmov d1, x10 -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.eq .LBB9_5 -; NOLSE-NEXT: .LBB9_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr x8, [x0] +; NOLSE-NEXT: fmov d1, x8 ; NOLSE-NEXT: fadd v2.2s, v1.2s, v0.2s -; NOLSE-NEXT: fmov x9, d1 ; NOLSE-NEXT: fmov x8, d2 -; NOLSE-NEXT: .LBB9_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr x10, [x0] -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.ne .LBB9_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, x8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB9_3 -; NOLSE-NEXT: b .LBB9_1 -; NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; NOLSE-NEXT: stlxr w9, x8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB9_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, d1 ; NOLSE-NEXT: ret ; @@ -1086,43 +936,17 @@ define <2 x float> @test_atomicrmw_fadd_v2f32_seq_cst_align8(ptr %ptr, <2 x floa define <2 x double> @test_atomicrmw_fadd_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fadd_v2f64_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr q1, [x0] -; NOLSE-NEXT: b .LBB10_2 ; NOLSE-NEXT: .LBB10_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 -; NOLSE-NEXT: fmov d1, x12 -; NOLSE-NEXT: cmp x13, x9 -; NOLSE-NEXT: ccmp x12, x11, #0, eq -; NOLSE-NEXT: mov v1.d[1], x13 -; NOLSE-NEXT: b.eq .LBB10_6 -; NOLSE-NEXT: .LBB10_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxp x8, x9, [x0] +; NOLSE-NEXT: fmov d1, x8 +; NOLSE-NEXT: mov v1.d[1], x9 ; NOLSE-NEXT: fadd v2.2d, v1.2d, v0.2d -; NOLSE-NEXT: mov x9, v1.d[1] -; NOLSE-NEXT: fmov x11, d1 ; NOLSE-NEXT: mov x8, v2.d[1] -; NOLSE-NEXT: fmov x10, d2 -; NOLSE-NEXT: .LBB10_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxp x12, x13, [x0] -; NOLSE-NEXT: cmp x12, x11 -; NOLSE-NEXT: cset w14, ne -; NOLSE-NEXT: cmp x13, x9 -; NOLSE-NEXT: cinc w14, w14, ne -; NOLSE-NEXT: cbz w14, .LBB10_5 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 -; NOLSE-NEXT: stlxp w14, x12, x13, [x0] -; NOLSE-NEXT: cbnz w14, .LBB10_3 -; NOLSE-NEXT: b .LBB10_1 -; NOLSE-NEXT: .LBB10_5: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 -; NOLSE-NEXT: stlxp w14, x10, x8, [x0] -; NOLSE-NEXT: cbnz w14, .LBB10_3 -; NOLSE-NEXT: b .LBB10_1 -; NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; NOLSE-NEXT: fmov x9, d2 +; NOLSE-NEXT: stlxp w10, x9, x8, [x0] +; NOLSE-NEXT: cbnz w10, .LBB10_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: mov v0.16b, v1.16b ; NOLSE-NEXT: ret ; diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll index 998d8ae0c1de..12a0c1169f2b 100644 --- a/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll +++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll @@ -10,33 +10,17 @@ define half @test_atomicrmw_fmax_f16_seq_cst_align2(ptr %ptr, half %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align2: ; NOLSE: // %bb.0: ; NOLSE-NEXT: fcvt s1, h0 -; NOLSE-NEXT: ldr h0, [x0] -; NOLSE-NEXT: b .LBB0_2 ; NOLSE-NEXT: .LBB0_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.eq .LBB0_5 -; NOLSE-NEXT: .LBB0_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB0_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w8, [x0] +; NOLSE-NEXT: fmov s0, w8 ; NOLSE-NEXT: fcvt s2, h0 -; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: fmaxnm s2, s2, s1 ; NOLSE-NEXT: fcvt h2, s2 ; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB0_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB0_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w10, [x0] -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.ne .LBB0_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB0_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB0_3 -; NOLSE-NEXT: b .LBB0_1 -; NOLSE-NEXT: .LBB0_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w9, w8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB0_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -111,33 +95,17 @@ define half @test_atomicrmw_fmax_f16_seq_cst_align4(ptr %ptr, half %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align4: ; NOLSE: // %bb.0: ; NOLSE-NEXT: fcvt s1, h0 -; NOLSE-NEXT: ldr h0, [x0] -; NOLSE-NEXT: b .LBB1_2 ; NOLSE-NEXT: .LBB1_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.eq .LBB1_5 -; NOLSE-NEXT: .LBB1_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB1_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w8, [x0] +; NOLSE-NEXT: fmov s0, w8 ; NOLSE-NEXT: fcvt s2, h0 -; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: fmaxnm s2, s2, s1 ; NOLSE-NEXT: fcvt h2, s2 ; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB1_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB1_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w10, [x0] -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.ne .LBB1_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB1_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB1_3 -; NOLSE-NEXT: b .LBB1_1 -; NOLSE-NEXT: .LBB1_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w9, w8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB1_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -214,19 +182,12 @@ define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align2(ptr %ptr, bfloat %value) ; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 ; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: mov w8, #32767 // =0x7fff -; NOLSE-NEXT: ldr h0, [x0] ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s1, w9 -; NOLSE-NEXT: b .LBB2_2 ; NOLSE-NEXT: .LBB2_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB2_2 Depth=1 -; NOLSE-NEXT: fmov s0, w11 -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.eq .LBB2_5 -; NOLSE-NEXT: .LBB2_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB2_3 Depth 2 -; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w9, [x0] +; NOLSE-NEXT: fmov s0, w9 ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s2, w9 ; NOLSE-NEXT: fmaxnm s2, s2, s1 @@ -235,21 +196,9 @@ define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align2(ptr %ptr, bfloat %value) ; NOLSE-NEXT: add w9, w9, w8 ; NOLSE-NEXT: add w9, w10, w9 ; NOLSE-NEXT: lsr w9, w9, #16 -; NOLSE-NEXT: fmov s2, w9 -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: fmov w10, s2 -; NOLSE-NEXT: .LBB2_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB2_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w11, [x0] -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.ne .LBB2_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB2_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w10, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB2_3 -; NOLSE-NEXT: b .LBB2_1 -; NOLSE-NEXT: .LBB2_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB2_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -328,19 +277,12 @@ define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align4(ptr %ptr, bfloat %value) ; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 ; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: mov w8, #32767 // =0x7fff -; NOLSE-NEXT: ldr h0, [x0] ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s1, w9 -; NOLSE-NEXT: b .LBB3_2 ; NOLSE-NEXT: .LBB3_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB3_2 Depth=1 -; NOLSE-NEXT: fmov s0, w11 -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.eq .LBB3_5 -; NOLSE-NEXT: .LBB3_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB3_3 Depth 2 -; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w9, [x0] +; NOLSE-NEXT: fmov s0, w9 ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s2, w9 ; NOLSE-NEXT: fmaxnm s2, s2, s1 @@ -349,21 +291,9 @@ define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align4(ptr %ptr, bfloat %value) ; NOLSE-NEXT: add w9, w9, w8 ; NOLSE-NEXT: add w9, w10, w9 ; NOLSE-NEXT: lsr w9, w9, #16 -; NOLSE-NEXT: fmov s2, w9 -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: fmov w10, s2 -; NOLSE-NEXT: .LBB3_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB3_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w11, [x0] -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.ne .LBB3_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB3_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w10, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB3_3 -; NOLSE-NEXT: b .LBB3_1 -; NOLSE-NEXT: .LBB3_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB3_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -439,31 +369,15 @@ define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align4(ptr %ptr, bfloat %value) define float @test_atomicrmw_fmax_f32_seq_cst_align4(ptr %ptr, float %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align4: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr s1, [x0] -; NOLSE-NEXT: b .LBB4_2 ; NOLSE-NEXT: .LBB4_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=1 -; NOLSE-NEXT: fmov s1, w10 -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.eq .LBB4_5 -; NOLSE-NEXT: .LBB4_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB4_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr w8, [x0] +; NOLSE-NEXT: fmov s1, w8 ; NOLSE-NEXT: fmaxnm s2, s1, s0 -; NOLSE-NEXT: fmov w9, s1 ; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB4_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB4_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr w10, [x0] -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.ne .LBB4_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB4_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB4_3 -; NOLSE-NEXT: b .LBB4_1 -; NOLSE-NEXT: .LBB4_5: // %atomicrmw.end +; NOLSE-NEXT: stlxr w9, w8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB4_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov s0, s1 ; NOLSE-NEXT: ret ; @@ -526,31 +440,15 @@ define float @test_atomicrmw_fmax_f32_seq_cst_align4(ptr %ptr, float %value) #0 define double @test_atomicrmw_fmax_f32_seq_cst_align8(ptr %ptr, double %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr d1, [x0] -; NOLSE-NEXT: b .LBB5_2 ; NOLSE-NEXT: .LBB5_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB5_2 Depth=1 -; NOLSE-NEXT: fmov d1, x10 -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.eq .LBB5_5 -; NOLSE-NEXT: .LBB5_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB5_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr x8, [x0] +; NOLSE-NEXT: fmov d1, x8 ; NOLSE-NEXT: fmaxnm d2, d1, d0 -; NOLSE-NEXT: fmov x9, d1 ; NOLSE-NEXT: fmov x8, d2 -; NOLSE-NEXT: .LBB5_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB5_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr x10, [x0] -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.ne .LBB5_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB5_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, x8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB5_3 -; NOLSE-NEXT: b .LBB5_1 -; NOLSE-NEXT: .LBB5_5: // %atomicrmw.end +; NOLSE-NEXT: stlxr w9, x8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB5_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, d1 ; NOLSE-NEXT: ret ; @@ -753,41 +651,25 @@ define <2 x half> @test_atomicrmw_fmax_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; NOLSE: // %bb.0: ; NOLSE-NEXT: // kill: def $d0 killed $d0 def $q0 ; NOLSE-NEXT: mov h1, v0.h[1] -; NOLSE-NEXT: fcvt s2, h0 -; NOLSE-NEXT: ldr s0, [x0] +; NOLSE-NEXT: fcvt s0, h0 ; NOLSE-NEXT: fcvt s1, h1 -; NOLSE-NEXT: b .LBB7_2 ; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.eq .LBB7_5 -; NOLSE-NEXT: .LBB7_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB7_3 Depth 2 -; NOLSE-NEXT: mov h3, v0.h[1] -; NOLSE-NEXT: fcvt s4, h0 -; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr w8, [x0] +; NOLSE-NEXT: fmov s2, w8 +; NOLSE-NEXT: mov h3, v2.h[1] +; NOLSE-NEXT: fcvt s2, h2 ; NOLSE-NEXT: fcvt s3, h3 -; NOLSE-NEXT: fmaxnm s4, s4, s2 +; NOLSE-NEXT: fmaxnm s2, s2, s0 ; NOLSE-NEXT: fmaxnm s3, s3, s1 -; NOLSE-NEXT: fcvt h4, s4 +; NOLSE-NEXT: fcvt h2, s2 ; NOLSE-NEXT: fcvt h3, s3 -; NOLSE-NEXT: mov v4.h[1], v3.h[0] -; NOLSE-NEXT: fmov w8, s4 -; NOLSE-NEXT: .LBB7_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr w10, [x0] -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.ne .LBB7_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB7_3 -; NOLSE-NEXT: b .LBB7_1 -; NOLSE-NEXT: .LBB7_5: // %atomicrmw.end -; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: mov v2.h[1], v3.h[0] +; NOLSE-NEXT: fmov w9, s2 +; NOLSE-NEXT: stlxr w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB7_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, x8 ; NOLSE-NEXT: ret ; ; LSE-LABEL: test_atomicrmw_fmax_v2f16_seq_cst_align4: @@ -888,58 +770,42 @@ define <2 x bfloat> @test_atomicrmw_fmax_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; NOLSE-NEXT: mov h1, v0.h[1] ; NOLSE-NEXT: fmov w10, s0 ; NOLSE-NEXT: mov w8, #32767 // =0x7fff -; NOLSE-NEXT: ldr s0, [x0] ; NOLSE-NEXT: lsl w10, w10, #16 ; NOLSE-NEXT: fmov w9, s1 -; NOLSE-NEXT: fmov s2, w10 +; NOLSE-NEXT: fmov s1, w10 ; NOLSE-NEXT: lsl w9, w9, #16 -; NOLSE-NEXT: fmov s1, w9 -; NOLSE-NEXT: b .LBB8_2 +; NOLSE-NEXT: fmov s0, w9 ; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 -; NOLSE-NEXT: fmov s0, w11 -; NOLSE-NEXT: cmp w11, w9 -; NOLSE-NEXT: b.eq .LBB8_5 -; NOLSE-NEXT: .LBB8_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB8_3 Depth 2 -; NOLSE-NEXT: mov h3, v0.h[1] -; NOLSE-NEXT: fmov w10, s0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr w9, [x0] +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: mov h3, v2.h[1] +; NOLSE-NEXT: fmov w11, s2 +; NOLSE-NEXT: lsl w11, w11, #16 +; NOLSE-NEXT: fmov w10, s3 +; NOLSE-NEXT: fmov s3, w11 ; NOLSE-NEXT: lsl w10, w10, #16 -; NOLSE-NEXT: fmov w9, s3 -; NOLSE-NEXT: fmov s4, w10 -; NOLSE-NEXT: lsl w9, w9, #16 -; NOLSE-NEXT: fmaxnm s4, s4, s2 -; NOLSE-NEXT: fmov s3, w9 ; NOLSE-NEXT: fmaxnm s3, s3, s1 -; NOLSE-NEXT: fmov w10, s4 +; NOLSE-NEXT: fmov s2, w10 +; NOLSE-NEXT: fmaxnm s2, s2, s0 +; NOLSE-NEXT: fmov w11, s3 +; NOLSE-NEXT: ubfx w13, w11, #16, #1 +; NOLSE-NEXT: add w11, w11, w8 +; NOLSE-NEXT: fmov w10, s2 +; NOLSE-NEXT: add w11, w13, w11 +; NOLSE-NEXT: lsr w11, w11, #16 ; NOLSE-NEXT: ubfx w12, w10, #16, #1 ; NOLSE-NEXT: add w10, w10, w8 -; NOLSE-NEXT: fmov w9, s3 +; NOLSE-NEXT: fmov s3, w11 ; NOLSE-NEXT: add w10, w12, w10 ; NOLSE-NEXT: lsr w10, w10, #16 -; NOLSE-NEXT: ubfx w11, w9, #16, #1 -; NOLSE-NEXT: add w9, w9, w8 -; NOLSE-NEXT: fmov s4, w10 -; NOLSE-NEXT: add w9, w11, w9 -; NOLSE-NEXT: lsr w9, w9, #16 -; NOLSE-NEXT: fmov s3, w9 -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: mov v4.h[1], v3.h[0] -; NOLSE-NEXT: fmov w10, s4 -; NOLSE-NEXT: .LBB8_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr w11, [x0] -; NOLSE-NEXT: cmp w11, w9 -; NOLSE-NEXT: b.ne .LBB8_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, w10, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB8_3 -; NOLSE-NEXT: b .LBB8_1 -; NOLSE-NEXT: .LBB8_5: // %atomicrmw.end -; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: fmov s2, w10 +; NOLSE-NEXT: mov v3.h[1], v2.h[0] +; NOLSE-NEXT: fmov w10, s3 +; NOLSE-NEXT: stlxr w11, w10, [x0] +; NOLSE-NEXT: cbnz w11, .LBB8_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, x9 ; NOLSE-NEXT: ret ; ; LSE-LABEL: test_atomicrmw_fmax_v2bf16_seq_cst_align4: @@ -1047,31 +913,15 @@ define <2 x bfloat> @test_atomicrmw_fmax_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf define <2 x float> @test_atomicrmw_fmax_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmax_v2f32_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr d1, [x0] -; NOLSE-NEXT: b .LBB9_2 ; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 -; NOLSE-NEXT: fmov d1, x10 -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.eq .LBB9_5 -; NOLSE-NEXT: .LBB9_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr x8, [x0] +; NOLSE-NEXT: fmov d1, x8 ; NOLSE-NEXT: fmaxnm v2.2s, v1.2s, v0.2s -; NOLSE-NEXT: fmov x9, d1 ; NOLSE-NEXT: fmov x8, d2 -; NOLSE-NEXT: .LBB9_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr x10, [x0] -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.ne .LBB9_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, x8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB9_3 -; NOLSE-NEXT: b .LBB9_1 -; NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; NOLSE-NEXT: stlxr w9, x8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB9_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, d1 ; NOLSE-NEXT: ret ; @@ -1149,43 +999,17 @@ define <2 x float> @test_atomicrmw_fmax_v2f32_seq_cst_align8(ptr %ptr, <2 x floa define <2 x double> @test_atomicrmw_fmax_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmax_v2f64_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr q1, [x0] -; NOLSE-NEXT: b .LBB10_2 ; NOLSE-NEXT: .LBB10_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 -; NOLSE-NEXT: fmov d1, x12 -; NOLSE-NEXT: cmp x13, x9 -; NOLSE-NEXT: ccmp x12, x11, #0, eq -; NOLSE-NEXT: mov v1.d[1], x13 -; NOLSE-NEXT: b.eq .LBB10_6 -; NOLSE-NEXT: .LBB10_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxp x8, x9, [x0] +; NOLSE-NEXT: fmov d1, x8 +; NOLSE-NEXT: mov v1.d[1], x9 ; NOLSE-NEXT: fmaxnm v2.2d, v1.2d, v0.2d -; NOLSE-NEXT: mov x9, v1.d[1] -; NOLSE-NEXT: fmov x11, d1 ; NOLSE-NEXT: mov x8, v2.d[1] -; NOLSE-NEXT: fmov x10, d2 -; NOLSE-NEXT: .LBB10_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxp x12, x13, [x0] -; NOLSE-NEXT: cmp x12, x11 -; NOLSE-NEXT: cset w14, ne -; NOLSE-NEXT: cmp x13, x9 -; NOLSE-NEXT: cinc w14, w14, ne -; NOLSE-NEXT: cbz w14, .LBB10_5 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 -; NOLSE-NEXT: stlxp w14, x12, x13, [x0] -; NOLSE-NEXT: cbnz w14, .LBB10_3 -; NOLSE-NEXT: b .LBB10_1 -; NOLSE-NEXT: .LBB10_5: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 -; NOLSE-NEXT: stlxp w14, x10, x8, [x0] -; NOLSE-NEXT: cbnz w14, .LBB10_3 -; NOLSE-NEXT: b .LBB10_1 -; NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; NOLSE-NEXT: fmov x9, d2 +; NOLSE-NEXT: stlxp w10, x9, x8, [x0] +; NOLSE-NEXT: cbnz w10, .LBB10_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: mov v0.16b, v1.16b ; NOLSE-NEXT: ret ; diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll index 2697dbf5b219..71765f435d94 100644 --- a/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll +++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll @@ -10,33 +10,17 @@ define half @test_atomicrmw_fmin_f16_seq_cst_align2(ptr %ptr, half %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align2: ; NOLSE: // %bb.0: ; NOLSE-NEXT: fcvt s1, h0 -; NOLSE-NEXT: ldr h0, [x0] -; NOLSE-NEXT: b .LBB0_2 ; NOLSE-NEXT: .LBB0_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.eq .LBB0_5 -; NOLSE-NEXT: .LBB0_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB0_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w8, [x0] +; NOLSE-NEXT: fmov s0, w8 ; NOLSE-NEXT: fcvt s2, h0 -; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: fminnm s2, s2, s1 ; NOLSE-NEXT: fcvt h2, s2 ; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB0_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB0_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w10, [x0] -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.ne .LBB0_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB0_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB0_3 -; NOLSE-NEXT: b .LBB0_1 -; NOLSE-NEXT: .LBB0_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w9, w8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB0_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -111,33 +95,17 @@ define half @test_atomicrmw_fmin_f16_seq_cst_align4(ptr %ptr, half %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align4: ; NOLSE: // %bb.0: ; NOLSE-NEXT: fcvt s1, h0 -; NOLSE-NEXT: ldr h0, [x0] -; NOLSE-NEXT: b .LBB1_2 ; NOLSE-NEXT: .LBB1_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.eq .LBB1_5 -; NOLSE-NEXT: .LBB1_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB1_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w8, [x0] +; NOLSE-NEXT: fmov s0, w8 ; NOLSE-NEXT: fcvt s2, h0 -; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: fminnm s2, s2, s1 ; NOLSE-NEXT: fcvt h2, s2 ; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB1_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB1_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w10, [x0] -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.ne .LBB1_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB1_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB1_3 -; NOLSE-NEXT: b .LBB1_1 -; NOLSE-NEXT: .LBB1_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w9, w8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB1_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -214,19 +182,12 @@ define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align2(ptr %ptr, bfloat %value) ; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 ; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: mov w8, #32767 // =0x7fff -; NOLSE-NEXT: ldr h0, [x0] ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s1, w9 -; NOLSE-NEXT: b .LBB2_2 ; NOLSE-NEXT: .LBB2_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB2_2 Depth=1 -; NOLSE-NEXT: fmov s0, w11 -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.eq .LBB2_5 -; NOLSE-NEXT: .LBB2_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB2_3 Depth 2 -; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w9, [x0] +; NOLSE-NEXT: fmov s0, w9 ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s2, w9 ; NOLSE-NEXT: fminnm s2, s2, s1 @@ -235,21 +196,9 @@ define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align2(ptr %ptr, bfloat %value) ; NOLSE-NEXT: add w9, w9, w8 ; NOLSE-NEXT: add w9, w10, w9 ; NOLSE-NEXT: lsr w9, w9, #16 -; NOLSE-NEXT: fmov s2, w9 -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: fmov w10, s2 -; NOLSE-NEXT: .LBB2_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB2_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w11, [x0] -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.ne .LBB2_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB2_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w10, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB2_3 -; NOLSE-NEXT: b .LBB2_1 -; NOLSE-NEXT: .LBB2_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB2_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -328,19 +277,12 @@ define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align4(ptr %ptr, bfloat %value) ; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 ; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: mov w8, #32767 // =0x7fff -; NOLSE-NEXT: ldr h0, [x0] ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s1, w9 -; NOLSE-NEXT: b .LBB3_2 ; NOLSE-NEXT: .LBB3_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB3_2 Depth=1 -; NOLSE-NEXT: fmov s0, w11 -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.eq .LBB3_5 -; NOLSE-NEXT: .LBB3_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB3_3 Depth 2 -; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w9, [x0] +; NOLSE-NEXT: fmov s0, w9 ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s2, w9 ; NOLSE-NEXT: fminnm s2, s2, s1 @@ -349,21 +291,9 @@ define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align4(ptr %ptr, bfloat %value) ; NOLSE-NEXT: add w9, w9, w8 ; NOLSE-NEXT: add w9, w10, w9 ; NOLSE-NEXT: lsr w9, w9, #16 -; NOLSE-NEXT: fmov s2, w9 -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: fmov w10, s2 -; NOLSE-NEXT: .LBB3_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB3_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w11, [x0] -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.ne .LBB3_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB3_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w10, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB3_3 -; NOLSE-NEXT: b .LBB3_1 -; NOLSE-NEXT: .LBB3_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB3_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -439,31 +369,15 @@ define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align4(ptr %ptr, bfloat %value) define float @test_atomicrmw_fmin_f32_seq_cst_align4(ptr %ptr, float %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align4: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr s1, [x0] -; NOLSE-NEXT: b .LBB4_2 ; NOLSE-NEXT: .LBB4_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=1 -; NOLSE-NEXT: fmov s1, w10 -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.eq .LBB4_5 -; NOLSE-NEXT: .LBB4_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB4_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr w8, [x0] +; NOLSE-NEXT: fmov s1, w8 ; NOLSE-NEXT: fminnm s2, s1, s0 -; NOLSE-NEXT: fmov w9, s1 ; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB4_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB4_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr w10, [x0] -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.ne .LBB4_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB4_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB4_3 -; NOLSE-NEXT: b .LBB4_1 -; NOLSE-NEXT: .LBB4_5: // %atomicrmw.end +; NOLSE-NEXT: stlxr w9, w8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB4_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov s0, s1 ; NOLSE-NEXT: ret ; @@ -526,31 +440,15 @@ define float @test_atomicrmw_fmin_f32_seq_cst_align4(ptr %ptr, float %value) #0 define double @test_atomicrmw_fmin_f32_seq_cst_align8(ptr %ptr, double %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr d1, [x0] -; NOLSE-NEXT: b .LBB5_2 ; NOLSE-NEXT: .LBB5_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB5_2 Depth=1 -; NOLSE-NEXT: fmov d1, x10 -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.eq .LBB5_5 -; NOLSE-NEXT: .LBB5_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB5_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr x8, [x0] +; NOLSE-NEXT: fmov d1, x8 ; NOLSE-NEXT: fminnm d2, d1, d0 -; NOLSE-NEXT: fmov x9, d1 ; NOLSE-NEXT: fmov x8, d2 -; NOLSE-NEXT: .LBB5_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB5_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr x10, [x0] -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.ne .LBB5_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB5_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, x8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB5_3 -; NOLSE-NEXT: b .LBB5_1 -; NOLSE-NEXT: .LBB5_5: // %atomicrmw.end +; NOLSE-NEXT: stlxr w9, x8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB5_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, d1 ; NOLSE-NEXT: ret ; @@ -753,41 +651,25 @@ define <2 x half> @test_atomicrmw_fmin_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; NOLSE: // %bb.0: ; NOLSE-NEXT: // kill: def $d0 killed $d0 def $q0 ; NOLSE-NEXT: mov h1, v0.h[1] -; NOLSE-NEXT: fcvt s2, h0 -; NOLSE-NEXT: ldr s0, [x0] +; NOLSE-NEXT: fcvt s0, h0 ; NOLSE-NEXT: fcvt s1, h1 -; NOLSE-NEXT: b .LBB7_2 ; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.eq .LBB7_5 -; NOLSE-NEXT: .LBB7_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB7_3 Depth 2 -; NOLSE-NEXT: mov h3, v0.h[1] -; NOLSE-NEXT: fcvt s4, h0 -; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr w8, [x0] +; NOLSE-NEXT: fmov s2, w8 +; NOLSE-NEXT: mov h3, v2.h[1] +; NOLSE-NEXT: fcvt s2, h2 ; NOLSE-NEXT: fcvt s3, h3 -; NOLSE-NEXT: fminnm s4, s4, s2 +; NOLSE-NEXT: fminnm s2, s2, s0 ; NOLSE-NEXT: fminnm s3, s3, s1 -; NOLSE-NEXT: fcvt h4, s4 +; NOLSE-NEXT: fcvt h2, s2 ; NOLSE-NEXT: fcvt h3, s3 -; NOLSE-NEXT: mov v4.h[1], v3.h[0] -; NOLSE-NEXT: fmov w8, s4 -; NOLSE-NEXT: .LBB7_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr w10, [x0] -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.ne .LBB7_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB7_3 -; NOLSE-NEXT: b .LBB7_1 -; NOLSE-NEXT: .LBB7_5: // %atomicrmw.end -; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: mov v2.h[1], v3.h[0] +; NOLSE-NEXT: fmov w9, s2 +; NOLSE-NEXT: stlxr w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB7_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, x8 ; NOLSE-NEXT: ret ; ; LSE-LABEL: test_atomicrmw_fmin_v2f16_seq_cst_align4: @@ -888,58 +770,42 @@ define <2 x bfloat> @test_atomicrmw_fmin_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; NOLSE-NEXT: mov h1, v0.h[1] ; NOLSE-NEXT: fmov w10, s0 ; NOLSE-NEXT: mov w8, #32767 // =0x7fff -; NOLSE-NEXT: ldr s0, [x0] ; NOLSE-NEXT: lsl w10, w10, #16 ; NOLSE-NEXT: fmov w9, s1 -; NOLSE-NEXT: fmov s2, w10 +; NOLSE-NEXT: fmov s1, w10 ; NOLSE-NEXT: lsl w9, w9, #16 -; NOLSE-NEXT: fmov s1, w9 -; NOLSE-NEXT: b .LBB8_2 +; NOLSE-NEXT: fmov s0, w9 ; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 -; NOLSE-NEXT: fmov s0, w11 -; NOLSE-NEXT: cmp w11, w9 -; NOLSE-NEXT: b.eq .LBB8_5 -; NOLSE-NEXT: .LBB8_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB8_3 Depth 2 -; NOLSE-NEXT: mov h3, v0.h[1] -; NOLSE-NEXT: fmov w10, s0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr w9, [x0] +; NOLSE-NEXT: fmov s2, w9 +; NOLSE-NEXT: mov h3, v2.h[1] +; NOLSE-NEXT: fmov w11, s2 +; NOLSE-NEXT: lsl w11, w11, #16 +; NOLSE-NEXT: fmov w10, s3 +; NOLSE-NEXT: fmov s3, w11 ; NOLSE-NEXT: lsl w10, w10, #16 -; NOLSE-NEXT: fmov w9, s3 -; NOLSE-NEXT: fmov s4, w10 -; NOLSE-NEXT: lsl w9, w9, #16 -; NOLSE-NEXT: fminnm s4, s4, s2 -; NOLSE-NEXT: fmov s3, w9 ; NOLSE-NEXT: fminnm s3, s3, s1 -; NOLSE-NEXT: fmov w10, s4 +; NOLSE-NEXT: fmov s2, w10 +; NOLSE-NEXT: fminnm s2, s2, s0 +; NOLSE-NEXT: fmov w11, s3 +; NOLSE-NEXT: ubfx w13, w11, #16, #1 +; NOLSE-NEXT: add w11, w11, w8 +; NOLSE-NEXT: fmov w10, s2 +; NOLSE-NEXT: add w11, w13, w11 +; NOLSE-NEXT: lsr w11, w11, #16 ; NOLSE-NEXT: ubfx w12, w10, #16, #1 ; NOLSE-NEXT: add w10, w10, w8 -; NOLSE-NEXT: fmov w9, s3 +; NOLSE-NEXT: fmov s3, w11 ; NOLSE-NEXT: add w10, w12, w10 ; NOLSE-NEXT: lsr w10, w10, #16 -; NOLSE-NEXT: ubfx w11, w9, #16, #1 -; NOLSE-NEXT: add w9, w9, w8 -; NOLSE-NEXT: fmov s4, w10 -; NOLSE-NEXT: add w9, w11, w9 -; NOLSE-NEXT: lsr w9, w9, #16 -; NOLSE-NEXT: fmov s3, w9 -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: mov v4.h[1], v3.h[0] -; NOLSE-NEXT: fmov w10, s4 -; NOLSE-NEXT: .LBB8_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr w11, [x0] -; NOLSE-NEXT: cmp w11, w9 -; NOLSE-NEXT: b.ne .LBB8_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, w10, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB8_3 -; NOLSE-NEXT: b .LBB8_1 -; NOLSE-NEXT: .LBB8_5: // %atomicrmw.end -; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: fmov s2, w10 +; NOLSE-NEXT: mov v3.h[1], v2.h[0] +; NOLSE-NEXT: fmov w10, s3 +; NOLSE-NEXT: stlxr w11, w10, [x0] +; NOLSE-NEXT: cbnz w11, .LBB8_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, x9 ; NOLSE-NEXT: ret ; ; LSE-LABEL: test_atomicrmw_fmin_v2bf16_seq_cst_align4: @@ -1047,31 +913,15 @@ define <2 x bfloat> @test_atomicrmw_fmin_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf define <2 x float> @test_atomicrmw_fmin_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmin_v2f32_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr d1, [x0] -; NOLSE-NEXT: b .LBB9_2 ; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 -; NOLSE-NEXT: fmov d1, x10 -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.eq .LBB9_5 -; NOLSE-NEXT: .LBB9_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr x8, [x0] +; NOLSE-NEXT: fmov d1, x8 ; NOLSE-NEXT: fminnm v2.2s, v1.2s, v0.2s -; NOLSE-NEXT: fmov x9, d1 ; NOLSE-NEXT: fmov x8, d2 -; NOLSE-NEXT: .LBB9_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr x10, [x0] -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.ne .LBB9_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, x8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB9_3 -; NOLSE-NEXT: b .LBB9_1 -; NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; NOLSE-NEXT: stlxr w9, x8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB9_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, d1 ; NOLSE-NEXT: ret ; @@ -1149,43 +999,17 @@ define <2 x float> @test_atomicrmw_fmin_v2f32_seq_cst_align8(ptr %ptr, <2 x floa define <2 x double> @test_atomicrmw_fmin_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmin_v2f64_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr q1, [x0] -; NOLSE-NEXT: b .LBB10_2 ; NOLSE-NEXT: .LBB10_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 -; NOLSE-NEXT: fmov d1, x12 -; NOLSE-NEXT: cmp x13, x9 -; NOLSE-NEXT: ccmp x12, x11, #0, eq -; NOLSE-NEXT: mov v1.d[1], x13 -; NOLSE-NEXT: b.eq .LBB10_6 -; NOLSE-NEXT: .LBB10_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxp x8, x9, [x0] +; NOLSE-NEXT: fmov d1, x8 +; NOLSE-NEXT: mov v1.d[1], x9 ; NOLSE-NEXT: fminnm v2.2d, v1.2d, v0.2d -; NOLSE-NEXT: mov x9, v1.d[1] -; NOLSE-NEXT: fmov x11, d1 ; NOLSE-NEXT: mov x8, v2.d[1] -; NOLSE-NEXT: fmov x10, d2 -; NOLSE-NEXT: .LBB10_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxp x12, x13, [x0] -; NOLSE-NEXT: cmp x12, x11 -; NOLSE-NEXT: cset w14, ne -; NOLSE-NEXT: cmp x13, x9 -; NOLSE-NEXT: cinc w14, w14, ne -; NOLSE-NEXT: cbz w14, .LBB10_5 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 -; NOLSE-NEXT: stlxp w14, x12, x13, [x0] -; NOLSE-NEXT: cbnz w14, .LBB10_3 -; NOLSE-NEXT: b .LBB10_1 -; NOLSE-NEXT: .LBB10_5: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 -; NOLSE-NEXT: stlxp w14, x10, x8, [x0] -; NOLSE-NEXT: cbnz w14, .LBB10_3 -; NOLSE-NEXT: b .LBB10_1 -; NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; NOLSE-NEXT: fmov x9, d2 +; NOLSE-NEXT: stlxp w10, x9, x8, [x0] +; NOLSE-NEXT: cbnz w10, .LBB10_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: mov v0.16b, v1.16b ; NOLSE-NEXT: ret ; diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll index f41ddcb81d5c..67e164037d5c 100644 --- a/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll +++ b/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll @@ -7,33 +7,17 @@ define half @test_atomicrmw_fsub_f16_seq_cst_align2(ptr %ptr, half %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align2: ; NOLSE: // %bb.0: ; NOLSE-NEXT: fcvt s1, h0 -; NOLSE-NEXT: ldr h0, [x0] -; NOLSE-NEXT: b .LBB0_2 ; NOLSE-NEXT: .LBB0_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.eq .LBB0_5 -; NOLSE-NEXT: .LBB0_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB0_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w8, [x0] +; NOLSE-NEXT: fmov s0, w8 ; NOLSE-NEXT: fcvt s2, h0 -; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: fsub s2, s2, s1 ; NOLSE-NEXT: fcvt h2, s2 ; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB0_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB0_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w10, [x0] -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.ne .LBB0_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB0_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB0_3 -; NOLSE-NEXT: b .LBB0_1 -; NOLSE-NEXT: .LBB0_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w9, w8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB0_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -108,33 +92,17 @@ define half @test_atomicrmw_fsub_f16_seq_cst_align4(ptr %ptr, half %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align4: ; NOLSE: // %bb.0: ; NOLSE-NEXT: fcvt s1, h0 -; NOLSE-NEXT: ldr h0, [x0] -; NOLSE-NEXT: b .LBB1_2 ; NOLSE-NEXT: .LBB1_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.eq .LBB1_5 -; NOLSE-NEXT: .LBB1_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB1_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w8, [x0] +; NOLSE-NEXT: fmov s0, w8 ; NOLSE-NEXT: fcvt s2, h0 -; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: fsub s2, s2, s1 ; NOLSE-NEXT: fcvt h2, s2 ; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB1_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB1_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w10, [x0] -; NOLSE-NEXT: cmp w10, w9, uxth -; NOLSE-NEXT: b.ne .LBB1_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB1_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB1_3 -; NOLSE-NEXT: b .LBB1_1 -; NOLSE-NEXT: .LBB1_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w9, w8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB1_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -211,19 +179,12 @@ define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align2(ptr %ptr, bfloat %value) ; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 ; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: mov w8, #32767 // =0x7fff -; NOLSE-NEXT: ldr h0, [x0] ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s1, w9 -; NOLSE-NEXT: b .LBB2_2 ; NOLSE-NEXT: .LBB2_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB2_2 Depth=1 -; NOLSE-NEXT: fmov s0, w11 -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.eq .LBB2_5 -; NOLSE-NEXT: .LBB2_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB2_3 Depth 2 -; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w9, [x0] +; NOLSE-NEXT: fmov s0, w9 ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s2, w9 ; NOLSE-NEXT: fsub s2, s2, s1 @@ -232,21 +193,9 @@ define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align2(ptr %ptr, bfloat %value) ; NOLSE-NEXT: add w9, w9, w8 ; NOLSE-NEXT: add w9, w10, w9 ; NOLSE-NEXT: lsr w9, w9, #16 -; NOLSE-NEXT: fmov s2, w9 -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: fmov w10, s2 -; NOLSE-NEXT: .LBB2_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB2_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w11, [x0] -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.ne .LBB2_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB2_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w10, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB2_3 -; NOLSE-NEXT: b .LBB2_1 -; NOLSE-NEXT: .LBB2_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB2_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -325,19 +274,12 @@ define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align4(ptr %ptr, bfloat %value) ; NOLSE-NEXT: // kill: def $h0 killed $h0 def $s0 ; NOLSE-NEXT: fmov w9, s0 ; NOLSE-NEXT: mov w8, #32767 // =0x7fff -; NOLSE-NEXT: ldr h0, [x0] ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s1, w9 -; NOLSE-NEXT: b .LBB3_2 ; NOLSE-NEXT: .LBB3_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB3_2 Depth=1 -; NOLSE-NEXT: fmov s0, w11 -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.eq .LBB3_5 -; NOLSE-NEXT: .LBB3_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB3_3 Depth 2 -; NOLSE-NEXT: fmov w9, s0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrh w9, [x0] +; NOLSE-NEXT: fmov s0, w9 ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s2, w9 ; NOLSE-NEXT: fsub s2, s2, s1 @@ -346,21 +288,9 @@ define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align4(ptr %ptr, bfloat %value) ; NOLSE-NEXT: add w9, w9, w8 ; NOLSE-NEXT: add w9, w10, w9 ; NOLSE-NEXT: lsr w9, w9, #16 -; NOLSE-NEXT: fmov s2, w9 -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: fmov w10, s2 -; NOLSE-NEXT: .LBB3_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB3_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxrh w11, [x0] -; NOLSE-NEXT: cmp w11, w9, uxth -; NOLSE-NEXT: b.ne .LBB3_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB3_3 Depth=2 -; NOLSE-NEXT: stlxrh wzr, w10, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB3_3 -; NOLSE-NEXT: b .LBB3_1 -; NOLSE-NEXT: .LBB3_5: // %atomicrmw.end +; NOLSE-NEXT: stlxrh w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB3_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: // kill: def $h0 killed $h0 killed $s0 ; NOLSE-NEXT: ret ; @@ -436,31 +366,15 @@ define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align4(ptr %ptr, bfloat %value) define float @test_atomicrmw_fsub_f32_seq_cst_align4(ptr %ptr, float %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align4: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr s1, [x0] -; NOLSE-NEXT: b .LBB4_2 ; NOLSE-NEXT: .LBB4_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=1 -; NOLSE-NEXT: fmov s1, w10 -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.eq .LBB4_5 -; NOLSE-NEXT: .LBB4_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB4_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr w8, [x0] +; NOLSE-NEXT: fmov s1, w8 ; NOLSE-NEXT: fsub s2, s1, s0 -; NOLSE-NEXT: fmov w9, s1 ; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB4_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB4_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr w10, [x0] -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.ne .LBB4_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB4_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB4_3 -; NOLSE-NEXT: b .LBB4_1 -; NOLSE-NEXT: .LBB4_5: // %atomicrmw.end +; NOLSE-NEXT: stlxr w9, w8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB4_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov s0, s1 ; NOLSE-NEXT: ret ; @@ -523,31 +437,15 @@ define float @test_atomicrmw_fsub_f32_seq_cst_align4(ptr %ptr, float %value) #0 define double @test_atomicrmw_fsub_f32_seq_cst_align8(ptr %ptr, double %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr d1, [x0] -; NOLSE-NEXT: b .LBB5_2 ; NOLSE-NEXT: .LBB5_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB5_2 Depth=1 -; NOLSE-NEXT: fmov d1, x10 -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.eq .LBB5_5 -; NOLSE-NEXT: .LBB5_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB5_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr x8, [x0] +; NOLSE-NEXT: fmov d1, x8 ; NOLSE-NEXT: fsub d2, d1, d0 -; NOLSE-NEXT: fmov x9, d1 ; NOLSE-NEXT: fmov x8, d2 -; NOLSE-NEXT: .LBB5_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB5_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr x10, [x0] -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.ne .LBB5_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB5_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, x8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB5_3 -; NOLSE-NEXT: b .LBB5_1 -; NOLSE-NEXT: .LBB5_5: // %atomicrmw.end +; NOLSE-NEXT: stlxr w9, x8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB5_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, d1 ; NOLSE-NEXT: ret ; @@ -748,35 +646,19 @@ define fp128 @test_atomicrmw_fsub_fp128_seq_cst_align16(ptr %ptr, fp128 %value) define <2 x half> @test_atomicrmw_fsub_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fsub_v2f16_seq_cst_align4: ; NOLSE: // %bb.0: -; NOLSE-NEXT: fcvtl v1.4s, v0.4h -; NOLSE-NEXT: ldr s0, [x0] -; NOLSE-NEXT: b .LBB7_2 +; NOLSE-NEXT: fcvtl v0.4s, v0.4h ; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.eq .LBB7_5 -; NOLSE-NEXT: .LBB7_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB7_3 Depth 2 -; NOLSE-NEXT: fcvtl v2.4s, v0.4h -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: fsub v2.4s, v2.4s, v1.4s -; NOLSE-NEXT: fcvtn v2.4h, v2.4s -; NOLSE-NEXT: fmov w8, s2 -; NOLSE-NEXT: .LBB7_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr w10, [x0] -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.ne .LBB7_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB7_3 -; NOLSE-NEXT: b .LBB7_1 -; NOLSE-NEXT: .LBB7_5: // %atomicrmw.end -; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr w8, [x0] +; NOLSE-NEXT: fmov s1, w8 +; NOLSE-NEXT: fcvtl v1.4s, v1.4h +; NOLSE-NEXT: fsub v1.4s, v1.4s, v0.4s +; NOLSE-NEXT: fcvtn v1.4h, v1.4s +; NOLSE-NEXT: fmov w9, s1 +; NOLSE-NEXT: stlxr w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB7_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, x8 ; NOLSE-NEXT: ret ; ; LSE-LABEL: test_atomicrmw_fsub_v2f16_seq_cst_align4: @@ -867,38 +749,22 @@ define <2 x bfloat> @test_atomicrmw_fsub_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; NOLSE: // %bb.0: ; NOLSE-NEXT: movi v1.4s, #1 ; NOLSE-NEXT: movi v2.4s, #127, msl #8 -; NOLSE-NEXT: shll v3.4s, v0.4h, #16 -; NOLSE-NEXT: ldr s0, [x0] -; NOLSE-NEXT: b .LBB8_2 +; NOLSE-NEXT: shll v0.4s, v0.4h, #16 ; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 -; NOLSE-NEXT: fmov s0, w10 -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.eq .LBB8_5 -; NOLSE-NEXT: .LBB8_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB8_3 Depth 2 -; NOLSE-NEXT: shll v4.4s, v0.4h, #16 -; NOLSE-NEXT: fmov w9, s0 -; NOLSE-NEXT: fsub v4.4s, v4.4s, v3.4s -; NOLSE-NEXT: ushr v5.4s, v4.4s, #16 -; NOLSE-NEXT: and v5.16b, v5.16b, v1.16b -; NOLSE-NEXT: add v4.4s, v5.4s, v4.4s -; NOLSE-NEXT: addhn v4.4h, v4.4s, v2.4s -; NOLSE-NEXT: fmov w8, s4 -; NOLSE-NEXT: .LBB8_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr w10, [x0] -; NOLSE-NEXT: cmp w10, w9 -; NOLSE-NEXT: b.ne .LBB8_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, w8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB8_3 -; NOLSE-NEXT: b .LBB8_1 -; NOLSE-NEXT: .LBB8_5: // %atomicrmw.end -; NOLSE-NEXT: // kill: def $d0 killed $d0 killed $q0 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr w8, [x0] +; NOLSE-NEXT: fmov s3, w8 +; NOLSE-NEXT: shll v3.4s, v3.4h, #16 +; NOLSE-NEXT: fsub v3.4s, v3.4s, v0.4s +; NOLSE-NEXT: ushr v4.4s, v3.4s, #16 +; NOLSE-NEXT: and v4.16b, v4.16b, v1.16b +; NOLSE-NEXT: add v3.4s, v4.4s, v3.4s +; NOLSE-NEXT: addhn v3.4h, v3.4s, v2.4s +; NOLSE-NEXT: fmov w9, s3 +; NOLSE-NEXT: stlxr w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB8_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end +; NOLSE-NEXT: fmov d0, x8 ; NOLSE-NEXT: ret ; ; LSE-LABEL: test_atomicrmw_fsub_v2bf16_seq_cst_align4: @@ -984,31 +850,15 @@ define <2 x bfloat> @test_atomicrmw_fsub_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf define <2 x float> @test_atomicrmw_fsub_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fsub_v2f32_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr d1, [x0] -; NOLSE-NEXT: b .LBB9_2 ; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 -; NOLSE-NEXT: fmov d1, x10 -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.eq .LBB9_5 -; NOLSE-NEXT: .LBB9_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr x8, [x0] +; NOLSE-NEXT: fmov d1, x8 ; NOLSE-NEXT: fsub v2.2s, v1.2s, v0.2s -; NOLSE-NEXT: fmov x9, d1 ; NOLSE-NEXT: fmov x8, d2 -; NOLSE-NEXT: .LBB9_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxr x10, [x0] -; NOLSE-NEXT: cmp x10, x9 -; NOLSE-NEXT: b.ne .LBB9_1 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 -; NOLSE-NEXT: stlxr wzr, x8, [x0] -; NOLSE-NEXT: cbnz wzr, .LBB9_3 -; NOLSE-NEXT: b .LBB9_1 -; NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; NOLSE-NEXT: stlxr w9, x8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB9_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, d1 ; NOLSE-NEXT: ret ; @@ -1086,43 +936,17 @@ define <2 x float> @test_atomicrmw_fsub_v2f32_seq_cst_align8(ptr %ptr, <2 x floa define <2 x double> @test_atomicrmw_fsub_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fsub_v2f64_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: ldr q1, [x0] -; NOLSE-NEXT: b .LBB10_2 ; NOLSE-NEXT: .LBB10_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 -; NOLSE-NEXT: fmov d1, x12 -; NOLSE-NEXT: cmp x13, x9 -; NOLSE-NEXT: ccmp x12, x11, #0, eq -; NOLSE-NEXT: mov v1.d[1], x13 -; NOLSE-NEXT: b.eq .LBB10_6 -; NOLSE-NEXT: .LBB10_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxp x8, x9, [x0] +; NOLSE-NEXT: fmov d1, x8 +; NOLSE-NEXT: mov v1.d[1], x9 ; NOLSE-NEXT: fsub v2.2d, v1.2d, v0.2d -; NOLSE-NEXT: mov x9, v1.d[1] -; NOLSE-NEXT: fmov x11, d1 ; NOLSE-NEXT: mov x8, v2.d[1] -; NOLSE-NEXT: fmov x10, d2 -; NOLSE-NEXT: .LBB10_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxp x12, x13, [x0] -; NOLSE-NEXT: cmp x12, x11 -; NOLSE-NEXT: cset w14, ne -; NOLSE-NEXT: cmp x13, x9 -; NOLSE-NEXT: cinc w14, w14, ne -; NOLSE-NEXT: cbz w14, .LBB10_5 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 -; NOLSE-NEXT: stlxp w14, x12, x13, [x0] -; NOLSE-NEXT: cbnz w14, .LBB10_3 -; NOLSE-NEXT: b .LBB10_1 -; NOLSE-NEXT: .LBB10_5: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 -; NOLSE-NEXT: stlxp w14, x10, x8, [x0] -; NOLSE-NEXT: cbnz w14, .LBB10_3 -; NOLSE-NEXT: b .LBB10_1 -; NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; NOLSE-NEXT: fmov x9, d2 +; NOLSE-NEXT: stlxp w10, x9, x8, [x0] +; NOLSE-NEXT: cbnz w10, .LBB10_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: mov v0.16b, v1.16b ; NOLSE-NEXT: ret ; -- Gitee From cfe0439a644538bcb33b0f46a1be0ecb38b50320 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Thu, 29 Aug 2024 20:12:14 +0400 Subject: [PATCH 3/3] AArch64: Delete tests of fp128 atomicrmw fmin/fmax These are getting different output on some build hosts for some reason. The stack offsets of temporaries are different. --- llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll | 285 +++++--------------- llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll | 285 +++++--------------- 2 files changed, 146 insertions(+), 424 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll index 12a0c1169f2b..bfe0d20ca814 100644 --- a/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll +++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll @@ -3,8 +3,7 @@ ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+lse -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=LSE %s ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=-lse,-fp-armv8 -O1 < %s | FileCheck -check-prefix=SOFTFP-NOLSE %s -; FIXME: Windows hosts assigns stack slots to different offsets for some reason. -; UNSUPPORTED: system-windows +; FIXME: Restore test of fp128 case define half @test_atomicrmw_fmax_f16_seq_cst_align2(ptr %ptr, half %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align2: @@ -508,144 +507,6 @@ define double @test_atomicrmw_fmax_f32_seq_cst_align8(ptr %ptr, double %value) # ret double %res } -define fp128 @test_atomicrmw_fmax_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 { -; NOLSE-LABEL: test_atomicrmw_fmax_fp128_seq_cst_align16: -; NOLSE: // %bb.0: -; NOLSE-NEXT: sub sp, sp, #96 -; NOLSE-NEXT: ldr q1, [x0] -; NOLSE-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill -; NOLSE-NEXT: mov x19, x0 -; NOLSE-NEXT: str q0, [sp] // 16-byte Folded Spill -; NOLSE-NEXT: b .LBB6_2 -; NOLSE-NEXT: .LBB6_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 -; NOLSE-NEXT: stp x12, x13, [sp, #32] -; NOLSE-NEXT: cmp x13, x10 -; NOLSE-NEXT: ldr q1, [sp, #32] -; NOLSE-NEXT: ccmp x12, x11, #0, eq -; NOLSE-NEXT: b.eq .LBB6_6 -; NOLSE-NEXT: .LBB6_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB6_3 Depth 2 -; NOLSE-NEXT: mov v0.16b, v1.16b -; NOLSE-NEXT: str q1, [sp, #16] // 16-byte Folded Spill -; NOLSE-NEXT: ldr q1, [sp] // 16-byte Folded Reload -; NOLSE-NEXT: bl fmaxl -; NOLSE-NEXT: str q0, [sp, #48] -; NOLSE-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload -; NOLSE-NEXT: ldp x9, x8, [sp, #48] -; NOLSE-NEXT: str q0, [sp, #64] -; NOLSE-NEXT: ldp x11, x10, [sp, #64] -; NOLSE-NEXT: .LBB6_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxp x12, x13, [x19] -; NOLSE-NEXT: cmp x12, x11 -; NOLSE-NEXT: cset w14, ne -; NOLSE-NEXT: cmp x13, x10 -; NOLSE-NEXT: cinc w14, w14, ne -; NOLSE-NEXT: cbz w14, .LBB6_5 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 -; NOLSE-NEXT: stlxp w14, x12, x13, [x19] -; NOLSE-NEXT: cbnz w14, .LBB6_3 -; NOLSE-NEXT: b .LBB6_1 -; NOLSE-NEXT: .LBB6_5: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 -; NOLSE-NEXT: stlxp w14, x9, x8, [x19] -; NOLSE-NEXT: cbnz w14, .LBB6_3 -; NOLSE-NEXT: b .LBB6_1 -; NOLSE-NEXT: .LBB6_6: // %atomicrmw.end -; NOLSE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload -; NOLSE-NEXT: mov v0.16b, v1.16b -; NOLSE-NEXT: add sp, sp, #96 -; NOLSE-NEXT: ret -; -; LSE-LABEL: test_atomicrmw_fmax_fp128_seq_cst_align16: -; LSE: // %bb.0: -; LSE-NEXT: sub sp, sp, #96 -; LSE-NEXT: ldr q1, [x0] -; LSE-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill -; LSE-NEXT: mov x19, x0 -; LSE-NEXT: str q0, [sp] // 16-byte Folded Spill -; LSE-NEXT: .LBB6_1: // %atomicrmw.start -; LSE-NEXT: // =>This Inner Loop Header: Depth=1 -; LSE-NEXT: mov v0.16b, v1.16b -; LSE-NEXT: str q1, [sp, #16] // 16-byte Folded Spill -; LSE-NEXT: ldr q1, [sp] // 16-byte Folded Reload -; LSE-NEXT: bl fmaxl -; LSE-NEXT: str q0, [sp, #48] -; LSE-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload -; LSE-NEXT: ldp x0, x1, [sp, #48] -; LSE-NEXT: str q0, [sp, #64] -; LSE-NEXT: ldp x2, x3, [sp, #64] -; LSE-NEXT: mov x4, x2 -; LSE-NEXT: mov x5, x3 -; LSE-NEXT: caspal x4, x5, x0, x1, [x19] -; LSE-NEXT: stp x4, x5, [sp, #32] -; LSE-NEXT: cmp x5, x3 -; LSE-NEXT: ldr q1, [sp, #32] -; LSE-NEXT: ccmp x4, x2, #0, eq -; LSE-NEXT: b.ne .LBB6_1 -; LSE-NEXT: // %bb.2: // %atomicrmw.end -; LSE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload -; LSE-NEXT: mov v0.16b, v1.16b -; LSE-NEXT: add sp, sp, #96 -; LSE-NEXT: ret -; -; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmax_fp128_seq_cst_align16: -; SOFTFP-NOLSE: // %bb.0: -; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill -; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill -; SOFTFP-NOLSE-NEXT: mov x20, x0 -; SOFTFP-NOLSE-NEXT: mov x19, x3 -; SOFTFP-NOLSE-NEXT: ldp x0, x1, [x0] -; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill -; SOFTFP-NOLSE-NEXT: mov x21, x2 -; SOFTFP-NOLSE-NEXT: b .LBB6_2 -; SOFTFP-NOLSE-NEXT: .LBB6_1: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 -; SOFTFP-NOLSE-NEXT: cmp x1, x22 -; SOFTFP-NOLSE-NEXT: ccmp x0, x23, #0, eq -; SOFTFP-NOLSE-NEXT: b.eq .LBB6_6 -; SOFTFP-NOLSE-NEXT: .LBB6_2: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 -; SOFTFP-NOLSE-NEXT: // Child Loop BB6_3 Depth 2 -; SOFTFP-NOLSE-NEXT: mov x2, x21 -; SOFTFP-NOLSE-NEXT: mov x3, x19 -; SOFTFP-NOLSE-NEXT: mov x22, x1 -; SOFTFP-NOLSE-NEXT: mov x23, x0 -; SOFTFP-NOLSE-NEXT: bl fmaxl -; SOFTFP-NOLSE-NEXT: mov x8, x0 -; SOFTFP-NOLSE-NEXT: mov x9, x1 -; SOFTFP-NOLSE-NEXT: .LBB6_3: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 -; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; SOFTFP-NOLSE-NEXT: ldaxp x0, x1, [x20] -; SOFTFP-NOLSE-NEXT: cmp x0, x23 -; SOFTFP-NOLSE-NEXT: cset w10, ne -; SOFTFP-NOLSE-NEXT: cmp x1, x22 -; SOFTFP-NOLSE-NEXT: cinc w10, w10, ne -; SOFTFP-NOLSE-NEXT: cbz w10, .LBB6_5 -; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 -; SOFTFP-NOLSE-NEXT: stlxp w10, x0, x1, [x20] -; SOFTFP-NOLSE-NEXT: cbnz w10, .LBB6_3 -; SOFTFP-NOLSE-NEXT: b .LBB6_1 -; SOFTFP-NOLSE-NEXT: .LBB6_5: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 -; SOFTFP-NOLSE-NEXT: stlxp w10, x8, x9, [x20] -; SOFTFP-NOLSE-NEXT: cbnz w10, .LBB6_3 -; SOFTFP-NOLSE-NEXT: b .LBB6_1 -; SOFTFP-NOLSE-NEXT: .LBB6_6: // %atomicrmw.end -; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload -; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload -; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload -; SOFTFP-NOLSE-NEXT: ret - %res = atomicrmw fmax ptr %ptr, fp128 %value seq_cst, align 16 - ret fp128 %res -} - define <2 x half> @test_atomicrmw_fmax_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmax_v2f16_seq_cst_align4: ; NOLSE: // %bb.0: @@ -653,7 +514,7 @@ define <2 x half> @test_atomicrmw_fmax_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; NOLSE-NEXT: mov h1, v0.h[1] ; NOLSE-NEXT: fcvt s0, h0 ; NOLSE-NEXT: fcvt s1, h1 -; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start +; NOLSE-NEXT: .LBB6_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 ; NOLSE-NEXT: ldaxr w8, [x0] ; NOLSE-NEXT: fmov s2, w8 @@ -667,7 +528,7 @@ define <2 x half> @test_atomicrmw_fmax_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; NOLSE-NEXT: mov v2.h[1], v3.h[0] ; NOLSE-NEXT: fmov w9, s2 ; NOLSE-NEXT: stlxr w10, w9, [x0] -; NOLSE-NEXT: cbnz w10, .LBB7_1 +; NOLSE-NEXT: cbnz w10, .LBB6_1 ; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, x8 ; NOLSE-NEXT: ret @@ -679,7 +540,7 @@ define <2 x half> @test_atomicrmw_fmax_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; LSE-NEXT: fcvt s2, h0 ; LSE-NEXT: ldr s0, [x0] ; LSE-NEXT: fcvt s1, h1 -; LSE-NEXT: .LBB7_1: // %atomicrmw.start +; LSE-NEXT: .LBB6_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 ; LSE-NEXT: mov h3, v0.h[1] ; LSE-NEXT: fcvt s4, h0 @@ -695,7 +556,7 @@ define <2 x half> @test_atomicrmw_fmax_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; LSE-NEXT: casal w10, w9, [x0] ; LSE-NEXT: fmov s0, w10 ; LSE-NEXT: cmp w10, w8 -; LSE-NEXT: b.ne .LBB7_1 +; LSE-NEXT: b.ne .LBB6_1 ; LSE-NEXT: // %bb.2: // %atomicrmw.end ; LSE-NEXT: // kill: def $d0 killed $d0 killed $q0 ; LSE-NEXT: ret @@ -711,16 +572,16 @@ define <2 x half> @test_atomicrmw_fmax_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill ; SOFTFP-NOLSE-NEXT: mov w19, w2 ; SOFTFP-NOLSE-NEXT: mov x20, x0 -; SOFTFP-NOLSE-NEXT: b .LBB7_2 -; SOFTFP-NOLSE-NEXT: .LBB7_1: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 +; SOFTFP-NOLSE-NEXT: b .LBB6_2 +; SOFTFP-NOLSE-NEXT: .LBB6_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 ; SOFTFP-NOLSE-NEXT: lsr w23, w8, #16 ; SOFTFP-NOLSE-NEXT: cmp w8, w21 ; SOFTFP-NOLSE-NEXT: mov w21, w8 -; SOFTFP-NOLSE-NEXT: b.eq .LBB7_5 -; SOFTFP-NOLSE-NEXT: .LBB7_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: b.eq .LBB6_5 +; SOFTFP-NOLSE-NEXT: .LBB6_2: // %atomicrmw.start ; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 -; SOFTFP-NOLSE-NEXT: // Child Loop BB7_3 Depth 2 +; SOFTFP-NOLSE-NEXT: // Child Loop BB6_3 Depth 2 ; SOFTFP-NOLSE-NEXT: and w0, w19, #0xffff ; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee ; SOFTFP-NOLSE-NEXT: mov w24, w0 @@ -740,18 +601,18 @@ define <2 x half> @test_atomicrmw_fmax_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee ; SOFTFP-NOLSE-NEXT: bfi w21, w23, #16, #16 ; SOFTFP-NOLSE-NEXT: bfi w0, w24, #16, #16 -; SOFTFP-NOLSE-NEXT: .LBB7_3: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 +; SOFTFP-NOLSE-NEXT: .LBB6_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 ; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 ; SOFTFP-NOLSE-NEXT: ldaxr w8, [x20] ; SOFTFP-NOLSE-NEXT: cmp w8, w21 -; SOFTFP-NOLSE-NEXT: b.ne .LBB7_1 +; SOFTFP-NOLSE-NEXT: b.ne .LBB6_1 ; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 ; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x20] -; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB7_3 -; SOFTFP-NOLSE-NEXT: b .LBB7_1 -; SOFTFP-NOLSE-NEXT: .LBB7_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB6_3 +; SOFTFP-NOLSE-NEXT: b .LBB6_1 +; SOFTFP-NOLSE-NEXT: .LBB6_5: // %atomicrmw.end ; SOFTFP-NOLSE-NEXT: mov w0, w21 ; SOFTFP-NOLSE-NEXT: mov w1, w23 ; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload @@ -775,7 +636,7 @@ define <2 x bfloat> @test_atomicrmw_fmax_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; NOLSE-NEXT: fmov s1, w10 ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s0, w9 -; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start +; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 ; NOLSE-NEXT: ldaxr w9, [x0] ; NOLSE-NEXT: fmov s2, w9 @@ -803,7 +664,7 @@ define <2 x bfloat> @test_atomicrmw_fmax_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; NOLSE-NEXT: mov v3.h[1], v2.h[0] ; NOLSE-NEXT: fmov w10, s3 ; NOLSE-NEXT: stlxr w11, w10, [x0] -; NOLSE-NEXT: cbnz w11, .LBB8_1 +; NOLSE-NEXT: cbnz w11, .LBB7_1 ; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, x9 ; NOLSE-NEXT: ret @@ -820,7 +681,7 @@ define <2 x bfloat> @test_atomicrmw_fmax_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; LSE-NEXT: fmov s2, w10 ; LSE-NEXT: lsl w9, w9, #16 ; LSE-NEXT: fmov s1, w9 -; LSE-NEXT: .LBB8_1: // %atomicrmw.start +; LSE-NEXT: .LBB7_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 ; LSE-NEXT: mov h3, v0.h[1] ; LSE-NEXT: fmov w10, s0 @@ -850,7 +711,7 @@ define <2 x bfloat> @test_atomicrmw_fmax_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; LSE-NEXT: casal w11, w10, [x0] ; LSE-NEXT: fmov s0, w11 ; LSE-NEXT: cmp w11, w9 -; LSE-NEXT: b.ne .LBB8_1 +; LSE-NEXT: b.ne .LBB7_1 ; LSE-NEXT: // %bb.2: // %atomicrmw.end ; LSE-NEXT: // kill: def $d0 killed $d0 killed $q0 ; LSE-NEXT: ret @@ -867,15 +728,15 @@ define <2 x bfloat> @test_atomicrmw_fmax_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; SOFTFP-NOLSE-NEXT: lsl w22, w8, #16 ; SOFTFP-NOLSE-NEXT: mov x19, x0 ; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill -; SOFTFP-NOLSE-NEXT: b .LBB8_2 -; SOFTFP-NOLSE-NEXT: .LBB8_1: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 +; SOFTFP-NOLSE-NEXT: b .LBB7_2 +; SOFTFP-NOLSE-NEXT: .LBB7_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 ; SOFTFP-NOLSE-NEXT: lsr w1, w21, #16 ; SOFTFP-NOLSE-NEXT: cmp w21, w23 -; SOFTFP-NOLSE-NEXT: b.eq .LBB8_5 -; SOFTFP-NOLSE-NEXT: .LBB8_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: b.eq .LBB7_5 +; SOFTFP-NOLSE-NEXT: .LBB7_2: // %atomicrmw.start ; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 -; SOFTFP-NOLSE-NEXT: // Child Loop BB8_3 Depth 2 +; SOFTFP-NOLSE-NEXT: // Child Loop BB7_3 Depth 2 ; SOFTFP-NOLSE-NEXT: lsl w23, w1, #16 ; SOFTFP-NOLSE-NEXT: mov w1, w20 ; SOFTFP-NOLSE-NEXT: mov w0, w23 @@ -888,18 +749,18 @@ define <2 x bfloat> @test_atomicrmw_fmax_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 ; SOFTFP-NOLSE-NEXT: bfxil w23, w21, #0, #16 ; SOFTFP-NOLSE-NEXT: bfi w0, w24, #16, #16 -; SOFTFP-NOLSE-NEXT: .LBB8_3: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 +; SOFTFP-NOLSE-NEXT: .LBB7_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 ; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 ; SOFTFP-NOLSE-NEXT: ldaxr w21, [x19] ; SOFTFP-NOLSE-NEXT: cmp w21, w23 -; SOFTFP-NOLSE-NEXT: b.ne .LBB8_1 +; SOFTFP-NOLSE-NEXT: b.ne .LBB7_1 ; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 ; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x19] -; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB8_3 -; SOFTFP-NOLSE-NEXT: b .LBB8_1 -; SOFTFP-NOLSE-NEXT: .LBB8_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB7_3 +; SOFTFP-NOLSE-NEXT: b .LBB7_1 +; SOFTFP-NOLSE-NEXT: .LBB7_5: // %atomicrmw.end ; SOFTFP-NOLSE-NEXT: mov w0, w21 ; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload ; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload @@ -913,14 +774,14 @@ define <2 x bfloat> @test_atomicrmw_fmax_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf define <2 x float> @test_atomicrmw_fmax_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmax_v2f32_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start +; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 ; NOLSE-NEXT: ldaxr x8, [x0] ; NOLSE-NEXT: fmov d1, x8 ; NOLSE-NEXT: fmaxnm v2.2s, v1.2s, v0.2s ; NOLSE-NEXT: fmov x8, d2 ; NOLSE-NEXT: stlxr w9, x8, [x0] -; NOLSE-NEXT: cbnz w9, .LBB9_1 +; NOLSE-NEXT: cbnz w9, .LBB8_1 ; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, d1 ; NOLSE-NEXT: ret @@ -928,7 +789,7 @@ define <2 x float> @test_atomicrmw_fmax_v2f32_seq_cst_align8(ptr %ptr, <2 x floa ; LSE-LABEL: test_atomicrmw_fmax_v2f32_seq_cst_align8: ; LSE: // %bb.0: ; LSE-NEXT: ldr d1, [x0] -; LSE-NEXT: .LBB9_1: // %atomicrmw.start +; LSE-NEXT: .LBB8_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 ; LSE-NEXT: fmaxnm v2.2s, v1.2s, v0.2s ; LSE-NEXT: fmov x8, d1 @@ -937,7 +798,7 @@ define <2 x float> @test_atomicrmw_fmax_v2f32_seq_cst_align8(ptr %ptr, <2 x floa ; LSE-NEXT: casal x10, x9, [x0] ; LSE-NEXT: fmov d1, x10 ; LSE-NEXT: cmp x10, x8 -; LSE-NEXT: b.ne .LBB9_1 +; LSE-NEXT: b.ne .LBB8_1 ; LSE-NEXT: // %bb.2: // %atomicrmw.end ; LSE-NEXT: fmov d0, d1 ; LSE-NEXT: ret @@ -952,16 +813,16 @@ define <2 x float> @test_atomicrmw_fmax_v2f32_seq_cst_align8(ptr %ptr, <2 x floa ; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill ; SOFTFP-NOLSE-NEXT: mov w19, w2 ; SOFTFP-NOLSE-NEXT: mov x20, x0 -; SOFTFP-NOLSE-NEXT: b .LBB9_2 -; SOFTFP-NOLSE-NEXT: .LBB9_1: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 +; SOFTFP-NOLSE-NEXT: b .LBB8_2 +; SOFTFP-NOLSE-NEXT: .LBB8_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 ; SOFTFP-NOLSE-NEXT: lsr x22, x23, #32 ; SOFTFP-NOLSE-NEXT: cmp x23, x8 ; SOFTFP-NOLSE-NEXT: // kill: def $w22 killed $w22 killed $x22 def $x22 -; SOFTFP-NOLSE-NEXT: b.eq .LBB9_5 -; SOFTFP-NOLSE-NEXT: .LBB9_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: b.eq .LBB8_5 +; SOFTFP-NOLSE-NEXT: .LBB8_2: // %atomicrmw.start ; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 -; SOFTFP-NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; SOFTFP-NOLSE-NEXT: // Child Loop BB8_3 Depth 2 ; SOFTFP-NOLSE-NEXT: mov w0, w22 ; SOFTFP-NOLSE-NEXT: mov w1, w19 ; SOFTFP-NOLSE-NEXT: bl fmaxf @@ -973,18 +834,18 @@ define <2 x float> @test_atomicrmw_fmax_v2f32_seq_cst_align8(ptr %ptr, <2 x floa ; SOFTFP-NOLSE-NEXT: mov w9, w0 ; SOFTFP-NOLSE-NEXT: orr x9, x9, x24, lsl #32 ; SOFTFP-NOLSE-NEXT: orr x8, x8, x22, lsl #32 -; SOFTFP-NOLSE-NEXT: .LBB9_3: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 +; SOFTFP-NOLSE-NEXT: .LBB8_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 ; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 ; SOFTFP-NOLSE-NEXT: ldaxr x23, [x20] ; SOFTFP-NOLSE-NEXT: cmp x23, x8 -; SOFTFP-NOLSE-NEXT: b.ne .LBB9_1 +; SOFTFP-NOLSE-NEXT: b.ne .LBB8_1 ; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 ; SOFTFP-NOLSE-NEXT: stlxr wzr, x9, [x20] -; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB9_3 -; SOFTFP-NOLSE-NEXT: b .LBB9_1 -; SOFTFP-NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB8_3 +; SOFTFP-NOLSE-NEXT: b .LBB8_1 +; SOFTFP-NOLSE-NEXT: .LBB8_5: // %atomicrmw.end ; SOFTFP-NOLSE-NEXT: mov w0, w23 ; SOFTFP-NOLSE-NEXT: mov w1, w22 ; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload @@ -999,7 +860,7 @@ define <2 x float> @test_atomicrmw_fmax_v2f32_seq_cst_align8(ptr %ptr, <2 x floa define <2 x double> @test_atomicrmw_fmax_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmax_v2f64_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: .LBB10_1: // %atomicrmw.start +; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 ; NOLSE-NEXT: ldaxp x8, x9, [x0] ; NOLSE-NEXT: fmov d1, x8 @@ -1008,7 +869,7 @@ define <2 x double> @test_atomicrmw_fmax_v2f64_seq_cst_align8(ptr %ptr, <2 x dou ; NOLSE-NEXT: mov x8, v2.d[1] ; NOLSE-NEXT: fmov x9, d2 ; NOLSE-NEXT: stlxp w10, x9, x8, [x0] -; NOLSE-NEXT: cbnz w10, .LBB10_1 +; NOLSE-NEXT: cbnz w10, .LBB9_1 ; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: mov v0.16b, v1.16b ; NOLSE-NEXT: ret @@ -1016,7 +877,7 @@ define <2 x double> @test_atomicrmw_fmax_v2f64_seq_cst_align8(ptr %ptr, <2 x dou ; LSE-LABEL: test_atomicrmw_fmax_v2f64_seq_cst_align8: ; LSE: // %bb.0: ; LSE-NEXT: ldr q1, [x0] -; LSE-NEXT: .LBB10_1: // %atomicrmw.start +; LSE-NEXT: .LBB9_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 ; LSE-NEXT: fmaxnm v2.2d, v1.2d, v0.2d ; LSE-NEXT: mov x3, v1.d[1] @@ -1030,7 +891,7 @@ define <2 x double> @test_atomicrmw_fmax_v2f64_seq_cst_align8(ptr %ptr, <2 x dou ; LSE-NEXT: cmp x7, x3 ; LSE-NEXT: ccmp x6, x2, #0, eq ; LSE-NEXT: mov v1.d[1], x7 -; LSE-NEXT: b.ne .LBB10_1 +; LSE-NEXT: b.ne .LBB9_1 ; LSE-NEXT: // %bb.2: // %atomicrmw.end ; LSE-NEXT: mov v0.16b, v1.16b ; LSE-NEXT: ret @@ -1045,15 +906,15 @@ define <2 x double> @test_atomicrmw_fmax_v2f64_seq_cst_align8(ptr %ptr, <2 x dou ; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill ; SOFTFP-NOLSE-NEXT: mov x21, x2 ; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill -; SOFTFP-NOLSE-NEXT: b .LBB10_2 -; SOFTFP-NOLSE-NEXT: .LBB10_1: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 +; SOFTFP-NOLSE-NEXT: b .LBB9_2 +; SOFTFP-NOLSE-NEXT: .LBB9_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 ; SOFTFP-NOLSE-NEXT: cmp x1, x22 ; SOFTFP-NOLSE-NEXT: ccmp x0, x23, #0, eq -; SOFTFP-NOLSE-NEXT: b.eq .LBB10_6 -; SOFTFP-NOLSE-NEXT: .LBB10_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: b.eq .LBB9_6 +; SOFTFP-NOLSE-NEXT: .LBB9_2: // %atomicrmw.start ; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 -; SOFTFP-NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; SOFTFP-NOLSE-NEXT: // Child Loop BB9_3 Depth 2 ; SOFTFP-NOLSE-NEXT: mov x22, x1 ; SOFTFP-NOLSE-NEXT: mov x23, x0 ; SOFTFP-NOLSE-NEXT: mov x0, x1 @@ -1064,26 +925,26 @@ define <2 x double> @test_atomicrmw_fmax_v2f64_seq_cst_align8(ptr %ptr, <2 x dou ; SOFTFP-NOLSE-NEXT: mov x1, x21 ; SOFTFP-NOLSE-NEXT: bl fmax ; SOFTFP-NOLSE-NEXT: mov x8, x0 -; SOFTFP-NOLSE-NEXT: .LBB10_3: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 +; SOFTFP-NOLSE-NEXT: .LBB9_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 ; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 ; SOFTFP-NOLSE-NEXT: ldaxp x0, x1, [x20] ; SOFTFP-NOLSE-NEXT: cmp x0, x23 ; SOFTFP-NOLSE-NEXT: cset w9, ne ; SOFTFP-NOLSE-NEXT: cmp x1, x22 ; SOFTFP-NOLSE-NEXT: cinc w9, w9, ne -; SOFTFP-NOLSE-NEXT: cbz w9, .LBB10_5 +; SOFTFP-NOLSE-NEXT: cbz w9, .LBB9_5 ; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 ; SOFTFP-NOLSE-NEXT: stlxp w9, x0, x1, [x20] -; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB10_3 -; SOFTFP-NOLSE-NEXT: b .LBB10_1 -; SOFTFP-NOLSE-NEXT: .LBB10_5: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB9_3 +; SOFTFP-NOLSE-NEXT: b .LBB9_1 +; SOFTFP-NOLSE-NEXT: .LBB9_5: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 ; SOFTFP-NOLSE-NEXT: stlxp w9, x8, x24, [x20] -; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB10_3 -; SOFTFP-NOLSE-NEXT: b .LBB10_1 -; SOFTFP-NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB9_3 +; SOFTFP-NOLSE-NEXT: b .LBB9_1 +; SOFTFP-NOLSE-NEXT: .LBB9_6: // %atomicrmw.end ; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload ; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload ; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll index 71765f435d94..6b7d2df04446 100644 --- a/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll +++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll @@ -3,8 +3,7 @@ ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+lse -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=LSE %s ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=-lse,-fp-armv8 -O1 < %s | FileCheck -check-prefix=SOFTFP-NOLSE %s -; FIXME: Windows hosts assigns stack slots to different offsets for some reason. -; UNSUPPORTED: system-windows +; FIXME: Restore test of fp128 case define half @test_atomicrmw_fmin_f16_seq_cst_align2(ptr %ptr, half %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align2: @@ -508,144 +507,6 @@ define double @test_atomicrmw_fmin_f32_seq_cst_align8(ptr %ptr, double %value) # ret double %res } -define fp128 @test_atomicrmw_fmin_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 { -; NOLSE-LABEL: test_atomicrmw_fmin_fp128_seq_cst_align16: -; NOLSE: // %bb.0: -; NOLSE-NEXT: sub sp, sp, #96 -; NOLSE-NEXT: ldr q1, [x0] -; NOLSE-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill -; NOLSE-NEXT: mov x19, x0 -; NOLSE-NEXT: str q0, [sp] // 16-byte Folded Spill -; NOLSE-NEXT: b .LBB6_2 -; NOLSE-NEXT: .LBB6_1: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 -; NOLSE-NEXT: stp x12, x13, [sp, #32] -; NOLSE-NEXT: cmp x13, x10 -; NOLSE-NEXT: ldr q1, [sp, #32] -; NOLSE-NEXT: ccmp x12, x11, #0, eq -; NOLSE-NEXT: b.eq .LBB6_6 -; NOLSE-NEXT: .LBB6_2: // %atomicrmw.start -; NOLSE-NEXT: // =>This Loop Header: Depth=1 -; NOLSE-NEXT: // Child Loop BB6_3 Depth 2 -; NOLSE-NEXT: mov v0.16b, v1.16b -; NOLSE-NEXT: str q1, [sp, #16] // 16-byte Folded Spill -; NOLSE-NEXT: ldr q1, [sp] // 16-byte Folded Reload -; NOLSE-NEXT: bl fminl -; NOLSE-NEXT: str q0, [sp, #48] -; NOLSE-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload -; NOLSE-NEXT: ldp x9, x8, [sp, #48] -; NOLSE-NEXT: str q0, [sp, #64] -; NOLSE-NEXT: ldp x11, x10, [sp, #64] -; NOLSE-NEXT: .LBB6_3: // %atomicrmw.start -; NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 -; NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; NOLSE-NEXT: ldaxp x12, x13, [x19] -; NOLSE-NEXT: cmp x12, x11 -; NOLSE-NEXT: cset w14, ne -; NOLSE-NEXT: cmp x13, x10 -; NOLSE-NEXT: cinc w14, w14, ne -; NOLSE-NEXT: cbz w14, .LBB6_5 -; NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 -; NOLSE-NEXT: stlxp w14, x12, x13, [x19] -; NOLSE-NEXT: cbnz w14, .LBB6_3 -; NOLSE-NEXT: b .LBB6_1 -; NOLSE-NEXT: .LBB6_5: // %atomicrmw.start -; NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 -; NOLSE-NEXT: stlxp w14, x9, x8, [x19] -; NOLSE-NEXT: cbnz w14, .LBB6_3 -; NOLSE-NEXT: b .LBB6_1 -; NOLSE-NEXT: .LBB6_6: // %atomicrmw.end -; NOLSE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload -; NOLSE-NEXT: mov v0.16b, v1.16b -; NOLSE-NEXT: add sp, sp, #96 -; NOLSE-NEXT: ret -; -; LSE-LABEL: test_atomicrmw_fmin_fp128_seq_cst_align16: -; LSE: // %bb.0: -; LSE-NEXT: sub sp, sp, #96 -; LSE-NEXT: ldr q1, [x0] -; LSE-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill -; LSE-NEXT: mov x19, x0 -; LSE-NEXT: str q0, [sp] // 16-byte Folded Spill -; LSE-NEXT: .LBB6_1: // %atomicrmw.start -; LSE-NEXT: // =>This Inner Loop Header: Depth=1 -; LSE-NEXT: mov v0.16b, v1.16b -; LSE-NEXT: str q1, [sp, #16] // 16-byte Folded Spill -; LSE-NEXT: ldr q1, [sp] // 16-byte Folded Reload -; LSE-NEXT: bl fminl -; LSE-NEXT: str q0, [sp, #48] -; LSE-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload -; LSE-NEXT: ldp x0, x1, [sp, #48] -; LSE-NEXT: str q0, [sp, #64] -; LSE-NEXT: ldp x2, x3, [sp, #64] -; LSE-NEXT: mov x4, x2 -; LSE-NEXT: mov x5, x3 -; LSE-NEXT: caspal x4, x5, x0, x1, [x19] -; LSE-NEXT: stp x4, x5, [sp, #32] -; LSE-NEXT: cmp x5, x3 -; LSE-NEXT: ldr q1, [sp, #32] -; LSE-NEXT: ccmp x4, x2, #0, eq -; LSE-NEXT: b.ne .LBB6_1 -; LSE-NEXT: // %bb.2: // %atomicrmw.end -; LSE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload -; LSE-NEXT: mov v0.16b, v1.16b -; LSE-NEXT: add sp, sp, #96 -; LSE-NEXT: ret -; -; SOFTFP-NOLSE-LABEL: test_atomicrmw_fmin_fp128_seq_cst_align16: -; SOFTFP-NOLSE: // %bb.0: -; SOFTFP-NOLSE-NEXT: stp x30, x23, [sp, #-48]! // 16-byte Folded Spill -; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill -; SOFTFP-NOLSE-NEXT: mov x20, x0 -; SOFTFP-NOLSE-NEXT: mov x19, x3 -; SOFTFP-NOLSE-NEXT: ldp x0, x1, [x0] -; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill -; SOFTFP-NOLSE-NEXT: mov x21, x2 -; SOFTFP-NOLSE-NEXT: b .LBB6_2 -; SOFTFP-NOLSE-NEXT: .LBB6_1: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 -; SOFTFP-NOLSE-NEXT: cmp x1, x22 -; SOFTFP-NOLSE-NEXT: ccmp x0, x23, #0, eq -; SOFTFP-NOLSE-NEXT: b.eq .LBB6_6 -; SOFTFP-NOLSE-NEXT: .LBB6_2: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 -; SOFTFP-NOLSE-NEXT: // Child Loop BB6_3 Depth 2 -; SOFTFP-NOLSE-NEXT: mov x2, x21 -; SOFTFP-NOLSE-NEXT: mov x3, x19 -; SOFTFP-NOLSE-NEXT: mov x22, x1 -; SOFTFP-NOLSE-NEXT: mov x23, x0 -; SOFTFP-NOLSE-NEXT: bl fminl -; SOFTFP-NOLSE-NEXT: mov x8, x0 -; SOFTFP-NOLSE-NEXT: mov x9, x1 -; SOFTFP-NOLSE-NEXT: .LBB6_3: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 -; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 -; SOFTFP-NOLSE-NEXT: ldaxp x0, x1, [x20] -; SOFTFP-NOLSE-NEXT: cmp x0, x23 -; SOFTFP-NOLSE-NEXT: cset w10, ne -; SOFTFP-NOLSE-NEXT: cmp x1, x22 -; SOFTFP-NOLSE-NEXT: cinc w10, w10, ne -; SOFTFP-NOLSE-NEXT: cbz w10, .LBB6_5 -; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 -; SOFTFP-NOLSE-NEXT: stlxp w10, x0, x1, [x20] -; SOFTFP-NOLSE-NEXT: cbnz w10, .LBB6_3 -; SOFTFP-NOLSE-NEXT: b .LBB6_1 -; SOFTFP-NOLSE-NEXT: .LBB6_5: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 -; SOFTFP-NOLSE-NEXT: stlxp w10, x8, x9, [x20] -; SOFTFP-NOLSE-NEXT: cbnz w10, .LBB6_3 -; SOFTFP-NOLSE-NEXT: b .LBB6_1 -; SOFTFP-NOLSE-NEXT: .LBB6_6: // %atomicrmw.end -; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload -; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload -; SOFTFP-NOLSE-NEXT: ldp x30, x23, [sp], #48 // 16-byte Folded Reload -; SOFTFP-NOLSE-NEXT: ret - %res = atomicrmw fmin ptr %ptr, fp128 %value seq_cst, align 16 - ret fp128 %res -} - define <2 x half> @test_atomicrmw_fmin_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmin_v2f16_seq_cst_align4: ; NOLSE: // %bb.0: @@ -653,7 +514,7 @@ define <2 x half> @test_atomicrmw_fmin_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; NOLSE-NEXT: mov h1, v0.h[1] ; NOLSE-NEXT: fcvt s0, h0 ; NOLSE-NEXT: fcvt s1, h1 -; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start +; NOLSE-NEXT: .LBB6_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 ; NOLSE-NEXT: ldaxr w8, [x0] ; NOLSE-NEXT: fmov s2, w8 @@ -667,7 +528,7 @@ define <2 x half> @test_atomicrmw_fmin_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; NOLSE-NEXT: mov v2.h[1], v3.h[0] ; NOLSE-NEXT: fmov w9, s2 ; NOLSE-NEXT: stlxr w10, w9, [x0] -; NOLSE-NEXT: cbnz w10, .LBB7_1 +; NOLSE-NEXT: cbnz w10, .LBB6_1 ; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, x8 ; NOLSE-NEXT: ret @@ -679,7 +540,7 @@ define <2 x half> @test_atomicrmw_fmin_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; LSE-NEXT: fcvt s2, h0 ; LSE-NEXT: ldr s0, [x0] ; LSE-NEXT: fcvt s1, h1 -; LSE-NEXT: .LBB7_1: // %atomicrmw.start +; LSE-NEXT: .LBB6_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 ; LSE-NEXT: mov h3, v0.h[1] ; LSE-NEXT: fcvt s4, h0 @@ -695,7 +556,7 @@ define <2 x half> @test_atomicrmw_fmin_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; LSE-NEXT: casal w10, w9, [x0] ; LSE-NEXT: fmov s0, w10 ; LSE-NEXT: cmp w10, w8 -; LSE-NEXT: b.ne .LBB7_1 +; LSE-NEXT: b.ne .LBB6_1 ; LSE-NEXT: // %bb.2: // %atomicrmw.end ; LSE-NEXT: // kill: def $d0 killed $d0 killed $q0 ; LSE-NEXT: ret @@ -711,16 +572,16 @@ define <2 x half> @test_atomicrmw_fmin_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill ; SOFTFP-NOLSE-NEXT: mov w19, w2 ; SOFTFP-NOLSE-NEXT: mov x20, x0 -; SOFTFP-NOLSE-NEXT: b .LBB7_2 -; SOFTFP-NOLSE-NEXT: .LBB7_1: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 +; SOFTFP-NOLSE-NEXT: b .LBB6_2 +; SOFTFP-NOLSE-NEXT: .LBB6_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=1 ; SOFTFP-NOLSE-NEXT: lsr w23, w8, #16 ; SOFTFP-NOLSE-NEXT: cmp w8, w21 ; SOFTFP-NOLSE-NEXT: mov w21, w8 -; SOFTFP-NOLSE-NEXT: b.eq .LBB7_5 -; SOFTFP-NOLSE-NEXT: .LBB7_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: b.eq .LBB6_5 +; SOFTFP-NOLSE-NEXT: .LBB6_2: // %atomicrmw.start ; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 -; SOFTFP-NOLSE-NEXT: // Child Loop BB7_3 Depth 2 +; SOFTFP-NOLSE-NEXT: // Child Loop BB6_3 Depth 2 ; SOFTFP-NOLSE-NEXT: and w0, w19, #0xffff ; SOFTFP-NOLSE-NEXT: bl __gnu_h2f_ieee ; SOFTFP-NOLSE-NEXT: mov w24, w0 @@ -740,18 +601,18 @@ define <2 x half> @test_atomicrmw_fmin_v2f16_seq_cst_align4(ptr %ptr, <2 x half> ; SOFTFP-NOLSE-NEXT: bl __gnu_f2h_ieee ; SOFTFP-NOLSE-NEXT: bfi w21, w23, #16, #16 ; SOFTFP-NOLSE-NEXT: bfi w0, w24, #16, #16 -; SOFTFP-NOLSE-NEXT: .LBB7_3: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 +; SOFTFP-NOLSE-NEXT: .LBB6_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB6_2 Depth=1 ; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 ; SOFTFP-NOLSE-NEXT: ldaxr w8, [x20] ; SOFTFP-NOLSE-NEXT: cmp w8, w21 -; SOFTFP-NOLSE-NEXT: b.ne .LBB7_1 +; SOFTFP-NOLSE-NEXT: b.ne .LBB6_1 ; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB6_3 Depth=2 ; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x20] -; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB7_3 -; SOFTFP-NOLSE-NEXT: b .LBB7_1 -; SOFTFP-NOLSE-NEXT: .LBB7_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB6_3 +; SOFTFP-NOLSE-NEXT: b .LBB6_1 +; SOFTFP-NOLSE-NEXT: .LBB6_5: // %atomicrmw.end ; SOFTFP-NOLSE-NEXT: mov w0, w21 ; SOFTFP-NOLSE-NEXT: mov w1, w23 ; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload @@ -775,7 +636,7 @@ define <2 x bfloat> @test_atomicrmw_fmin_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; NOLSE-NEXT: fmov s1, w10 ; NOLSE-NEXT: lsl w9, w9, #16 ; NOLSE-NEXT: fmov s0, w9 -; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start +; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 ; NOLSE-NEXT: ldaxr w9, [x0] ; NOLSE-NEXT: fmov s2, w9 @@ -803,7 +664,7 @@ define <2 x bfloat> @test_atomicrmw_fmin_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; NOLSE-NEXT: mov v3.h[1], v2.h[0] ; NOLSE-NEXT: fmov w10, s3 ; NOLSE-NEXT: stlxr w11, w10, [x0] -; NOLSE-NEXT: cbnz w11, .LBB8_1 +; NOLSE-NEXT: cbnz w11, .LBB7_1 ; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, x9 ; NOLSE-NEXT: ret @@ -820,7 +681,7 @@ define <2 x bfloat> @test_atomicrmw_fmin_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; LSE-NEXT: fmov s2, w10 ; LSE-NEXT: lsl w9, w9, #16 ; LSE-NEXT: fmov s1, w9 -; LSE-NEXT: .LBB8_1: // %atomicrmw.start +; LSE-NEXT: .LBB7_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 ; LSE-NEXT: mov h3, v0.h[1] ; LSE-NEXT: fmov w10, s0 @@ -850,7 +711,7 @@ define <2 x bfloat> @test_atomicrmw_fmin_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; LSE-NEXT: casal w11, w10, [x0] ; LSE-NEXT: fmov s0, w11 ; LSE-NEXT: cmp w11, w9 -; LSE-NEXT: b.ne .LBB8_1 +; LSE-NEXT: b.ne .LBB7_1 ; LSE-NEXT: // %bb.2: // %atomicrmw.end ; LSE-NEXT: // kill: def $d0 killed $d0 killed $q0 ; LSE-NEXT: ret @@ -867,15 +728,15 @@ define <2 x bfloat> @test_atomicrmw_fmin_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; SOFTFP-NOLSE-NEXT: lsl w22, w8, #16 ; SOFTFP-NOLSE-NEXT: mov x19, x0 ; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill -; SOFTFP-NOLSE-NEXT: b .LBB8_2 -; SOFTFP-NOLSE-NEXT: .LBB8_1: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 +; SOFTFP-NOLSE-NEXT: b .LBB7_2 +; SOFTFP-NOLSE-NEXT: .LBB7_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=1 ; SOFTFP-NOLSE-NEXT: lsr w1, w21, #16 ; SOFTFP-NOLSE-NEXT: cmp w21, w23 -; SOFTFP-NOLSE-NEXT: b.eq .LBB8_5 -; SOFTFP-NOLSE-NEXT: .LBB8_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: b.eq .LBB7_5 +; SOFTFP-NOLSE-NEXT: .LBB7_2: // %atomicrmw.start ; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 -; SOFTFP-NOLSE-NEXT: // Child Loop BB8_3 Depth 2 +; SOFTFP-NOLSE-NEXT: // Child Loop BB7_3 Depth 2 ; SOFTFP-NOLSE-NEXT: lsl w23, w1, #16 ; SOFTFP-NOLSE-NEXT: mov w1, w20 ; SOFTFP-NOLSE-NEXT: mov w0, w23 @@ -888,18 +749,18 @@ define <2 x bfloat> @test_atomicrmw_fmin_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf ; SOFTFP-NOLSE-NEXT: bl __truncsfbf2 ; SOFTFP-NOLSE-NEXT: bfxil w23, w21, #0, #16 ; SOFTFP-NOLSE-NEXT: bfi w0, w24, #16, #16 -; SOFTFP-NOLSE-NEXT: .LBB8_3: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 +; SOFTFP-NOLSE-NEXT: .LBB7_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB7_2 Depth=1 ; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 ; SOFTFP-NOLSE-NEXT: ldaxr w21, [x19] ; SOFTFP-NOLSE-NEXT: cmp w21, w23 -; SOFTFP-NOLSE-NEXT: b.ne .LBB8_1 +; SOFTFP-NOLSE-NEXT: b.ne .LBB7_1 ; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB7_3 Depth=2 ; SOFTFP-NOLSE-NEXT: stlxr wzr, w0, [x19] -; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB8_3 -; SOFTFP-NOLSE-NEXT: b .LBB8_1 -; SOFTFP-NOLSE-NEXT: .LBB8_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB7_3 +; SOFTFP-NOLSE-NEXT: b .LBB7_1 +; SOFTFP-NOLSE-NEXT: .LBB7_5: // %atomicrmw.end ; SOFTFP-NOLSE-NEXT: mov w0, w21 ; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload ; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload @@ -913,14 +774,14 @@ define <2 x bfloat> @test_atomicrmw_fmin_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf define <2 x float> @test_atomicrmw_fmin_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmin_v2f32_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start +; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 ; NOLSE-NEXT: ldaxr x8, [x0] ; NOLSE-NEXT: fmov d1, x8 ; NOLSE-NEXT: fminnm v2.2s, v1.2s, v0.2s ; NOLSE-NEXT: fmov x8, d2 ; NOLSE-NEXT: stlxr w9, x8, [x0] -; NOLSE-NEXT: cbnz w9, .LBB9_1 +; NOLSE-NEXT: cbnz w9, .LBB8_1 ; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: fmov d0, d1 ; NOLSE-NEXT: ret @@ -928,7 +789,7 @@ define <2 x float> @test_atomicrmw_fmin_v2f32_seq_cst_align8(ptr %ptr, <2 x floa ; LSE-LABEL: test_atomicrmw_fmin_v2f32_seq_cst_align8: ; LSE: // %bb.0: ; LSE-NEXT: ldr d1, [x0] -; LSE-NEXT: .LBB9_1: // %atomicrmw.start +; LSE-NEXT: .LBB8_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 ; LSE-NEXT: fminnm v2.2s, v1.2s, v0.2s ; LSE-NEXT: fmov x8, d1 @@ -937,7 +798,7 @@ define <2 x float> @test_atomicrmw_fmin_v2f32_seq_cst_align8(ptr %ptr, <2 x floa ; LSE-NEXT: casal x10, x9, [x0] ; LSE-NEXT: fmov d1, x10 ; LSE-NEXT: cmp x10, x8 -; LSE-NEXT: b.ne .LBB9_1 +; LSE-NEXT: b.ne .LBB8_1 ; LSE-NEXT: // %bb.2: // %atomicrmw.end ; LSE-NEXT: fmov d0, d1 ; LSE-NEXT: ret @@ -952,16 +813,16 @@ define <2 x float> @test_atomicrmw_fmin_v2f32_seq_cst_align8(ptr %ptr, <2 x floa ; SOFTFP-NOLSE-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill ; SOFTFP-NOLSE-NEXT: mov w19, w2 ; SOFTFP-NOLSE-NEXT: mov x20, x0 -; SOFTFP-NOLSE-NEXT: b .LBB9_2 -; SOFTFP-NOLSE-NEXT: .LBB9_1: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 +; SOFTFP-NOLSE-NEXT: b .LBB8_2 +; SOFTFP-NOLSE-NEXT: .LBB8_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=1 ; SOFTFP-NOLSE-NEXT: lsr x22, x23, #32 ; SOFTFP-NOLSE-NEXT: cmp x23, x8 ; SOFTFP-NOLSE-NEXT: // kill: def $w22 killed $w22 killed $x22 def $x22 -; SOFTFP-NOLSE-NEXT: b.eq .LBB9_5 -; SOFTFP-NOLSE-NEXT: .LBB9_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: b.eq .LBB8_5 +; SOFTFP-NOLSE-NEXT: .LBB8_2: // %atomicrmw.start ; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 -; SOFTFP-NOLSE-NEXT: // Child Loop BB9_3 Depth 2 +; SOFTFP-NOLSE-NEXT: // Child Loop BB8_3 Depth 2 ; SOFTFP-NOLSE-NEXT: mov w0, w22 ; SOFTFP-NOLSE-NEXT: mov w1, w19 ; SOFTFP-NOLSE-NEXT: bl fminf @@ -973,18 +834,18 @@ define <2 x float> @test_atomicrmw_fmin_v2f32_seq_cst_align8(ptr %ptr, <2 x floa ; SOFTFP-NOLSE-NEXT: mov w9, w0 ; SOFTFP-NOLSE-NEXT: orr x9, x9, x24, lsl #32 ; SOFTFP-NOLSE-NEXT: orr x8, x8, x22, lsl #32 -; SOFTFP-NOLSE-NEXT: .LBB9_3: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 +; SOFTFP-NOLSE-NEXT: .LBB8_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB8_2 Depth=1 ; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 ; SOFTFP-NOLSE-NEXT: ldaxr x23, [x20] ; SOFTFP-NOLSE-NEXT: cmp x23, x8 -; SOFTFP-NOLSE-NEXT: b.ne .LBB9_1 +; SOFTFP-NOLSE-NEXT: b.ne .LBB8_1 ; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB8_3 Depth=2 ; SOFTFP-NOLSE-NEXT: stlxr wzr, x9, [x20] -; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB9_3 -; SOFTFP-NOLSE-NEXT: b .LBB9_1 -; SOFTFP-NOLSE-NEXT: .LBB9_5: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: cbnz wzr, .LBB8_3 +; SOFTFP-NOLSE-NEXT: b .LBB8_1 +; SOFTFP-NOLSE-NEXT: .LBB8_5: // %atomicrmw.end ; SOFTFP-NOLSE-NEXT: mov w0, w23 ; SOFTFP-NOLSE-NEXT: mov w1, w22 ; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload @@ -999,7 +860,7 @@ define <2 x float> @test_atomicrmw_fmin_v2f32_seq_cst_align8(ptr %ptr, <2 x floa define <2 x double> @test_atomicrmw_fmin_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 { ; NOLSE-LABEL: test_atomicrmw_fmin_v2f64_seq_cst_align8: ; NOLSE: // %bb.0: -; NOLSE-NEXT: .LBB10_1: // %atomicrmw.start +; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start ; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 ; NOLSE-NEXT: ldaxp x8, x9, [x0] ; NOLSE-NEXT: fmov d1, x8 @@ -1008,7 +869,7 @@ define <2 x double> @test_atomicrmw_fmin_v2f64_seq_cst_align8(ptr %ptr, <2 x dou ; NOLSE-NEXT: mov x8, v2.d[1] ; NOLSE-NEXT: fmov x9, d2 ; NOLSE-NEXT: stlxp w10, x9, x8, [x0] -; NOLSE-NEXT: cbnz w10, .LBB10_1 +; NOLSE-NEXT: cbnz w10, .LBB9_1 ; NOLSE-NEXT: // %bb.2: // %atomicrmw.end ; NOLSE-NEXT: mov v0.16b, v1.16b ; NOLSE-NEXT: ret @@ -1016,7 +877,7 @@ define <2 x double> @test_atomicrmw_fmin_v2f64_seq_cst_align8(ptr %ptr, <2 x dou ; LSE-LABEL: test_atomicrmw_fmin_v2f64_seq_cst_align8: ; LSE: // %bb.0: ; LSE-NEXT: ldr q1, [x0] -; LSE-NEXT: .LBB10_1: // %atomicrmw.start +; LSE-NEXT: .LBB9_1: // %atomicrmw.start ; LSE-NEXT: // =>This Inner Loop Header: Depth=1 ; LSE-NEXT: fminnm v2.2d, v1.2d, v0.2d ; LSE-NEXT: mov x3, v1.d[1] @@ -1030,7 +891,7 @@ define <2 x double> @test_atomicrmw_fmin_v2f64_seq_cst_align8(ptr %ptr, <2 x dou ; LSE-NEXT: cmp x7, x3 ; LSE-NEXT: ccmp x6, x2, #0, eq ; LSE-NEXT: mov v1.d[1], x7 -; LSE-NEXT: b.ne .LBB10_1 +; LSE-NEXT: b.ne .LBB9_1 ; LSE-NEXT: // %bb.2: // %atomicrmw.end ; LSE-NEXT: mov v0.16b, v1.16b ; LSE-NEXT: ret @@ -1045,15 +906,15 @@ define <2 x double> @test_atomicrmw_fmin_v2f64_seq_cst_align8(ptr %ptr, <2 x dou ; SOFTFP-NOLSE-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill ; SOFTFP-NOLSE-NEXT: mov x21, x2 ; SOFTFP-NOLSE-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill -; SOFTFP-NOLSE-NEXT: b .LBB10_2 -; SOFTFP-NOLSE-NEXT: .LBB10_1: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_2 Depth=1 +; SOFTFP-NOLSE-NEXT: b .LBB9_2 +; SOFTFP-NOLSE-NEXT: .LBB9_1: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=1 ; SOFTFP-NOLSE-NEXT: cmp x1, x22 ; SOFTFP-NOLSE-NEXT: ccmp x0, x23, #0, eq -; SOFTFP-NOLSE-NEXT: b.eq .LBB10_6 -; SOFTFP-NOLSE-NEXT: .LBB10_2: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: b.eq .LBB9_6 +; SOFTFP-NOLSE-NEXT: .LBB9_2: // %atomicrmw.start ; SOFTFP-NOLSE-NEXT: // =>This Loop Header: Depth=1 -; SOFTFP-NOLSE-NEXT: // Child Loop BB10_3 Depth 2 +; SOFTFP-NOLSE-NEXT: // Child Loop BB9_3 Depth 2 ; SOFTFP-NOLSE-NEXT: mov x22, x1 ; SOFTFP-NOLSE-NEXT: mov x23, x0 ; SOFTFP-NOLSE-NEXT: mov x0, x1 @@ -1064,26 +925,26 @@ define <2 x double> @test_atomicrmw_fmin_v2f64_seq_cst_align8(ptr %ptr, <2 x dou ; SOFTFP-NOLSE-NEXT: mov x1, x21 ; SOFTFP-NOLSE-NEXT: bl fmin ; SOFTFP-NOLSE-NEXT: mov x8, x0 -; SOFTFP-NOLSE-NEXT: .LBB10_3: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // Parent Loop BB10_2 Depth=1 +; SOFTFP-NOLSE-NEXT: .LBB9_3: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // Parent Loop BB9_2 Depth=1 ; SOFTFP-NOLSE-NEXT: // => This Inner Loop Header: Depth=2 ; SOFTFP-NOLSE-NEXT: ldaxp x0, x1, [x20] ; SOFTFP-NOLSE-NEXT: cmp x0, x23 ; SOFTFP-NOLSE-NEXT: cset w9, ne ; SOFTFP-NOLSE-NEXT: cmp x1, x22 ; SOFTFP-NOLSE-NEXT: cinc w9, w9, ne -; SOFTFP-NOLSE-NEXT: cbz w9, .LBB10_5 +; SOFTFP-NOLSE-NEXT: cbz w9, .LBB9_5 ; SOFTFP-NOLSE-NEXT: // %bb.4: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 ; SOFTFP-NOLSE-NEXT: stlxp w9, x0, x1, [x20] -; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB10_3 -; SOFTFP-NOLSE-NEXT: b .LBB10_1 -; SOFTFP-NOLSE-NEXT: .LBB10_5: // %atomicrmw.start -; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB10_3 Depth=2 +; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB9_3 +; SOFTFP-NOLSE-NEXT: b .LBB9_1 +; SOFTFP-NOLSE-NEXT: .LBB9_5: // %atomicrmw.start +; SOFTFP-NOLSE-NEXT: // in Loop: Header=BB9_3 Depth=2 ; SOFTFP-NOLSE-NEXT: stlxp w9, x8, x24, [x20] -; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB10_3 -; SOFTFP-NOLSE-NEXT: b .LBB10_1 -; SOFTFP-NOLSE-NEXT: .LBB10_6: // %atomicrmw.end +; SOFTFP-NOLSE-NEXT: cbnz w9, .LBB9_3 +; SOFTFP-NOLSE-NEXT: b .LBB9_1 +; SOFTFP-NOLSE-NEXT: .LBB9_6: // %atomicrmw.end ; SOFTFP-NOLSE-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload ; SOFTFP-NOLSE-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload ; SOFTFP-NOLSE-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload -- Gitee