1//=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// AArch64 Atomic operand code-gen constructs.
11//
12//===----------------------------------------------------------------------===//
13
14//===----------------------------------
15// Atomic fences
16//===----------------------------------
17def : Pat<(atomic_fence (i64 4), (imm)), (DMB (i32 0x9))>;
18def : Pat<(atomic_fence (imm), (imm)), (DMB (i32 0xb))>;
19
20//===----------------------------------
21// Atomic loads
22//===----------------------------------
23
24// When they're actually atomic, only one addressing mode (GPR64sp) is
25// supported, but when they're relaxed and anything can be used, all the
26// standard modes would be valid and may give efficiency gains.
27
28// A atomic load operation that actually needs acquire semantics.
29class acquiring_load<PatFrag base>
30  : PatFrag<(ops node:$ptr), (base node:$ptr), [{
31  AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
32  assert(Ordering != AcquireRelease && "unexpected load ordering");
33  return Ordering == Acquire || Ordering == SequentiallyConsistent;
34}]>;
35
36// An atomic load operation that does not need either acquire or release
37// semantics.
38class relaxed_load<PatFrag base>
39  : PatFrag<(ops node:$ptr), (base node:$ptr), [{
40  AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
41  return Ordering == Monotonic || Ordering == Unordered;
42}]>;
43
44// 8-bit loads
45def : Pat<(acquiring_load<atomic_load_8>  GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
46def : Pat<(relaxed_load<atomic_load_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm,
47                                                     ro_Wextend8:$offset)),
48          (LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>;
49def : Pat<(relaxed_load<atomic_load_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm,
50                                                     ro_Xextend8:$offset)),
51          (LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>;
52def : Pat<(relaxed_load<atomic_load_8> (am_indexed8 GPR64sp:$Rn,
53                                                    uimm12s1:$offset)),
54          (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>;
55def : Pat<(relaxed_load<atomic_load_8>
56               (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
57          (LDURBBi GPR64sp:$Rn, simm9:$offset)>;
58
59// 16-bit loads
60def : Pat<(acquiring_load<atomic_load_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;
61def : Pat<(relaxed_load<atomic_load_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
62                                                       ro_Wextend16:$extend)),
63          (LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
64def : Pat<(relaxed_load<atomic_load_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
65                                                       ro_Xextend16:$extend)),
66          (LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
67def : Pat<(relaxed_load<atomic_load_16> (am_indexed16 GPR64sp:$Rn,
68                                                      uimm12s2:$offset)),
69          (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>;
70def : Pat<(relaxed_load<atomic_load_16>
71               (am_unscaled16 GPR64sp:$Rn, simm9:$offset)),
72          (LDURHHi GPR64sp:$Rn, simm9:$offset)>;
73
74// 32-bit loads
75def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>;
76def : Pat<(relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
77                                                       ro_Wextend32:$extend)),
78          (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
79def : Pat<(relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
80                                                       ro_Xextend32:$extend)),
81          (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
82def : Pat<(relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn,
83                                                      uimm12s4:$offset)),
84          (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>;
85def : Pat<(relaxed_load<atomic_load_32>
86               (am_unscaled32 GPR64sp:$Rn, simm9:$offset)),
87          (LDURWi GPR64sp:$Rn, simm9:$offset)>;
88
89// 64-bit loads
90def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>;
91def : Pat<(relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
92                                                       ro_Wextend64:$extend)),
93          (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
94def : Pat<(relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
95                                                       ro_Xextend64:$extend)),
96          (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
97def : Pat<(relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn,
98                                                      uimm12s8:$offset)),
99          (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>;
100def : Pat<(relaxed_load<atomic_load_64>
101               (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
102          (LDURXi GPR64sp:$Rn, simm9:$offset)>;
103
104//===----------------------------------
105// Atomic stores
106//===----------------------------------
107
108// When they're actually atomic, only one addressing mode (GPR64sp) is
109// supported, but when they're relaxed and anything can be used, all the
110// standard modes would be valid and may give efficiency gains.
111
112// A store operation that actually needs release semantics.
113class releasing_store<PatFrag base>
114  : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
115  AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
116  assert(Ordering != AcquireRelease && "unexpected store ordering");
117  return Ordering == Release || Ordering == SequentiallyConsistent;
118}]>;
119
120// An atomic store operation that doesn't actually need to be atomic on AArch64.
121class relaxed_store<PatFrag base>
122  : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
123  AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
124  return Ordering == Monotonic || Ordering == Unordered;
125}]>;
126
127// 8-bit stores
128def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val),
129          (STLRB GPR32:$val, GPR64sp:$ptr)>;
130def : Pat<(relaxed_store<atomic_store_8>
131               (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend),
132               GPR32:$val),
133          (STRBBroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend)>;
134def : Pat<(relaxed_store<atomic_store_8>
135               (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend),
136               GPR32:$val),
137          (STRBBroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend)>;
138def : Pat<(relaxed_store<atomic_store_8>
139               (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset), GPR32:$val),
140          (STRBBui GPR32:$val, GPR64sp:$Rn, uimm12s1:$offset)>;
141def : Pat<(relaxed_store<atomic_store_8>
142               (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
143          (STURBBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
144
145// 16-bit stores
146def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val),
147          (STLRH GPR32:$val, GPR64sp:$ptr)>;
148def : Pat<(relaxed_store<atomic_store_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
149                                                         ro_Wextend16:$extend),
150                                          GPR32:$val),
151          (STRHHroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
152def : Pat<(relaxed_store<atomic_store_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
153                                                         ro_Xextend16:$extend),
154                                          GPR32:$val),
155          (STRHHroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
156def : Pat<(relaxed_store<atomic_store_16>
157              (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset), GPR32:$val),
158          (STRHHui GPR32:$val, GPR64sp:$Rn, uimm12s2:$offset)>;
159def : Pat<(relaxed_store<atomic_store_16>
160               (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
161          (STURHHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
162
163// 32-bit stores
164def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val),
165          (STLRW GPR32:$val, GPR64sp:$ptr)>;
166def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
167                                                         ro_Wextend32:$extend),
168                                          GPR32:$val),
169          (STRWroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
170def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
171                                                         ro_Xextend32:$extend),
172                                          GPR32:$val),
173          (STRWroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
174def : Pat<(relaxed_store<atomic_store_32>
175              (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), GPR32:$val),
176          (STRWui GPR32:$val, GPR64sp:$Rn, uimm12s4:$offset)>;
177def : Pat<(relaxed_store<atomic_store_32>
178               (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
179          (STURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
180
181// 64-bit stores
182def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val),
183          (STLRX GPR64:$val, GPR64sp:$ptr)>;
184def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
185                                                         ro_Wextend16:$extend),
186                                          GPR64:$val),
187          (STRXroW GPR64:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
188def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
189                                                         ro_Xextend16:$extend),
190                                          GPR64:$val),
191          (STRXroX GPR64:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
192def : Pat<(relaxed_store<atomic_store_64>
193              (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), GPR64:$val),
194          (STRXui GPR64:$val, GPR64sp:$Rn, uimm12s8:$offset)>;
195def : Pat<(relaxed_store<atomic_store_64>
196               (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val),
197          (STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>;
198
199//===----------------------------------
200// Low-level exclusive operations
201//===----------------------------------
202
203// Load-exclusives.
204
205def ldxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
206  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
207}]>;
208
209def ldxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
210  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
211}]>;
212
213def ldxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
214  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
215}]>;
216
217def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
218  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
219}]>;
220
221def : Pat<(ldxr_1 GPR64sp:$addr),
222          (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
223def : Pat<(ldxr_2 GPR64sp:$addr),
224          (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
225def : Pat<(ldxr_4 GPR64sp:$addr),
226          (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
227def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>;
228
229def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff),
230          (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
231def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff),
232          (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
233def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff),
234          (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
235
236// Load-exclusives.
237
238def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
239  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
240}]>;
241
242def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
243  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
244}]>;
245
246def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
247  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
248}]>;
249
250def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
251  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
252}]>;
253
254def : Pat<(ldaxr_1 GPR64sp:$addr),
255          (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
256def : Pat<(ldaxr_2 GPR64sp:$addr),
257          (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
258def : Pat<(ldaxr_4 GPR64sp:$addr),
259          (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
260def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>;
261
262def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff),
263          (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
264def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff),
265          (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
266def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff),
267          (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
268
269// Store-exclusives.
270
271def stxr_1 : PatFrag<(ops node:$val, node:$ptr),
272                     (int_aarch64_stxr node:$val, node:$ptr), [{
273  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
274}]>;
275
276def stxr_2 : PatFrag<(ops node:$val, node:$ptr),
277                     (int_aarch64_stxr node:$val, node:$ptr), [{
278  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
279}]>;
280
281def stxr_4 : PatFrag<(ops node:$val, node:$ptr),
282                     (int_aarch64_stxr node:$val, node:$ptr), [{
283  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
284}]>;
285
286def stxr_8 : PatFrag<(ops node:$val, node:$ptr),
287                     (int_aarch64_stxr node:$val, node:$ptr), [{
288  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
289}]>;
290
291
292def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr),
293          (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
294def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr),
295          (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
296def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr),
297          (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
298def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr),
299          (STXRX GPR64:$val, GPR64sp:$addr)>;
300
301def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
302          (STXRB GPR32:$val, GPR64sp:$addr)>;
303def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
304          (STXRH GPR32:$val, GPR64sp:$addr)>;
305def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr),
306          (STXRW GPR32:$val, GPR64sp:$addr)>;
307
308def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
309          (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
310def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
311          (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
312def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
313          (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
314
315// Store-release-exclusives.
316
317def stlxr_1 : PatFrag<(ops node:$val, node:$ptr),
318                     (int_aarch64_stlxr node:$val, node:$ptr), [{
319  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
320}]>;
321
322def stlxr_2 : PatFrag<(ops node:$val, node:$ptr),
323                     (int_aarch64_stlxr node:$val, node:$ptr), [{
324  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
325}]>;
326
327def stlxr_4 : PatFrag<(ops node:$val, node:$ptr),
328                     (int_aarch64_stlxr node:$val, node:$ptr), [{
329  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
330}]>;
331
332def stlxr_8 : PatFrag<(ops node:$val, node:$ptr),
333                     (int_aarch64_stlxr node:$val, node:$ptr), [{
334  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
335}]>;
336
337
338def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr),
339          (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
340def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr),
341          (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
342def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr),
343          (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
344def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr),
345          (STLXRX GPR64:$val, GPR64sp:$addr)>;
346
347def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
348          (STLXRB GPR32:$val, GPR64sp:$addr)>;
349def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
350          (STLXRH GPR32:$val, GPR64sp:$addr)>;
351def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr),
352          (STLXRW GPR32:$val, GPR64sp:$addr)>;
353
354def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
355          (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
356def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
357          (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
358def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
359          (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
360
361
362// And clear exclusive.
363
364def : Pat<(int_aarch64_clrex), (CLREX 0xf)>;
365