1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SystemZTargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SystemZISelLowering.h"
14#include "SystemZCallingConv.h"
15#include "SystemZConstantPoolValue.h"
16#include "SystemZMachineFunctionInfo.h"
17#include "SystemZTargetMachine.h"
18#include "llvm/CodeGen/CallingConvLower.h"
19#include "llvm/CodeGen/ISDOpcodes.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
23#include "llvm/IR/GlobalAlias.h"
24#include "llvm/IR/IntrinsicInst.h"
25#include "llvm/IR/Intrinsics.h"
26#include "llvm/IR/IntrinsicsS390.h"
27#include "llvm/Support/CommandLine.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/KnownBits.h"
30#include <cctype>
31#include <optional>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "systemz-lower"
36
37// Temporarily let this be disabled by default until all known problems
38// related to argument extensions are fixed.
39static cl::opt<bool> EnableIntArgExtCheck(
40 "argext-abi-check", cl::init(Val: false),
41 cl::desc("Verify that narrow int args are properly extended per the "
42 "SystemZ ABI."));
43
44namespace {
45// Represents information about a comparison.
46struct Comparison {
47 Comparison(SDValue Op0In, SDValue Op1In, SDValue ChainIn)
48 : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
49 Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
50
51 // The operands to the comparison.
52 SDValue Op0, Op1;
53
54 // Chain if this is a strict floating-point comparison.
55 SDValue Chain;
56
57 // The opcode that should be used to compare Op0 and Op1.
58 unsigned Opcode;
59
60 // A SystemZICMP value. Only used for integer comparisons.
61 unsigned ICmpType;
62
63 // The mask of CC values that Opcode can produce.
64 unsigned CCValid;
65
66 // The mask of CC values for which the original condition is true.
67 unsigned CCMask;
68};
69} // end anonymous namespace
70
71// Classify VT as either 32 or 64 bit.
72static bool is32Bit(EVT VT) {
73 switch (VT.getSimpleVT().SimpleTy) {
74 case MVT::i32:
75 return true;
76 case MVT::i64:
77 return false;
78 default:
79 llvm_unreachable("Unsupported type");
80 }
81}
82
83// Return a version of MachineOperand that can be safely used before the
84// final use.
85static MachineOperand earlyUseOperand(MachineOperand Op) {
86 if (Op.isReg())
87 Op.setIsKill(false);
88 return Op;
89}
90
91SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
92 const SystemZSubtarget &STI)
93 : TargetLowering(TM), Subtarget(STI) {
94 MVT PtrVT = MVT::getIntegerVT(BitWidth: TM.getPointerSizeInBits(AS: 0));
95
96 auto *Regs = STI.getSpecialRegisters();
97
98 // Set up the register classes.
99 if (Subtarget.hasHighWord())
100 addRegisterClass(VT: MVT::i32, RC: &SystemZ::GRX32BitRegClass);
101 else
102 addRegisterClass(VT: MVT::i32, RC: &SystemZ::GR32BitRegClass);
103 addRegisterClass(VT: MVT::i64, RC: &SystemZ::GR64BitRegClass);
104 if (!useSoftFloat()) {
105 if (Subtarget.hasVector()) {
106 addRegisterClass(VT: MVT::f16, RC: &SystemZ::VR16BitRegClass);
107 addRegisterClass(VT: MVT::f32, RC: &SystemZ::VR32BitRegClass);
108 addRegisterClass(VT: MVT::f64, RC: &SystemZ::VR64BitRegClass);
109 } else {
110 addRegisterClass(VT: MVT::f16, RC: &SystemZ::FP16BitRegClass);
111 addRegisterClass(VT: MVT::f32, RC: &SystemZ::FP32BitRegClass);
112 addRegisterClass(VT: MVT::f64, RC: &SystemZ::FP64BitRegClass);
113 }
114 if (Subtarget.hasVectorEnhancements1())
115 addRegisterClass(VT: MVT::f128, RC: &SystemZ::VR128BitRegClass);
116 else
117 addRegisterClass(VT: MVT::f128, RC: &SystemZ::FP128BitRegClass);
118
119 if (Subtarget.hasVector()) {
120 addRegisterClass(VT: MVT::v16i8, RC: &SystemZ::VR128BitRegClass);
121 addRegisterClass(VT: MVT::v8i16, RC: &SystemZ::VR128BitRegClass);
122 addRegisterClass(VT: MVT::v4i32, RC: &SystemZ::VR128BitRegClass);
123 addRegisterClass(VT: MVT::v2i64, RC: &SystemZ::VR128BitRegClass);
124 addRegisterClass(VT: MVT::v4f32, RC: &SystemZ::VR128BitRegClass);
125 addRegisterClass(VT: MVT::v2f64, RC: &SystemZ::VR128BitRegClass);
126 }
127
128 if (Subtarget.hasVector())
129 addRegisterClass(VT: MVT::i128, RC: &SystemZ::VR128BitRegClass);
130 }
131
132 // Compute derived properties from the register classes
133 computeRegisterProperties(TRI: Subtarget.getRegisterInfo());
134
135 // Set up special registers.
136 setStackPointerRegisterToSaveRestore(Regs->getStackPointerRegister());
137
138 // TODO: It may be better to default to latency-oriented scheduling, however
139 // LLVM's current latency-oriented scheduler can't handle physreg definitions
140 // such as SystemZ has with CC, so set this to the register-pressure
141 // scheduler, because it can.
142 setSchedulingPreference(Sched::RegPressure);
143
144 setBooleanContents(ZeroOrOneBooleanContent);
145 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
146
147 setMaxAtomicSizeInBitsSupported(128);
148
149 // Instructions are strings of 2-byte aligned 2-byte values.
150 setMinFunctionAlignment(Align(2));
151 // For performance reasons we prefer 16-byte alignment.
152 setPrefFunctionAlignment(Align(16));
153
154 // Handle operations that are handled in a similar way for all types.
155 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
156 I <= MVT::LAST_FP_VALUETYPE;
157 ++I) {
158 MVT VT = MVT::SimpleValueType(I);
159 if (isTypeLegal(VT)) {
160 // Lower SET_CC into an IPM-based sequence.
161 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
162 setOperationAction(Op: ISD::STRICT_FSETCC, VT, Action: Custom);
163 setOperationAction(Op: ISD::STRICT_FSETCCS, VT, Action: Custom);
164
165 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE).
166 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
167
168 // Lower SELECT_CC and BR_CC into separate comparisons and branches.
169 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Custom);
170 setOperationAction(Op: ISD::BR_CC, VT, Action: Custom);
171 }
172 }
173
174 // Expand jump table branches as address arithmetic followed by an
175 // indirect jump.
176 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
177
178 // Expand BRCOND into a BR_CC (see above).
179 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Expand);
180
181 // Handle integer types except i128.
182 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
183 I <= MVT::LAST_INTEGER_VALUETYPE;
184 ++I) {
185 MVT VT = MVT::SimpleValueType(I);
186 if (isTypeLegal(VT) && VT != MVT::i128) {
187 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
188
189 // Expand individual DIV and REMs into DIVREMs.
190 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
191 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
192 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
193 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
194 setOperationAction(Op: ISD::SDIVREM, VT, Action: Custom);
195 setOperationAction(Op: ISD::UDIVREM, VT, Action: Custom);
196
197 // Support addition/subtraction with overflow.
198 setOperationAction(Op: ISD::SADDO, VT, Action: Custom);
199 setOperationAction(Op: ISD::SSUBO, VT, Action: Custom);
200
201 // Support addition/subtraction with carry.
202 setOperationAction(Op: ISD::UADDO, VT, Action: Custom);
203 setOperationAction(Op: ISD::USUBO, VT, Action: Custom);
204
205 // Support carry in as value rather than glue.
206 setOperationAction(Op: ISD::UADDO_CARRY, VT, Action: Custom);
207 setOperationAction(Op: ISD::USUBO_CARRY, VT, Action: Custom);
208
209 // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are
210 // available, or if the operand is constant.
211 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT, Action: Custom);
212
213 // Use POPCNT on z196 and above.
214 if (Subtarget.hasPopulationCount())
215 setOperationAction(Op: ISD::CTPOP, VT, Action: Custom);
216 else
217 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
218
219 // No special instructions for these.
220 setOperationAction(Op: ISD::CTTZ, VT, Action: Expand);
221 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
222
223 // Use *MUL_LOHI where possible instead of MULH*.
224 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
225 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
226 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Custom);
227 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Custom);
228
229 // The fp<=>i32/i64 conversions are all Legal except for f16 and for
230 // unsigned on z10 (only z196 and above have native support for
231 // unsigned conversions).
232 for (auto Op : {ISD::FP_TO_SINT, ISD::STRICT_FP_TO_SINT,
233 ISD::SINT_TO_FP, ISD::STRICT_SINT_TO_FP})
234 setOperationAction(Op, VT, Action: Custom);
235 for (auto Op : {ISD::FP_TO_UINT, ISD::STRICT_FP_TO_UINT})
236 setOperationAction(Op, VT, Action: Custom);
237 for (auto Op : {ISD::UINT_TO_FP, ISD::STRICT_UINT_TO_FP}) {
238 // Handle unsigned 32-bit input types as signed 64-bit types on z10.
239 auto OpAction =
240 (!Subtarget.hasFPExtension() && VT == MVT::i32) ? Promote : Custom;
241 setOperationAction(Op, VT, Action: OpAction);
242 }
243 }
244 }
245
246 // Handle i128 if legal.
247 if (isTypeLegal(VT: MVT::i128)) {
248 // No special instructions for these.
249 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i128, Action: Expand);
250 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i128, Action: Expand);
251 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i128, Action: Expand);
252 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i128, Action: Expand);
253 setOperationAction(Op: ISD::ROTR, VT: MVT::i128, Action: Expand);
254 setOperationAction(Op: ISD::ROTL, VT: MVT::i128, Action: Expand);
255
256 // We may be able to use VSLDB/VSLD/VSRD for these.
257 setOperationAction(Op: ISD::FSHL, VT: MVT::i128, Action: Custom);
258 setOperationAction(Op: ISD::FSHR, VT: MVT::i128, Action: Custom);
259
260 // No special instructions for these before z17.
261 if (!Subtarget.hasVectorEnhancements3()) {
262 setOperationAction(Op: ISD::MUL, VT: MVT::i128, Action: Expand);
263 setOperationAction(Op: ISD::MULHS, VT: MVT::i128, Action: Expand);
264 setOperationAction(Op: ISD::MULHU, VT: MVT::i128, Action: Expand);
265 setOperationAction(Op: ISD::SDIV, VT: MVT::i128, Action: Expand);
266 setOperationAction(Op: ISD::UDIV, VT: MVT::i128, Action: Expand);
267 setOperationAction(Op: ISD::SREM, VT: MVT::i128, Action: Expand);
268 setOperationAction(Op: ISD::UREM, VT: MVT::i128, Action: Expand);
269 setOperationAction(Op: ISD::CTLZ, VT: MVT::i128, Action: Expand);
270 setOperationAction(Op: ISD::CTTZ, VT: MVT::i128, Action: Expand);
271 } else {
272 // Even if we do have a legal 128-bit multiply, we do not
273 // want 64-bit multiply-high operations to use it.
274 setOperationAction(Op: ISD::MULHS, VT: MVT::i64, Action: Custom);
275 setOperationAction(Op: ISD::MULHU, VT: MVT::i64, Action: Custom);
276 }
277
278 // Support addition/subtraction with carry.
279 setOperationAction(Op: ISD::UADDO, VT: MVT::i128, Action: Custom);
280 setOperationAction(Op: ISD::USUBO, VT: MVT::i128, Action: Custom);
281 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i128, Action: Custom);
282 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i128, Action: Custom);
283
284 // Use VPOPCT and add up partial results.
285 setOperationAction(Op: ISD::CTPOP, VT: MVT::i128, Action: Custom);
286
287 // Additional instructions available with z17.
288 if (Subtarget.hasVectorEnhancements3()) {
289 setOperationAction(Op: ISD::ABS, VT: MVT::i128, Action: Legal);
290 }
291 }
292
293 // These need custom handling in order to handle the f16 conversions.
294 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i128, Action: Custom);
295 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i128, Action: Custom);
296 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i128, Action: Custom);
297 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i128, Action: Custom);
298 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i128, Action: Custom);
299 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i128, Action: Custom);
300 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i128, Action: Custom);
301 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i128, Action: Custom);
302
303 // Type legalization will convert 8- and 16-bit atomic operations into
304 // forms that operate on i32s (but still keeping the original memory VT).
305 // Lower them into full i32 operations.
306 setOperationAction(Op: ISD::ATOMIC_SWAP, VT: MVT::i32, Action: Custom);
307 setOperationAction(Op: ISD::ATOMIC_LOAD_ADD, VT: MVT::i32, Action: Custom);
308 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT: MVT::i32, Action: Custom);
309 setOperationAction(Op: ISD::ATOMIC_LOAD_AND, VT: MVT::i32, Action: Custom);
310 setOperationAction(Op: ISD::ATOMIC_LOAD_OR, VT: MVT::i32, Action: Custom);
311 setOperationAction(Op: ISD::ATOMIC_LOAD_XOR, VT: MVT::i32, Action: Custom);
312 setOperationAction(Op: ISD::ATOMIC_LOAD_NAND, VT: MVT::i32, Action: Custom);
313 setOperationAction(Op: ISD::ATOMIC_LOAD_MIN, VT: MVT::i32, Action: Custom);
314 setOperationAction(Op: ISD::ATOMIC_LOAD_MAX, VT: MVT::i32, Action: Custom);
315 setOperationAction(Op: ISD::ATOMIC_LOAD_UMIN, VT: MVT::i32, Action: Custom);
316 setOperationAction(Op: ISD::ATOMIC_LOAD_UMAX, VT: MVT::i32, Action: Custom);
317
318 // Whether or not i128 is not a legal type, we need to custom lower
319 // the atomic operations in order to exploit SystemZ instructions.
320 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i128, Action: Custom);
321 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i128, Action: Custom);
322 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f128, Action: Custom);
323 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f128, Action: Custom);
324
325 // Mark sign/zero extending atomic loads as legal, which will make
326 // DAGCombiner fold extensions into atomic loads if possible.
327 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i64,
328 MemVTs: {MVT::i8, MVT::i16, MVT::i32}, Action: Legal);
329 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i32,
330 MemVTs: {MVT::i8, MVT::i16}, Action: Legal);
331 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i16,
332 MemVT: MVT::i8, Action: Legal);
333
334 // We can use the CC result of compare-and-swap to implement
335 // the "success" result of ATOMIC_CMP_SWAP_WITH_SUCCESS.
336 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i32, Action: Custom);
337 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i64, Action: Custom);
338 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i128, Action: Custom);
339
340 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
341
342 // Traps are legal, as we will convert them to "j .+2".
343 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
344
345 // We have native support for a 64-bit CTLZ, via FLOGR.
346 setOperationAction(Op: ISD::CTLZ, VT: MVT::i32, Action: Promote);
347 setOperationAction(Op: ISD::CTLZ_ZERO_UNDEF, VT: MVT::i32, Action: Promote);
348 setOperationAction(Op: ISD::CTLZ, VT: MVT::i64, Action: Legal);
349
350 // On z17 we have native support for a 64-bit CTTZ.
351 if (Subtarget.hasMiscellaneousExtensions4()) {
352 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Promote);
353 setOperationAction(Op: ISD::CTTZ_ZERO_UNDEF, VT: MVT::i32, Action: Promote);
354 setOperationAction(Op: ISD::CTTZ, VT: MVT::i64, Action: Legal);
355 }
356
357 // On z15 we have native support for a 64-bit CTPOP.
358 if (Subtarget.hasMiscellaneousExtensions3()) {
359 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Promote);
360 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Legal);
361 }
362
363 // Give LowerOperation the chance to replace 64-bit ORs with subregs.
364 setOperationAction(Op: ISD::OR, VT: MVT::i64, Action: Custom);
365
366 // Expand 128 bit shifts without using a libcall.
367 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i64, Action: Expand);
368 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i64, Action: Expand);
369 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i64, Action: Expand);
370
371 // Also expand 256 bit shifts if i128 is a legal type.
372 if (isTypeLegal(VT: MVT::i128)) {
373 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i128, Action: Expand);
374 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i128, Action: Expand);
375 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i128, Action: Expand);
376 }
377
378 // Handle bitcast from fp128 to i128.
379 if (!isTypeLegal(VT: MVT::i128))
380 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
381
382 // We have native instructions for i8, i16 and i32 extensions, but not i1.
383 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
384 for (MVT VT : MVT::integer_valuetypes()) {
385 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
386 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
387 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
388 }
389
390 // Handle the various types of symbolic address.
391 setOperationAction(Op: ISD::ConstantPool, VT: PtrVT, Action: Custom);
392 setOperationAction(Op: ISD::GlobalAddress, VT: PtrVT, Action: Custom);
393 setOperationAction(Op: ISD::GlobalTLSAddress, VT: PtrVT, Action: Custom);
394 setOperationAction(Op: ISD::BlockAddress, VT: PtrVT, Action: Custom);
395 setOperationAction(Op: ISD::JumpTable, VT: PtrVT, Action: Custom);
396
397 // We need to handle dynamic allocations specially because of the
398 // 160-byte area at the bottom of the stack.
399 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: PtrVT, Action: Custom);
400 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: PtrVT, Action: Custom);
401
402 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Custom);
403 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Custom);
404
405 // Handle prefetches with PFD or PFDRL.
406 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
407
408 // Handle readcyclecounter with STCKF.
409 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Custom);
410
411 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
412 // Assume by default that all vector operations need to be expanded.
413 for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode)
414 if (getOperationAction(Op: Opcode, VT) == Legal)
415 setOperationAction(Op: Opcode, VT, Action: Expand);
416
417 // Likewise all truncating stores and extending loads.
418 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
419 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
420 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
421 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
422 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
423 }
424
425 if (isTypeLegal(VT)) {
426 // These operations are legal for anything that can be stored in a
427 // vector register, even if there is no native support for the format
428 // as such. In particular, we can do these for v4f32 even though there
429 // are no specific instructions for that format.
430 setOperationAction(Op: ISD::LOAD, VT, Action: Legal);
431 setOperationAction(Op: ISD::STORE, VT, Action: Legal);
432 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
433 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
434 setOperationAction(Op: ISD::UNDEF, VT, Action: Legal);
435
436 // Likewise, except that we need to replace the nodes with something
437 // more specific.
438 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
439 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
440 }
441 }
442
443 // Handle integer vector types.
444 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
445 if (isTypeLegal(VT)) {
446 // These operations have direct equivalents.
447 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Legal);
448 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Legal);
449 setOperationAction(Op: ISD::ADD, VT, Action: Legal);
450 setOperationAction(Op: ISD::SUB, VT, Action: Legal);
451 if (VT != MVT::v2i64 || Subtarget.hasVectorEnhancements3()) {
452 setOperationAction(Op: ISD::MUL, VT, Action: Legal);
453 setOperationAction(Op: ISD::MULHS, VT, Action: Legal);
454 setOperationAction(Op: ISD::MULHU, VT, Action: Legal);
455 }
456 if (Subtarget.hasVectorEnhancements3() &&
457 VT != MVT::v16i8 && VT != MVT::v8i16) {
458 setOperationAction(Op: ISD::SDIV, VT, Action: Legal);
459 setOperationAction(Op: ISD::UDIV, VT, Action: Legal);
460 setOperationAction(Op: ISD::SREM, VT, Action: Legal);
461 setOperationAction(Op: ISD::UREM, VT, Action: Legal);
462 }
463 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
464 setOperationAction(Op: ISD::AND, VT, Action: Legal);
465 setOperationAction(Op: ISD::OR, VT, Action: Legal);
466 setOperationAction(Op: ISD::XOR, VT, Action: Legal);
467 if (Subtarget.hasVectorEnhancements1())
468 setOperationAction(Op: ISD::CTPOP, VT, Action: Legal);
469 else
470 setOperationAction(Op: ISD::CTPOP, VT, Action: Custom);
471 setOperationAction(Op: ISD::CTTZ, VT, Action: Legal);
472 setOperationAction(Op: ISD::CTLZ, VT, Action: Legal);
473
474 // Convert a GPR scalar to a vector by inserting it into element 0.
475 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
476
477 // Use a series of unpacks for extensions.
478 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Custom);
479 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT, Action: Custom);
480
481 // Detect shifts/rotates by a scalar amount and convert them into
482 // V*_BY_SCALAR.
483 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
484 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
485 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
486 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
487
488 // Add ISD::VECREDUCE_ADD as custom in order to implement
489 // it with VZERO+VSUM
490 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
491
492 // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
493 // and inverting the result as necessary.
494 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
495 }
496 }
497
498 if (Subtarget.hasVector()) {
499 // There should be no need to check for float types other than v2f64
500 // since <2 x f32> isn't a legal type.
501 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
502 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2f64, Action: Legal);
503 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
504 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2f64, Action: Legal);
505 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
506 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2f64, Action: Legal);
507 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
508 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2f64, Action: Legal);
509
510 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
511 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2f64, Action: Legal);
512 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
513 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2f64, Action: Legal);
514 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
515 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2f64, Action: Legal);
516 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
517 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2f64, Action: Legal);
518 }
519
520 if (Subtarget.hasVectorEnhancements2()) {
521 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
522 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4f32, Action: Legal);
523 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
524 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4f32, Action: Legal);
525 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
526 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4f32, Action: Legal);
527 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
528 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4f32, Action: Legal);
529
530 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
531 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4f32, Action: Legal);
532 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
533 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4f32, Action: Legal);
534 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
535 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4f32, Action: Legal);
536 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
537 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4f32, Action: Legal);
538 }
539
540 // Handle floating-point types.
541 if (!useSoftFloat()) {
542 // Promote all f16 operations to float, with some exceptions below.
543 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
544 setOperationAction(Op: Opc, VT: MVT::f16, Action: Promote);
545 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Expand);
546 for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) {
547 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f16, Action: Expand);
548 setTruncStoreAction(ValVT: VT, MemVT: MVT::f16, Action: Expand);
549 }
550 for (auto Op : {ISD::LOAD, ISD::ATOMIC_LOAD, ISD::STORE, ISD::ATOMIC_STORE})
551 setOperationAction(Op, VT: MVT::f16, Action: Subtarget.hasVector() ? Legal : Custom);
552 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f16, Action: LibCall);
553 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: LibCall);
554 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
555 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f16, Action: Custom);
556 for (auto Op : {ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN})
557 setOperationAction(Op, VT: MVT::f16, Action: Legal);
558 }
559
560 for (unsigned I = MVT::FIRST_FP_VALUETYPE;
561 I <= MVT::LAST_FP_VALUETYPE;
562 ++I) {
563 MVT VT = MVT::SimpleValueType(I);
564 if (isTypeLegal(VT) && VT != MVT::f16) {
565 // We can use FI for FRINT.
566 setOperationAction(Op: ISD::FRINT, VT, Action: Legal);
567
568 // We can use the extended form of FI for other rounding operations.
569 if (Subtarget.hasFPExtension()) {
570 setOperationAction(Op: ISD::FNEARBYINT, VT, Action: Legal);
571 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
572 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
573 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
574 setOperationAction(Op: ISD::FROUND, VT, Action: Legal);
575 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
576 }
577
578 // No special instructions for these.
579 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
580 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
581 setOperationAction(Op: ISD::FSINCOS, VT, Action: Expand);
582 setOperationAction(Op: ISD::FREM, VT, Action: Expand);
583 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
584
585 // Special treatment.
586 setOperationAction(Op: ISD::IS_FPCLASS, VT, Action: Custom);
587
588 // Handle constrained floating-point operations.
589 setOperationAction(Op: ISD::STRICT_FADD, VT, Action: Legal);
590 setOperationAction(Op: ISD::STRICT_FSUB, VT, Action: Legal);
591 setOperationAction(Op: ISD::STRICT_FMUL, VT, Action: Legal);
592 setOperationAction(Op: ISD::STRICT_FDIV, VT, Action: Legal);
593 setOperationAction(Op: ISD::STRICT_FMA, VT, Action: Legal);
594 setOperationAction(Op: ISD::STRICT_FSQRT, VT, Action: Legal);
595 setOperationAction(Op: ISD::STRICT_FRINT, VT, Action: Legal);
596 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT, Action: Legal);
597 if (Subtarget.hasFPExtension()) {
598 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT, Action: Legal);
599 setOperationAction(Op: ISD::STRICT_FFLOOR, VT, Action: Legal);
600 setOperationAction(Op: ISD::STRICT_FCEIL, VT, Action: Legal);
601 setOperationAction(Op: ISD::STRICT_FTRUNC, VT, Action: Legal);
602 setOperationAction(Op: ISD::STRICT_FROUND, VT, Action: Legal);
603 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT, Action: Legal);
604 }
605
606 // Extension from f16 needs libcall.
607 setOperationAction(Op: ISD::FP_EXTEND, VT, Action: Custom);
608 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT, Action: Custom);
609 }
610 }
611
612 // Handle floating-point vector types.
613 if (Subtarget.hasVector()) {
614 // Scalar-to-vector conversion is just a subreg.
615 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Legal);
616 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2f64, Action: Legal);
617
618 // Some insertions and extractions can be done directly but others
619 // need to go via integers.
620 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
621 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
622 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
623 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
624
625 // These operations have direct equivalents.
626 setOperationAction(Op: ISD::FADD, VT: MVT::v2f64, Action: Legal);
627 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f64, Action: Legal);
628 setOperationAction(Op: ISD::FSUB, VT: MVT::v2f64, Action: Legal);
629 setOperationAction(Op: ISD::FMUL, VT: MVT::v2f64, Action: Legal);
630 setOperationAction(Op: ISD::FMA, VT: MVT::v2f64, Action: Legal);
631 setOperationAction(Op: ISD::FDIV, VT: MVT::v2f64, Action: Legal);
632 setOperationAction(Op: ISD::FABS, VT: MVT::v2f64, Action: Legal);
633 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f64, Action: Legal);
634 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f64, Action: Legal);
635 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v2f64, Action: Legal);
636 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f64, Action: Legal);
637 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f64, Action: Legal);
638 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f64, Action: Legal);
639 setOperationAction(Op: ISD::FROUND, VT: MVT::v2f64, Action: Legal);
640 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v2f64, Action: Legal);
641
642 // Handle constrained floating-point operations.
643 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v2f64, Action: Legal);
644 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v2f64, Action: Legal);
645 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v2f64, Action: Legal);
646 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v2f64, Action: Legal);
647 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v2f64, Action: Legal);
648 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v2f64, Action: Legal);
649 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v2f64, Action: Legal);
650 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v2f64, Action: Legal);
651 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v2f64, Action: Legal);
652 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v2f64, Action: Legal);
653 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v2f64, Action: Legal);
654 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v2f64, Action: Legal);
655 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT: MVT::v2f64, Action: Legal);
656
657 setOperationAction(Op: ISD::SETCC, VT: MVT::v2f64, Action: Custom);
658 setOperationAction(Op: ISD::SETCC, VT: MVT::v4f32, Action: Custom);
659 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::v2f64, Action: Custom);
660 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::v4f32, Action: Custom);
661 if (Subtarget.hasVectorEnhancements1()) {
662 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::v2f64, Action: Custom);
663 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::v4f32, Action: Custom);
664 }
665 }
666
667 // The vector enhancements facility 1 has instructions for these.
668 if (Subtarget.hasVectorEnhancements1()) {
669 setOperationAction(Op: ISD::FADD, VT: MVT::v4f32, Action: Legal);
670 setOperationAction(Op: ISD::FNEG, VT: MVT::v4f32, Action: Legal);
671 setOperationAction(Op: ISD::FSUB, VT: MVT::v4f32, Action: Legal);
672 setOperationAction(Op: ISD::FMUL, VT: MVT::v4f32, Action: Legal);
673 setOperationAction(Op: ISD::FMA, VT: MVT::v4f32, Action: Legal);
674 setOperationAction(Op: ISD::FDIV, VT: MVT::v4f32, Action: Legal);
675 setOperationAction(Op: ISD::FABS, VT: MVT::v4f32, Action: Legal);
676 setOperationAction(Op: ISD::FSQRT, VT: MVT::v4f32, Action: Legal);
677 setOperationAction(Op: ISD::FRINT, VT: MVT::v4f32, Action: Legal);
678 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v4f32, Action: Legal);
679 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v4f32, Action: Legal);
680 setOperationAction(Op: ISD::FCEIL, VT: MVT::v4f32, Action: Legal);
681 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v4f32, Action: Legal);
682 setOperationAction(Op: ISD::FROUND, VT: MVT::v4f32, Action: Legal);
683 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v4f32, Action: Legal);
684
685 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f64, Action: Legal);
686 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::f64, Action: Legal);
687 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f64, Action: Legal);
688 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::f64, Action: Legal);
689
690 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v2f64, Action: Legal);
691 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v2f64, Action: Legal);
692 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v2f64, Action: Legal);
693 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v2f64, Action: Legal);
694
695 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f32, Action: Legal);
696 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::f32, Action: Legal);
697 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f32, Action: Legal);
698 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::f32, Action: Legal);
699
700 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v4f32, Action: Legal);
701 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v4f32, Action: Legal);
702 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v4f32, Action: Legal);
703 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v4f32, Action: Legal);
704
705 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f128, Action: Legal);
706 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::f128, Action: Legal);
707 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f128, Action: Legal);
708 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::f128, Action: Legal);
709
710 // Handle constrained floating-point operations.
711 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v4f32, Action: Legal);
712 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v4f32, Action: Legal);
713 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v4f32, Action: Legal);
714 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v4f32, Action: Legal);
715 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v4f32, Action: Legal);
716 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v4f32, Action: Legal);
717 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v4f32, Action: Legal);
718 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v4f32, Action: Legal);
719 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v4f32, Action: Legal);
720 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v4f32, Action: Legal);
721 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v4f32, Action: Legal);
722 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v4f32, Action: Legal);
723 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT: MVT::v4f32, Action: Legal);
724 for (auto VT : { MVT::f32, MVT::f64, MVT::f128,
725 MVT::v4f32, MVT::v2f64 }) {
726 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT, Action: Legal);
727 setOperationAction(Op: ISD::STRICT_FMINNUM, VT, Action: Legal);
728 setOperationAction(Op: ISD::STRICT_FMAXIMUM, VT, Action: Legal);
729 setOperationAction(Op: ISD::STRICT_FMINIMUM, VT, Action: Legal);
730 }
731 }
732
733 // We only have fused f128 multiply-addition on vector registers.
734 if (!Subtarget.hasVectorEnhancements1()) {
735 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Expand);
736 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f128, Action: Expand);
737 }
738
739 // We don't have a copysign instruction on vector registers.
740 if (Subtarget.hasVectorEnhancements1())
741 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f128, Action: Expand);
742
743 // Needed so that we don't try to implement f128 constant loads using
744 // a load-and-extend of a f80 constant (in cases where the constant
745 // would fit in an f80).
746 for (MVT VT : MVT::fp_valuetypes())
747 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f80, Action: Expand);
748
749 // We don't have extending load instruction on vector registers.
750 if (Subtarget.hasVectorEnhancements1()) {
751 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
752 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
753 }
754
755 // Floating-point truncation and stores need to be done separately.
756 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
757 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
758 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
759
760 // We have 64-bit FPR<->GPR moves, but need special handling for
761 // 32-bit forms.
762 if (!Subtarget.hasVector()) {
763 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
764 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Custom);
765 }
766
767 // VASTART and VACOPY need to deal with the SystemZ-specific varargs
768 // structure, but VAEND is a no-op.
769 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
770 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Custom);
771 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
772
773 if (Subtarget.isTargetzOS()) {
774 // Handle address space casts between mixed sized pointers.
775 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i32, Action: Custom);
776 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i64, Action: Custom);
777 }
778
779 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
780
781 // Codes for which we want to perform some z-specific combinations.
782 setTargetDAGCombine({ISD::ZERO_EXTEND,
783 ISD::SIGN_EXTEND,
784 ISD::SIGN_EXTEND_INREG,
785 ISD::LOAD,
786 ISD::STORE,
787 ISD::VECTOR_SHUFFLE,
788 ISD::EXTRACT_VECTOR_ELT,
789 ISD::FP_ROUND,
790 ISD::STRICT_FP_ROUND,
791 ISD::FP_EXTEND,
792 ISD::SINT_TO_FP,
793 ISD::UINT_TO_FP,
794 ISD::STRICT_FP_EXTEND,
795 ISD::FCOPYSIGN,
796 ISD::BSWAP,
797 ISD::SETCC,
798 ISD::SRL,
799 ISD::SRA,
800 ISD::MUL,
801 ISD::SDIV,
802 ISD::UDIV,
803 ISD::SREM,
804 ISD::UREM,
805 ISD::INTRINSIC_VOID,
806 ISD::INTRINSIC_W_CHAIN});
807
808 // Handle intrinsics.
809 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
810 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
811
812 // We're not using SJLJ for exception handling, but they're implemented
813 // solely to support use of __builtin_setjmp / __builtin_longjmp.
814 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
815 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
816
817 // We want to use MVC in preference to even a single load/store pair.
818 MaxStoresPerMemcpy = Subtarget.hasVector() ? 2 : 0;
819 MaxStoresPerMemcpyOptSize = 0;
820
821 // The main memset sequence is a byte store followed by an MVC.
822 // Two STC or MV..I stores win over that, but the kind of fused stores
823 // generated by target-independent code don't when the byte value is
824 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better
825 // than "STC;MVC". Handle the choice in target-specific code instead.
826 MaxStoresPerMemset = Subtarget.hasVector() ? 2 : 0;
827 MaxStoresPerMemsetOptSize = 0;
828
829 // Default to having -disable-strictnode-mutation on
830 IsStrictFPEnabled = true;
831}
832
833bool SystemZTargetLowering::useSoftFloat() const {
834 return Subtarget.hasSoftFloat();
835}
836
837EVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL,
838 LLVMContext &, EVT VT) const {
839 if (!VT.isVector())
840 return MVT::i32;
841 return VT.changeVectorElementTypeToInteger();
842}
843
844bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(
845 const MachineFunction &MF, EVT VT) const {
846 if (useSoftFloat())
847 return false;
848
849 VT = VT.getScalarType();
850
851 if (!VT.isSimple())
852 return false;
853
854 switch (VT.getSimpleVT().SimpleTy) {
855 case MVT::f32:
856 case MVT::f64:
857 return true;
858 case MVT::f128:
859 return Subtarget.hasVectorEnhancements1();
860 default:
861 break;
862 }
863
864 return false;
865}
866
867// Return true if the constant can be generated with a vector instruction,
868// such as VGM, VGMB or VREPI.
869bool SystemZVectorConstantInfo::isVectorConstantLegal(
870 const SystemZSubtarget &Subtarget) {
871 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
872 if (!Subtarget.hasVector() ||
873 (isFP128 && !Subtarget.hasVectorEnhancements1()))
874 return false;
875
876 // Try using VECTOR GENERATE BYTE MASK. This is the architecturally-
877 // preferred way of creating all-zero and all-one vectors so give it
878 // priority over other methods below.
879 unsigned Mask = 0;
880 unsigned I = 0;
881 for (; I < SystemZ::VectorBytes; ++I) {
882 uint64_t Byte = IntBits.lshr(shiftAmt: I * 8).trunc(width: 8).getZExtValue();
883 if (Byte == 0xff)
884 Mask |= 1ULL << I;
885 else if (Byte != 0)
886 break;
887 }
888 if (I == SystemZ::VectorBytes) {
889 Opcode = SystemZISD::BYTE_MASK;
890 OpVals.push_back(Elt: Mask);
891 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 8), NumElements: 16);
892 return true;
893 }
894
895 if (SplatBitSize > 64)
896 return false;
897
898 auto TryValue = [&](uint64_t Value) -> bool {
899 // Try VECTOR REPLICATE IMMEDIATE
900 int64_t SignedValue = SignExtend64(X: Value, B: SplatBitSize);
901 if (isInt<16>(x: SignedValue)) {
902 OpVals.push_back(Elt: ((unsigned) SignedValue));
903 Opcode = SystemZISD::REPLICATE;
904 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplatBitSize),
905 NumElements: SystemZ::VectorBits / SplatBitSize);
906 return true;
907 }
908 // Try VECTOR GENERATE MASK
909 unsigned Start, End;
910 if (TII->isRxSBGMask(Mask: Value, BitSize: SplatBitSize, Start, End)) {
911 // isRxSBGMask returns the bit numbers for a full 64-bit value, with 0
912 // denoting 1 << 63 and 63 denoting 1. Convert them to bit numbers for
913 // an SplatBitSize value, so that 0 denotes 1 << (SplatBitSize-1).
914 OpVals.push_back(Elt: Start - (64 - SplatBitSize));
915 OpVals.push_back(Elt: End - (64 - SplatBitSize));
916 Opcode = SystemZISD::ROTATE_MASK;
917 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplatBitSize),
918 NumElements: SystemZ::VectorBits / SplatBitSize);
919 return true;
920 }
921 return false;
922 };
923
924 // First try assuming that any undefined bits above the highest set bit
925 // and below the lowest set bit are 1s. This increases the likelihood of
926 // being able to use a sign-extended element value in VECTOR REPLICATE
927 // IMMEDIATE or a wraparound mask in VECTOR GENERATE MASK.
928 uint64_t SplatBitsZ = SplatBits.getZExtValue();
929 uint64_t SplatUndefZ = SplatUndef.getZExtValue();
930 unsigned LowerBits = llvm::countr_zero(Val: SplatBitsZ);
931 unsigned UpperBits = llvm::countl_zero(Val: SplatBitsZ);
932 uint64_t Lower = SplatUndefZ & maskTrailingOnes<uint64_t>(N: LowerBits);
933 uint64_t Upper = SplatUndefZ & maskLeadingOnes<uint64_t>(N: UpperBits);
934 if (TryValue(SplatBitsZ | Upper | Lower))
935 return true;
936
937 // Now try assuming that any undefined bits between the first and
938 // last defined set bits are set. This increases the chances of
939 // using a non-wraparound mask.
940 uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
941 return TryValue(SplatBitsZ | Middle);
942}
943
944SystemZVectorConstantInfo::SystemZVectorConstantInfo(APInt IntImm) {
945 if (IntImm.isSingleWord()) {
946 IntBits = APInt(128, IntImm.getZExtValue());
947 IntBits <<= (SystemZ::VectorBits - IntImm.getBitWidth());
948 } else
949 IntBits = IntImm;
950 assert(IntBits.getBitWidth() == 128 && "Unsupported APInt.");
951
952 // Find the smallest splat.
953 SplatBits = IntImm;
954 unsigned Width = SplatBits.getBitWidth();
955 while (Width > 8) {
956 unsigned HalfSize = Width / 2;
957 APInt HighValue = SplatBits.lshr(shiftAmt: HalfSize).trunc(width: HalfSize);
958 APInt LowValue = SplatBits.trunc(width: HalfSize);
959
960 // If the two halves do not match, stop here.
961 if (HighValue != LowValue || 8 > HalfSize)
962 break;
963
964 SplatBits = HighValue;
965 Width = HalfSize;
966 }
967 SplatUndef = 0;
968 SplatBitSize = Width;
969}
970
971SystemZVectorConstantInfo::SystemZVectorConstantInfo(BuildVectorSDNode *BVN) {
972 assert(BVN->isConstant() && "Expected a constant BUILD_VECTOR");
973 bool HasAnyUndefs;
974
975 // Get IntBits by finding the 128 bit splat.
976 BVN->isConstantSplat(SplatValue&: IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 128,
977 isBigEndian: true);
978
979 // Get SplatBits by finding the 8 bit or greater splat.
980 BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 8,
981 isBigEndian: true);
982}
983
984bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
985 bool ForCodeSize) const {
986 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR.
987 if (Imm.isZero() || Imm.isNegZero())
988 return true;
989
990 return SystemZVectorConstantInfo(Imm).isVectorConstantLegal(Subtarget);
991}
992
993MachineBasicBlock *
994SystemZTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
995 MachineBasicBlock *MBB) const {
996 DebugLoc DL = MI.getDebugLoc();
997 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
998 const SystemZRegisterInfo *TRI = Subtarget.getRegisterInfo();
999
1000 MachineFunction *MF = MBB->getParent();
1001 MachineRegisterInfo &MRI = MF->getRegInfo();
1002
1003 const BasicBlock *BB = MBB->getBasicBlock();
1004 MachineFunction::iterator I = ++MBB->getIterator();
1005
1006 Register DstReg = MI.getOperand(i: 0).getReg();
1007 const TargetRegisterClass *RC = MRI.getRegClass(Reg: DstReg);
1008 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
1009 (void)TRI;
1010 Register MainDstReg = MRI.createVirtualRegister(RegClass: RC);
1011 Register RestoreDstReg = MRI.createVirtualRegister(RegClass: RC);
1012
1013 MVT PVT = getPointerTy(DL: MF->getDataLayout());
1014 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1015 // For v = setjmp(buf), we generate.
1016 // Algorithm:
1017 //
1018 // ---------
1019 // | thisMBB |
1020 // ---------
1021 // |
1022 // ------------------------
1023 // | |
1024 // ---------- ---------------
1025 // | mainMBB | | restoreMBB |
1026 // | v = 0 | | v = 1 |
1027 // ---------- ---------------
1028 // | |
1029 // -------------------------
1030 // |
1031 // -----------------------------
1032 // | sinkMBB |
1033 // | phi(v_mainMBB,v_restoreMBB) |
1034 // -----------------------------
1035 // thisMBB:
1036 // buf[FPOffset] = Frame Pointer if hasFP.
1037 // buf[LabelOffset] = restoreMBB <-- takes address of restoreMBB.
1038 // buf[BCOffset] = Backchain value if building with -mbackchain.
1039 // buf[SPOffset] = Stack Pointer.
1040 // buf[LPOffset] = We never write this slot with R13, gcc stores R13 always.
1041 // SjLjSetup restoreMBB
1042 // mainMBB:
1043 // v_main = 0
1044 // sinkMBB:
1045 // v = phi(v_main, v_restore)
1046 // restoreMBB:
1047 // v_restore = 1
1048
1049 MachineBasicBlock *ThisMBB = MBB;
1050 MachineBasicBlock *MainMBB = MF->CreateMachineBasicBlock(BB);
1051 MachineBasicBlock *SinkMBB = MF->CreateMachineBasicBlock(BB);
1052 MachineBasicBlock *RestoreMBB = MF->CreateMachineBasicBlock(BB);
1053
1054 MF->insert(MBBI: I, MBB: MainMBB);
1055 MF->insert(MBBI: I, MBB: SinkMBB);
1056 MF->push_back(MBB: RestoreMBB);
1057 RestoreMBB->setMachineBlockAddressTaken();
1058
1059 MachineInstrBuilder MIB;
1060
1061 // Transfer the remainder of BB and its successor edges to sinkMBB.
1062 SinkMBB->splice(Where: SinkMBB->begin(), Other: MBB,
1063 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
1064 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
1065
1066 // thisMBB:
1067 const int64_t FPOffset = 0; // Slot 1.
1068 const int64_t LabelOffset = 1 * PVT.getStoreSize(); // Slot 2.
1069 const int64_t BCOffset = 2 * PVT.getStoreSize(); // Slot 3.
1070 const int64_t SPOffset = 3 * PVT.getStoreSize(); // Slot 4.
1071
1072 // Buf address.
1073 Register BufReg = MI.getOperand(i: 1).getReg();
1074
1075 const TargetRegisterClass *PtrRC = getRegClassFor(VT: PVT);
1076 Register LabelReg = MRI.createVirtualRegister(RegClass: PtrRC);
1077
1078 // Prepare IP for longjmp.
1079 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LARL), DestReg: LabelReg)
1080 .addMBB(MBB: RestoreMBB);
1081 // Store IP for return from jmp, slot 2, offset = 1.
1082 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1083 .addReg(RegNo: LabelReg)
1084 .addReg(RegNo: BufReg)
1085 .addImm(Val: LabelOffset)
1086 .addReg(RegNo: 0);
1087
1088 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1089 bool HasFP = Subtarget.getFrameLowering()->hasFP(MF: *MF);
1090 if (HasFP) {
1091 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1092 .addReg(RegNo: SpecialRegs->getFramePointerRegister())
1093 .addReg(RegNo: BufReg)
1094 .addImm(Val: FPOffset)
1095 .addReg(RegNo: 0);
1096 }
1097
1098 // Store SP.
1099 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1100 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1101 .addReg(RegNo: BufReg)
1102 .addImm(Val: SPOffset)
1103 .addReg(RegNo: 0);
1104
1105 // Slot 3(Offset = 2) Backchain value (if building with -mbackchain).
1106 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1107 if (BackChain) {
1108 Register BCReg = MRI.createVirtualRegister(RegClass: PtrRC);
1109 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1110 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: BCReg)
1111 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1112 .addImm(Val: TFL->getBackchainOffset(MF&: *MF))
1113 .addReg(RegNo: 0);
1114
1115 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1116 .addReg(RegNo: BCReg)
1117 .addReg(RegNo: BufReg)
1118 .addImm(Val: BCOffset)
1119 .addReg(RegNo: 0);
1120 }
1121
1122 // Setup.
1123 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::EH_SjLj_Setup))
1124 .addMBB(MBB: RestoreMBB);
1125
1126 const SystemZRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1127 MIB.addRegMask(Mask: RegInfo->getNoPreservedMask());
1128
1129 ThisMBB->addSuccessor(Succ: MainMBB);
1130 ThisMBB->addSuccessor(Succ: RestoreMBB);
1131
1132 // mainMBB:
1133 BuildMI(BB: MainMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: MainDstReg).addImm(Val: 0);
1134 MainMBB->addSuccessor(Succ: SinkMBB);
1135
1136 // sinkMBB:
1137 BuildMI(BB&: *SinkMBB, I: SinkMBB->begin(), MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: DstReg)
1138 .addReg(RegNo: MainDstReg)
1139 .addMBB(MBB: MainMBB)
1140 .addReg(RegNo: RestoreDstReg)
1141 .addMBB(MBB: RestoreMBB);
1142
1143 // restoreMBB.
1144 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: RestoreDstReg).addImm(Val: 1);
1145 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: SinkMBB);
1146 RestoreMBB->addSuccessor(Succ: SinkMBB);
1147
1148 MI.eraseFromParent();
1149
1150 return SinkMBB;
1151}
1152
1153MachineBasicBlock *
1154SystemZTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
1155 MachineBasicBlock *MBB) const {
1156
1157 DebugLoc DL = MI.getDebugLoc();
1158 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1159
1160 MachineFunction *MF = MBB->getParent();
1161 MachineRegisterInfo &MRI = MF->getRegInfo();
1162
1163 MVT PVT = getPointerTy(DL: MF->getDataLayout());
1164 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1165 Register BufReg = MI.getOperand(i: 0).getReg();
1166 const TargetRegisterClass *RC = MRI.getRegClass(Reg: BufReg);
1167 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1168
1169 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
1170 Register BCReg = MRI.createVirtualRegister(RegClass: RC);
1171
1172 MachineInstrBuilder MIB;
1173
1174 const int64_t FPOffset = 0;
1175 const int64_t LabelOffset = 1 * PVT.getStoreSize();
1176 const int64_t BCOffset = 2 * PVT.getStoreSize();
1177 const int64_t SPOffset = 3 * PVT.getStoreSize();
1178 const int64_t LPOffset = 4 * PVT.getStoreSize();
1179
1180 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: Tmp)
1181 .addReg(RegNo: BufReg)
1182 .addImm(Val: LabelOffset)
1183 .addReg(RegNo: 0);
1184
1185 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG),
1186 DestReg: SpecialRegs->getFramePointerRegister())
1187 .addReg(RegNo: BufReg)
1188 .addImm(Val: FPOffset)
1189 .addReg(RegNo: 0);
1190
1191 // We are restoring R13 even though we never stored in setjmp from llvm,
1192 // as gcc always stores R13 in builtin_setjmp. We could have mixed code
1193 // gcc setjmp and llvm longjmp.
1194 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: SystemZ::R13D)
1195 .addReg(RegNo: BufReg)
1196 .addImm(Val: LPOffset)
1197 .addReg(RegNo: 0);
1198
1199 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1200 if (BackChain) {
1201 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: BCReg)
1202 .addReg(RegNo: BufReg)
1203 .addImm(Val: BCOffset)
1204 .addReg(RegNo: 0);
1205 }
1206
1207 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG),
1208 DestReg: SpecialRegs->getStackPointerRegister())
1209 .addReg(RegNo: BufReg)
1210 .addImm(Val: SPOffset)
1211 .addReg(RegNo: 0);
1212
1213 if (BackChain) {
1214 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1215 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1216 .addReg(RegNo: BCReg)
1217 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1218 .addImm(Val: TFL->getBackchainOffset(MF&: *MF))
1219 .addReg(RegNo: 0);
1220 }
1221
1222 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BR)).addReg(RegNo: Tmp);
1223
1224 MI.eraseFromParent();
1225 return MBB;
1226}
1227
1228/// Returns true if stack probing through inline assembly is requested.
1229bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
1230 // If the function specifically requests inline stack probes, emit them.
1231 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
1232 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
1233 "inline-asm";
1234 return false;
1235}
1236
1237TargetLowering::AtomicExpansionKind
1238SystemZTargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
1239 return AtomicExpansionKind::None;
1240}
1241
1242TargetLowering::AtomicExpansionKind
1243SystemZTargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
1244 return AtomicExpansionKind::None;
1245}
1246
1247TargetLowering::AtomicExpansionKind
1248SystemZTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
1249 // Don't expand subword operations as they require special treatment.
1250 if (RMW->getType()->isIntegerTy(Bitwidth: 8) || RMW->getType()->isIntegerTy(Bitwidth: 16))
1251 return AtomicExpansionKind::None;
1252
1253 // Don't expand if there is a target instruction available.
1254 if (Subtarget.hasInterlockedAccess1() &&
1255 (RMW->getType()->isIntegerTy(Bitwidth: 32) || RMW->getType()->isIntegerTy(Bitwidth: 64)) &&
1256 (RMW->getOperation() == AtomicRMWInst::BinOp::Add ||
1257 RMW->getOperation() == AtomicRMWInst::BinOp::Sub ||
1258 RMW->getOperation() == AtomicRMWInst::BinOp::And ||
1259 RMW->getOperation() == AtomicRMWInst::BinOp::Or ||
1260 RMW->getOperation() == AtomicRMWInst::BinOp::Xor))
1261 return AtomicExpansionKind::None;
1262
1263 return AtomicExpansionKind::CmpXChg;
1264}
1265
1266bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
1267 // We can use CGFI or CLGFI.
1268 return isInt<32>(x: Imm) || isUInt<32>(x: Imm);
1269}
1270
1271bool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const {
1272 // We can use ALGFI or SLGFI.
1273 return isUInt<32>(x: Imm) || isUInt<32>(x: -Imm);
1274}
1275
1276bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(
1277 EVT VT, unsigned, Align, MachineMemOperand::Flags, unsigned *Fast) const {
1278 // Unaligned accesses should never be slower than the expanded version.
1279 // We check specifically for aligned accesses in the few cases where
1280 // they are required.
1281 if (Fast)
1282 *Fast = 1;
1283 return true;
1284}
1285
1286bool SystemZTargetLowering::hasAndNot(SDValue Y) const {
1287 EVT VT = Y.getValueType();
1288
1289 // We can use NC(G)RK for types in GPRs ...
1290 if (VT == MVT::i32 || VT == MVT::i64)
1291 return Subtarget.hasMiscellaneousExtensions3();
1292
1293 // ... or VNC for types in VRs.
1294 if (VT.isVector() || VT == MVT::i128)
1295 return Subtarget.hasVector();
1296
1297 return false;
1298}
1299
1300// Information about the addressing mode for a memory access.
1301struct AddressingMode {
1302 // True if a long displacement is supported.
1303 bool LongDisplacement;
1304
1305 // True if use of index register is supported.
1306 bool IndexReg;
1307
1308 AddressingMode(bool LongDispl, bool IdxReg) :
1309 LongDisplacement(LongDispl), IndexReg(IdxReg) {}
1310};
1311
1312// Return the desired addressing mode for a Load which has only one use (in
1313// the same block) which is a Store.
1314static AddressingMode getLoadStoreAddrMode(bool HasVector,
1315 Type *Ty) {
1316 // With vector support a Load->Store combination may be combined to either
1317 // an MVC or vector operations and it seems to work best to allow the
1318 // vector addressing mode.
1319 if (HasVector)
1320 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1321
1322 // Otherwise only the MVC case is special.
1323 bool MVC = Ty->isIntegerTy(Bitwidth: 8);
1324 return AddressingMode(!MVC/*LongDispl*/, !MVC/*IdxReg*/);
1325}
1326
1327// Return the addressing mode which seems most desirable given an LLVM
1328// Instruction pointer.
1329static AddressingMode
1330supportedAddressingMode(Instruction *I, bool HasVector) {
1331 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
1332 switch (II->getIntrinsicID()) {
1333 default: break;
1334 case Intrinsic::memset:
1335 case Intrinsic::memmove:
1336 case Intrinsic::memcpy:
1337 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1338 }
1339 }
1340
1341 if (isa<LoadInst>(Val: I) && I->hasOneUse()) {
1342 auto *SingleUser = cast<Instruction>(Val: *I->user_begin());
1343 if (SingleUser->getParent() == I->getParent()) {
1344 if (isa<ICmpInst>(Val: SingleUser)) {
1345 if (auto *C = dyn_cast<ConstantInt>(Val: SingleUser->getOperand(i: 1)))
1346 if (C->getBitWidth() <= 64 &&
1347 (isInt<16>(x: C->getSExtValue()) || isUInt<16>(x: C->getZExtValue())))
1348 // Comparison of memory with 16 bit signed / unsigned immediate
1349 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1350 } else if (isa<StoreInst>(Val: SingleUser))
1351 // Load->Store
1352 return getLoadStoreAddrMode(HasVector, Ty: I->getType());
1353 }
1354 } else if (auto *StoreI = dyn_cast<StoreInst>(Val: I)) {
1355 if (auto *LoadI = dyn_cast<LoadInst>(Val: StoreI->getValueOperand()))
1356 if (LoadI->hasOneUse() && LoadI->getParent() == I->getParent())
1357 // Load->Store
1358 return getLoadStoreAddrMode(HasVector, Ty: LoadI->getType());
1359 }
1360
1361 if (HasVector && (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I))) {
1362
1363 // * Use LDE instead of LE/LEY for z13 to avoid partial register
1364 // dependencies (LDE only supports small offsets).
1365 // * Utilize the vector registers to hold floating point
1366 // values (vector load / store instructions only support small
1367 // offsets).
1368
1369 Type *MemAccessTy = (isa<LoadInst>(Val: I) ? I->getType() :
1370 I->getOperand(i: 0)->getType());
1371 bool IsFPAccess = MemAccessTy->isFloatingPointTy();
1372 bool IsVectorAccess = MemAccessTy->isVectorTy();
1373
1374 // A store of an extracted vector element will be combined into a VSTE type
1375 // instruction.
1376 if (!IsVectorAccess && isa<StoreInst>(Val: I)) {
1377 Value *DataOp = I->getOperand(i: 0);
1378 if (isa<ExtractElementInst>(Val: DataOp))
1379 IsVectorAccess = true;
1380 }
1381
1382 // A load which gets inserted into a vector element will be combined into a
1383 // VLE type instruction.
1384 if (!IsVectorAccess && isa<LoadInst>(Val: I) && I->hasOneUse()) {
1385 User *LoadUser = *I->user_begin();
1386 if (isa<InsertElementInst>(Val: LoadUser))
1387 IsVectorAccess = true;
1388 }
1389
1390 if (IsFPAccess || IsVectorAccess)
1391 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1392 }
1393
1394 return AddressingMode(true/*LongDispl*/, true/*IdxReg*/);
1395}
1396
1397bool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1398 const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const {
1399 // Punt on globals for now, although they can be used in limited
1400 // RELATIVE LONG cases.
1401 if (AM.BaseGV)
1402 return false;
1403
1404 // Require a 20-bit signed offset.
1405 if (!isInt<20>(x: AM.BaseOffs))
1406 return false;
1407
1408 bool RequireD12 =
1409 Subtarget.hasVector() && (Ty->isVectorTy() || Ty->isIntegerTy(Bitwidth: 128));
1410 AddressingMode SupportedAM(!RequireD12, true);
1411 if (I != nullptr)
1412 SupportedAM = supportedAddressingMode(I, HasVector: Subtarget.hasVector());
1413
1414 if (!SupportedAM.LongDisplacement && !isUInt<12>(x: AM.BaseOffs))
1415 return false;
1416
1417 if (!SupportedAM.IndexReg)
1418 // No indexing allowed.
1419 return AM.Scale == 0;
1420 else
1421 // Indexing is OK but no scale factor can be applied.
1422 return AM.Scale == 0 || AM.Scale == 1;
1423}
1424
1425bool SystemZTargetLowering::findOptimalMemOpLowering(
1426 LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
1427 const MemOp &Op, unsigned DstAS, unsigned SrcAS,
1428 const AttributeList &FuncAttributes) const {
1429 const int MVCFastLen = 16;
1430
1431 if (Limit != ~unsigned(0)) {
1432 // Don't expand Op into scalar loads/stores in these cases:
1433 if (Op.isMemcpy() && Op.allowOverlap() && Op.size() <= MVCFastLen)
1434 return false; // Small memcpy: Use MVC
1435 if (Op.isMemset() && Op.size() - 1 <= MVCFastLen)
1436 return false; // Small memset (first byte with STC/MVI): Use MVC
1437 if (Op.isZeroMemset())
1438 return false; // Memset zero: Use XC
1439 }
1440
1441 return TargetLowering::findOptimalMemOpLowering(Context, MemOps, Limit, Op,
1442 DstAS, SrcAS, FuncAttributes);
1443}
1444
1445EVT SystemZTargetLowering::getOptimalMemOpType(
1446 LLVMContext &Context, const MemOp &Op,
1447 const AttributeList &FuncAttributes) const {
1448 return Subtarget.hasVector() ? MVT::v2i64 : MVT::Other;
1449}
1450
1451bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const {
1452 if (!FromType->isIntegerTy() || !ToType->isIntegerTy())
1453 return false;
1454 unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue();
1455 unsigned ToBits = ToType->getPrimitiveSizeInBits().getFixedValue();
1456 return FromBits > ToBits;
1457}
1458
1459bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const {
1460 if (!FromVT.isInteger() || !ToVT.isInteger())
1461 return false;
1462 unsigned FromBits = FromVT.getFixedSizeInBits();
1463 unsigned ToBits = ToVT.getFixedSizeInBits();
1464 return FromBits > ToBits;
1465}
1466
1467//===----------------------------------------------------------------------===//
1468// Inline asm support
1469//===----------------------------------------------------------------------===//
1470
1471TargetLowering::ConstraintType
1472SystemZTargetLowering::getConstraintType(StringRef Constraint) const {
1473 if (Constraint.size() == 1) {
1474 switch (Constraint[0]) {
1475 case 'a': // Address register
1476 case 'd': // Data register (equivalent to 'r')
1477 case 'f': // Floating-point register
1478 case 'h': // High-part register
1479 case 'r': // General-purpose register
1480 case 'v': // Vector register
1481 return C_RegisterClass;
1482
1483 case 'Q': // Memory with base and unsigned 12-bit displacement
1484 case 'R': // Likewise, plus an index
1485 case 'S': // Memory with base and signed 20-bit displacement
1486 case 'T': // Likewise, plus an index
1487 case 'm': // Equivalent to 'T'.
1488 return C_Memory;
1489
1490 case 'I': // Unsigned 8-bit constant
1491 case 'J': // Unsigned 12-bit constant
1492 case 'K': // Signed 16-bit constant
1493 case 'L': // Signed 20-bit displacement (on all targets we support)
1494 case 'M': // 0x7fffffff
1495 return C_Immediate;
1496
1497 default:
1498 break;
1499 }
1500 } else if (Constraint.size() == 2 && Constraint[0] == 'Z') {
1501 switch (Constraint[1]) {
1502 case 'Q': // Address with base and unsigned 12-bit displacement
1503 case 'R': // Likewise, plus an index
1504 case 'S': // Address with base and signed 20-bit displacement
1505 case 'T': // Likewise, plus an index
1506 return C_Address;
1507
1508 default:
1509 break;
1510 }
1511 }
1512 return TargetLowering::getConstraintType(Constraint);
1513}
1514
1515TargetLowering::ConstraintWeight
1516SystemZTargetLowering::getSingleConstraintMatchWeight(
1517 AsmOperandInfo &Info, const char *Constraint) const {
1518 ConstraintWeight Weight = CW_Invalid;
1519 Value *CallOperandVal = Info.CallOperandVal;
1520 // If we don't have a value, we can't do a match,
1521 // but allow it at the lowest weight.
1522 if (!CallOperandVal)
1523 return CW_Default;
1524 Type *type = CallOperandVal->getType();
1525 // Look at the constraint type.
1526 switch (*Constraint) {
1527 default:
1528 Weight = TargetLowering::getSingleConstraintMatchWeight(info&: Info, constraint: Constraint);
1529 break;
1530
1531 case 'a': // Address register
1532 case 'd': // Data register (equivalent to 'r')
1533 case 'h': // High-part register
1534 case 'r': // General-purpose register
1535 Weight =
1536 CallOperandVal->getType()->isIntegerTy() ? CW_Register : CW_Default;
1537 break;
1538
1539 case 'f': // Floating-point register
1540 if (!useSoftFloat())
1541 Weight = type->isFloatingPointTy() ? CW_Register : CW_Default;
1542 break;
1543
1544 case 'v': // Vector register
1545 if (Subtarget.hasVector())
1546 Weight = (type->isVectorTy() || type->isFloatingPointTy()) ? CW_Register
1547 : CW_Default;
1548 break;
1549
1550 case 'I': // Unsigned 8-bit constant
1551 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1552 if (isUInt<8>(x: C->getZExtValue()))
1553 Weight = CW_Constant;
1554 break;
1555
1556 case 'J': // Unsigned 12-bit constant
1557 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1558 if (isUInt<12>(x: C->getZExtValue()))
1559 Weight = CW_Constant;
1560 break;
1561
1562 case 'K': // Signed 16-bit constant
1563 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1564 if (isInt<16>(x: C->getSExtValue()))
1565 Weight = CW_Constant;
1566 break;
1567
1568 case 'L': // Signed 20-bit displacement (on all targets we support)
1569 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1570 if (isInt<20>(x: C->getSExtValue()))
1571 Weight = CW_Constant;
1572 break;
1573
1574 case 'M': // 0x7fffffff
1575 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1576 if (C->getZExtValue() == 0x7fffffff)
1577 Weight = CW_Constant;
1578 break;
1579 }
1580 return Weight;
1581}
1582
1583// Parse a "{tNNN}" register constraint for which the register type "t"
1584// has already been verified. MC is the class associated with "t" and
1585// Map maps 0-based register numbers to LLVM register numbers.
1586static std::pair<unsigned, const TargetRegisterClass *>
1587parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC,
1588 const unsigned *Map, unsigned Size) {
1589 assert(*(Constraint.end()-1) == '}' && "Missing '}'");
1590 if (isdigit(Constraint[2])) {
1591 unsigned Index;
1592 bool Failed =
1593 Constraint.slice(Start: 2, End: Constraint.size() - 1).getAsInteger(Radix: 10, Result&: Index);
1594 if (!Failed && Index < Size && Map[Index])
1595 return std::make_pair(x: Map[Index], y&: RC);
1596 }
1597 return std::make_pair(x: 0U, y: nullptr);
1598}
1599
1600std::pair<unsigned, const TargetRegisterClass *>
1601SystemZTargetLowering::getRegForInlineAsmConstraint(
1602 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
1603 if (Constraint.size() == 1) {
1604 // GCC Constraint Letters
1605 switch (Constraint[0]) {
1606 default: break;
1607 case 'd': // Data register (equivalent to 'r')
1608 case 'r': // General-purpose register
1609 if (VT.getSizeInBits() == 64)
1610 return std::make_pair(x: 0U, y: &SystemZ::GR64BitRegClass);
1611 else if (VT.getSizeInBits() == 128)
1612 return std::make_pair(x: 0U, y: &SystemZ::GR128BitRegClass);
1613 return std::make_pair(x: 0U, y: &SystemZ::GR32BitRegClass);
1614
1615 case 'a': // Address register
1616 if (VT == MVT::i64)
1617 return std::make_pair(x: 0U, y: &SystemZ::ADDR64BitRegClass);
1618 else if (VT == MVT::i128)
1619 return std::make_pair(x: 0U, y: &SystemZ::ADDR128BitRegClass);
1620 return std::make_pair(x: 0U, y: &SystemZ::ADDR32BitRegClass);
1621
1622 case 'h': // High-part register (an LLVM extension)
1623 return std::make_pair(x: 0U, y: &SystemZ::GRH32BitRegClass);
1624
1625 case 'f': // Floating-point register
1626 if (!useSoftFloat()) {
1627 if (VT.getSizeInBits() == 16)
1628 return std::make_pair(x: 0U, y: &SystemZ::FP16BitRegClass);
1629 else if (VT.getSizeInBits() == 64)
1630 return std::make_pair(x: 0U, y: &SystemZ::FP64BitRegClass);
1631 else if (VT.getSizeInBits() == 128)
1632 return std::make_pair(x: 0U, y: &SystemZ::FP128BitRegClass);
1633 return std::make_pair(x: 0U, y: &SystemZ::FP32BitRegClass);
1634 }
1635 break;
1636
1637 case 'v': // Vector register
1638 if (Subtarget.hasVector()) {
1639 if (VT.getSizeInBits() == 16)
1640 return std::make_pair(x: 0U, y: &SystemZ::VR16BitRegClass);
1641 if (VT.getSizeInBits() == 32)
1642 return std::make_pair(x: 0U, y: &SystemZ::VR32BitRegClass);
1643 if (VT.getSizeInBits() == 64)
1644 return std::make_pair(x: 0U, y: &SystemZ::VR64BitRegClass);
1645 return std::make_pair(x: 0U, y: &SystemZ::VR128BitRegClass);
1646 }
1647 break;
1648 }
1649 }
1650 if (Constraint.starts_with(Prefix: "{")) {
1651
1652 // A clobber constraint (e.g. ~{f0}) will have MVT::Other which is illegal
1653 // to check the size on.
1654 auto getVTSizeInBits = [&VT]() {
1655 return VT == MVT::Other ? 0 : VT.getSizeInBits();
1656 };
1657
1658 // We need to override the default register parsing for GPRs and FPRs
1659 // because the interpretation depends on VT. The internal names of
1660 // the registers are also different from the external names
1661 // (F0D and F0S instead of F0, etc.).
1662 if (Constraint[1] == 'r') {
1663 if (getVTSizeInBits() == 32)
1664 return parseRegisterNumber(Constraint, RC: &SystemZ::GR32BitRegClass,
1665 Map: SystemZMC::GR32Regs, Size: 16);
1666 if (getVTSizeInBits() == 128)
1667 return parseRegisterNumber(Constraint, RC: &SystemZ::GR128BitRegClass,
1668 Map: SystemZMC::GR128Regs, Size: 16);
1669 return parseRegisterNumber(Constraint, RC: &SystemZ::GR64BitRegClass,
1670 Map: SystemZMC::GR64Regs, Size: 16);
1671 }
1672 if (Constraint[1] == 'f') {
1673 if (useSoftFloat())
1674 return std::make_pair(
1675 x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
1676 if (getVTSizeInBits() == 16)
1677 return parseRegisterNumber(Constraint, RC: &SystemZ::FP16BitRegClass,
1678 Map: SystemZMC::FP16Regs, Size: 16);
1679 if (getVTSizeInBits() == 32)
1680 return parseRegisterNumber(Constraint, RC: &SystemZ::FP32BitRegClass,
1681 Map: SystemZMC::FP32Regs, Size: 16);
1682 if (getVTSizeInBits() == 128)
1683 return parseRegisterNumber(Constraint, RC: &SystemZ::FP128BitRegClass,
1684 Map: SystemZMC::FP128Regs, Size: 16);
1685 return parseRegisterNumber(Constraint, RC: &SystemZ::FP64BitRegClass,
1686 Map: SystemZMC::FP64Regs, Size: 16);
1687 }
1688 if (Constraint[1] == 'v') {
1689 if (!Subtarget.hasVector())
1690 return std::make_pair(
1691 x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
1692 if (getVTSizeInBits() == 16)
1693 return parseRegisterNumber(Constraint, RC: &SystemZ::VR16BitRegClass,
1694 Map: SystemZMC::VR16Regs, Size: 32);
1695 if (getVTSizeInBits() == 32)
1696 return parseRegisterNumber(Constraint, RC: &SystemZ::VR32BitRegClass,
1697 Map: SystemZMC::VR32Regs, Size: 32);
1698 if (getVTSizeInBits() == 64)
1699 return parseRegisterNumber(Constraint, RC: &SystemZ::VR64BitRegClass,
1700 Map: SystemZMC::VR64Regs, Size: 32);
1701 return parseRegisterNumber(Constraint, RC: &SystemZ::VR128BitRegClass,
1702 Map: SystemZMC::VR128Regs, Size: 32);
1703 }
1704 }
1705 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
1706}
1707
1708// FIXME? Maybe this could be a TableGen attribute on some registers and
1709// this table could be generated automatically from RegInfo.
1710Register
1711SystemZTargetLowering::getRegisterByName(const char *RegName, LLT VT,
1712 const MachineFunction &MF) const {
1713 Register Reg =
1714 StringSwitch<Register>(RegName)
1715 .Case(S: "r4", Value: Subtarget.isTargetXPLINK64() ? SystemZ::R4D
1716 : SystemZ::NoRegister)
1717 .Case(S: "r15",
1718 Value: Subtarget.isTargetELF() ? SystemZ::R15D : SystemZ::NoRegister)
1719 .Default(Value: Register());
1720
1721 return Reg;
1722}
1723
1724Register SystemZTargetLowering::getExceptionPointerRegister(
1725 const Constant *PersonalityFn) const {
1726 return Subtarget.isTargetXPLINK64() ? SystemZ::R1D : SystemZ::R6D;
1727}
1728
1729Register SystemZTargetLowering::getExceptionSelectorRegister(
1730 const Constant *PersonalityFn) const {
1731 return Subtarget.isTargetXPLINK64() ? SystemZ::R2D : SystemZ::R7D;
1732}
1733
1734void SystemZTargetLowering::LowerAsmOperandForConstraint(
1735 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
1736 SelectionDAG &DAG) const {
1737 // Only support length 1 constraints for now.
1738 if (Constraint.size() == 1) {
1739 switch (Constraint[0]) {
1740 case 'I': // Unsigned 8-bit constant
1741 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1742 if (isUInt<8>(x: C->getZExtValue()))
1743 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1744 VT: Op.getValueType()));
1745 return;
1746
1747 case 'J': // Unsigned 12-bit constant
1748 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1749 if (isUInt<12>(x: C->getZExtValue()))
1750 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1751 VT: Op.getValueType()));
1752 return;
1753
1754 case 'K': // Signed 16-bit constant
1755 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1756 if (isInt<16>(x: C->getSExtValue()))
1757 Ops.push_back(x: DAG.getSignedTargetConstant(
1758 Val: C->getSExtValue(), DL: SDLoc(Op), VT: Op.getValueType()));
1759 return;
1760
1761 case 'L': // Signed 20-bit displacement (on all targets we support)
1762 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1763 if (isInt<20>(x: C->getSExtValue()))
1764 Ops.push_back(x: DAG.getSignedTargetConstant(
1765 Val: C->getSExtValue(), DL: SDLoc(Op), VT: Op.getValueType()));
1766 return;
1767
1768 case 'M': // 0x7fffffff
1769 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1770 if (C->getZExtValue() == 0x7fffffff)
1771 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1772 VT: Op.getValueType()));
1773 return;
1774 }
1775 }
1776 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
1777}
1778
1779//===----------------------------------------------------------------------===//
1780// Calling conventions
1781//===----------------------------------------------------------------------===//
1782
1783#include "SystemZGenCallingConv.inc"
1784
1785const MCPhysReg *SystemZTargetLowering::getScratchRegisters(
1786 CallingConv::ID) const {
1787 static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
1788 SystemZ::R14D, 0 };
1789 return ScratchRegs;
1790}
1791
1792bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType,
1793 Type *ToType) const {
1794 return isTruncateFree(FromType, ToType);
1795}
1796
1797bool SystemZTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
1798 return CI->isTailCall();
1799}
1800
1801// Value is a value that has been passed to us in the location described by VA
1802// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining
1803// any loads onto Chain.
1804static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL,
1805 CCValAssign &VA, SDValue Chain,
1806 SDValue Value) {
1807 // If the argument has been promoted from a smaller type, insert an
1808 // assertion to capture this.
1809 if (VA.getLocInfo() == CCValAssign::SExt)
1810 Value = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: Value,
1811 N2: DAG.getValueType(VA.getValVT()));
1812 else if (VA.getLocInfo() == CCValAssign::ZExt)
1813 Value = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: Value,
1814 N2: DAG.getValueType(VA.getValVT()));
1815
1816 if (VA.isExtInLoc())
1817 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Value);
1818 else if (VA.getLocInfo() == CCValAssign::BCvt) {
1819 // If this is a short vector argument loaded from the stack,
1820 // extend from i64 to full vector size and then bitcast.
1821 assert(VA.getLocVT() == MVT::i64);
1822 assert(VA.getValVT().isVector());
1823 Value = DAG.getBuildVector(VT: MVT::v2i64, DL, Ops: {Value, DAG.getUNDEF(VT: MVT::i64)});
1824 Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Value);
1825 } else
1826 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo");
1827 return Value;
1828}
1829
1830// Value is a value of type VA.getValVT() that we need to copy into
1831// the location described by VA. Return a copy of Value converted to
1832// VA.getValVT(). The caller is responsible for handling indirect values.
1833static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL,
1834 CCValAssign &VA, SDValue Value) {
1835 switch (VA.getLocInfo()) {
1836 case CCValAssign::SExt:
1837 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1838 case CCValAssign::ZExt:
1839 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1840 case CCValAssign::AExt:
1841 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1842 case CCValAssign::BCvt: {
1843 assert(VA.getLocVT() == MVT::i64 || VA.getLocVT() == MVT::i128);
1844 assert(VA.getValVT().isVector() || VA.getValVT() == MVT::f32 ||
1845 VA.getValVT() == MVT::f64 || VA.getValVT() == MVT::f128);
1846 // For an f32 vararg we need to first promote it to an f64 and then
1847 // bitcast it to an i64.
1848 if (VA.getValVT() == MVT::f32 && VA.getLocVT() == MVT::i64)
1849 Value = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f64, Operand: Value);
1850 MVT BitCastToType = VA.getValVT().isVector() && VA.getLocVT() == MVT::i64
1851 ? MVT::v2i64
1852 : VA.getLocVT();
1853 Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: BitCastToType, Operand: Value);
1854 // For ELF, this is a short vector argument to be stored to the stack,
1855 // bitcast to v2i64 and then extract first element.
1856 if (BitCastToType == MVT::v2i64)
1857 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: VA.getLocVT(), N1: Value,
1858 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
1859 return Value;
1860 }
1861 case CCValAssign::Full:
1862 return Value;
1863 default:
1864 llvm_unreachable("Unhandled getLocInfo()");
1865 }
1866}
1867
1868static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
1869 SDLoc DL(In);
1870 SDValue Lo, Hi;
1871 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128)) {
1872 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64, Operand: In);
1873 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64,
1874 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i128, N1: In,
1875 N2: DAG.getConstant(Val: 64, DL, VT: MVT::i32)));
1876 } else {
1877 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: In, DL, LoVT: MVT::i64, HiVT: MVT::i64);
1878 }
1879
1880 // FIXME: If v2i64 were a legal type, we could use it instead of
1881 // Untyped here. This might enable improved folding.
1882 SDNode *Pair = DAG.getMachineNode(Opcode: SystemZ::PAIR128, dl: DL,
1883 VT: MVT::Untyped, Op1: Hi, Op2: Lo);
1884 return SDValue(Pair, 0);
1885}
1886
1887static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
1888 SDLoc DL(In);
1889 SDValue Hi = DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h64,
1890 DL, VT: MVT::i64, Operand: In);
1891 SDValue Lo = DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_l64,
1892 DL, VT: MVT::i64, Operand: In);
1893
1894 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128)) {
1895 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i128, Operand: Lo);
1896 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i128, Operand: Hi);
1897 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i128, N1: Hi,
1898 N2: DAG.getConstant(Val: 64, DL, VT: MVT::i32));
1899 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i128, N1: Lo, N2: Hi);
1900 } else {
1901 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i128, N1: Lo, N2: Hi);
1902 }
1903}
1904
1905bool SystemZTargetLowering::splitValueIntoRegisterParts(
1906 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
1907 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
1908 EVT ValueVT = Val.getValueType();
1909 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
1910 // Inline assembly operand.
1911 Parts[0] = lowerI128ToGR128(DAG, In: DAG.getBitcast(VT: MVT::i128, V: Val));
1912 return true;
1913 }
1914
1915 return false;
1916}
1917
1918SDValue SystemZTargetLowering::joinRegisterPartsIntoValue(
1919 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
1920 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
1921 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
1922 // Inline assembly operand.
1923 SDValue Res = lowerGR128ToI128(DAG, In: Parts[0]);
1924 return DAG.getBitcast(VT: ValueVT, V: Res);
1925 }
1926
1927 return SDValue();
1928}
1929
1930SDValue SystemZTargetLowering::LowerFormalArguments(
1931 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1932 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1933 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1934 MachineFunction &MF = DAG.getMachineFunction();
1935 MachineFrameInfo &MFI = MF.getFrameInfo();
1936 MachineRegisterInfo &MRI = MF.getRegInfo();
1937 SystemZMachineFunctionInfo *FuncInfo =
1938 MF.getInfo<SystemZMachineFunctionInfo>();
1939 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
1940 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
1941
1942 // Assign locations to all of the incoming arguments.
1943 SmallVector<CCValAssign, 16> ArgLocs;
1944 SystemZCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1945 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_SystemZ);
1946 FuncInfo->setSizeOfFnParams(CCInfo.getStackSize());
1947
1948 unsigned NumFixedGPRs = 0;
1949 unsigned NumFixedFPRs = 0;
1950 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
1951 SDValue ArgValue;
1952 CCValAssign &VA = ArgLocs[I];
1953 EVT LocVT = VA.getLocVT();
1954 if (VA.isRegLoc()) {
1955 // Arguments passed in registers
1956 const TargetRegisterClass *RC;
1957 switch (LocVT.getSimpleVT().SimpleTy) {
1958 default:
1959 // Integers smaller than i64 should be promoted to i64.
1960 llvm_unreachable("Unexpected argument type");
1961 case MVT::i32:
1962 NumFixedGPRs += 1;
1963 RC = &SystemZ::GR32BitRegClass;
1964 break;
1965 case MVT::i64:
1966 NumFixedGPRs += 1;
1967 RC = &SystemZ::GR64BitRegClass;
1968 break;
1969 case MVT::f16:
1970 NumFixedFPRs += 1;
1971 RC = &SystemZ::FP16BitRegClass;
1972 break;
1973 case MVT::f32:
1974 NumFixedFPRs += 1;
1975 RC = &SystemZ::FP32BitRegClass;
1976 break;
1977 case MVT::f64:
1978 NumFixedFPRs += 1;
1979 RC = &SystemZ::FP64BitRegClass;
1980 break;
1981 case MVT::f128:
1982 NumFixedFPRs += 2;
1983 RC = &SystemZ::FP128BitRegClass;
1984 break;
1985 case MVT::v16i8:
1986 case MVT::v8i16:
1987 case MVT::v4i32:
1988 case MVT::v2i64:
1989 case MVT::v4f32:
1990 case MVT::v2f64:
1991 RC = &SystemZ::VR128BitRegClass;
1992 break;
1993 }
1994
1995 Register VReg = MRI.createVirtualRegister(RegClass: RC);
1996 MRI.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
1997 ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: LocVT);
1998 } else {
1999 assert(VA.isMemLoc() && "Argument not register or memory");
2000
2001 // Create the frame index object for this incoming parameter.
2002 // FIXME: Pre-include call frame size in the offset, should not
2003 // need to manually add it here.
2004 int64_t ArgSPOffset = VA.getLocMemOffset();
2005 if (Subtarget.isTargetXPLINK64()) {
2006 auto &XPRegs =
2007 Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
2008 ArgSPOffset += XPRegs.getCallFrameSize();
2009 }
2010 int FI =
2011 MFI.CreateFixedObject(Size: LocVT.getSizeInBits() / 8, SPOffset: ArgSPOffset, IsImmutable: true);
2012
2013 // Create the SelectionDAG nodes corresponding to a load
2014 // from this parameter. Unpromoted ints and floats are
2015 // passed as right-justified 8-byte values.
2016 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
2017 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32 ||
2018 VA.getLocVT() == MVT::f16) {
2019 unsigned SlotOffs = VA.getLocVT() == MVT::f16 ? 6 : 4;
2020 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FIN,
2021 N2: DAG.getIntPtrConstant(Val: SlotOffs, DL));
2022 }
2023 ArgValue = DAG.getLoad(VT: LocVT, dl: DL, Chain, Ptr: FIN,
2024 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
2025 }
2026
2027 // Convert the value of the argument register into the value that's
2028 // being passed.
2029 if (VA.getLocInfo() == CCValAssign::Indirect) {
2030 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain, Ptr: ArgValue,
2031 PtrInfo: MachinePointerInfo()));
2032 // If the original argument was split (e.g. i128), we need
2033 // to load all parts of it here (using the same address).
2034 unsigned ArgIndex = Ins[I].OrigArgIndex;
2035 assert (Ins[I].PartOffset == 0);
2036 while (I + 1 != E && Ins[I + 1].OrigArgIndex == ArgIndex) {
2037 CCValAssign &PartVA = ArgLocs[I + 1];
2038 unsigned PartOffset = Ins[I + 1].PartOffset;
2039 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: ArgValue,
2040 N2: DAG.getIntPtrConstant(Val: PartOffset, DL));
2041 InVals.push_back(Elt: DAG.getLoad(VT: PartVA.getValVT(), dl: DL, Chain, Ptr: Address,
2042 PtrInfo: MachinePointerInfo()));
2043 ++I;
2044 }
2045 } else
2046 InVals.push_back(Elt: convertLocVTToValVT(DAG, DL, VA, Chain, Value: ArgValue));
2047 }
2048
2049 if (IsVarArg && Subtarget.isTargetXPLINK64()) {
2050 // Save the number of non-varargs registers for later use by va_start, etc.
2051 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2052 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2053
2054 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2055 Subtarget.getSpecialRegisters());
2056
2057 // Likewise the address (in the form of a frame index) of where the
2058 // first stack vararg would be. The 1-byte size here is arbitrary.
2059 // FIXME: Pre-include call frame size in the offset, should not
2060 // need to manually add it here.
2061 int64_t VarArgOffset = CCInfo.getStackSize() + Regs->getCallFrameSize();
2062 int FI = MFI.CreateFixedObject(Size: 1, SPOffset: VarArgOffset, IsImmutable: true);
2063 FuncInfo->setVarArgsFrameIndex(FI);
2064 }
2065
2066 if (IsVarArg && Subtarget.isTargetELF()) {
2067 // Save the number of non-varargs registers for later use by va_start, etc.
2068 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2069 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2070
2071 // Likewise the address (in the form of a frame index) of where the
2072 // first stack vararg would be. The 1-byte size here is arbitrary.
2073 int64_t VarArgsOffset = CCInfo.getStackSize();
2074 FuncInfo->setVarArgsFrameIndex(
2075 MFI.CreateFixedObject(Size: 1, SPOffset: VarArgsOffset, IsImmutable: true));
2076
2077 // ...and a similar frame index for the caller-allocated save area
2078 // that will be used to store the incoming registers.
2079 int64_t RegSaveOffset =
2080 -SystemZMC::ELFCallFrameSize + TFL->getRegSpillOffset(MF, Reg: SystemZ::R2D) - 16;
2081 unsigned RegSaveIndex = MFI.CreateFixedObject(Size: 1, SPOffset: RegSaveOffset, IsImmutable: true);
2082 FuncInfo->setRegSaveFrameIndex(RegSaveIndex);
2083
2084 // Store the FPR varargs in the reserved frame slots. (We store the
2085 // GPRs as part of the prologue.)
2086 if (NumFixedFPRs < SystemZ::ELFNumArgFPRs && !useSoftFloat()) {
2087 SDValue MemOps[SystemZ::ELFNumArgFPRs];
2088 for (unsigned I = NumFixedFPRs; I < SystemZ::ELFNumArgFPRs; ++I) {
2089 unsigned Offset = TFL->getRegSpillOffset(MF, Reg: SystemZ::ELFArgFPRs[I]);
2090 int FI =
2091 MFI.CreateFixedObject(Size: 8, SPOffset: -SystemZMC::ELFCallFrameSize + Offset, IsImmutable: true);
2092 SDValue FIN = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
2093 Register VReg = MF.addLiveIn(PReg: SystemZ::ELFArgFPRs[I],
2094 RC: &SystemZ::FP64BitRegClass);
2095 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::f64);
2096 MemOps[I] = DAG.getStore(Chain: ArgValue.getValue(R: 1), dl: DL, Val: ArgValue, Ptr: FIN,
2097 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
2098 }
2099 // Join the stores, which are independent of one another.
2100 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
2101 Ops: ArrayRef(&MemOps[NumFixedFPRs],
2102 SystemZ::ELFNumArgFPRs - NumFixedFPRs));
2103 }
2104 }
2105
2106 if (Subtarget.isTargetXPLINK64()) {
2107 // Create virual register for handling incoming "ADA" special register (R5)
2108 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
2109 Register ADAvReg = MRI.createVirtualRegister(RegClass: RC);
2110 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2111 Subtarget.getSpecialRegisters());
2112 MRI.addLiveIn(Reg: Regs->getADARegister(), vreg: ADAvReg);
2113 FuncInfo->setADAVirtualRegister(ADAvReg);
2114 }
2115 return Chain;
2116}
2117
2118static bool canUseSiblingCall(const CCState &ArgCCInfo,
2119 SmallVectorImpl<CCValAssign> &ArgLocs,
2120 SmallVectorImpl<ISD::OutputArg> &Outs) {
2121 // Punt if there are any indirect or stack arguments, or if the call
2122 // needs the callee-saved argument register R6, or if the call uses
2123 // the callee-saved register arguments SwiftSelf and SwiftError.
2124 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2125 CCValAssign &VA = ArgLocs[I];
2126 if (VA.getLocInfo() == CCValAssign::Indirect)
2127 return false;
2128 if (!VA.isRegLoc())
2129 return false;
2130 Register Reg = VA.getLocReg();
2131 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
2132 return false;
2133 if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftError())
2134 return false;
2135 }
2136 return true;
2137}
2138
2139static SDValue getADAEntry(SelectionDAG &DAG, SDValue Val, SDLoc DL,
2140 unsigned Offset, bool LoadAdr = false) {
2141 MachineFunction &MF = DAG.getMachineFunction();
2142 SystemZMachineFunctionInfo *MFI = MF.getInfo<SystemZMachineFunctionInfo>();
2143 Register ADAvReg = MFI->getADAVirtualRegister();
2144 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
2145
2146 SDValue Reg = DAG.getRegister(Reg: ADAvReg, VT: PtrVT);
2147 SDValue Ofs = DAG.getTargetConstant(Val: Offset, DL, VT: PtrVT);
2148
2149 SDValue Result = DAG.getNode(Opcode: SystemZISD::ADA_ENTRY, DL, VT: PtrVT, N1: Val, N2: Reg, N3: Ofs);
2150 if (!LoadAdr)
2151 Result = DAG.getLoad(
2152 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result, PtrInfo: MachinePointerInfo(), Alignment: Align(8),
2153 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
2154
2155 return Result;
2156}
2157
2158// ADA access using Global value
2159// Note: for functions, address of descriptor is returned
2160static SDValue getADAEntry(SelectionDAG &DAG, const GlobalValue *GV, SDLoc DL,
2161 EVT PtrVT) {
2162 unsigned ADAtype;
2163 bool LoadAddr = false;
2164 const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: GV);
2165 bool IsFunction =
2166 (isa<Function>(Val: GV)) || (GA && isa<Function>(Val: GA->getAliaseeObject()));
2167 bool IsInternal = (GV->hasInternalLinkage() || GV->hasPrivateLinkage());
2168
2169 if (IsFunction) {
2170 if (IsInternal) {
2171 ADAtype = SystemZII::MO_ADA_DIRECT_FUNC_DESC;
2172 LoadAddr = true;
2173 } else
2174 ADAtype = SystemZII::MO_ADA_INDIRECT_FUNC_DESC;
2175 } else {
2176 ADAtype = SystemZII::MO_ADA_DATA_SYMBOL_ADDR;
2177 }
2178 SDValue Val = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: ADAtype);
2179
2180 return getADAEntry(DAG, Val, DL, Offset: 0, LoadAdr: LoadAddr);
2181}
2182
2183static bool getzOSCalleeAndADA(SelectionDAG &DAG, SDValue &Callee, SDValue &ADA,
2184 SDLoc &DL, SDValue &Chain) {
2185 unsigned ADADelta = 0; // ADA offset in desc.
2186 unsigned EPADelta = 8; // EPA offset in desc.
2187 MachineFunction &MF = DAG.getMachineFunction();
2188 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
2189
2190 // XPLink calling convention.
2191 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2192 bool IsInternal = (G->getGlobal()->hasInternalLinkage() ||
2193 G->getGlobal()->hasPrivateLinkage());
2194 if (IsInternal) {
2195 SystemZMachineFunctionInfo *MFI =
2196 MF.getInfo<SystemZMachineFunctionInfo>();
2197 Register ADAvReg = MFI->getADAVirtualRegister();
2198 ADA = DAG.getCopyFromReg(Chain, dl: DL, Reg: ADAvReg, VT: PtrVT);
2199 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL, VT: PtrVT);
2200 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2201 return true;
2202 } else {
2203 SDValue GA = DAG.getTargetGlobalAddress(
2204 GV: G->getGlobal(), DL, VT: PtrVT, offset: 0, TargetFlags: SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2205 ADA = getADAEntry(DAG, Val: GA, DL, Offset: ADADelta);
2206 Callee = getADAEntry(DAG, Val: GA, DL, Offset: EPADelta);
2207 }
2208 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2209 SDValue ES = DAG.getTargetExternalSymbol(
2210 Sym: E->getSymbol(), VT: PtrVT, TargetFlags: SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2211 ADA = getADAEntry(DAG, Val: ES, DL, Offset: ADADelta);
2212 Callee = getADAEntry(DAG, Val: ES, DL, Offset: EPADelta);
2213 } else {
2214 // Function pointer case
2215 ADA = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Callee,
2216 N2: DAG.getConstant(Val: ADADelta, DL, VT: PtrVT));
2217 ADA = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: ADA,
2218 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2219 Callee = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Callee,
2220 N2: DAG.getConstant(Val: EPADelta, DL, VT: PtrVT));
2221 Callee = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Callee,
2222 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2223 }
2224 return false;
2225}
2226
2227SDValue
2228SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
2229 SmallVectorImpl<SDValue> &InVals) const {
2230 SelectionDAG &DAG = CLI.DAG;
2231 SDLoc &DL = CLI.DL;
2232 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2233 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2234 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2235 SDValue Chain = CLI.Chain;
2236 SDValue Callee = CLI.Callee;
2237 bool &IsTailCall = CLI.IsTailCall;
2238 CallingConv::ID CallConv = CLI.CallConv;
2239 bool IsVarArg = CLI.IsVarArg;
2240 MachineFunction &MF = DAG.getMachineFunction();
2241 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
2242 LLVMContext &Ctx = *DAG.getContext();
2243 SystemZCallingConventionRegisters *Regs = Subtarget.getSpecialRegisters();
2244
2245 // FIXME: z/OS support to be added in later.
2246 if (Subtarget.isTargetXPLINK64())
2247 IsTailCall = false;
2248
2249 // Integer args <=32 bits should have an extension attribute.
2250 verifyNarrowIntegerArgs_Call(Outs, F: &MF.getFunction(), Callee);
2251
2252 // Analyze the operands of the call, assigning locations to each operand.
2253 SmallVector<CCValAssign, 16> ArgLocs;
2254 SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx);
2255 ArgCCInfo.AnalyzeCallOperands(Outs, Fn: CC_SystemZ);
2256
2257 // We don't support GuaranteedTailCallOpt, only automatically-detected
2258 // sibling calls.
2259 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs, Outs))
2260 IsTailCall = false;
2261
2262 // Get a count of how many bytes are to be pushed on the stack.
2263 unsigned NumBytes = ArgCCInfo.getStackSize();
2264
2265 // Mark the start of the call.
2266 if (!IsTailCall)
2267 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL);
2268
2269 // Copy argument values to their designated locations.
2270 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass;
2271 SmallVector<SDValue, 8> MemOpChains;
2272 SDValue StackPtr;
2273 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2274 CCValAssign &VA = ArgLocs[I];
2275 SDValue ArgValue = OutVals[I];
2276
2277 if (VA.getLocInfo() == CCValAssign::Indirect) {
2278 // Store the argument in a stack slot and pass its address.
2279 unsigned ArgIndex = Outs[I].OrigArgIndex;
2280 EVT SlotVT;
2281 if (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) {
2282 // Allocate the full stack space for a promoted (and split) argument.
2283 Type *OrigArgType = CLI.Args[Outs[I].OrigArgIndex].Ty;
2284 EVT OrigArgVT = getValueType(DL: MF.getDataLayout(), Ty: OrigArgType);
2285 MVT PartVT = getRegisterTypeForCallingConv(Context&: Ctx, CC: CLI.CallConv, VT: OrigArgVT);
2286 unsigned N = getNumRegistersForCallingConv(Context&: Ctx, CC: CLI.CallConv, VT: OrigArgVT);
2287 SlotVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: PartVT.getSizeInBits() * N);
2288 } else {
2289 SlotVT = Outs[I].VT;
2290 }
2291 SDValue SpillSlot = DAG.CreateStackTemporary(VT: SlotVT);
2292 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
2293 MemOpChains.push_back(
2294 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: SpillSlot,
2295 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
2296 // If the original argument was split (e.g. i128), we need
2297 // to store all parts of it here (and pass just one address).
2298 assert (Outs[I].PartOffset == 0);
2299 while (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) {
2300 SDValue PartValue = OutVals[I + 1];
2301 unsigned PartOffset = Outs[I + 1].PartOffset;
2302 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SpillSlot,
2303 N2: DAG.getIntPtrConstant(Val: PartOffset, DL));
2304 MemOpChains.push_back(
2305 Elt: DAG.getStore(Chain, dl: DL, Val: PartValue, Ptr: Address,
2306 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
2307 assert((PartOffset + PartValue.getValueType().getStoreSize() <=
2308 SlotVT.getStoreSize()) && "Not enough space for argument part!");
2309 ++I;
2310 }
2311 ArgValue = SpillSlot;
2312 } else
2313 ArgValue = convertValVTToLocVT(DAG, DL, VA, Value: ArgValue);
2314
2315 if (VA.isRegLoc()) {
2316 // In XPLINK64, for the 128-bit vararg case, ArgValue is bitcasted to a
2317 // MVT::i128 type. We decompose the 128-bit type to a pair of its high
2318 // and low values.
2319 if (VA.getLocVT() == MVT::i128)
2320 ArgValue = lowerI128ToGR128(DAG, In: ArgValue);
2321 // Queue up the argument copies and emit them at the end.
2322 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgValue));
2323 } else {
2324 assert(VA.isMemLoc() && "Argument not register or memory");
2325
2326 // Work out the address of the stack slot. Unpromoted ints and
2327 // floats are passed as right-justified 8-byte values.
2328 if (!StackPtr.getNode())
2329 StackPtr = DAG.getCopyFromReg(Chain, dl: DL,
2330 Reg: Regs->getStackPointerRegister(), VT: PtrVT);
2331 unsigned Offset = Regs->getStackPointerBias() + Regs->getCallFrameSize() +
2332 VA.getLocMemOffset();
2333 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
2334 Offset += 4;
2335 else if (VA.getLocVT() == MVT::f16)
2336 Offset += 6;
2337 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
2338 N2: DAG.getIntPtrConstant(Val: Offset, DL));
2339
2340 // Emit the store.
2341 MemOpChains.push_back(
2342 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: Address, PtrInfo: MachinePointerInfo()));
2343
2344 // Although long doubles or vectors are passed through the stack when
2345 // they are vararg (non-fixed arguments), if a long double or vector
2346 // occupies the third and fourth slot of the argument list GPR3 should
2347 // still shadow the third slot of the argument list.
2348 if (Subtarget.isTargetXPLINK64() && VA.needsCustom()) {
2349 SDValue ShadowArgValue =
2350 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: ArgValue,
2351 N2: DAG.getIntPtrConstant(Val: 1, DL));
2352 RegsToPass.push_back(Elt: std::make_pair(x: SystemZ::R3D, y&: ShadowArgValue));
2353 }
2354 }
2355 }
2356
2357 // Join the stores, which are independent of one another.
2358 if (!MemOpChains.empty())
2359 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
2360
2361 // Accept direct calls by converting symbolic call addresses to the
2362 // associated Target* opcodes. Force %r1 to be used for indirect
2363 // tail calls.
2364 SDValue Glue;
2365
2366 if (Subtarget.isTargetXPLINK64()) {
2367 SDValue ADA;
2368 bool IsBRASL = getzOSCalleeAndADA(DAG, Callee, ADA, DL, Chain);
2369 if (!IsBRASL) {
2370 unsigned CalleeReg = static_cast<SystemZXPLINK64Registers *>(Regs)
2371 ->getAddressOfCalleeRegister();
2372 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: CalleeReg, N: Callee, Glue);
2373 Glue = Chain.getValue(R: 1);
2374 Callee = DAG.getRegister(Reg: CalleeReg, VT: Callee.getValueType());
2375 }
2376 RegsToPass.push_back(Elt: std::make_pair(
2377 x: static_cast<SystemZXPLINK64Registers *>(Regs)->getADARegister(), y&: ADA));
2378 } else {
2379 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2380 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL, VT: PtrVT);
2381 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2382 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2383 Callee = DAG.getTargetExternalSymbol(Sym: E->getSymbol(), VT: PtrVT);
2384 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2385 } else if (IsTailCall) {
2386 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R1D, N: Callee, Glue);
2387 Glue = Chain.getValue(R: 1);
2388 Callee = DAG.getRegister(Reg: SystemZ::R1D, VT: Callee.getValueType());
2389 }
2390 }
2391
2392 // Build a sequence of copy-to-reg nodes, chained and glued together.
2393 for (const auto &[Reg, N] : RegsToPass) {
2394 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N, Glue);
2395 Glue = Chain.getValue(R: 1);
2396 }
2397
2398 // The first call operand is the chain and the second is the target address.
2399 SmallVector<SDValue, 8> Ops;
2400 Ops.push_back(Elt: Chain);
2401 Ops.push_back(Elt: Callee);
2402
2403 // Add argument registers to the end of the list so that they are
2404 // known live into the call.
2405 for (const auto &[Reg, N] : RegsToPass)
2406 Ops.push_back(Elt: DAG.getRegister(Reg, VT: N.getValueType()));
2407
2408 // Add a register mask operand representing the call-preserved registers.
2409 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2410 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2411 assert(Mask && "Missing call preserved mask for calling convention");
2412 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
2413
2414 // Glue the call to the argument copies, if any.
2415 if (Glue.getNode())
2416 Ops.push_back(Elt: Glue);
2417
2418 // Emit the call.
2419 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
2420 if (IsTailCall) {
2421 SDValue Ret = DAG.getNode(Opcode: SystemZISD::SIBCALL, DL, VTList: NodeTys, Ops);
2422 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
2423 return Ret;
2424 }
2425 Chain = DAG.getNode(Opcode: SystemZISD::CALL, DL, VTList: NodeTys, Ops);
2426 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
2427 Glue = Chain.getValue(R: 1);
2428
2429 // Mark the end of the call, which is glued to the call itself.
2430 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL);
2431 Glue = Chain.getValue(R: 1);
2432
2433 // Assign locations to each value returned by this call.
2434 SmallVector<CCValAssign, 16> RetLocs;
2435 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
2436 RetCCInfo.AnalyzeCallResult(Ins, Fn: RetCC_SystemZ);
2437
2438 // Copy all of the result registers out of their specified physreg.
2439 for (CCValAssign &VA : RetLocs) {
2440 // Copy the value out, gluing the copy to the end of the call sequence.
2441 SDValue RetValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(),
2442 VT: VA.getLocVT(), Glue);
2443 Chain = RetValue.getValue(R: 1);
2444 Glue = RetValue.getValue(R: 2);
2445
2446 // Convert the value of the return register into the value that's
2447 // being returned.
2448 InVals.push_back(Elt: convertLocVTToValVT(DAG, DL, VA, Chain, Value: RetValue));
2449 }
2450
2451 return Chain;
2452}
2453
2454// Generate a call taking the given operands as arguments and returning a
2455// result of type RetVT.
2456std::pair<SDValue, SDValue> SystemZTargetLowering::makeExternalCall(
2457 SDValue Chain, SelectionDAG &DAG, const char *CalleeName, EVT RetVT,
2458 ArrayRef<SDValue> Ops, CallingConv::ID CallConv, bool IsSigned, SDLoc DL,
2459 bool DoesNotReturn, bool IsReturnValueUsed) const {
2460 TargetLowering::ArgListTy Args;
2461 Args.reserve(n: Ops.size());
2462
2463 TargetLowering::ArgListEntry Entry;
2464 for (SDValue Op : Ops) {
2465 Entry.Node = Op;
2466 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(Context&: *DAG.getContext());
2467 Entry.IsSExt = shouldSignExtendTypeInLibCall(Ty: Entry.Ty, IsSigned);
2468 Entry.IsZExt = !Entry.IsSExt;
2469 Args.push_back(x: Entry);
2470 }
2471
2472 SDValue Callee =
2473 DAG.getExternalSymbol(Sym: CalleeName, VT: getPointerTy(DL: DAG.getDataLayout()));
2474
2475 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2476 TargetLowering::CallLoweringInfo CLI(DAG);
2477 bool SignExtend = shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2478 CLI.setDebugLoc(DL)
2479 .setChain(Chain)
2480 .setCallee(CC: CallConv, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2481 .setNoReturn(DoesNotReturn)
2482 .setDiscardResult(!IsReturnValueUsed)
2483 .setSExtResult(SignExtend)
2484 .setZExtResult(!SignExtend);
2485 return LowerCallTo(CLI);
2486}
2487
2488bool SystemZTargetLowering::CanLowerReturn(
2489 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
2490 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
2491 const Type *RetTy) const {
2492 // Special case that we cannot easily detect in RetCC_SystemZ since
2493 // i128 may not be a legal type.
2494 for (auto &Out : Outs)
2495 if (Out.ArgVT == MVT::i128)
2496 return false;
2497
2498 SmallVector<CCValAssign, 16> RetLocs;
2499 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Context);
2500 return RetCCInfo.CheckReturn(Outs, Fn: RetCC_SystemZ);
2501}
2502
2503SDValue
2504SystemZTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2505 bool IsVarArg,
2506 const SmallVectorImpl<ISD::OutputArg> &Outs,
2507 const SmallVectorImpl<SDValue> &OutVals,
2508 const SDLoc &DL, SelectionDAG &DAG) const {
2509 MachineFunction &MF = DAG.getMachineFunction();
2510
2511 // Integer args <=32 bits should have an extension attribute.
2512 verifyNarrowIntegerArgs_Ret(Outs, F: &MF.getFunction());
2513
2514 // Assign locations to each returned value.
2515 SmallVector<CCValAssign, 16> RetLocs;
2516 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
2517 RetCCInfo.AnalyzeReturn(Outs, Fn: RetCC_SystemZ);
2518
2519 // Quick exit for void returns
2520 if (RetLocs.empty())
2521 return DAG.getNode(Opcode: SystemZISD::RET_GLUE, DL, VT: MVT::Other, Operand: Chain);
2522
2523 if (CallConv == CallingConv::GHC)
2524 report_fatal_error(reason: "GHC functions return void only");
2525
2526 // Copy the result values into the output registers.
2527 SDValue Glue;
2528 SmallVector<SDValue, 4> RetOps;
2529 RetOps.push_back(Elt: Chain);
2530 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
2531 CCValAssign &VA = RetLocs[I];
2532 SDValue RetValue = OutVals[I];
2533
2534 // Make the return register live on exit.
2535 assert(VA.isRegLoc() && "Can only return in registers!");
2536
2537 // Promote the value as required.
2538 RetValue = convertValVTToLocVT(DAG, DL, VA, Value: RetValue);
2539
2540 // Chain and glue the copies together.
2541 Register Reg = VA.getLocReg();
2542 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N: RetValue, Glue);
2543 Glue = Chain.getValue(R: 1);
2544 RetOps.push_back(Elt: DAG.getRegister(Reg, VT: VA.getLocVT()));
2545 }
2546
2547 // Update chain and glue.
2548 RetOps[0] = Chain;
2549 if (Glue.getNode())
2550 RetOps.push_back(Elt: Glue);
2551
2552 return DAG.getNode(Opcode: SystemZISD::RET_GLUE, DL, VT: MVT::Other, Ops: RetOps);
2553}
2554
2555// Return true if Op is an intrinsic node with chain that returns the CC value
2556// as its only (other) argument. Provide the associated SystemZISD opcode and
2557// the mask of valid CC values if so.
2558static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode,
2559 unsigned &CCValid) {
2560 unsigned Id = Op.getConstantOperandVal(i: 1);
2561 switch (Id) {
2562 case Intrinsic::s390_tbegin:
2563 Opcode = SystemZISD::TBEGIN;
2564 CCValid = SystemZ::CCMASK_TBEGIN;
2565 return true;
2566
2567 case Intrinsic::s390_tbegin_nofloat:
2568 Opcode = SystemZISD::TBEGIN_NOFLOAT;
2569 CCValid = SystemZ::CCMASK_TBEGIN;
2570 return true;
2571
2572 case Intrinsic::s390_tend:
2573 Opcode = SystemZISD::TEND;
2574 CCValid = SystemZ::CCMASK_TEND;
2575 return true;
2576
2577 default:
2578 return false;
2579 }
2580}
2581
2582// Return true if Op is an intrinsic node without chain that returns the
2583// CC value as its final argument. Provide the associated SystemZISD
2584// opcode and the mask of valid CC values if so.
2585static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) {
2586 unsigned Id = Op.getConstantOperandVal(i: 0);
2587 switch (Id) {
2588 case Intrinsic::s390_vpkshs:
2589 case Intrinsic::s390_vpksfs:
2590 case Intrinsic::s390_vpksgs:
2591 Opcode = SystemZISD::PACKS_CC;
2592 CCValid = SystemZ::CCMASK_VCMP;
2593 return true;
2594
2595 case Intrinsic::s390_vpklshs:
2596 case Intrinsic::s390_vpklsfs:
2597 case Intrinsic::s390_vpklsgs:
2598 Opcode = SystemZISD::PACKLS_CC;
2599 CCValid = SystemZ::CCMASK_VCMP;
2600 return true;
2601
2602 case Intrinsic::s390_vceqbs:
2603 case Intrinsic::s390_vceqhs:
2604 case Intrinsic::s390_vceqfs:
2605 case Intrinsic::s390_vceqgs:
2606 case Intrinsic::s390_vceqqs:
2607 Opcode = SystemZISD::VICMPES;
2608 CCValid = SystemZ::CCMASK_VCMP;
2609 return true;
2610
2611 case Intrinsic::s390_vchbs:
2612 case Intrinsic::s390_vchhs:
2613 case Intrinsic::s390_vchfs:
2614 case Intrinsic::s390_vchgs:
2615 case Intrinsic::s390_vchqs:
2616 Opcode = SystemZISD::VICMPHS;
2617 CCValid = SystemZ::CCMASK_VCMP;
2618 return true;
2619
2620 case Intrinsic::s390_vchlbs:
2621 case Intrinsic::s390_vchlhs:
2622 case Intrinsic::s390_vchlfs:
2623 case Intrinsic::s390_vchlgs:
2624 case Intrinsic::s390_vchlqs:
2625 Opcode = SystemZISD::VICMPHLS;
2626 CCValid = SystemZ::CCMASK_VCMP;
2627 return true;
2628
2629 case Intrinsic::s390_vtm:
2630 Opcode = SystemZISD::VTM;
2631 CCValid = SystemZ::CCMASK_VCMP;
2632 return true;
2633
2634 case Intrinsic::s390_vfaebs:
2635 case Intrinsic::s390_vfaehs:
2636 case Intrinsic::s390_vfaefs:
2637 Opcode = SystemZISD::VFAE_CC;
2638 CCValid = SystemZ::CCMASK_ANY;
2639 return true;
2640
2641 case Intrinsic::s390_vfaezbs:
2642 case Intrinsic::s390_vfaezhs:
2643 case Intrinsic::s390_vfaezfs:
2644 Opcode = SystemZISD::VFAEZ_CC;
2645 CCValid = SystemZ::CCMASK_ANY;
2646 return true;
2647
2648 case Intrinsic::s390_vfeebs:
2649 case Intrinsic::s390_vfeehs:
2650 case Intrinsic::s390_vfeefs:
2651 Opcode = SystemZISD::VFEE_CC;
2652 CCValid = SystemZ::CCMASK_ANY;
2653 return true;
2654
2655 case Intrinsic::s390_vfeezbs:
2656 case Intrinsic::s390_vfeezhs:
2657 case Intrinsic::s390_vfeezfs:
2658 Opcode = SystemZISD::VFEEZ_CC;
2659 CCValid = SystemZ::CCMASK_ANY;
2660 return true;
2661
2662 case Intrinsic::s390_vfenebs:
2663 case Intrinsic::s390_vfenehs:
2664 case Intrinsic::s390_vfenefs:
2665 Opcode = SystemZISD::VFENE_CC;
2666 CCValid = SystemZ::CCMASK_ANY;
2667 return true;
2668
2669 case Intrinsic::s390_vfenezbs:
2670 case Intrinsic::s390_vfenezhs:
2671 case Intrinsic::s390_vfenezfs:
2672 Opcode = SystemZISD::VFENEZ_CC;
2673 CCValid = SystemZ::CCMASK_ANY;
2674 return true;
2675
2676 case Intrinsic::s390_vistrbs:
2677 case Intrinsic::s390_vistrhs:
2678 case Intrinsic::s390_vistrfs:
2679 Opcode = SystemZISD::VISTR_CC;
2680 CCValid = SystemZ::CCMASK_0 | SystemZ::CCMASK_3;
2681 return true;
2682
2683 case Intrinsic::s390_vstrcbs:
2684 case Intrinsic::s390_vstrchs:
2685 case Intrinsic::s390_vstrcfs:
2686 Opcode = SystemZISD::VSTRC_CC;
2687 CCValid = SystemZ::CCMASK_ANY;
2688 return true;
2689
2690 case Intrinsic::s390_vstrczbs:
2691 case Intrinsic::s390_vstrczhs:
2692 case Intrinsic::s390_vstrczfs:
2693 Opcode = SystemZISD::VSTRCZ_CC;
2694 CCValid = SystemZ::CCMASK_ANY;
2695 return true;
2696
2697 case Intrinsic::s390_vstrsb:
2698 case Intrinsic::s390_vstrsh:
2699 case Intrinsic::s390_vstrsf:
2700 Opcode = SystemZISD::VSTRS_CC;
2701 CCValid = SystemZ::CCMASK_ANY;
2702 return true;
2703
2704 case Intrinsic::s390_vstrszb:
2705 case Intrinsic::s390_vstrszh:
2706 case Intrinsic::s390_vstrszf:
2707 Opcode = SystemZISD::VSTRSZ_CC;
2708 CCValid = SystemZ::CCMASK_ANY;
2709 return true;
2710
2711 case Intrinsic::s390_vfcedbs:
2712 case Intrinsic::s390_vfcesbs:
2713 Opcode = SystemZISD::VFCMPES;
2714 CCValid = SystemZ::CCMASK_VCMP;
2715 return true;
2716
2717 case Intrinsic::s390_vfchdbs:
2718 case Intrinsic::s390_vfchsbs:
2719 Opcode = SystemZISD::VFCMPHS;
2720 CCValid = SystemZ::CCMASK_VCMP;
2721 return true;
2722
2723 case Intrinsic::s390_vfchedbs:
2724 case Intrinsic::s390_vfchesbs:
2725 Opcode = SystemZISD::VFCMPHES;
2726 CCValid = SystemZ::CCMASK_VCMP;
2727 return true;
2728
2729 case Intrinsic::s390_vftcidb:
2730 case Intrinsic::s390_vftcisb:
2731 Opcode = SystemZISD::VFTCI;
2732 CCValid = SystemZ::CCMASK_VCMP;
2733 return true;
2734
2735 case Intrinsic::s390_tdc:
2736 Opcode = SystemZISD::TDC;
2737 CCValid = SystemZ::CCMASK_TDC;
2738 return true;
2739
2740 default:
2741 return false;
2742 }
2743}
2744
2745// Emit an intrinsic with chain and an explicit CC register result.
2746static SDNode *emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op,
2747 unsigned Opcode) {
2748 // Copy all operands except the intrinsic ID.
2749 unsigned NumOps = Op.getNumOperands();
2750 SmallVector<SDValue, 6> Ops;
2751 Ops.reserve(N: NumOps - 1);
2752 Ops.push_back(Elt: Op.getOperand(i: 0));
2753 for (unsigned I = 2; I < NumOps; ++I)
2754 Ops.push_back(Elt: Op.getOperand(i: I));
2755
2756 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
2757 SDVTList RawVTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
2758 SDValue Intr = DAG.getNode(Opcode, DL: SDLoc(Op), VTList: RawVTs, Ops);
2759 SDValue OldChain = SDValue(Op.getNode(), 1);
2760 SDValue NewChain = SDValue(Intr.getNode(), 1);
2761 DAG.ReplaceAllUsesOfValueWith(From: OldChain, To: NewChain);
2762 return Intr.getNode();
2763}
2764
2765// Emit an intrinsic with an explicit CC register result.
2766static SDNode *emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op,
2767 unsigned Opcode) {
2768 // Copy all operands except the intrinsic ID.
2769 SDLoc DL(Op);
2770 unsigned NumOps = Op.getNumOperands();
2771 SmallVector<SDValue, 6> Ops;
2772 Ops.reserve(N: NumOps - 1);
2773 for (unsigned I = 1; I < NumOps; ++I) {
2774 SDValue CurrOper = Op.getOperand(i: I);
2775 if (CurrOper.getValueType() == MVT::f16) {
2776 assert((Op.getConstantOperandVal(0) == Intrinsic::s390_tdc && I == 1) &&
2777 "Unhandled intrinsic with f16 operand.");
2778 CurrOper = DAG.getFPExtendOrRound(Op: CurrOper, DL, VT: MVT::f32);
2779 }
2780 Ops.push_back(Elt: CurrOper);
2781 }
2782
2783 SDValue Intr = DAG.getNode(Opcode, DL, VTList: Op->getVTList(), Ops);
2784 return Intr.getNode();
2785}
2786
2787// CC is a comparison that will be implemented using an integer or
2788// floating-point comparison. Return the condition code mask for
2789// a branch on true. In the integer case, CCMASK_CMP_UO is set for
2790// unsigned comparisons and clear for signed ones. In the floating-point
2791// case, CCMASK_CMP_UO has its normal mask meaning (unordered).
2792static unsigned CCMaskForCondCode(ISD::CondCode CC) {
2793#define CONV(X) \
2794 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
2795 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
2796 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
2797
2798 switch (CC) {
2799 default:
2800 llvm_unreachable("Invalid integer condition!");
2801
2802 CONV(EQ);
2803 CONV(NE);
2804 CONV(GT);
2805 CONV(GE);
2806 CONV(LT);
2807 CONV(LE);
2808
2809 case ISD::SETO: return SystemZ::CCMASK_CMP_O;
2810 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO;
2811 }
2812#undef CONV
2813}
2814
2815// If C can be converted to a comparison against zero, adjust the operands
2816// as necessary.
2817static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
2818 if (C.ICmpType == SystemZICMP::UnsignedOnly)
2819 return;
2820
2821 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val: C.Op1.getNode());
2822 if (!ConstOp1 || ConstOp1->getValueSizeInBits(ResNo: 0) > 64)
2823 return;
2824
2825 int64_t Value = ConstOp1->getSExtValue();
2826 if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) ||
2827 (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) ||
2828 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) ||
2829 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) {
2830 C.CCMask ^= SystemZ::CCMASK_CMP_EQ;
2831 C.Op1 = DAG.getConstant(Val: 0, DL, VT: C.Op1.getValueType());
2832 }
2833}
2834
2835// If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI,
2836// adjust the operands as necessary.
2837static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL,
2838 Comparison &C) {
2839 // For us to make any changes, it must a comparison between a single-use
2840 // load and a constant.
2841 if (!C.Op0.hasOneUse() ||
2842 C.Op0.getOpcode() != ISD::LOAD ||
2843 C.Op1.getOpcode() != ISD::Constant)
2844 return;
2845
2846 // We must have an 8- or 16-bit load.
2847 auto *Load = cast<LoadSDNode>(Val&: C.Op0);
2848 unsigned NumBits = Load->getMemoryVT().getSizeInBits();
2849 if ((NumBits != 8 && NumBits != 16) ||
2850 NumBits != Load->getMemoryVT().getStoreSizeInBits())
2851 return;
2852
2853 // The load must be an extending one and the constant must be within the
2854 // range of the unextended value.
2855 auto *ConstOp1 = cast<ConstantSDNode>(Val&: C.Op1);
2856 if (!ConstOp1 || ConstOp1->getValueSizeInBits(ResNo: 0) > 64)
2857 return;
2858 uint64_t Value = ConstOp1->getZExtValue();
2859 uint64_t Mask = (1 << NumBits) - 1;
2860 if (Load->getExtensionType() == ISD::SEXTLOAD) {
2861 // Make sure that ConstOp1 is in range of C.Op0.
2862 int64_t SignedValue = ConstOp1->getSExtValue();
2863 if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask)
2864 return;
2865 if (C.ICmpType != SystemZICMP::SignedOnly) {
2866 // Unsigned comparison between two sign-extended values is equivalent
2867 // to unsigned comparison between two zero-extended values.
2868 Value &= Mask;
2869 } else if (NumBits == 8) {
2870 // Try to treat the comparison as unsigned, so that we can use CLI.
2871 // Adjust CCMask and Value as necessary.
2872 if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT)
2873 // Test whether the high bit of the byte is set.
2874 Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT;
2875 else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE)
2876 // Test whether the high bit of the byte is clear.
2877 Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT;
2878 else
2879 // No instruction exists for this combination.
2880 return;
2881 C.ICmpType = SystemZICMP::UnsignedOnly;
2882 }
2883 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) {
2884 if (Value > Mask)
2885 return;
2886 // If the constant is in range, we can use any comparison.
2887 C.ICmpType = SystemZICMP::Any;
2888 } else
2889 return;
2890
2891 // Make sure that the first operand is an i32 of the right extension type.
2892 ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ?
2893 ISD::SEXTLOAD :
2894 ISD::ZEXTLOAD);
2895 if (C.Op0.getValueType() != MVT::i32 ||
2896 Load->getExtensionType() != ExtType) {
2897 C.Op0 = DAG.getExtLoad(ExtType, dl: SDLoc(Load), VT: MVT::i32, Chain: Load->getChain(),
2898 Ptr: Load->getBasePtr(), PtrInfo: Load->getPointerInfo(),
2899 MemVT: Load->getMemoryVT(), Alignment: Load->getAlign(),
2900 MMOFlags: Load->getMemOperand()->getFlags());
2901 // Update the chain uses.
2902 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Load, 1), To: C.Op0.getValue(R: 1));
2903 }
2904
2905 // Make sure that the second operand is an i32 with the right value.
2906 if (C.Op1.getValueType() != MVT::i32 ||
2907 Value != ConstOp1->getZExtValue())
2908 C.Op1 = DAG.getConstant(Val: (uint32_t)Value, DL, VT: MVT::i32);
2909}
2910
2911// Return true if Op is either an unextended load, or a load suitable
2912// for integer register-memory comparisons of type ICmpType.
2913static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) {
2914 auto *Load = dyn_cast<LoadSDNode>(Val: Op.getNode());
2915 if (Load) {
2916 // There are no instructions to compare a register with a memory byte.
2917 if (Load->getMemoryVT() == MVT::i8)
2918 return false;
2919 // Otherwise decide on extension type.
2920 switch (Load->getExtensionType()) {
2921 case ISD::NON_EXTLOAD:
2922 return true;
2923 case ISD::SEXTLOAD:
2924 return ICmpType != SystemZICMP::UnsignedOnly;
2925 case ISD::ZEXTLOAD:
2926 return ICmpType != SystemZICMP::SignedOnly;
2927 default:
2928 break;
2929 }
2930 }
2931 return false;
2932}
2933
2934// Return true if it is better to swap the operands of C.
2935static bool shouldSwapCmpOperands(const Comparison &C) {
2936 // Leave i128 and f128 comparisons alone, since they have no memory forms.
2937 if (C.Op0.getValueType() == MVT::i128)
2938 return false;
2939 if (C.Op0.getValueType() == MVT::f128)
2940 return false;
2941
2942 // Always keep a floating-point constant second, since comparisons with
2943 // zero can use LOAD TEST and comparisons with other constants make a
2944 // natural memory operand.
2945 if (isa<ConstantFPSDNode>(Val: C.Op1))
2946 return false;
2947
2948 // Never swap comparisons with zero since there are many ways to optimize
2949 // those later.
2950 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val: C.Op1);
2951 if (ConstOp1 && ConstOp1->getZExtValue() == 0)
2952 return false;
2953
2954 // Also keep natural memory operands second if the loaded value is
2955 // only used here. Several comparisons have memory forms.
2956 if (isNaturalMemoryOperand(Op: C.Op1, ICmpType: C.ICmpType) && C.Op1.hasOneUse())
2957 return false;
2958
2959 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't.
2960 // In that case we generally prefer the memory to be second.
2961 if (isNaturalMemoryOperand(Op: C.Op0, ICmpType: C.ICmpType) && C.Op0.hasOneUse()) {
2962 // The only exceptions are when the second operand is a constant and
2963 // we can use things like CHHSI.
2964 if (!ConstOp1)
2965 return true;
2966 // The unsigned memory-immediate instructions can handle 16-bit
2967 // unsigned integers.
2968 if (C.ICmpType != SystemZICMP::SignedOnly &&
2969 isUInt<16>(x: ConstOp1->getZExtValue()))
2970 return false;
2971 // The signed memory-immediate instructions can handle 16-bit
2972 // signed integers.
2973 if (C.ICmpType != SystemZICMP::UnsignedOnly &&
2974 isInt<16>(x: ConstOp1->getSExtValue()))
2975 return false;
2976 return true;
2977 }
2978
2979 // Try to promote the use of CGFR and CLGFR.
2980 unsigned Opcode0 = C.Op0.getOpcode();
2981 if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND)
2982 return true;
2983 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND)
2984 return true;
2985 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::AND &&
2986 C.Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
2987 C.Op0.getConstantOperandVal(i: 1) == 0xffffffff)
2988 return true;
2989
2990 return false;
2991}
2992
2993// Check whether C tests for equality between X and Y and whether X - Y
2994// or Y - X is also computed. In that case it's better to compare the
2995// result of the subtraction against zero.
2996static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL,
2997 Comparison &C) {
2998 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
2999 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3000 for (SDNode *N : C.Op0->users()) {
3001 if (N->getOpcode() == ISD::SUB &&
3002 ((N->getOperand(Num: 0) == C.Op0 && N->getOperand(Num: 1) == C.Op1) ||
3003 (N->getOperand(Num: 0) == C.Op1 && N->getOperand(Num: 1) == C.Op0))) {
3004 // Disable the nsw and nuw flags: the backend needs to handle
3005 // overflow as well during comparison elimination.
3006 N->dropFlags(Mask: SDNodeFlags::NoWrap);
3007 C.Op0 = SDValue(N, 0);
3008 C.Op1 = DAG.getConstant(Val: 0, DL, VT: N->getValueType(ResNo: 0));
3009 return;
3010 }
3011 }
3012 }
3013}
3014
3015// Check whether C compares a floating-point value with zero and if that
3016// floating-point value is also negated. In this case we can use the
3017// negation to set CC, so avoiding separate LOAD AND TEST and
3018// LOAD (NEGATIVE/COMPLEMENT) instructions.
3019static void adjustForFNeg(Comparison &C) {
3020 // This optimization is invalid for strict comparisons, since FNEG
3021 // does not raise any exceptions.
3022 if (C.Chain)
3023 return;
3024 auto *C1 = dyn_cast<ConstantFPSDNode>(Val&: C.Op1);
3025 if (C1 && C1->isZero()) {
3026 for (SDNode *N : C.Op0->users()) {
3027 if (N->getOpcode() == ISD::FNEG) {
3028 C.Op0 = SDValue(N, 0);
3029 C.CCMask = SystemZ::reverseCCMask(CCMask: C.CCMask);
3030 return;
3031 }
3032 }
3033 }
3034}
3035
3036// Check whether C compares (shl X, 32) with 0 and whether X is
3037// also sign-extended. In that case it is better to test the result
3038// of the sign extension using LTGFR.
3039//
3040// This case is important because InstCombine transforms a comparison
3041// with (sext (trunc X)) into a comparison with (shl X, 32).
3042static void adjustForLTGFR(Comparison &C) {
3043 // Check for a comparison between (shl X, 32) and 0.
3044 if (C.Op0.getOpcode() == ISD::SHL && C.Op0.getValueType() == MVT::i64 &&
3045 C.Op1.getOpcode() == ISD::Constant && C.Op1->getAsZExtVal() == 0) {
3046 auto *C1 = dyn_cast<ConstantSDNode>(Val: C.Op0.getOperand(i: 1));
3047 if (C1 && C1->getZExtValue() == 32) {
3048 SDValue ShlOp0 = C.Op0.getOperand(i: 0);
3049 // See whether X has any SIGN_EXTEND_INREG uses.
3050 for (SDNode *N : ShlOp0->users()) {
3051 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3052 cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT() == MVT::i32) {
3053 C.Op0 = SDValue(N, 0);
3054 return;
3055 }
3056 }
3057 }
3058 }
3059}
3060
3061// If C compares the truncation of an extending load, try to compare
3062// the untruncated value instead. This exposes more opportunities to
3063// reuse CC.
3064static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL,
3065 Comparison &C) {
3066 if (C.Op0.getOpcode() == ISD::TRUNCATE &&
3067 C.Op0.getOperand(i: 0).getOpcode() == ISD::LOAD &&
3068 C.Op1.getOpcode() == ISD::Constant &&
3069 cast<ConstantSDNode>(Val&: C.Op1)->getValueSizeInBits(ResNo: 0) <= 64 &&
3070 C.Op1->getAsZExtVal() == 0) {
3071 auto *L = cast<LoadSDNode>(Val: C.Op0.getOperand(i: 0));
3072 if (L->getMemoryVT().getStoreSizeInBits().getFixedValue() <=
3073 C.Op0.getValueSizeInBits().getFixedValue()) {
3074 unsigned Type = L->getExtensionType();
3075 if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) ||
3076 (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) {
3077 C.Op0 = C.Op0.getOperand(i: 0);
3078 C.Op1 = DAG.getConstant(Val: 0, DL, VT: C.Op0.getValueType());
3079 }
3080 }
3081 }
3082}
3083
3084// Return true if shift operation N has an in-range constant shift value.
3085// Store it in ShiftVal if so.
3086static bool isSimpleShift(SDValue N, unsigned &ShiftVal) {
3087 auto *Shift = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
3088 if (!Shift)
3089 return false;
3090
3091 uint64_t Amount = Shift->getZExtValue();
3092 if (Amount >= N.getValueSizeInBits())
3093 return false;
3094
3095 ShiftVal = Amount;
3096 return true;
3097}
3098
3099// Check whether an AND with Mask is suitable for a TEST UNDER MASK
3100// instruction and whether the CC value is descriptive enough to handle
3101// a comparison of type Opcode between the AND result and CmpVal.
3102// CCMask says which comparison result is being tested and BitSize is
3103// the number of bits in the operands. If TEST UNDER MASK can be used,
3104// return the corresponding CC mask, otherwise return 0.
3105static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask,
3106 uint64_t Mask, uint64_t CmpVal,
3107 unsigned ICmpType) {
3108 assert(Mask != 0 && "ANDs with zero should have been removed by now");
3109
3110 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL.
3111 if (!SystemZ::isImmLL(Val: Mask) && !SystemZ::isImmLH(Val: Mask) &&
3112 !SystemZ::isImmHL(Val: Mask) && !SystemZ::isImmHH(Val: Mask))
3113 return 0;
3114
3115 // Work out the masks for the lowest and highest bits.
3116 uint64_t High = llvm::bit_floor(Value: Mask);
3117 uint64_t Low = uint64_t(1) << llvm::countr_zero(Val: Mask);
3118
3119 // Signed ordered comparisons are effectively unsigned if the sign
3120 // bit is dropped.
3121 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly);
3122
3123 // Check for equality comparisons with 0, or the equivalent.
3124 if (CmpVal == 0) {
3125 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3126 return SystemZ::CCMASK_TM_ALL_0;
3127 if (CCMask == SystemZ::CCMASK_CMP_NE)
3128 return SystemZ::CCMASK_TM_SOME_1;
3129 }
3130 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <= Low) {
3131 if (CCMask == SystemZ::CCMASK_CMP_LT)
3132 return SystemZ::CCMASK_TM_ALL_0;
3133 if (CCMask == SystemZ::CCMASK_CMP_GE)
3134 return SystemZ::CCMASK_TM_SOME_1;
3135 }
3136 if (EffectivelyUnsigned && CmpVal < Low) {
3137 if (CCMask == SystemZ::CCMASK_CMP_LE)
3138 return SystemZ::CCMASK_TM_ALL_0;
3139 if (CCMask == SystemZ::CCMASK_CMP_GT)
3140 return SystemZ::CCMASK_TM_SOME_1;
3141 }
3142
3143 // Check for equality comparisons with the mask, or the equivalent.
3144 if (CmpVal == Mask) {
3145 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3146 return SystemZ::CCMASK_TM_ALL_1;
3147 if (CCMask == SystemZ::CCMASK_CMP_NE)
3148 return SystemZ::CCMASK_TM_SOME_0;
3149 }
3150 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) {
3151 if (CCMask == SystemZ::CCMASK_CMP_GT)
3152 return SystemZ::CCMASK_TM_ALL_1;
3153 if (CCMask == SystemZ::CCMASK_CMP_LE)
3154 return SystemZ::CCMASK_TM_SOME_0;
3155 }
3156 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) {
3157 if (CCMask == SystemZ::CCMASK_CMP_GE)
3158 return SystemZ::CCMASK_TM_ALL_1;
3159 if (CCMask == SystemZ::CCMASK_CMP_LT)
3160 return SystemZ::CCMASK_TM_SOME_0;
3161 }
3162
3163 // Check for ordered comparisons with the top bit.
3164 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) {
3165 if (CCMask == SystemZ::CCMASK_CMP_LE)
3166 return SystemZ::CCMASK_TM_MSB_0;
3167 if (CCMask == SystemZ::CCMASK_CMP_GT)
3168 return SystemZ::CCMASK_TM_MSB_1;
3169 }
3170 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) {
3171 if (CCMask == SystemZ::CCMASK_CMP_LT)
3172 return SystemZ::CCMASK_TM_MSB_0;
3173 if (CCMask == SystemZ::CCMASK_CMP_GE)
3174 return SystemZ::CCMASK_TM_MSB_1;
3175 }
3176
3177 // If there are just two bits, we can do equality checks for Low and High
3178 // as well.
3179 if (Mask == Low + High) {
3180 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low)
3181 return SystemZ::CCMASK_TM_MIXED_MSB_0;
3182 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low)
3183 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY;
3184 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High)
3185 return SystemZ::CCMASK_TM_MIXED_MSB_1;
3186 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High)
3187 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY;
3188 }
3189
3190 // Looks like we've exhausted our options.
3191 return 0;
3192}
3193
3194// See whether C can be implemented as a TEST UNDER MASK instruction.
3195// Update the arguments with the TM version if so.
3196static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL,
3197 Comparison &C) {
3198 // Use VECTOR TEST UNDER MASK for i128 operations.
3199 if (C.Op0.getValueType() == MVT::i128) {
3200 // We can use VTM for EQ/NE comparisons of x & y against 0.
3201 if (C.Op0.getOpcode() == ISD::AND &&
3202 (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3203 C.CCMask == SystemZ::CCMASK_CMP_NE)) {
3204 auto *Mask = dyn_cast<ConstantSDNode>(Val&: C.Op1);
3205 if (Mask && Mask->getAPIntValue() == 0) {
3206 C.Opcode = SystemZISD::VTM;
3207 C.Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: C.Op0.getOperand(i: 1));
3208 C.Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: C.Op0.getOperand(i: 0));
3209 C.CCValid = SystemZ::CCMASK_VCMP;
3210 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3211 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3212 else
3213 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3214 }
3215 }
3216 return;
3217 }
3218
3219 // Check that we have a comparison with a constant.
3220 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val&: C.Op1);
3221 if (!ConstOp1)
3222 return;
3223 uint64_t CmpVal = ConstOp1->getZExtValue();
3224
3225 // Check whether the nonconstant input is an AND with a constant mask.
3226 Comparison NewC(C);
3227 uint64_t MaskVal;
3228 ConstantSDNode *Mask = nullptr;
3229 if (C.Op0.getOpcode() == ISD::AND) {
3230 NewC.Op0 = C.Op0.getOperand(i: 0);
3231 NewC.Op1 = C.Op0.getOperand(i: 1);
3232 Mask = dyn_cast<ConstantSDNode>(Val&: NewC.Op1);
3233 if (!Mask)
3234 return;
3235 MaskVal = Mask->getZExtValue();
3236 } else {
3237 // There is no instruction to compare with a 64-bit immediate
3238 // so use TMHH instead if possible. We need an unsigned ordered
3239 // comparison with an i64 immediate.
3240 if (NewC.Op0.getValueType() != MVT::i64 ||
3241 NewC.CCMask == SystemZ::CCMASK_CMP_EQ ||
3242 NewC.CCMask == SystemZ::CCMASK_CMP_NE ||
3243 NewC.ICmpType == SystemZICMP::SignedOnly)
3244 return;
3245 // Convert LE and GT comparisons into LT and GE.
3246 if (NewC.CCMask == SystemZ::CCMASK_CMP_LE ||
3247 NewC.CCMask == SystemZ::CCMASK_CMP_GT) {
3248 if (CmpVal == uint64_t(-1))
3249 return;
3250 CmpVal += 1;
3251 NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ;
3252 }
3253 // If the low N bits of Op1 are zero than the low N bits of Op0 can
3254 // be masked off without changing the result.
3255 MaskVal = -(CmpVal & -CmpVal);
3256 NewC.ICmpType = SystemZICMP::UnsignedOnly;
3257 }
3258 if (!MaskVal)
3259 return;
3260
3261 // Check whether the combination of mask, comparison value and comparison
3262 // type are suitable.
3263 unsigned BitSize = NewC.Op0.getValueSizeInBits();
3264 unsigned NewCCMask, ShiftVal;
3265 if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3266 NewC.Op0.getOpcode() == ISD::SHL &&
3267 isSimpleShift(N: NewC.Op0, ShiftVal) &&
3268 (MaskVal >> ShiftVal != 0) &&
3269 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
3270 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask,
3271 Mask: MaskVal >> ShiftVal,
3272 CmpVal: CmpVal >> ShiftVal,
3273 ICmpType: SystemZICMP::Any))) {
3274 NewC.Op0 = NewC.Op0.getOperand(i: 0);
3275 MaskVal >>= ShiftVal;
3276 } else if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3277 NewC.Op0.getOpcode() == ISD::SRL &&
3278 isSimpleShift(N: NewC.Op0, ShiftVal) &&
3279 (MaskVal << ShiftVal != 0) &&
3280 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
3281 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask,
3282 Mask: MaskVal << ShiftVal,
3283 CmpVal: CmpVal << ShiftVal,
3284 ICmpType: SystemZICMP::UnsignedOnly))) {
3285 NewC.Op0 = NewC.Op0.getOperand(i: 0);
3286 MaskVal <<= ShiftVal;
3287 } else {
3288 NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask, Mask: MaskVal, CmpVal,
3289 ICmpType: NewC.ICmpType);
3290 if (!NewCCMask)
3291 return;
3292 }
3293
3294 // Go ahead and make the change.
3295 C.Opcode = SystemZISD::TM;
3296 C.Op0 = NewC.Op0;
3297 if (Mask && Mask->getZExtValue() == MaskVal)
3298 C.Op1 = SDValue(Mask, 0);
3299 else
3300 C.Op1 = DAG.getConstant(Val: MaskVal, DL, VT: C.Op0.getValueType());
3301 C.CCValid = SystemZ::CCMASK_TM;
3302 C.CCMask = NewCCMask;
3303}
3304
3305// Implement i128 comparison in vector registers.
3306static void adjustICmp128(SelectionDAG &DAG, const SDLoc &DL,
3307 Comparison &C) {
3308 if (C.Opcode != SystemZISD::ICMP)
3309 return;
3310 if (C.Op0.getValueType() != MVT::i128)
3311 return;
3312
3313 // Recognize vector comparison reductions.
3314 if ((C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3315 C.CCMask == SystemZ::CCMASK_CMP_NE) &&
3316 (isNullConstant(V: C.Op1) || isAllOnesConstant(V: C.Op1))) {
3317 bool CmpEq = C.CCMask == SystemZ::CCMASK_CMP_EQ;
3318 bool CmpNull = isNullConstant(V: C.Op1);
3319 SDValue Src = peekThroughBitcasts(V: C.Op0);
3320 if (Src.hasOneUse() && isBitwiseNot(V: Src)) {
3321 Src = Src.getOperand(i: 0);
3322 CmpNull = !CmpNull;
3323 }
3324 unsigned Opcode = 0;
3325 if (Src.hasOneUse()) {
3326 switch (Src.getOpcode()) {
3327 case SystemZISD::VICMPE: Opcode = SystemZISD::VICMPES; break;
3328 case SystemZISD::VICMPH: Opcode = SystemZISD::VICMPHS; break;
3329 case SystemZISD::VICMPHL: Opcode = SystemZISD::VICMPHLS; break;
3330 case SystemZISD::VFCMPE: Opcode = SystemZISD::VFCMPES; break;
3331 case SystemZISD::VFCMPH: Opcode = SystemZISD::VFCMPHS; break;
3332 case SystemZISD::VFCMPHE: Opcode = SystemZISD::VFCMPHES; break;
3333 default: break;
3334 }
3335 }
3336 if (Opcode) {
3337 C.Opcode = Opcode;
3338 C.Op0 = Src->getOperand(Num: 0);
3339 C.Op1 = Src->getOperand(Num: 1);
3340 C.CCValid = SystemZ::CCMASK_VCMP;
3341 C.CCMask = CmpNull ? SystemZ::CCMASK_VCMP_NONE : SystemZ::CCMASK_VCMP_ALL;
3342 if (!CmpEq)
3343 C.CCMask ^= C.CCValid;
3344 return;
3345 }
3346 }
3347
3348 // Everything below here is not useful if we have native i128 compares.
3349 if (DAG.getSubtarget<SystemZSubtarget>().hasVectorEnhancements3())
3350 return;
3351
3352 // (In-)Equality comparisons can be implemented via VCEQGS.
3353 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3354 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3355 C.Opcode = SystemZISD::VICMPES;
3356 C.Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: C.Op0);
3357 C.Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: C.Op1);
3358 C.CCValid = SystemZ::CCMASK_VCMP;
3359 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3360 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3361 else
3362 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3363 return;
3364 }
3365
3366 // Normalize other comparisons to GT.
3367 bool Swap = false, Invert = false;
3368 switch (C.CCMask) {
3369 case SystemZ::CCMASK_CMP_GT: break;
3370 case SystemZ::CCMASK_CMP_LT: Swap = true; break;
3371 case SystemZ::CCMASK_CMP_LE: Invert = true; break;
3372 case SystemZ::CCMASK_CMP_GE: Swap = Invert = true; break;
3373 default: llvm_unreachable("Invalid integer condition!");
3374 }
3375 if (Swap)
3376 std::swap(a&: C.Op0, b&: C.Op1);
3377
3378 if (C.ICmpType == SystemZICMP::UnsignedOnly)
3379 C.Opcode = SystemZISD::UCMP128HI;
3380 else
3381 C.Opcode = SystemZISD::SCMP128HI;
3382 C.CCValid = SystemZ::CCMASK_ANY;
3383 C.CCMask = SystemZ::CCMASK_1;
3384
3385 if (Invert)
3386 C.CCMask ^= C.CCValid;
3387}
3388
3389// See whether the comparison argument contains a redundant AND
3390// and remove it if so. This sometimes happens due to the generic
3391// BRCOND expansion.
3392static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL,
3393 Comparison &C) {
3394 if (C.Op0.getOpcode() != ISD::AND)
3395 return;
3396 auto *Mask = dyn_cast<ConstantSDNode>(Val: C.Op0.getOperand(i: 1));
3397 if (!Mask || Mask->getValueSizeInBits(ResNo: 0) > 64)
3398 return;
3399 KnownBits Known = DAG.computeKnownBits(Op: C.Op0.getOperand(i: 0));
3400 if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue())
3401 return;
3402
3403 C.Op0 = C.Op0.getOperand(i: 0);
3404}
3405
3406// Return a Comparison that tests the condition-code result of intrinsic
3407// node Call against constant integer CC using comparison code Cond.
3408// Opcode is the opcode of the SystemZISD operation for the intrinsic
3409// and CCValid is the set of possible condition-code results.
3410static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
3411 SDValue Call, unsigned CCValid, uint64_t CC,
3412 ISD::CondCode Cond) {
3413 Comparison C(Call, SDValue(), SDValue());
3414 C.Opcode = Opcode;
3415 C.CCValid = CCValid;
3416 if (Cond == ISD::SETEQ)
3417 // bit 3 for CC==0, bit 0 for CC==3, always false for CC>3.
3418 C.CCMask = CC < 4 ? 1 << (3 - CC) : 0;
3419 else if (Cond == ISD::SETNE)
3420 // ...and the inverse of that.
3421 C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1;
3422 else if (Cond == ISD::SETLT || Cond == ISD::SETULT)
3423 // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3,
3424 // always true for CC>3.
3425 C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;
3426 else if (Cond == ISD::SETGE || Cond == ISD::SETUGE)
3427 // ...and the inverse of that.
3428 C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;
3429 else if (Cond == ISD::SETLE || Cond == ISD::SETULE)
3430 // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true),
3431 // always true for CC>3.
3432 C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;
3433 else if (Cond == ISD::SETGT || Cond == ISD::SETUGT)
3434 // ...and the inverse of that.
3435 C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;
3436 else
3437 llvm_unreachable("Unexpected integer comparison type");
3438 C.CCMask &= CCValid;
3439 return C;
3440}
3441
3442// Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1.
3443static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
3444 ISD::CondCode Cond, const SDLoc &DL,
3445 SDValue Chain = SDValue(),
3446 bool IsSignaling = false) {
3447 if (CmpOp1.getOpcode() == ISD::Constant) {
3448 assert(!Chain);
3449 unsigned Opcode, CCValid;
3450 if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
3451 CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(NUses: 1, Value: 0) &&
3452 isIntrinsicWithCCAndChain(Op: CmpOp0, Opcode, CCValid))
3453 return getIntrinsicCmp(DAG, Opcode, Call: CmpOp0, CCValid,
3454 CC: CmpOp1->getAsZExtVal(), Cond);
3455 if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
3456 CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 &&
3457 isIntrinsicWithCC(Op: CmpOp0, Opcode, CCValid))
3458 return getIntrinsicCmp(DAG, Opcode, Call: CmpOp0, CCValid,
3459 CC: CmpOp1->getAsZExtVal(), Cond);
3460 }
3461 Comparison C(CmpOp0, CmpOp1, Chain);
3462 C.CCMask = CCMaskForCondCode(CC: Cond);
3463 if (C.Op0.getValueType().isFloatingPoint()) {
3464 C.CCValid = SystemZ::CCMASK_FCMP;
3465 if (!C.Chain)
3466 C.Opcode = SystemZISD::FCMP;
3467 else if (!IsSignaling)
3468 C.Opcode = SystemZISD::STRICT_FCMP;
3469 else
3470 C.Opcode = SystemZISD::STRICT_FCMPS;
3471 adjustForFNeg(C);
3472 } else {
3473 assert(!C.Chain);
3474 C.CCValid = SystemZ::CCMASK_ICMP;
3475 C.Opcode = SystemZISD::ICMP;
3476 // Choose the type of comparison. Equality and inequality tests can
3477 // use either signed or unsigned comparisons. The choice also doesn't
3478 // matter if both sign bits are known to be clear. In those cases we
3479 // want to give the main isel code the freedom to choose whichever
3480 // form fits best.
3481 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3482 C.CCMask == SystemZ::CCMASK_CMP_NE ||
3483 (DAG.SignBitIsZero(Op: C.Op0) && DAG.SignBitIsZero(Op: C.Op1)))
3484 C.ICmpType = SystemZICMP::Any;
3485 else if (C.CCMask & SystemZ::CCMASK_CMP_UO)
3486 C.ICmpType = SystemZICMP::UnsignedOnly;
3487 else
3488 C.ICmpType = SystemZICMP::SignedOnly;
3489 C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
3490 adjustForRedundantAnd(DAG, DL, C);
3491 adjustZeroCmp(DAG, DL, C);
3492 adjustSubwordCmp(DAG, DL, C);
3493 adjustForSubtraction(DAG, DL, C);
3494 adjustForLTGFR(C);
3495 adjustICmpTruncate(DAG, DL, C);
3496 }
3497
3498 if (shouldSwapCmpOperands(C)) {
3499 std::swap(a&: C.Op0, b&: C.Op1);
3500 C.CCMask = SystemZ::reverseCCMask(CCMask: C.CCMask);
3501 }
3502
3503 adjustForTestUnderMask(DAG, DL, C);
3504 adjustICmp128(DAG, DL, C);
3505 return C;
3506}
3507
3508// Emit the comparison instruction described by C.
3509static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
3510 if (!C.Op1.getNode()) {
3511 SDNode *Node;
3512 switch (C.Op0.getOpcode()) {
3513 case ISD::INTRINSIC_W_CHAIN:
3514 Node = emitIntrinsicWithCCAndChain(DAG, Op: C.Op0, Opcode: C.Opcode);
3515 return SDValue(Node, 0);
3516 case ISD::INTRINSIC_WO_CHAIN:
3517 Node = emitIntrinsicWithCC(DAG, Op: C.Op0, Opcode: C.Opcode);
3518 return SDValue(Node, Node->getNumValues() - 1);
3519 default:
3520 llvm_unreachable("Invalid comparison operands");
3521 }
3522 }
3523 if (C.Opcode == SystemZISD::ICMP)
3524 return DAG.getNode(Opcode: SystemZISD::ICMP, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1,
3525 N3: DAG.getTargetConstant(Val: C.ICmpType, DL, VT: MVT::i32));
3526 if (C.Opcode == SystemZISD::TM) {
3527 bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
3528 bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1));
3529 return DAG.getNode(Opcode: SystemZISD::TM, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1,
3530 N3: DAG.getTargetConstant(Val: RegisterOnly, DL, VT: MVT::i32));
3531 }
3532 if (C.Opcode == SystemZISD::VICMPES ||
3533 C.Opcode == SystemZISD::VICMPHS ||
3534 C.Opcode == SystemZISD::VICMPHLS ||
3535 C.Opcode == SystemZISD::VFCMPES ||
3536 C.Opcode == SystemZISD::VFCMPHS ||
3537 C.Opcode == SystemZISD::VFCMPHES) {
3538 EVT IntVT = C.Op0.getValueType().changeVectorElementTypeToInteger();
3539 SDVTList VTs = DAG.getVTList(VT1: IntVT, VT2: MVT::i32);
3540 SDValue Val = DAG.getNode(Opcode: C.Opcode, DL, VTList: VTs, N1: C.Op0, N2: C.Op1);
3541 return SDValue(Val.getNode(), 1);
3542 }
3543 if (C.Chain) {
3544 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
3545 return DAG.getNode(Opcode: C.Opcode, DL, VTList: VTs, N1: C.Chain, N2: C.Op0, N3: C.Op1);
3546 }
3547 return DAG.getNode(Opcode: C.Opcode, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1);
3548}
3549
3550// Implement a 32-bit *MUL_LOHI operation by extending both operands to
3551// 64 bits. Extend is the extension type to use. Store the high part
3552// in Hi and the low part in Lo.
3553static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend,
3554 SDValue Op0, SDValue Op1, SDValue &Hi,
3555 SDValue &Lo) {
3556 Op0 = DAG.getNode(Opcode: Extend, DL, VT: MVT::i64, Operand: Op0);
3557 Op1 = DAG.getNode(Opcode: Extend, DL, VT: MVT::i64, Operand: Op1);
3558 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
3559 Hi = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Mul,
3560 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
3561 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Hi);
3562 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Mul);
3563}
3564
3565// Lower a binary operation that produces two VT results, one in each
3566// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation,
3567// and Opcode performs the GR128 operation. Store the even register result
3568// in Even and the odd register result in Odd.
3569static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
3570 unsigned Opcode, SDValue Op0, SDValue Op1,
3571 SDValue &Even, SDValue &Odd) {
3572 SDValue Result = DAG.getNode(Opcode, DL, VT: MVT::Untyped, N1: Op0, N2: Op1);
3573 bool Is32Bit = is32Bit(VT);
3574 Even = DAG.getTargetExtractSubreg(SRIdx: SystemZ::even128(Is32bit: Is32Bit), DL, VT, Operand: Result);
3575 Odd = DAG.getTargetExtractSubreg(SRIdx: SystemZ::odd128(Is32bit: Is32Bit), DL, VT, Operand: Result);
3576}
3577
3578// Return an i32 value that is 1 if the CC value produced by CCReg is
3579// in the mask CCMask and 0 otherwise. CC is known to have a value
3580// in CCValid, so other values can be ignored.
3581static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
3582 unsigned CCValid, unsigned CCMask) {
3583 SDValue Ops[] = {DAG.getConstant(Val: 1, DL, VT: MVT::i32),
3584 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
3585 DAG.getTargetConstant(Val: CCValid, DL, VT: MVT::i32),
3586 DAG.getTargetConstant(Val: CCMask, DL, VT: MVT::i32), CCReg};
3587 return DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: MVT::i32, Ops);
3588}
3589
3590// Return the SystemISD vector comparison operation for CC, or 0 if it cannot
3591// be done directly. Mode is CmpMode::Int for integer comparisons, CmpMode::FP
3592// for regular floating-point comparisons, CmpMode::StrictFP for strict (quiet)
3593// floating-point comparisons, and CmpMode::SignalingFP for strict signaling
3594// floating-point comparisons.
3595enum class CmpMode { Int, FP, StrictFP, SignalingFP };
3596static unsigned getVectorComparison(ISD::CondCode CC, CmpMode Mode) {
3597 switch (CC) {
3598 case ISD::SETOEQ:
3599 case ISD::SETEQ:
3600 switch (Mode) {
3601 case CmpMode::Int: return SystemZISD::VICMPE;
3602 case CmpMode::FP: return SystemZISD::VFCMPE;
3603 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPE;
3604 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPES;
3605 }
3606 llvm_unreachable("Bad mode");
3607
3608 case ISD::SETOGE:
3609 case ISD::SETGE:
3610 switch (Mode) {
3611 case CmpMode::Int: return 0;
3612 case CmpMode::FP: return SystemZISD::VFCMPHE;
3613 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPHE;
3614 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHES;
3615 }
3616 llvm_unreachable("Bad mode");
3617
3618 case ISD::SETOGT:
3619 case ISD::SETGT:
3620 switch (Mode) {
3621 case CmpMode::Int: return SystemZISD::VICMPH;
3622 case CmpMode::FP: return SystemZISD::VFCMPH;
3623 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPH;
3624 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHS;
3625 }
3626 llvm_unreachable("Bad mode");
3627
3628 case ISD::SETUGT:
3629 switch (Mode) {
3630 case CmpMode::Int: return SystemZISD::VICMPHL;
3631 case CmpMode::FP: return 0;
3632 case CmpMode::StrictFP: return 0;
3633 case CmpMode::SignalingFP: return 0;
3634 }
3635 llvm_unreachable("Bad mode");
3636
3637 default:
3638 return 0;
3639 }
3640}
3641
3642// Return the SystemZISD vector comparison operation for CC or its inverse,
3643// or 0 if neither can be done directly. Indicate in Invert whether the
3644// result is for the inverse of CC. Mode is as above.
3645static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, CmpMode Mode,
3646 bool &Invert) {
3647 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3648 Invert = false;
3649 return Opcode;
3650 }
3651
3652 CC = ISD::getSetCCInverse(Operation: CC, Type: Mode == CmpMode::Int ? MVT::i32 : MVT::f32);
3653 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3654 Invert = true;
3655 return Opcode;
3656 }
3657
3658 return 0;
3659}
3660
3661// Return a v2f64 that contains the extended form of elements Start and Start+1
3662// of v4f32 value Op. If Chain is nonnull, return the strict form.
3663static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
3664 SDValue Op, SDValue Chain) {
3665 int Mask[] = { Start, -1, Start + 1, -1 };
3666 Op = DAG.getVectorShuffle(VT: MVT::v4f32, dl: DL, N1: Op, N2: DAG.getUNDEF(VT: MVT::v4f32), Mask);
3667 if (Chain) {
3668 SDVTList VTs = DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other);
3669 return DAG.getNode(Opcode: SystemZISD::STRICT_VEXTEND, DL, VTList: VTs, N1: Chain, N2: Op);
3670 }
3671 return DAG.getNode(Opcode: SystemZISD::VEXTEND, DL, VT: MVT::v2f64, Operand: Op);
3672}
3673
3674// Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode,
3675// producing a result of type VT. If Chain is nonnull, return the strict form.
3676SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
3677 const SDLoc &DL, EVT VT,
3678 SDValue CmpOp0,
3679 SDValue CmpOp1,
3680 SDValue Chain) const {
3681 // There is no hardware support for v4f32 (unless we have the vector
3682 // enhancements facility 1), so extend the vector into two v2f64s
3683 // and compare those.
3684 if (CmpOp0.getValueType() == MVT::v4f32 &&
3685 !Subtarget.hasVectorEnhancements1()) {
3686 SDValue H0 = expandV4F32ToV2F64(DAG, Start: 0, DL, Op: CmpOp0, Chain);
3687 SDValue L0 = expandV4F32ToV2F64(DAG, Start: 2, DL, Op: CmpOp0, Chain);
3688 SDValue H1 = expandV4F32ToV2F64(DAG, Start: 0, DL, Op: CmpOp1, Chain);
3689 SDValue L1 = expandV4F32ToV2F64(DAG, Start: 2, DL, Op: CmpOp1, Chain);
3690 if (Chain) {
3691 SDVTList VTs = DAG.getVTList(VT1: MVT::v2i64, VT2: MVT::Other);
3692 SDValue HRes = DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: H0, N3: H1);
3693 SDValue LRes = DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: L0, N3: L1);
3694 SDValue Res = DAG.getNode(Opcode: SystemZISD::PACK, DL, VT, N1: HRes, N2: LRes);
3695 SDValue Chains[6] = { H0.getValue(R: 1), L0.getValue(R: 1),
3696 H1.getValue(R: 1), L1.getValue(R: 1),
3697 HRes.getValue(R: 1), LRes.getValue(R: 1) };
3698 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
3699 SDValue Ops[2] = { Res, NewChain };
3700 return DAG.getMergeValues(Ops, dl: DL);
3701 }
3702 SDValue HRes = DAG.getNode(Opcode, DL, VT: MVT::v2i64, N1: H0, N2: H1);
3703 SDValue LRes = DAG.getNode(Opcode, DL, VT: MVT::v2i64, N1: L0, N2: L1);
3704 return DAG.getNode(Opcode: SystemZISD::PACK, DL, VT, N1: HRes, N2: LRes);
3705 }
3706 if (Chain) {
3707 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::Other);
3708 return DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: CmpOp0, N3: CmpOp1);
3709 }
3710 return DAG.getNode(Opcode, DL, VT, N1: CmpOp0, N2: CmpOp1);
3711}
3712
3713// Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing
3714// an integer mask of type VT. If Chain is nonnull, we have a strict
3715// floating-point comparison. If in addition IsSignaling is true, we have
3716// a strict signaling floating-point comparison.
3717SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
3718 const SDLoc &DL, EVT VT,
3719 ISD::CondCode CC,
3720 SDValue CmpOp0,
3721 SDValue CmpOp1,
3722 SDValue Chain,
3723 bool IsSignaling) const {
3724 bool IsFP = CmpOp0.getValueType().isFloatingPoint();
3725 assert (!Chain || IsFP);
3726 assert (!IsSignaling || Chain);
3727 CmpMode Mode = IsSignaling ? CmpMode::SignalingFP :
3728 Chain ? CmpMode::StrictFP : IsFP ? CmpMode::FP : CmpMode::Int;
3729 bool Invert = false;
3730 SDValue Cmp;
3731 switch (CC) {
3732 // Handle tests for order using (or (ogt y x) (oge x y)).
3733 case ISD::SETUO:
3734 Invert = true;
3735 [[fallthrough]];
3736 case ISD::SETO: {
3737 assert(IsFP && "Unexpected integer comparison");
3738 SDValue LT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3739 DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3740 SDValue GE = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGE, Mode),
3741 DL, VT, CmpOp0, CmpOp1, Chain);
3742 Cmp = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LT, N2: GE);
3743 if (Chain)
3744 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
3745 N1: LT.getValue(R: 1), N2: GE.getValue(R: 1));
3746 break;
3747 }
3748
3749 // Handle <> tests using (or (ogt y x) (ogt x y)).
3750 case ISD::SETUEQ:
3751 Invert = true;
3752 [[fallthrough]];
3753 case ISD::SETONE: {
3754 assert(IsFP && "Unexpected integer comparison");
3755 SDValue LT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3756 DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3757 SDValue GT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3758 DL, VT, CmpOp0, CmpOp1, Chain);
3759 Cmp = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LT, N2: GT);
3760 if (Chain)
3761 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
3762 N1: LT.getValue(R: 1), N2: GT.getValue(R: 1));
3763 break;
3764 }
3765
3766 // Otherwise a single comparison is enough. It doesn't really
3767 // matter whether we try the inversion or the swap first, since
3768 // there are no cases where both work.
3769 default:
3770 // Optimize sign-bit comparisons to signed compares.
3771 if (Mode == CmpMode::Int && (CC == ISD::SETEQ || CC == ISD::SETNE) &&
3772 ISD::isConstantSplatVectorAllZeros(N: CmpOp1.getNode())) {
3773 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
3774 APInt Mask;
3775 if (CmpOp0.getOpcode() == ISD::AND
3776 && ISD::isConstantSplatVector(N: CmpOp0.getOperand(i: 1).getNode(), SplatValue&: Mask)
3777 && Mask == APInt::getSignMask(BitWidth: EltSize)) {
3778 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
3779 CmpOp0 = CmpOp0.getOperand(i: 0);
3780 }
3781 }
3782 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
3783 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1, Chain);
3784 else {
3785 CC = ISD::getSetCCSwappedOperands(Operation: CC);
3786 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
3787 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3788 else
3789 llvm_unreachable("Unhandled comparison");
3790 }
3791 if (Chain)
3792 Chain = Cmp.getValue(R: 1);
3793 break;
3794 }
3795 if (Invert) {
3796 SDValue Mask =
3797 DAG.getSplatBuildVector(VT, DL, Op: DAG.getAllOnesConstant(DL, VT: MVT::i64));
3798 Cmp = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Cmp, N2: Mask);
3799 }
3800 if (Chain && Chain.getNode() != Cmp.getNode()) {
3801 SDValue Ops[2] = { Cmp, Chain };
3802 Cmp = DAG.getMergeValues(Ops, dl: DL);
3803 }
3804 return Cmp;
3805}
3806
3807SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
3808 SelectionDAG &DAG) const {
3809 SDValue CmpOp0 = Op.getOperand(i: 0);
3810 SDValue CmpOp1 = Op.getOperand(i: 1);
3811 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
3812 SDLoc DL(Op);
3813 EVT VT = Op.getValueType();
3814 if (VT.isVector())
3815 return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
3816
3817 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
3818 SDValue CCReg = emitCmp(DAG, DL, C);
3819 return emitSETCC(DAG, DL, CCReg, CCValid: C.CCValid, CCMask: C.CCMask);
3820}
3821
3822SDValue SystemZTargetLowering::lowerSTRICT_FSETCC(SDValue Op,
3823 SelectionDAG &DAG,
3824 bool IsSignaling) const {
3825 SDValue Chain = Op.getOperand(i: 0);
3826 SDValue CmpOp0 = Op.getOperand(i: 1);
3827 SDValue CmpOp1 = Op.getOperand(i: 2);
3828 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 3))->get();
3829 SDLoc DL(Op);
3830 EVT VT = Op.getNode()->getValueType(ResNo: 0);
3831 if (VT.isVector()) {
3832 SDValue Res = lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1,
3833 Chain, IsSignaling);
3834 return Res.getValue(R: Op.getResNo());
3835 }
3836
3837 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL, Chain, IsSignaling));
3838 SDValue CCReg = emitCmp(DAG, DL, C);
3839 CCReg->setFlags(Op->getFlags());
3840 SDValue Result = emitSETCC(DAG, DL, CCReg, CCValid: C.CCValid, CCMask: C.CCMask);
3841 SDValue Ops[2] = { Result, CCReg.getValue(R: 1) };
3842 return DAG.getMergeValues(Ops, dl: DL);
3843}
3844
3845SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
3846 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 1))->get();
3847 SDValue CmpOp0 = Op.getOperand(i: 2);
3848 SDValue CmpOp1 = Op.getOperand(i: 3);
3849 SDValue Dest = Op.getOperand(i: 4);
3850 SDLoc DL(Op);
3851
3852 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
3853 SDValue CCReg = emitCmp(DAG, DL, C);
3854 return DAG.getNode(
3855 Opcode: SystemZISD::BR_CCMASK, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0),
3856 N2: DAG.getTargetConstant(Val: C.CCValid, DL, VT: MVT::i32),
3857 N3: DAG.getTargetConstant(Val: C.CCMask, DL, VT: MVT::i32), N4: Dest, N5: CCReg);
3858}
3859
3860// Return true if Pos is CmpOp and Neg is the negative of CmpOp,
3861// allowing Pos and Neg to be wider than CmpOp.
3862static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
3863 return (Neg.getOpcode() == ISD::SUB &&
3864 Neg.getOperand(i: 0).getOpcode() == ISD::Constant &&
3865 Neg.getConstantOperandVal(i: 0) == 0 && Neg.getOperand(i: 1) == Pos &&
3866 (Pos == CmpOp || (Pos.getOpcode() == ISD::SIGN_EXTEND &&
3867 Pos.getOperand(i: 0) == CmpOp)));
3868}
3869
3870// Return the absolute or negative absolute of Op; IsNegative decides which.
3871static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
3872 bool IsNegative) {
3873 Op = DAG.getNode(Opcode: ISD::ABS, DL, VT: Op.getValueType(), Operand: Op);
3874 if (IsNegative)
3875 Op = DAG.getNode(Opcode: ISD::SUB, DL, VT: Op.getValueType(),
3876 N1: DAG.getConstant(Val: 0, DL, VT: Op.getValueType()), N2: Op);
3877 return Op;
3878}
3879
3880static SDValue getI128Select(SelectionDAG &DAG, const SDLoc &DL,
3881 Comparison C, SDValue TrueOp, SDValue FalseOp) {
3882 EVT VT = MVT::i128;
3883 unsigned Op;
3884
3885 if (C.CCMask == SystemZ::CCMASK_CMP_NE ||
3886 C.CCMask == SystemZ::CCMASK_CMP_GE ||
3887 C.CCMask == SystemZ::CCMASK_CMP_LE) {
3888 std::swap(a&: TrueOp, b&: FalseOp);
3889 C.CCMask ^= C.CCValid;
3890 }
3891 if (C.CCMask == SystemZ::CCMASK_CMP_LT) {
3892 std::swap(a&: C.Op0, b&: C.Op1);
3893 C.CCMask = SystemZ::CCMASK_CMP_GT;
3894 }
3895 switch (C.CCMask) {
3896 case SystemZ::CCMASK_CMP_EQ:
3897 Op = SystemZISD::VICMPE;
3898 break;
3899 case SystemZ::CCMASK_CMP_GT:
3900 if (C.ICmpType == SystemZICMP::UnsignedOnly)
3901 Op = SystemZISD::VICMPHL;
3902 else
3903 Op = SystemZISD::VICMPH;
3904 break;
3905 default:
3906 llvm_unreachable("Unhandled comparison");
3907 break;
3908 }
3909
3910 SDValue Mask = DAG.getNode(Opcode: Op, DL, VT, N1: C.Op0, N2: C.Op1);
3911 TrueOp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: TrueOp, N2: Mask);
3912 FalseOp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: FalseOp, N2: DAG.getNOT(DL, Val: Mask, VT));
3913 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: TrueOp, N2: FalseOp);
3914}
3915
3916SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
3917 SelectionDAG &DAG) const {
3918 SDValue CmpOp0 = Op.getOperand(i: 0);
3919 SDValue CmpOp1 = Op.getOperand(i: 1);
3920 SDValue TrueOp = Op.getOperand(i: 2);
3921 SDValue FalseOp = Op.getOperand(i: 3);
3922 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
3923 SDLoc DL(Op);
3924
3925 // SELECT_CC involving f16 will not have the cmp-ops promoted by the
3926 // legalizer, as it will be handled according to the type of the resulting
3927 // value. Extend them here if needed.
3928 if (CmpOp0.getSimpleValueType() == MVT::f16) {
3929 CmpOp0 = DAG.getFPExtendOrRound(Op: CmpOp0, DL: SDLoc(CmpOp0), VT: MVT::f32);
3930 CmpOp1 = DAG.getFPExtendOrRound(Op: CmpOp1, DL: SDLoc(CmpOp1), VT: MVT::f32);
3931 }
3932
3933 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
3934
3935 // Check for absolute and negative-absolute selections, including those
3936 // where the comparison value is sign-extended (for LPGFR and LNGFR).
3937 // This check supplements the one in DAGCombiner.
3938 if (C.Opcode == SystemZISD::ICMP && C.CCMask != SystemZ::CCMASK_CMP_EQ &&
3939 C.CCMask != SystemZ::CCMASK_CMP_NE &&
3940 C.Op1.getOpcode() == ISD::Constant &&
3941 cast<ConstantSDNode>(Val&: C.Op1)->getValueSizeInBits(ResNo: 0) <= 64 &&
3942 C.Op1->getAsZExtVal() == 0) {
3943 if (isAbsolute(CmpOp: C.Op0, Pos: TrueOp, Neg: FalseOp))
3944 return getAbsolute(DAG, DL, Op: TrueOp, IsNegative: C.CCMask & SystemZ::CCMASK_CMP_LT);
3945 if (isAbsolute(CmpOp: C.Op0, Pos: FalseOp, Neg: TrueOp))
3946 return getAbsolute(DAG, DL, Op: FalseOp, IsNegative: C.CCMask & SystemZ::CCMASK_CMP_GT);
3947 }
3948
3949 if (Subtarget.hasVectorEnhancements3() &&
3950 C.Opcode == SystemZISD::ICMP &&
3951 C.Op0.getValueType() == MVT::i128 &&
3952 TrueOp.getValueType() == MVT::i128) {
3953 return getI128Select(DAG, DL, C, TrueOp, FalseOp);
3954 }
3955
3956 SDValue CCReg = emitCmp(DAG, DL, C);
3957 SDValue Ops[] = {TrueOp, FalseOp,
3958 DAG.getTargetConstant(Val: C.CCValid, DL, VT: MVT::i32),
3959 DAG.getTargetConstant(Val: C.CCMask, DL, VT: MVT::i32), CCReg};
3960
3961 return DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: Op.getValueType(), Ops);
3962}
3963
3964SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
3965 SelectionDAG &DAG) const {
3966 SDLoc DL(Node);
3967 const GlobalValue *GV = Node->getGlobal();
3968 int64_t Offset = Node->getOffset();
3969 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3970 CodeModel::Model CM = DAG.getTarget().getCodeModel();
3971
3972 SDValue Result;
3973 if (Subtarget.isPC32DBLSymbol(GV, CM)) {
3974 if (isInt<32>(x: Offset)) {
3975 // Assign anchors at 1<<12 byte boundaries.
3976 uint64_t Anchor = Offset & ~uint64_t(0xfff);
3977 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: Anchor);
3978 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
3979
3980 // The offset can be folded into the address if it is aligned to a
3981 // halfword.
3982 Offset -= Anchor;
3983 if (Offset != 0 && (Offset & 1) == 0) {
3984 SDValue Full =
3985 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: Anchor + Offset);
3986 Result = DAG.getNode(Opcode: SystemZISD::PCREL_OFFSET, DL, VT: PtrVT, N1: Full, N2: Result);
3987 Offset = 0;
3988 }
3989 } else {
3990 // Conservatively load a constant offset greater than 32 bits into a
3991 // register below.
3992 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT);
3993 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
3994 }
3995 } else if (Subtarget.isTargetELF()) {
3996 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: SystemZII::MO_GOT);
3997 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
3998 Result = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result,
3999 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
4000 } else if (Subtarget.isTargetzOS()) {
4001 Result = getADAEntry(DAG, GV, DL, PtrVT);
4002 } else
4003 llvm_unreachable("Unexpected Subtarget");
4004
4005 // If there was a non-zero offset that we didn't fold, create an explicit
4006 // addition for it.
4007 if (Offset != 0)
4008 Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Result,
4009 N2: DAG.getSignedConstant(Val: Offset, DL, VT: PtrVT));
4010
4011 return Result;
4012}
4013
4014SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
4015 SelectionDAG &DAG,
4016 unsigned Opcode,
4017 SDValue GOTOffset) const {
4018 SDLoc DL(Node);
4019 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4020 SDValue Chain = DAG.getEntryNode();
4021 SDValue Glue;
4022
4023 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4024 CallingConv::GHC)
4025 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4026
4027 // __tls_get_offset takes the GOT offset in %r2 and the GOT in %r12.
4028 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(VT: PtrVT);
4029 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R12D, N: GOT, Glue);
4030 Glue = Chain.getValue(R: 1);
4031 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R2D, N: GOTOffset, Glue);
4032 Glue = Chain.getValue(R: 1);
4033
4034 // The first call operand is the chain and the second is the TLS symbol.
4035 SmallVector<SDValue, 8> Ops;
4036 Ops.push_back(Elt: Chain);
4037 Ops.push_back(Elt: DAG.getTargetGlobalAddress(GV: Node->getGlobal(), DL,
4038 VT: Node->getValueType(ResNo: 0),
4039 offset: 0, TargetFlags: 0));
4040
4041 // Add argument registers to the end of the list so that they are
4042 // known live into the call.
4043 Ops.push_back(Elt: DAG.getRegister(Reg: SystemZ::R2D, VT: PtrVT));
4044 Ops.push_back(Elt: DAG.getRegister(Reg: SystemZ::R12D, VT: PtrVT));
4045
4046 // Add a register mask operand representing the call-preserved registers.
4047 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
4048 const uint32_t *Mask =
4049 TRI->getCallPreservedMask(MF: DAG.getMachineFunction(), CallingConv::C);
4050 assert(Mask && "Missing call preserved mask for calling convention");
4051 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
4052
4053 // Glue the call to the argument copies.
4054 Ops.push_back(Elt: Glue);
4055
4056 // Emit the call.
4057 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
4058 Chain = DAG.getNode(Opcode, DL, VTList: NodeTys, Ops);
4059 Glue = Chain.getValue(R: 1);
4060
4061 // Copy the return value from %r2.
4062 return DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::R2D, VT: PtrVT, Glue);
4063}
4064
4065SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL,
4066 SelectionDAG &DAG) const {
4067 SDValue Chain = DAG.getEntryNode();
4068 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4069
4070 // The high part of the thread pointer is in access register 0.
4071 SDValue TPHi = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::A0, VT: MVT::i32);
4072 TPHi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PtrVT, Operand: TPHi);
4073
4074 // The low part of the thread pointer is in access register 1.
4075 SDValue TPLo = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::A1, VT: MVT::i32);
4076 TPLo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: PtrVT, Operand: TPLo);
4077
4078 // Merge them into a single 64-bit address.
4079 SDValue TPHiShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: TPHi,
4080 N2: DAG.getConstant(Val: 32, DL, VT: PtrVT));
4081 return DAG.getNode(Opcode: ISD::OR, DL, VT: PtrVT, N1: TPHiShifted, N2: TPLo);
4082}
4083
4084SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
4085 SelectionDAG &DAG) const {
4086 if (DAG.getTarget().useEmulatedTLS())
4087 return LowerToTLSEmulatedModel(GA: Node, DAG);
4088 SDLoc DL(Node);
4089 const GlobalValue *GV = Node->getGlobal();
4090 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4091 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
4092
4093 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4094 CallingConv::GHC)
4095 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4096
4097 SDValue TP = lowerThreadPointer(DL, DAG);
4098
4099 // Get the offset of GA from the thread pointer, based on the TLS model.
4100 SDValue Offset;
4101 switch (model) {
4102 case TLSModel::GeneralDynamic: {
4103 // Load the GOT offset of the tls_index (module ID / per-symbol offset).
4104 SystemZConstantPoolValue *CPV =
4105 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::TLSGD);
4106
4107 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4108 Offset = DAG.getLoad(
4109 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4110 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4111
4112 // Call __tls_get_offset to retrieve the offset.
4113 Offset = lowerTLSGetOffset(Node, DAG, Opcode: SystemZISD::TLS_GDCALL, GOTOffset: Offset);
4114 break;
4115 }
4116
4117 case TLSModel::LocalDynamic: {
4118 // Load the GOT offset of the module ID.
4119 SystemZConstantPoolValue *CPV =
4120 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::TLSLDM);
4121
4122 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4123 Offset = DAG.getLoad(
4124 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4125 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4126
4127 // Call __tls_get_offset to retrieve the module base offset.
4128 Offset = lowerTLSGetOffset(Node, DAG, Opcode: SystemZISD::TLS_LDCALL, GOTOffset: Offset);
4129
4130 // Note: The SystemZLDCleanupPass will remove redundant computations
4131 // of the module base offset. Count total number of local-dynamic
4132 // accesses to trigger execution of that pass.
4133 SystemZMachineFunctionInfo* MFI =
4134 DAG.getMachineFunction().getInfo<SystemZMachineFunctionInfo>();
4135 MFI->incNumLocalDynamicTLSAccesses();
4136
4137 // Add the per-symbol offset.
4138 CPV = SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::DTPOFF);
4139
4140 SDValue DTPOffset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4141 DTPOffset = DAG.getLoad(
4142 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: DTPOffset,
4143 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4144
4145 Offset = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Offset, N2: DTPOffset);
4146 break;
4147 }
4148
4149 case TLSModel::InitialExec: {
4150 // Load the offset from the GOT.
4151 Offset = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0,
4152 TargetFlags: SystemZII::MO_INDNTPOFF);
4153 Offset = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Offset);
4154 Offset =
4155 DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4156 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
4157 break;
4158 }
4159
4160 case TLSModel::LocalExec: {
4161 // Force the offset into the constant pool and load it from there.
4162 SystemZConstantPoolValue *CPV =
4163 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::NTPOFF);
4164
4165 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4166 Offset = DAG.getLoad(
4167 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4168 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4169 break;
4170 }
4171 }
4172
4173 // Add the base and offset together.
4174 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TP, N2: Offset);
4175}
4176
4177SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
4178 SelectionDAG &DAG) const {
4179 SDLoc DL(Node);
4180 const BlockAddress *BA = Node->getBlockAddress();
4181 int64_t Offset = Node->getOffset();
4182 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4183
4184 SDValue Result = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset);
4185 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4186 return Result;
4187}
4188
4189SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
4190 SelectionDAG &DAG) const {
4191 SDLoc DL(JT);
4192 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4193 SDValue Result = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT);
4194
4195 // Use LARL to load the address of the table.
4196 return DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4197}
4198
4199SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP,
4200 SelectionDAG &DAG) const {
4201 SDLoc DL(CP);
4202 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4203
4204 SDValue Result;
4205 if (CP->isMachineConstantPoolEntry())
4206 Result =
4207 DAG.getTargetConstantPool(C: CP->getMachineCPVal(), VT: PtrVT, Align: CP->getAlign());
4208 else
4209 Result = DAG.getTargetConstantPool(C: CP->getConstVal(), VT: PtrVT, Align: CP->getAlign(),
4210 Offset: CP->getOffset());
4211
4212 // Use LARL to load the address of the constant pool entry.
4213 return DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4214}
4215
4216SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
4217 SelectionDAG &DAG) const {
4218 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4219 MachineFunction &MF = DAG.getMachineFunction();
4220 MachineFrameInfo &MFI = MF.getFrameInfo();
4221 MFI.setFrameAddressIsTaken(true);
4222
4223 SDLoc DL(Op);
4224 unsigned Depth = Op.getConstantOperandVal(i: 0);
4225 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4226
4227 // By definition, the frame address is the address of the back chain. (In
4228 // the case of packed stack without backchain, return the address where the
4229 // backchain would have been stored. This will either be an unused space or
4230 // contain a saved register).
4231 int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
4232 SDValue BackChain = DAG.getFrameIndex(FI: BackChainIdx, VT: PtrVT);
4233
4234 if (Depth > 0) {
4235 // FIXME The frontend should detect this case.
4236 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4237 report_fatal_error(reason: "Unsupported stack frame traversal count");
4238
4239 SDValue Offset = DAG.getConstant(Val: TFL->getBackchainOffset(MF), DL, VT: PtrVT);
4240 while (Depth--) {
4241 BackChain = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: BackChain,
4242 PtrInfo: MachinePointerInfo());
4243 BackChain = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: BackChain, N2: Offset);
4244 }
4245 }
4246
4247 return BackChain;
4248}
4249
4250SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
4251 SelectionDAG &DAG) const {
4252 MachineFunction &MF = DAG.getMachineFunction();
4253 MachineFrameInfo &MFI = MF.getFrameInfo();
4254 MFI.setReturnAddressIsTaken(true);
4255
4256 SDLoc DL(Op);
4257 unsigned Depth = Op.getConstantOperandVal(i: 0);
4258 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4259
4260 if (Depth > 0) {
4261 // FIXME The frontend should detect this case.
4262 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4263 report_fatal_error(reason: "Unsupported stack frame traversal count");
4264
4265 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
4266 const auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4267 int Offset = TFL->getReturnAddressOffset(MF);
4268 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FrameAddr,
4269 N2: DAG.getSignedConstant(Val: Offset, DL, VT: PtrVT));
4270 return DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr,
4271 PtrInfo: MachinePointerInfo());
4272 }
4273
4274 // Return R14D (Elf) / R7D (XPLINK), which has the return address. Mark it an
4275 // implicit live-in.
4276 SystemZCallingConventionRegisters *CCR = Subtarget.getSpecialRegisters();
4277 Register LinkReg = MF.addLiveIn(PReg: CCR->getReturnFunctionAddressRegister(),
4278 RC: &SystemZ::GR64BitRegClass);
4279 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: LinkReg, VT: PtrVT);
4280}
4281
4282SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
4283 SelectionDAG &DAG) const {
4284 SDLoc DL(Op);
4285 SDValue In = Op.getOperand(i: 0);
4286 EVT InVT = In.getValueType();
4287 EVT ResVT = Op.getValueType();
4288
4289 // Convert loads directly. This is normally done by DAGCombiner,
4290 // but we need this case for bitcasts that are created during lowering
4291 // and which are then lowered themselves.
4292 if (auto *LoadN = dyn_cast<LoadSDNode>(Val&: In))
4293 if (ISD::isNormalLoad(N: LoadN)) {
4294 SDValue NewLoad = DAG.getLoad(VT: ResVT, dl: DL, Chain: LoadN->getChain(),
4295 Ptr: LoadN->getBasePtr(), MMO: LoadN->getMemOperand());
4296 // Update the chain uses.
4297 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LoadN, 1), To: NewLoad.getValue(R: 1));
4298 return NewLoad;
4299 }
4300
4301 if (InVT == MVT::i32 && ResVT == MVT::f32) {
4302 SDValue In64;
4303 if (Subtarget.hasHighWord()) {
4304 SDNode *U64 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL,
4305 VT: MVT::i64);
4306 In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h32, DL,
4307 VT: MVT::i64, Operand: SDValue(U64, 0), Subreg: In);
4308 } else {
4309 In64 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: In);
4310 In64 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: In64,
4311 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
4312 }
4313 SDValue Out64 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: In64);
4314 return DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h32,
4315 DL, VT: MVT::f32, Operand: Out64);
4316 }
4317 if (InVT == MVT::f32 && ResVT == MVT::i32) {
4318 SDNode *U64 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::f64);
4319 SDValue In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h32, DL,
4320 VT: MVT::f64, Operand: SDValue(U64, 0), Subreg: In);
4321 SDValue Out64 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: In64);
4322 if (Subtarget.hasHighWord())
4323 return DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h32, DL,
4324 VT: MVT::i32, Operand: Out64);
4325 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Out64,
4326 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
4327 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Shift);
4328 }
4329 llvm_unreachable("Unexpected bitcast combination");
4330}
4331
4332SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
4333 SelectionDAG &DAG) const {
4334
4335 if (Subtarget.isTargetXPLINK64())
4336 return lowerVASTART_XPLINK(Op, DAG);
4337 else
4338 return lowerVASTART_ELF(Op, DAG);
4339}
4340
4341SDValue SystemZTargetLowering::lowerVASTART_XPLINK(SDValue Op,
4342 SelectionDAG &DAG) const {
4343 MachineFunction &MF = DAG.getMachineFunction();
4344 SystemZMachineFunctionInfo *FuncInfo =
4345 MF.getInfo<SystemZMachineFunctionInfo>();
4346
4347 SDLoc DL(Op);
4348
4349 // vastart just stores the address of the VarArgsFrameIndex slot into the
4350 // memory location argument.
4351 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4352 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4353 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4354 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: FR, Ptr: Op.getOperand(i: 1),
4355 PtrInfo: MachinePointerInfo(SV));
4356}
4357
4358SDValue SystemZTargetLowering::lowerVASTART_ELF(SDValue Op,
4359 SelectionDAG &DAG) const {
4360 MachineFunction &MF = DAG.getMachineFunction();
4361 SystemZMachineFunctionInfo *FuncInfo =
4362 MF.getInfo<SystemZMachineFunctionInfo>();
4363 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4364
4365 SDValue Chain = Op.getOperand(i: 0);
4366 SDValue Addr = Op.getOperand(i: 1);
4367 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4368 SDLoc DL(Op);
4369
4370 // The initial values of each field.
4371 const unsigned NumFields = 4;
4372 SDValue Fields[NumFields] = {
4373 DAG.getConstant(Val: FuncInfo->getVarArgsFirstGPR(), DL, VT: PtrVT),
4374 DAG.getConstant(Val: FuncInfo->getVarArgsFirstFPR(), DL, VT: PtrVT),
4375 DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT),
4376 DAG.getFrameIndex(FI: FuncInfo->getRegSaveFrameIndex(), VT: PtrVT)
4377 };
4378
4379 // Store each field into its respective slot.
4380 SDValue MemOps[NumFields];
4381 unsigned Offset = 0;
4382 for (unsigned I = 0; I < NumFields; ++I) {
4383 SDValue FieldAddr = Addr;
4384 if (Offset != 0)
4385 FieldAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FieldAddr,
4386 N2: DAG.getIntPtrConstant(Val: Offset, DL));
4387 MemOps[I] = DAG.getStore(Chain, dl: DL, Val: Fields[I], Ptr: FieldAddr,
4388 PtrInfo: MachinePointerInfo(SV, Offset));
4389 Offset += 8;
4390 }
4391 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOps);
4392}
4393
4394SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
4395 SelectionDAG &DAG) const {
4396 SDValue Chain = Op.getOperand(i: 0);
4397 SDValue DstPtr = Op.getOperand(i: 1);
4398 SDValue SrcPtr = Op.getOperand(i: 2);
4399 const Value *DstSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 3))->getValue();
4400 const Value *SrcSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
4401 SDLoc DL(Op);
4402
4403 uint32_t Sz =
4404 Subtarget.isTargetXPLINK64() ? getTargetMachine().getPointerSize(AS: 0) : 32;
4405 return DAG.getMemcpy(Chain, dl: DL, Dst: DstPtr, Src: SrcPtr, Size: DAG.getIntPtrConstant(Val: Sz, DL),
4406 Alignment: Align(8), /*isVolatile*/ isVol: false, /*AlwaysInline*/ false,
4407 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(DstSV),
4408 SrcPtrInfo: MachinePointerInfo(SrcSV));
4409}
4410
4411SDValue
4412SystemZTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
4413 SelectionDAG &DAG) const {
4414 if (Subtarget.isTargetXPLINK64())
4415 return lowerDYNAMIC_STACKALLOC_XPLINK(Op, DAG);
4416 else
4417 return lowerDYNAMIC_STACKALLOC_ELF(Op, DAG);
4418}
4419
4420SDValue
4421SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_XPLINK(SDValue Op,
4422 SelectionDAG &DAG) const {
4423 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4424 MachineFunction &MF = DAG.getMachineFunction();
4425 bool RealignOpt = !MF.getFunction().hasFnAttribute(Kind: "no-realign-stack");
4426 SDValue Chain = Op.getOperand(i: 0);
4427 SDValue Size = Op.getOperand(i: 1);
4428 SDValue Align = Op.getOperand(i: 2);
4429 SDLoc DL(Op);
4430
4431 // If user has set the no alignment function attribute, ignore
4432 // alloca alignments.
4433 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4434
4435 uint64_t StackAlign = TFI->getStackAlignment();
4436 uint64_t RequiredAlign = std::max(a: AlignVal, b: StackAlign);
4437 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4438
4439 SDValue NeededSpace = Size;
4440
4441 // Add extra space for alignment if needed.
4442 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4443 if (ExtraAlignSpace)
4444 NeededSpace = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: NeededSpace,
4445 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: PtrVT));
4446
4447 bool IsSigned = false;
4448 bool DoesNotReturn = false;
4449 bool IsReturnValueUsed = false;
4450 EVT VT = Op.getValueType();
4451 SDValue AllocaCall =
4452 makeExternalCall(Chain, DAG, CalleeName: "@@ALCAXP", RetVT: VT, Ops: ArrayRef(NeededSpace),
4453 CallConv: CallingConv::C, IsSigned, DL, DoesNotReturn,
4454 IsReturnValueUsed)
4455 .first;
4456
4457 // Perform a CopyFromReg from %GPR4 (stack pointer register). Chain and Glue
4458 // to end of call in order to ensure it isn't broken up from the call
4459 // sequence.
4460 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
4461 Register SPReg = Regs.getStackPointerRegister();
4462 Chain = AllocaCall.getValue(R: 1);
4463 SDValue Glue = AllocaCall.getValue(R: 2);
4464 SDValue NewSPRegNode = DAG.getCopyFromReg(Chain, dl: DL, Reg: SPReg, VT: PtrVT, Glue);
4465 Chain = NewSPRegNode.getValue(R: 1);
4466
4467 MVT PtrMVT = getPointerMemTy(DL: MF.getDataLayout());
4468 SDValue ArgAdjust = DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: PtrMVT);
4469 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrMVT, N1: NewSPRegNode, N2: ArgAdjust);
4470
4471 // Dynamically realign if needed.
4472 if (ExtraAlignSpace) {
4473 Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Result,
4474 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: PtrVT));
4475 Result = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Result,
4476 N2: DAG.getConstant(Val: ~(RequiredAlign - 1), DL, VT: PtrVT));
4477 }
4478
4479 SDValue Ops[2] = {Result, Chain};
4480 return DAG.getMergeValues(Ops, dl: DL);
4481}
4482
4483SDValue
4484SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_ELF(SDValue Op,
4485 SelectionDAG &DAG) const {
4486 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4487 MachineFunction &MF = DAG.getMachineFunction();
4488 bool RealignOpt = !MF.getFunction().hasFnAttribute(Kind: "no-realign-stack");
4489 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
4490
4491 SDValue Chain = Op.getOperand(i: 0);
4492 SDValue Size = Op.getOperand(i: 1);
4493 SDValue Align = Op.getOperand(i: 2);
4494 SDLoc DL(Op);
4495
4496 // If user has set the no alignment function attribute, ignore
4497 // alloca alignments.
4498 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4499
4500 uint64_t StackAlign = TFI->getStackAlignment();
4501 uint64_t RequiredAlign = std::max(a: AlignVal, b: StackAlign);
4502 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4503
4504 Register SPReg = getStackPointerRegisterToSaveRestore();
4505 SDValue NeededSpace = Size;
4506
4507 // Get a reference to the stack pointer.
4508 SDValue OldSP = DAG.getCopyFromReg(Chain, dl: DL, Reg: SPReg, VT: MVT::i64);
4509
4510 // If we need a backchain, save it now.
4511 SDValue Backchain;
4512 if (StoreBackchain)
4513 Backchain = DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: getBackchainAddress(SP: OldSP, DAG),
4514 PtrInfo: MachinePointerInfo());
4515
4516 // Add extra space for alignment if needed.
4517 if (ExtraAlignSpace)
4518 NeededSpace = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: NeededSpace,
4519 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: MVT::i64));
4520
4521 // Get the new stack pointer value.
4522 SDValue NewSP;
4523 if (hasInlineStackProbe(MF)) {
4524 NewSP = DAG.getNode(Opcode: SystemZISD::PROBED_ALLOCA, DL,
4525 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N1: Chain, N2: OldSP, N3: NeededSpace);
4526 Chain = NewSP.getValue(R: 1);
4527 }
4528 else {
4529 NewSP = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: OldSP, N2: NeededSpace);
4530 // Copy the new stack pointer back.
4531 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SPReg, N: NewSP);
4532 }
4533
4534 // The allocated data lives above the 160 bytes allocated for the standard
4535 // frame, plus any outgoing stack arguments. We don't know how much that
4536 // amounts to yet, so emit a special ADJDYNALLOC placeholder.
4537 SDValue ArgAdjust = DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: MVT::i64);
4538 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: NewSP, N2: ArgAdjust);
4539
4540 // Dynamically realign if needed.
4541 if (RequiredAlign > StackAlign) {
4542 Result =
4543 DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Result,
4544 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: MVT::i64));
4545 Result =
4546 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, N1: Result,
4547 N2: DAG.getConstant(Val: ~(RequiredAlign - 1), DL, VT: MVT::i64));
4548 }
4549
4550 if (StoreBackchain)
4551 Chain = DAG.getStore(Chain, dl: DL, Val: Backchain, Ptr: getBackchainAddress(SP: NewSP, DAG),
4552 PtrInfo: MachinePointerInfo());
4553
4554 SDValue Ops[2] = { Result, Chain };
4555 return DAG.getMergeValues(Ops, dl: DL);
4556}
4557
4558SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
4559 SDValue Op, SelectionDAG &DAG) const {
4560 SDLoc DL(Op);
4561
4562 return DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: MVT::i64);
4563}
4564
4565SDValue SystemZTargetLowering::lowerMULH(SDValue Op,
4566 SelectionDAG &DAG,
4567 unsigned Opcode) const {
4568 EVT VT = Op.getValueType();
4569 SDLoc DL(Op);
4570 SDValue Even, Odd;
4571
4572 // This custom expander is only used on z17 and later for 64-bit types.
4573 assert(!is32Bit(VT));
4574 assert(Subtarget.hasMiscellaneousExtensions2());
4575
4576 // SystemZISD::xMUL_LOHI returns the low result in the odd register and
4577 // the high result in the even register. Return the latter.
4578 lowerGR128Binary(DAG, DL, VT, Opcode,
4579 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even, Odd);
4580 return Even;
4581}
4582
4583SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
4584 SelectionDAG &DAG) const {
4585 EVT VT = Op.getValueType();
4586 SDLoc DL(Op);
4587 SDValue Ops[2];
4588 if (is32Bit(VT))
4589 // Just do a normal 64-bit multiplication and extract the results.
4590 // We define this so that it can be used for constant division.
4591 lowerMUL_LOHI32(DAG, DL, Extend: ISD::SIGN_EXTEND, Op0: Op.getOperand(i: 0),
4592 Op1: Op.getOperand(i: 1), Hi&: Ops[1], Lo&: Ops[0]);
4593 else if (Subtarget.hasMiscellaneousExtensions2())
4594 // SystemZISD::SMUL_LOHI returns the low result in the odd register and
4595 // the high result in the even register. ISD::SMUL_LOHI is defined to
4596 // return the low half first, so the results are in reverse order.
4597 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::SMUL_LOHI,
4598 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4599 else {
4600 // Do a full 128-bit multiplication based on SystemZISD::UMUL_LOHI:
4601 //
4602 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64)
4603 //
4604 // but using the fact that the upper halves are either all zeros
4605 // or all ones:
4606 //
4607 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64)
4608 //
4609 // and grouping the right terms together since they are quicker than the
4610 // multiplication:
4611 //
4612 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64)
4613 SDValue C63 = DAG.getConstant(Val: 63, DL, VT: MVT::i64);
4614 SDValue LL = Op.getOperand(i: 0);
4615 SDValue RL = Op.getOperand(i: 1);
4616 SDValue LH = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LL, N2: C63);
4617 SDValue RH = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: RL, N2: C63);
4618 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4619 // the high result in the even register. ISD::SMUL_LOHI is defined to
4620 // return the low half first, so the results are in reverse order.
4621 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UMUL_LOHI,
4622 Op0: LL, Op1: RL, Even&: Ops[1], Odd&: Ops[0]);
4623 SDValue NegLLTimesRH = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LL, N2: RH);
4624 SDValue NegLHTimesRL = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LH, N2: RL);
4625 SDValue NegSum = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: NegLLTimesRH, N2: NegLHTimesRL);
4626 Ops[1] = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Ops[1], N2: NegSum);
4627 }
4628 return DAG.getMergeValues(Ops, dl: DL);
4629}
4630
4631SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op,
4632 SelectionDAG &DAG) const {
4633 EVT VT = Op.getValueType();
4634 SDLoc DL(Op);
4635 SDValue Ops[2];
4636 if (is32Bit(VT))
4637 // Just do a normal 64-bit multiplication and extract the results.
4638 // We define this so that it can be used for constant division.
4639 lowerMUL_LOHI32(DAG, DL, Extend: ISD::ZERO_EXTEND, Op0: Op.getOperand(i: 0),
4640 Op1: Op.getOperand(i: 1), Hi&: Ops[1], Lo&: Ops[0]);
4641 else
4642 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4643 // the high result in the even register. ISD::UMUL_LOHI is defined to
4644 // return the low half first, so the results are in reverse order.
4645 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UMUL_LOHI,
4646 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4647 return DAG.getMergeValues(Ops, dl: DL);
4648}
4649
4650SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op,
4651 SelectionDAG &DAG) const {
4652 SDValue Op0 = Op.getOperand(i: 0);
4653 SDValue Op1 = Op.getOperand(i: 1);
4654 EVT VT = Op.getValueType();
4655 SDLoc DL(Op);
4656
4657 // We use DSGF for 32-bit division. This means the first operand must
4658 // always be 64-bit, and the second operand should be 32-bit whenever
4659 // that is possible, to improve performance.
4660 if (is32Bit(VT))
4661 Op0 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: Op0);
4662 else if (DAG.ComputeNumSignBits(Op: Op1) > 32)
4663 Op1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Op1);
4664
4665 // DSG(F) returns the remainder in the even register and the
4666 // quotient in the odd register.
4667 SDValue Ops[2];
4668 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::SDIVREM, Op0, Op1, Even&: Ops[1], Odd&: Ops[0]);
4669 return DAG.getMergeValues(Ops, dl: DL);
4670}
4671
4672SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op,
4673 SelectionDAG &DAG) const {
4674 EVT VT = Op.getValueType();
4675 SDLoc DL(Op);
4676
4677 // DL(G) returns the remainder in the even register and the
4678 // quotient in the odd register.
4679 SDValue Ops[2];
4680 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UDIVREM,
4681 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4682 return DAG.getMergeValues(Ops, dl: DL);
4683}
4684
4685SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
4686 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation");
4687
4688 // Get the known-zero masks for each operand.
4689 SDValue Ops[] = {Op.getOperand(i: 0), Op.getOperand(i: 1)};
4690 KnownBits Known[2] = {DAG.computeKnownBits(Op: Ops[0]),
4691 DAG.computeKnownBits(Op: Ops[1])};
4692
4693 // See if the upper 32 bits of one operand and the lower 32 bits of the
4694 // other are known zero. They are the low and high operands respectively.
4695 uint64_t Masks[] = { Known[0].Zero.getZExtValue(),
4696 Known[1].Zero.getZExtValue() };
4697 unsigned High, Low;
4698 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff)
4699 High = 1, Low = 0;
4700 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff)
4701 High = 0, Low = 1;
4702 else
4703 return Op;
4704
4705 SDValue LowOp = Ops[Low];
4706 SDValue HighOp = Ops[High];
4707
4708 // If the high part is a constant, we're better off using IILH.
4709 if (HighOp.getOpcode() == ISD::Constant)
4710 return Op;
4711
4712 // If the low part is a constant that is outside the range of LHI,
4713 // then we're better off using IILF.
4714 if (LowOp.getOpcode() == ISD::Constant) {
4715 int64_t Value = int32_t(LowOp->getAsZExtVal());
4716 if (!isInt<16>(x: Value))
4717 return Op;
4718 }
4719
4720 // Check whether the high part is an AND that doesn't change the
4721 // high 32 bits and just masks out low bits. We can skip it if so.
4722 if (HighOp.getOpcode() == ISD::AND &&
4723 HighOp.getOperand(i: 1).getOpcode() == ISD::Constant) {
4724 SDValue HighOp0 = HighOp.getOperand(i: 0);
4725 uint64_t Mask = HighOp.getConstantOperandVal(i: 1);
4726 if (DAG.MaskedValueIsZero(Op: HighOp0, Mask: APInt(64, ~(Mask | 0xffffffff))))
4727 HighOp = HighOp0;
4728 }
4729
4730 // Take advantage of the fact that all GR32 operations only change the
4731 // low 32 bits by truncating Low to an i32 and inserting it directly
4732 // using a subreg. The interesting cases are those where the truncation
4733 // can be folded.
4734 SDLoc DL(Op);
4735 SDValue Low32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: LowOp);
4736 return DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_l32, DL,
4737 VT: MVT::i64, Operand: HighOp, Subreg: Low32);
4738}
4739
4740// Lower SADDO/SSUBO/UADDO/USUBO nodes.
4741SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
4742 SelectionDAG &DAG) const {
4743 SDNode *N = Op.getNode();
4744 SDValue LHS = N->getOperand(Num: 0);
4745 SDValue RHS = N->getOperand(Num: 1);
4746 SDLoc DL(N);
4747
4748 if (N->getValueType(ResNo: 0) == MVT::i128) {
4749 unsigned BaseOp = 0;
4750 unsigned FlagOp = 0;
4751 bool IsBorrow = false;
4752 switch (Op.getOpcode()) {
4753 default: llvm_unreachable("Unknown instruction!");
4754 case ISD::UADDO:
4755 BaseOp = ISD::ADD;
4756 FlagOp = SystemZISD::VACC;
4757 break;
4758 case ISD::USUBO:
4759 BaseOp = ISD::SUB;
4760 FlagOp = SystemZISD::VSCBI;
4761 IsBorrow = true;
4762 break;
4763 }
4764 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VT: MVT::i128, N1: LHS, N2: RHS);
4765 SDValue Flag = DAG.getNode(Opcode: FlagOp, DL, VT: MVT::i128, N1: LHS, N2: RHS);
4766 Flag = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: MVT::i128, N1: Flag,
4767 N2: DAG.getValueType(MVT::i1));
4768 Flag = DAG.getZExtOrTrunc(Op: Flag, DL, VT: N->getValueType(ResNo: 1));
4769 if (IsBorrow)
4770 Flag = DAG.getNode(Opcode: ISD::XOR, DL, VT: Flag.getValueType(),
4771 N1: Flag, N2: DAG.getConstant(Val: 1, DL, VT: Flag.getValueType()));
4772 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: Flag);
4773 }
4774
4775 unsigned BaseOp = 0;
4776 unsigned CCValid = 0;
4777 unsigned CCMask = 0;
4778
4779 switch (Op.getOpcode()) {
4780 default: llvm_unreachable("Unknown instruction!");
4781 case ISD::SADDO:
4782 BaseOp = SystemZISD::SADDO;
4783 CCValid = SystemZ::CCMASK_ARITH;
4784 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
4785 break;
4786 case ISD::SSUBO:
4787 BaseOp = SystemZISD::SSUBO;
4788 CCValid = SystemZ::CCMASK_ARITH;
4789 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
4790 break;
4791 case ISD::UADDO:
4792 BaseOp = SystemZISD::UADDO;
4793 CCValid = SystemZ::CCMASK_LOGICAL;
4794 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
4795 break;
4796 case ISD::USUBO:
4797 BaseOp = SystemZISD::USUBO;
4798 CCValid = SystemZ::CCMASK_LOGICAL;
4799 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
4800 break;
4801 }
4802
4803 SDVTList VTs = DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::i32);
4804 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VTList: VTs, N1: LHS, N2: RHS);
4805
4806 SDValue SetCC = emitSETCC(DAG, DL, CCReg: Result.getValue(R: 1), CCValid, CCMask);
4807 if (N->getValueType(ResNo: 1) == MVT::i1)
4808 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: SetCC);
4809
4810 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: SetCC);
4811}
4812
4813static bool isAddCarryChain(SDValue Carry) {
4814 while (Carry.getOpcode() == ISD::UADDO_CARRY &&
4815 Carry->getValueType(ResNo: 0) != MVT::i128)
4816 Carry = Carry.getOperand(i: 2);
4817 return Carry.getOpcode() == ISD::UADDO &&
4818 Carry->getValueType(ResNo: 0) != MVT::i128;
4819}
4820
4821static bool isSubBorrowChain(SDValue Carry) {
4822 while (Carry.getOpcode() == ISD::USUBO_CARRY &&
4823 Carry->getValueType(ResNo: 0) != MVT::i128)
4824 Carry = Carry.getOperand(i: 2);
4825 return Carry.getOpcode() == ISD::USUBO &&
4826 Carry->getValueType(ResNo: 0) != MVT::i128;
4827}
4828
4829// Lower UADDO_CARRY/USUBO_CARRY nodes.
4830SDValue SystemZTargetLowering::lowerUADDSUBO_CARRY(SDValue Op,
4831 SelectionDAG &DAG) const {
4832
4833 SDNode *N = Op.getNode();
4834 MVT VT = N->getSimpleValueType(ResNo: 0);
4835
4836 // Let legalize expand this if it isn't a legal type yet.
4837 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
4838 return SDValue();
4839
4840 SDValue LHS = N->getOperand(Num: 0);
4841 SDValue RHS = N->getOperand(Num: 1);
4842 SDValue Carry = Op.getOperand(i: 2);
4843 SDLoc DL(N);
4844
4845 if (VT == MVT::i128) {
4846 unsigned BaseOp = 0;
4847 unsigned FlagOp = 0;
4848 bool IsBorrow = false;
4849 switch (Op.getOpcode()) {
4850 default: llvm_unreachable("Unknown instruction!");
4851 case ISD::UADDO_CARRY:
4852 BaseOp = SystemZISD::VAC;
4853 FlagOp = SystemZISD::VACCC;
4854 break;
4855 case ISD::USUBO_CARRY:
4856 BaseOp = SystemZISD::VSBI;
4857 FlagOp = SystemZISD::VSBCBI;
4858 IsBorrow = true;
4859 break;
4860 }
4861 if (IsBorrow)
4862 Carry = DAG.getNode(Opcode: ISD::XOR, DL, VT: Carry.getValueType(),
4863 N1: Carry, N2: DAG.getConstant(Val: 1, DL, VT: Carry.getValueType()));
4864 Carry = DAG.getZExtOrTrunc(Op: Carry, DL, VT: MVT::i128);
4865 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VT: MVT::i128, N1: LHS, N2: RHS, N3: Carry);
4866 SDValue Flag = DAG.getNode(Opcode: FlagOp, DL, VT: MVT::i128, N1: LHS, N2: RHS, N3: Carry);
4867 Flag = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: MVT::i128, N1: Flag,
4868 N2: DAG.getValueType(MVT::i1));
4869 Flag = DAG.getZExtOrTrunc(Op: Flag, DL, VT: N->getValueType(ResNo: 1));
4870 if (IsBorrow)
4871 Flag = DAG.getNode(Opcode: ISD::XOR, DL, VT: Flag.getValueType(),
4872 N1: Flag, N2: DAG.getConstant(Val: 1, DL, VT: Flag.getValueType()));
4873 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: Flag);
4874 }
4875
4876 unsigned BaseOp = 0;
4877 unsigned CCValid = 0;
4878 unsigned CCMask = 0;
4879
4880 switch (Op.getOpcode()) {
4881 default: llvm_unreachable("Unknown instruction!");
4882 case ISD::UADDO_CARRY:
4883 if (!isAddCarryChain(Carry))
4884 return SDValue();
4885
4886 BaseOp = SystemZISD::ADDCARRY;
4887 CCValid = SystemZ::CCMASK_LOGICAL;
4888 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
4889 break;
4890 case ISD::USUBO_CARRY:
4891 if (!isSubBorrowChain(Carry))
4892 return SDValue();
4893
4894 BaseOp = SystemZISD::SUBCARRY;
4895 CCValid = SystemZ::CCMASK_LOGICAL;
4896 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
4897 break;
4898 }
4899
4900 // Set the condition code from the carry flag.
4901 Carry = DAG.getNode(Opcode: SystemZISD::GET_CCMASK, DL, VT: MVT::i32, N1: Carry,
4902 N2: DAG.getConstant(Val: CCValid, DL, VT: MVT::i32),
4903 N3: DAG.getConstant(Val: CCMask, DL, VT: MVT::i32));
4904
4905 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
4906 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VTList: VTs, N1: LHS, N2: RHS, N3: Carry);
4907
4908 SDValue SetCC = emitSETCC(DAG, DL, CCReg: Result.getValue(R: 1), CCValid, CCMask);
4909 if (N->getValueType(ResNo: 1) == MVT::i1)
4910 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: SetCC);
4911
4912 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: SetCC);
4913}
4914
4915SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
4916 SelectionDAG &DAG) const {
4917 EVT VT = Op.getValueType();
4918 SDLoc DL(Op);
4919 Op = Op.getOperand(i: 0);
4920
4921 if (VT.getScalarSizeInBits() == 128) {
4922 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: Op);
4923 Op = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: MVT::v2i64, Operand: Op);
4924 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v2i64, DL,
4925 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i64));
4926 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
4927 return Op;
4928 }
4929
4930 // Handle vector types via VPOPCT.
4931 if (VT.isVector()) {
4932 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Op);
4933 Op = DAG.getNode(Opcode: SystemZISD::POPCNT, DL, VT: MVT::v16i8, Operand: Op);
4934 switch (VT.getScalarSizeInBits()) {
4935 case 8:
4936 break;
4937 case 16: {
4938 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
4939 SDValue Shift = DAG.getConstant(Val: 8, DL, VT: MVT::i32);
4940 SDValue Tmp = DAG.getNode(Opcode: SystemZISD::VSHL_BY_SCALAR, DL, VT, N1: Op, N2: Shift);
4941 Op = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op, N2: Tmp);
4942 Op = DAG.getNode(Opcode: SystemZISD::VSRL_BY_SCALAR, DL, VT, N1: Op, N2: Shift);
4943 break;
4944 }
4945 case 32: {
4946 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v16i8, DL,
4947 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
4948 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
4949 break;
4950 }
4951 case 64: {
4952 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v16i8, DL,
4953 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
4954 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::v4i32, N1: Op, N2: Tmp);
4955 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
4956 break;
4957 }
4958 default:
4959 llvm_unreachable("Unexpected type");
4960 }
4961 return Op;
4962 }
4963
4964 // Get the known-zero mask for the operand.
4965 KnownBits Known = DAG.computeKnownBits(Op);
4966 unsigned NumSignificantBits = Known.getMaxValue().getActiveBits();
4967 if (NumSignificantBits == 0)
4968 return DAG.getConstant(Val: 0, DL, VT);
4969
4970 // Skip known-zero high parts of the operand.
4971 int64_t OrigBitSize = VT.getSizeInBits();
4972 int64_t BitSize = llvm::bit_ceil(Value: NumSignificantBits);
4973 BitSize = std::min(a: BitSize, b: OrigBitSize);
4974
4975 // The POPCNT instruction counts the number of bits in each byte.
4976 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op);
4977 Op = DAG.getNode(Opcode: SystemZISD::POPCNT, DL, VT: MVT::i64, Operand: Op);
4978 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
4979
4980 // Add up per-byte counts in a binary tree. All bits of Op at
4981 // position larger than BitSize remain zero throughout.
4982 for (int64_t I = BitSize / 2; I >= 8; I = I / 2) {
4983 SDValue Tmp = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Op, N2: DAG.getConstant(Val: I, DL, VT));
4984 if (BitSize != OrigBitSize)
4985 Tmp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Tmp,
4986 N2: DAG.getConstant(Val: ((uint64_t)1 << BitSize) - 1, DL, VT));
4987 Op = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op, N2: Tmp);
4988 }
4989
4990 // Extract overall result from high byte.
4991 if (BitSize > 8)
4992 Op = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Op,
4993 N2: DAG.getConstant(Val: BitSize - 8, DL, VT));
4994
4995 return Op;
4996}
4997
4998SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
4999 SelectionDAG &DAG) const {
5000 SDLoc DL(Op);
5001 AtomicOrdering FenceOrdering =
5002 static_cast<AtomicOrdering>(Op.getConstantOperandVal(i: 1));
5003 SyncScope::ID FenceSSID =
5004 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
5005
5006 // The only fence that needs an instruction is a sequentially-consistent
5007 // cross-thread fence.
5008 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
5009 FenceSSID == SyncScope::System) {
5010 return SDValue(DAG.getMachineNode(Opcode: SystemZ::Serialize, dl: DL, VT: MVT::Other,
5011 Op1: Op.getOperand(i: 0)),
5012 0);
5013 }
5014
5015 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
5016 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL, VT: MVT::Other, Operand: Op.getOperand(i: 0));
5017}
5018
5019SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
5020 SelectionDAG &DAG) const {
5021 EVT RegVT = Op.getValueType();
5022 if (RegVT.getSizeInBits() == 128)
5023 return lowerATOMIC_LDST_I128(Op, DAG);
5024 return lowerLoadF16(Op, DAG);
5025}
5026
5027SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
5028 SelectionDAG &DAG) const {
5029 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5030 if (Node->getMemoryVT().getSizeInBits() == 128)
5031 return lowerATOMIC_LDST_I128(Op, DAG);
5032 return lowerStoreF16(Op, DAG);
5033}
5034
5035SDValue SystemZTargetLowering::lowerATOMIC_LDST_I128(SDValue Op,
5036 SelectionDAG &DAG) const {
5037 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5038 assert(
5039 (Node->getMemoryVT() == MVT::i128 || Node->getMemoryVT() == MVT::f128) &&
5040 "Only custom lowering i128 or f128.");
5041 // Use same code to handle both legal and non-legal i128 types.
5042 SmallVector<SDValue, 2> Results;
5043 LowerOperationWrapper(N: Node, Results, DAG);
5044 return DAG.getMergeValues(Ops: Results, dl: SDLoc(Op));
5045}
5046
5047// Prepare for a Compare And Swap for a subword operation. This needs to be
5048// done in memory with 4 bytes at natural alignment.
5049static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL,
5050 SDValue &AlignedAddr, SDValue &BitShift,
5051 SDValue &NegBitShift) {
5052 EVT PtrVT = Addr.getValueType();
5053 EVT WideVT = MVT::i32;
5054
5055 // Get the address of the containing word.
5056 AlignedAddr = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Addr,
5057 N2: DAG.getSignedConstant(Val: -4, DL, VT: PtrVT));
5058
5059 // Get the number of bits that the word must be rotated left in order
5060 // to bring the field to the top bits of a GR32.
5061 BitShift = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: Addr,
5062 N2: DAG.getConstant(Val: 3, DL, VT: PtrVT));
5063 BitShift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: WideVT, Operand: BitShift);
5064
5065 // Get the complementing shift amount, for rotating a field in the top
5066 // bits back to its proper position.
5067 NegBitShift = DAG.getNode(Opcode: ISD::SUB, DL, VT: WideVT,
5068 N1: DAG.getConstant(Val: 0, DL, VT: WideVT), N2: BitShift);
5069
5070}
5071
5072// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first
5073// two into the fullword ATOMIC_LOADW_* operation given by Opcode.
5074SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
5075 SelectionDAG &DAG,
5076 unsigned Opcode) const {
5077 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5078
5079 // 32-bit operations need no special handling.
5080 EVT NarrowVT = Node->getMemoryVT();
5081 EVT WideVT = MVT::i32;
5082 if (NarrowVT == WideVT)
5083 return Op;
5084
5085 int64_t BitSize = NarrowVT.getSizeInBits();
5086 SDValue ChainIn = Node->getChain();
5087 SDValue Addr = Node->getBasePtr();
5088 SDValue Src2 = Node->getVal();
5089 MachineMemOperand *MMO = Node->getMemOperand();
5090 SDLoc DL(Node);
5091
5092 // Convert atomic subtracts of constants into additions.
5093 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
5094 if (auto *Const = dyn_cast<ConstantSDNode>(Val&: Src2)) {
5095 Opcode = SystemZISD::ATOMIC_LOADW_ADD;
5096 Src2 = DAG.getSignedConstant(Val: -Const->getSExtValue(), DL,
5097 VT: Src2.getValueType());
5098 }
5099
5100 SDValue AlignedAddr, BitShift, NegBitShift;
5101 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5102
5103 // Extend the source operand to 32 bits and prepare it for the inner loop.
5104 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other
5105 // operations require the source to be shifted in advance. (This shift
5106 // can be folded if the source is constant.) For AND and NAND, the lower
5107 // bits must be set, while for other opcodes they should be left clear.
5108 if (Opcode != SystemZISD::ATOMIC_SWAPW)
5109 Src2 = DAG.getNode(Opcode: ISD::SHL, DL, VT: WideVT, N1: Src2,
5110 N2: DAG.getConstant(Val: 32 - BitSize, DL, VT: WideVT));
5111 if (Opcode == SystemZISD::ATOMIC_LOADW_AND ||
5112 Opcode == SystemZISD::ATOMIC_LOADW_NAND)
5113 Src2 = DAG.getNode(Opcode: ISD::OR, DL, VT: WideVT, N1: Src2,
5114 N2: DAG.getConstant(Val: uint32_t(-1) >> BitSize, DL, VT: WideVT));
5115
5116 // Construct the ATOMIC_LOADW_* node.
5117 SDVTList VTList = DAG.getVTList(VT1: WideVT, VT2: MVT::Other);
5118 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
5119 DAG.getConstant(Val: BitSize, DL, VT: WideVT) };
5120 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList, Ops,
5121 MemVT: NarrowVT, MMO);
5122
5123 // Rotate the result of the final CS so that the field is in the lower
5124 // bits of a GR32, then truncate it.
5125 SDValue ResultShift = DAG.getNode(Opcode: ISD::ADD, DL, VT: WideVT, N1: BitShift,
5126 N2: DAG.getConstant(Val: BitSize, DL, VT: WideVT));
5127 SDValue Result = DAG.getNode(Opcode: ISD::ROTL, DL, VT: WideVT, N1: AtomicOp, N2: ResultShift);
5128
5129 SDValue RetOps[2] = { Result, AtomicOp.getValue(R: 1) };
5130 return DAG.getMergeValues(Ops: RetOps, dl: DL);
5131}
5132
5133// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations into
5134// ATOMIC_LOADW_SUBs and convert 32- and 64-bit operations into additions.
5135SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
5136 SelectionDAG &DAG) const {
5137 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5138 EVT MemVT = Node->getMemoryVT();
5139 if (MemVT == MVT::i32 || MemVT == MVT::i64) {
5140 // A full-width operation: negate and use LAA(G).
5141 assert(Op.getValueType() == MemVT && "Mismatched VTs");
5142 assert(Subtarget.hasInterlockedAccess1() &&
5143 "Should have been expanded by AtomicExpand pass.");
5144 SDValue Src2 = Node->getVal();
5145 SDLoc DL(Src2);
5146 SDValue NegSrc2 =
5147 DAG.getNode(Opcode: ISD::SUB, DL, VT: MemVT, N1: DAG.getConstant(Val: 0, DL, VT: MemVT), N2: Src2);
5148 return DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT,
5149 Chain: Node->getChain(), Ptr: Node->getBasePtr(), Val: NegSrc2,
5150 MMO: Node->getMemOperand());
5151 }
5152
5153 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_SUB);
5154}
5155
5156// Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
5157SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
5158 SelectionDAG &DAG) const {
5159 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5160 SDValue ChainIn = Node->getOperand(Num: 0);
5161 SDValue Addr = Node->getOperand(Num: 1);
5162 SDValue CmpVal = Node->getOperand(Num: 2);
5163 SDValue SwapVal = Node->getOperand(Num: 3);
5164 MachineMemOperand *MMO = Node->getMemOperand();
5165 SDLoc DL(Node);
5166
5167 if (Node->getMemoryVT() == MVT::i128) {
5168 // Use same code to handle both legal and non-legal i128 types.
5169 SmallVector<SDValue, 3> Results;
5170 LowerOperationWrapper(N: Node, Results, DAG);
5171 return DAG.getMergeValues(Ops: Results, dl: DL);
5172 }
5173
5174 // We have native support for 32-bit and 64-bit compare and swap, but we
5175 // still need to expand extracting the "success" result from the CC.
5176 EVT NarrowVT = Node->getMemoryVT();
5177 EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
5178 if (NarrowVT == WideVT) {
5179 SDVTList Tys = DAG.getVTList(VT1: WideVT, VT2: MVT::i32, VT3: MVT::Other);
5180 SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
5181 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAP,
5182 dl: DL, VTList: Tys, Ops, MemVT: NarrowVT, MMO);
5183 SDValue Success = emitSETCC(DAG, DL, CCReg: AtomicOp.getValue(R: 1),
5184 CCValid: SystemZ::CCMASK_CS, CCMask: SystemZ::CCMASK_CS_EQ);
5185
5186 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 0), To: AtomicOp.getValue(R: 0));
5187 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 1), To: Success);
5188 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 2), To: AtomicOp.getValue(R: 2));
5189 return SDValue();
5190 }
5191
5192 // Convert 8-bit and 16-bit compare and swap to a loop, implemented
5193 // via a fullword ATOMIC_CMP_SWAPW operation.
5194 int64_t BitSize = NarrowVT.getSizeInBits();
5195
5196 SDValue AlignedAddr, BitShift, NegBitShift;
5197 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5198
5199 // Construct the ATOMIC_CMP_SWAPW node.
5200 SDVTList VTList = DAG.getVTList(VT1: WideVT, VT2: MVT::i32, VT3: MVT::Other);
5201 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
5202 NegBitShift, DAG.getConstant(Val: BitSize, DL, VT: WideVT) };
5203 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAPW, dl: DL,
5204 VTList, Ops, MemVT: NarrowVT, MMO);
5205 SDValue Success = emitSETCC(DAG, DL, CCReg: AtomicOp.getValue(R: 1),
5206 CCValid: SystemZ::CCMASK_ICMP, CCMask: SystemZ::CCMASK_CMP_EQ);
5207
5208 // emitAtomicCmpSwapW() will zero extend the result (original value).
5209 SDValue OrigVal = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: WideVT, N1: AtomicOp.getValue(R: 0),
5210 N2: DAG.getValueType(NarrowVT));
5211 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 0), To: OrigVal);
5212 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 1), To: Success);
5213 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 2), To: AtomicOp.getValue(R: 2));
5214 return SDValue();
5215}
5216
5217MachineMemOperand::Flags
5218SystemZTargetLowering::getTargetMMOFlags(const Instruction &I) const {
5219 // Because of how we convert atomic_load and atomic_store to normal loads and
5220 // stores in the DAG, we need to ensure that the MMOs are marked volatile
5221 // since DAGCombine hasn't been updated to account for atomic, but non
5222 // volatile loads. (See D57601)
5223 if (auto *SI = dyn_cast<StoreInst>(Val: &I))
5224 if (SI->isAtomic())
5225 return MachineMemOperand::MOVolatile;
5226 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
5227 if (LI->isAtomic())
5228 return MachineMemOperand::MOVolatile;
5229 if (auto *AI = dyn_cast<AtomicRMWInst>(Val: &I))
5230 if (AI->isAtomic())
5231 return MachineMemOperand::MOVolatile;
5232 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
5233 if (AI->isAtomic())
5234 return MachineMemOperand::MOVolatile;
5235 return MachineMemOperand::MONone;
5236}
5237
5238SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
5239 SelectionDAG &DAG) const {
5240 MachineFunction &MF = DAG.getMachineFunction();
5241 auto *Regs = Subtarget.getSpecialRegisters();
5242 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
5243 report_fatal_error(reason: "Variable-sized stack allocations are not supported "
5244 "in GHC calling convention");
5245 return DAG.getCopyFromReg(Chain: Op.getOperand(i: 0), dl: SDLoc(Op),
5246 Reg: Regs->getStackPointerRegister(), VT: Op.getValueType());
5247}
5248
5249SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op,
5250 SelectionDAG &DAG) const {
5251 MachineFunction &MF = DAG.getMachineFunction();
5252 auto *Regs = Subtarget.getSpecialRegisters();
5253 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
5254
5255 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
5256 report_fatal_error(reason: "Variable-sized stack allocations are not supported "
5257 "in GHC calling convention");
5258
5259 SDValue Chain = Op.getOperand(i: 0);
5260 SDValue NewSP = Op.getOperand(i: 1);
5261 SDValue Backchain;
5262 SDLoc DL(Op);
5263
5264 if (StoreBackchain) {
5265 SDValue OldSP = DAG.getCopyFromReg(
5266 Chain, dl: DL, Reg: Regs->getStackPointerRegister(), VT: MVT::i64);
5267 Backchain = DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: getBackchainAddress(SP: OldSP, DAG),
5268 PtrInfo: MachinePointerInfo());
5269 }
5270
5271 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: Regs->getStackPointerRegister(), N: NewSP);
5272
5273 if (StoreBackchain)
5274 Chain = DAG.getStore(Chain, dl: DL, Val: Backchain, Ptr: getBackchainAddress(SP: NewSP, DAG),
5275 PtrInfo: MachinePointerInfo());
5276
5277 return Chain;
5278}
5279
5280SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
5281 SelectionDAG &DAG) const {
5282 bool IsData = Op.getConstantOperandVal(i: 4);
5283 if (!IsData)
5284 // Just preserve the chain.
5285 return Op.getOperand(i: 0);
5286
5287 SDLoc DL(Op);
5288 bool IsWrite = Op.getConstantOperandVal(i: 2);
5289 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
5290 auto *Node = cast<MemIntrinsicSDNode>(Val: Op.getNode());
5291 SDValue Ops[] = {Op.getOperand(i: 0), DAG.getTargetConstant(Val: Code, DL, VT: MVT::i32),
5292 Op.getOperand(i: 1)};
5293 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::PREFETCH, dl: DL,
5294 VTList: Node->getVTList(), Ops,
5295 MemVT: Node->getMemoryVT(), MMO: Node->getMemOperand());
5296}
5297
5298// Convert condition code in CCReg to an i32 value.
5299static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg) {
5300 SDLoc DL(CCReg);
5301 SDValue IPM = DAG.getNode(Opcode: SystemZISD::IPM, DL, VT: MVT::i32, Operand: CCReg);
5302 return DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: IPM,
5303 N2: DAG.getConstant(Val: SystemZ::IPM_CC, DL, VT: MVT::i32));
5304}
5305
5306SDValue
5307SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
5308 SelectionDAG &DAG) const {
5309 unsigned Opcode, CCValid;
5310 if (isIntrinsicWithCCAndChain(Op, Opcode, CCValid)) {
5311 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
5312 SDNode *Node = emitIntrinsicWithCCAndChain(DAG, Op, Opcode);
5313 SDValue CC = getCCResult(DAG, CCReg: SDValue(Node, 0));
5314 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Op.getNode(), 0), To: CC);
5315 return SDValue();
5316 }
5317
5318 return SDValue();
5319}
5320
5321SDValue
5322SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
5323 SelectionDAG &DAG) const {
5324 unsigned Opcode, CCValid;
5325 if (isIntrinsicWithCC(Op, Opcode, CCValid)) {
5326 SDNode *Node = emitIntrinsicWithCC(DAG, Op, Opcode);
5327 if (Op->getNumValues() == 1)
5328 return getCCResult(DAG, CCReg: SDValue(Node, 0));
5329 assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result");
5330 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op), VTList: Op->getVTList(),
5331 N1: SDValue(Node, 0), N2: getCCResult(DAG, CCReg: SDValue(Node, 1)));
5332 }
5333
5334 unsigned Id = Op.getConstantOperandVal(i: 0);
5335 switch (Id) {
5336 case Intrinsic::thread_pointer:
5337 return lowerThreadPointer(DL: SDLoc(Op), DAG);
5338
5339 case Intrinsic::s390_vpdi:
5340 return DAG.getNode(Opcode: SystemZISD::PERMUTE_DWORDS, DL: SDLoc(Op), VT: Op.getValueType(),
5341 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5342
5343 case Intrinsic::s390_vperm:
5344 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL: SDLoc(Op), VT: Op.getValueType(),
5345 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5346
5347 case Intrinsic::s390_vuphb:
5348 case Intrinsic::s390_vuphh:
5349 case Intrinsic::s390_vuphf:
5350 case Intrinsic::s390_vuphg:
5351 return DAG.getNode(Opcode: SystemZISD::UNPACK_HIGH, DL: SDLoc(Op), VT: Op.getValueType(),
5352 Operand: Op.getOperand(i: 1));
5353
5354 case Intrinsic::s390_vuplhb:
5355 case Intrinsic::s390_vuplhh:
5356 case Intrinsic::s390_vuplhf:
5357 case Intrinsic::s390_vuplhg:
5358 return DAG.getNode(Opcode: SystemZISD::UNPACKL_HIGH, DL: SDLoc(Op), VT: Op.getValueType(),
5359 Operand: Op.getOperand(i: 1));
5360
5361 case Intrinsic::s390_vuplb:
5362 case Intrinsic::s390_vuplhw:
5363 case Intrinsic::s390_vuplf:
5364 case Intrinsic::s390_vuplg:
5365 return DAG.getNode(Opcode: SystemZISD::UNPACK_LOW, DL: SDLoc(Op), VT: Op.getValueType(),
5366 Operand: Op.getOperand(i: 1));
5367
5368 case Intrinsic::s390_vupllb:
5369 case Intrinsic::s390_vupllh:
5370 case Intrinsic::s390_vupllf:
5371 case Intrinsic::s390_vupllg:
5372 return DAG.getNode(Opcode: SystemZISD::UNPACKL_LOW, DL: SDLoc(Op), VT: Op.getValueType(),
5373 Operand: Op.getOperand(i: 1));
5374
5375 case Intrinsic::s390_vsumb:
5376 case Intrinsic::s390_vsumh:
5377 case Intrinsic::s390_vsumgh:
5378 case Intrinsic::s390_vsumgf:
5379 case Intrinsic::s390_vsumqf:
5380 case Intrinsic::s390_vsumqg:
5381 return DAG.getNode(Opcode: SystemZISD::VSUM, DL: SDLoc(Op), VT: Op.getValueType(),
5382 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5383
5384 case Intrinsic::s390_vaq:
5385 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5386 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5387 case Intrinsic::s390_vaccb:
5388 case Intrinsic::s390_vacch:
5389 case Intrinsic::s390_vaccf:
5390 case Intrinsic::s390_vaccg:
5391 case Intrinsic::s390_vaccq:
5392 return DAG.getNode(Opcode: SystemZISD::VACC, DL: SDLoc(Op), VT: Op.getValueType(),
5393 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5394 case Intrinsic::s390_vacq:
5395 return DAG.getNode(Opcode: SystemZISD::VAC, DL: SDLoc(Op), VT: Op.getValueType(),
5396 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5397 case Intrinsic::s390_vacccq:
5398 return DAG.getNode(Opcode: SystemZISD::VACCC, DL: SDLoc(Op), VT: Op.getValueType(),
5399 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5400
5401 case Intrinsic::s390_vsq:
5402 return DAG.getNode(Opcode: ISD::SUB, DL: SDLoc(Op), VT: Op.getValueType(),
5403 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5404 case Intrinsic::s390_vscbib:
5405 case Intrinsic::s390_vscbih:
5406 case Intrinsic::s390_vscbif:
5407 case Intrinsic::s390_vscbig:
5408 case Intrinsic::s390_vscbiq:
5409 return DAG.getNode(Opcode: SystemZISD::VSCBI, DL: SDLoc(Op), VT: Op.getValueType(),
5410 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5411 case Intrinsic::s390_vsbiq:
5412 return DAG.getNode(Opcode: SystemZISD::VSBI, DL: SDLoc(Op), VT: Op.getValueType(),
5413 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5414 case Intrinsic::s390_vsbcbiq:
5415 return DAG.getNode(Opcode: SystemZISD::VSBCBI, DL: SDLoc(Op), VT: Op.getValueType(),
5416 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5417
5418 case Intrinsic::s390_vmhb:
5419 case Intrinsic::s390_vmhh:
5420 case Intrinsic::s390_vmhf:
5421 case Intrinsic::s390_vmhg:
5422 case Intrinsic::s390_vmhq:
5423 return DAG.getNode(Opcode: ISD::MULHS, DL: SDLoc(Op), VT: Op.getValueType(),
5424 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5425 case Intrinsic::s390_vmlhb:
5426 case Intrinsic::s390_vmlhh:
5427 case Intrinsic::s390_vmlhf:
5428 case Intrinsic::s390_vmlhg:
5429 case Intrinsic::s390_vmlhq:
5430 return DAG.getNode(Opcode: ISD::MULHU, DL: SDLoc(Op), VT: Op.getValueType(),
5431 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5432
5433 case Intrinsic::s390_vmahb:
5434 case Intrinsic::s390_vmahh:
5435 case Intrinsic::s390_vmahf:
5436 case Intrinsic::s390_vmahg:
5437 case Intrinsic::s390_vmahq:
5438 return DAG.getNode(Opcode: SystemZISD::VMAH, DL: SDLoc(Op), VT: Op.getValueType(),
5439 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5440 case Intrinsic::s390_vmalhb:
5441 case Intrinsic::s390_vmalhh:
5442 case Intrinsic::s390_vmalhf:
5443 case Intrinsic::s390_vmalhg:
5444 case Intrinsic::s390_vmalhq:
5445 return DAG.getNode(Opcode: SystemZISD::VMALH, DL: SDLoc(Op), VT: Op.getValueType(),
5446 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5447
5448 case Intrinsic::s390_vmeb:
5449 case Intrinsic::s390_vmeh:
5450 case Intrinsic::s390_vmef:
5451 case Intrinsic::s390_vmeg:
5452 return DAG.getNode(Opcode: SystemZISD::VME, DL: SDLoc(Op), VT: Op.getValueType(),
5453 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5454 case Intrinsic::s390_vmleb:
5455 case Intrinsic::s390_vmleh:
5456 case Intrinsic::s390_vmlef:
5457 case Intrinsic::s390_vmleg:
5458 return DAG.getNode(Opcode: SystemZISD::VMLE, DL: SDLoc(Op), VT: Op.getValueType(),
5459 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5460 case Intrinsic::s390_vmob:
5461 case Intrinsic::s390_vmoh:
5462 case Intrinsic::s390_vmof:
5463 case Intrinsic::s390_vmog:
5464 return DAG.getNode(Opcode: SystemZISD::VMO, DL: SDLoc(Op), VT: Op.getValueType(),
5465 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5466 case Intrinsic::s390_vmlob:
5467 case Intrinsic::s390_vmloh:
5468 case Intrinsic::s390_vmlof:
5469 case Intrinsic::s390_vmlog:
5470 return DAG.getNode(Opcode: SystemZISD::VMLO, DL: SDLoc(Op), VT: Op.getValueType(),
5471 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5472
5473 case Intrinsic::s390_vmaeb:
5474 case Intrinsic::s390_vmaeh:
5475 case Intrinsic::s390_vmaef:
5476 case Intrinsic::s390_vmaeg:
5477 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5478 N1: DAG.getNode(Opcode: SystemZISD::VME, DL: SDLoc(Op), VT: Op.getValueType(),
5479 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5480 N2: Op.getOperand(i: 3));
5481 case Intrinsic::s390_vmaleb:
5482 case Intrinsic::s390_vmaleh:
5483 case Intrinsic::s390_vmalef:
5484 case Intrinsic::s390_vmaleg:
5485 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5486 N1: DAG.getNode(Opcode: SystemZISD::VMLE, DL: SDLoc(Op), VT: Op.getValueType(),
5487 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5488 N2: Op.getOperand(i: 3));
5489 case Intrinsic::s390_vmaob:
5490 case Intrinsic::s390_vmaoh:
5491 case Intrinsic::s390_vmaof:
5492 case Intrinsic::s390_vmaog:
5493 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5494 N1: DAG.getNode(Opcode: SystemZISD::VMO, DL: SDLoc(Op), VT: Op.getValueType(),
5495 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5496 N2: Op.getOperand(i: 3));
5497 case Intrinsic::s390_vmalob:
5498 case Intrinsic::s390_vmaloh:
5499 case Intrinsic::s390_vmalof:
5500 case Intrinsic::s390_vmalog:
5501 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5502 N1: DAG.getNode(Opcode: SystemZISD::VMLO, DL: SDLoc(Op), VT: Op.getValueType(),
5503 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5504 N2: Op.getOperand(i: 3));
5505 }
5506
5507 return SDValue();
5508}
5509
5510namespace {
5511// Says that SystemZISD operation Opcode can be used to perform the equivalent
5512// of a VPERM with permute vector Bytes. If Opcode takes three operands,
5513// Operand is the constant third operand, otherwise it is the number of
5514// bytes in each element of the result.
5515struct Permute {
5516 unsigned Opcode;
5517 unsigned Operand;
5518 unsigned char Bytes[SystemZ::VectorBytes];
5519};
5520}
5521
5522static const Permute PermuteForms[] = {
5523 // VMRHG
5524 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 8,
5525 .Bytes: { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
5526 // VMRHF
5527 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 4,
5528 .Bytes: { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
5529 // VMRHH
5530 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 2,
5531 .Bytes: { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
5532 // VMRHB
5533 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 1,
5534 .Bytes: { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
5535 // VMRLG
5536 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 8,
5537 .Bytes: { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
5538 // VMRLF
5539 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 4,
5540 .Bytes: { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
5541 // VMRLH
5542 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 2,
5543 .Bytes: { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
5544 // VMRLB
5545 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 1,
5546 .Bytes: { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
5547 // VPKG
5548 { .Opcode: SystemZISD::PACK, .Operand: 4,
5549 .Bytes: { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
5550 // VPKF
5551 { .Opcode: SystemZISD::PACK, .Operand: 2,
5552 .Bytes: { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
5553 // VPKH
5554 { .Opcode: SystemZISD::PACK, .Operand: 1,
5555 .Bytes: { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
5556 // VPDI V1, V2, 4 (low half of V1, high half of V2)
5557 { .Opcode: SystemZISD::PERMUTE_DWORDS, .Operand: 4,
5558 .Bytes: { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
5559 // VPDI V1, V2, 1 (high half of V1, low half of V2)
5560 { .Opcode: SystemZISD::PERMUTE_DWORDS, .Operand: 1,
5561 .Bytes: { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
5562};
5563
5564// Called after matching a vector shuffle against a particular pattern.
5565// Both the original shuffle and the pattern have two vector operands.
5566// OpNos[0] is the operand of the original shuffle that should be used for
5567// operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything.
5568// OpNos[1] is the same for operand 1 of the pattern. Resolve these -1s and
5569// set OpNo0 and OpNo1 to the shuffle operands that should actually be used
5570// for operands 0 and 1 of the pattern.
5571static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1) {
5572 if (OpNos[0] < 0) {
5573 if (OpNos[1] < 0)
5574 return false;
5575 OpNo0 = OpNo1 = OpNos[1];
5576 } else if (OpNos[1] < 0) {
5577 OpNo0 = OpNo1 = OpNos[0];
5578 } else {
5579 OpNo0 = OpNos[0];
5580 OpNo1 = OpNos[1];
5581 }
5582 return true;
5583}
5584
5585// Bytes is a VPERM-like permute vector, except that -1 is used for
5586// undefined bytes. Return true if the VPERM can be implemented using P.
5587// When returning true set OpNo0 to the VPERM operand that should be
5588// used for operand 0 of P and likewise OpNo1 for operand 1 of P.
5589//
5590// For example, if swapping the VPERM operands allows P to match, OpNo0
5591// will be 1 and OpNo1 will be 0. If instead Bytes only refers to one
5592// operand, but rewriting it to use two duplicated operands allows it to
5593// match P, then OpNo0 and OpNo1 will be the same.
5594static bool matchPermute(const SmallVectorImpl<int> &Bytes, const Permute &P,
5595 unsigned &OpNo0, unsigned &OpNo1) {
5596 int OpNos[] = { -1, -1 };
5597 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5598 int Elt = Bytes[I];
5599 if (Elt >= 0) {
5600 // Make sure that the two permute vectors use the same suboperand
5601 // byte number. Only the operand numbers (the high bits) are
5602 // allowed to differ.
5603 if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))
5604 return false;
5605 int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes;
5606 int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes;
5607 // Make sure that the operand mappings are consistent with previous
5608 // elements.
5609 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5610 return false;
5611 OpNos[ModelOpNo] = RealOpNo;
5612 }
5613 }
5614 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5615}
5616
5617// As above, but search for a matching permute.
5618static const Permute *matchPermute(const SmallVectorImpl<int> &Bytes,
5619 unsigned &OpNo0, unsigned &OpNo1) {
5620 for (auto &P : PermuteForms)
5621 if (matchPermute(Bytes, P, OpNo0, OpNo1))
5622 return &P;
5623 return nullptr;
5624}
5625
5626// Bytes is a VPERM-like permute vector, except that -1 is used for
5627// undefined bytes. This permute is an operand of an outer permute.
5628// See whether redistributing the -1 bytes gives a shuffle that can be
5629// implemented using P. If so, set Transform to a VPERM-like permute vector
5630// that, when applied to the result of P, gives the original permute in Bytes.
5631static bool matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5632 const Permute &P,
5633 SmallVectorImpl<int> &Transform) {
5634 unsigned To = 0;
5635 for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) {
5636 int Elt = Bytes[From];
5637 if (Elt < 0)
5638 // Byte number From of the result is undefined.
5639 Transform[From] = -1;
5640 else {
5641 while (P.Bytes[To] != Elt) {
5642 To += 1;
5643 if (To == SystemZ::VectorBytes)
5644 return false;
5645 }
5646 Transform[From] = To;
5647 }
5648 }
5649 return true;
5650}
5651
5652// As above, but search for a matching permute.
5653static const Permute *matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5654 SmallVectorImpl<int> &Transform) {
5655 for (auto &P : PermuteForms)
5656 if (matchDoublePermute(Bytes, P, Transform))
5657 return &P;
5658 return nullptr;
5659}
5660
5661// Convert the mask of the given shuffle op into a byte-level mask,
5662// as if it had type vNi8.
5663static bool getVPermMask(SDValue ShuffleOp,
5664 SmallVectorImpl<int> &Bytes) {
5665 EVT VT = ShuffleOp.getValueType();
5666 unsigned NumElements = VT.getVectorNumElements();
5667 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5668
5669 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Val&: ShuffleOp)) {
5670 Bytes.resize(N: NumElements * BytesPerElement, NV: -1);
5671 for (unsigned I = 0; I < NumElements; ++I) {
5672 int Index = VSN->getMaskElt(Idx: I);
5673 if (Index >= 0)
5674 for (unsigned J = 0; J < BytesPerElement; ++J)
5675 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5676 }
5677 return true;
5678 }
5679 if (SystemZISD::SPLAT == ShuffleOp.getOpcode() &&
5680 isa<ConstantSDNode>(Val: ShuffleOp.getOperand(i: 1))) {
5681 unsigned Index = ShuffleOp.getConstantOperandVal(i: 1);
5682 Bytes.resize(N: NumElements * BytesPerElement, NV: -1);
5683 for (unsigned I = 0; I < NumElements; ++I)
5684 for (unsigned J = 0; J < BytesPerElement; ++J)
5685 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5686 return true;
5687 }
5688 return false;
5689}
5690
5691// Bytes is a VPERM-like permute vector, except that -1 is used for
5692// undefined bytes. See whether bytes [Start, Start + BytesPerElement) of
5693// the result come from a contiguous sequence of bytes from one input.
5694// Set Base to the selector for the first byte if so.
5695static bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start,
5696 unsigned BytesPerElement, int &Base) {
5697 Base = -1;
5698 for (unsigned I = 0; I < BytesPerElement; ++I) {
5699 if (Bytes[Start + I] >= 0) {
5700 unsigned Elem = Bytes[Start + I];
5701 if (Base < 0) {
5702 Base = Elem - I;
5703 // Make sure the bytes would come from one input operand.
5704 if (unsigned(Base) % Bytes.size() + BytesPerElement > Bytes.size())
5705 return false;
5706 } else if (unsigned(Base) != Elem - I)
5707 return false;
5708 }
5709 }
5710 return true;
5711}
5712
5713// Bytes is a VPERM-like permute vector, except that -1 is used for
5714// undefined bytes. Return true if it can be performed using VSLDB.
5715// When returning true, set StartIndex to the shift amount and OpNo0
5716// and OpNo1 to the VPERM operands that should be used as the first
5717// and second shift operand respectively.
5718static bool isShlDoublePermute(const SmallVectorImpl<int> &Bytes,
5719 unsigned &StartIndex, unsigned &OpNo0,
5720 unsigned &OpNo1) {
5721 int OpNos[] = { -1, -1 };
5722 int Shift = -1;
5723 for (unsigned I = 0; I < 16; ++I) {
5724 int Index = Bytes[I];
5725 if (Index >= 0) {
5726 int ExpectedShift = (Index - I) % SystemZ::VectorBytes;
5727 int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes;
5728 int RealOpNo = unsigned(Index) / SystemZ::VectorBytes;
5729 if (Shift < 0)
5730 Shift = ExpectedShift;
5731 else if (Shift != ExpectedShift)
5732 return false;
5733 // Make sure that the operand mappings are consistent with previous
5734 // elements.
5735 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5736 return false;
5737 OpNos[ModelOpNo] = RealOpNo;
5738 }
5739 }
5740 StartIndex = Shift;
5741 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5742}
5743
5744// Create a node that performs P on operands Op0 and Op1, casting the
5745// operands to the appropriate type. The type of the result is determined by P.
5746static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5747 const Permute &P, SDValue Op0, SDValue Op1) {
5748 // VPDI (PERMUTE_DWORDS) always operates on v2i64s. The input
5749 // elements of a PACK are twice as wide as the outputs.
5750 unsigned InBytes = (P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 :
5751 P.Opcode == SystemZISD::PACK ? P.Operand * 2 :
5752 P.Operand);
5753 // Cast both operands to the appropriate type.
5754 MVT InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: InBytes * 8),
5755 NumElements: SystemZ::VectorBytes / InBytes);
5756 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op0);
5757 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op1);
5758 SDValue Op;
5759 if (P.Opcode == SystemZISD::PERMUTE_DWORDS) {
5760 SDValue Op2 = DAG.getTargetConstant(Val: P.Operand, DL, VT: MVT::i32);
5761 Op = DAG.getNode(Opcode: SystemZISD::PERMUTE_DWORDS, DL, VT: InVT, N1: Op0, N2: Op1, N3: Op2);
5762 } else if (P.Opcode == SystemZISD::PACK) {
5763 MVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: P.Operand * 8),
5764 NumElements: SystemZ::VectorBytes / P.Operand);
5765 Op = DAG.getNode(Opcode: SystemZISD::PACK, DL, VT: OutVT, N1: Op0, N2: Op1);
5766 } else {
5767 Op = DAG.getNode(Opcode: P.Opcode, DL, VT: InVT, N1: Op0, N2: Op1);
5768 }
5769 return Op;
5770}
5771
5772static bool isZeroVector(SDValue N) {
5773 if (N->getOpcode() == ISD::BITCAST)
5774 N = N->getOperand(Num: 0);
5775 if (N->getOpcode() == ISD::SPLAT_VECTOR)
5776 if (auto *Op = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0)))
5777 return Op->getZExtValue() == 0;
5778 return ISD::isBuildVectorAllZeros(N: N.getNode());
5779}
5780
5781// Return the index of the zero/undef vector, or UINT32_MAX if not found.
5782static uint32_t findZeroVectorIdx(SDValue *Ops, unsigned Num) {
5783 for (unsigned I = 0; I < Num ; I++)
5784 if (isZeroVector(N: Ops[I]))
5785 return I;
5786 return UINT32_MAX;
5787}
5788
5789// Bytes is a VPERM-like permute vector, except that -1 is used for
5790// undefined bytes. Implement it on operands Ops[0] and Ops[1] using
5791// VSLDB or VPERM.
5792static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5793 SDValue *Ops,
5794 const SmallVectorImpl<int> &Bytes) {
5795 for (unsigned I = 0; I < 2; ++I)
5796 Ops[I] = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Ops[I]);
5797
5798 // First see whether VSLDB can be used.
5799 unsigned StartIndex, OpNo0, OpNo1;
5800 if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
5801 return DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE, DL, VT: MVT::v16i8, N1: Ops[OpNo0],
5802 N2: Ops[OpNo1],
5803 N3: DAG.getTargetConstant(Val: StartIndex, DL, VT: MVT::i32));
5804
5805 // Fall back on VPERM. Construct an SDNode for the permute vector. Try to
5806 // eliminate a zero vector by reusing any zero index in the permute vector.
5807 unsigned ZeroVecIdx = findZeroVectorIdx(Ops: &Ops[0], Num: 2);
5808 if (ZeroVecIdx != UINT32_MAX) {
5809 bool MaskFirst = true;
5810 int ZeroIdx = -1;
5811 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5812 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
5813 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
5814 if (OpNo == ZeroVecIdx && I == 0) {
5815 // If the first byte is zero, use mask as first operand.
5816 ZeroIdx = 0;
5817 break;
5818 }
5819 if (OpNo != ZeroVecIdx && Byte == 0) {
5820 // If mask contains a zero, use it by placing that vector first.
5821 ZeroIdx = I + SystemZ::VectorBytes;
5822 MaskFirst = false;
5823 break;
5824 }
5825 }
5826 if (ZeroIdx != -1) {
5827 SDValue IndexNodes[SystemZ::VectorBytes];
5828 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5829 if (Bytes[I] >= 0) {
5830 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
5831 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
5832 if (OpNo == ZeroVecIdx)
5833 IndexNodes[I] = DAG.getConstant(Val: ZeroIdx, DL, VT: MVT::i32);
5834 else {
5835 unsigned BIdx = MaskFirst ? Byte + SystemZ::VectorBytes : Byte;
5836 IndexNodes[I] = DAG.getConstant(Val: BIdx, DL, VT: MVT::i32);
5837 }
5838 } else
5839 IndexNodes[I] = DAG.getUNDEF(VT: MVT::i32);
5840 }
5841 SDValue Mask = DAG.getBuildVector(VT: MVT::v16i8, DL, Ops: IndexNodes);
5842 SDValue Src = ZeroVecIdx == 0 ? Ops[1] : Ops[0];
5843 if (MaskFirst)
5844 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Mask, N2: Src,
5845 N3: Mask);
5846 else
5847 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Src, N2: Mask,
5848 N3: Mask);
5849 }
5850 }
5851
5852 SDValue IndexNodes[SystemZ::VectorBytes];
5853 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
5854 if (Bytes[I] >= 0)
5855 IndexNodes[I] = DAG.getConstant(Val: Bytes[I], DL, VT: MVT::i32);
5856 else
5857 IndexNodes[I] = DAG.getUNDEF(VT: MVT::i32);
5858 SDValue Op2 = DAG.getBuildVector(VT: MVT::v16i8, DL, Ops: IndexNodes);
5859 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Ops[0],
5860 N2: (!Ops[1].isUndef() ? Ops[1] : Ops[0]), N3: Op2);
5861}
5862
5863namespace {
5864// Describes a general N-operand vector shuffle.
5865struct GeneralShuffle {
5866 GeneralShuffle(EVT vt)
5867 : VT(vt), UnpackFromEltSize(UINT_MAX), UnpackLow(false) {}
5868 void addUndef();
5869 bool add(SDValue, unsigned);
5870 SDValue getNode(SelectionDAG &, const SDLoc &);
5871 void tryPrepareForUnpack();
5872 bool unpackWasPrepared() { return UnpackFromEltSize <= 4; }
5873 SDValue insertUnpackIfPrepared(SelectionDAG &DAG, const SDLoc &DL, SDValue Op);
5874
5875 // The operands of the shuffle.
5876 SmallVector<SDValue, SystemZ::VectorBytes> Ops;
5877
5878 // Index I is -1 if byte I of the result is undefined. Otherwise the
5879 // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand
5880 // Bytes[I] / SystemZ::VectorBytes.
5881 SmallVector<int, SystemZ::VectorBytes> Bytes;
5882
5883 // The type of the shuffle result.
5884 EVT VT;
5885
5886 // Holds a value of 1, 2 or 4 if a final unpack has been prepared for.
5887 unsigned UnpackFromEltSize;
5888 // True if the final unpack uses the low half.
5889 bool UnpackLow;
5890};
5891} // namespace
5892
5893// Add an extra undefined element to the shuffle.
5894void GeneralShuffle::addUndef() {
5895 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5896 for (unsigned I = 0; I < BytesPerElement; ++I)
5897 Bytes.push_back(Elt: -1);
5898}
5899
5900// Add an extra element to the shuffle, taking it from element Elem of Op.
5901// A null Op indicates a vector input whose value will be calculated later;
5902// there is at most one such input per shuffle and it always has the same
5903// type as the result. Aborts and returns false if the source vector elements
5904// of an EXTRACT_VECTOR_ELT are smaller than the destination elements. Per
5905// LLVM they become implicitly extended, but this is rare and not optimized.
5906bool GeneralShuffle::add(SDValue Op, unsigned Elem) {
5907 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5908
5909 // The source vector can have wider elements than the result,
5910 // either through an explicit TRUNCATE or because of type legalization.
5911 // We want the least significant part.
5912 EVT FromVT = Op.getNode() ? Op.getValueType() : VT;
5913 unsigned FromBytesPerElement = FromVT.getVectorElementType().getStoreSize();
5914
5915 // Return false if the source elements are smaller than their destination
5916 // elements.
5917 if (FromBytesPerElement < BytesPerElement)
5918 return false;
5919
5920 unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes +
5921 (FromBytesPerElement - BytesPerElement));
5922
5923 // Look through things like shuffles and bitcasts.
5924 while (Op.getNode()) {
5925 if (Op.getOpcode() == ISD::BITCAST)
5926 Op = Op.getOperand(i: 0);
5927 else if (Op.getOpcode() == ISD::VECTOR_SHUFFLE && Op.hasOneUse()) {
5928 // See whether the bytes we need come from a contiguous part of one
5929 // operand.
5930 SmallVector<int, SystemZ::VectorBytes> OpBytes;
5931 if (!getVPermMask(ShuffleOp: Op, Bytes&: OpBytes))
5932 break;
5933 int NewByte;
5934 if (!getShuffleInput(Bytes: OpBytes, Start: Byte, BytesPerElement, Base&: NewByte))
5935 break;
5936 if (NewByte < 0) {
5937 addUndef();
5938 return true;
5939 }
5940 Op = Op.getOperand(i: unsigned(NewByte) / SystemZ::VectorBytes);
5941 Byte = unsigned(NewByte) % SystemZ::VectorBytes;
5942 } else if (Op.isUndef()) {
5943 addUndef();
5944 return true;
5945 } else
5946 break;
5947 }
5948
5949 // Make sure that the source of the extraction is in Ops.
5950 unsigned OpNo = 0;
5951 for (; OpNo < Ops.size(); ++OpNo)
5952 if (Ops[OpNo] == Op)
5953 break;
5954 if (OpNo == Ops.size())
5955 Ops.push_back(Elt: Op);
5956
5957 // Add the element to Bytes.
5958 unsigned Base = OpNo * SystemZ::VectorBytes + Byte;
5959 for (unsigned I = 0; I < BytesPerElement; ++I)
5960 Bytes.push_back(Elt: Base + I);
5961
5962 return true;
5963}
5964
5965// Return SDNodes for the completed shuffle.
5966SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
5967 assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector");
5968
5969 if (Ops.size() == 0)
5970 return DAG.getUNDEF(VT);
5971
5972 // Use a single unpack if possible as the last operation.
5973 tryPrepareForUnpack();
5974
5975 // Make sure that there are at least two shuffle operands.
5976 if (Ops.size() == 1)
5977 Ops.push_back(Elt: DAG.getUNDEF(VT: MVT::v16i8));
5978
5979 // Create a tree of shuffles, deferring root node until after the loop.
5980 // Try to redistribute the undefined elements of non-root nodes so that
5981 // the non-root shuffles match something like a pack or merge, then adjust
5982 // the parent node's permute vector to compensate for the new order.
5983 // Among other things, this copes with vectors like <2 x i16> that were
5984 // padded with undefined elements during type legalization.
5985 //
5986 // In the best case this redistribution will lead to the whole tree
5987 // using packs and merges. It should rarely be a loss in other cases.
5988 unsigned Stride = 1;
5989 for (; Stride * 2 < Ops.size(); Stride *= 2) {
5990 for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) {
5991 SDValue SubOps[] = { Ops[I], Ops[I + Stride] };
5992
5993 // Create a mask for just these two operands.
5994 SmallVector<int, SystemZ::VectorBytes> NewBytes(SystemZ::VectorBytes);
5995 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
5996 unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes;
5997 unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes;
5998 if (OpNo == I)
5999 NewBytes[J] = Byte;
6000 else if (OpNo == I + Stride)
6001 NewBytes[J] = SystemZ::VectorBytes + Byte;
6002 else
6003 NewBytes[J] = -1;
6004 }
6005 // See if it would be better to reorganize NewMask to avoid using VPERM.
6006 SmallVector<int, SystemZ::VectorBytes> NewBytesMap(SystemZ::VectorBytes);
6007 if (const Permute *P = matchDoublePermute(Bytes: NewBytes, Transform&: NewBytesMap)) {
6008 Ops[I] = getPermuteNode(DAG, DL, P: *P, Op0: SubOps[0], Op1: SubOps[1]);
6009 // Applying NewBytesMap to Ops[I] gets back to NewBytes.
6010 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
6011 if (NewBytes[J] >= 0) {
6012 assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes &&
6013 "Invalid double permute");
6014 Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J];
6015 } else
6016 assert(NewBytesMap[J] < 0 && "Invalid double permute");
6017 }
6018 } else {
6019 // Just use NewBytes on the operands.
6020 Ops[I] = getGeneralPermuteNode(DAG, DL, Ops: SubOps, Bytes: NewBytes);
6021 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J)
6022 if (NewBytes[J] >= 0)
6023 Bytes[J] = I * SystemZ::VectorBytes + J;
6024 }
6025 }
6026 }
6027
6028 // Now we just have 2 inputs. Put the second operand in Ops[1].
6029 if (Stride > 1) {
6030 Ops[1] = Ops[Stride];
6031 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6032 if (Bytes[I] >= int(SystemZ::VectorBytes))
6033 Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;
6034 }
6035
6036 // Look for an instruction that can do the permute without resorting
6037 // to VPERM.
6038 unsigned OpNo0, OpNo1;
6039 SDValue Op;
6040 if (unpackWasPrepared() && Ops[1].isUndef())
6041 Op = Ops[0];
6042 else if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
6043 Op = getPermuteNode(DAG, DL, P: *P, Op0: Ops[OpNo0], Op1: Ops[OpNo1]);
6044 else
6045 Op = getGeneralPermuteNode(DAG, DL, Ops: &Ops[0], Bytes);
6046
6047 Op = insertUnpackIfPrepared(DAG, DL, Op);
6048
6049 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
6050}
6051
6052#ifndef NDEBUG
6053static void dumpBytes(const SmallVectorImpl<int> &Bytes, std::string Msg) {
6054 dbgs() << Msg.c_str() << " { ";
6055 for (unsigned I = 0; I < Bytes.size(); I++)
6056 dbgs() << Bytes[I] << " ";
6057 dbgs() << "}\n";
6058}
6059#endif
6060
6061// If the Bytes vector matches an unpack operation, prepare to do the unpack
6062// after all else by removing the zero vector and the effect of the unpack on
6063// Bytes.
6064void GeneralShuffle::tryPrepareForUnpack() {
6065 uint32_t ZeroVecOpNo = findZeroVectorIdx(Ops: &Ops[0], Num: Ops.size());
6066 if (ZeroVecOpNo == UINT32_MAX || Ops.size() == 1)
6067 return;
6068
6069 // Only do this if removing the zero vector reduces the depth, otherwise
6070 // the critical path will increase with the final unpack.
6071 if (Ops.size() > 2 &&
6072 Log2_32_Ceil(Value: Ops.size()) == Log2_32_Ceil(Value: Ops.size() - 1))
6073 return;
6074
6075 // Find an unpack that would allow removing the zero vector from Ops.
6076 UnpackFromEltSize = 1;
6077 for (; UnpackFromEltSize <= 4; UnpackFromEltSize *= 2) {
6078 bool MatchUnpack = true;
6079 SmallVector<int, SystemZ::VectorBytes> SrcBytes;
6080 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes; Elt++) {
6081 unsigned ToEltSize = UnpackFromEltSize * 2;
6082 bool IsZextByte = (Elt % ToEltSize) < UnpackFromEltSize;
6083 if (!IsZextByte)
6084 SrcBytes.push_back(Elt: Bytes[Elt]);
6085 if (Bytes[Elt] != -1) {
6086 unsigned OpNo = unsigned(Bytes[Elt]) / SystemZ::VectorBytes;
6087 if (IsZextByte != (OpNo == ZeroVecOpNo)) {
6088 MatchUnpack = false;
6089 break;
6090 }
6091 }
6092 }
6093 if (MatchUnpack) {
6094 if (Ops.size() == 2) {
6095 // Don't use unpack if a single source operand needs rearrangement.
6096 bool CanUseUnpackLow = true, CanUseUnpackHigh = true;
6097 for (unsigned i = 0; i < SystemZ::VectorBytes / 2; i++) {
6098 if (SrcBytes[i] == -1)
6099 continue;
6100 if (SrcBytes[i] % 16 != int(i))
6101 CanUseUnpackHigh = false;
6102 if (SrcBytes[i] % 16 != int(i + SystemZ::VectorBytes / 2))
6103 CanUseUnpackLow = false;
6104 if (!CanUseUnpackLow && !CanUseUnpackHigh) {
6105 UnpackFromEltSize = UINT_MAX;
6106 return;
6107 }
6108 }
6109 if (!CanUseUnpackHigh)
6110 UnpackLow = true;
6111 }
6112 break;
6113 }
6114 }
6115 if (UnpackFromEltSize > 4)
6116 return;
6117
6118 LLVM_DEBUG(dbgs() << "Preparing for final unpack of element size "
6119 << UnpackFromEltSize << ". Zero vector is Op#" << ZeroVecOpNo
6120 << ".\n";
6121 dumpBytes(Bytes, "Original Bytes vector:"););
6122
6123 // Apply the unpack in reverse to the Bytes array.
6124 unsigned B = 0;
6125 if (UnpackLow) {
6126 while (B < SystemZ::VectorBytes / 2)
6127 Bytes[B++] = -1;
6128 }
6129 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes;) {
6130 Elt += UnpackFromEltSize;
6131 for (unsigned i = 0; i < UnpackFromEltSize; i++, Elt++, B++)
6132 Bytes[B] = Bytes[Elt];
6133 }
6134 if (!UnpackLow) {
6135 while (B < SystemZ::VectorBytes)
6136 Bytes[B++] = -1;
6137 }
6138
6139 // Remove the zero vector from Ops
6140 Ops.erase(CI: &Ops[ZeroVecOpNo]);
6141 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6142 if (Bytes[I] >= 0) {
6143 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
6144 if (OpNo > ZeroVecOpNo)
6145 Bytes[I] -= SystemZ::VectorBytes;
6146 }
6147
6148 LLVM_DEBUG(dumpBytes(Bytes, "Resulting Bytes vector, zero vector removed:");
6149 dbgs() << "\n";);
6150}
6151
6152SDValue GeneralShuffle::insertUnpackIfPrepared(SelectionDAG &DAG,
6153 const SDLoc &DL,
6154 SDValue Op) {
6155 if (!unpackWasPrepared())
6156 return Op;
6157 unsigned InBits = UnpackFromEltSize * 8;
6158 EVT InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: InBits),
6159 NumElements: SystemZ::VectorBits / InBits);
6160 SDValue PackedOp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op);
6161 unsigned OutBits = InBits * 2;
6162 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: OutBits),
6163 NumElements: SystemZ::VectorBits / OutBits);
6164 return DAG.getNode(Opcode: UnpackLow ? SystemZISD::UNPACKL_LOW
6165 : SystemZISD::UNPACKL_HIGH,
6166 DL, VT: OutVT, Operand: PackedOp);
6167}
6168
6169// Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
6170static bool isScalarToVector(SDValue Op) {
6171 for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I)
6172 if (!Op.getOperand(i: I).isUndef())
6173 return false;
6174 return true;
6175}
6176
6177// Return a vector of type VT that contains Value in the first element.
6178// The other elements don't matter.
6179static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6180 SDValue Value) {
6181 // If we have a constant, replicate it to all elements and let the
6182 // BUILD_VECTOR lowering take care of it.
6183 if (Value.getOpcode() == ISD::Constant ||
6184 Value.getOpcode() == ISD::ConstantFP) {
6185 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Value);
6186 return DAG.getBuildVector(VT, DL, Ops);
6187 }
6188 if (Value.isUndef())
6189 return DAG.getUNDEF(VT);
6190 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Value);
6191}
6192
6193// Return a vector of type VT in which Op0 is in element 0 and Op1 is in
6194// element 1. Used for cases in which replication is cheap.
6195static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6196 SDValue Op0, SDValue Op1) {
6197 if (Op0.isUndef()) {
6198 if (Op1.isUndef())
6199 return DAG.getUNDEF(VT);
6200 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op1);
6201 }
6202 if (Op1.isUndef())
6203 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op0);
6204 return DAG.getNode(Opcode: SystemZISD::MERGE_HIGH, DL, VT,
6205 N1: buildScalarToVector(DAG, DL, VT, Value: Op0),
6206 N2: buildScalarToVector(DAG, DL, VT, Value: Op1));
6207}
6208
6209// Extend GPR scalars Op0 and Op1 to doublewords and return a v2i64
6210// vector for them.
6211static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
6212 SDValue Op1) {
6213 if (Op0.isUndef() && Op1.isUndef())
6214 return DAG.getUNDEF(VT: MVT::v2i64);
6215 // If one of the two inputs is undefined then replicate the other one,
6216 // in order to avoid using another register unnecessarily.
6217 if (Op0.isUndef())
6218 Op0 = Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op1);
6219 else if (Op1.isUndef())
6220 Op0 = Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
6221 else {
6222 Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
6223 Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op1);
6224 }
6225 return DAG.getNode(Opcode: SystemZISD::JOIN_DWORDS, DL, VT: MVT::v2i64, N1: Op0, N2: Op1);
6226}
6227
6228// If a BUILD_VECTOR contains some EXTRACT_VECTOR_ELTs, it's usually
6229// better to use VECTOR_SHUFFLEs on them, only using BUILD_VECTOR for
6230// the non-EXTRACT_VECTOR_ELT elements. See if the given BUILD_VECTOR
6231// would benefit from this representation and return it if so.
6232static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
6233 BuildVectorSDNode *BVN) {
6234 EVT VT = BVN->getValueType(ResNo: 0);
6235 unsigned NumElements = VT.getVectorNumElements();
6236
6237 // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation
6238 // on byte vectors. If there are non-EXTRACT_VECTOR_ELT elements that still
6239 // need a BUILD_VECTOR, add an additional placeholder operand for that
6240 // BUILD_VECTOR and store its operands in ResidueOps.
6241 GeneralShuffle GS(VT);
6242 SmallVector<SDValue, SystemZ::VectorBytes> ResidueOps;
6243 bool FoundOne = false;
6244 for (unsigned I = 0; I < NumElements; ++I) {
6245 SDValue Op = BVN->getOperand(Num: I);
6246 if (Op.getOpcode() == ISD::TRUNCATE)
6247 Op = Op.getOperand(i: 0);
6248 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
6249 Op.getOperand(i: 1).getOpcode() == ISD::Constant) {
6250 unsigned Elem = Op.getConstantOperandVal(i: 1);
6251 if (!GS.add(Op: Op.getOperand(i: 0), Elem))
6252 return SDValue();
6253 FoundOne = true;
6254 } else if (Op.isUndef()) {
6255 GS.addUndef();
6256 } else {
6257 if (!GS.add(Op: SDValue(), Elem: ResidueOps.size()))
6258 return SDValue();
6259 ResidueOps.push_back(Elt: BVN->getOperand(Num: I));
6260 }
6261 }
6262
6263 // Nothing to do if there are no EXTRACT_VECTOR_ELTs.
6264 if (!FoundOne)
6265 return SDValue();
6266
6267 // Create the BUILD_VECTOR for the remaining elements, if any.
6268 if (!ResidueOps.empty()) {
6269 while (ResidueOps.size() < NumElements)
6270 ResidueOps.push_back(Elt: DAG.getUNDEF(VT: ResidueOps[0].getValueType()));
6271 for (auto &Op : GS.Ops) {
6272 if (!Op.getNode()) {
6273 Op = DAG.getBuildVector(VT, DL: SDLoc(BVN), Ops: ResidueOps);
6274 break;
6275 }
6276 }
6277 }
6278 return GS.getNode(DAG, DL: SDLoc(BVN));
6279}
6280
6281bool SystemZTargetLowering::isVectorElementLoad(SDValue Op) const {
6282 if (Op.getOpcode() == ISD::LOAD && cast<LoadSDNode>(Val&: Op)->isUnindexed())
6283 return true;
6284 if (auto *AL = dyn_cast<AtomicSDNode>(Val&: Op))
6285 if (AL->getOpcode() == ISD::ATOMIC_LOAD)
6286 return true;
6287 if (Subtarget.hasVectorEnhancements2() && Op.getOpcode() == SystemZISD::LRV)
6288 return true;
6289 return false;
6290}
6291
6292// Combine GPR scalar values Elems into a vector of type VT.
6293SDValue
6294SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6295 SmallVectorImpl<SDValue> &Elems) const {
6296 // See whether there is a single replicated value.
6297 SDValue Single;
6298 unsigned int NumElements = Elems.size();
6299 unsigned int Count = 0;
6300 for (auto Elem : Elems) {
6301 if (!Elem.isUndef()) {
6302 if (!Single.getNode())
6303 Single = Elem;
6304 else if (Elem != Single) {
6305 Single = SDValue();
6306 break;
6307 }
6308 Count += 1;
6309 }
6310 }
6311 // There are three cases here:
6312 //
6313 // - if the only defined element is a loaded one, the best sequence
6314 // is a replicating load.
6315 //
6316 // - otherwise, if the only defined element is an i64 value, we will
6317 // end up with the same VLVGP sequence regardless of whether we short-cut
6318 // for replication or fall through to the later code.
6319 //
6320 // - otherwise, if the only defined element is an i32 or smaller value,
6321 // we would need 2 instructions to replicate it: VLVGP followed by VREPx.
6322 // This is only a win if the single defined element is used more than once.
6323 // In other cases we're better off using a single VLVGx.
6324 if (Single.getNode() && (Count > 1 || isVectorElementLoad(Op: Single)))
6325 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Single);
6326
6327 // If all elements are loads, use VLREP/VLEs (below).
6328 bool AllLoads = true;
6329 for (auto Elem : Elems)
6330 if (!isVectorElementLoad(Op: Elem)) {
6331 AllLoads = false;
6332 break;
6333 }
6334
6335 // The best way of building a v2i64 from two i64s is to use VLVGP.
6336 if (VT == MVT::v2i64 && !AllLoads)
6337 return joinDwords(DAG, DL, Op0: Elems[0], Op1: Elems[1]);
6338
6339 // Use a 64-bit merge high to combine two doubles.
6340 if (VT == MVT::v2f64 && !AllLoads)
6341 return buildMergeScalars(DAG, DL, VT, Op0: Elems[0], Op1: Elems[1]);
6342
6343 // Build v4f32 values directly from the FPRs:
6344 //
6345 // <Axxx> <Bxxx> <Cxxxx> <Dxxx>
6346 // V V VMRHF
6347 // <ABxx> <CDxx>
6348 // V VMRHG
6349 // <ABCD>
6350 if (VT == MVT::v4f32 && !AllLoads) {
6351 SDValue Op01 = buildMergeScalars(DAG, DL, VT, Op0: Elems[0], Op1: Elems[1]);
6352 SDValue Op23 = buildMergeScalars(DAG, DL, VT, Op0: Elems[2], Op1: Elems[3]);
6353 // Avoid unnecessary undefs by reusing the other operand.
6354 if (Op01.isUndef())
6355 Op01 = Op23;
6356 else if (Op23.isUndef())
6357 Op23 = Op01;
6358 // Merging identical replications is a no-op.
6359 if (Op01.getOpcode() == SystemZISD::REPLICATE && Op01 == Op23)
6360 return Op01;
6361 Op01 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: Op01);
6362 Op23 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: Op23);
6363 SDValue Op = DAG.getNode(Opcode: SystemZISD::MERGE_HIGH,
6364 DL, VT: MVT::v2i64, N1: Op01, N2: Op23);
6365 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
6366 }
6367
6368 // Collect the constant terms.
6369 SmallVector<SDValue, SystemZ::VectorBytes> Constants(NumElements, SDValue());
6370 SmallVector<bool, SystemZ::VectorBytes> Done(NumElements, false);
6371
6372 unsigned NumConstants = 0;
6373 for (unsigned I = 0; I < NumElements; ++I) {
6374 SDValue Elem = Elems[I];
6375 if (Elem.getOpcode() == ISD::Constant ||
6376 Elem.getOpcode() == ISD::ConstantFP) {
6377 NumConstants += 1;
6378 Constants[I] = Elem;
6379 Done[I] = true;
6380 }
6381 }
6382 // If there was at least one constant, fill in the other elements of
6383 // Constants with undefs to get a full vector constant and use that
6384 // as the starting point.
6385 SDValue Result;
6386 SDValue ReplicatedVal;
6387 if (NumConstants > 0) {
6388 for (unsigned I = 0; I < NumElements; ++I)
6389 if (!Constants[I].getNode())
6390 Constants[I] = DAG.getUNDEF(VT: Elems[I].getValueType());
6391 Result = DAG.getBuildVector(VT, DL, Ops: Constants);
6392 } else {
6393 // Otherwise try to use VLREP or VLVGP to start the sequence in order to
6394 // avoid a false dependency on any previous contents of the vector
6395 // register.
6396
6397 // Use a VLREP if at least one element is a load. Make sure to replicate
6398 // the load with the most elements having its value.
6399 std::map<const SDNode*, unsigned> UseCounts;
6400 SDNode *LoadMaxUses = nullptr;
6401 for (unsigned I = 0; I < NumElements; ++I)
6402 if (isVectorElementLoad(Op: Elems[I])) {
6403 SDNode *Ld = Elems[I].getNode();
6404 unsigned Count = ++UseCounts[Ld];
6405 if (LoadMaxUses == nullptr || UseCounts[LoadMaxUses] < Count)
6406 LoadMaxUses = Ld;
6407 }
6408 if (LoadMaxUses != nullptr) {
6409 ReplicatedVal = SDValue(LoadMaxUses, 0);
6410 Result = DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: ReplicatedVal);
6411 } else {
6412 // Try to use VLVGP.
6413 unsigned I1 = NumElements / 2 - 1;
6414 unsigned I2 = NumElements - 1;
6415 bool Def1 = !Elems[I1].isUndef();
6416 bool Def2 = !Elems[I2].isUndef();
6417 if (Def1 || Def2) {
6418 SDValue Elem1 = Elems[Def1 ? I1 : I2];
6419 SDValue Elem2 = Elems[Def2 ? I2 : I1];
6420 Result = DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
6421 Operand: joinDwords(DAG, DL, Op0: Elem1, Op1: Elem2));
6422 Done[I1] = true;
6423 Done[I2] = true;
6424 } else
6425 Result = DAG.getUNDEF(VT);
6426 }
6427 }
6428
6429 // Use VLVGx to insert the other elements.
6430 for (unsigned I = 0; I < NumElements; ++I)
6431 if (!Done[I] && !Elems[I].isUndef() && Elems[I] != ReplicatedVal)
6432 Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: Result, N2: Elems[I],
6433 N3: DAG.getConstant(Val: I, DL, VT: MVT::i32));
6434 return Result;
6435}
6436
6437SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
6438 SelectionDAG &DAG) const {
6439 auto *BVN = cast<BuildVectorSDNode>(Val: Op.getNode());
6440 SDLoc DL(Op);
6441 EVT VT = Op.getValueType();
6442
6443 if (BVN->isConstant()) {
6444 if (SystemZVectorConstantInfo(BVN).isVectorConstantLegal(Subtarget))
6445 return Op;
6446
6447 // Fall back to loading it from memory.
6448 return SDValue();
6449 }
6450
6451 // See if we should use shuffles to construct the vector from other vectors.
6452 if (SDValue Res = tryBuildVectorShuffle(DAG, BVN))
6453 return Res;
6454
6455 // Detect SCALAR_TO_VECTOR conversions.
6456 if (isOperationLegal(Op: ISD::SCALAR_TO_VECTOR, VT) && isScalarToVector(Op))
6457 return buildScalarToVector(DAG, DL, VT, Value: Op.getOperand(i: 0));
6458
6459 // Otherwise use buildVector to build the vector up from GPRs.
6460 unsigned NumElements = Op.getNumOperands();
6461 SmallVector<SDValue, SystemZ::VectorBytes> Ops(NumElements);
6462 for (unsigned I = 0; I < NumElements; ++I)
6463 Ops[I] = Op.getOperand(i: I);
6464 return buildVector(DAG, DL, VT, Elems&: Ops);
6465}
6466
6467SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
6468 SelectionDAG &DAG) const {
6469 auto *VSN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
6470 SDLoc DL(Op);
6471 EVT VT = Op.getValueType();
6472 unsigned NumElements = VT.getVectorNumElements();
6473
6474 if (VSN->isSplat()) {
6475 SDValue Op0 = Op.getOperand(i: 0);
6476 unsigned Index = VSN->getSplatIndex();
6477 assert(Index < VT.getVectorNumElements() &&
6478 "Splat index should be defined and in first operand");
6479 // See whether the value we're splatting is directly available as a scalar.
6480 if ((Index == 0 && Op0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
6481 Op0.getOpcode() == ISD::BUILD_VECTOR)
6482 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op0.getOperand(i: Index));
6483 // Otherwise keep it as a vector-to-vector operation.
6484 return DAG.getNode(Opcode: SystemZISD::SPLAT, DL, VT, N1: Op.getOperand(i: 0),
6485 N2: DAG.getTargetConstant(Val: Index, DL, VT: MVT::i32));
6486 }
6487
6488 GeneralShuffle GS(VT);
6489 for (unsigned I = 0; I < NumElements; ++I) {
6490 int Elt = VSN->getMaskElt(Idx: I);
6491 if (Elt < 0)
6492 GS.addUndef();
6493 else if (!GS.add(Op: Op.getOperand(i: unsigned(Elt) / NumElements),
6494 Elem: unsigned(Elt) % NumElements))
6495 return SDValue();
6496 }
6497 return GS.getNode(DAG, DL: SDLoc(VSN));
6498}
6499
6500SDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
6501 SelectionDAG &DAG) const {
6502 SDLoc DL(Op);
6503 // Just insert the scalar into element 0 of an undefined vector.
6504 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL,
6505 VT: Op.getValueType(), N1: DAG.getUNDEF(VT: Op.getValueType()),
6506 N2: Op.getOperand(i: 0), N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
6507}
6508
6509SDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
6510 SelectionDAG &DAG) const {
6511 // Handle insertions of floating-point values.
6512 SDLoc DL(Op);
6513 SDValue Op0 = Op.getOperand(i: 0);
6514 SDValue Op1 = Op.getOperand(i: 1);
6515 SDValue Op2 = Op.getOperand(i: 2);
6516 EVT VT = Op.getValueType();
6517
6518 // Insertions into constant indices of a v2f64 can be done using VPDI.
6519 // However, if the inserted value is a bitcast or a constant then it's
6520 // better to use GPRs, as below.
6521 if (VT == MVT::v2f64 &&
6522 Op1.getOpcode() != ISD::BITCAST &&
6523 Op1.getOpcode() != ISD::ConstantFP &&
6524 Op2.getOpcode() == ISD::Constant) {
6525 uint64_t Index = Op2->getAsZExtVal();
6526 unsigned Mask = VT.getVectorNumElements() - 1;
6527 if (Index <= Mask)
6528 return Op;
6529 }
6530
6531 // Otherwise bitcast to the equivalent integer form and insert via a GPR.
6532 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits());
6533 MVT IntVecVT = MVT::getVectorVT(VT: IntVT, NumElements: VT.getVectorNumElements());
6534 SDValue Res = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: IntVecVT,
6535 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0),
6536 N2: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Op1), N3: Op2);
6537 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Res);
6538}
6539
6540SDValue
6541SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
6542 SelectionDAG &DAG) const {
6543 // Handle extractions of floating-point values.
6544 SDLoc DL(Op);
6545 SDValue Op0 = Op.getOperand(i: 0);
6546 SDValue Op1 = Op.getOperand(i: 1);
6547 EVT VT = Op.getValueType();
6548 EVT VecVT = Op0.getValueType();
6549
6550 // Extractions of constant indices can be done directly.
6551 if (auto *CIndexN = dyn_cast<ConstantSDNode>(Val&: Op1)) {
6552 uint64_t Index = CIndexN->getZExtValue();
6553 unsigned Mask = VecVT.getVectorNumElements() - 1;
6554 if (Index <= Mask)
6555 return Op;
6556 }
6557
6558 // Otherwise bitcast to the equivalent integer form and extract via a GPR.
6559 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getSizeInBits());
6560 MVT IntVecVT = MVT::getVectorVT(VT: IntVT, NumElements: VecVT.getVectorNumElements());
6561 SDValue Res = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IntVT,
6562 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0), N2: Op1);
6563 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Res);
6564}
6565
6566SDValue SystemZTargetLowering::
6567lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6568 SDValue PackedOp = Op.getOperand(i: 0);
6569 EVT OutVT = Op.getValueType();
6570 EVT InVT = PackedOp.getValueType();
6571 unsigned ToBits = OutVT.getScalarSizeInBits();
6572 unsigned FromBits = InVT.getScalarSizeInBits();
6573 unsigned StartOffset = 0;
6574
6575 // If the input is a VECTOR_SHUFFLE, there are a number of important
6576 // cases where we can directly implement the sign-extension of the
6577 // original input lanes of the shuffle.
6578 if (PackedOp.getOpcode() == ISD::VECTOR_SHUFFLE) {
6579 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: PackedOp.getNode());
6580 ArrayRef<int> ShuffleMask = SVN->getMask();
6581 int OutNumElts = OutVT.getVectorNumElements();
6582
6583 // Recognize the special case where the sign-extension can be done
6584 // by the VSEG instruction. Handled via the default expander.
6585 if (ToBits == 64 && OutNumElts == 2) {
6586 int NumElem = ToBits / FromBits;
6587 if (ShuffleMask[0] == NumElem - 1 && ShuffleMask[1] == 2 * NumElem - 1)
6588 return SDValue();
6589 }
6590
6591 // Recognize the special case where we can fold the shuffle by
6592 // replacing some of the UNPACK_HIGH with UNPACK_LOW.
6593 int StartOffsetCandidate = -1;
6594 for (int Elt = 0; Elt < OutNumElts; Elt++) {
6595 if (ShuffleMask[Elt] == -1)
6596 continue;
6597 if (ShuffleMask[Elt] % OutNumElts == Elt) {
6598 if (StartOffsetCandidate == -1)
6599 StartOffsetCandidate = ShuffleMask[Elt] - Elt;
6600 if (StartOffsetCandidate == ShuffleMask[Elt] - Elt)
6601 continue;
6602 }
6603 StartOffsetCandidate = -1;
6604 break;
6605 }
6606 if (StartOffsetCandidate != -1) {
6607 StartOffset = StartOffsetCandidate;
6608 PackedOp = PackedOp.getOperand(i: 0);
6609 }
6610 }
6611
6612 do {
6613 FromBits *= 2;
6614 unsigned OutNumElts = SystemZ::VectorBits / FromBits;
6615 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: FromBits), NumElements: OutNumElts);
6616 unsigned Opcode = SystemZISD::UNPACK_HIGH;
6617 if (StartOffset >= OutNumElts) {
6618 Opcode = SystemZISD::UNPACK_LOW;
6619 StartOffset -= OutNumElts;
6620 }
6621 PackedOp = DAG.getNode(Opcode, DL: SDLoc(PackedOp), VT: OutVT, Operand: PackedOp);
6622 } while (FromBits != ToBits);
6623 return PackedOp;
6624}
6625
6626// Lower a ZERO_EXTEND_VECTOR_INREG to a vector shuffle with a zero vector.
6627SDValue SystemZTargetLowering::
6628lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6629 SDValue PackedOp = Op.getOperand(i: 0);
6630 SDLoc DL(Op);
6631 EVT OutVT = Op.getValueType();
6632 EVT InVT = PackedOp.getValueType();
6633 unsigned InNumElts = InVT.getVectorNumElements();
6634 unsigned OutNumElts = OutVT.getVectorNumElements();
6635 unsigned NumInPerOut = InNumElts / OutNumElts;
6636
6637 SDValue ZeroVec =
6638 DAG.getSplatVector(VT: InVT, DL, Op: DAG.getConstant(Val: 0, DL, VT: InVT.getScalarType()));
6639
6640 SmallVector<int, 16> Mask(InNumElts);
6641 unsigned ZeroVecElt = InNumElts;
6642 for (unsigned PackedElt = 0; PackedElt < OutNumElts; PackedElt++) {
6643 unsigned MaskElt = PackedElt * NumInPerOut;
6644 unsigned End = MaskElt + NumInPerOut - 1;
6645 for (; MaskElt < End; MaskElt++)
6646 Mask[MaskElt] = ZeroVecElt++;
6647 Mask[MaskElt] = PackedElt;
6648 }
6649 SDValue Shuf = DAG.getVectorShuffle(VT: InVT, dl: DL, N1: PackedOp, N2: ZeroVec, Mask);
6650 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: OutVT, Operand: Shuf);
6651}
6652
6653SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
6654 unsigned ByScalar) const {
6655 // Look for cases where a vector shift can use the *_BY_SCALAR form.
6656 SDValue Op0 = Op.getOperand(i: 0);
6657 SDValue Op1 = Op.getOperand(i: 1);
6658 SDLoc DL(Op);
6659 EVT VT = Op.getValueType();
6660 unsigned ElemBitSize = VT.getScalarSizeInBits();
6661
6662 // See whether the shift vector is a splat represented as BUILD_VECTOR.
6663 if (auto *BVN = dyn_cast<BuildVectorSDNode>(Val&: Op1)) {
6664 APInt SplatBits, SplatUndef;
6665 unsigned SplatBitSize;
6666 bool HasAnyUndefs;
6667 // Check for constant splats. Use ElemBitSize as the minimum element
6668 // width and reject splats that need wider elements.
6669 if (BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6670 MinSplatBits: ElemBitSize, isBigEndian: true) &&
6671 SplatBitSize == ElemBitSize) {
6672 SDValue Shift = DAG.getConstant(Val: SplatBits.getZExtValue() & 0xfff,
6673 DL, VT: MVT::i32);
6674 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6675 }
6676 // Check for variable splats.
6677 BitVector UndefElements;
6678 SDValue Splat = BVN->getSplatValue(UndefElements: &UndefElements);
6679 if (Splat) {
6680 // Since i32 is the smallest legal type, we either need a no-op
6681 // or a truncation.
6682 SDValue Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Splat);
6683 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6684 }
6685 }
6686
6687 // See whether the shift vector is a splat represented as SHUFFLE_VECTOR,
6688 // and the shift amount is directly available in a GPR.
6689 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Val&: Op1)) {
6690 if (VSN->isSplat()) {
6691 SDValue VSNOp0 = VSN->getOperand(Num: 0);
6692 unsigned Index = VSN->getSplatIndex();
6693 assert(Index < VT.getVectorNumElements() &&
6694 "Splat index should be defined and in first operand");
6695 if ((Index == 0 && VSNOp0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
6696 VSNOp0.getOpcode() == ISD::BUILD_VECTOR) {
6697 // Since i32 is the smallest legal type, we either need a no-op
6698 // or a truncation.
6699 SDValue Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32,
6700 Operand: VSNOp0.getOperand(i: Index));
6701 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6702 }
6703 }
6704 }
6705
6706 // Otherwise just treat the current form as legal.
6707 return Op;
6708}
6709
6710SDValue SystemZTargetLowering::lowerFSHL(SDValue Op, SelectionDAG &DAG) const {
6711 SDLoc DL(Op);
6712
6713 // i128 FSHL with a constant amount that is a multiple of 8 can be
6714 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
6715 // facility, FSHL with a constant amount less than 8 can be implemented
6716 // via SHL_DOUBLE_BIT, and FSHL with other constant amounts by a
6717 // combination of the two.
6718 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2))) {
6719 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
6720 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
6721 SDValue Op0 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 0));
6722 SDValue Op1 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 1));
6723 SmallVector<int, 16> Mask(16);
6724 for (unsigned Elt = 0; Elt < 16; Elt++)
6725 Mask[Elt] = (ShiftAmt >> 3) + Elt;
6726 SDValue Shuf1 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op1, Mask);
6727 if ((ShiftAmt & 7) == 0)
6728 return DAG.getBitcast(VT: MVT::i128, V: Shuf1);
6729 SDValue Shuf2 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op1, N2: Op1, Mask);
6730 SDValue Val =
6731 DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Shuf1, N2: Shuf2,
6732 N3: DAG.getTargetConstant(Val: ShiftAmt & 7, DL, VT: MVT::i32));
6733 return DAG.getBitcast(VT: MVT::i128, V: Val);
6734 }
6735 }
6736
6737 return SDValue();
6738}
6739
6740SDValue SystemZTargetLowering::lowerFSHR(SDValue Op, SelectionDAG &DAG) const {
6741 SDLoc DL(Op);
6742
6743 // i128 FSHR with a constant amount that is a multiple of 8 can be
6744 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
6745 // facility, FSHR with a constant amount less than 8 can be implemented
6746 // via SHL_DOUBLE_BIT, and FSHR with other constant amounts by a
6747 // combination of the two.
6748 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2))) {
6749 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
6750 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
6751 SDValue Op0 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 0));
6752 SDValue Op1 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 1));
6753 SmallVector<int, 16> Mask(16);
6754 for (unsigned Elt = 0; Elt < 16; Elt++)
6755 Mask[Elt] = 16 - (ShiftAmt >> 3) + Elt;
6756 SDValue Shuf1 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op1, Mask);
6757 if ((ShiftAmt & 7) == 0)
6758 return DAG.getBitcast(VT: MVT::i128, V: Shuf1);
6759 SDValue Shuf2 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op0, Mask);
6760 SDValue Val =
6761 DAG.getNode(Opcode: SystemZISD::SHR_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Shuf2, N2: Shuf1,
6762 N3: DAG.getTargetConstant(Val: ShiftAmt & 7, DL, VT: MVT::i32));
6763 return DAG.getBitcast(VT: MVT::i128, V: Val);
6764 }
6765 }
6766
6767 return SDValue();
6768}
6769
6770static SDValue lowerAddrSpaceCast(SDValue Op, SelectionDAG &DAG) {
6771 SDLoc DL(Op);
6772 SDValue Src = Op.getOperand(i: 0);
6773 MVT DstVT = Op.getSimpleValueType();
6774
6775 AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Val: Op.getNode());
6776 unsigned SrcAS = N->getSrcAddressSpace();
6777
6778 assert(SrcAS != N->getDestAddressSpace() &&
6779 "addrspacecast must be between different address spaces");
6780
6781 // addrspacecast [0 <- 1] : Assinging a ptr32 value to a 64-bit pointer.
6782 // addrspacecast [1 <- 0] : Assigining a 64-bit pointer to a ptr32 value.
6783 if (SrcAS == SYSTEMZAS::PTR32 && DstVT == MVT::i64) {
6784 Op = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Src,
6785 N2: DAG.getConstant(Val: 0x7fffffff, DL, VT: MVT::i32));
6786 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: DstVT, Operand: Op);
6787 } else if (DstVT == MVT::i32) {
6788 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Src);
6789 Op = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Op,
6790 N2: DAG.getConstant(Val: 0x7fffffff, DL, VT: MVT::i32));
6791 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: DstVT, Operand: Op);
6792 } else {
6793 report_fatal_error(reason: "Bad address space in addrspacecast");
6794 }
6795 return Op;
6796}
6797
6798SDValue SystemZTargetLowering::lowerFP_EXTEND(SDValue Op,
6799 SelectionDAG &DAG) const {
6800 SDValue In = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0);
6801 if (In.getSimpleValueType() != MVT::f16)
6802 return Op; // Legal
6803 return SDValue(); // Let legalizer emit the libcall.
6804}
6805
6806SDValue SystemZTargetLowering::useLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
6807 MVT VT, SDValue Arg, SDLoc DL,
6808 SDValue Chain, bool IsStrict) const {
6809 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
6810 MakeLibCallOptions CallOptions;
6811 SDValue Result;
6812 std::tie(args&: Result, args&: Chain) =
6813 makeLibCall(DAG, LC, RetVT: VT, Ops: Arg, CallOptions, dl: DL, Chain);
6814 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: DL) : Result;
6815}
6816
6817SDValue SystemZTargetLowering::lower_FP_TO_INT(SDValue Op,
6818 SelectionDAG &DAG) const {
6819 bool IsSigned = (Op->getOpcode() == ISD::FP_TO_SINT ||
6820 Op->getOpcode() == ISD::STRICT_FP_TO_SINT);
6821 bool IsStrict = Op->isStrictFPOpcode();
6822 SDLoc DL(Op);
6823 MVT VT = Op.getSimpleValueType();
6824 SDValue InOp = Op.getOperand(i: IsStrict ? 1 : 0);
6825 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
6826 EVT InVT = InOp.getValueType();
6827
6828 // FP to unsigned is not directly supported on z10. Promoting an i32
6829 // result to (signed) i64 doesn't generate an inexact condition (fp
6830 // exception) for values that are outside the i32 range but in the i64
6831 // range, so use the default expansion.
6832 if (!Subtarget.hasFPExtension() && !IsSigned)
6833 // Expand i32/i64. F16 values will be recognized to fit and extended.
6834 return SDValue();
6835
6836 // Conversion from f16 is done via f32.
6837 if (InOp.getSimpleValueType() == MVT::f16) {
6838 SmallVector<SDValue, 2> Results;
6839 LowerOperationWrapper(N: Op.getNode(), Results, DAG);
6840 return DAG.getMergeValues(Ops: Results, dl: DL);
6841 }
6842
6843 if (VT == MVT::i128) {
6844 RTLIB::Libcall LC =
6845 IsSigned ? RTLIB::getFPTOSINT(OpVT: InVT, RetVT: VT) : RTLIB::getFPTOUINT(OpVT: InVT, RetVT: VT);
6846 return useLibCall(DAG, LC, VT, Arg: InOp, DL, Chain, IsStrict);
6847 }
6848
6849 return Op; // Legal
6850}
6851
6852SDValue SystemZTargetLowering::lower_INT_TO_FP(SDValue Op,
6853 SelectionDAG &DAG) const {
6854 bool IsSigned = (Op->getOpcode() == ISD::SINT_TO_FP ||
6855 Op->getOpcode() == ISD::STRICT_SINT_TO_FP);
6856 bool IsStrict = Op->isStrictFPOpcode();
6857 SDLoc DL(Op);
6858 MVT VT = Op.getSimpleValueType();
6859 SDValue InOp = Op.getOperand(i: IsStrict ? 1 : 0);
6860 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
6861 EVT InVT = InOp.getValueType();
6862
6863 // Conversion to f16 is done via f32.
6864 if (VT == MVT::f16) {
6865 SmallVector<SDValue, 2> Results;
6866 LowerOperationWrapper(N: Op.getNode(), Results, DAG);
6867 return DAG.getMergeValues(Ops: Results, dl: DL);
6868 }
6869
6870 // Unsigned to fp is not directly supported on z10.
6871 if (!Subtarget.hasFPExtension() && !IsSigned)
6872 return SDValue(); // Expand i64.
6873
6874 if (InVT == MVT::i128) {
6875 RTLIB::Libcall LC =
6876 IsSigned ? RTLIB::getSINTTOFP(OpVT: InVT, RetVT: VT) : RTLIB::getUINTTOFP(OpVT: InVT, RetVT: VT);
6877 return useLibCall(DAG, LC, VT, Arg: InOp, DL, Chain, IsStrict);
6878 }
6879
6880 return Op; // Legal
6881}
6882
6883// Shift the lower 2 bytes of Op to the left in order to insert into the
6884// upper 2 bytes of the FP register.
6885static SDValue convertToF16(SDValue Op, SelectionDAG &DAG) {
6886 assert(Op.getSimpleValueType() == MVT::i64 &&
6887 "Expexted to convert i64 to f16.");
6888 SDLoc DL(Op);
6889 SDValue Shft = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: Op,
6890 N2: DAG.getConstant(Val: 48, DL, VT: MVT::i64));
6891 SDValue BCast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Shft);
6892 SDValue F16Val =
6893 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h16, DL, VT: MVT::f16, Operand: BCast);
6894 return F16Val;
6895}
6896
6897// Extract Op into GPR and shift the 2 f16 bytes to the right.
6898static SDValue convertFromF16(SDValue Op, SDLoc DL, SelectionDAG &DAG) {
6899 assert(Op.getSimpleValueType() == MVT::f16 &&
6900 "Expected to convert f16 to i64.");
6901 SDNode *U32 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::f64);
6902 SDValue In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h16, DL, VT: MVT::f64,
6903 Operand: SDValue(U32, 0), Subreg: Op);
6904 SDValue BCast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: In64);
6905 SDValue Shft = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: BCast,
6906 N2: DAG.getConstant(Val: 48, DL, VT: MVT::i32));
6907 return Shft;
6908}
6909
6910// Lower an f16 LOAD in case of no vector support.
6911SDValue SystemZTargetLowering::lowerLoadF16(SDValue Op,
6912 SelectionDAG &DAG) const {
6913 EVT RegVT = Op.getValueType();
6914 assert(RegVT == MVT::f16 && "Expected to lower an f16 load.");
6915 (void)RegVT;
6916
6917 // Load as integer.
6918 SDLoc DL(Op);
6919 SDValue NewLd;
6920 if (auto *AtomicLd = dyn_cast<AtomicSDNode>(Val: Op.getNode())) {
6921 assert(EVT(RegVT) == AtomicLd->getMemoryVT() && "Unhandled f16 load");
6922 NewLd = DAG.getAtomicLoad(ExtType: ISD::EXTLOAD, dl: DL, MemVT: MVT::i16, VT: MVT::i64,
6923 Chain: AtomicLd->getChain(), Ptr: AtomicLd->getBasePtr(),
6924 MMO: AtomicLd->getMemOperand());
6925 } else {
6926 LoadSDNode *Ld = cast<LoadSDNode>(Val: Op.getNode());
6927 assert(EVT(RegVT) == Ld->getMemoryVT() && "Unhandled f16 load");
6928 NewLd = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: MVT::i64, Chain: Ld->getChain(),
6929 Ptr: Ld->getBasePtr(), PtrInfo: Ld->getPointerInfo(), MemVT: MVT::i16,
6930 Alignment: Ld->getBaseAlign(), MMOFlags: Ld->getMemOperand()->getFlags());
6931 }
6932 SDValue F16Val = convertToF16(Op: NewLd, DAG);
6933 return DAG.getMergeValues(Ops: {F16Val, NewLd.getValue(R: 1)}, dl: DL);
6934}
6935
6936// Lower an f16 STORE in case of no vector support.
6937SDValue SystemZTargetLowering::lowerStoreF16(SDValue Op,
6938 SelectionDAG &DAG) const {
6939 SDLoc DL(Op);
6940 SDValue Shft = convertFromF16(Op: Op->getOperand(Num: 1), DL, DAG);
6941
6942 if (auto *AtomicSt = dyn_cast<AtomicSDNode>(Val: Op.getNode()))
6943 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MVT::i16, Chain: AtomicSt->getChain(),
6944 Ptr: Shft, Val: AtomicSt->getBasePtr(),
6945 MMO: AtomicSt->getMemOperand());
6946
6947 StoreSDNode *St = cast<StoreSDNode>(Val: Op.getNode());
6948 return DAG.getTruncStore(Chain: St->getChain(), dl: DL, Val: Shft, Ptr: St->getBasePtr(), SVT: MVT::i16,
6949 MMO: St->getMemOperand());
6950}
6951
6952SDValue SystemZTargetLowering::lowerIS_FPCLASS(SDValue Op,
6953 SelectionDAG &DAG) const {
6954 SDLoc DL(Op);
6955 MVT ResultVT = Op.getSimpleValueType();
6956 SDValue Arg = Op.getOperand(i: 0);
6957 unsigned Check = Op.getConstantOperandVal(i: 1);
6958
6959 unsigned TDCMask = 0;
6960 if (Check & fcSNan)
6961 TDCMask |= SystemZ::TDCMASK_SNAN_PLUS | SystemZ::TDCMASK_SNAN_MINUS;
6962 if (Check & fcQNan)
6963 TDCMask |= SystemZ::TDCMASK_QNAN_PLUS | SystemZ::TDCMASK_QNAN_MINUS;
6964 if (Check & fcPosInf)
6965 TDCMask |= SystemZ::TDCMASK_INFINITY_PLUS;
6966 if (Check & fcNegInf)
6967 TDCMask |= SystemZ::TDCMASK_INFINITY_MINUS;
6968 if (Check & fcPosNormal)
6969 TDCMask |= SystemZ::TDCMASK_NORMAL_PLUS;
6970 if (Check & fcNegNormal)
6971 TDCMask |= SystemZ::TDCMASK_NORMAL_MINUS;
6972 if (Check & fcPosSubnormal)
6973 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_PLUS;
6974 if (Check & fcNegSubnormal)
6975 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_MINUS;
6976 if (Check & fcPosZero)
6977 TDCMask |= SystemZ::TDCMASK_ZERO_PLUS;
6978 if (Check & fcNegZero)
6979 TDCMask |= SystemZ::TDCMASK_ZERO_MINUS;
6980 SDValue TDCMaskV = DAG.getConstant(Val: TDCMask, DL, VT: MVT::i64);
6981
6982 if (Arg.getSimpleValueType() == MVT::f16)
6983 Arg = DAG.getFPExtendOrRound(Op: Arg, DL: SDLoc(Arg), VT: MVT::f32);
6984 SDValue Intr = DAG.getNode(Opcode: SystemZISD::TDC, DL, VT: ResultVT, N1: Arg, N2: TDCMaskV);
6985 return getCCResult(DAG, CCReg: Intr);
6986}
6987
6988SDValue SystemZTargetLowering::lowerREADCYCLECOUNTER(SDValue Op,
6989 SelectionDAG &DAG) const {
6990 SDLoc DL(Op);
6991 SDValue Chain = Op.getOperand(i: 0);
6992
6993 // STCKF only supports a memory operand, so we have to use a temporary.
6994 SDValue StackPtr = DAG.CreateStackTemporary(VT: MVT::i64);
6995 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
6996 MachinePointerInfo MPI =
6997 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
6998
6999 // Use STCFK to store the TOD clock into the temporary.
7000 SDValue StoreOps[] = {Chain, StackPtr};
7001 Chain = DAG.getMemIntrinsicNode(
7002 Opcode: SystemZISD::STCKF, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops: StoreOps, MemVT: MVT::i64,
7003 PtrInfo: MPI, Alignment: MaybeAlign(), Flags: MachineMemOperand::MOStore);
7004
7005 // And read it back from there.
7006 return DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: StackPtr, PtrInfo: MPI);
7007}
7008
7009SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
7010 SelectionDAG &DAG) const {
7011 switch (Op.getOpcode()) {
7012 case ISD::FRAMEADDR:
7013 return lowerFRAMEADDR(Op, DAG);
7014 case ISD::RETURNADDR:
7015 return lowerRETURNADDR(Op, DAG);
7016 case ISD::BR_CC:
7017 return lowerBR_CC(Op, DAG);
7018 case ISD::SELECT_CC:
7019 return lowerSELECT_CC(Op, DAG);
7020 case ISD::SETCC:
7021 return lowerSETCC(Op, DAG);
7022 case ISD::STRICT_FSETCC:
7023 return lowerSTRICT_FSETCC(Op, DAG, IsSignaling: false);
7024 case ISD::STRICT_FSETCCS:
7025 return lowerSTRICT_FSETCC(Op, DAG, IsSignaling: true);
7026 case ISD::GlobalAddress:
7027 return lowerGlobalAddress(Node: cast<GlobalAddressSDNode>(Val&: Op), DAG);
7028 case ISD::GlobalTLSAddress:
7029 return lowerGlobalTLSAddress(Node: cast<GlobalAddressSDNode>(Val&: Op), DAG);
7030 case ISD::BlockAddress:
7031 return lowerBlockAddress(Node: cast<BlockAddressSDNode>(Val&: Op), DAG);
7032 case ISD::JumpTable:
7033 return lowerJumpTable(JT: cast<JumpTableSDNode>(Val&: Op), DAG);
7034 case ISD::ConstantPool:
7035 return lowerConstantPool(CP: cast<ConstantPoolSDNode>(Val&: Op), DAG);
7036 case ISD::BITCAST:
7037 return lowerBITCAST(Op, DAG);
7038 case ISD::VASTART:
7039 return lowerVASTART(Op, DAG);
7040 case ISD::VACOPY:
7041 return lowerVACOPY(Op, DAG);
7042 case ISD::DYNAMIC_STACKALLOC:
7043 return lowerDYNAMIC_STACKALLOC(Op, DAG);
7044 case ISD::GET_DYNAMIC_AREA_OFFSET:
7045 return lowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
7046 case ISD::MULHS:
7047 return lowerMULH(Op, DAG, Opcode: SystemZISD::SMUL_LOHI);
7048 case ISD::MULHU:
7049 return lowerMULH(Op, DAG, Opcode: SystemZISD::UMUL_LOHI);
7050 case ISD::SMUL_LOHI:
7051 return lowerSMUL_LOHI(Op, DAG);
7052 case ISD::UMUL_LOHI:
7053 return lowerUMUL_LOHI(Op, DAG);
7054 case ISD::SDIVREM:
7055 return lowerSDIVREM(Op, DAG);
7056 case ISD::UDIVREM:
7057 return lowerUDIVREM(Op, DAG);
7058 case ISD::SADDO:
7059 case ISD::SSUBO:
7060 case ISD::UADDO:
7061 case ISD::USUBO:
7062 return lowerXALUO(Op, DAG);
7063 case ISD::UADDO_CARRY:
7064 case ISD::USUBO_CARRY:
7065 return lowerUADDSUBO_CARRY(Op, DAG);
7066 case ISD::OR:
7067 return lowerOR(Op, DAG);
7068 case ISD::CTPOP:
7069 return lowerCTPOP(Op, DAG);
7070 case ISD::VECREDUCE_ADD:
7071 return lowerVECREDUCE_ADD(Op, DAG);
7072 case ISD::ATOMIC_FENCE:
7073 return lowerATOMIC_FENCE(Op, DAG);
7074 case ISD::ATOMIC_SWAP:
7075 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_SWAPW);
7076 case ISD::ATOMIC_STORE:
7077 return lowerATOMIC_STORE(Op, DAG);
7078 case ISD::ATOMIC_LOAD:
7079 return lowerATOMIC_LOAD(Op, DAG);
7080 case ISD::ATOMIC_LOAD_ADD:
7081 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_ADD);
7082 case ISD::ATOMIC_LOAD_SUB:
7083 return lowerATOMIC_LOAD_SUB(Op, DAG);
7084 case ISD::ATOMIC_LOAD_AND:
7085 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_AND);
7086 case ISD::ATOMIC_LOAD_OR:
7087 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_OR);
7088 case ISD::ATOMIC_LOAD_XOR:
7089 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_XOR);
7090 case ISD::ATOMIC_LOAD_NAND:
7091 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_NAND);
7092 case ISD::ATOMIC_LOAD_MIN:
7093 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_MIN);
7094 case ISD::ATOMIC_LOAD_MAX:
7095 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_MAX);
7096 case ISD::ATOMIC_LOAD_UMIN:
7097 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_UMIN);
7098 case ISD::ATOMIC_LOAD_UMAX:
7099 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_UMAX);
7100 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
7101 return lowerATOMIC_CMP_SWAP(Op, DAG);
7102 case ISD::STACKSAVE:
7103 return lowerSTACKSAVE(Op, DAG);
7104 case ISD::STACKRESTORE:
7105 return lowerSTACKRESTORE(Op, DAG);
7106 case ISD::PREFETCH:
7107 return lowerPREFETCH(Op, DAG);
7108 case ISD::INTRINSIC_W_CHAIN:
7109 return lowerINTRINSIC_W_CHAIN(Op, DAG);
7110 case ISD::INTRINSIC_WO_CHAIN:
7111 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
7112 case ISD::BUILD_VECTOR:
7113 return lowerBUILD_VECTOR(Op, DAG);
7114 case ISD::VECTOR_SHUFFLE:
7115 return lowerVECTOR_SHUFFLE(Op, DAG);
7116 case ISD::SCALAR_TO_VECTOR:
7117 return lowerSCALAR_TO_VECTOR(Op, DAG);
7118 case ISD::INSERT_VECTOR_ELT:
7119 return lowerINSERT_VECTOR_ELT(Op, DAG);
7120 case ISD::EXTRACT_VECTOR_ELT:
7121 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7122 case ISD::SIGN_EXTEND_VECTOR_INREG:
7123 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
7124 case ISD::ZERO_EXTEND_VECTOR_INREG:
7125 return lowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
7126 case ISD::SHL:
7127 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSHL_BY_SCALAR);
7128 case ISD::SRL:
7129 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSRL_BY_SCALAR);
7130 case ISD::SRA:
7131 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSRA_BY_SCALAR);
7132 case ISD::ADDRSPACECAST:
7133 return lowerAddrSpaceCast(Op, DAG);
7134 case ISD::ROTL:
7135 return lowerShift(Op, DAG, ByScalar: SystemZISD::VROTL_BY_SCALAR);
7136 case ISD::FSHL:
7137 return lowerFSHL(Op, DAG);
7138 case ISD::FSHR:
7139 return lowerFSHR(Op, DAG);
7140 case ISD::FP_EXTEND:
7141 case ISD::STRICT_FP_EXTEND:
7142 return lowerFP_EXTEND(Op, DAG);
7143 case ISD::FP_TO_UINT:
7144 case ISD::FP_TO_SINT:
7145 case ISD::STRICT_FP_TO_UINT:
7146 case ISD::STRICT_FP_TO_SINT:
7147 return lower_FP_TO_INT(Op, DAG);
7148 case ISD::UINT_TO_FP:
7149 case ISD::SINT_TO_FP:
7150 case ISD::STRICT_UINT_TO_FP:
7151 case ISD::STRICT_SINT_TO_FP:
7152 return lower_INT_TO_FP(Op, DAG);
7153 case ISD::LOAD:
7154 return lowerLoadF16(Op, DAG);
7155 case ISD::STORE:
7156 return lowerStoreF16(Op, DAG);
7157 case ISD::IS_FPCLASS:
7158 return lowerIS_FPCLASS(Op, DAG);
7159 case ISD::GET_ROUNDING:
7160 return lowerGET_ROUNDING(Op, DAG);
7161 case ISD::READCYCLECOUNTER:
7162 return lowerREADCYCLECOUNTER(Op, DAG);
7163 case ISD::EH_SJLJ_SETJMP:
7164 case ISD::EH_SJLJ_LONGJMP:
7165 // These operations are legal on our platform, but we cannot actually
7166 // set the operation action to Legal as common code would treat this
7167 // as equivalent to Expand. Instead, we keep the operation action to
7168 // Custom and just leave them unchanged here.
7169 return Op;
7170
7171 default:
7172 llvm_unreachable("Unexpected node to lower");
7173 }
7174}
7175
7176static SDValue expandBitCastI128ToF128(SelectionDAG &DAG, SDValue Src,
7177 const SDLoc &SL) {
7178 // If i128 is legal, just use a normal bitcast.
7179 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128))
7180 return DAG.getBitcast(VT: MVT::f128, V: Src);
7181
7182 // Otherwise, f128 must live in FP128, so do a partwise move.
7183 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
7184 &SystemZ::FP128BitRegClass);
7185
7186 SDValue Hi, Lo;
7187 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Src, DL: SL, LoVT: MVT::i64, HiVT: MVT::i64);
7188
7189 Hi = DAG.getBitcast(VT: MVT::f64, V: Hi);
7190 Lo = DAG.getBitcast(VT: MVT::f64, V: Lo);
7191
7192 SDNode *Pair = DAG.getMachineNode(
7193 Opcode: SystemZ::REG_SEQUENCE, dl: SL, VT: MVT::f128,
7194 Ops: {DAG.getTargetConstant(Val: SystemZ::FP128BitRegClassID, DL: SL, VT: MVT::i32), Lo,
7195 DAG.getTargetConstant(Val: SystemZ::subreg_l64, DL: SL, VT: MVT::i32), Hi,
7196 DAG.getTargetConstant(Val: SystemZ::subreg_h64, DL: SL, VT: MVT::i32)});
7197 return SDValue(Pair, 0);
7198}
7199
7200static SDValue expandBitCastF128ToI128(SelectionDAG &DAG, SDValue Src,
7201 const SDLoc &SL) {
7202 // If i128 is legal, just use a normal bitcast.
7203 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128))
7204 return DAG.getBitcast(VT: MVT::i128, V: Src);
7205
7206 // Otherwise, f128 must live in FP128, so do a partwise move.
7207 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
7208 &SystemZ::FP128BitRegClass);
7209
7210 SDValue LoFP =
7211 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_l64, DL: SL, VT: MVT::f64, Operand: Src);
7212 SDValue HiFP =
7213 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h64, DL: SL, VT: MVT::f64, Operand: Src);
7214 SDValue Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: LoFP);
7215 SDValue Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: HiFP);
7216
7217 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i128, N1: Lo, N2: Hi);
7218}
7219
7220// Lower operations with invalid operand or result types.
7221void
7222SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
7223 SmallVectorImpl<SDValue> &Results,
7224 SelectionDAG &DAG) const {
7225 switch (N->getOpcode()) {
7226 case ISD::ATOMIC_LOAD: {
7227 SDLoc DL(N);
7228 SDVTList Tys = DAG.getVTList(VT1: MVT::Untyped, VT2: MVT::Other);
7229 SDValue Ops[] = { N->getOperand(Num: 0), N->getOperand(Num: 1) };
7230 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7231 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_LOAD_128,
7232 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7233
7234 SDValue Lowered = lowerGR128ToI128(DAG, In: Res);
7235 if (N->getValueType(ResNo: 0) == MVT::f128)
7236 Lowered = expandBitCastI128ToF128(DAG, Src: Lowered, SL: DL);
7237 Results.push_back(Elt: Lowered);
7238 Results.push_back(Elt: Res.getValue(R: 1));
7239 break;
7240 }
7241 case ISD::ATOMIC_STORE: {
7242 SDLoc DL(N);
7243 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
7244 SDValue Val = N->getOperand(Num: 1);
7245 if (Val.getValueType() == MVT::f128)
7246 Val = expandBitCastF128ToI128(DAG, Src: Val, SL: DL);
7247 Val = lowerI128ToGR128(DAG, In: Val);
7248
7249 SDValue Ops[] = {N->getOperand(Num: 0), Val, N->getOperand(Num: 2)};
7250 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7251 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_STORE_128,
7252 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7253 // We have to enforce sequential consistency by performing a
7254 // serialization operation after the store.
7255 if (cast<AtomicSDNode>(Val: N)->getSuccessOrdering() ==
7256 AtomicOrdering::SequentiallyConsistent)
7257 Res = SDValue(DAG.getMachineNode(Opcode: SystemZ::Serialize, dl: DL,
7258 VT: MVT::Other, Op1: Res), 0);
7259 Results.push_back(Elt: Res);
7260 break;
7261 }
7262 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
7263 SDLoc DL(N);
7264 SDVTList Tys = DAG.getVTList(VT1: MVT::Untyped, VT2: MVT::i32, VT3: MVT::Other);
7265 SDValue Ops[] = { N->getOperand(Num: 0), N->getOperand(Num: 1),
7266 lowerI128ToGR128(DAG, In: N->getOperand(Num: 2)),
7267 lowerI128ToGR128(DAG, In: N->getOperand(Num: 3)) };
7268 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7269 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAP_128,
7270 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7271 SDValue Success = emitSETCC(DAG, DL, CCReg: Res.getValue(R: 1),
7272 CCValid: SystemZ::CCMASK_CS, CCMask: SystemZ::CCMASK_CS_EQ);
7273 Success = DAG.getZExtOrTrunc(Op: Success, DL, VT: N->getValueType(ResNo: 1));
7274 Results.push_back(Elt: lowerGR128ToI128(DAG, In: Res));
7275 Results.push_back(Elt: Success);
7276 Results.push_back(Elt: Res.getValue(R: 2));
7277 break;
7278 }
7279 case ISD::BITCAST: {
7280 if (useSoftFloat())
7281 return;
7282 SDLoc DL(N);
7283 SDValue Src = N->getOperand(Num: 0);
7284 EVT SrcVT = Src.getValueType();
7285 EVT ResVT = N->getValueType(ResNo: 0);
7286 if (ResVT == MVT::i128 && SrcVT == MVT::f128)
7287 Results.push_back(Elt: expandBitCastF128ToI128(DAG, Src, SL: DL));
7288 else if (SrcVT == MVT::i16 && ResVT == MVT::f16) {
7289 if (Subtarget.hasVector()) {
7290 SDValue In32 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Src);
7291 Results.push_back(Elt: SDValue(
7292 DAG.getMachineNode(Opcode: SystemZ::LEFR_16, dl: DL, VT: MVT::f16, Op1: In32), 0));
7293 } else {
7294 SDValue In64 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Src);
7295 Results.push_back(Elt: convertToF16(Op: In64, DAG));
7296 }
7297 } else if (SrcVT == MVT::f16 && ResVT == MVT::i16) {
7298 SDValue ExtractedVal =
7299 Subtarget.hasVector()
7300 ? SDValue(DAG.getMachineNode(Opcode: SystemZ::LFER_16, dl: DL, VT: MVT::i32, Op1: Src),
7301 0)
7302 : convertFromF16(Op: Src, DL, DAG);
7303 Results.push_back(Elt: DAG.getZExtOrTrunc(Op: ExtractedVal, DL, VT: ResVT));
7304 }
7305 break;
7306 }
7307 case ISD::UINT_TO_FP:
7308 case ISD::SINT_TO_FP:
7309 case ISD::STRICT_UINT_TO_FP:
7310 case ISD::STRICT_SINT_TO_FP: {
7311 if (useSoftFloat())
7312 return;
7313 bool IsStrict = N->isStrictFPOpcode();
7314 SDLoc DL(N);
7315 SDValue InOp = N->getOperand(Num: IsStrict ? 1 : 0);
7316 EVT ResVT = N->getValueType(ResNo: 0);
7317 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : DAG.getEntryNode();
7318 if (ResVT == MVT::f16) {
7319 if (!IsStrict) {
7320 SDValue OpF32 = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::f32, Operand: InOp);
7321 Results.push_back(Elt: DAG.getFPExtendOrRound(Op: OpF32, DL, VT: MVT::f16));
7322 } else {
7323 SDValue OpF32 =
7324 DAG.getNode(Opcode: N->getOpcode(), DL, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
7325 Ops: {Chain, InOp});
7326 SDValue F16Res;
7327 std::tie(args&: F16Res, args&: Chain) = DAG.getStrictFPExtendOrRound(
7328 Op: OpF32, Chain: OpF32.getValue(R: 1), DL, VT: MVT::f16);
7329 Results.push_back(Elt: F16Res);
7330 Results.push_back(Elt: Chain);
7331 }
7332 }
7333 break;
7334 }
7335 case ISD::FP_TO_UINT:
7336 case ISD::FP_TO_SINT:
7337 case ISD::STRICT_FP_TO_UINT:
7338 case ISD::STRICT_FP_TO_SINT: {
7339 if (useSoftFloat())
7340 return;
7341 bool IsStrict = N->isStrictFPOpcode();
7342 SDLoc DL(N);
7343 EVT ResVT = N->getValueType(ResNo: 0);
7344 SDValue InOp = N->getOperand(Num: IsStrict ? 1 : 0);
7345 EVT InVT = InOp->getValueType(ResNo: 0);
7346 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : DAG.getEntryNode();
7347 if (InVT == MVT::f16) {
7348 if (!IsStrict) {
7349 SDValue InF32 = DAG.getFPExtendOrRound(Op: InOp, DL, VT: MVT::f32);
7350 Results.push_back(Elt: DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: InF32));
7351 } else {
7352 SDValue InF32;
7353 std::tie(args&: InF32, args&: Chain) =
7354 DAG.getStrictFPExtendOrRound(Op: InOp, Chain, DL, VT: MVT::f32);
7355 SDValue OpF32 =
7356 DAG.getNode(Opcode: N->getOpcode(), DL, VTList: DAG.getVTList(VT1: ResVT, VT2: MVT::Other),
7357 Ops: {Chain, InF32});
7358 Results.push_back(Elt: OpF32);
7359 Results.push_back(Elt: OpF32.getValue(R: 1));
7360 }
7361 }
7362 break;
7363 }
7364 default:
7365 llvm_unreachable("Unexpected node to lower");
7366 }
7367}
7368
7369void
7370SystemZTargetLowering::ReplaceNodeResults(SDNode *N,
7371 SmallVectorImpl<SDValue> &Results,
7372 SelectionDAG &DAG) const {
7373 return LowerOperationWrapper(N, Results, DAG);
7374}
7375
7376const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const {
7377#define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME
7378 switch ((SystemZISD::NodeType)Opcode) {
7379 case SystemZISD::FIRST_NUMBER: break;
7380 OPCODE(RET_GLUE);
7381 OPCODE(CALL);
7382 OPCODE(SIBCALL);
7383 OPCODE(TLS_GDCALL);
7384 OPCODE(TLS_LDCALL);
7385 OPCODE(PCREL_WRAPPER);
7386 OPCODE(PCREL_OFFSET);
7387 OPCODE(ICMP);
7388 OPCODE(FCMP);
7389 OPCODE(STRICT_FCMP);
7390 OPCODE(STRICT_FCMPS);
7391 OPCODE(TM);
7392 OPCODE(BR_CCMASK);
7393 OPCODE(SELECT_CCMASK);
7394 OPCODE(ADJDYNALLOC);
7395 OPCODE(PROBED_ALLOCA);
7396 OPCODE(POPCNT);
7397 OPCODE(SMUL_LOHI);
7398 OPCODE(UMUL_LOHI);
7399 OPCODE(SDIVREM);
7400 OPCODE(UDIVREM);
7401 OPCODE(SADDO);
7402 OPCODE(SSUBO);
7403 OPCODE(UADDO);
7404 OPCODE(USUBO);
7405 OPCODE(ADDCARRY);
7406 OPCODE(SUBCARRY);
7407 OPCODE(GET_CCMASK);
7408 OPCODE(MVC);
7409 OPCODE(NC);
7410 OPCODE(OC);
7411 OPCODE(XC);
7412 OPCODE(CLC);
7413 OPCODE(MEMSET_MVC);
7414 OPCODE(STPCPY);
7415 OPCODE(STRCMP);
7416 OPCODE(SEARCH_STRING);
7417 OPCODE(IPM);
7418 OPCODE(TBEGIN);
7419 OPCODE(TBEGIN_NOFLOAT);
7420 OPCODE(TEND);
7421 OPCODE(BYTE_MASK);
7422 OPCODE(ROTATE_MASK);
7423 OPCODE(REPLICATE);
7424 OPCODE(JOIN_DWORDS);
7425 OPCODE(SPLAT);
7426 OPCODE(MERGE_HIGH);
7427 OPCODE(MERGE_LOW);
7428 OPCODE(SHL_DOUBLE);
7429 OPCODE(PERMUTE_DWORDS);
7430 OPCODE(PERMUTE);
7431 OPCODE(PACK);
7432 OPCODE(PACKS_CC);
7433 OPCODE(PACKLS_CC);
7434 OPCODE(UNPACK_HIGH);
7435 OPCODE(UNPACKL_HIGH);
7436 OPCODE(UNPACK_LOW);
7437 OPCODE(UNPACKL_LOW);
7438 OPCODE(VSHL_BY_SCALAR);
7439 OPCODE(VSRL_BY_SCALAR);
7440 OPCODE(VSRA_BY_SCALAR);
7441 OPCODE(VROTL_BY_SCALAR);
7442 OPCODE(SHL_DOUBLE_BIT);
7443 OPCODE(SHR_DOUBLE_BIT);
7444 OPCODE(VSUM);
7445 OPCODE(VACC);
7446 OPCODE(VSCBI);
7447 OPCODE(VAC);
7448 OPCODE(VSBI);
7449 OPCODE(VACCC);
7450 OPCODE(VSBCBI);
7451 OPCODE(VMAH);
7452 OPCODE(VMALH);
7453 OPCODE(VME);
7454 OPCODE(VMLE);
7455 OPCODE(VMO);
7456 OPCODE(VMLO);
7457 OPCODE(VICMPE);
7458 OPCODE(VICMPH);
7459 OPCODE(VICMPHL);
7460 OPCODE(VICMPES);
7461 OPCODE(VICMPHS);
7462 OPCODE(VICMPHLS);
7463 OPCODE(VFCMPE);
7464 OPCODE(STRICT_VFCMPE);
7465 OPCODE(STRICT_VFCMPES);
7466 OPCODE(VFCMPH);
7467 OPCODE(STRICT_VFCMPH);
7468 OPCODE(STRICT_VFCMPHS);
7469 OPCODE(VFCMPHE);
7470 OPCODE(STRICT_VFCMPHE);
7471 OPCODE(STRICT_VFCMPHES);
7472 OPCODE(VFCMPES);
7473 OPCODE(VFCMPHS);
7474 OPCODE(VFCMPHES);
7475 OPCODE(VFTCI);
7476 OPCODE(VEXTEND);
7477 OPCODE(STRICT_VEXTEND);
7478 OPCODE(VROUND);
7479 OPCODE(STRICT_VROUND);
7480 OPCODE(VTM);
7481 OPCODE(SCMP128HI);
7482 OPCODE(UCMP128HI);
7483 OPCODE(VFAE_CC);
7484 OPCODE(VFAEZ_CC);
7485 OPCODE(VFEE_CC);
7486 OPCODE(VFEEZ_CC);
7487 OPCODE(VFENE_CC);
7488 OPCODE(VFENEZ_CC);
7489 OPCODE(VISTR_CC);
7490 OPCODE(VSTRC_CC);
7491 OPCODE(VSTRCZ_CC);
7492 OPCODE(VSTRS_CC);
7493 OPCODE(VSTRSZ_CC);
7494 OPCODE(TDC);
7495 OPCODE(ATOMIC_SWAPW);
7496 OPCODE(ATOMIC_LOADW_ADD);
7497 OPCODE(ATOMIC_LOADW_SUB);
7498 OPCODE(ATOMIC_LOADW_AND);
7499 OPCODE(ATOMIC_LOADW_OR);
7500 OPCODE(ATOMIC_LOADW_XOR);
7501 OPCODE(ATOMIC_LOADW_NAND);
7502 OPCODE(ATOMIC_LOADW_MIN);
7503 OPCODE(ATOMIC_LOADW_MAX);
7504 OPCODE(ATOMIC_LOADW_UMIN);
7505 OPCODE(ATOMIC_LOADW_UMAX);
7506 OPCODE(ATOMIC_CMP_SWAPW);
7507 OPCODE(ATOMIC_CMP_SWAP);
7508 OPCODE(ATOMIC_LOAD_128);
7509 OPCODE(ATOMIC_STORE_128);
7510 OPCODE(ATOMIC_CMP_SWAP_128);
7511 OPCODE(LRV);
7512 OPCODE(STRV);
7513 OPCODE(VLER);
7514 OPCODE(VSTER);
7515 OPCODE(STCKF);
7516 OPCODE(PREFETCH);
7517 OPCODE(ADA_ENTRY);
7518 }
7519 return nullptr;
7520#undef OPCODE
7521}
7522
7523// Return true if VT is a vector whose elements are a whole number of bytes
7524// in width. Also check for presence of vector support.
7525bool SystemZTargetLowering::canTreatAsByteVector(EVT VT) const {
7526 if (!Subtarget.hasVector())
7527 return false;
7528
7529 return VT.isVector() && VT.getScalarSizeInBits() % 8 == 0 && VT.isSimple();
7530}
7531
7532// Try to simplify an EXTRACT_VECTOR_ELT from a vector of type VecVT
7533// producing a result of type ResVT. Op is a possibly bitcast version
7534// of the input vector and Index is the index (based on type VecVT) that
7535// should be extracted. Return the new extraction if a simplification
7536// was possible or if Force is true.
7537SDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT,
7538 EVT VecVT, SDValue Op,
7539 unsigned Index,
7540 DAGCombinerInfo &DCI,
7541 bool Force) const {
7542 SelectionDAG &DAG = DCI.DAG;
7543
7544 // The number of bytes being extracted.
7545 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7546
7547 for (;;) {
7548 unsigned Opcode = Op.getOpcode();
7549 if (Opcode == ISD::BITCAST)
7550 // Look through bitcasts.
7551 Op = Op.getOperand(i: 0);
7552 else if ((Opcode == ISD::VECTOR_SHUFFLE || Opcode == SystemZISD::SPLAT) &&
7553 canTreatAsByteVector(VT: Op.getValueType())) {
7554 // Get a VPERM-like permute mask and see whether the bytes covered
7555 // by the extracted element are a contiguous sequence from one
7556 // source operand.
7557 SmallVector<int, SystemZ::VectorBytes> Bytes;
7558 if (!getVPermMask(ShuffleOp: Op, Bytes))
7559 break;
7560 int First;
7561 if (!getShuffleInput(Bytes, Start: Index * BytesPerElement,
7562 BytesPerElement, Base&: First))
7563 break;
7564 if (First < 0)
7565 return DAG.getUNDEF(VT: ResVT);
7566 // Make sure the contiguous sequence starts at a multiple of the
7567 // original element size.
7568 unsigned Byte = unsigned(First) % Bytes.size();
7569 if (Byte % BytesPerElement != 0)
7570 break;
7571 // We can get the extracted value directly from an input.
7572 Index = Byte / BytesPerElement;
7573 Op = Op.getOperand(i: unsigned(First) / Bytes.size());
7574 Force = true;
7575 } else if (Opcode == ISD::BUILD_VECTOR &&
7576 canTreatAsByteVector(VT: Op.getValueType())) {
7577 // We can only optimize this case if the BUILD_VECTOR elements are
7578 // at least as wide as the extracted value.
7579 EVT OpVT = Op.getValueType();
7580 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7581 if (OpBytesPerElement < BytesPerElement)
7582 break;
7583 // Make sure that the least-significant bit of the extracted value
7584 // is the least significant bit of an input.
7585 unsigned End = (Index + 1) * BytesPerElement;
7586 if (End % OpBytesPerElement != 0)
7587 break;
7588 // We're extracting the low part of one operand of the BUILD_VECTOR.
7589 Op = Op.getOperand(i: End / OpBytesPerElement - 1);
7590 if (!Op.getValueType().isInteger()) {
7591 EVT VT = MVT::getIntegerVT(BitWidth: Op.getValueSizeInBits());
7592 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
7593 DCI.AddToWorklist(N: Op.getNode());
7594 }
7595 EVT VT = MVT::getIntegerVT(BitWidth: ResVT.getSizeInBits());
7596 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
7597 if (VT != ResVT) {
7598 DCI.AddToWorklist(N: Op.getNode());
7599 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ResVT, Operand: Op);
7600 }
7601 return Op;
7602 } else if ((Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
7603 Opcode == ISD::ZERO_EXTEND_VECTOR_INREG ||
7604 Opcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
7605 canTreatAsByteVector(VT: Op.getValueType()) &&
7606 canTreatAsByteVector(VT: Op.getOperand(i: 0).getValueType())) {
7607 // Make sure that only the unextended bits are significant.
7608 EVT ExtVT = Op.getValueType();
7609 EVT OpVT = Op.getOperand(i: 0).getValueType();
7610 unsigned ExtBytesPerElement = ExtVT.getVectorElementType().getStoreSize();
7611 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7612 unsigned Byte = Index * BytesPerElement;
7613 unsigned SubByte = Byte % ExtBytesPerElement;
7614 unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;
7615 if (SubByte < MinSubByte ||
7616 SubByte + BytesPerElement > ExtBytesPerElement)
7617 break;
7618 // Get the byte offset of the unextended element
7619 Byte = Byte / ExtBytesPerElement * OpBytesPerElement;
7620 // ...then add the byte offset relative to that element.
7621 Byte += SubByte - MinSubByte;
7622 if (Byte % BytesPerElement != 0)
7623 break;
7624 Op = Op.getOperand(i: 0);
7625 Index = Byte / BytesPerElement;
7626 Force = true;
7627 } else
7628 break;
7629 }
7630 if (Force) {
7631 if (Op.getValueType() != VecVT) {
7632 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VecVT, Operand: Op);
7633 DCI.AddToWorklist(N: Op.getNode());
7634 }
7635 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ResVT, N1: Op,
7636 N2: DAG.getConstant(Val: Index, DL, VT: MVT::i32));
7637 }
7638 return SDValue();
7639}
7640
7641// Optimize vector operations in scalar value Op on the basis that Op
7642// is truncated to TruncVT.
7643SDValue SystemZTargetLowering::combineTruncateExtract(
7644 const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const {
7645 // If we have (trunc (extract_vector_elt X, Y)), try to turn it into
7646 // (extract_vector_elt (bitcast X), Y'), where (bitcast X) has elements
7647 // of type TruncVT.
7648 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7649 TruncVT.getSizeInBits() % 8 == 0) {
7650 SDValue Vec = Op.getOperand(i: 0);
7651 EVT VecVT = Vec.getValueType();
7652 if (canTreatAsByteVector(VT: VecVT)) {
7653 if (auto *IndexN = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
7654 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7655 unsigned TruncBytes = TruncVT.getStoreSize();
7656 if (BytesPerElement % TruncBytes == 0) {
7657 // Calculate the value of Y' in the above description. We are
7658 // splitting the original elements into Scale equal-sized pieces
7659 // and for truncation purposes want the last (least-significant)
7660 // of these pieces for IndexN. This is easiest to do by calculating
7661 // the start index of the following element and then subtracting 1.
7662 unsigned Scale = BytesPerElement / TruncBytes;
7663 unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;
7664
7665 // Defer the creation of the bitcast from X to combineExtract,
7666 // which might be able to optimize the extraction.
7667 VecVT = EVT::getVectorVT(Context&: *DCI.DAG.getContext(),
7668 VT: MVT::getIntegerVT(BitWidth: TruncBytes * 8),
7669 NumElements: VecVT.getStoreSize() / TruncBytes);
7670 EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT);
7671 return combineExtract(DL, ResVT, VecVT, Op: Vec, Index: NewIndex, DCI, Force: true);
7672 }
7673 }
7674 }
7675 }
7676 return SDValue();
7677}
7678
7679SDValue SystemZTargetLowering::combineZERO_EXTEND(
7680 SDNode *N, DAGCombinerInfo &DCI) const {
7681 // Convert (zext (select_ccmask C1, C2)) into (select_ccmask C1', C2')
7682 SelectionDAG &DAG = DCI.DAG;
7683 SDValue N0 = N->getOperand(Num: 0);
7684 EVT VT = N->getValueType(ResNo: 0);
7685 if (N0.getOpcode() == SystemZISD::SELECT_CCMASK) {
7686 auto *TrueOp = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0));
7687 auto *FalseOp = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
7688 if (TrueOp && FalseOp) {
7689 SDLoc DL(N0);
7690 SDValue Ops[] = { DAG.getConstant(Val: TrueOp->getZExtValue(), DL, VT),
7691 DAG.getConstant(Val: FalseOp->getZExtValue(), DL, VT),
7692 N0.getOperand(i: 2), N0.getOperand(i: 3), N0.getOperand(i: 4) };
7693 SDValue NewSelect = DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT, Ops);
7694 // If N0 has multiple uses, change other uses as well.
7695 if (!N0.hasOneUse()) {
7696 SDValue TruncSelect =
7697 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N0.getValueType(), Operand: NewSelect);
7698 DCI.CombineTo(N: N0.getNode(), Res: TruncSelect);
7699 }
7700 return NewSelect;
7701 }
7702 }
7703 // Convert (zext (xor (trunc X), C)) into (xor (trunc X), C') if the size
7704 // of the result is smaller than the size of X and all the truncated bits
7705 // of X are already zero.
7706 if (N0.getOpcode() == ISD::XOR &&
7707 N0.hasOneUse() && N0.getOperand(i: 0).hasOneUse() &&
7708 N0.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
7709 N0.getOperand(i: 1).getOpcode() == ISD::Constant) {
7710 SDValue X = N0.getOperand(i: 0).getOperand(i: 0);
7711 if (VT.isScalarInteger() && VT.getSizeInBits() < X.getValueSizeInBits()) {
7712 KnownBits Known = DAG.computeKnownBits(Op: X);
7713 APInt TruncatedBits = APInt::getBitsSet(numBits: X.getValueSizeInBits(),
7714 loBit: N0.getValueSizeInBits(),
7715 hiBit: VT.getSizeInBits());
7716 if (TruncatedBits.isSubsetOf(RHS: Known.Zero)) {
7717 X = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(X), VT, Operand: X);
7718 APInt Mask = N0.getConstantOperandAPInt(i: 1).zext(width: VT.getSizeInBits());
7719 return DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N0), VT,
7720 N1: X, N2: DAG.getConstant(Val: Mask, DL: SDLoc(N0), VT));
7721 }
7722 }
7723 }
7724 // Recognize patterns for VECTOR SUBTRACT COMPUTE BORROW INDICATION
7725 // and VECTOR ADD COMPUTE CARRY for i128:
7726 // (zext (setcc_uge X Y)) --> (VSCBI X Y)
7727 // (zext (setcc_ule Y X)) --> (VSCBI X Y)
7728 // (zext (setcc_ult (add X Y) X/Y) -> (VACC X Y)
7729 // (zext (setcc_ugt X/Y (add X Y)) -> (VACC X Y)
7730 // For vector types, these patterns are recognized in the .td file.
7731 if (N0.getOpcode() == ISD::SETCC && isTypeLegal(VT) && VT == MVT::i128 &&
7732 N0.getOperand(i: 0).getValueType() == VT) {
7733 SDValue Op0 = N0.getOperand(i: 0);
7734 SDValue Op1 = N0.getOperand(i: 1);
7735 const ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
7736 switch (CC) {
7737 case ISD::SETULE:
7738 std::swap(a&: Op0, b&: Op1);
7739 [[fallthrough]];
7740 case ISD::SETUGE:
7741 return DAG.getNode(Opcode: SystemZISD::VSCBI, DL: SDLoc(N0), VT, N1: Op0, N2: Op1);
7742 case ISD::SETUGT:
7743 std::swap(a&: Op0, b&: Op1);
7744 [[fallthrough]];
7745 case ISD::SETULT:
7746 if (Op0->hasOneUse() && Op0->getOpcode() == ISD::ADD &&
7747 (Op0->getOperand(Num: 0) == Op1 || Op0->getOperand(Num: 1) == Op1))
7748 return DAG.getNode(Opcode: SystemZISD::VACC, DL: SDLoc(N0), VT, N1: Op0->getOperand(Num: 0),
7749 N2: Op0->getOperand(Num: 1));
7750 break;
7751 default:
7752 break;
7753 }
7754 }
7755
7756 return SDValue();
7757}
7758
7759SDValue SystemZTargetLowering::combineSIGN_EXTEND_INREG(
7760 SDNode *N, DAGCombinerInfo &DCI) const {
7761 // Convert (sext_in_reg (setcc LHS, RHS, COND), i1)
7762 // and (sext_in_reg (any_extend (setcc LHS, RHS, COND)), i1)
7763 // into (select_cc LHS, RHS, -1, 0, COND)
7764 SelectionDAG &DAG = DCI.DAG;
7765 SDValue N0 = N->getOperand(Num: 0);
7766 EVT VT = N->getValueType(ResNo: 0);
7767 EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
7768 if (N0.hasOneUse() && N0.getOpcode() == ISD::ANY_EXTEND)
7769 N0 = N0.getOperand(i: 0);
7770 if (EVT == MVT::i1 && N0.hasOneUse() && N0.getOpcode() == ISD::SETCC) {
7771 SDLoc DL(N0);
7772 SDValue Ops[] = { N0.getOperand(i: 0), N0.getOperand(i: 1),
7773 DAG.getAllOnesConstant(DL, VT),
7774 DAG.getConstant(Val: 0, DL, VT), N0.getOperand(i: 2) };
7775 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT, Ops);
7776 }
7777 return SDValue();
7778}
7779
7780SDValue SystemZTargetLowering::combineSIGN_EXTEND(
7781 SDNode *N, DAGCombinerInfo &DCI) const {
7782 // Convert (sext (ashr (shl X, C1), C2)) to
7783 // (ashr (shl (anyext X), C1'), C2')), since wider shifts are as
7784 // cheap as narrower ones.
7785 SelectionDAG &DAG = DCI.DAG;
7786 SDValue N0 = N->getOperand(Num: 0);
7787 EVT VT = N->getValueType(ResNo: 0);
7788 if (N0.hasOneUse() && N0.getOpcode() == ISD::SRA) {
7789 auto *SraAmt = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
7790 SDValue Inner = N0.getOperand(i: 0);
7791 if (SraAmt && Inner.hasOneUse() && Inner.getOpcode() == ISD::SHL) {
7792 if (auto *ShlAmt = dyn_cast<ConstantSDNode>(Val: Inner.getOperand(i: 1))) {
7793 unsigned Extra = (VT.getSizeInBits() - N0.getValueSizeInBits());
7794 unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra;
7795 unsigned NewSraAmt = SraAmt->getZExtValue() + Extra;
7796 EVT ShiftVT = N0.getOperand(i: 1).getValueType();
7797 SDValue Ext = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(Inner), VT,
7798 Operand: Inner.getOperand(i: 0));
7799 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(Inner), VT, N1: Ext,
7800 N2: DAG.getConstant(Val: NewShlAmt, DL: SDLoc(Inner),
7801 VT: ShiftVT));
7802 return DAG.getNode(Opcode: ISD::SRA, DL: SDLoc(N0), VT, N1: Shl,
7803 N2: DAG.getConstant(Val: NewSraAmt, DL: SDLoc(N0), VT: ShiftVT));
7804 }
7805 }
7806 }
7807
7808 return SDValue();
7809}
7810
7811SDValue SystemZTargetLowering::combineMERGE(
7812 SDNode *N, DAGCombinerInfo &DCI) const {
7813 SelectionDAG &DAG = DCI.DAG;
7814 unsigned Opcode = N->getOpcode();
7815 SDValue Op0 = N->getOperand(Num: 0);
7816 SDValue Op1 = N->getOperand(Num: 1);
7817 if (Op0.getOpcode() == ISD::BITCAST)
7818 Op0 = Op0.getOperand(i: 0);
7819 if (ISD::isBuildVectorAllZeros(N: Op0.getNode())) {
7820 // (z_merge_* 0, 0) -> 0. This is mostly useful for using VLLEZF
7821 // for v4f32.
7822 if (Op1 == N->getOperand(Num: 0))
7823 return Op1;
7824 // (z_merge_? 0, X) -> (z_unpackl_? 0, X).
7825 EVT VT = Op1.getValueType();
7826 unsigned ElemBytes = VT.getVectorElementType().getStoreSize();
7827 if (ElemBytes <= 4) {
7828 Opcode = (Opcode == SystemZISD::MERGE_HIGH ?
7829 SystemZISD::UNPACKL_HIGH : SystemZISD::UNPACKL_LOW);
7830 EVT InVT = VT.changeVectorElementTypeToInteger();
7831 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ElemBytes * 16),
7832 NumElements: SystemZ::VectorBytes / ElemBytes / 2);
7833 if (VT != InVT) {
7834 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: InVT, Operand: Op1);
7835 DCI.AddToWorklist(N: Op1.getNode());
7836 }
7837 SDValue Op = DAG.getNode(Opcode, DL: SDLoc(N), VT: OutVT, Operand: Op1);
7838 DCI.AddToWorklist(N: Op.getNode());
7839 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT, Operand: Op);
7840 }
7841 }
7842 return SDValue();
7843}
7844
7845static bool isI128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
7846 SDNode *&HiPart) {
7847 LoPart = HiPart = nullptr;
7848
7849 // Scan through all users.
7850 for (SDUse &Use : LD->uses()) {
7851 // Skip the uses of the chain.
7852 if (Use.getResNo() != 0)
7853 continue;
7854
7855 // Verify every user is a TRUNCATE to i64 of the low or high half.
7856 SDNode *User = Use.getUser();
7857 bool IsLoPart = true;
7858 if (User->getOpcode() == ISD::SRL &&
7859 User->getOperand(Num: 1).getOpcode() == ISD::Constant &&
7860 User->getConstantOperandVal(Num: 1) == 64 && User->hasOneUse()) {
7861 User = *User->user_begin();
7862 IsLoPart = false;
7863 }
7864 if (User->getOpcode() != ISD::TRUNCATE || User->getValueType(ResNo: 0) != MVT::i64)
7865 return false;
7866
7867 if (IsLoPart) {
7868 if (LoPart)
7869 return false;
7870 LoPart = User;
7871 } else {
7872 if (HiPart)
7873 return false;
7874 HiPart = User;
7875 }
7876 }
7877 return true;
7878}
7879
7880static bool isF128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
7881 SDNode *&HiPart) {
7882 LoPart = HiPart = nullptr;
7883
7884 // Scan through all users.
7885 for (SDUse &Use : LD->uses()) {
7886 // Skip the uses of the chain.
7887 if (Use.getResNo() != 0)
7888 continue;
7889
7890 // Verify every user is an EXTRACT_SUBREG of the low or high half.
7891 SDNode *User = Use.getUser();
7892 if (!User->hasOneUse() || !User->isMachineOpcode() ||
7893 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
7894 return false;
7895
7896 switch (User->getConstantOperandVal(Num: 1)) {
7897 case SystemZ::subreg_l64:
7898 if (LoPart)
7899 return false;
7900 LoPart = User;
7901 break;
7902 case SystemZ::subreg_h64:
7903 if (HiPart)
7904 return false;
7905 HiPart = User;
7906 break;
7907 default:
7908 return false;
7909 }
7910 }
7911 return true;
7912}
7913
7914SDValue SystemZTargetLowering::combineLOAD(
7915 SDNode *N, DAGCombinerInfo &DCI) const {
7916 SelectionDAG &DAG = DCI.DAG;
7917 EVT LdVT = N->getValueType(ResNo: 0);
7918 if (auto *LN = dyn_cast<LoadSDNode>(Val: N)) {
7919 if (LN->getAddressSpace() == SYSTEMZAS::PTR32) {
7920 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
7921 MVT LoadNodeVT = LN->getBasePtr().getSimpleValueType();
7922 if (PtrVT != LoadNodeVT) {
7923 SDLoc DL(LN);
7924 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(
7925 dl: DL, VT: PtrVT, Ptr: LN->getBasePtr(), SrcAS: SYSTEMZAS::PTR32, DestAS: 0);
7926 return DAG.getExtLoad(ExtType: LN->getExtensionType(), dl: DL, VT: LN->getValueType(ResNo: 0),
7927 Chain: LN->getChain(), Ptr: AddrSpaceCast, MemVT: LN->getMemoryVT(),
7928 MMO: LN->getMemOperand());
7929 }
7930 }
7931 }
7932 SDLoc DL(N);
7933
7934 // Replace a 128-bit load that is used solely to move its value into GPRs
7935 // by separate loads of both halves.
7936 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
7937 if (LD->isSimple() && ISD::isNormalLoad(N: LD)) {
7938 SDNode *LoPart, *HiPart;
7939 if ((LdVT == MVT::i128 && isI128MovedToParts(LD, LoPart, HiPart)) ||
7940 (LdVT == MVT::f128 && isF128MovedToParts(LD, LoPart, HiPart))) {
7941 // Rewrite each extraction as an independent load.
7942 SmallVector<SDValue, 2> ArgChains;
7943 if (HiPart) {
7944 SDValue EltLoad = DAG.getLoad(
7945 VT: HiPart->getValueType(ResNo: 0), dl: DL, Chain: LD->getChain(), Ptr: LD->getBasePtr(),
7946 PtrInfo: LD->getPointerInfo(), Alignment: LD->getBaseAlign(),
7947 MMOFlags: LD->getMemOperand()->getFlags(), AAInfo: LD->getAAInfo());
7948
7949 DCI.CombineTo(N: HiPart, Res: EltLoad, AddTo: true);
7950 ArgChains.push_back(Elt: EltLoad.getValue(R: 1));
7951 }
7952 if (LoPart) {
7953 SDValue EltLoad = DAG.getLoad(
7954 VT: LoPart->getValueType(ResNo: 0), dl: DL, Chain: LD->getChain(),
7955 Ptr: DAG.getObjectPtrOffset(SL: DL, Ptr: LD->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: 8)),
7956 PtrInfo: LD->getPointerInfo().getWithOffset(O: 8), Alignment: LD->getBaseAlign(),
7957 MMOFlags: LD->getMemOperand()->getFlags(), AAInfo: LD->getAAInfo());
7958
7959 DCI.CombineTo(N: LoPart, Res: EltLoad, AddTo: true);
7960 ArgChains.push_back(Elt: EltLoad.getValue(R: 1));
7961 }
7962
7963 // Collect all chains via TokenFactor.
7964 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ArgChains);
7965 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: Chain);
7966 DCI.AddToWorklist(N: Chain.getNode());
7967 return SDValue(N, 0);
7968 }
7969 }
7970
7971 if (LdVT.isVector() || LdVT.isInteger())
7972 return SDValue();
7973 // Transform a scalar load that is REPLICATEd as well as having other
7974 // use(s) to the form where the other use(s) use the first element of the
7975 // REPLICATE instead of the load. Otherwise instruction selection will not
7976 // produce a VLREP. Avoid extracting to a GPR, so only do this for floating
7977 // point loads.
7978
7979 SDValue Replicate;
7980 SmallVector<SDNode*, 8> OtherUses;
7981 for (SDUse &Use : N->uses()) {
7982 if (Use.getUser()->getOpcode() == SystemZISD::REPLICATE) {
7983 if (Replicate)
7984 return SDValue(); // Should never happen
7985 Replicate = SDValue(Use.getUser(), 0);
7986 } else if (Use.getResNo() == 0)
7987 OtherUses.push_back(Elt: Use.getUser());
7988 }
7989 if (!Replicate || OtherUses.empty())
7990 return SDValue();
7991
7992 SDValue Extract0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: LdVT,
7993 N1: Replicate, N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7994 // Update uses of the loaded Value while preserving old chains.
7995 for (SDNode *U : OtherUses) {
7996 SmallVector<SDValue, 8> Ops;
7997 for (SDValue Op : U->ops())
7998 Ops.push_back(Elt: (Op.getNode() == N && Op.getResNo() == 0) ? Extract0 : Op);
7999 DAG.UpdateNodeOperands(N: U, Ops);
8000 }
8001 return SDValue(N, 0);
8002}
8003
8004bool SystemZTargetLowering::canLoadStoreByteSwapped(EVT VT) const {
8005 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64)
8006 return true;
8007 if (Subtarget.hasVectorEnhancements2())
8008 if (VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v2i64 || VT == MVT::i128)
8009 return true;
8010 return false;
8011}
8012
8013static bool isVectorElementSwap(ArrayRef<int> M, EVT VT) {
8014 if (!VT.isVector() || !VT.isSimple() ||
8015 VT.getSizeInBits() != 128 ||
8016 VT.getScalarSizeInBits() % 8 != 0)
8017 return false;
8018
8019 unsigned NumElts = VT.getVectorNumElements();
8020 for (unsigned i = 0; i < NumElts; ++i) {
8021 if (M[i] < 0) continue; // ignore UNDEF indices
8022 if ((unsigned) M[i] != NumElts - 1 - i)
8023 return false;
8024 }
8025
8026 return true;
8027}
8028
8029static bool isOnlyUsedByStores(SDValue StoredVal, SelectionDAG &DAG) {
8030 for (auto *U : StoredVal->users()) {
8031 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: U)) {
8032 EVT CurrMemVT = ST->getMemoryVT().getScalarType();
8033 if (CurrMemVT.isRound() && CurrMemVT.getStoreSize() <= 16)
8034 continue;
8035 } else if (isa<BuildVectorSDNode>(Val: U)) {
8036 SDValue BuildVector = SDValue(U, 0);
8037 if (DAG.isSplatValue(V: BuildVector, AllowUndefs: true/*AllowUndefs*/) &&
8038 isOnlyUsedByStores(StoredVal: BuildVector, DAG))
8039 continue;
8040 }
8041 return false;
8042 }
8043 return true;
8044}
8045
8046static bool isI128MovedFromParts(SDValue Val, SDValue &LoPart,
8047 SDValue &HiPart) {
8048 if (Val.getOpcode() != ISD::OR || !Val.getNode()->hasOneUse())
8049 return false;
8050
8051 SDValue Op0 = Val.getOperand(i: 0);
8052 SDValue Op1 = Val.getOperand(i: 1);
8053
8054 if (Op0.getOpcode() == ISD::SHL)
8055 std::swap(a&: Op0, b&: Op1);
8056 if (Op1.getOpcode() != ISD::SHL || !Op1.getNode()->hasOneUse() ||
8057 Op1.getOperand(i: 1).getOpcode() != ISD::Constant ||
8058 Op1.getConstantOperandVal(i: 1) != 64)
8059 return false;
8060 Op1 = Op1.getOperand(i: 0);
8061
8062 if (Op0.getOpcode() != ISD::ZERO_EXTEND || !Op0.getNode()->hasOneUse() ||
8063 Op0.getOperand(i: 0).getValueType() != MVT::i64)
8064 return false;
8065 if (Op1.getOpcode() != ISD::ANY_EXTEND || !Op1.getNode()->hasOneUse() ||
8066 Op1.getOperand(i: 0).getValueType() != MVT::i64)
8067 return false;
8068
8069 LoPart = Op0.getOperand(i: 0);
8070 HiPart = Op1.getOperand(i: 0);
8071 return true;
8072}
8073
8074static bool isF128MovedFromParts(SDValue Val, SDValue &LoPart,
8075 SDValue &HiPart) {
8076 if (!Val.getNode()->hasOneUse() || !Val.isMachineOpcode() ||
8077 Val.getMachineOpcode() != TargetOpcode::REG_SEQUENCE)
8078 return false;
8079
8080 if (Val->getNumOperands() != 5 ||
8081 Val->getOperand(Num: 0)->getAsZExtVal() != SystemZ::FP128BitRegClassID ||
8082 Val->getOperand(Num: 2)->getAsZExtVal() != SystemZ::subreg_l64 ||
8083 Val->getOperand(Num: 4)->getAsZExtVal() != SystemZ::subreg_h64)
8084 return false;
8085
8086 LoPart = Val->getOperand(Num: 1);
8087 HiPart = Val->getOperand(Num: 3);
8088 return true;
8089}
8090
8091SDValue SystemZTargetLowering::combineSTORE(
8092 SDNode *N, DAGCombinerInfo &DCI) const {
8093 SelectionDAG &DAG = DCI.DAG;
8094 auto *SN = cast<StoreSDNode>(Val: N);
8095 auto &Op1 = N->getOperand(Num: 1);
8096 EVT MemVT = SN->getMemoryVT();
8097
8098 if (SN->getAddressSpace() == SYSTEMZAS::PTR32) {
8099 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8100 MVT StoreNodeVT = SN->getBasePtr().getSimpleValueType();
8101 if (PtrVT != StoreNodeVT) {
8102 SDLoc DL(SN);
8103 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(dl: DL, VT: PtrVT, Ptr: SN->getBasePtr(),
8104 SrcAS: SYSTEMZAS::PTR32, DestAS: 0);
8105 return DAG.getStore(Chain: SN->getChain(), dl: DL, Val: SN->getValue(), Ptr: AddrSpaceCast,
8106 PtrInfo: SN->getPointerInfo(), Alignment: SN->getBaseAlign(),
8107 MMOFlags: SN->getMemOperand()->getFlags(), AAInfo: SN->getAAInfo());
8108 }
8109 }
8110
8111 // If we have (truncstoreiN (extract_vector_elt X, Y), Z) then it is better
8112 // for the extraction to be done on a vMiN value, so that we can use VSTE.
8113 // If X has wider elements then convert it to:
8114 // (truncstoreiN (extract_vector_elt (bitcast X), Y2), Z).
8115 if (MemVT.isInteger() && SN->isTruncatingStore()) {
8116 if (SDValue Value =
8117 combineTruncateExtract(DL: SDLoc(N), TruncVT: MemVT, Op: SN->getValue(), DCI)) {
8118 DCI.AddToWorklist(N: Value.getNode());
8119
8120 // Rewrite the store with the new form of stored value.
8121 return DAG.getTruncStore(Chain: SN->getChain(), dl: SDLoc(SN), Val: Value,
8122 Ptr: SN->getBasePtr(), SVT: SN->getMemoryVT(),
8123 MMO: SN->getMemOperand());
8124 }
8125 }
8126 // Combine STORE (BSWAP) into STRVH/STRV/STRVG/VSTBR
8127 if (!SN->isTruncatingStore() &&
8128 Op1.getOpcode() == ISD::BSWAP &&
8129 Op1.getNode()->hasOneUse() &&
8130 canLoadStoreByteSwapped(VT: Op1.getValueType())) {
8131
8132 SDValue BSwapOp = Op1.getOperand(i: 0);
8133
8134 if (BSwapOp.getValueType() == MVT::i16)
8135 BSwapOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: MVT::i32, Operand: BSwapOp);
8136
8137 SDValue Ops[] = {
8138 N->getOperand(Num: 0), BSwapOp, N->getOperand(Num: 2)
8139 };
8140
8141 return
8142 DAG.getMemIntrinsicNode(Opcode: SystemZISD::STRV, dl: SDLoc(N), VTList: DAG.getVTList(VT: MVT::Other),
8143 Ops, MemVT, MMO: SN->getMemOperand());
8144 }
8145 // Combine STORE (element-swap) into VSTER
8146 if (!SN->isTruncatingStore() &&
8147 Op1.getOpcode() == ISD::VECTOR_SHUFFLE &&
8148 Op1.getNode()->hasOneUse() &&
8149 Subtarget.hasVectorEnhancements2()) {
8150 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op1.getNode());
8151 ArrayRef<int> ShuffleMask = SVN->getMask();
8152 if (isVectorElementSwap(M: ShuffleMask, VT: Op1.getValueType())) {
8153 SDValue Ops[] = {
8154 N->getOperand(Num: 0), Op1.getOperand(i: 0), N->getOperand(Num: 2)
8155 };
8156
8157 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::VSTER, dl: SDLoc(N),
8158 VTList: DAG.getVTList(VT: MVT::Other),
8159 Ops, MemVT, MMO: SN->getMemOperand());
8160 }
8161 }
8162
8163 // Combine STORE (READCYCLECOUNTER) into STCKF.
8164 if (!SN->isTruncatingStore() &&
8165 Op1.getOpcode() == ISD::READCYCLECOUNTER &&
8166 Op1.hasOneUse() &&
8167 N->getOperand(Num: 0).reachesChainWithoutSideEffects(Dest: SDValue(Op1.getNode(), 1))) {
8168 SDValue Ops[] = { Op1.getOperand(i: 0), N->getOperand(Num: 2) };
8169 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::STCKF, dl: SDLoc(N),
8170 VTList: DAG.getVTList(VT: MVT::Other),
8171 Ops, MemVT, MMO: SN->getMemOperand());
8172 }
8173
8174 // Transform a store of a 128-bit value moved from parts into two stores.
8175 if (SN->isSimple() && ISD::isNormalStore(N: SN)) {
8176 SDValue LoPart, HiPart;
8177 if ((MemVT == MVT::i128 && isI128MovedFromParts(Val: Op1, LoPart, HiPart)) ||
8178 (MemVT == MVT::f128 && isF128MovedFromParts(Val: Op1, LoPart, HiPart))) {
8179 SDLoc DL(SN);
8180 SDValue Chain0 = DAG.getStore(
8181 Chain: SN->getChain(), dl: DL, Val: HiPart, Ptr: SN->getBasePtr(), PtrInfo: SN->getPointerInfo(),
8182 Alignment: SN->getBaseAlign(), MMOFlags: SN->getMemOperand()->getFlags(), AAInfo: SN->getAAInfo());
8183 SDValue Chain1 = DAG.getStore(
8184 Chain: SN->getChain(), dl: DL, Val: LoPart,
8185 Ptr: DAG.getObjectPtrOffset(SL: DL, Ptr: SN->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: 8)),
8186 PtrInfo: SN->getPointerInfo().getWithOffset(O: 8), Alignment: SN->getBaseAlign(),
8187 MMOFlags: SN->getMemOperand()->getFlags(), AAInfo: SN->getAAInfo());
8188
8189 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Chain0, N2: Chain1);
8190 }
8191 }
8192
8193 // Replicate a reg or immediate with VREP instead of scalar multiply or
8194 // immediate load. It seems best to do this during the first DAGCombine as
8195 // it is straight-forward to handle the zero-extend node in the initial
8196 // DAG, and also not worry about the keeping the new MemVT legal (e.g. when
8197 // extracting an i16 element from a v16i8 vector).
8198 if (Subtarget.hasVector() && DCI.Level == BeforeLegalizeTypes &&
8199 isOnlyUsedByStores(StoredVal: Op1, DAG)) {
8200 SDValue Word = SDValue();
8201 EVT WordVT;
8202
8203 // Find a replicated immediate and return it if found in Word and its
8204 // type in WordVT.
8205 auto FindReplicatedImm = [&](ConstantSDNode *C, unsigned TotBytes) {
8206 // Some constants are better handled with a scalar store.
8207 if (C->getAPIntValue().getBitWidth() > 64 || C->isAllOnes() ||
8208 isInt<16>(x: C->getSExtValue()) || MemVT.getStoreSize() <= 2)
8209 return;
8210
8211 APInt Val = C->getAPIntValue();
8212 // Truncate Val in case of a truncating store.
8213 if (!llvm::isUIntN(N: TotBytes * 8, x: Val.getZExtValue())) {
8214 assert(SN->isTruncatingStore() &&
8215 "Non-truncating store and immediate value does not fit?");
8216 Val = Val.trunc(width: TotBytes * 8);
8217 }
8218
8219 SystemZVectorConstantInfo VCI(APInt(TotBytes * 8, Val.getZExtValue()));
8220 if (VCI.isVectorConstantLegal(Subtarget) &&
8221 VCI.Opcode == SystemZISD::REPLICATE) {
8222 Word = DAG.getConstant(Val: VCI.OpVals[0], DL: SDLoc(SN), VT: MVT::i32);
8223 WordVT = VCI.VecVT.getScalarType();
8224 }
8225 };
8226
8227 // Find a replicated register and return it if found in Word and its type
8228 // in WordVT.
8229 auto FindReplicatedReg = [&](SDValue MulOp) {
8230 EVT MulVT = MulOp.getValueType();
8231 if (MulOp->getOpcode() == ISD::MUL &&
8232 (MulVT == MVT::i16 || MulVT == MVT::i32 || MulVT == MVT::i64)) {
8233 // Find a zero extended value and its type.
8234 SDValue LHS = MulOp->getOperand(Num: 0);
8235 if (LHS->getOpcode() == ISD::ZERO_EXTEND)
8236 WordVT = LHS->getOperand(Num: 0).getValueType();
8237 else if (LHS->getOpcode() == ISD::AssertZext)
8238 WordVT = cast<VTSDNode>(Val: LHS->getOperand(Num: 1))->getVT();
8239 else
8240 return;
8241 // Find a replicating constant, e.g. 0x00010001.
8242 if (auto *C = dyn_cast<ConstantSDNode>(Val: MulOp->getOperand(Num: 1))) {
8243 SystemZVectorConstantInfo VCI(
8244 APInt(MulVT.getSizeInBits(), C->getZExtValue()));
8245 if (VCI.isVectorConstantLegal(Subtarget) &&
8246 VCI.Opcode == SystemZISD::REPLICATE && VCI.OpVals[0] == 1 &&
8247 WordVT == VCI.VecVT.getScalarType())
8248 Word = DAG.getZExtOrTrunc(Op: LHS->getOperand(Num: 0), DL: SDLoc(SN), VT: WordVT);
8249 }
8250 }
8251 };
8252
8253 if (isa<BuildVectorSDNode>(Val: Op1) &&
8254 DAG.isSplatValue(V: Op1, AllowUndefs: true/*AllowUndefs*/)) {
8255 SDValue SplatVal = Op1->getOperand(Num: 0);
8256 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatVal))
8257 FindReplicatedImm(C, SplatVal.getValueType().getStoreSize());
8258 else
8259 FindReplicatedReg(SplatVal);
8260 } else {
8261 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op1))
8262 FindReplicatedImm(C, MemVT.getStoreSize());
8263 else
8264 FindReplicatedReg(Op1);
8265 }
8266
8267 if (Word != SDValue()) {
8268 assert(MemVT.getSizeInBits() % WordVT.getSizeInBits() == 0 &&
8269 "Bad type handling");
8270 unsigned NumElts = MemVT.getSizeInBits() / WordVT.getSizeInBits();
8271 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WordVT, NumElements: NumElts);
8272 SDValue SplatVal = DAG.getSplatVector(VT: SplatVT, DL: SDLoc(SN), Op: Word);
8273 return DAG.getStore(Chain: SN->getChain(), dl: SDLoc(SN), Val: SplatVal,
8274 Ptr: SN->getBasePtr(), MMO: SN->getMemOperand());
8275 }
8276 }
8277
8278 return SDValue();
8279}
8280
8281SDValue SystemZTargetLowering::combineVECTOR_SHUFFLE(
8282 SDNode *N, DAGCombinerInfo &DCI) const {
8283 SelectionDAG &DAG = DCI.DAG;
8284 // Combine element-swap (LOAD) into VLER
8285 if (ISD::isNON_EXTLoad(N: N->getOperand(Num: 0).getNode()) &&
8286 N->getOperand(Num: 0).hasOneUse() &&
8287 Subtarget.hasVectorEnhancements2()) {
8288 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
8289 ArrayRef<int> ShuffleMask = SVN->getMask();
8290 if (isVectorElementSwap(M: ShuffleMask, VT: N->getValueType(ResNo: 0))) {
8291 SDValue Load = N->getOperand(Num: 0);
8292 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
8293
8294 // Create the element-swapping load.
8295 SDValue Ops[] = {
8296 LD->getChain(), // Chain
8297 LD->getBasePtr() // Ptr
8298 };
8299 SDValue ESLoad =
8300 DAG.getMemIntrinsicNode(Opcode: SystemZISD::VLER, dl: SDLoc(N),
8301 VTList: DAG.getVTList(VT1: LD->getValueType(ResNo: 0), VT2: MVT::Other),
8302 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
8303
8304 // First, combine the VECTOR_SHUFFLE away. This makes the value produced
8305 // by the load dead.
8306 DCI.CombineTo(N, Res: ESLoad);
8307
8308 // Next, combine the load away, we give it a bogus result value but a real
8309 // chain result. The result value is dead because the shuffle is dead.
8310 DCI.CombineTo(N: Load.getNode(), Res0: ESLoad, Res1: ESLoad.getValue(R: 1));
8311
8312 // Return N so it doesn't get rechecked!
8313 return SDValue(N, 0);
8314 }
8315 }
8316
8317 return SDValue();
8318}
8319
8320SDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT(
8321 SDNode *N, DAGCombinerInfo &DCI) const {
8322 SelectionDAG &DAG = DCI.DAG;
8323
8324 if (!Subtarget.hasVector())
8325 return SDValue();
8326
8327 // Look through bitcasts that retain the number of vector elements.
8328 SDValue Op = N->getOperand(Num: 0);
8329 if (Op.getOpcode() == ISD::BITCAST &&
8330 Op.getValueType().isVector() &&
8331 Op.getOperand(i: 0).getValueType().isVector() &&
8332 Op.getValueType().getVectorNumElements() ==
8333 Op.getOperand(i: 0).getValueType().getVectorNumElements())
8334 Op = Op.getOperand(i: 0);
8335
8336 // Pull BSWAP out of a vector extraction.
8337 if (Op.getOpcode() == ISD::BSWAP && Op.hasOneUse()) {
8338 EVT VecVT = Op.getValueType();
8339 EVT EltVT = VecVT.getVectorElementType();
8340 Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N), VT: EltVT,
8341 N1: Op.getOperand(i: 0), N2: N->getOperand(Num: 1));
8342 DCI.AddToWorklist(N: Op.getNode());
8343 Op = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: EltVT, Operand: Op);
8344 if (EltVT != N->getValueType(ResNo: 0)) {
8345 DCI.AddToWorklist(N: Op.getNode());
8346 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
8347 }
8348 return Op;
8349 }
8350
8351 // Try to simplify a vector extraction.
8352 if (auto *IndexN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
8353 SDValue Op0 = N->getOperand(Num: 0);
8354 EVT VecVT = Op0.getValueType();
8355 if (canTreatAsByteVector(VT: VecVT))
8356 return combineExtract(DL: SDLoc(N), ResVT: N->getValueType(ResNo: 0), VecVT, Op: Op0,
8357 Index: IndexN->getZExtValue(), DCI, Force: false);
8358 }
8359 return SDValue();
8360}
8361
8362SDValue SystemZTargetLowering::combineJOIN_DWORDS(
8363 SDNode *N, DAGCombinerInfo &DCI) const {
8364 SelectionDAG &DAG = DCI.DAG;
8365 // (join_dwords X, X) == (replicate X)
8366 if (N->getOperand(Num: 0) == N->getOperand(Num: 1))
8367 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8368 Operand: N->getOperand(Num: 0));
8369 return SDValue();
8370}
8371
8372static SDValue MergeInputChains(SDNode *N1, SDNode *N2) {
8373 SDValue Chain1 = N1->getOperand(Num: 0);
8374 SDValue Chain2 = N2->getOperand(Num: 0);
8375
8376 // Trivial case: both nodes take the same chain.
8377 if (Chain1 == Chain2)
8378 return Chain1;
8379
8380 // FIXME - we could handle more complex cases via TokenFactor,
8381 // assuming we can verify that this would not create a cycle.
8382 return SDValue();
8383}
8384
8385SDValue SystemZTargetLowering::combineFP_ROUND(
8386 SDNode *N, DAGCombinerInfo &DCI) const {
8387
8388 if (!Subtarget.hasVector())
8389 return SDValue();
8390
8391 // (fpround (extract_vector_elt X 0))
8392 // (fpround (extract_vector_elt X 1)) ->
8393 // (extract_vector_elt (VROUND X) 0)
8394 // (extract_vector_elt (VROUND X) 2)
8395 //
8396 // This is a special case since the target doesn't really support v2f32s.
8397 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8398 SelectionDAG &DAG = DCI.DAG;
8399 SDValue Op0 = N->getOperand(Num: OpNo);
8400 if (N->getValueType(ResNo: 0) == MVT::f32 && Op0.hasOneUse() &&
8401 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8402 Op0.getOperand(i: 0).getValueType() == MVT::v2f64 &&
8403 Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
8404 Op0.getConstantOperandVal(i: 1) == 0) {
8405 SDValue Vec = Op0.getOperand(i: 0);
8406 for (auto *U : Vec->users()) {
8407 if (U != Op0.getNode() && U->hasOneUse() &&
8408 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8409 U->getOperand(Num: 0) == Vec &&
8410 U->getOperand(Num: 1).getOpcode() == ISD::Constant &&
8411 U->getConstantOperandVal(Num: 1) == 1) {
8412 SDValue OtherRound = SDValue(*U->user_begin(), 0);
8413 if (OtherRound.getOpcode() == N->getOpcode() &&
8414 OtherRound.getOperand(i: OpNo) == SDValue(U, 0) &&
8415 OtherRound.getValueType() == MVT::f32) {
8416 SDValue VRound, Chain;
8417 if (N->isStrictFPOpcode()) {
8418 Chain = MergeInputChains(N1: N, N2: OtherRound.getNode());
8419 if (!Chain)
8420 continue;
8421 VRound = DAG.getNode(Opcode: SystemZISD::STRICT_VROUND, DL: SDLoc(N),
8422 ResultTys: {MVT::v4f32, MVT::Other}, Ops: {Chain, Vec});
8423 Chain = VRound.getValue(R: 1);
8424 } else
8425 VRound = DAG.getNode(Opcode: SystemZISD::VROUND, DL: SDLoc(N),
8426 VT: MVT::v4f32, Operand: Vec);
8427 DCI.AddToWorklist(N: VRound.getNode());
8428 SDValue Extract1 =
8429 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(U), VT: MVT::f32,
8430 N1: VRound, N2: DAG.getConstant(Val: 2, DL: SDLoc(U), VT: MVT::i32));
8431 DCI.AddToWorklist(N: Extract1.getNode());
8432 DAG.ReplaceAllUsesOfValueWith(From: OtherRound, To: Extract1);
8433 if (Chain)
8434 DAG.ReplaceAllUsesOfValueWith(From: OtherRound.getValue(R: 1), To: Chain);
8435 SDValue Extract0 =
8436 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(Op0), VT: MVT::f32,
8437 N1: VRound, N2: DAG.getConstant(Val: 0, DL: SDLoc(Op0), VT: MVT::i32));
8438 if (Chain)
8439 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op0),
8440 VTList: N->getVTList(), N1: Extract0, N2: Chain);
8441 return Extract0;
8442 }
8443 }
8444 }
8445 }
8446 return SDValue();
8447}
8448
8449SDValue SystemZTargetLowering::combineFP_EXTEND(
8450 SDNode *N, DAGCombinerInfo &DCI) const {
8451
8452 if (!Subtarget.hasVector())
8453 return SDValue();
8454
8455 // (fpextend (extract_vector_elt X 0))
8456 // (fpextend (extract_vector_elt X 2)) ->
8457 // (extract_vector_elt (VEXTEND X) 0)
8458 // (extract_vector_elt (VEXTEND X) 1)
8459 //
8460 // This is a special case since the target doesn't really support v2f32s.
8461 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8462 SelectionDAG &DAG = DCI.DAG;
8463 SDValue Op0 = N->getOperand(Num: OpNo);
8464 if (N->getValueType(ResNo: 0) == MVT::f64 && Op0.hasOneUse() &&
8465 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8466 Op0.getOperand(i: 0).getValueType() == MVT::v4f32 &&
8467 Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
8468 Op0.getConstantOperandVal(i: 1) == 0) {
8469 SDValue Vec = Op0.getOperand(i: 0);
8470 for (auto *U : Vec->users()) {
8471 if (U != Op0.getNode() && U->hasOneUse() &&
8472 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8473 U->getOperand(Num: 0) == Vec &&
8474 U->getOperand(Num: 1).getOpcode() == ISD::Constant &&
8475 U->getConstantOperandVal(Num: 1) == 2) {
8476 SDValue OtherExtend = SDValue(*U->user_begin(), 0);
8477 if (OtherExtend.getOpcode() == N->getOpcode() &&
8478 OtherExtend.getOperand(i: OpNo) == SDValue(U, 0) &&
8479 OtherExtend.getValueType() == MVT::f64) {
8480 SDValue VExtend, Chain;
8481 if (N->isStrictFPOpcode()) {
8482 Chain = MergeInputChains(N1: N, N2: OtherExtend.getNode());
8483 if (!Chain)
8484 continue;
8485 VExtend = DAG.getNode(Opcode: SystemZISD::STRICT_VEXTEND, DL: SDLoc(N),
8486 ResultTys: {MVT::v2f64, MVT::Other}, Ops: {Chain, Vec});
8487 Chain = VExtend.getValue(R: 1);
8488 } else
8489 VExtend = DAG.getNode(Opcode: SystemZISD::VEXTEND, DL: SDLoc(N),
8490 VT: MVT::v2f64, Operand: Vec);
8491 DCI.AddToWorklist(N: VExtend.getNode());
8492 SDValue Extract1 =
8493 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(U), VT: MVT::f64,
8494 N1: VExtend, N2: DAG.getConstant(Val: 1, DL: SDLoc(U), VT: MVT::i32));
8495 DCI.AddToWorklist(N: Extract1.getNode());
8496 DAG.ReplaceAllUsesOfValueWith(From: OtherExtend, To: Extract1);
8497 if (Chain)
8498 DAG.ReplaceAllUsesOfValueWith(From: OtherExtend.getValue(R: 1), To: Chain);
8499 SDValue Extract0 =
8500 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(Op0), VT: MVT::f64,
8501 N1: VExtend, N2: DAG.getConstant(Val: 0, DL: SDLoc(Op0), VT: MVT::i32));
8502 if (Chain)
8503 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op0),
8504 VTList: N->getVTList(), N1: Extract0, N2: Chain);
8505 return Extract0;
8506 }
8507 }
8508 }
8509 }
8510 return SDValue();
8511}
8512
8513SDValue SystemZTargetLowering::combineINT_TO_FP(
8514 SDNode *N, DAGCombinerInfo &DCI) const {
8515 if (DCI.Level != BeforeLegalizeTypes)
8516 return SDValue();
8517 SelectionDAG &DAG = DCI.DAG;
8518 LLVMContext &Ctx = *DAG.getContext();
8519 unsigned Opcode = N->getOpcode();
8520 EVT OutVT = N->getValueType(ResNo: 0);
8521 Type *OutLLVMTy = OutVT.getTypeForEVT(Context&: Ctx);
8522 SDValue Op = N->getOperand(Num: 0);
8523 unsigned OutScalarBits = OutLLVMTy->getScalarSizeInBits();
8524 unsigned InScalarBits = Op->getValueType(ResNo: 0).getScalarSizeInBits();
8525
8526 // Insert an extension before type-legalization to avoid scalarization, e.g.:
8527 // v2f64 = uint_to_fp v2i16
8528 // =>
8529 // v2f64 = uint_to_fp (v2i64 zero_extend v2i16)
8530 if (OutLLVMTy->isVectorTy() && OutScalarBits > InScalarBits &&
8531 OutScalarBits <= 64) {
8532 unsigned NumElts = cast<FixedVectorType>(Val: OutLLVMTy)->getNumElements();
8533 EVT ExtVT = EVT::getVectorVT(
8534 Context&: Ctx, VT: EVT::getIntegerVT(Context&: Ctx, BitWidth: OutLLVMTy->getScalarSizeInBits()), NumElements: NumElts);
8535 unsigned ExtOpcode =
8536 (Opcode == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND);
8537 SDValue ExtOp = DAG.getNode(Opcode: ExtOpcode, DL: SDLoc(N), VT: ExtVT, Operand: Op);
8538 return DAG.getNode(Opcode, DL: SDLoc(N), VT: OutVT, Operand: ExtOp);
8539 }
8540 return SDValue();
8541}
8542
8543SDValue SystemZTargetLowering::combineFCOPYSIGN(
8544 SDNode *N, DAGCombinerInfo &DCI) const {
8545 SelectionDAG &DAG = DCI.DAG;
8546 EVT VT = N->getValueType(ResNo: 0);
8547 SDValue ValOp = N->getOperand(Num: 0);
8548 SDValue SignOp = N->getOperand(Num: 1);
8549
8550 // Remove the rounding which is not needed.
8551 if (SignOp.getOpcode() == ISD::FP_ROUND) {
8552 SDValue WideOp = SignOp.getOperand(i: 0);
8553 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N), VT, N1: ValOp, N2: WideOp);
8554 }
8555
8556 return SDValue();
8557}
8558
8559SDValue SystemZTargetLowering::combineBSWAP(
8560 SDNode *N, DAGCombinerInfo &DCI) const {
8561 SelectionDAG &DAG = DCI.DAG;
8562 // Combine BSWAP (LOAD) into LRVH/LRV/LRVG/VLBR
8563 if (ISD::isNON_EXTLoad(N: N->getOperand(Num: 0).getNode()) &&
8564 N->getOperand(Num: 0).hasOneUse() &&
8565 canLoadStoreByteSwapped(VT: N->getValueType(ResNo: 0))) {
8566 SDValue Load = N->getOperand(Num: 0);
8567 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
8568
8569 // Create the byte-swapping load.
8570 SDValue Ops[] = {
8571 LD->getChain(), // Chain
8572 LD->getBasePtr() // Ptr
8573 };
8574 EVT LoadVT = N->getValueType(ResNo: 0);
8575 if (LoadVT == MVT::i16)
8576 LoadVT = MVT::i32;
8577 SDValue BSLoad =
8578 DAG.getMemIntrinsicNode(Opcode: SystemZISD::LRV, dl: SDLoc(N),
8579 VTList: DAG.getVTList(VT1: LoadVT, VT2: MVT::Other),
8580 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
8581
8582 // If this is an i16 load, insert the truncate.
8583 SDValue ResVal = BSLoad;
8584 if (N->getValueType(ResNo: 0) == MVT::i16)
8585 ResVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: MVT::i16, Operand: BSLoad);
8586
8587 // First, combine the bswap away. This makes the value produced by the
8588 // load dead.
8589 DCI.CombineTo(N, Res: ResVal);
8590
8591 // Next, combine the load away, we give it a bogus result value but a real
8592 // chain result. The result value is dead because the bswap is dead.
8593 DCI.CombineTo(N: Load.getNode(), Res0: ResVal, Res1: BSLoad.getValue(R: 1));
8594
8595 // Return N so it doesn't get rechecked!
8596 return SDValue(N, 0);
8597 }
8598
8599 // Look through bitcasts that retain the number of vector elements.
8600 SDValue Op = N->getOperand(Num: 0);
8601 if (Op.getOpcode() == ISD::BITCAST &&
8602 Op.getValueType().isVector() &&
8603 Op.getOperand(i: 0).getValueType().isVector() &&
8604 Op.getValueType().getVectorNumElements() ==
8605 Op.getOperand(i: 0).getValueType().getVectorNumElements())
8606 Op = Op.getOperand(i: 0);
8607
8608 // Push BSWAP into a vector insertion if at least one side then simplifies.
8609 if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT && Op.hasOneUse()) {
8610 SDValue Vec = Op.getOperand(i: 0);
8611 SDValue Elt = Op.getOperand(i: 1);
8612 SDValue Idx = Op.getOperand(i: 2);
8613
8614 if (DAG.isConstantIntBuildVectorOrConstantInt(N: Vec) ||
8615 Vec.getOpcode() == ISD::BSWAP || Vec.isUndef() ||
8616 DAG.isConstantIntBuildVectorOrConstantInt(N: Elt) ||
8617 Elt.getOpcode() == ISD::BSWAP || Elt.isUndef() ||
8618 (canLoadStoreByteSwapped(VT: N->getValueType(ResNo: 0)) &&
8619 ISD::isNON_EXTLoad(N: Elt.getNode()) && Elt.hasOneUse())) {
8620 EVT VecVT = N->getValueType(ResNo: 0);
8621 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
8622 if (VecVT != Vec.getValueType()) {
8623 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Vec);
8624 DCI.AddToWorklist(N: Vec.getNode());
8625 }
8626 if (EltVT != Elt.getValueType()) {
8627 Elt = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: EltVT, Operand: Elt);
8628 DCI.AddToWorklist(N: Elt.getNode());
8629 }
8630 Vec = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Vec);
8631 DCI.AddToWorklist(N: Vec.getNode());
8632 Elt = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: EltVT, Operand: Elt);
8633 DCI.AddToWorklist(N: Elt.getNode());
8634 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: VecVT,
8635 N1: Vec, N2: Elt, N3: Idx);
8636 }
8637 }
8638
8639 // Push BSWAP into a vector shuffle if at least one side then simplifies.
8640 ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(Val&: Op);
8641 if (SV && Op.hasOneUse()) {
8642 SDValue Op0 = Op.getOperand(i: 0);
8643 SDValue Op1 = Op.getOperand(i: 1);
8644
8645 if (DAG.isConstantIntBuildVectorOrConstantInt(N: Op0) ||
8646 Op0.getOpcode() == ISD::BSWAP || Op0.isUndef() ||
8647 DAG.isConstantIntBuildVectorOrConstantInt(N: Op1) ||
8648 Op1.getOpcode() == ISD::BSWAP || Op1.isUndef()) {
8649 EVT VecVT = N->getValueType(ResNo: 0);
8650 if (VecVT != Op0.getValueType()) {
8651 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Op0);
8652 DCI.AddToWorklist(N: Op0.getNode());
8653 }
8654 if (VecVT != Op1.getValueType()) {
8655 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Op1);
8656 DCI.AddToWorklist(N: Op1.getNode());
8657 }
8658 Op0 = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Op0);
8659 DCI.AddToWorklist(N: Op0.getNode());
8660 Op1 = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Op1);
8661 DCI.AddToWorklist(N: Op1.getNode());
8662 return DAG.getVectorShuffle(VT: VecVT, dl: SDLoc(N), N1: Op0, N2: Op1, Mask: SV->getMask());
8663 }
8664 }
8665
8666 return SDValue();
8667}
8668
8669SDValue SystemZTargetLowering::combineSETCC(
8670 SDNode *N, DAGCombinerInfo &DCI) const {
8671 SelectionDAG &DAG = DCI.DAG;
8672 const ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
8673 const SDValue LHS = N->getOperand(Num: 0);
8674 const SDValue RHS = N->getOperand(Num: 1);
8675 bool CmpNull = isNullConstant(V: RHS);
8676 bool CmpAllOnes = isAllOnesConstant(V: RHS);
8677 EVT VT = N->getValueType(ResNo: 0);
8678 SDLoc DL(N);
8679
8680 // Match icmp_eq/ne(bitcast(icmp(X,Y)),0/-1) reduction patterns, and
8681 // change the outer compare to a i128 compare. This will normally
8682 // allow the reduction to be recognized in adjustICmp128, and even if
8683 // not, the i128 compare will still generate better code.
8684 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && (CmpNull || CmpAllOnes)) {
8685 SDValue Src = peekThroughBitcasts(V: LHS);
8686 if (Src.getOpcode() == ISD::SETCC &&
8687 Src.getValueType().isFixedLengthVector() &&
8688 Src.getValueType().getScalarType() == MVT::i1) {
8689 EVT CmpVT = Src.getOperand(i: 0).getValueType();
8690 if (CmpVT.getSizeInBits() == 128) {
8691 EVT IntVT = CmpVT.changeVectorElementTypeToInteger();
8692 SDValue LHS =
8693 DAG.getBitcast(VT: MVT::i128, V: DAG.getSExtOrTrunc(Op: Src, DL, VT: IntVT));
8694 SDValue RHS = CmpNull ? DAG.getConstant(Val: 0, DL, VT: MVT::i128)
8695 : DAG.getAllOnesConstant(DL, VT: MVT::i128);
8696 return DAG.getNode(Opcode: ISD::SETCC, DL, VT, N1: LHS, N2: RHS, N3: N->getOperand(Num: 2),
8697 Flags: N->getFlags());
8698 }
8699 }
8700 }
8701
8702 return SDValue();
8703}
8704
8705static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask) {
8706 // We have a SELECT_CCMASK or BR_CCMASK comparing the condition code
8707 // set by the CCReg instruction using the CCValid / CCMask masks,
8708 // If the CCReg instruction is itself a ICMP testing the condition
8709 // code set by some other instruction, see whether we can directly
8710 // use that condition code.
8711
8712 // Verify that we have an ICMP against some constant.
8713 if (CCValid != SystemZ::CCMASK_ICMP)
8714 return false;
8715 auto *ICmp = CCReg.getNode();
8716 if (ICmp->getOpcode() != SystemZISD::ICMP)
8717 return false;
8718 auto *CompareLHS = ICmp->getOperand(Num: 0).getNode();
8719 auto *CompareRHS = dyn_cast<ConstantSDNode>(Val: ICmp->getOperand(Num: 1));
8720 if (!CompareRHS)
8721 return false;
8722
8723 // Optimize the case where CompareLHS is a SELECT_CCMASK.
8724 if (CompareLHS->getOpcode() == SystemZISD::SELECT_CCMASK) {
8725 // Verify that we have an appropriate mask for a EQ or NE comparison.
8726 bool Invert = false;
8727 if (CCMask == SystemZ::CCMASK_CMP_NE)
8728 Invert = !Invert;
8729 else if (CCMask != SystemZ::CCMASK_CMP_EQ)
8730 return false;
8731
8732 // Verify that the ICMP compares against one of select values.
8733 auto *TrueVal = dyn_cast<ConstantSDNode>(Val: CompareLHS->getOperand(Num: 0));
8734 if (!TrueVal)
8735 return false;
8736 auto *FalseVal = dyn_cast<ConstantSDNode>(Val: CompareLHS->getOperand(Num: 1));
8737 if (!FalseVal)
8738 return false;
8739 if (CompareRHS->getAPIntValue() == FalseVal->getAPIntValue())
8740 Invert = !Invert;
8741 else if (CompareRHS->getAPIntValue() != TrueVal->getAPIntValue())
8742 return false;
8743
8744 // Compute the effective CC mask for the new branch or select.
8745 auto *NewCCValid = dyn_cast<ConstantSDNode>(Val: CompareLHS->getOperand(Num: 2));
8746 auto *NewCCMask = dyn_cast<ConstantSDNode>(Val: CompareLHS->getOperand(Num: 3));
8747 if (!NewCCValid || !NewCCMask)
8748 return false;
8749 CCValid = NewCCValid->getZExtValue();
8750 CCMask = NewCCMask->getZExtValue();
8751 if (Invert)
8752 CCMask ^= CCValid;
8753
8754 // Return the updated CCReg link.
8755 CCReg = CompareLHS->getOperand(Num: 4);
8756 return true;
8757 }
8758
8759 // Optimize the case where CompareRHS is (SRA (SHL (IPM))).
8760 if (CompareLHS->getOpcode() == ISD::SRA) {
8761 auto *SRACount = dyn_cast<ConstantSDNode>(Val: CompareLHS->getOperand(Num: 1));
8762 if (!SRACount || SRACount->getZExtValue() != 30)
8763 return false;
8764 auto *SHL = CompareLHS->getOperand(Num: 0).getNode();
8765 if (SHL->getOpcode() != ISD::SHL)
8766 return false;
8767 auto *SHLCount = dyn_cast<ConstantSDNode>(Val: SHL->getOperand(Num: 1));
8768 if (!SHLCount || SHLCount->getZExtValue() != 30 - SystemZ::IPM_CC)
8769 return false;
8770 auto *IPM = SHL->getOperand(Num: 0).getNode();
8771 if (IPM->getOpcode() != SystemZISD::IPM)
8772 return false;
8773
8774 // Avoid introducing CC spills (because SRA would clobber CC).
8775 if (!CompareLHS->hasOneUse())
8776 return false;
8777 // Verify that the ICMP compares against zero.
8778 if (CompareRHS->getZExtValue() != 0)
8779 return false;
8780
8781 // Compute the effective CC mask for the new branch or select.
8782 CCMask = SystemZ::reverseCCMask(CCMask);
8783
8784 // Return the updated CCReg link.
8785 CCReg = IPM->getOperand(Num: 0);
8786 return true;
8787 }
8788
8789 return false;
8790}
8791
8792SDValue SystemZTargetLowering::combineBR_CCMASK(
8793 SDNode *N, DAGCombinerInfo &DCI) const {
8794 SelectionDAG &DAG = DCI.DAG;
8795
8796 // Combine BR_CCMASK (ICMP (SELECT_CCMASK)) into a single BR_CCMASK.
8797 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
8798 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
8799 if (!CCValid || !CCMask)
8800 return SDValue();
8801
8802 int CCValidVal = CCValid->getZExtValue();
8803 int CCMaskVal = CCMask->getZExtValue();
8804 SDValue Chain = N->getOperand(Num: 0);
8805 SDValue CCReg = N->getOperand(Num: 4);
8806
8807 if (combineCCMask(CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal))
8808 return DAG.getNode(Opcode: SystemZISD::BR_CCMASK, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8809 N1: Chain,
8810 N2: DAG.getTargetConstant(Val: CCValidVal, DL: SDLoc(N), VT: MVT::i32),
8811 N3: DAG.getTargetConstant(Val: CCMaskVal, DL: SDLoc(N), VT: MVT::i32),
8812 N4: N->getOperand(Num: 3), N5: CCReg);
8813 return SDValue();
8814}
8815
8816SDValue SystemZTargetLowering::combineSELECT_CCMASK(
8817 SDNode *N, DAGCombinerInfo &DCI) const {
8818 SelectionDAG &DAG = DCI.DAG;
8819
8820 // Combine SELECT_CCMASK (ICMP (SELECT_CCMASK)) into a single SELECT_CCMASK.
8821 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
8822 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3));
8823 if (!CCValid || !CCMask)
8824 return SDValue();
8825
8826 int CCValidVal = CCValid->getZExtValue();
8827 int CCMaskVal = CCMask->getZExtValue();
8828 SDValue CCReg = N->getOperand(Num: 4);
8829
8830 if (combineCCMask(CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal))
8831 return DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8832 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
8833 N3: DAG.getTargetConstant(Val: CCValidVal, DL: SDLoc(N), VT: MVT::i32),
8834 N4: DAG.getTargetConstant(Val: CCMaskVal, DL: SDLoc(N), VT: MVT::i32),
8835 N5: CCReg);
8836 return SDValue();
8837}
8838
8839
8840SDValue SystemZTargetLowering::combineGET_CCMASK(
8841 SDNode *N, DAGCombinerInfo &DCI) const {
8842
8843 // Optimize away GET_CCMASK (SELECT_CCMASK) if the CC masks are compatible
8844 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
8845 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
8846 if (!CCValid || !CCMask)
8847 return SDValue();
8848 int CCValidVal = CCValid->getZExtValue();
8849 int CCMaskVal = CCMask->getZExtValue();
8850
8851 SDValue Select = N->getOperand(Num: 0);
8852 if (Select->getOpcode() == ISD::TRUNCATE)
8853 Select = Select->getOperand(Num: 0);
8854 if (Select->getOpcode() != SystemZISD::SELECT_CCMASK)
8855 return SDValue();
8856
8857 auto *SelectCCValid = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 2));
8858 auto *SelectCCMask = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 3));
8859 if (!SelectCCValid || !SelectCCMask)
8860 return SDValue();
8861 int SelectCCValidVal = SelectCCValid->getZExtValue();
8862 int SelectCCMaskVal = SelectCCMask->getZExtValue();
8863
8864 auto *TrueVal = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 0));
8865 auto *FalseVal = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 1));
8866 if (!TrueVal || !FalseVal)
8867 return SDValue();
8868 if (TrueVal->getZExtValue() == 1 && FalseVal->getZExtValue() == 0)
8869 ;
8870 else if (TrueVal->getZExtValue() == 0 && FalseVal->getZExtValue() == 1)
8871 SelectCCMaskVal ^= SelectCCValidVal;
8872 else
8873 return SDValue();
8874
8875 if (SelectCCValidVal & ~CCValidVal)
8876 return SDValue();
8877 if (SelectCCMaskVal != (CCMaskVal & SelectCCValidVal))
8878 return SDValue();
8879
8880 return Select->getOperand(Num: 4);
8881}
8882
8883SDValue SystemZTargetLowering::combineIntDIVREM(
8884 SDNode *N, DAGCombinerInfo &DCI) const {
8885 SelectionDAG &DAG = DCI.DAG;
8886 EVT VT = N->getValueType(ResNo: 0);
8887 // In the case where the divisor is a vector of constants a cheaper
8888 // sequence of instructions can replace the divide. BuildSDIV is called to
8889 // do this during DAG combining, but it only succeeds when it can build a
8890 // multiplication node. The only option for SystemZ is ISD::SMUL_LOHI, and
8891 // since it is not Legal but Custom it can only happen before
8892 // legalization. Therefore we must scalarize this early before Combine
8893 // 1. For widened vectors, this is already the result of type legalization.
8894 if (DCI.Level == BeforeLegalizeTypes && VT.isVector() && isTypeLegal(VT) &&
8895 DAG.isConstantIntBuildVectorOrConstantInt(N: N->getOperand(Num: 1)))
8896 return DAG.UnrollVectorOp(N);
8897 return SDValue();
8898}
8899
8900
8901// Transform a right shift of a multiply-and-add into a multiply-and-add-high.
8902// This is closely modeled after the common-code combineShiftToMULH.
8903SDValue SystemZTargetLowering::combineShiftToMulAddHigh(
8904 SDNode *N, DAGCombinerInfo &DCI) const {
8905 SelectionDAG &DAG = DCI.DAG;
8906 SDLoc DL(N);
8907
8908 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
8909 "SRL or SRA node is required here!");
8910
8911 if (!Subtarget.hasVector())
8912 return SDValue();
8913
8914 // Check the shift amount. Proceed with the transformation if the shift
8915 // amount is constant.
8916 ConstantSDNode *ShiftAmtSrc = isConstOrConstSplat(N: N->getOperand(Num: 1));
8917 if (!ShiftAmtSrc)
8918 return SDValue();
8919
8920 // The operation feeding into the shift must be an add.
8921 SDValue ShiftOperand = N->getOperand(Num: 0);
8922 if (ShiftOperand.getOpcode() != ISD::ADD)
8923 return SDValue();
8924
8925 // One operand of the add must be a multiply.
8926 SDValue MulOp = ShiftOperand.getOperand(i: 0);
8927 SDValue AddOp = ShiftOperand.getOperand(i: 1);
8928 if (MulOp.getOpcode() != ISD::MUL) {
8929 if (AddOp.getOpcode() != ISD::MUL)
8930 return SDValue();
8931 std::swap(a&: MulOp, b&: AddOp);
8932 }
8933
8934 // All operands must be equivalent extend nodes.
8935 SDValue LeftOp = MulOp.getOperand(i: 0);
8936 SDValue RightOp = MulOp.getOperand(i: 1);
8937
8938 bool IsSignExt = LeftOp.getOpcode() == ISD::SIGN_EXTEND;
8939 bool IsZeroExt = LeftOp.getOpcode() == ISD::ZERO_EXTEND;
8940
8941 if (!IsSignExt && !IsZeroExt)
8942 return SDValue();
8943
8944 EVT NarrowVT = LeftOp.getOperand(i: 0).getValueType();
8945 unsigned NarrowVTSize = NarrowVT.getScalarSizeInBits();
8946
8947 SDValue MulhRightOp;
8948 if (ConstantSDNode *Constant = isConstOrConstSplat(N: RightOp)) {
8949 unsigned ActiveBits = IsSignExt
8950 ? Constant->getAPIntValue().getSignificantBits()
8951 : Constant->getAPIntValue().getActiveBits();
8952 if (ActiveBits > NarrowVTSize)
8953 return SDValue();
8954 MulhRightOp = DAG.getConstant(
8955 Val: Constant->getAPIntValue().trunc(width: NarrowVT.getScalarSizeInBits()), DL,
8956 VT: NarrowVT);
8957 } else {
8958 if (LeftOp.getOpcode() != RightOp.getOpcode())
8959 return SDValue();
8960 // Check that the two extend nodes are the same type.
8961 if (NarrowVT != RightOp.getOperand(i: 0).getValueType())
8962 return SDValue();
8963 MulhRightOp = RightOp.getOperand(i: 0);
8964 }
8965
8966 SDValue MulhAddOp;
8967 if (ConstantSDNode *Constant = isConstOrConstSplat(N: AddOp)) {
8968 unsigned ActiveBits = IsSignExt
8969 ? Constant->getAPIntValue().getSignificantBits()
8970 : Constant->getAPIntValue().getActiveBits();
8971 if (ActiveBits > NarrowVTSize)
8972 return SDValue();
8973 MulhAddOp = DAG.getConstant(
8974 Val: Constant->getAPIntValue().trunc(width: NarrowVT.getScalarSizeInBits()), DL,
8975 VT: NarrowVT);
8976 } else {
8977 if (LeftOp.getOpcode() != AddOp.getOpcode())
8978 return SDValue();
8979 // Check that the two extend nodes are the same type.
8980 if (NarrowVT != AddOp.getOperand(i: 0).getValueType())
8981 return SDValue();
8982 MulhAddOp = AddOp.getOperand(i: 0);
8983 }
8984
8985 EVT WideVT = LeftOp.getValueType();
8986 // Proceed with the transformation if the wide types match.
8987 assert((WideVT == RightOp.getValueType()) &&
8988 "Cannot have a multiply node with two different operand types.");
8989 assert((WideVT == AddOp.getValueType()) &&
8990 "Cannot have an add node with two different operand types.");
8991
8992 // Proceed with the transformation if the wide type is twice as large
8993 // as the narrow type.
8994 if (WideVT.getScalarSizeInBits() != 2 * NarrowVTSize)
8995 return SDValue();
8996
8997 // Check the shift amount with the narrow type size.
8998 // Proceed with the transformation if the shift amount is the width
8999 // of the narrow type.
9000 unsigned ShiftAmt = ShiftAmtSrc->getZExtValue();
9001 if (ShiftAmt != NarrowVTSize)
9002 return SDValue();
9003
9004 // Proceed if we support the multiply-and-add-high operation.
9005 if (!(NarrowVT == MVT::v16i8 || NarrowVT == MVT::v8i16 ||
9006 NarrowVT == MVT::v4i32 ||
9007 (Subtarget.hasVectorEnhancements3() &&
9008 (NarrowVT == MVT::v2i64 || NarrowVT == MVT::i128))))
9009 return SDValue();
9010
9011 // Emit the VMAH (signed) or VMALH (unsigned) operation.
9012 SDValue Result = DAG.getNode(Opcode: IsSignExt ? SystemZISD::VMAH : SystemZISD::VMALH,
9013 DL, VT: NarrowVT, N1: LeftOp.getOperand(i: 0),
9014 N2: MulhRightOp, N3: MulhAddOp);
9015 bool IsSigned = N->getOpcode() == ISD::SRA;
9016 return DAG.getExtOrTrunc(IsSigned, Op: Result, DL, VT: WideVT);
9017}
9018
9019// Op is an operand of a multiplication. Check whether this can be folded
9020// into an even/odd widening operation; if so, return the opcode to be used
9021// and update Op to the appropriate sub-operand. Note that the caller must
9022// verify that *both* operands of the multiplication support the operation.
9023static unsigned detectEvenOddMultiplyOperand(const SelectionDAG &DAG,
9024 const SystemZSubtarget &Subtarget,
9025 SDValue &Op) {
9026 EVT VT = Op.getValueType();
9027
9028 // Check for (sign/zero_extend_vector_inreg (vector_shuffle)) corresponding
9029 // to selecting the even or odd vector elements.
9030 if (VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
9031 (Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
9032 Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG)) {
9033 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
9034 unsigned NumElts = VT.getVectorNumElements();
9035 Op = Op.getOperand(i: 0);
9036 if (Op.getValueType().getVectorNumElements() == 2 * NumElts &&
9037 Op.getOpcode() == ISD::VECTOR_SHUFFLE) {
9038 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
9039 ArrayRef<int> ShuffleMask = SVN->getMask();
9040 bool CanUseEven = true, CanUseOdd = true;
9041 for (unsigned Elt = 0; Elt < NumElts; Elt++) {
9042 if (ShuffleMask[Elt] == -1)
9043 continue;
9044 if (unsigned(ShuffleMask[Elt]) != 2 * Elt)
9045 CanUseEven = false;
9046 if (unsigned(ShuffleMask[Elt]) != 2 * Elt + 1)
9047 CanUseEven = true;
9048 }
9049 Op = Op.getOperand(i: 0);
9050 if (CanUseEven)
9051 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9052 if (CanUseOdd)
9053 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9054 }
9055 }
9056
9057 // For z17, we can also support the v2i64->i128 case, which looks like
9058 // (sign/zero_extend (extract_vector_elt X 0/1))
9059 if (VT == MVT::i128 && Subtarget.hasVectorEnhancements3() &&
9060 (Op.getOpcode() == ISD::SIGN_EXTEND ||
9061 Op.getOpcode() == ISD::ZERO_EXTEND)) {
9062 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND;
9063 Op = Op.getOperand(i: 0);
9064 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
9065 Op.getOperand(i: 0).getValueType() == MVT::v2i64 &&
9066 Op.getOperand(i: 1).getOpcode() == ISD::Constant) {
9067 unsigned Elem = Op.getConstantOperandVal(i: 1);
9068 Op = Op.getOperand(i: 0);
9069 if (Elem == 0)
9070 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9071 if (Elem == 1)
9072 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9073 }
9074 }
9075
9076 return 0;
9077}
9078
9079SDValue SystemZTargetLowering::combineMUL(
9080 SDNode *N, DAGCombinerInfo &DCI) const {
9081 SelectionDAG &DAG = DCI.DAG;
9082
9083 // Detect even/odd widening multiplication.
9084 SDValue Op0 = N->getOperand(Num: 0);
9085 SDValue Op1 = N->getOperand(Num: 1);
9086 unsigned OpcodeCand0 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op&: Op0);
9087 unsigned OpcodeCand1 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op&: Op1);
9088 if (OpcodeCand0 && OpcodeCand0 == OpcodeCand1)
9089 return DAG.getNode(Opcode: OpcodeCand0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1);
9090
9091 return SDValue();
9092}
9093
9094SDValue SystemZTargetLowering::combineINTRINSIC(
9095 SDNode *N, DAGCombinerInfo &DCI) const {
9096 SelectionDAG &DAG = DCI.DAG;
9097
9098 unsigned Id = N->getConstantOperandVal(Num: 1);
9099 switch (Id) {
9100 // VECTOR LOAD (RIGHTMOST) WITH LENGTH with a length operand of 15
9101 // or larger is simply a vector load.
9102 case Intrinsic::s390_vll:
9103 case Intrinsic::s390_vlrl:
9104 if (auto *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2)))
9105 if (C->getZExtValue() >= 15)
9106 return DAG.getLoad(VT: N->getValueType(ResNo: 0), dl: SDLoc(N), Chain: N->getOperand(Num: 0),
9107 Ptr: N->getOperand(Num: 3), PtrInfo: MachinePointerInfo());
9108 break;
9109 // Likewise for VECTOR STORE (RIGHTMOST) WITH LENGTH.
9110 case Intrinsic::s390_vstl:
9111 case Intrinsic::s390_vstrl:
9112 if (auto *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3)))
9113 if (C->getZExtValue() >= 15)
9114 return DAG.getStore(Chain: N->getOperand(Num: 0), dl: SDLoc(N), Val: N->getOperand(Num: 2),
9115 Ptr: N->getOperand(Num: 4), PtrInfo: MachinePointerInfo());
9116 break;
9117 }
9118
9119 return SDValue();
9120}
9121
9122SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
9123 if (N->getOpcode() == SystemZISD::PCREL_WRAPPER)
9124 return N->getOperand(Num: 0);
9125 return N;
9126}
9127
9128SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
9129 DAGCombinerInfo &DCI) const {
9130 switch(N->getOpcode()) {
9131 default: break;
9132 case ISD::ZERO_EXTEND: return combineZERO_EXTEND(N, DCI);
9133 case ISD::SIGN_EXTEND: return combineSIGN_EXTEND(N, DCI);
9134 case ISD::SIGN_EXTEND_INREG: return combineSIGN_EXTEND_INREG(N, DCI);
9135 case SystemZISD::MERGE_HIGH:
9136 case SystemZISD::MERGE_LOW: return combineMERGE(N, DCI);
9137 case ISD::LOAD: return combineLOAD(N, DCI);
9138 case ISD::STORE: return combineSTORE(N, DCI);
9139 case ISD::VECTOR_SHUFFLE: return combineVECTOR_SHUFFLE(N, DCI);
9140 case ISD::EXTRACT_VECTOR_ELT: return combineEXTRACT_VECTOR_ELT(N, DCI);
9141 case SystemZISD::JOIN_DWORDS: return combineJOIN_DWORDS(N, DCI);
9142 case ISD::STRICT_FP_ROUND:
9143 case ISD::FP_ROUND: return combineFP_ROUND(N, DCI);
9144 case ISD::STRICT_FP_EXTEND:
9145 case ISD::FP_EXTEND: return combineFP_EXTEND(N, DCI);
9146 case ISD::SINT_TO_FP:
9147 case ISD::UINT_TO_FP: return combineINT_TO_FP(N, DCI);
9148 case ISD::FCOPYSIGN: return combineFCOPYSIGN(N, DCI);
9149 case ISD::BSWAP: return combineBSWAP(N, DCI);
9150 case ISD::SETCC: return combineSETCC(N, DCI);
9151 case SystemZISD::BR_CCMASK: return combineBR_CCMASK(N, DCI);
9152 case SystemZISD::SELECT_CCMASK: return combineSELECT_CCMASK(N, DCI);
9153 case SystemZISD::GET_CCMASK: return combineGET_CCMASK(N, DCI);
9154 case ISD::SRL:
9155 case ISD::SRA: return combineShiftToMulAddHigh(N, DCI);
9156 case ISD::MUL: return combineMUL(N, DCI);
9157 case ISD::SDIV:
9158 case ISD::UDIV:
9159 case ISD::SREM:
9160 case ISD::UREM: return combineIntDIVREM(N, DCI);
9161 case ISD::INTRINSIC_W_CHAIN:
9162 case ISD::INTRINSIC_VOID: return combineINTRINSIC(N, DCI);
9163 }
9164
9165 return SDValue();
9166}
9167
9168// Return the demanded elements for the OpNo source operand of Op. DemandedElts
9169// are for Op.
9170static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts,
9171 unsigned OpNo) {
9172 EVT VT = Op.getValueType();
9173 unsigned NumElts = (VT.isVector() ? VT.getVectorNumElements() : 1);
9174 APInt SrcDemE;
9175 unsigned Opcode = Op.getOpcode();
9176 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9177 unsigned Id = Op.getConstantOperandVal(i: 0);
9178 switch (Id) {
9179 case Intrinsic::s390_vpksh: // PACKS
9180 case Intrinsic::s390_vpksf:
9181 case Intrinsic::s390_vpksg:
9182 case Intrinsic::s390_vpkshs: // PACKS_CC
9183 case Intrinsic::s390_vpksfs:
9184 case Intrinsic::s390_vpksgs:
9185 case Intrinsic::s390_vpklsh: // PACKLS
9186 case Intrinsic::s390_vpklsf:
9187 case Intrinsic::s390_vpklsg:
9188 case Intrinsic::s390_vpklshs: // PACKLS_CC
9189 case Intrinsic::s390_vpklsfs:
9190 case Intrinsic::s390_vpklsgs:
9191 // VECTOR PACK truncates the elements of two source vectors into one.
9192 SrcDemE = DemandedElts;
9193 if (OpNo == 2)
9194 SrcDemE.lshrInPlace(ShiftAmt: NumElts / 2);
9195 SrcDemE = SrcDemE.trunc(width: NumElts / 2);
9196 break;
9197 // VECTOR UNPACK extends half the elements of the source vector.
9198 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9199 case Intrinsic::s390_vuphh:
9200 case Intrinsic::s390_vuphf:
9201 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9202 case Intrinsic::s390_vuplhh:
9203 case Intrinsic::s390_vuplhf:
9204 SrcDemE = APInt(NumElts * 2, 0);
9205 SrcDemE.insertBits(SubBits: DemandedElts, bitPosition: 0);
9206 break;
9207 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9208 case Intrinsic::s390_vuplhw:
9209 case Intrinsic::s390_vuplf:
9210 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9211 case Intrinsic::s390_vupllh:
9212 case Intrinsic::s390_vupllf:
9213 SrcDemE = APInt(NumElts * 2, 0);
9214 SrcDemE.insertBits(SubBits: DemandedElts, bitPosition: NumElts);
9215 break;
9216 case Intrinsic::s390_vpdi: {
9217 // VECTOR PERMUTE DWORD IMMEDIATE selects one element from each source.
9218 SrcDemE = APInt(NumElts, 0);
9219 if (!DemandedElts[OpNo - 1])
9220 break;
9221 unsigned Mask = Op.getConstantOperandVal(i: 3);
9222 unsigned MaskBit = ((OpNo - 1) ? 1 : 4);
9223 // Demand input element 0 or 1, given by the mask bit value.
9224 SrcDemE.setBit((Mask & MaskBit)? 1 : 0);
9225 break;
9226 }
9227 case Intrinsic::s390_vsldb: {
9228 // VECTOR SHIFT LEFT DOUBLE BY BYTE
9229 assert(VT == MVT::v16i8 && "Unexpected type.");
9230 unsigned FirstIdx = Op.getConstantOperandVal(i: 3);
9231 assert (FirstIdx > 0 && FirstIdx < 16 && "Unused operand.");
9232 unsigned NumSrc0Els = 16 - FirstIdx;
9233 SrcDemE = APInt(NumElts, 0);
9234 if (OpNo == 1) {
9235 APInt DemEls = DemandedElts.trunc(width: NumSrc0Els);
9236 SrcDemE.insertBits(SubBits: DemEls, bitPosition: FirstIdx);
9237 } else {
9238 APInt DemEls = DemandedElts.lshr(shiftAmt: NumSrc0Els);
9239 SrcDemE.insertBits(SubBits: DemEls, bitPosition: 0);
9240 }
9241 break;
9242 }
9243 case Intrinsic::s390_vperm:
9244 SrcDemE = APInt::getAllOnes(numBits: NumElts);
9245 break;
9246 default:
9247 llvm_unreachable("Unhandled intrinsic.");
9248 break;
9249 }
9250 } else {
9251 switch (Opcode) {
9252 case SystemZISD::JOIN_DWORDS:
9253 // Scalar operand.
9254 SrcDemE = APInt(1, 1);
9255 break;
9256 case SystemZISD::SELECT_CCMASK:
9257 SrcDemE = DemandedElts;
9258 break;
9259 default:
9260 llvm_unreachable("Unhandled opcode.");
9261 break;
9262 }
9263 }
9264 return SrcDemE;
9265}
9266
9267static void computeKnownBitsBinOp(const SDValue Op, KnownBits &Known,
9268 const APInt &DemandedElts,
9269 const SelectionDAG &DAG, unsigned Depth,
9270 unsigned OpNo) {
9271 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9272 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo: OpNo + 1);
9273 KnownBits LHSKnown =
9274 DAG.computeKnownBits(Op: Op.getOperand(i: OpNo), DemandedElts: Src0DemE, Depth: Depth + 1);
9275 KnownBits RHSKnown =
9276 DAG.computeKnownBits(Op: Op.getOperand(i: OpNo + 1), DemandedElts: Src1DemE, Depth: Depth + 1);
9277 Known = LHSKnown.intersectWith(RHS: RHSKnown);
9278}
9279
9280void
9281SystemZTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
9282 KnownBits &Known,
9283 const APInt &DemandedElts,
9284 const SelectionDAG &DAG,
9285 unsigned Depth) const {
9286 Known.resetAll();
9287
9288 // Intrinsic CC result is returned in the two low bits.
9289 unsigned Tmp0, Tmp1; // not used
9290 if (Op.getResNo() == 1 && isIntrinsicWithCC(Op, Opcode&: Tmp0, CCValid&: Tmp1)) {
9291 Known.Zero.setBitsFrom(2);
9292 return;
9293 }
9294 EVT VT = Op.getValueType();
9295 if (Op.getResNo() != 0 || VT == MVT::Untyped)
9296 return;
9297 assert (Known.getBitWidth() == VT.getScalarSizeInBits() &&
9298 "KnownBits does not match VT in bitwidth");
9299 assert ((!VT.isVector() ||
9300 (DemandedElts.getBitWidth() == VT.getVectorNumElements())) &&
9301 "DemandedElts does not match VT number of elements");
9302 unsigned BitWidth = Known.getBitWidth();
9303 unsigned Opcode = Op.getOpcode();
9304 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9305 bool IsLogical = false;
9306 unsigned Id = Op.getConstantOperandVal(i: 0);
9307 switch (Id) {
9308 case Intrinsic::s390_vpksh: // PACKS
9309 case Intrinsic::s390_vpksf:
9310 case Intrinsic::s390_vpksg:
9311 case Intrinsic::s390_vpkshs: // PACKS_CC
9312 case Intrinsic::s390_vpksfs:
9313 case Intrinsic::s390_vpksgs:
9314 case Intrinsic::s390_vpklsh: // PACKLS
9315 case Intrinsic::s390_vpklsf:
9316 case Intrinsic::s390_vpklsg:
9317 case Intrinsic::s390_vpklshs: // PACKLS_CC
9318 case Intrinsic::s390_vpklsfs:
9319 case Intrinsic::s390_vpklsgs:
9320 case Intrinsic::s390_vpdi:
9321 case Intrinsic::s390_vsldb:
9322 case Intrinsic::s390_vperm:
9323 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, OpNo: 1);
9324 break;
9325 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9326 case Intrinsic::s390_vuplhh:
9327 case Intrinsic::s390_vuplhf:
9328 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9329 case Intrinsic::s390_vupllh:
9330 case Intrinsic::s390_vupllf:
9331 IsLogical = true;
9332 [[fallthrough]];
9333 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9334 case Intrinsic::s390_vuphh:
9335 case Intrinsic::s390_vuphf:
9336 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9337 case Intrinsic::s390_vuplhw:
9338 case Intrinsic::s390_vuplf: {
9339 SDValue SrcOp = Op.getOperand(i: 1);
9340 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, OpNo: 0);
9341 Known = DAG.computeKnownBits(Op: SrcOp, DemandedElts: SrcDemE, Depth: Depth + 1);
9342 if (IsLogical) {
9343 Known = Known.zext(BitWidth);
9344 } else
9345 Known = Known.sext(BitWidth);
9346 break;
9347 }
9348 default:
9349 break;
9350 }
9351 } else {
9352 switch (Opcode) {
9353 case SystemZISD::JOIN_DWORDS:
9354 case SystemZISD::SELECT_CCMASK:
9355 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, OpNo: 0);
9356 break;
9357 case SystemZISD::REPLICATE: {
9358 SDValue SrcOp = Op.getOperand(i: 0);
9359 Known = DAG.computeKnownBits(Op: SrcOp, Depth: Depth + 1);
9360 if (Known.getBitWidth() < BitWidth && isa<ConstantSDNode>(Val: SrcOp))
9361 Known = Known.sext(BitWidth); // VREPI sign extends the immedate.
9362 break;
9363 }
9364 default:
9365 break;
9366 }
9367 }
9368
9369 // Known has the width of the source operand(s). Adjust if needed to match
9370 // the passed bitwidth.
9371 if (Known.getBitWidth() != BitWidth)
9372 Known = Known.anyextOrTrunc(BitWidth);
9373}
9374
9375static unsigned computeNumSignBitsBinOp(SDValue Op, const APInt &DemandedElts,
9376 const SelectionDAG &DAG, unsigned Depth,
9377 unsigned OpNo) {
9378 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9379 unsigned LHS = DAG.ComputeNumSignBits(Op: Op.getOperand(i: OpNo), DemandedElts: Src0DemE, Depth: Depth + 1);
9380 if (LHS == 1) return 1; // Early out.
9381 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo: OpNo + 1);
9382 unsigned RHS = DAG.ComputeNumSignBits(Op: Op.getOperand(i: OpNo + 1), DemandedElts: Src1DemE, Depth: Depth + 1);
9383 if (RHS == 1) return 1; // Early out.
9384 unsigned Common = std::min(a: LHS, b: RHS);
9385 unsigned SrcBitWidth = Op.getOperand(i: OpNo).getScalarValueSizeInBits();
9386 EVT VT = Op.getValueType();
9387 unsigned VTBits = VT.getScalarSizeInBits();
9388 if (SrcBitWidth > VTBits) { // PACK
9389 unsigned SrcExtraBits = SrcBitWidth - VTBits;
9390 if (Common > SrcExtraBits)
9391 return (Common - SrcExtraBits);
9392 return 1;
9393 }
9394 assert (SrcBitWidth == VTBits && "Expected operands of same bitwidth.");
9395 return Common;
9396}
9397
9398unsigned
9399SystemZTargetLowering::ComputeNumSignBitsForTargetNode(
9400 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
9401 unsigned Depth) const {
9402 if (Op.getResNo() != 0)
9403 return 1;
9404 unsigned Opcode = Op.getOpcode();
9405 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9406 unsigned Id = Op.getConstantOperandVal(i: 0);
9407 switch (Id) {
9408 case Intrinsic::s390_vpksh: // PACKS
9409 case Intrinsic::s390_vpksf:
9410 case Intrinsic::s390_vpksg:
9411 case Intrinsic::s390_vpkshs: // PACKS_CC
9412 case Intrinsic::s390_vpksfs:
9413 case Intrinsic::s390_vpksgs:
9414 case Intrinsic::s390_vpklsh: // PACKLS
9415 case Intrinsic::s390_vpklsf:
9416 case Intrinsic::s390_vpklsg:
9417 case Intrinsic::s390_vpklshs: // PACKLS_CC
9418 case Intrinsic::s390_vpklsfs:
9419 case Intrinsic::s390_vpklsgs:
9420 case Intrinsic::s390_vpdi:
9421 case Intrinsic::s390_vsldb:
9422 case Intrinsic::s390_vperm:
9423 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, OpNo: 1);
9424 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9425 case Intrinsic::s390_vuphh:
9426 case Intrinsic::s390_vuphf:
9427 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9428 case Intrinsic::s390_vuplhw:
9429 case Intrinsic::s390_vuplf: {
9430 SDValue PackedOp = Op.getOperand(i: 1);
9431 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, OpNo: 1);
9432 unsigned Tmp = DAG.ComputeNumSignBits(Op: PackedOp, DemandedElts: SrcDemE, Depth: Depth + 1);
9433 EVT VT = Op.getValueType();
9434 unsigned VTBits = VT.getScalarSizeInBits();
9435 Tmp += VTBits - PackedOp.getScalarValueSizeInBits();
9436 return Tmp;
9437 }
9438 default:
9439 break;
9440 }
9441 } else {
9442 switch (Opcode) {
9443 case SystemZISD::SELECT_CCMASK:
9444 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, OpNo: 0);
9445 default:
9446 break;
9447 }
9448 }
9449
9450 return 1;
9451}
9452
9453bool SystemZTargetLowering::
9454isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op,
9455 const APInt &DemandedElts, const SelectionDAG &DAG,
9456 bool PoisonOnly, unsigned Depth) const {
9457 switch (Op->getOpcode()) {
9458 case SystemZISD::PCREL_WRAPPER:
9459 case SystemZISD::PCREL_OFFSET:
9460 return true;
9461 }
9462 return false;
9463}
9464
9465unsigned
9466SystemZTargetLowering::getStackProbeSize(const MachineFunction &MF) const {
9467 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
9468 unsigned StackAlign = TFI->getStackAlignment();
9469 assert(StackAlign >=1 && isPowerOf2_32(StackAlign) &&
9470 "Unexpected stack alignment");
9471 // The default stack probe size is 4096 if the function has no
9472 // stack-probe-size attribute.
9473 unsigned StackProbeSize =
9474 MF.getFunction().getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
9475 // Round down to the stack alignment.
9476 StackProbeSize &= ~(StackAlign - 1);
9477 return StackProbeSize ? StackProbeSize : StackAlign;
9478}
9479
9480//===----------------------------------------------------------------------===//
9481// Custom insertion
9482//===----------------------------------------------------------------------===//
9483
9484// Force base value Base into a register before MI. Return the register.
9485static Register forceReg(MachineInstr &MI, MachineOperand &Base,
9486 const SystemZInstrInfo *TII) {
9487 MachineBasicBlock *MBB = MI.getParent();
9488 MachineFunction &MF = *MBB->getParent();
9489 MachineRegisterInfo &MRI = MF.getRegInfo();
9490
9491 if (Base.isReg()) {
9492 // Copy Base into a new virtual register to help register coalescing in
9493 // cases with multiple uses.
9494 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
9495 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::COPY), DestReg: Reg)
9496 .add(MO: Base);
9497 return Reg;
9498 }
9499
9500 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
9501 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::LA), DestReg: Reg)
9502 .add(MO: Base)
9503 .addImm(Val: 0)
9504 .addReg(RegNo: 0);
9505 return Reg;
9506}
9507
9508// The CC operand of MI might be missing a kill marker because there
9509// were multiple uses of CC, and ISel didn't know which to mark.
9510// Figure out whether MI should have had a kill marker.
9511static bool checkCCKill(MachineInstr &MI, MachineBasicBlock *MBB) {
9512 // Scan forward through BB for a use/def of CC.
9513 MachineBasicBlock::iterator miI(std::next(x: MachineBasicBlock::iterator(MI)));
9514 for (MachineBasicBlock::iterator miE = MBB->end(); miI != miE; ++miI) {
9515 const MachineInstr &MI = *miI;
9516 if (MI.readsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr))
9517 return false;
9518 if (MI.definesRegister(Reg: SystemZ::CC, /*TRI=*/nullptr))
9519 break; // Should have kill-flag - update below.
9520 }
9521
9522 // If we hit the end of the block, check whether CC is live into a
9523 // successor.
9524 if (miI == MBB->end()) {
9525 for (const MachineBasicBlock *Succ : MBB->successors())
9526 if (Succ->isLiveIn(Reg: SystemZ::CC))
9527 return false;
9528 }
9529
9530 return true;
9531}
9532
9533// Return true if it is OK for this Select pseudo-opcode to be cascaded
9534// together with other Select pseudo-opcodes into a single basic-block with
9535// a conditional jump around it.
9536static bool isSelectPseudo(MachineInstr &MI) {
9537 switch (MI.getOpcode()) {
9538 case SystemZ::Select32:
9539 case SystemZ::Select64:
9540 case SystemZ::Select128:
9541 case SystemZ::SelectF32:
9542 case SystemZ::SelectF64:
9543 case SystemZ::SelectF128:
9544 case SystemZ::SelectVR32:
9545 case SystemZ::SelectVR64:
9546 case SystemZ::SelectVR128:
9547 return true;
9548
9549 default:
9550 return false;
9551 }
9552}
9553
9554// Helper function, which inserts PHI functions into SinkMBB:
9555// %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
9556// where %FalseValue(i) and %TrueValue(i) are taken from Selects.
9557static void createPHIsForSelects(SmallVector<MachineInstr*, 8> &Selects,
9558 MachineBasicBlock *TrueMBB,
9559 MachineBasicBlock *FalseMBB,
9560 MachineBasicBlock *SinkMBB) {
9561 MachineFunction *MF = TrueMBB->getParent();
9562 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
9563
9564 MachineInstr *FirstMI = Selects.front();
9565 unsigned CCValid = FirstMI->getOperand(i: 3).getImm();
9566 unsigned CCMask = FirstMI->getOperand(i: 4).getImm();
9567
9568 MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
9569
9570 // As we are creating the PHIs, we have to be careful if there is more than
9571 // one. Later Selects may reference the results of earlier Selects, but later
9572 // PHIs have to reference the individual true/false inputs from earlier PHIs.
9573 // That also means that PHI construction must work forward from earlier to
9574 // later, and that the code must maintain a mapping from earlier PHI's
9575 // destination registers, and the registers that went into the PHI.
9576 DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
9577
9578 for (auto *MI : Selects) {
9579 Register DestReg = MI->getOperand(i: 0).getReg();
9580 Register TrueReg = MI->getOperand(i: 1).getReg();
9581 Register FalseReg = MI->getOperand(i: 2).getReg();
9582
9583 // If this Select we are generating is the opposite condition from
9584 // the jump we generated, then we have to swap the operands for the
9585 // PHI that is going to be generated.
9586 if (MI->getOperand(i: 4).getImm() == (CCValid ^ CCMask))
9587 std::swap(a&: TrueReg, b&: FalseReg);
9588
9589 if (auto It = RegRewriteTable.find(Val: TrueReg); It != RegRewriteTable.end())
9590 TrueReg = It->second.first;
9591
9592 if (auto It = RegRewriteTable.find(Val: FalseReg); It != RegRewriteTable.end())
9593 FalseReg = It->second.second;
9594
9595 DebugLoc DL = MI->getDebugLoc();
9596 BuildMI(BB&: *SinkMBB, I: SinkInsertionPoint, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg)
9597 .addReg(RegNo: TrueReg).addMBB(MBB: TrueMBB)
9598 .addReg(RegNo: FalseReg).addMBB(MBB: FalseMBB);
9599
9600 // Add this PHI to the rewrite table.
9601 RegRewriteTable[DestReg] = std::make_pair(x&: TrueReg, y&: FalseReg);
9602 }
9603
9604 MF->getProperties().resetNoPHIs();
9605}
9606
9607MachineBasicBlock *
9608SystemZTargetLowering::emitAdjCallStack(MachineInstr &MI,
9609 MachineBasicBlock *BB) const {
9610 MachineFunction &MF = *BB->getParent();
9611 MachineFrameInfo &MFI = MF.getFrameInfo();
9612 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
9613 assert(TFL->hasReservedCallFrame(MF) &&
9614 "ADJSTACKDOWN and ADJSTACKUP should be no-ops");
9615 (void)TFL;
9616 // Get the MaxCallFrameSize value and erase MI since it serves no further
9617 // purpose as the call frame is statically reserved in the prolog. Set
9618 // AdjustsStack as MI is *not* mapped as a frame instruction.
9619 uint32_t NumBytes = MI.getOperand(i: 0).getImm();
9620 if (NumBytes > MFI.getMaxCallFrameSize())
9621 MFI.setMaxCallFrameSize(NumBytes);
9622 MFI.setAdjustsStack(true);
9623
9624 MI.eraseFromParent();
9625 return BB;
9626}
9627
9628// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI.
9629MachineBasicBlock *
9630SystemZTargetLowering::emitSelect(MachineInstr &MI,
9631 MachineBasicBlock *MBB) const {
9632 assert(isSelectPseudo(MI) && "Bad call to emitSelect()");
9633 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9634
9635 unsigned CCValid = MI.getOperand(i: 3).getImm();
9636 unsigned CCMask = MI.getOperand(i: 4).getImm();
9637
9638 // If we have a sequence of Select* pseudo instructions using the
9639 // same condition code value, we want to expand all of them into
9640 // a single pair of basic blocks using the same condition.
9641 SmallVector<MachineInstr*, 8> Selects;
9642 SmallVector<MachineInstr*, 8> DbgValues;
9643 Selects.push_back(Elt: &MI);
9644 unsigned Count = 0;
9645 for (MachineInstr &NextMI : llvm::make_range(
9646 x: std::next(x: MachineBasicBlock::iterator(MI)), y: MBB->end())) {
9647 if (isSelectPseudo(MI&: NextMI)) {
9648 assert(NextMI.getOperand(3).getImm() == CCValid &&
9649 "Bad CCValid operands since CC was not redefined.");
9650 if (NextMI.getOperand(i: 4).getImm() == CCMask ||
9651 NextMI.getOperand(i: 4).getImm() == (CCValid ^ CCMask)) {
9652 Selects.push_back(Elt: &NextMI);
9653 continue;
9654 }
9655 break;
9656 }
9657 if (NextMI.definesRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) ||
9658 NextMI.usesCustomInsertionHook())
9659 break;
9660 bool User = false;
9661 for (auto *SelMI : Selects)
9662 if (NextMI.readsVirtualRegister(Reg: SelMI->getOperand(i: 0).getReg())) {
9663 User = true;
9664 break;
9665 }
9666 if (NextMI.isDebugInstr()) {
9667 if (User) {
9668 assert(NextMI.isDebugValue() && "Unhandled debug opcode.");
9669 DbgValues.push_back(Elt: &NextMI);
9670 }
9671 } else if (User || ++Count > 20)
9672 break;
9673 }
9674
9675 MachineInstr *LastMI = Selects.back();
9676 bool CCKilled = (LastMI->killsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) ||
9677 checkCCKill(MI&: *LastMI, MBB));
9678 MachineBasicBlock *StartMBB = MBB;
9679 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI: LastMI, MBB);
9680 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
9681
9682 // Unless CC was killed in the last Select instruction, mark it as
9683 // live-in to both FalseMBB and JoinMBB.
9684 if (!CCKilled) {
9685 FalseMBB->addLiveIn(PhysReg: SystemZ::CC);
9686 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
9687 }
9688
9689 // StartMBB:
9690 // BRC CCMask, JoinMBB
9691 // # fallthrough to FalseMBB
9692 MBB = StartMBB;
9693 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::BRC))
9694 .addImm(Val: CCValid).addImm(Val: CCMask).addMBB(MBB: JoinMBB);
9695 MBB->addSuccessor(Succ: JoinMBB);
9696 MBB->addSuccessor(Succ: FalseMBB);
9697
9698 // FalseMBB:
9699 // # fallthrough to JoinMBB
9700 MBB = FalseMBB;
9701 MBB->addSuccessor(Succ: JoinMBB);
9702
9703 // JoinMBB:
9704 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ]
9705 // ...
9706 MBB = JoinMBB;
9707 createPHIsForSelects(Selects, TrueMBB: StartMBB, FalseMBB, SinkMBB: MBB);
9708 for (auto *SelMI : Selects)
9709 SelMI->eraseFromParent();
9710
9711 MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI();
9712 for (auto *DbgMI : DbgValues)
9713 MBB->splice(Where: InsertPos, Other: StartMBB, From: DbgMI);
9714
9715 return JoinMBB;
9716}
9717
9718// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI.
9719// StoreOpcode is the store to use and Invert says whether the store should
9720// happen when the condition is false rather than true. If a STORE ON
9721// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0.
9722MachineBasicBlock *SystemZTargetLowering::emitCondStore(MachineInstr &MI,
9723 MachineBasicBlock *MBB,
9724 unsigned StoreOpcode,
9725 unsigned STOCOpcode,
9726 bool Invert) const {
9727 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9728
9729 Register SrcReg = MI.getOperand(i: 0).getReg();
9730 MachineOperand Base = MI.getOperand(i: 1);
9731 int64_t Disp = MI.getOperand(i: 2).getImm();
9732 Register IndexReg = MI.getOperand(i: 3).getReg();
9733 unsigned CCValid = MI.getOperand(i: 4).getImm();
9734 unsigned CCMask = MI.getOperand(i: 5).getImm();
9735 DebugLoc DL = MI.getDebugLoc();
9736
9737 StoreOpcode = TII->getOpcodeForOffset(Opcode: StoreOpcode, Offset: Disp);
9738
9739 // ISel pattern matching also adds a load memory operand of the same
9740 // address, so take special care to find the storing memory operand.
9741 MachineMemOperand *MMO = nullptr;
9742 for (auto *I : MI.memoperands())
9743 if (I->isStore()) {
9744 MMO = I;
9745 break;
9746 }
9747
9748 // Use STOCOpcode if possible. We could use different store patterns in
9749 // order to avoid matching the index register, but the performance trade-offs
9750 // might be more complicated in that case.
9751 if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) {
9752 if (Invert)
9753 CCMask ^= CCValid;
9754
9755 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: STOCOpcode))
9756 .addReg(RegNo: SrcReg)
9757 .add(MO: Base)
9758 .addImm(Val: Disp)
9759 .addImm(Val: CCValid)
9760 .addImm(Val: CCMask)
9761 .addMemOperand(MMO);
9762
9763 MI.eraseFromParent();
9764 return MBB;
9765 }
9766
9767 // Get the condition needed to branch around the store.
9768 if (!Invert)
9769 CCMask ^= CCValid;
9770
9771 MachineBasicBlock *StartMBB = MBB;
9772 MachineBasicBlock *JoinMBB = SystemZ::splitBlockBefore(MI, MBB);
9773 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
9774
9775 // Unless CC was killed in the CondStore instruction, mark it as
9776 // live-in to both FalseMBB and JoinMBB.
9777 if (!MI.killsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) &&
9778 !checkCCKill(MI, MBB: JoinMBB)) {
9779 FalseMBB->addLiveIn(PhysReg: SystemZ::CC);
9780 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
9781 }
9782
9783 // StartMBB:
9784 // BRC CCMask, JoinMBB
9785 // # fallthrough to FalseMBB
9786 MBB = StartMBB;
9787 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
9788 .addImm(Val: CCValid).addImm(Val: CCMask).addMBB(MBB: JoinMBB);
9789 MBB->addSuccessor(Succ: JoinMBB);
9790 MBB->addSuccessor(Succ: FalseMBB);
9791
9792 // FalseMBB:
9793 // store %SrcReg, %Disp(%Index,%Base)
9794 // # fallthrough to JoinMBB
9795 MBB = FalseMBB;
9796 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: StoreOpcode))
9797 .addReg(RegNo: SrcReg)
9798 .add(MO: Base)
9799 .addImm(Val: Disp)
9800 .addReg(RegNo: IndexReg)
9801 .addMemOperand(MMO);
9802 MBB->addSuccessor(Succ: JoinMBB);
9803
9804 MI.eraseFromParent();
9805 return JoinMBB;
9806}
9807
9808// Implement EmitInstrWithCustomInserter for pseudo [SU]Cmp128Hi instruction MI.
9809MachineBasicBlock *
9810SystemZTargetLowering::emitICmp128Hi(MachineInstr &MI,
9811 MachineBasicBlock *MBB,
9812 bool Unsigned) const {
9813 MachineFunction &MF = *MBB->getParent();
9814 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9815 MachineRegisterInfo &MRI = MF.getRegInfo();
9816
9817 // Synthetic instruction to compare 128-bit values.
9818 // Sets CC 1 if Op0 > Op1, sets a different CC otherwise.
9819 Register Op0 = MI.getOperand(i: 0).getReg();
9820 Register Op1 = MI.getOperand(i: 1).getReg();
9821
9822 MachineBasicBlock *StartMBB = MBB;
9823 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI, MBB);
9824 MachineBasicBlock *HiEqMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
9825
9826 // StartMBB:
9827 //
9828 // Use VECTOR ELEMENT COMPARE [LOGICAL] to compare the high parts.
9829 // Swap the inputs to get:
9830 // CC 1 if high(Op0) > high(Op1)
9831 // CC 2 if high(Op0) < high(Op1)
9832 // CC 0 if high(Op0) == high(Op1)
9833 //
9834 // If CC != 0, we'd done, so jump over the next instruction.
9835 //
9836 // VEC[L]G Op1, Op0
9837 // JNE JoinMBB
9838 // # fallthrough to HiEqMBB
9839 MBB = StartMBB;
9840 int HiOpcode = Unsigned? SystemZ::VECLG : SystemZ::VECG;
9841 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: HiOpcode))
9842 .addReg(RegNo: Op1).addReg(RegNo: Op0);
9843 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::BRC))
9844 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE).addMBB(MBB: JoinMBB);
9845 MBB->addSuccessor(Succ: JoinMBB);
9846 MBB->addSuccessor(Succ: HiEqMBB);
9847
9848 // HiEqMBB:
9849 //
9850 // Otherwise, use VECTOR COMPARE HIGH LOGICAL.
9851 // Since we already know the high parts are equal, the CC
9852 // result will only depend on the low parts:
9853 // CC 1 if low(Op0) > low(Op1)
9854 // CC 3 if low(Op0) <= low(Op1)
9855 //
9856 // VCHLGS Tmp, Op0, Op1
9857 // # fallthrough to JoinMBB
9858 MBB = HiEqMBB;
9859 Register Temp = MRI.createVirtualRegister(RegClass: &SystemZ::VR128BitRegClass);
9860 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::VCHLGS), DestReg: Temp)
9861 .addReg(RegNo: Op0).addReg(RegNo: Op1);
9862 MBB->addSuccessor(Succ: JoinMBB);
9863
9864 // Mark CC as live-in to JoinMBB.
9865 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
9866
9867 MI.eraseFromParent();
9868 return JoinMBB;
9869}
9870
9871// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_LOADW_* or
9872// ATOMIC_SWAPW instruction MI. BinOpcode is the instruction that performs
9873// the binary operation elided by "*", or 0 for ATOMIC_SWAPW. Invert says
9874// whether the field should be inverted after performing BinOpcode (e.g. for
9875// NAND).
9876MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary(
9877 MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode,
9878 bool Invert) const {
9879 MachineFunction &MF = *MBB->getParent();
9880 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9881 MachineRegisterInfo &MRI = MF.getRegInfo();
9882
9883 // Extract the operands. Base can be a register or a frame index.
9884 // Src2 can be a register or immediate.
9885 Register Dest = MI.getOperand(i: 0).getReg();
9886 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
9887 int64_t Disp = MI.getOperand(i: 2).getImm();
9888 MachineOperand Src2 = earlyUseOperand(Op: MI.getOperand(i: 3));
9889 Register BitShift = MI.getOperand(i: 4).getReg();
9890 Register NegBitShift = MI.getOperand(i: 5).getReg();
9891 unsigned BitSize = MI.getOperand(i: 6).getImm();
9892 DebugLoc DL = MI.getDebugLoc();
9893
9894 // Get the right opcodes for the displacement.
9895 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
9896 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
9897 assert(LOpcode && CSOpcode && "Displacement out of range");
9898
9899 // Create virtual registers for temporary results.
9900 Register OrigVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
9901 Register OldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
9902 Register NewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
9903 Register RotatedOldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
9904 Register RotatedNewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
9905
9906 // Insert a basic block for the main loop.
9907 MachineBasicBlock *StartMBB = MBB;
9908 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
9909 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
9910
9911 // StartMBB:
9912 // ...
9913 // %OrigVal = L Disp(%Base)
9914 // # fall through to LoopMBB
9915 MBB = StartMBB;
9916 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigVal).add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
9917 MBB->addSuccessor(Succ: LoopMBB);
9918
9919 // LoopMBB:
9920 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ]
9921 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
9922 // %RotatedNewVal = OP %RotatedOldVal, %Src2
9923 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
9924 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
9925 // JNE LoopMBB
9926 // # fall through to DoneMBB
9927 MBB = LoopMBB;
9928 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
9929 .addReg(RegNo: OrigVal).addMBB(MBB: StartMBB)
9930 .addReg(RegNo: Dest).addMBB(MBB: LoopMBB);
9931 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: RotatedOldVal)
9932 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: 0);
9933 if (Invert) {
9934 // Perform the operation normally and then invert every bit of the field.
9935 Register Tmp = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
9936 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: BinOpcode), DestReg: Tmp).addReg(RegNo: RotatedOldVal).add(MO: Src2);
9937 // XILF with the upper BitSize bits set.
9938 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::XILF), DestReg: RotatedNewVal)
9939 .addReg(RegNo: Tmp).addImm(Val: -1U << (32 - BitSize));
9940 } else if (BinOpcode)
9941 // A simply binary operation.
9942 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: BinOpcode), DestReg: RotatedNewVal)
9943 .addReg(RegNo: RotatedOldVal)
9944 .add(MO: Src2);
9945 else
9946 // Use RISBG to rotate Src2 into position and use it to replace the
9947 // field in RotatedOldVal.
9948 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RotatedNewVal)
9949 .addReg(RegNo: RotatedOldVal).addReg(RegNo: Src2.getReg())
9950 .addImm(Val: 32).addImm(Val: 31 + BitSize).addImm(Val: 32 - BitSize);
9951 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: NewVal)
9952 .addReg(RegNo: RotatedNewVal).addReg(RegNo: NegBitShift).addImm(Val: 0);
9953 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: Dest)
9954 .addReg(RegNo: OldVal)
9955 .addReg(RegNo: NewVal)
9956 .add(MO: Base)
9957 .addImm(Val: Disp);
9958 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
9959 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
9960 MBB->addSuccessor(Succ: LoopMBB);
9961 MBB->addSuccessor(Succ: DoneMBB);
9962
9963 MI.eraseFromParent();
9964 return DoneMBB;
9965}
9966
9967// Implement EmitInstrWithCustomInserter for subword pseudo
9968// ATOMIC_LOADW_{,U}{MIN,MAX} instruction MI. CompareOpcode is the
9969// instruction that should be used to compare the current field with the
9970// minimum or maximum value. KeepOldMask is the BRC condition-code mask
9971// for when the current field should be kept.
9972MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
9973 MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode,
9974 unsigned KeepOldMask) const {
9975 MachineFunction &MF = *MBB->getParent();
9976 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9977 MachineRegisterInfo &MRI = MF.getRegInfo();
9978
9979 // Extract the operands. Base can be a register or a frame index.
9980 Register Dest = MI.getOperand(i: 0).getReg();
9981 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
9982 int64_t Disp = MI.getOperand(i: 2).getImm();
9983 Register Src2 = MI.getOperand(i: 3).getReg();
9984 Register BitShift = MI.getOperand(i: 4).getReg();
9985 Register NegBitShift = MI.getOperand(i: 5).getReg();
9986 unsigned BitSize = MI.getOperand(i: 6).getImm();
9987 DebugLoc DL = MI.getDebugLoc();
9988
9989 // Get the right opcodes for the displacement.
9990 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
9991 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
9992 assert(LOpcode && CSOpcode && "Displacement out of range");
9993
9994 // Create virtual registers for temporary results.
9995 Register OrigVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
9996 Register OldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
9997 Register NewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
9998 Register RotatedOldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
9999 Register RotatedAltVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10000 Register RotatedNewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10001
10002 // Insert 3 basic blocks for the loop.
10003 MachineBasicBlock *StartMBB = MBB;
10004 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10005 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10006 MachineBasicBlock *UseAltMBB = SystemZ::emitBlockAfter(MBB: LoopMBB);
10007 MachineBasicBlock *UpdateMBB = SystemZ::emitBlockAfter(MBB: UseAltMBB);
10008
10009 // StartMBB:
10010 // ...
10011 // %OrigVal = L Disp(%Base)
10012 // # fall through to LoopMBB
10013 MBB = StartMBB;
10014 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigVal).add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10015 MBB->addSuccessor(Succ: LoopMBB);
10016
10017 // LoopMBB:
10018 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ]
10019 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
10020 // CompareOpcode %RotatedOldVal, %Src2
10021 // BRC KeepOldMask, UpdateMBB
10022 MBB = LoopMBB;
10023 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10024 .addReg(RegNo: OrigVal).addMBB(MBB: StartMBB)
10025 .addReg(RegNo: Dest).addMBB(MBB: UpdateMBB);
10026 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: RotatedOldVal)
10027 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: 0);
10028 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CompareOpcode))
10029 .addReg(RegNo: RotatedOldVal).addReg(RegNo: Src2);
10030 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10031 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: KeepOldMask).addMBB(MBB: UpdateMBB);
10032 MBB->addSuccessor(Succ: UpdateMBB);
10033 MBB->addSuccessor(Succ: UseAltMBB);
10034
10035 // UseAltMBB:
10036 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0
10037 // # fall through to UpdateMBB
10038 MBB = UseAltMBB;
10039 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RotatedAltVal)
10040 .addReg(RegNo: RotatedOldVal).addReg(RegNo: Src2)
10041 .addImm(Val: 32).addImm(Val: 31 + BitSize).addImm(Val: 0);
10042 MBB->addSuccessor(Succ: UpdateMBB);
10043
10044 // UpdateMBB:
10045 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ],
10046 // [ %RotatedAltVal, UseAltMBB ]
10047 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
10048 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
10049 // JNE LoopMBB
10050 // # fall through to DoneMBB
10051 MBB = UpdateMBB;
10052 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RotatedNewVal)
10053 .addReg(RegNo: RotatedOldVal).addMBB(MBB: LoopMBB)
10054 .addReg(RegNo: RotatedAltVal).addMBB(MBB: UseAltMBB);
10055 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: NewVal)
10056 .addReg(RegNo: RotatedNewVal).addReg(RegNo: NegBitShift).addImm(Val: 0);
10057 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: Dest)
10058 .addReg(RegNo: OldVal)
10059 .addReg(RegNo: NewVal)
10060 .add(MO: Base)
10061 .addImm(Val: Disp);
10062 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10063 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10064 MBB->addSuccessor(Succ: LoopMBB);
10065 MBB->addSuccessor(Succ: DoneMBB);
10066
10067 MI.eraseFromParent();
10068 return DoneMBB;
10069}
10070
10071// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_CMP_SWAPW
10072// instruction MI.
10073MachineBasicBlock *
10074SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
10075 MachineBasicBlock *MBB) const {
10076 MachineFunction &MF = *MBB->getParent();
10077 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10078 MachineRegisterInfo &MRI = MF.getRegInfo();
10079
10080 // Extract the operands. Base can be a register or a frame index.
10081 Register Dest = MI.getOperand(i: 0).getReg();
10082 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10083 int64_t Disp = MI.getOperand(i: 2).getImm();
10084 Register CmpVal = MI.getOperand(i: 3).getReg();
10085 Register OrigSwapVal = MI.getOperand(i: 4).getReg();
10086 Register BitShift = MI.getOperand(i: 5).getReg();
10087 Register NegBitShift = MI.getOperand(i: 6).getReg();
10088 int64_t BitSize = MI.getOperand(i: 7).getImm();
10089 DebugLoc DL = MI.getDebugLoc();
10090
10091 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
10092
10093 // Get the right opcodes for the displacement and zero-extension.
10094 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10095 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10096 unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
10097 assert(LOpcode && CSOpcode && "Displacement out of range");
10098
10099 // Create virtual registers for temporary results.
10100 Register OrigOldVal = MRI.createVirtualRegister(RegClass: RC);
10101 Register OldVal = MRI.createVirtualRegister(RegClass: RC);
10102 Register SwapVal = MRI.createVirtualRegister(RegClass: RC);
10103 Register StoreVal = MRI.createVirtualRegister(RegClass: RC);
10104 Register OldValRot = MRI.createVirtualRegister(RegClass: RC);
10105 Register RetryOldVal = MRI.createVirtualRegister(RegClass: RC);
10106 Register RetrySwapVal = MRI.createVirtualRegister(RegClass: RC);
10107
10108 // Insert 2 basic blocks for the loop.
10109 MachineBasicBlock *StartMBB = MBB;
10110 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10111 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10112 MachineBasicBlock *SetMBB = SystemZ::emitBlockAfter(MBB: LoopMBB);
10113
10114 // StartMBB:
10115 // ...
10116 // %OrigOldVal = L Disp(%Base)
10117 // # fall through to LoopMBB
10118 MBB = StartMBB;
10119 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigOldVal)
10120 .add(MO: Base)
10121 .addImm(Val: Disp)
10122 .addReg(RegNo: 0);
10123 MBB->addSuccessor(Succ: LoopMBB);
10124
10125 // LoopMBB:
10126 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
10127 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
10128 // %OldValRot = RLL %OldVal, BitSize(%BitShift)
10129 // ^^ The low BitSize bits contain the field
10130 // of interest.
10131 // %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0
10132 // ^^ Replace the upper 32-BitSize bits of the
10133 // swap value with those that we loaded and rotated.
10134 // %Dest = LL[CH] %OldValRot
10135 // CR %Dest, %CmpVal
10136 // JNE DoneMBB
10137 // # Fall through to SetMBB
10138 MBB = LoopMBB;
10139 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10140 .addReg(RegNo: OrigOldVal).addMBB(MBB: StartMBB)
10141 .addReg(RegNo: RetryOldVal).addMBB(MBB: SetMBB);
10142 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: SwapVal)
10143 .addReg(RegNo: OrigSwapVal).addMBB(MBB: StartMBB)
10144 .addReg(RegNo: RetrySwapVal).addMBB(MBB: SetMBB);
10145 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: OldValRot)
10146 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: BitSize);
10147 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RetrySwapVal)
10148 .addReg(RegNo: SwapVal).addReg(RegNo: OldValRot).addImm(Val: 32).addImm(Val: 63 - BitSize).addImm(Val: 0);
10149 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: ZExtOpcode), DestReg: Dest)
10150 .addReg(RegNo: OldValRot);
10151 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CR))
10152 .addReg(RegNo: Dest).addReg(RegNo: CmpVal);
10153 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10154 .addImm(Val: SystemZ::CCMASK_ICMP)
10155 .addImm(Val: SystemZ::CCMASK_CMP_NE).addMBB(MBB: DoneMBB);
10156 MBB->addSuccessor(Succ: DoneMBB);
10157 MBB->addSuccessor(Succ: SetMBB);
10158
10159 // SetMBB:
10160 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
10161 // ^^ Rotate the new field to its proper position.
10162 // %RetryOldVal = CS %OldVal, %StoreVal, Disp(%Base)
10163 // JNE LoopMBB
10164 // # fall through to ExitMBB
10165 MBB = SetMBB;
10166 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: StoreVal)
10167 .addReg(RegNo: RetrySwapVal).addReg(RegNo: NegBitShift).addImm(Val: -BitSize);
10168 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: RetryOldVal)
10169 .addReg(RegNo: OldVal)
10170 .addReg(RegNo: StoreVal)
10171 .add(MO: Base)
10172 .addImm(Val: Disp);
10173 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10174 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10175 MBB->addSuccessor(Succ: LoopMBB);
10176 MBB->addSuccessor(Succ: DoneMBB);
10177
10178 // If the CC def wasn't dead in the ATOMIC_CMP_SWAPW, mark CC as live-in
10179 // to the block after the loop. At this point, CC may have been defined
10180 // either by the CR in LoopMBB or by the CS in SetMBB.
10181 if (!MI.registerDefIsDead(Reg: SystemZ::CC, /*TRI=*/nullptr))
10182 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10183
10184 MI.eraseFromParent();
10185 return DoneMBB;
10186}
10187
10188// Emit a move from two GR64s to a GR128.
10189MachineBasicBlock *
10190SystemZTargetLowering::emitPair128(MachineInstr &MI,
10191 MachineBasicBlock *MBB) const {
10192 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10193 const DebugLoc &DL = MI.getDebugLoc();
10194
10195 Register Dest = MI.getOperand(i: 0).getReg();
10196 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest)
10197 .add(MO: MI.getOperand(i: 1))
10198 .addImm(Val: SystemZ::subreg_h64)
10199 .add(MO: MI.getOperand(i: 2))
10200 .addImm(Val: SystemZ::subreg_l64);
10201 MI.eraseFromParent();
10202 return MBB;
10203}
10204
10205// Emit an extension from a GR64 to a GR128. ClearEven is true
10206// if the high register of the GR128 value must be cleared or false if
10207// it's "don't care".
10208MachineBasicBlock *SystemZTargetLowering::emitExt128(MachineInstr &MI,
10209 MachineBasicBlock *MBB,
10210 bool ClearEven) const {
10211 MachineFunction &MF = *MBB->getParent();
10212 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10213 MachineRegisterInfo &MRI = MF.getRegInfo();
10214 DebugLoc DL = MI.getDebugLoc();
10215
10216 Register Dest = MI.getOperand(i: 0).getReg();
10217 Register Src = MI.getOperand(i: 1).getReg();
10218 Register In128 = MRI.createVirtualRegister(RegClass: &SystemZ::GR128BitRegClass);
10219
10220 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: In128);
10221 if (ClearEven) {
10222 Register NewIn128 = MRI.createVirtualRegister(RegClass: &SystemZ::GR128BitRegClass);
10223 Register Zero64 = MRI.createVirtualRegister(RegClass: &SystemZ::GR64BitRegClass);
10224
10225 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LLILL), DestReg: Zero64)
10226 .addImm(Val: 0);
10227 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: NewIn128)
10228 .addReg(RegNo: In128).addReg(RegNo: Zero64).addImm(Val: SystemZ::subreg_h64);
10229 In128 = NewIn128;
10230 }
10231 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: Dest)
10232 .addReg(RegNo: In128).addReg(RegNo: Src).addImm(Val: SystemZ::subreg_l64);
10233
10234 MI.eraseFromParent();
10235 return MBB;
10236}
10237
10238MachineBasicBlock *
10239SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI,
10240 MachineBasicBlock *MBB,
10241 unsigned Opcode, bool IsMemset) const {
10242 MachineFunction &MF = *MBB->getParent();
10243 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10244 MachineRegisterInfo &MRI = MF.getRegInfo();
10245 DebugLoc DL = MI.getDebugLoc();
10246
10247 MachineOperand DestBase = earlyUseOperand(Op: MI.getOperand(i: 0));
10248 uint64_t DestDisp = MI.getOperand(i: 1).getImm();
10249 MachineOperand SrcBase = MachineOperand::CreateReg(Reg: 0U, isDef: false);
10250 uint64_t SrcDisp;
10251
10252 // Fold the displacement Disp if it is out of range.
10253 auto foldDisplIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void {
10254 if (!isUInt<12>(x: Disp)) {
10255 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10256 unsigned Opcode = TII->getOpcodeForOffset(Opcode: SystemZ::LA, Offset: Disp);
10257 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode), DestReg: Reg)
10258 .add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10259 Base = MachineOperand::CreateReg(Reg, isDef: false);
10260 Disp = 0;
10261 }
10262 };
10263
10264 if (!IsMemset) {
10265 SrcBase = earlyUseOperand(Op: MI.getOperand(i: 2));
10266 SrcDisp = MI.getOperand(i: 3).getImm();
10267 } else {
10268 SrcBase = DestBase;
10269 SrcDisp = DestDisp++;
10270 foldDisplIfNeeded(DestBase, DestDisp);
10271 }
10272
10273 MachineOperand &LengthMO = MI.getOperand(i: IsMemset ? 2 : 4);
10274 bool IsImmForm = LengthMO.isImm();
10275 bool IsRegForm = !IsImmForm;
10276
10277 // Build and insert one Opcode of Length, with special treatment for memset.
10278 auto insertMemMemOp = [&](MachineBasicBlock *InsMBB,
10279 MachineBasicBlock::iterator InsPos,
10280 MachineOperand DBase, uint64_t DDisp,
10281 MachineOperand SBase, uint64_t SDisp,
10282 unsigned Length) -> void {
10283 assert(Length > 0 && Length <= 256 && "Building memory op with bad length.");
10284 if (IsMemset) {
10285 MachineOperand ByteMO = earlyUseOperand(Op: MI.getOperand(i: 3));
10286 if (ByteMO.isImm())
10287 BuildMI(BB&: *InsMBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode: SystemZ::MVI))
10288 .add(MO: SBase).addImm(Val: SDisp).add(MO: ByteMO);
10289 else
10290 BuildMI(BB&: *InsMBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STC))
10291 .add(MO: ByteMO).add(MO: SBase).addImm(Val: SDisp).addReg(RegNo: 0);
10292 if (--Length == 0)
10293 return;
10294 }
10295 BuildMI(BB&: *MBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode))
10296 .add(MO: DBase).addImm(Val: DDisp).addImm(Val: Length)
10297 .add(MO: SBase).addImm(Val: SDisp)
10298 .setMemRefs(MI.memoperands());
10299 };
10300
10301 bool NeedsLoop = false;
10302 uint64_t ImmLength = 0;
10303 Register LenAdjReg = SystemZ::NoRegister;
10304 if (IsImmForm) {
10305 ImmLength = LengthMO.getImm();
10306 ImmLength += IsMemset ? 2 : 1; // Add back the subtracted adjustment.
10307 if (ImmLength == 0) {
10308 MI.eraseFromParent();
10309 return MBB;
10310 }
10311 if (Opcode == SystemZ::CLC) {
10312 if (ImmLength > 3 * 256)
10313 // A two-CLC sequence is a clear win over a loop, not least because
10314 // it needs only one branch. A three-CLC sequence needs the same
10315 // number of branches as a loop (i.e. 2), but is shorter. That
10316 // brings us to lengths greater than 768 bytes. It seems relatively
10317 // likely that a difference will be found within the first 768 bytes,
10318 // so we just optimize for the smallest number of branch
10319 // instructions, in order to avoid polluting the prediction buffer
10320 // too much.
10321 NeedsLoop = true;
10322 } else if (ImmLength > 6 * 256)
10323 // The heuristic we use is to prefer loops for anything that would
10324 // require 7 or more MVCs. With these kinds of sizes there isn't much
10325 // to choose between straight-line code and looping code, since the
10326 // time will be dominated by the MVCs themselves.
10327 NeedsLoop = true;
10328 } else {
10329 NeedsLoop = true;
10330 LenAdjReg = LengthMO.getReg();
10331 }
10332
10333 // When generating more than one CLC, all but the last will need to
10334 // branch to the end when a difference is found.
10335 MachineBasicBlock *EndMBB =
10336 (Opcode == SystemZ::CLC && (ImmLength > 256 || NeedsLoop)
10337 ? SystemZ::splitBlockAfter(MI, MBB)
10338 : nullptr);
10339
10340 if (NeedsLoop) {
10341 Register StartCountReg =
10342 MRI.createVirtualRegister(RegClass: &SystemZ::GR64BitRegClass);
10343 if (IsImmForm) {
10344 TII->loadImmediate(MBB&: *MBB, MBBI: MI, Reg: StartCountReg, Value: ImmLength / 256);
10345 ImmLength &= 255;
10346 } else {
10347 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SRLG), DestReg: StartCountReg)
10348 .addReg(RegNo: LenAdjReg)
10349 .addReg(RegNo: 0)
10350 .addImm(Val: 8);
10351 }
10352
10353 bool HaveSingleBase = DestBase.isIdenticalTo(Other: SrcBase);
10354 auto loadZeroAddress = [&]() -> MachineOperand {
10355 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10356 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LGHI), DestReg: Reg).addImm(Val: 0);
10357 return MachineOperand::CreateReg(Reg, isDef: false);
10358 };
10359 if (DestBase.isReg() && DestBase.getReg() == SystemZ::NoRegister)
10360 DestBase = loadZeroAddress();
10361 if (SrcBase.isReg() && SrcBase.getReg() == SystemZ::NoRegister)
10362 SrcBase = HaveSingleBase ? DestBase : loadZeroAddress();
10363
10364 MachineBasicBlock *StartMBB = nullptr;
10365 MachineBasicBlock *LoopMBB = nullptr;
10366 MachineBasicBlock *NextMBB = nullptr;
10367 MachineBasicBlock *DoneMBB = nullptr;
10368 MachineBasicBlock *AllDoneMBB = nullptr;
10369
10370 Register StartSrcReg = forceReg(MI, Base&: SrcBase, TII);
10371 Register StartDestReg =
10372 (HaveSingleBase ? StartSrcReg : forceReg(MI, Base&: DestBase, TII));
10373
10374 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
10375 Register ThisSrcReg = MRI.createVirtualRegister(RegClass: RC);
10376 Register ThisDestReg =
10377 (HaveSingleBase ? ThisSrcReg : MRI.createVirtualRegister(RegClass: RC));
10378 Register NextSrcReg = MRI.createVirtualRegister(RegClass: RC);
10379 Register NextDestReg =
10380 (HaveSingleBase ? NextSrcReg : MRI.createVirtualRegister(RegClass: RC));
10381 RC = &SystemZ::GR64BitRegClass;
10382 Register ThisCountReg = MRI.createVirtualRegister(RegClass: RC);
10383 Register NextCountReg = MRI.createVirtualRegister(RegClass: RC);
10384
10385 if (IsRegForm) {
10386 AllDoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10387 StartMBB = SystemZ::emitBlockAfter(MBB);
10388 LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10389 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(MBB: LoopMBB) : LoopMBB);
10390 DoneMBB = SystemZ::emitBlockAfter(MBB: NextMBB);
10391
10392 // MBB:
10393 // # Jump to AllDoneMBB if LenAdjReg means 0, or fall thru to StartMBB.
10394 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10395 .addReg(RegNo: LenAdjReg).addImm(Val: IsMemset ? -2 : -1);
10396 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10397 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10398 .addMBB(MBB: AllDoneMBB);
10399 MBB->addSuccessor(Succ: AllDoneMBB);
10400 if (!IsMemset)
10401 MBB->addSuccessor(Succ: StartMBB);
10402 else {
10403 // MemsetOneCheckMBB:
10404 // # Jump to MemsetOneMBB for a memset of length 1, or
10405 // # fall thru to StartMBB.
10406 MachineBasicBlock *MemsetOneCheckMBB = SystemZ::emitBlockAfter(MBB);
10407 MachineBasicBlock *MemsetOneMBB = SystemZ::emitBlockAfter(MBB: &*MF.rbegin());
10408 MBB->addSuccessor(Succ: MemsetOneCheckMBB);
10409 MBB = MemsetOneCheckMBB;
10410 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10411 .addReg(RegNo: LenAdjReg).addImm(Val: -1);
10412 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10413 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10414 .addMBB(MBB: MemsetOneMBB);
10415 MBB->addSuccessor(Succ: MemsetOneMBB, Prob: {10, 100});
10416 MBB->addSuccessor(Succ: StartMBB, Prob: {90, 100});
10417
10418 // MemsetOneMBB:
10419 // # Jump back to AllDoneMBB after a single MVI or STC.
10420 MBB = MemsetOneMBB;
10421 insertMemMemOp(MBB, MBB->end(),
10422 MachineOperand::CreateReg(Reg: StartDestReg, isDef: false), DestDisp,
10423 MachineOperand::CreateReg(Reg: StartSrcReg, isDef: false), SrcDisp,
10424 1);
10425 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: AllDoneMBB);
10426 MBB->addSuccessor(Succ: AllDoneMBB);
10427 }
10428
10429 // StartMBB:
10430 // # Jump to DoneMBB if %StartCountReg is zero, or fall through to LoopMBB.
10431 MBB = StartMBB;
10432 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10433 .addReg(RegNo: StartCountReg).addImm(Val: 0);
10434 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10435 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10436 .addMBB(MBB: DoneMBB);
10437 MBB->addSuccessor(Succ: DoneMBB);
10438 MBB->addSuccessor(Succ: LoopMBB);
10439 }
10440 else {
10441 StartMBB = MBB;
10442 DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10443 LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10444 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(MBB: LoopMBB) : LoopMBB);
10445
10446 // StartMBB:
10447 // # fall through to LoopMBB
10448 MBB->addSuccessor(Succ: LoopMBB);
10449
10450 DestBase = MachineOperand::CreateReg(Reg: NextDestReg, isDef: false);
10451 SrcBase = MachineOperand::CreateReg(Reg: NextSrcReg, isDef: false);
10452 if (EndMBB && !ImmLength)
10453 // If the loop handled the whole CLC range, DoneMBB will be empty with
10454 // CC live-through into EndMBB, so add it as live-in.
10455 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10456 }
10457
10458 // LoopMBB:
10459 // %ThisDestReg = phi [ %StartDestReg, StartMBB ],
10460 // [ %NextDestReg, NextMBB ]
10461 // %ThisSrcReg = phi [ %StartSrcReg, StartMBB ],
10462 // [ %NextSrcReg, NextMBB ]
10463 // %ThisCountReg = phi [ %StartCountReg, StartMBB ],
10464 // [ %NextCountReg, NextMBB ]
10465 // ( PFD 2, 768+DestDisp(%ThisDestReg) )
10466 // Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg)
10467 // ( JLH EndMBB )
10468 //
10469 // The prefetch is used only for MVC. The JLH is used only for CLC.
10470 MBB = LoopMBB;
10471 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisDestReg)
10472 .addReg(RegNo: StartDestReg).addMBB(MBB: StartMBB)
10473 .addReg(RegNo: NextDestReg).addMBB(MBB: NextMBB);
10474 if (!HaveSingleBase)
10475 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisSrcReg)
10476 .addReg(RegNo: StartSrcReg).addMBB(MBB: StartMBB)
10477 .addReg(RegNo: NextSrcReg).addMBB(MBB: NextMBB);
10478 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisCountReg)
10479 .addReg(RegNo: StartCountReg).addMBB(MBB: StartMBB)
10480 .addReg(RegNo: NextCountReg).addMBB(MBB: NextMBB);
10481 if (Opcode == SystemZ::MVC)
10482 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PFD))
10483 .addImm(Val: SystemZ::PFD_WRITE)
10484 .addReg(RegNo: ThisDestReg).addImm(Val: DestDisp - IsMemset + 768).addReg(RegNo: 0);
10485 insertMemMemOp(MBB, MBB->end(),
10486 MachineOperand::CreateReg(Reg: ThisDestReg, isDef: false), DestDisp,
10487 MachineOperand::CreateReg(Reg: ThisSrcReg, isDef: false), SrcDisp, 256);
10488 if (EndMBB) {
10489 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10490 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10491 .addMBB(MBB: EndMBB);
10492 MBB->addSuccessor(Succ: EndMBB);
10493 MBB->addSuccessor(Succ: NextMBB);
10494 }
10495
10496 // NextMBB:
10497 // %NextDestReg = LA 256(%ThisDestReg)
10498 // %NextSrcReg = LA 256(%ThisSrcReg)
10499 // %NextCountReg = AGHI %ThisCountReg, -1
10500 // CGHI %NextCountReg, 0
10501 // JLH LoopMBB
10502 // # fall through to DoneMBB
10503 //
10504 // The AGHI, CGHI and JLH should be converted to BRCTG by later passes.
10505 MBB = NextMBB;
10506 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LA), DestReg: NextDestReg)
10507 .addReg(RegNo: ThisDestReg).addImm(Val: 256).addReg(RegNo: 0);
10508 if (!HaveSingleBase)
10509 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LA), DestReg: NextSrcReg)
10510 .addReg(RegNo: ThisSrcReg).addImm(Val: 256).addReg(RegNo: 0);
10511 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::AGHI), DestReg: NextCountReg)
10512 .addReg(RegNo: ThisCountReg).addImm(Val: -1);
10513 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10514 .addReg(RegNo: NextCountReg).addImm(Val: 0);
10515 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10516 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10517 .addMBB(MBB: LoopMBB);
10518 MBB->addSuccessor(Succ: LoopMBB);
10519 MBB->addSuccessor(Succ: DoneMBB);
10520
10521 MBB = DoneMBB;
10522 if (IsRegForm) {
10523 // DoneMBB:
10524 // # Make PHIs for RemDestReg/RemSrcReg as the loop may or may not run.
10525 // # Use EXecute Relative Long for the remainder of the bytes. The target
10526 // instruction of the EXRL will have a length field of 1 since 0 is an
10527 // illegal value. The number of bytes processed becomes (%LenAdjReg &
10528 // 0xff) + 1.
10529 // # Fall through to AllDoneMBB.
10530 Register RemSrcReg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10531 Register RemDestReg = HaveSingleBase ? RemSrcReg
10532 : MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10533 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RemDestReg)
10534 .addReg(RegNo: StartDestReg).addMBB(MBB: StartMBB)
10535 .addReg(RegNo: NextDestReg).addMBB(MBB: NextMBB);
10536 if (!HaveSingleBase)
10537 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RemSrcReg)
10538 .addReg(RegNo: StartSrcReg).addMBB(MBB: StartMBB)
10539 .addReg(RegNo: NextSrcReg).addMBB(MBB: NextMBB);
10540 if (IsMemset)
10541 insertMemMemOp(MBB, MBB->end(),
10542 MachineOperand::CreateReg(Reg: RemDestReg, isDef: false), DestDisp,
10543 MachineOperand::CreateReg(Reg: RemSrcReg, isDef: false), SrcDisp, 1);
10544 MachineInstrBuilder EXRL_MIB =
10545 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::EXRL_Pseudo))
10546 .addImm(Val: Opcode)
10547 .addReg(RegNo: LenAdjReg)
10548 .addReg(RegNo: RemDestReg).addImm(Val: DestDisp)
10549 .addReg(RegNo: RemSrcReg).addImm(Val: SrcDisp);
10550 MBB->addSuccessor(Succ: AllDoneMBB);
10551 MBB = AllDoneMBB;
10552 if (Opcode != SystemZ::MVC) {
10553 EXRL_MIB.addReg(RegNo: SystemZ::CC, flags: RegState::ImplicitDefine);
10554 if (EndMBB)
10555 MBB->addLiveIn(PhysReg: SystemZ::CC);
10556 }
10557 }
10558 MF.getProperties().resetNoPHIs();
10559 }
10560
10561 // Handle any remaining bytes with straight-line code.
10562 while (ImmLength > 0) {
10563 uint64_t ThisLength = std::min(a: ImmLength, b: uint64_t(256));
10564 // The previous iteration might have created out-of-range displacements.
10565 // Apply them using LA/LAY if so.
10566 foldDisplIfNeeded(DestBase, DestDisp);
10567 foldDisplIfNeeded(SrcBase, SrcDisp);
10568 insertMemMemOp(MBB, MI, DestBase, DestDisp, SrcBase, SrcDisp, ThisLength);
10569 DestDisp += ThisLength;
10570 SrcDisp += ThisLength;
10571 ImmLength -= ThisLength;
10572 // If there's another CLC to go, branch to the end if a difference
10573 // was found.
10574 if (EndMBB && ImmLength > 0) {
10575 MachineBasicBlock *NextMBB = SystemZ::splitBlockBefore(MI, MBB);
10576 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10577 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10578 .addMBB(MBB: EndMBB);
10579 MBB->addSuccessor(Succ: EndMBB);
10580 MBB->addSuccessor(Succ: NextMBB);
10581 MBB = NextMBB;
10582 }
10583 }
10584 if (EndMBB) {
10585 MBB->addSuccessor(Succ: EndMBB);
10586 MBB = EndMBB;
10587 MBB->addLiveIn(PhysReg: SystemZ::CC);
10588 }
10589
10590 MI.eraseFromParent();
10591 return MBB;
10592}
10593
10594// Decompose string pseudo-instruction MI into a loop that continually performs
10595// Opcode until CC != 3.
10596MachineBasicBlock *SystemZTargetLowering::emitStringWrapper(
10597 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
10598 MachineFunction &MF = *MBB->getParent();
10599 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10600 MachineRegisterInfo &MRI = MF.getRegInfo();
10601 DebugLoc DL = MI.getDebugLoc();
10602
10603 uint64_t End1Reg = MI.getOperand(i: 0).getReg();
10604 uint64_t Start1Reg = MI.getOperand(i: 1).getReg();
10605 uint64_t Start2Reg = MI.getOperand(i: 2).getReg();
10606 uint64_t CharReg = MI.getOperand(i: 3).getReg();
10607
10608 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass;
10609 uint64_t This1Reg = MRI.createVirtualRegister(RegClass: RC);
10610 uint64_t This2Reg = MRI.createVirtualRegister(RegClass: RC);
10611 uint64_t End2Reg = MRI.createVirtualRegister(RegClass: RC);
10612
10613 MachineBasicBlock *StartMBB = MBB;
10614 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10615 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10616
10617 // StartMBB:
10618 // # fall through to LoopMBB
10619 MBB->addSuccessor(Succ: LoopMBB);
10620
10621 // LoopMBB:
10622 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ]
10623 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ]
10624 // R0L = %CharReg
10625 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L
10626 // JO LoopMBB
10627 // # fall through to DoneMBB
10628 //
10629 // The load of R0L can be hoisted by post-RA LICM.
10630 MBB = LoopMBB;
10631
10632 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: This1Reg)
10633 .addReg(RegNo: Start1Reg).addMBB(MBB: StartMBB)
10634 .addReg(RegNo: End1Reg).addMBB(MBB: LoopMBB);
10635 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: This2Reg)
10636 .addReg(RegNo: Start2Reg).addMBB(MBB: StartMBB)
10637 .addReg(RegNo: End2Reg).addMBB(MBB: LoopMBB);
10638 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: SystemZ::R0L).addReg(RegNo: CharReg);
10639 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode))
10640 .addReg(RegNo: End1Reg, flags: RegState::Define).addReg(RegNo: End2Reg, flags: RegState::Define)
10641 .addReg(RegNo: This1Reg).addReg(RegNo: This2Reg);
10642 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10643 .addImm(Val: SystemZ::CCMASK_ANY).addImm(Val: SystemZ::CCMASK_3).addMBB(MBB: LoopMBB);
10644 MBB->addSuccessor(Succ: LoopMBB);
10645 MBB->addSuccessor(Succ: DoneMBB);
10646
10647 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10648
10649 MI.eraseFromParent();
10650 return DoneMBB;
10651}
10652
10653// Update TBEGIN instruction with final opcode and register clobbers.
10654MachineBasicBlock *SystemZTargetLowering::emitTransactionBegin(
10655 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode,
10656 bool NoFloat) const {
10657 MachineFunction &MF = *MBB->getParent();
10658 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
10659 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10660
10661 // Update opcode.
10662 MI.setDesc(TII->get(Opcode));
10663
10664 // We cannot handle a TBEGIN that clobbers the stack or frame pointer.
10665 // Make sure to add the corresponding GRSM bits if they are missing.
10666 uint64_t Control = MI.getOperand(i: 2).getImm();
10667 static const unsigned GPRControlBit[16] = {
10668 0x8000, 0x8000, 0x4000, 0x4000, 0x2000, 0x2000, 0x1000, 0x1000,
10669 0x0800, 0x0800, 0x0400, 0x0400, 0x0200, 0x0200, 0x0100, 0x0100
10670 };
10671 Control |= GPRControlBit[15];
10672 if (TFI->hasFP(MF))
10673 Control |= GPRControlBit[11];
10674 MI.getOperand(i: 2).setImm(Control);
10675
10676 // Add GPR clobbers.
10677 for (int I = 0; I < 16; I++) {
10678 if ((Control & GPRControlBit[I]) == 0) {
10679 unsigned Reg = SystemZMC::GR64Regs[I];
10680 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
10681 }
10682 }
10683
10684 // Add FPR/VR clobbers.
10685 if (!NoFloat && (Control & 4) != 0) {
10686 if (Subtarget.hasVector()) {
10687 for (unsigned Reg : SystemZMC::VR128Regs) {
10688 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
10689 }
10690 } else {
10691 for (unsigned Reg : SystemZMC::FP64Regs) {
10692 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
10693 }
10694 }
10695 }
10696
10697 return MBB;
10698}
10699
10700MachineBasicBlock *SystemZTargetLowering::emitLoadAndTestCmp0(
10701 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
10702 MachineFunction &MF = *MBB->getParent();
10703 MachineRegisterInfo *MRI = &MF.getRegInfo();
10704 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10705 DebugLoc DL = MI.getDebugLoc();
10706
10707 Register SrcReg = MI.getOperand(i: 0).getReg();
10708
10709 // Create new virtual register of the same class as source.
10710 const TargetRegisterClass *RC = MRI->getRegClass(Reg: SrcReg);
10711 Register DstReg = MRI->createVirtualRegister(RegClass: RC);
10712
10713 // Replace pseudo with a normal load-and-test that models the def as
10714 // well.
10715 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode), DestReg: DstReg)
10716 .addReg(RegNo: SrcReg)
10717 .setMIFlags(MI.getFlags());
10718 MI.eraseFromParent();
10719
10720 return MBB;
10721}
10722
10723MachineBasicBlock *SystemZTargetLowering::emitProbedAlloca(
10724 MachineInstr &MI, MachineBasicBlock *MBB) const {
10725 MachineFunction &MF = *MBB->getParent();
10726 MachineRegisterInfo *MRI = &MF.getRegInfo();
10727 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10728 DebugLoc DL = MI.getDebugLoc();
10729 const unsigned ProbeSize = getStackProbeSize(MF);
10730 Register DstReg = MI.getOperand(i: 0).getReg();
10731 Register SizeReg = MI.getOperand(i: 2).getReg();
10732
10733 MachineBasicBlock *StartMBB = MBB;
10734 MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
10735 MachineBasicBlock *LoopTestMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10736 MachineBasicBlock *LoopBodyMBB = SystemZ::emitBlockAfter(MBB: LoopTestMBB);
10737 MachineBasicBlock *TailTestMBB = SystemZ::emitBlockAfter(MBB: LoopBodyMBB);
10738 MachineBasicBlock *TailMBB = SystemZ::emitBlockAfter(MBB: TailTestMBB);
10739
10740 MachineMemOperand *VolLdMMO = MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
10741 F: MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad, Size: 8, BaseAlignment: Align(1));
10742
10743 Register PHIReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10744 Register IncReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10745
10746 // LoopTestMBB
10747 // BRC TailTestMBB
10748 // # fallthrough to LoopBodyMBB
10749 StartMBB->addSuccessor(Succ: LoopTestMBB);
10750 MBB = LoopTestMBB;
10751 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: PHIReg)
10752 .addReg(RegNo: SizeReg)
10753 .addMBB(MBB: StartMBB)
10754 .addReg(RegNo: IncReg)
10755 .addMBB(MBB: LoopBodyMBB);
10756 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CLGFI))
10757 .addReg(RegNo: PHIReg)
10758 .addImm(Val: ProbeSize);
10759 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10760 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_LT)
10761 .addMBB(MBB: TailTestMBB);
10762 MBB->addSuccessor(Succ: LoopBodyMBB);
10763 MBB->addSuccessor(Succ: TailTestMBB);
10764
10765 // LoopBodyMBB: Allocate and probe by means of a volatile compare.
10766 // J LoopTestMBB
10767 MBB = LoopBodyMBB;
10768 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGFI), DestReg: IncReg)
10769 .addReg(RegNo: PHIReg)
10770 .addImm(Val: ProbeSize);
10771 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGFI), DestReg: SystemZ::R15D)
10772 .addReg(RegNo: SystemZ::R15D)
10773 .addImm(Val: ProbeSize);
10774 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CG)).addReg(RegNo: SystemZ::R15D)
10775 .addReg(RegNo: SystemZ::R15D).addImm(Val: ProbeSize - 8).addReg(RegNo: 0)
10776 .setMemRefs(VolLdMMO);
10777 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: LoopTestMBB);
10778 MBB->addSuccessor(Succ: LoopTestMBB);
10779
10780 // TailTestMBB
10781 // BRC DoneMBB
10782 // # fallthrough to TailMBB
10783 MBB = TailTestMBB;
10784 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10785 .addReg(RegNo: PHIReg)
10786 .addImm(Val: 0);
10787 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10788 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10789 .addMBB(MBB: DoneMBB);
10790 MBB->addSuccessor(Succ: TailMBB);
10791 MBB->addSuccessor(Succ: DoneMBB);
10792
10793 // TailMBB
10794 // # fallthrough to DoneMBB
10795 MBB = TailMBB;
10796 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGR), DestReg: SystemZ::R15D)
10797 .addReg(RegNo: SystemZ::R15D)
10798 .addReg(RegNo: PHIReg);
10799 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CG)).addReg(RegNo: SystemZ::R15D)
10800 .addReg(RegNo: SystemZ::R15D).addImm(Val: -8).addReg(RegNo: PHIReg)
10801 .setMemRefs(VolLdMMO);
10802 MBB->addSuccessor(Succ: DoneMBB);
10803
10804 // DoneMBB
10805 MBB = DoneMBB;
10806 BuildMI(BB&: *MBB, I: MBB->begin(), MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
10807 .addReg(RegNo: SystemZ::R15D);
10808
10809 MI.eraseFromParent();
10810 return DoneMBB;
10811}
10812
10813SDValue SystemZTargetLowering::
10814getBackchainAddress(SDValue SP, SelectionDAG &DAG) const {
10815 MachineFunction &MF = DAG.getMachineFunction();
10816 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
10817 SDLoc DL(SP);
10818 return DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: SP,
10819 N2: DAG.getIntPtrConstant(Val: TFL->getBackchainOffset(MF), DL));
10820}
10821
10822MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter(
10823 MachineInstr &MI, MachineBasicBlock *MBB) const {
10824 switch (MI.getOpcode()) {
10825 case SystemZ::ADJCALLSTACKDOWN:
10826 case SystemZ::ADJCALLSTACKUP:
10827 return emitAdjCallStack(MI, BB: MBB);
10828
10829 case SystemZ::Select32:
10830 case SystemZ::Select64:
10831 case SystemZ::Select128:
10832 case SystemZ::SelectF32:
10833 case SystemZ::SelectF64:
10834 case SystemZ::SelectF128:
10835 case SystemZ::SelectVR32:
10836 case SystemZ::SelectVR64:
10837 case SystemZ::SelectVR128:
10838 return emitSelect(MI, MBB);
10839
10840 case SystemZ::CondStore8Mux:
10841 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STCMux, STOCOpcode: 0, Invert: false);
10842 case SystemZ::CondStore8MuxInv:
10843 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STCMux, STOCOpcode: 0, Invert: true);
10844 case SystemZ::CondStore16Mux:
10845 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STHMux, STOCOpcode: 0, Invert: false);
10846 case SystemZ::CondStore16MuxInv:
10847 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STHMux, STOCOpcode: 0, Invert: true);
10848 case SystemZ::CondStore32Mux:
10849 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STMux, STOCOpcode: SystemZ::STOCMux, Invert: false);
10850 case SystemZ::CondStore32MuxInv:
10851 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STMux, STOCOpcode: SystemZ::STOCMux, Invert: true);
10852 case SystemZ::CondStore8:
10853 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STC, STOCOpcode: 0, Invert: false);
10854 case SystemZ::CondStore8Inv:
10855 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STC, STOCOpcode: 0, Invert: true);
10856 case SystemZ::CondStore16:
10857 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STH, STOCOpcode: 0, Invert: false);
10858 case SystemZ::CondStore16Inv:
10859 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STH, STOCOpcode: 0, Invert: true);
10860 case SystemZ::CondStore32:
10861 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::ST, STOCOpcode: SystemZ::STOC, Invert: false);
10862 case SystemZ::CondStore32Inv:
10863 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::ST, STOCOpcode: SystemZ::STOC, Invert: true);
10864 case SystemZ::CondStore64:
10865 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STG, STOCOpcode: SystemZ::STOCG, Invert: false);
10866 case SystemZ::CondStore64Inv:
10867 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STG, STOCOpcode: SystemZ::STOCG, Invert: true);
10868 case SystemZ::CondStoreF32:
10869 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STE, STOCOpcode: 0, Invert: false);
10870 case SystemZ::CondStoreF32Inv:
10871 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STE, STOCOpcode: 0, Invert: true);
10872 case SystemZ::CondStoreF64:
10873 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STD, STOCOpcode: 0, Invert: false);
10874 case SystemZ::CondStoreF64Inv:
10875 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STD, STOCOpcode: 0, Invert: true);
10876
10877 case SystemZ::SCmp128Hi:
10878 return emitICmp128Hi(MI, MBB, Unsigned: false);
10879 case SystemZ::UCmp128Hi:
10880 return emitICmp128Hi(MI, MBB, Unsigned: true);
10881
10882 case SystemZ::PAIR128:
10883 return emitPair128(MI, MBB);
10884 case SystemZ::AEXT128:
10885 return emitExt128(MI, MBB, ClearEven: false);
10886 case SystemZ::ZEXT128:
10887 return emitExt128(MI, MBB, ClearEven: true);
10888
10889 case SystemZ::ATOMIC_SWAPW:
10890 return emitAtomicLoadBinary(MI, MBB, BinOpcode: 0);
10891
10892 case SystemZ::ATOMIC_LOADW_AR:
10893 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::AR);
10894 case SystemZ::ATOMIC_LOADW_AFI:
10895 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::AFI);
10896
10897 case SystemZ::ATOMIC_LOADW_SR:
10898 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::SR);
10899
10900 case SystemZ::ATOMIC_LOADW_NR:
10901 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NR);
10902 case SystemZ::ATOMIC_LOADW_NILH:
10903 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NILH);
10904
10905 case SystemZ::ATOMIC_LOADW_OR:
10906 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::OR);
10907 case SystemZ::ATOMIC_LOADW_OILH:
10908 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::OILH);
10909
10910 case SystemZ::ATOMIC_LOADW_XR:
10911 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::XR);
10912 case SystemZ::ATOMIC_LOADW_XILF:
10913 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::XILF);
10914
10915 case SystemZ::ATOMIC_LOADW_NRi:
10916 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NR, Invert: true);
10917 case SystemZ::ATOMIC_LOADW_NILHi:
10918 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NILH, Invert: true);
10919
10920 case SystemZ::ATOMIC_LOADW_MIN:
10921 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CR, KeepOldMask: SystemZ::CCMASK_CMP_LE);
10922 case SystemZ::ATOMIC_LOADW_MAX:
10923 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CR, KeepOldMask: SystemZ::CCMASK_CMP_GE);
10924 case SystemZ::ATOMIC_LOADW_UMIN:
10925 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CLR, KeepOldMask: SystemZ::CCMASK_CMP_LE);
10926 case SystemZ::ATOMIC_LOADW_UMAX:
10927 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CLR, KeepOldMask: SystemZ::CCMASK_CMP_GE);
10928
10929 case SystemZ::ATOMIC_CMP_SWAPW:
10930 return emitAtomicCmpSwapW(MI, MBB);
10931 case SystemZ::MVCImm:
10932 case SystemZ::MVCReg:
10933 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::MVC);
10934 case SystemZ::NCImm:
10935 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::NC);
10936 case SystemZ::OCImm:
10937 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::OC);
10938 case SystemZ::XCImm:
10939 case SystemZ::XCReg:
10940 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::XC);
10941 case SystemZ::CLCImm:
10942 case SystemZ::CLCReg:
10943 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::CLC);
10944 case SystemZ::MemsetImmImm:
10945 case SystemZ::MemsetImmReg:
10946 case SystemZ::MemsetRegImm:
10947 case SystemZ::MemsetRegReg:
10948 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::MVC, IsMemset: true/*IsMemset*/);
10949 case SystemZ::CLSTLoop:
10950 return emitStringWrapper(MI, MBB, Opcode: SystemZ::CLST);
10951 case SystemZ::MVSTLoop:
10952 return emitStringWrapper(MI, MBB, Opcode: SystemZ::MVST);
10953 case SystemZ::SRSTLoop:
10954 return emitStringWrapper(MI, MBB, Opcode: SystemZ::SRST);
10955 case SystemZ::TBEGIN:
10956 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGIN, NoFloat: false);
10957 case SystemZ::TBEGIN_nofloat:
10958 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGIN, NoFloat: true);
10959 case SystemZ::TBEGINC:
10960 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGINC, NoFloat: true);
10961 case SystemZ::LTEBRCompare_Pseudo:
10962 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTEBR);
10963 case SystemZ::LTDBRCompare_Pseudo:
10964 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTDBR);
10965 case SystemZ::LTXBRCompare_Pseudo:
10966 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTXBR);
10967
10968 case SystemZ::PROBED_ALLOCA:
10969 return emitProbedAlloca(MI, MBB);
10970 case SystemZ::EH_SjLj_SetJmp:
10971 return emitEHSjLjSetJmp(MI, MBB);
10972 case SystemZ::EH_SjLj_LongJmp:
10973 return emitEHSjLjLongJmp(MI, MBB);
10974
10975 case TargetOpcode::STACKMAP:
10976 case TargetOpcode::PATCHPOINT:
10977 return emitPatchPoint(MI, MBB);
10978
10979 default:
10980 llvm_unreachable("Unexpected instr type to insert");
10981 }
10982}
10983
10984// This is only used by the isel schedulers, and is needed only to prevent
10985// compiler from crashing when list-ilp is used.
10986const TargetRegisterClass *
10987SystemZTargetLowering::getRepRegClassFor(MVT VT) const {
10988 if (VT == MVT::Untyped)
10989 return &SystemZ::ADDR128BitRegClass;
10990 return TargetLowering::getRepRegClassFor(VT);
10991}
10992
10993SDValue SystemZTargetLowering::lowerGET_ROUNDING(SDValue Op,
10994 SelectionDAG &DAG) const {
10995 SDLoc dl(Op);
10996 /*
10997 The rounding method is in FPC Byte 3 bits 6-7, and has the following
10998 settings:
10999 00 Round to nearest
11000 01 Round to 0
11001 10 Round to +inf
11002 11 Round to -inf
11003
11004 FLT_ROUNDS, on the other hand, expects the following:
11005 -1 Undefined
11006 0 Round to 0
11007 1 Round to nearest
11008 2 Round to +inf
11009 3 Round to -inf
11010 */
11011
11012 // Save FPC to register.
11013 SDValue Chain = Op.getOperand(i: 0);
11014 SDValue EFPC(
11015 DAG.getMachineNode(Opcode: SystemZ::EFPC, dl, ResultTys: {MVT::i32, MVT::Other}, Ops: Chain), 0);
11016 Chain = EFPC.getValue(R: 1);
11017
11018 // Transform as necessary
11019 SDValue CWD1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: EFPC,
11020 N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32));
11021 // RetVal = (CWD1 ^ (CWD1 >> 1)) ^ 1
11022 SDValue CWD2 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD1,
11023 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: CWD1,
11024 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32)));
11025
11026 SDValue RetVal = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD2,
11027 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11028 RetVal = DAG.getZExtOrTrunc(Op: RetVal, DL: dl, VT: Op.getValueType());
11029
11030 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl);
11031}
11032
11033SDValue SystemZTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
11034 SelectionDAG &DAG) const {
11035 EVT VT = Op.getValueType();
11036 Op = Op.getOperand(i: 0);
11037 EVT OpVT = Op.getValueType();
11038
11039 assert(OpVT.isVector() && "Operand type for VECREDUCE_ADD is not a vector.");
11040
11041 SDLoc DL(Op);
11042
11043 // load a 0 vector for the third operand of VSUM.
11044 SDValue Zero = DAG.getSplatBuildVector(VT: OpVT, DL, Op: DAG.getConstant(Val: 0, DL, VT));
11045
11046 // execute VSUM.
11047 switch (OpVT.getScalarSizeInBits()) {
11048 case 8:
11049 case 16:
11050 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::v4i32, N1: Op, N2: Zero);
11051 [[fallthrough]];
11052 case 32:
11053 case 64:
11054 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::i128, N1: Op,
11055 N2: DAG.getBitcast(VT: Op.getValueType(), V: Zero));
11056 break;
11057 case 128:
11058 break; // VSUM over v1i128 should not happen and would be a noop
11059 default:
11060 llvm_unreachable("Unexpected scalar size.");
11061 }
11062 // Cast to original vector type, retrieve last element.
11063 return DAG.getNode(
11064 Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: DAG.getBitcast(VT: OpVT, V: Op),
11065 N2: DAG.getConstant(Val: OpVT.getVectorNumElements() - 1, DL, VT: MVT::i32));
11066}
11067
11068static void printFunctionArgExts(const Function *F, raw_fd_ostream &OS) {
11069 FunctionType *FT = F->getFunctionType();
11070 const AttributeList &Attrs = F->getAttributes();
11071 if (Attrs.hasRetAttrs())
11072 OS << Attrs.getAsString(Index: AttributeList::ReturnIndex) << " ";
11073 OS << *F->getReturnType() << " @" << F->getName() << "(";
11074 for (unsigned I = 0, E = FT->getNumParams(); I != E; ++I) {
11075 if (I)
11076 OS << ", ";
11077 OS << *FT->getParamType(i: I);
11078 AttributeSet ArgAttrs = Attrs.getParamAttrs(ArgNo: I);
11079 for (auto A : {Attribute::SExt, Attribute::ZExt, Attribute::NoExt})
11080 if (ArgAttrs.hasAttribute(Kind: A))
11081 OS << " " << Attribute::getNameFromAttrKind(AttrKind: A);
11082 }
11083 OS << ")\n";
11084}
11085
11086bool SystemZTargetLowering::isInternal(const Function *Fn) const {
11087 std::map<const Function *, bool>::iterator Itr = IsInternalCache.find(x: Fn);
11088 if (Itr == IsInternalCache.end())
11089 Itr = IsInternalCache
11090 .insert(x: std::pair<const Function *, bool>(
11091 Fn, (Fn->hasLocalLinkage() && !Fn->hasAddressTaken())))
11092 .first;
11093 return Itr->second;
11094}
11095
11096void SystemZTargetLowering::
11097verifyNarrowIntegerArgs_Call(const SmallVectorImpl<ISD::OutputArg> &Outs,
11098 const Function *F, SDValue Callee) const {
11099 // Temporarily only do the check when explicitly requested, until it can be
11100 // enabled by default.
11101 if (!EnableIntArgExtCheck)
11102 return;
11103
11104 bool IsInternal = false;
11105 const Function *CalleeFn = nullptr;
11106 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
11107 if ((CalleeFn = dyn_cast<Function>(Val: G->getGlobal())))
11108 IsInternal = isInternal(Fn: CalleeFn);
11109 if (!IsInternal && !verifyNarrowIntegerArgs(Outs)) {
11110 errs() << "ERROR: Missing extension attribute of passed "
11111 << "value in call to function:\n" << "Callee: ";
11112 if (CalleeFn != nullptr)
11113 printFunctionArgExts(F: CalleeFn, OS&: errs());
11114 else
11115 errs() << "-\n";
11116 errs() << "Caller: ";
11117 printFunctionArgExts(F, OS&: errs());
11118 llvm_unreachable("");
11119 }
11120}
11121
11122void SystemZTargetLowering::
11123verifyNarrowIntegerArgs_Ret(const SmallVectorImpl<ISD::OutputArg> &Outs,
11124 const Function *F) const {
11125 // Temporarily only do the check when explicitly requested, until it can be
11126 // enabled by default.
11127 if (!EnableIntArgExtCheck)
11128 return;
11129
11130 if (!isInternal(Fn: F) && !verifyNarrowIntegerArgs(Outs)) {
11131 errs() << "ERROR: Missing extension attribute of returned "
11132 << "value from function:\n";
11133 printFunctionArgExts(F, OS&: errs());
11134 llvm_unreachable("");
11135 }
11136}
11137
11138// Verify that narrow integer arguments are extended as required by the ABI.
11139// Return false if an error is found.
11140bool SystemZTargetLowering::verifyNarrowIntegerArgs(
11141 const SmallVectorImpl<ISD::OutputArg> &Outs) const {
11142 if (!Subtarget.isTargetELF())
11143 return true;
11144
11145 if (EnableIntArgExtCheck.getNumOccurrences()) {
11146 if (!EnableIntArgExtCheck)
11147 return true;
11148 } else if (!getTargetMachine().Options.VerifyArgABICompliance)
11149 return true;
11150
11151 for (unsigned i = 0; i < Outs.size(); ++i) {
11152 MVT VT = Outs[i].VT;
11153 ISD::ArgFlagsTy Flags = Outs[i].Flags;
11154 if (VT.isInteger()) {
11155 assert((VT == MVT::i32 || VT.getSizeInBits() >= 64) &&
11156 "Unexpected integer argument VT.");
11157 if (VT == MVT::i32 &&
11158 !Flags.isSExt() && !Flags.isZExt() && !Flags.isNoExt())
11159 return false;
11160 }
11161 }
11162
11163 return true;
11164}
11165

source code of llvm/lib/Target/SystemZ/SystemZISelLowering.cpp