1 | //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file performs vector type splitting and scalarization for LegalizeTypes. |
10 | // Scalarization is the act of changing a computation in an illegal one-element |
11 | // vector type to be a computation in its scalar element type. For example, |
12 | // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed |
13 | // as a base case when scalarizing vector arithmetic like <4 x f32>, which |
14 | // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 |
15 | // types. |
16 | // Splitting is the act of changing a computation in an invalid vector type to |
17 | // be a computation in two vectors of half the size. For example, implementing |
18 | // <128 x f32> operations in terms of two <64 x f32> operations. |
19 | // |
20 | //===----------------------------------------------------------------------===// |
21 | |
22 | #include "LegalizeTypes.h" |
23 | #include "llvm/ADT/SmallBitVector.h" |
24 | #include "llvm/Analysis/MemoryLocation.h" |
25 | #include "llvm/Analysis/VectorUtils.h" |
26 | #include "llvm/IR/DataLayout.h" |
27 | #include "llvm/Support/ErrorHandling.h" |
28 | #include "llvm/Support/TypeSize.h" |
29 | #include "llvm/Support/raw_ostream.h" |
30 | #include <numeric> |
31 | |
32 | using namespace llvm; |
33 | |
34 | #define DEBUG_TYPE "legalize-types" |
35 | |
36 | //===----------------------------------------------------------------------===// |
37 | // Result Vector Scalarization: <1 x ty> -> ty. |
38 | //===----------------------------------------------------------------------===// |
39 | |
40 | void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { |
41 | LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": " ; |
42 | N->dump(&DAG)); |
43 | SDValue R = SDValue(); |
44 | |
45 | switch (N->getOpcode()) { |
46 | default: |
47 | #ifndef NDEBUG |
48 | dbgs() << "ScalarizeVectorResult #" << ResNo << ": " ; |
49 | N->dump(G: &DAG); |
50 | dbgs() << "\n" ; |
51 | #endif |
52 | report_fatal_error(reason: "Do not know how to scalarize the result of this " |
53 | "operator!\n" ); |
54 | |
55 | case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; |
56 | case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; |
57 | case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; |
58 | case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; |
59 | case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; |
60 | case ISD::FPOWI: R = ScalarizeVecRes_ExpOp(N); break; |
61 | case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; |
62 | case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break; |
63 | case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; |
64 | case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; |
65 | case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break; |
66 | case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; |
67 | case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; |
68 | case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; |
69 | case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; |
70 | case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; |
71 | case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break; |
72 | case ISD::ANY_EXTEND_VECTOR_INREG: |
73 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
74 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
75 | R = ScalarizeVecRes_VecInregOp(N); |
76 | break; |
77 | case ISD::ABS: |
78 | case ISD::ANY_EXTEND: |
79 | case ISD::BITREVERSE: |
80 | case ISD::BSWAP: |
81 | case ISD::CTLZ: |
82 | case ISD::CTLZ_ZERO_UNDEF: |
83 | case ISD::CTPOP: |
84 | case ISD::CTTZ: |
85 | case ISD::CTTZ_ZERO_UNDEF: |
86 | case ISD::FABS: |
87 | case ISD::FCEIL: |
88 | case ISD::FCOS: |
89 | case ISD::FEXP: |
90 | case ISD::FEXP2: |
91 | case ISD::FEXP10: |
92 | case ISD::FFLOOR: |
93 | case ISD::FLOG: |
94 | case ISD::FLOG10: |
95 | case ISD::FLOG2: |
96 | case ISD::FNEARBYINT: |
97 | case ISD::FNEG: |
98 | case ISD::FREEZE: |
99 | case ISD::ARITH_FENCE: |
100 | case ISD::FP_EXTEND: |
101 | case ISD::FP_TO_SINT: |
102 | case ISD::FP_TO_UINT: |
103 | case ISD::FRINT: |
104 | case ISD::LRINT: |
105 | case ISD::LLRINT: |
106 | case ISD::FROUND: |
107 | case ISD::FROUNDEVEN: |
108 | case ISD::FSIN: |
109 | case ISD::FSQRT: |
110 | case ISD::FTRUNC: |
111 | case ISD::SIGN_EXTEND: |
112 | case ISD::SINT_TO_FP: |
113 | case ISD::TRUNCATE: |
114 | case ISD::UINT_TO_FP: |
115 | case ISD::ZERO_EXTEND: |
116 | case ISD::FCANONICALIZE: |
117 | R = ScalarizeVecRes_UnaryOp(N); |
118 | break; |
119 | case ISD::FFREXP: |
120 | R = ScalarizeVecRes_FFREXP(N, ResNo); |
121 | break; |
122 | case ISD::ADD: |
123 | case ISD::AND: |
124 | case ISD::FADD: |
125 | case ISD::FCOPYSIGN: |
126 | case ISD::FDIV: |
127 | case ISD::FMUL: |
128 | case ISD::FMINNUM: |
129 | case ISD::FMAXNUM: |
130 | case ISD::FMINNUM_IEEE: |
131 | case ISD::FMAXNUM_IEEE: |
132 | case ISD::FMINIMUM: |
133 | case ISD::FMAXIMUM: |
134 | case ISD::FLDEXP: |
135 | case ISD::SMIN: |
136 | case ISD::SMAX: |
137 | case ISD::UMIN: |
138 | case ISD::UMAX: |
139 | |
140 | case ISD::SADDSAT: |
141 | case ISD::UADDSAT: |
142 | case ISD::SSUBSAT: |
143 | case ISD::USUBSAT: |
144 | case ISD::SSHLSAT: |
145 | case ISD::USHLSAT: |
146 | |
147 | case ISD::FPOW: |
148 | case ISD::FREM: |
149 | case ISD::FSUB: |
150 | case ISD::MUL: |
151 | case ISD::MULHS: |
152 | case ISD::MULHU: |
153 | case ISD::OR: |
154 | case ISD::SDIV: |
155 | case ISD::SREM: |
156 | case ISD::SUB: |
157 | case ISD::UDIV: |
158 | case ISD::UREM: |
159 | case ISD::XOR: |
160 | case ISD::SHL: |
161 | case ISD::SRA: |
162 | case ISD::SRL: |
163 | case ISD::ROTL: |
164 | case ISD::ROTR: |
165 | R = ScalarizeVecRes_BinOp(N); |
166 | break; |
167 | case ISD::FMA: |
168 | case ISD::FSHL: |
169 | case ISD::FSHR: |
170 | R = ScalarizeVecRes_TernaryOp(N); |
171 | break; |
172 | |
173 | #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ |
174 | case ISD::STRICT_##DAGN: |
175 | #include "llvm/IR/ConstrainedOps.def" |
176 | R = ScalarizeVecRes_StrictFPOp(N); |
177 | break; |
178 | |
179 | case ISD::FP_TO_UINT_SAT: |
180 | case ISD::FP_TO_SINT_SAT: |
181 | R = ScalarizeVecRes_FP_TO_XINT_SAT(N); |
182 | break; |
183 | |
184 | case ISD::UADDO: |
185 | case ISD::SADDO: |
186 | case ISD::USUBO: |
187 | case ISD::SSUBO: |
188 | case ISD::UMULO: |
189 | case ISD::SMULO: |
190 | R = ScalarizeVecRes_OverflowOp(N, ResNo); |
191 | break; |
192 | case ISD::SMULFIX: |
193 | case ISD::SMULFIXSAT: |
194 | case ISD::UMULFIX: |
195 | case ISD::UMULFIXSAT: |
196 | case ISD::SDIVFIX: |
197 | case ISD::SDIVFIXSAT: |
198 | case ISD::UDIVFIX: |
199 | case ISD::UDIVFIXSAT: |
200 | R = ScalarizeVecRes_FIX(N); |
201 | break; |
202 | } |
203 | |
204 | // If R is null, the sub-method took care of registering the result. |
205 | if (R.getNode()) |
206 | SetScalarizedVector(Op: SDValue(N, ResNo), Result: R); |
207 | } |
208 | |
209 | SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { |
210 | SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
211 | SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1)); |
212 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), |
213 | VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags()); |
214 | } |
215 | |
216 | SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { |
217 | SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
218 | SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1)); |
219 | SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2)); |
220 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1, |
221 | N3: Op2, Flags: N->getFlags()); |
222 | } |
223 | |
224 | SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) { |
225 | SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
226 | SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1)); |
227 | SDValue Op2 = N->getOperand(Num: 2); |
228 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1, |
229 | N3: Op2, Flags: N->getFlags()); |
230 | } |
231 | |
232 | SDValue DAGTypeLegalizer::ScalarizeVecRes_FFREXP(SDNode *N, unsigned ResNo) { |
233 | assert(N->getValueType(0).getVectorNumElements() == 1 && |
234 | "Unexpected vector type!" ); |
235 | SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
236 | |
237 | EVT VT0 = N->getValueType(ResNo: 0); |
238 | EVT VT1 = N->getValueType(ResNo: 1); |
239 | SDLoc dl(N); |
240 | |
241 | SDNode *ScalarNode = |
242 | DAG.getNode(Opcode: N->getOpcode(), DL: dl, |
243 | ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt) |
244 | .getNode(); |
245 | |
246 | // Replace the other vector result not being explicitly scalarized here. |
247 | unsigned OtherNo = 1 - ResNo; |
248 | EVT OtherVT = N->getValueType(ResNo: OtherNo); |
249 | if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) { |
250 | SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo)); |
251 | } else { |
252 | SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT, |
253 | Operand: SDValue(ScalarNode, OtherNo)); |
254 | ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal); |
255 | } |
256 | |
257 | return SDValue(ScalarNode, ResNo); |
258 | } |
259 | |
260 | SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) { |
261 | EVT VT = N->getValueType(ResNo: 0).getVectorElementType(); |
262 | unsigned NumOpers = N->getNumOperands(); |
263 | SDValue Chain = N->getOperand(Num: 0); |
264 | EVT ValueVTs[] = {VT, MVT::Other}; |
265 | SDLoc dl(N); |
266 | |
267 | SmallVector<SDValue, 4> Opers(NumOpers); |
268 | |
269 | // The Chain is the first operand. |
270 | Opers[0] = Chain; |
271 | |
272 | // Now process the remaining operands. |
273 | for (unsigned i = 1; i < NumOpers; ++i) { |
274 | SDValue Oper = N->getOperand(Num: i); |
275 | EVT OperVT = Oper.getValueType(); |
276 | |
277 | if (OperVT.isVector()) { |
278 | if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector) |
279 | Oper = GetScalarizedVector(Op: Oper); |
280 | else |
281 | Oper = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, |
282 | VT: OperVT.getVectorElementType(), N1: Oper, |
283 | N2: DAG.getVectorIdxConstant(Val: 0, DL: dl)); |
284 | } |
285 | |
286 | Opers[i] = Oper; |
287 | } |
288 | |
289 | SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(ValueVTs), |
290 | Ops: Opers, Flags: N->getFlags()); |
291 | |
292 | // Legalize the chain result - switch anything that used the old chain to |
293 | // use the new one. |
294 | ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1)); |
295 | return Result; |
296 | } |
297 | |
298 | SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N, |
299 | unsigned ResNo) { |
300 | SDLoc DL(N); |
301 | EVT ResVT = N->getValueType(ResNo: 0); |
302 | EVT OvVT = N->getValueType(ResNo: 1); |
303 | |
304 | SDValue ScalarLHS, ScalarRHS; |
305 | if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) { |
306 | ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
307 | ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1)); |
308 | } else { |
309 | SmallVector<SDValue, 1> ElemsLHS, ElemsRHS; |
310 | DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS); |
311 | DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS); |
312 | ScalarLHS = ElemsLHS[0]; |
313 | ScalarRHS = ElemsRHS[0]; |
314 | } |
315 | |
316 | SDVTList ScalarVTs = DAG.getVTList( |
317 | VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType()); |
318 | SDNode *ScalarNode = DAG.getNode( |
319 | Opcode: N->getOpcode(), DL, VTList: ScalarVTs, N1: ScalarLHS, N2: ScalarRHS).getNode(); |
320 | ScalarNode->setFlags(N->getFlags()); |
321 | |
322 | // Replace the other vector result not being explicitly scalarized here. |
323 | unsigned OtherNo = 1 - ResNo; |
324 | EVT OtherVT = N->getValueType(ResNo: OtherNo); |
325 | if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) { |
326 | SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo)); |
327 | } else { |
328 | SDValue OtherVal = DAG.getNode( |
329 | Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo)); |
330 | ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal); |
331 | } |
332 | |
333 | return SDValue(ScalarNode, ResNo); |
334 | } |
335 | |
336 | SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, |
337 | unsigned ResNo) { |
338 | SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); |
339 | return GetScalarizedVector(Op); |
340 | } |
341 | |
342 | SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { |
343 | SDValue Op = N->getOperand(Num: 0); |
344 | if (Op.getValueType().isVector() |
345 | && Op.getValueType().getVectorNumElements() == 1 |
346 | && !isSimpleLegalType(VT: Op.getValueType())) |
347 | Op = GetScalarizedVector(Op); |
348 | EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType(); |
349 | return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), |
350 | VT: NewVT, Operand: Op); |
351 | } |
352 | |
353 | SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { |
354 | EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType(); |
355 | SDValue InOp = N->getOperand(Num: 0); |
356 | // The BUILD_VECTOR operands may be of wider element types and |
357 | // we may need to truncate them back to the requested return type. |
358 | if (EltVT.isInteger()) |
359 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp); |
360 | return InOp; |
361 | } |
362 | |
363 | SDValue DAGTypeLegalizer::(SDNode *N) { |
364 | return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N), |
365 | VT: N->getValueType(ResNo: 0).getVectorElementType(), |
366 | N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1)); |
367 | } |
368 | |
369 | SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { |
370 | SDLoc DL(N); |
371 | SDValue Op = N->getOperand(Num: 0); |
372 | EVT OpVT = Op.getValueType(); |
373 | // The result needs scalarizing, but it's not a given that the source does. |
374 | // See similar logic in ScalarizeVecRes_UnaryOp. |
375 | if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) { |
376 | Op = GetScalarizedVector(Op); |
377 | } else { |
378 | EVT VT = OpVT.getVectorElementType(); |
379 | Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Op, |
380 | N2: DAG.getVectorIdxConstant(Val: 0, DL)); |
381 | } |
382 | return DAG.getNode(Opcode: ISD::FP_ROUND, DL, |
383 | VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op, |
384 | N2: N->getOperand(Num: 1)); |
385 | } |
386 | |
387 | SDValue DAGTypeLegalizer::ScalarizeVecRes_ExpOp(SDNode *N) { |
388 | SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
389 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op, |
390 | N2: N->getOperand(Num: 1)); |
391 | } |
392 | |
393 | SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { |
394 | // The value to insert may have a wider type than the vector element type, |
395 | // so be sure to truncate it to the element type if necessary. |
396 | SDValue Op = N->getOperand(Num: 1); |
397 | EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType(); |
398 | if (Op.getValueType() != EltVT) |
399 | // FIXME: Can this happen for floating point types? |
400 | Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op); |
401 | return Op; |
402 | } |
403 | |
404 | SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { |
405 | assert(N->isUnindexed() && "Indexed vector load?" ); |
406 | |
407 | SDValue Result = DAG.getLoad( |
408 | AM: ISD::UNINDEXED, ExtType: N->getExtensionType(), |
409 | VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(), |
410 | Ptr: N->getBasePtr(), Offset: DAG.getUNDEF(VT: N->getBasePtr().getValueType()), |
411 | PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(), |
412 | Alignment: N->getOriginalAlign(), MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo()); |
413 | |
414 | // Legalize the chain result - switch anything that used the old chain to |
415 | // use the new one. |
416 | ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1)); |
417 | return Result; |
418 | } |
419 | |
420 | SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { |
421 | // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. |
422 | EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType(); |
423 | SDValue Op = N->getOperand(Num: 0); |
424 | EVT OpVT = Op.getValueType(); |
425 | SDLoc DL(N); |
426 | // The result needs scalarizing, but it's not a given that the source does. |
427 | // This is a workaround for targets where it's impossible to scalarize the |
428 | // result of a conversion, because the source type is legal. |
429 | // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32} |
430 | // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is |
431 | // legal and was not scalarized. |
432 | // See the similar logic in ScalarizeVecRes_SETCC |
433 | if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) { |
434 | Op = GetScalarizedVector(Op); |
435 | } else { |
436 | EVT VT = OpVT.getVectorElementType(); |
437 | Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Op, |
438 | N2: DAG.getVectorIdxConstant(Val: 0, DL)); |
439 | } |
440 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags()); |
441 | } |
442 | |
443 | SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { |
444 | EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType(); |
445 | EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType(); |
446 | SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
447 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT, |
448 | N1: LHS, N2: DAG.getValueType(ExtVT)); |
449 | } |
450 | |
451 | SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) { |
452 | SDLoc DL(N); |
453 | SDValue Op = N->getOperand(Num: 0); |
454 | |
455 | EVT OpVT = Op.getValueType(); |
456 | EVT OpEltVT = OpVT.getVectorElementType(); |
457 | EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType(); |
458 | |
459 | if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) { |
460 | Op = GetScalarizedVector(Op); |
461 | } else { |
462 | Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: OpEltVT, N1: Op, |
463 | N2: DAG.getVectorIdxConstant(Val: 0, DL)); |
464 | } |
465 | |
466 | switch (N->getOpcode()) { |
467 | case ISD::ANY_EXTEND_VECTOR_INREG: |
468 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op); |
469 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
470 | return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op); |
471 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
472 | return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op); |
473 | } |
474 | |
475 | llvm_unreachable("Illegal extend_vector_inreg opcode" ); |
476 | } |
477 | |
478 | SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { |
479 | // If the operand is wider than the vector element type then it is implicitly |
480 | // truncated. Make that explicit here. |
481 | EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType(); |
482 | SDValue InOp = N->getOperand(Num: 0); |
483 | if (InOp.getValueType() != EltVT) |
484 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp); |
485 | return InOp; |
486 | } |
487 | |
488 | SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { |
489 | SDValue Cond = N->getOperand(Num: 0); |
490 | EVT OpVT = Cond.getValueType(); |
491 | SDLoc DL(N); |
492 | // The vselect result and true/value operands needs scalarizing, but it's |
493 | // not a given that the Cond does. For instance, in AVX512 v1i1 is legal. |
494 | // See the similar logic in ScalarizeVecRes_SETCC |
495 | if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) { |
496 | Cond = GetScalarizedVector(Op: Cond); |
497 | } else { |
498 | EVT VT = OpVT.getVectorElementType(); |
499 | Cond = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Cond, |
500 | N2: DAG.getVectorIdxConstant(Val: 0, DL)); |
501 | } |
502 | |
503 | SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1)); |
504 | TargetLowering::BooleanContent ScalarBool = |
505 | TLI.getBooleanContents(isVec: false, isFloat: false); |
506 | TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false); |
507 | |
508 | // If integer and float booleans have different contents then we can't |
509 | // reliably optimize in all cases. There is a full explanation for this in |
510 | // DAGCombiner::visitSELECT() where the same issue affects folding |
511 | // (select C, 0, 1) to (xor C, 1). |
512 | if (TLI.getBooleanContents(isVec: false, isFloat: false) != |
513 | TLI.getBooleanContents(isVec: false, isFloat: true)) { |
514 | // At least try the common case where the boolean is generated by a |
515 | // comparison. |
516 | if (Cond->getOpcode() == ISD::SETCC) { |
517 | EVT OpVT = Cond->getOperand(Num: 0).getValueType(); |
518 | ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType()); |
519 | VecBool = TLI.getBooleanContents(Type: OpVT); |
520 | } else |
521 | ScalarBool = TargetLowering::UndefinedBooleanContent; |
522 | } |
523 | |
524 | EVT CondVT = Cond.getValueType(); |
525 | if (ScalarBool != VecBool) { |
526 | switch (ScalarBool) { |
527 | case TargetLowering::UndefinedBooleanContent: |
528 | break; |
529 | case TargetLowering::ZeroOrOneBooleanContent: |
530 | assert(VecBool == TargetLowering::UndefinedBooleanContent || |
531 | VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent); |
532 | // Vector read from all ones, scalar expects a single 1 so mask. |
533 | Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT, |
534 | N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT)); |
535 | break; |
536 | case TargetLowering::ZeroOrNegativeOneBooleanContent: |
537 | assert(VecBool == TargetLowering::UndefinedBooleanContent || |
538 | VecBool == TargetLowering::ZeroOrOneBooleanContent); |
539 | // Vector reads from a one, scalar from all ones so sign extend. |
540 | Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT, |
541 | Cond, DAG.getValueType(MVT::i1)); |
542 | break; |
543 | } |
544 | } |
545 | |
546 | // Truncate the condition if needed |
547 | auto BoolVT = getSetCCResultType(VT: CondVT); |
548 | if (BoolVT.bitsLT(VT: CondVT)) |
549 | Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond); |
550 | |
551 | return DAG.getSelect(DL: SDLoc(N), |
552 | VT: LHS.getValueType(), Cond, LHS, |
553 | RHS: GetScalarizedVector(Op: N->getOperand(Num: 2))); |
554 | } |
555 | |
556 | SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { |
557 | SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1)); |
558 | return DAG.getSelect(DL: SDLoc(N), |
559 | VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS, |
560 | RHS: GetScalarizedVector(Op: N->getOperand(Num: 2))); |
561 | } |
562 | |
563 | SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { |
564 | SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2)); |
565 | return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(), |
566 | N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), |
567 | N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)), |
568 | N5: N->getOperand(Num: 4)); |
569 | } |
570 | |
571 | SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { |
572 | return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType()); |
573 | } |
574 | |
575 | SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { |
576 | // Figure out if the scalar is the LHS or RHS and return it. |
577 | SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0); |
578 | if (Arg.isUndef()) |
579 | return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType()); |
580 | unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero(); |
581 | return GetScalarizedVector(Op: N->getOperand(Num: Op)); |
582 | } |
583 | |
584 | SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) { |
585 | SDValue Src = N->getOperand(Num: 0); |
586 | EVT SrcVT = Src.getValueType(); |
587 | SDLoc dl(N); |
588 | |
589 | // Handle case where result is scalarized but operand is not |
590 | if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector) |
591 | Src = GetScalarizedVector(Op: Src); |
592 | else |
593 | Src = DAG.getNode( |
594 | Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src, |
595 | N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()))); |
596 | |
597 | EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType(); |
598 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1)); |
599 | } |
600 | |
601 | SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { |
602 | assert(N->getValueType(0).isVector() && |
603 | N->getOperand(0).getValueType().isVector() && |
604 | "Operand types must be vectors" ); |
605 | SDValue LHS = N->getOperand(Num: 0); |
606 | SDValue RHS = N->getOperand(Num: 1); |
607 | EVT OpVT = LHS.getValueType(); |
608 | EVT NVT = N->getValueType(ResNo: 0).getVectorElementType(); |
609 | SDLoc DL(N); |
610 | |
611 | // The result needs scalarizing, but it's not a given that the source does. |
612 | if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) { |
613 | LHS = GetScalarizedVector(Op: LHS); |
614 | RHS = GetScalarizedVector(Op: RHS); |
615 | } else { |
616 | EVT VT = OpVT.getVectorElementType(); |
617 | LHS = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: LHS, |
618 | N2: DAG.getVectorIdxConstant(Val: 0, DL)); |
619 | RHS = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: RHS, |
620 | N2: DAG.getVectorIdxConstant(Val: 0, DL)); |
621 | } |
622 | |
623 | // Turn it into a scalar SETCC. |
624 | SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, |
625 | N->getOperand(Num: 2)); |
626 | // Vectors may have a different boolean contents to scalars. Promote the |
627 | // value appropriately. |
628 | ISD::NodeType ExtendCode = |
629 | TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT)); |
630 | return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res); |
631 | } |
632 | |
633 | SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) { |
634 | SDLoc DL(N); |
635 | SDValue Arg = N->getOperand(Num: 0); |
636 | SDValue Test = N->getOperand(Num: 1); |
637 | EVT ArgVT = Arg.getValueType(); |
638 | EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType(); |
639 | |
640 | if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) { |
641 | Arg = GetScalarizedVector(Op: Arg); |
642 | } else { |
643 | EVT VT = ArgVT.getVectorElementType(); |
644 | Arg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Arg, |
645 | N2: DAG.getVectorIdxConstant(Val: 0, DL)); |
646 | } |
647 | |
648 | SDValue Res = |
649 | DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags()); |
650 | // Vectors may have a different boolean contents to scalars. Promote the |
651 | // value appropriately. |
652 | ISD::NodeType ExtendCode = |
653 | TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT)); |
654 | return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res); |
655 | } |
656 | |
657 | //===----------------------------------------------------------------------===// |
658 | // Operand Vector Scalarization <1 x ty> -> ty. |
659 | //===----------------------------------------------------------------------===// |
660 | |
661 | bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { |
662 | LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": " ; |
663 | N->dump(&DAG)); |
664 | SDValue Res = SDValue(); |
665 | |
666 | switch (N->getOpcode()) { |
667 | default: |
668 | #ifndef NDEBUG |
669 | dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": " ; |
670 | N->dump(G: &DAG); |
671 | dbgs() << "\n" ; |
672 | #endif |
673 | report_fatal_error(reason: "Do not know how to scalarize this operator's " |
674 | "operand!\n" ); |
675 | case ISD::BITCAST: |
676 | Res = ScalarizeVecOp_BITCAST(N); |
677 | break; |
678 | case ISD::ANY_EXTEND: |
679 | case ISD::ZERO_EXTEND: |
680 | case ISD::SIGN_EXTEND: |
681 | case ISD::TRUNCATE: |
682 | case ISD::FP_TO_SINT: |
683 | case ISD::FP_TO_UINT: |
684 | case ISD::SINT_TO_FP: |
685 | case ISD::UINT_TO_FP: |
686 | case ISD::LRINT: |
687 | case ISD::LLRINT: |
688 | Res = ScalarizeVecOp_UnaryOp(N); |
689 | break; |
690 | case ISD::STRICT_SINT_TO_FP: |
691 | case ISD::STRICT_UINT_TO_FP: |
692 | case ISD::STRICT_FP_TO_SINT: |
693 | case ISD::STRICT_FP_TO_UINT: |
694 | Res = ScalarizeVecOp_UnaryOp_StrictFP(N); |
695 | break; |
696 | case ISD::CONCAT_VECTORS: |
697 | Res = ScalarizeVecOp_CONCAT_VECTORS(N); |
698 | break; |
699 | case ISD::EXTRACT_VECTOR_ELT: |
700 | Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); |
701 | break; |
702 | case ISD::VSELECT: |
703 | Res = ScalarizeVecOp_VSELECT(N); |
704 | break; |
705 | case ISD::SETCC: |
706 | Res = ScalarizeVecOp_VSETCC(N); |
707 | break; |
708 | case ISD::STORE: |
709 | Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo); |
710 | break; |
711 | case ISD::STRICT_FP_ROUND: |
712 | Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo); |
713 | break; |
714 | case ISD::FP_ROUND: |
715 | Res = ScalarizeVecOp_FP_ROUND(N, OpNo); |
716 | break; |
717 | case ISD::STRICT_FP_EXTEND: |
718 | Res = ScalarizeVecOp_STRICT_FP_EXTEND(N); |
719 | break; |
720 | case ISD::FP_EXTEND: |
721 | Res = ScalarizeVecOp_FP_EXTEND(N); |
722 | break; |
723 | case ISD::VECREDUCE_FADD: |
724 | case ISD::VECREDUCE_FMUL: |
725 | case ISD::VECREDUCE_ADD: |
726 | case ISD::VECREDUCE_MUL: |
727 | case ISD::VECREDUCE_AND: |
728 | case ISD::VECREDUCE_OR: |
729 | case ISD::VECREDUCE_XOR: |
730 | case ISD::VECREDUCE_SMAX: |
731 | case ISD::VECREDUCE_SMIN: |
732 | case ISD::VECREDUCE_UMAX: |
733 | case ISD::VECREDUCE_UMIN: |
734 | case ISD::VECREDUCE_FMAX: |
735 | case ISD::VECREDUCE_FMIN: |
736 | case ISD::VECREDUCE_FMAXIMUM: |
737 | case ISD::VECREDUCE_FMINIMUM: |
738 | Res = ScalarizeVecOp_VECREDUCE(N); |
739 | break; |
740 | case ISD::VECREDUCE_SEQ_FADD: |
741 | case ISD::VECREDUCE_SEQ_FMUL: |
742 | Res = ScalarizeVecOp_VECREDUCE_SEQ(N); |
743 | break; |
744 | } |
745 | |
746 | // If the result is null, the sub-method took care of registering results etc. |
747 | if (!Res.getNode()) return false; |
748 | |
749 | // If the result is N, the sub-method updated N in place. Tell the legalizer |
750 | // core about this. |
751 | if (Res.getNode() == N) |
752 | return true; |
753 | |
754 | assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && |
755 | "Invalid operand expansion" ); |
756 | |
757 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
758 | return false; |
759 | } |
760 | |
761 | /// If the value to convert is a vector that needs to be scalarized, it must be |
762 | /// <1 x ty>. Convert the element instead. |
763 | SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { |
764 | SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
765 | return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), |
766 | VT: N->getValueType(ResNo: 0), Operand: Elt); |
767 | } |
768 | |
769 | /// If the input is a vector that needs to be scalarized, it must be <1 x ty>. |
770 | /// Do the operation on the element instead. |
771 | SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) { |
772 | assert(N->getValueType(0).getVectorNumElements() == 1 && |
773 | "Unexpected vector type!" ); |
774 | SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
775 | SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), |
776 | VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt); |
777 | // Revectorize the result so the types line up with what the uses of this |
778 | // expression expect. |
779 | return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op); |
780 | } |
781 | |
782 | /// If the input is a vector that needs to be scalarized, it must be <1 x ty>. |
783 | /// Do the strict FP operation on the element instead. |
784 | SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) { |
785 | assert(N->getValueType(0).getVectorNumElements() == 1 && |
786 | "Unexpected vector type!" ); |
787 | SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1)); |
788 | SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), |
789 | { N->getValueType(0).getScalarType(), MVT::Other }, |
790 | { N->getOperand(0), Elt }); |
791 | // Legalize the chain result - switch anything that used the old chain to |
792 | // use the new one. |
793 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
794 | // Revectorize the result so the types line up with what the uses of this |
795 | // expression expect. |
796 | Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res); |
797 | |
798 | // Do our own replacement and return SDValue() to tell the caller that we |
799 | // handled all replacements since caller can only handle a single result. |
800 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
801 | return SDValue(); |
802 | } |
803 | |
804 | /// The vectors to concatenate have length one - use a BUILD_VECTOR instead. |
805 | SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { |
806 | SmallVector<SDValue, 8> Ops(N->getNumOperands()); |
807 | for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) |
808 | Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i)); |
809 | return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops); |
810 | } |
811 | |
812 | /// If the input is a vector that needs to be scalarized, it must be <1 x ty>, |
813 | /// so just return the element, ignoring the index. |
814 | SDValue DAGTypeLegalizer::(SDNode *N) { |
815 | EVT VT = N->getValueType(ResNo: 0); |
816 | SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
817 | if (Res.getValueType() != VT) |
818 | Res = VT.isFloatingPoint() |
819 | ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res) |
820 | : DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res); |
821 | return Res; |
822 | } |
823 | |
824 | /// If the input condition is a vector that needs to be scalarized, it must be |
825 | /// <1 x i1>, so just convert to a normal ISD::SELECT |
826 | /// (still with vector output type since that was acceptable if we got here). |
827 | SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) { |
828 | SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
829 | EVT VT = N->getValueType(ResNo: 0); |
830 | |
831 | return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1), |
832 | N3: N->getOperand(Num: 2)); |
833 | } |
834 | |
835 | /// If the operand is a vector that needs to be scalarized then the |
836 | /// result must be v1i1, so just convert to a scalar SETCC and wrap |
837 | /// with a scalar_to_vector since the res type is legal if we got here |
838 | SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) { |
839 | assert(N->getValueType(0).isVector() && |
840 | N->getOperand(0).getValueType().isVector() && |
841 | "Operand types must be vectors" ); |
842 | assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type" ); |
843 | |
844 | EVT VT = N->getValueType(ResNo: 0); |
845 | SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
846 | SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1)); |
847 | |
848 | EVT OpVT = N->getOperand(Num: 0).getValueType(); |
849 | EVT NVT = VT.getVectorElementType(); |
850 | SDLoc DL(N); |
851 | // Turn it into a scalar SETCC. |
852 | SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, |
853 | N->getOperand(Num: 2)); |
854 | |
855 | // Vectors may have a different boolean contents to scalars. Promote the |
856 | // value appropriately. |
857 | ISD::NodeType ExtendCode = |
858 | TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT)); |
859 | |
860 | Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res); |
861 | |
862 | return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res); |
863 | } |
864 | |
865 | /// If the value to store is a vector that needs to be scalarized, it must be |
866 | /// <1 x ty>. Just store the element. |
867 | SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ |
868 | assert(N->isUnindexed() && "Indexed store of one-element vector?" ); |
869 | assert(OpNo == 1 && "Do not know how to scalarize this operand!" ); |
870 | SDLoc dl(N); |
871 | |
872 | if (N->isTruncatingStore()) |
873 | return DAG.getTruncStore( |
874 | Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)), |
875 | Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(), |
876 | SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getOriginalAlign(), |
877 | MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo()); |
878 | |
879 | return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)), |
880 | Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(), |
881 | Alignment: N->getOriginalAlign(), MMOFlags: N->getMemOperand()->getFlags(), |
882 | AAInfo: N->getAAInfo()); |
883 | } |
884 | |
885 | /// If the value to round is a vector that needs to be scalarized, it must be |
886 | /// <1 x ty>. Convert the element instead. |
887 | SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) { |
888 | assert(OpNo == 0 && "Wrong operand for scalarization!" ); |
889 | SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
890 | SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N), |
891 | VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt, |
892 | N2: N->getOperand(Num: 1)); |
893 | return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res); |
894 | } |
895 | |
896 | SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, |
897 | unsigned OpNo) { |
898 | assert(OpNo == 1 && "Wrong operand for scalarization!" ); |
899 | SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1)); |
900 | SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), |
901 | { N->getValueType(0).getVectorElementType(), |
902 | MVT::Other }, |
903 | { N->getOperand(0), Elt, N->getOperand(2) }); |
904 | // Legalize the chain result - switch anything that used the old chain to |
905 | // use the new one. |
906 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
907 | |
908 | Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res); |
909 | |
910 | // Do our own replacement and return SDValue() to tell the caller that we |
911 | // handled all replacements since caller can only handle a single result. |
912 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
913 | return SDValue(); |
914 | } |
915 | |
916 | /// If the value to extend is a vector that needs to be scalarized, it must be |
917 | /// <1 x ty>. Convert the element instead. |
918 | SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) { |
919 | SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
920 | SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), |
921 | VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt); |
922 | return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res); |
923 | } |
924 | |
925 | /// If the value to extend is a vector that needs to be scalarized, it must be |
926 | /// <1 x ty>. Convert the element instead. |
927 | SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) { |
928 | SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1)); |
929 | SDValue Res = |
930 | DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), |
931 | {N->getValueType(0).getVectorElementType(), MVT::Other}, |
932 | {N->getOperand(0), Elt}); |
933 | // Legalize the chain result - switch anything that used the old chain to |
934 | // use the new one. |
935 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
936 | |
937 | Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res); |
938 | |
939 | // Do our own replacement and return SDValue() to tell the caller that we |
940 | // handled all replacements since caller can only handle a single result. |
941 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
942 | return SDValue(); |
943 | } |
944 | |
945 | SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) { |
946 | SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
947 | // Result type may be wider than element type. |
948 | if (Res.getValueType() != N->getValueType(ResNo: 0)) |
949 | Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res); |
950 | return Res; |
951 | } |
952 | |
953 | SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) { |
954 | SDValue AccOp = N->getOperand(Num: 0); |
955 | SDValue VecOp = N->getOperand(Num: 1); |
956 | |
957 | unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode()); |
958 | |
959 | SDValue Op = GetScalarizedVector(Op: VecOp); |
960 | return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), |
961 | N1: AccOp, N2: Op, Flags: N->getFlags()); |
962 | } |
963 | |
964 | //===----------------------------------------------------------------------===// |
965 | // Result Vector Splitting |
966 | //===----------------------------------------------------------------------===// |
967 | |
968 | /// This method is called when the specified result of the specified node is |
969 | /// found to need vector splitting. At this point, the node may also have |
970 | /// invalid operands or may have other results that need legalization, we just |
971 | /// know that (at least) one result needs vector splitting. |
972 | void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { |
973 | LLVM_DEBUG(dbgs() << "Split node result: " ; N->dump(&DAG)); |
974 | SDValue Lo, Hi; |
975 | |
976 | // See if the target wants to custom expand this node. |
977 | if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) |
978 | return; |
979 | |
980 | switch (N->getOpcode()) { |
981 | default: |
982 | #ifndef NDEBUG |
983 | dbgs() << "SplitVectorResult #" << ResNo << ": " ; |
984 | N->dump(G: &DAG); |
985 | dbgs() << "\n" ; |
986 | #endif |
987 | report_fatal_error(reason: "Do not know how to split the result of this " |
988 | "operator!\n" ); |
989 | |
990 | case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; |
991 | case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break; |
992 | case ISD::VSELECT: |
993 | case ISD::SELECT: |
994 | case ISD::VP_MERGE: |
995 | case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break; |
996 | case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; |
997 | case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; |
998 | case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break; |
999 | case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; |
1000 | case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; |
1001 | case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; |
1002 | case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break; |
1003 | case ISD::FPOWI: |
1004 | case ISD::FLDEXP: |
1005 | case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break; |
1006 | case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break; |
1007 | case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; |
1008 | case ISD::SPLAT_VECTOR: |
1009 | case ISD::SCALAR_TO_VECTOR: |
1010 | SplitVecRes_ScalarOp(N, Lo, Hi); |
1011 | break; |
1012 | case ISD::STEP_VECTOR: |
1013 | SplitVecRes_STEP_VECTOR(N, Lo, Hi); |
1014 | break; |
1015 | case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; |
1016 | case ISD::LOAD: |
1017 | SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi); |
1018 | break; |
1019 | case ISD::VP_LOAD: |
1020 | SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi); |
1021 | break; |
1022 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: |
1023 | SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi); |
1024 | break; |
1025 | case ISD::MLOAD: |
1026 | SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi); |
1027 | break; |
1028 | case ISD::MGATHER: |
1029 | case ISD::VP_GATHER: |
1030 | SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true); |
1031 | break; |
1032 | case ISD::SETCC: |
1033 | case ISD::VP_SETCC: |
1034 | SplitVecRes_SETCC(N, Lo, Hi); |
1035 | break; |
1036 | case ISD::VECTOR_REVERSE: |
1037 | SplitVecRes_VECTOR_REVERSE(N, Lo, Hi); |
1038 | break; |
1039 | case ISD::VECTOR_SHUFFLE: |
1040 | SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi); |
1041 | break; |
1042 | case ISD::VECTOR_SPLICE: |
1043 | SplitVecRes_VECTOR_SPLICE(N, Lo, Hi); |
1044 | break; |
1045 | case ISD::VECTOR_DEINTERLEAVE: |
1046 | SplitVecRes_VECTOR_DEINTERLEAVE(N); |
1047 | return; |
1048 | case ISD::VECTOR_INTERLEAVE: |
1049 | SplitVecRes_VECTOR_INTERLEAVE(N); |
1050 | return; |
1051 | case ISD::VAARG: |
1052 | SplitVecRes_VAARG(N, Lo, Hi); |
1053 | break; |
1054 | |
1055 | case ISD::ANY_EXTEND_VECTOR_INREG: |
1056 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
1057 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
1058 | SplitVecRes_ExtVecInRegOp(N, Lo, Hi); |
1059 | break; |
1060 | |
1061 | case ISD::ABS: |
1062 | case ISD::VP_ABS: |
1063 | case ISD::BITREVERSE: |
1064 | case ISD::VP_BITREVERSE: |
1065 | case ISD::BSWAP: |
1066 | case ISD::VP_BSWAP: |
1067 | case ISD::CTLZ: |
1068 | case ISD::VP_CTLZ: |
1069 | case ISD::CTTZ: |
1070 | case ISD::VP_CTTZ: |
1071 | case ISD::CTLZ_ZERO_UNDEF: |
1072 | case ISD::VP_CTLZ_ZERO_UNDEF: |
1073 | case ISD::CTTZ_ZERO_UNDEF: |
1074 | case ISD::VP_CTTZ_ZERO_UNDEF: |
1075 | case ISD::CTPOP: |
1076 | case ISD::VP_CTPOP: |
1077 | case ISD::FABS: case ISD::VP_FABS: |
1078 | case ISD::FCEIL: |
1079 | case ISD::VP_FCEIL: |
1080 | case ISD::FCOS: |
1081 | case ISD::FEXP: |
1082 | case ISD::FEXP2: |
1083 | case ISD::FEXP10: |
1084 | case ISD::FFLOOR: |
1085 | case ISD::VP_FFLOOR: |
1086 | case ISD::FLOG: |
1087 | case ISD::FLOG10: |
1088 | case ISD::FLOG2: |
1089 | case ISD::FNEARBYINT: |
1090 | case ISD::VP_FNEARBYINT: |
1091 | case ISD::FNEG: case ISD::VP_FNEG: |
1092 | case ISD::FREEZE: |
1093 | case ISD::ARITH_FENCE: |
1094 | case ISD::FP_EXTEND: |
1095 | case ISD::VP_FP_EXTEND: |
1096 | case ISD::FP_ROUND: |
1097 | case ISD::VP_FP_ROUND: |
1098 | case ISD::FP_TO_SINT: |
1099 | case ISD::VP_FP_TO_SINT: |
1100 | case ISD::FP_TO_UINT: |
1101 | case ISD::VP_FP_TO_UINT: |
1102 | case ISD::FRINT: |
1103 | case ISD::VP_FRINT: |
1104 | case ISD::LRINT: |
1105 | case ISD::VP_LRINT: |
1106 | case ISD::LLRINT: |
1107 | case ISD::VP_LLRINT: |
1108 | case ISD::FROUND: |
1109 | case ISD::VP_FROUND: |
1110 | case ISD::FROUNDEVEN: |
1111 | case ISD::VP_FROUNDEVEN: |
1112 | case ISD::FSIN: |
1113 | case ISD::FSQRT: case ISD::VP_SQRT: |
1114 | case ISD::FTRUNC: |
1115 | case ISD::VP_FROUNDTOZERO: |
1116 | case ISD::SINT_TO_FP: |
1117 | case ISD::VP_SINT_TO_FP: |
1118 | case ISD::TRUNCATE: |
1119 | case ISD::VP_TRUNCATE: |
1120 | case ISD::UINT_TO_FP: |
1121 | case ISD::VP_UINT_TO_FP: |
1122 | case ISD::FCANONICALIZE: |
1123 | SplitVecRes_UnaryOp(N, Lo, Hi); |
1124 | break; |
1125 | case ISD::FFREXP: |
1126 | SplitVecRes_FFREXP(N, ResNo, Lo, Hi); |
1127 | break; |
1128 | |
1129 | case ISD::ANY_EXTEND: |
1130 | case ISD::SIGN_EXTEND: |
1131 | case ISD::ZERO_EXTEND: |
1132 | case ISD::VP_SIGN_EXTEND: |
1133 | case ISD::VP_ZERO_EXTEND: |
1134 | SplitVecRes_ExtendOp(N, Lo, Hi); |
1135 | break; |
1136 | |
1137 | case ISD::ADD: case ISD::VP_ADD: |
1138 | case ISD::SUB: case ISD::VP_SUB: |
1139 | case ISD::MUL: case ISD::VP_MUL: |
1140 | case ISD::MULHS: |
1141 | case ISD::MULHU: |
1142 | case ISD::FADD: case ISD::VP_FADD: |
1143 | case ISD::FSUB: case ISD::VP_FSUB: |
1144 | case ISD::FMUL: case ISD::VP_FMUL: |
1145 | case ISD::FMINNUM: case ISD::VP_FMINNUM: |
1146 | case ISD::FMAXNUM: case ISD::VP_FMAXNUM: |
1147 | case ISD::FMINIMUM: |
1148 | case ISD::VP_FMINIMUM: |
1149 | case ISD::FMAXIMUM: |
1150 | case ISD::VP_FMAXIMUM: |
1151 | case ISD::SDIV: case ISD::VP_SDIV: |
1152 | case ISD::UDIV: case ISD::VP_UDIV: |
1153 | case ISD::FDIV: case ISD::VP_FDIV: |
1154 | case ISD::FPOW: |
1155 | case ISD::AND: case ISD::VP_AND: |
1156 | case ISD::OR: case ISD::VP_OR: |
1157 | case ISD::XOR: case ISD::VP_XOR: |
1158 | case ISD::SHL: case ISD::VP_SHL: |
1159 | case ISD::SRA: case ISD::VP_ASHR: |
1160 | case ISD::SRL: case ISD::VP_LSHR: |
1161 | case ISD::UREM: case ISD::VP_UREM: |
1162 | case ISD::SREM: case ISD::VP_SREM: |
1163 | case ISD::FREM: case ISD::VP_FREM: |
1164 | case ISD::SMIN: case ISD::VP_SMIN: |
1165 | case ISD::SMAX: case ISD::VP_SMAX: |
1166 | case ISD::UMIN: case ISD::VP_UMIN: |
1167 | case ISD::UMAX: case ISD::VP_UMAX: |
1168 | case ISD::SADDSAT: case ISD::VP_SADDSAT: |
1169 | case ISD::UADDSAT: case ISD::VP_UADDSAT: |
1170 | case ISD::SSUBSAT: case ISD::VP_SSUBSAT: |
1171 | case ISD::USUBSAT: case ISD::VP_USUBSAT: |
1172 | case ISD::SSHLSAT: |
1173 | case ISD::USHLSAT: |
1174 | case ISD::ROTL: |
1175 | case ISD::ROTR: |
1176 | case ISD::VP_FCOPYSIGN: |
1177 | SplitVecRes_BinOp(N, Lo, Hi); |
1178 | break; |
1179 | case ISD::FMA: case ISD::VP_FMA: |
1180 | case ISD::FSHL: |
1181 | case ISD::VP_FSHL: |
1182 | case ISD::FSHR: |
1183 | case ISD::VP_FSHR: |
1184 | SplitVecRes_TernaryOp(N, Lo, Hi); |
1185 | break; |
1186 | |
1187 | #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ |
1188 | case ISD::STRICT_##DAGN: |
1189 | #include "llvm/IR/ConstrainedOps.def" |
1190 | SplitVecRes_StrictFPOp(N, Lo, Hi); |
1191 | break; |
1192 | |
1193 | case ISD::FP_TO_UINT_SAT: |
1194 | case ISD::FP_TO_SINT_SAT: |
1195 | SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi); |
1196 | break; |
1197 | |
1198 | case ISD::UADDO: |
1199 | case ISD::SADDO: |
1200 | case ISD::USUBO: |
1201 | case ISD::SSUBO: |
1202 | case ISD::UMULO: |
1203 | case ISD::SMULO: |
1204 | SplitVecRes_OverflowOp(N, ResNo, Lo, Hi); |
1205 | break; |
1206 | case ISD::SMULFIX: |
1207 | case ISD::SMULFIXSAT: |
1208 | case ISD::UMULFIX: |
1209 | case ISD::UMULFIXSAT: |
1210 | case ISD::SDIVFIX: |
1211 | case ISD::SDIVFIXSAT: |
1212 | case ISD::UDIVFIX: |
1213 | case ISD::UDIVFIXSAT: |
1214 | SplitVecRes_FIX(N, Lo, Hi); |
1215 | break; |
1216 | case ISD::EXPERIMENTAL_VP_REVERSE: |
1217 | SplitVecRes_VP_REVERSE(N, Lo, Hi); |
1218 | break; |
1219 | } |
1220 | |
1221 | // If Lo/Hi is null, the sub-method took care of registering results etc. |
1222 | if (Lo.getNode()) |
1223 | SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi); |
1224 | } |
1225 | |
1226 | void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT, |
1227 | MachinePointerInfo &MPI, SDValue &Ptr, |
1228 | uint64_t *ScaledOffset) { |
1229 | SDLoc DL(N); |
1230 | unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8; |
1231 | |
1232 | if (MemVT.isScalableVector()) { |
1233 | SDNodeFlags Flags; |
1234 | SDValue BytesIncrement = DAG.getVScale( |
1235 | DL, VT: Ptr.getValueType(), |
1236 | MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize)); |
1237 | MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace()); |
1238 | Flags.setNoUnsignedWrap(true); |
1239 | if (ScaledOffset) |
1240 | *ScaledOffset += IncrementSize; |
1241 | Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement, |
1242 | Flags); |
1243 | } else { |
1244 | MPI = N->getPointerInfo().getWithOffset(O: IncrementSize); |
1245 | // Increment the pointer to the other half. |
1246 | Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize)); |
1247 | } |
1248 | } |
1249 | |
1250 | std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) { |
1251 | return SplitMask(Mask, DL: SDLoc(Mask)); |
1252 | } |
1253 | |
1254 | std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask, |
1255 | const SDLoc &DL) { |
1256 | SDValue MaskLo, MaskHi; |
1257 | EVT MaskVT = Mask.getValueType(); |
1258 | if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector) |
1259 | GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi); |
1260 | else |
1261 | std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL); |
1262 | return std::make_pair(x&: MaskLo, y&: MaskHi); |
1263 | } |
1264 | |
1265 | void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) { |
1266 | SDValue LHSLo, LHSHi; |
1267 | GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi); |
1268 | SDValue RHSLo, RHSHi; |
1269 | GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi); |
1270 | SDLoc dl(N); |
1271 | |
1272 | const SDNodeFlags Flags = N->getFlags(); |
1273 | unsigned Opcode = N->getOpcode(); |
1274 | if (N->getNumOperands() == 2) { |
1275 | Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags); |
1276 | Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags); |
1277 | return; |
1278 | } |
1279 | |
1280 | assert(N->getNumOperands() == 4 && "Unexpected number of operands!" ); |
1281 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
1282 | |
1283 | SDValue MaskLo, MaskHi; |
1284 | std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2)); |
1285 | |
1286 | SDValue EVLLo, EVLHi; |
1287 | std::tie(args&: EVLLo, args&: EVLHi) = |
1288 | DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl); |
1289 | |
1290 | Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), |
1291 | Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags); |
1292 | Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), |
1293 | Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags); |
1294 | } |
1295 | |
1296 | void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, |
1297 | SDValue &Hi) { |
1298 | SDValue Op0Lo, Op0Hi; |
1299 | GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi); |
1300 | SDValue Op1Lo, Op1Hi; |
1301 | GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi); |
1302 | SDValue Op2Lo, Op2Hi; |
1303 | GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi); |
1304 | SDLoc dl(N); |
1305 | |
1306 | const SDNodeFlags Flags = N->getFlags(); |
1307 | unsigned Opcode = N->getOpcode(); |
1308 | if (N->getNumOperands() == 3) { |
1309 | Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags); |
1310 | Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags); |
1311 | return; |
1312 | } |
1313 | |
1314 | assert(N->getNumOperands() == 5 && "Unexpected number of operands!" ); |
1315 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
1316 | |
1317 | SDValue MaskLo, MaskHi; |
1318 | std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3)); |
1319 | |
1320 | SDValue EVLLo, EVLHi; |
1321 | std::tie(args&: EVLLo, args&: EVLHi) = |
1322 | DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL: dl); |
1323 | |
1324 | Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), |
1325 | Ops: {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags); |
1326 | Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), |
1327 | Ops: {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags); |
1328 | } |
1329 | |
1330 | void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) { |
1331 | SDValue LHSLo, LHSHi; |
1332 | GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi); |
1333 | SDValue RHSLo, RHSHi; |
1334 | GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi); |
1335 | SDLoc dl(N); |
1336 | SDValue Op2 = N->getOperand(Num: 2); |
1337 | |
1338 | unsigned Opcode = N->getOpcode(); |
1339 | Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2, |
1340 | Flags: N->getFlags()); |
1341 | Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2, |
1342 | Flags: N->getFlags()); |
1343 | } |
1344 | |
1345 | void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, |
1346 | SDValue &Hi) { |
1347 | // We know the result is a vector. The input may be either a vector or a |
1348 | // scalar value. |
1349 | EVT LoVT, HiVT; |
1350 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
1351 | SDLoc dl(N); |
1352 | |
1353 | SDValue InOp = N->getOperand(Num: 0); |
1354 | EVT InVT = InOp.getValueType(); |
1355 | |
1356 | // Handle some special cases efficiently. |
1357 | switch (getTypeAction(VT: InVT)) { |
1358 | case TargetLowering::TypeLegal: |
1359 | case TargetLowering::TypePromoteInteger: |
1360 | case TargetLowering::TypePromoteFloat: |
1361 | case TargetLowering::TypeSoftPromoteHalf: |
1362 | case TargetLowering::TypeSoftenFloat: |
1363 | case TargetLowering::TypeScalarizeVector: |
1364 | case TargetLowering::TypeWidenVector: |
1365 | break; |
1366 | case TargetLowering::TypeExpandInteger: |
1367 | case TargetLowering::TypeExpandFloat: |
1368 | // A scalar to vector conversion, where the scalar needs expansion. |
1369 | // If the vector is being split in two then we can just convert the |
1370 | // expanded pieces. |
1371 | if (LoVT == HiVT) { |
1372 | GetExpandedOp(Op: InOp, Lo, Hi); |
1373 | if (DAG.getDataLayout().isBigEndian()) |
1374 | std::swap(a&: Lo, b&: Hi); |
1375 | Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo); |
1376 | Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi); |
1377 | return; |
1378 | } |
1379 | break; |
1380 | case TargetLowering::TypeSplitVector: |
1381 | // If the input is a vector that needs to be split, convert each split |
1382 | // piece of the input now. |
1383 | GetSplitVector(Op: InOp, Lo, Hi); |
1384 | Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo); |
1385 | Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi); |
1386 | return; |
1387 | case TargetLowering::TypeScalarizeScalableVector: |
1388 | report_fatal_error(reason: "Scalarization of scalable vectors is not supported." ); |
1389 | } |
1390 | |
1391 | if (LoVT.isScalableVector()) { |
1392 | auto [InLo, InHi] = DAG.SplitVectorOperand(N, OpNo: 0); |
1393 | Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: InLo); |
1394 | Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: InHi); |
1395 | return; |
1396 | } |
1397 | |
1398 | // In the general case, convert the input to an integer and split it by hand. |
1399 | EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits()); |
1400 | EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits()); |
1401 | if (DAG.getDataLayout().isBigEndian()) |
1402 | std::swap(a&: LoIntVT, b&: HiIntVT); |
1403 | |
1404 | SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi); |
1405 | |
1406 | if (DAG.getDataLayout().isBigEndian()) |
1407 | std::swap(a&: Lo, b&: Hi); |
1408 | Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo); |
1409 | Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi); |
1410 | } |
1411 | |
1412 | void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, |
1413 | SDValue &Hi) { |
1414 | EVT LoVT, HiVT; |
1415 | SDLoc dl(N); |
1416 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
1417 | unsigned LoNumElts = LoVT.getVectorNumElements(); |
1418 | SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); |
1419 | Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps); |
1420 | |
1421 | SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); |
1422 | Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps); |
1423 | } |
1424 | |
1425 | void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, |
1426 | SDValue &Hi) { |
1427 | assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS" ); |
1428 | SDLoc dl(N); |
1429 | unsigned NumSubvectors = N->getNumOperands() / 2; |
1430 | if (NumSubvectors == 1) { |
1431 | Lo = N->getOperand(Num: 0); |
1432 | Hi = N->getOperand(Num: 1); |
1433 | return; |
1434 | } |
1435 | |
1436 | EVT LoVT, HiVT; |
1437 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
1438 | |
1439 | SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); |
1440 | Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps); |
1441 | |
1442 | SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); |
1443 | Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps); |
1444 | } |
1445 | |
1446 | void DAGTypeLegalizer::(SDNode *N, SDValue &Lo, |
1447 | SDValue &Hi) { |
1448 | SDValue Vec = N->getOperand(Num: 0); |
1449 | SDValue Idx = N->getOperand(Num: 1); |
1450 | SDLoc dl(N); |
1451 | |
1452 | EVT LoVT, HiVT; |
1453 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
1454 | |
1455 | Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx); |
1456 | uint64_t IdxVal = Idx->getAsZExtVal(); |
1457 | Hi = DAG.getNode( |
1458 | Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec, |
1459 | N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl)); |
1460 | } |
1461 | |
1462 | void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, |
1463 | SDValue &Hi) { |
1464 | SDValue Vec = N->getOperand(Num: 0); |
1465 | SDValue SubVec = N->getOperand(Num: 1); |
1466 | SDValue Idx = N->getOperand(Num: 2); |
1467 | SDLoc dl(N); |
1468 | GetSplitVector(Op: Vec, Lo, Hi); |
1469 | |
1470 | EVT VecVT = Vec.getValueType(); |
1471 | EVT LoVT = Lo.getValueType(); |
1472 | EVT SubVecVT = SubVec.getValueType(); |
1473 | unsigned VecElems = VecVT.getVectorMinNumElements(); |
1474 | unsigned SubElems = SubVecVT.getVectorMinNumElements(); |
1475 | unsigned LoElems = LoVT.getVectorMinNumElements(); |
1476 | |
1477 | // If we know the index is in the first half, and we know the subvector |
1478 | // doesn't cross the boundary between the halves, we can avoid spilling the |
1479 | // vector, and insert into the lower half of the split vector directly. |
1480 | unsigned IdxVal = Idx->getAsZExtVal(); |
1481 | if (IdxVal + SubElems <= LoElems) { |
1482 | Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx); |
1483 | return; |
1484 | } |
1485 | // Similarly if the subvector is fully in the high half, but mind that we |
1486 | // can't tell whether a fixed-length subvector is fully within the high half |
1487 | // of a scalable vector. |
1488 | if (VecVT.isScalableVector() == SubVecVT.isScalableVector() && |
1489 | IdxVal >= LoElems && IdxVal + SubElems <= VecElems) { |
1490 | Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec, |
1491 | N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl)); |
1492 | return; |
1493 | } |
1494 | |
1495 | // Spill the vector to the stack. |
1496 | // In cases where the vector is illegal it will be broken down into parts |
1497 | // and stored in parts - we should use the alignment for the smallest part. |
1498 | Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false); |
1499 | SDValue StackPtr = |
1500 | DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign); |
1501 | auto &MF = DAG.getMachineFunction(); |
1502 | auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex(); |
1503 | auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex); |
1504 | |
1505 | SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo, |
1506 | Alignment: SmallestAlign); |
1507 | |
1508 | // Store the new subvector into the specified index. |
1509 | SDValue SubVecPtr = |
1510 | TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx); |
1511 | Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr, |
1512 | PtrInfo: MachinePointerInfo::getUnknownStack(MF)); |
1513 | |
1514 | // Load the Lo part from the stack slot. |
1515 | Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo, |
1516 | Alignment: SmallestAlign); |
1517 | |
1518 | // Increment the pointer to the other part. |
1519 | auto *Load = cast<LoadSDNode>(Val&: Lo); |
1520 | MachinePointerInfo MPI = Load->getPointerInfo(); |
1521 | IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr); |
1522 | |
1523 | // Load the Hi part from the stack slot. |
1524 | Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign); |
1525 | } |
1526 | |
1527 | // Handle splitting an FP where the second operand does not match the first |
1528 | // type. The second operand may be a scalar, or a vector that has exactly as |
1529 | // many elements as the first |
1530 | void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo, |
1531 | SDValue &Hi) { |
1532 | SDValue LHSLo, LHSHi; |
1533 | GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi); |
1534 | SDLoc DL(N); |
1535 | |
1536 | SDValue RHSLo, RHSHi; |
1537 | SDValue RHS = N->getOperand(Num: 1); |
1538 | EVT RHSVT = RHS.getValueType(); |
1539 | if (RHSVT.isVector()) { |
1540 | if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector) |
1541 | GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi); |
1542 | else |
1543 | std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS)); |
1544 | |
1545 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo); |
1546 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi); |
1547 | } else { |
1548 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS); |
1549 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS); |
1550 | } |
1551 | } |
1552 | |
1553 | void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo, |
1554 | SDValue &Hi) { |
1555 | SDLoc DL(N); |
1556 | SDValue ArgLo, ArgHi; |
1557 | SDValue Test = N->getOperand(Num: 1); |
1558 | SDValue FpValue = N->getOperand(Num: 0); |
1559 | if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector) |
1560 | GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi); |
1561 | else |
1562 | std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue)); |
1563 | EVT LoVT, HiVT; |
1564 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
1565 | |
1566 | Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags()); |
1567 | Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags()); |
1568 | } |
1569 | |
1570 | void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, |
1571 | SDValue &Hi) { |
1572 | SDValue LHSLo, LHSHi; |
1573 | GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi); |
1574 | SDLoc dl(N); |
1575 | |
1576 | EVT LoVT, HiVT; |
1577 | std::tie(args&: LoVT, args&: HiVT) = |
1578 | DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT()); |
1579 | |
1580 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, |
1581 | N2: DAG.getValueType(LoVT)); |
1582 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, |
1583 | N2: DAG.getValueType(HiVT)); |
1584 | } |
1585 | |
1586 | void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, |
1587 | SDValue &Hi) { |
1588 | unsigned Opcode = N->getOpcode(); |
1589 | SDValue N0 = N->getOperand(Num: 0); |
1590 | |
1591 | SDLoc dl(N); |
1592 | SDValue InLo, InHi; |
1593 | |
1594 | if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector) |
1595 | GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi); |
1596 | else |
1597 | std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0); |
1598 | |
1599 | EVT InLoVT = InLo.getValueType(); |
1600 | unsigned InNumElements = InLoVT.getVectorNumElements(); |
1601 | |
1602 | EVT OutLoVT, OutHiVT; |
1603 | std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
1604 | unsigned OutNumElements = OutLoVT.getVectorNumElements(); |
1605 | assert((2 * OutNumElements) <= InNumElements && |
1606 | "Illegal extend vector in reg split" ); |
1607 | |
1608 | // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the |
1609 | // input vector (i.e. we only use InLo): |
1610 | // OutLo will extend the first OutNumElements from InLo. |
1611 | // OutHi will extend the next OutNumElements from InLo. |
1612 | |
1613 | // Shuffle the elements from InLo for OutHi into the bottom elements to |
1614 | // create a 'fake' InHi. |
1615 | SmallVector<int, 8> SplitHi(InNumElements, -1); |
1616 | for (unsigned i = 0; i != OutNumElements; ++i) |
1617 | SplitHi[i] = i + OutNumElements; |
1618 | InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getUNDEF(VT: InLoVT), Mask: SplitHi); |
1619 | |
1620 | Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo); |
1621 | Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi); |
1622 | } |
1623 | |
1624 | void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, |
1625 | SDValue &Hi) { |
1626 | unsigned NumOps = N->getNumOperands(); |
1627 | SDValue Chain = N->getOperand(Num: 0); |
1628 | EVT LoVT, HiVT; |
1629 | SDLoc dl(N); |
1630 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
1631 | |
1632 | SmallVector<SDValue, 4> OpsLo(NumOps); |
1633 | SmallVector<SDValue, 4> OpsHi(NumOps); |
1634 | |
1635 | // The Chain is the first operand. |
1636 | OpsLo[0] = Chain; |
1637 | OpsHi[0] = Chain; |
1638 | |
1639 | // Now process the remaining operands. |
1640 | for (unsigned i = 1; i < NumOps; ++i) { |
1641 | SDValue Op = N->getOperand(Num: i); |
1642 | SDValue OpLo = Op; |
1643 | SDValue OpHi = Op; |
1644 | |
1645 | EVT InVT = Op.getValueType(); |
1646 | if (InVT.isVector()) { |
1647 | // If the input also splits, handle it directly for a |
1648 | // compile time speedup. Otherwise split it by hand. |
1649 | if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) |
1650 | GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi); |
1651 | else |
1652 | std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i); |
1653 | } |
1654 | |
1655 | OpsLo[i] = OpLo; |
1656 | OpsHi[i] = OpHi; |
1657 | } |
1658 | |
1659 | EVT LoValueVTs[] = {LoVT, MVT::Other}; |
1660 | EVT HiValueVTs[] = {HiVT, MVT::Other}; |
1661 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(LoValueVTs), Ops: OpsLo, |
1662 | Flags: N->getFlags()); |
1663 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(HiValueVTs), Ops: OpsHi, |
1664 | Flags: N->getFlags()); |
1665 | |
1666 | // Build a factor node to remember that this Op is independent of the |
1667 | // other one. |
1668 | Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, |
1669 | Lo.getValue(R: 1), Hi.getValue(R: 1)); |
1670 | |
1671 | // Legalize the chain result - switch anything that used the old chain to |
1672 | // use the new one. |
1673 | ReplaceValueWith(From: SDValue(N, 1), To: Chain); |
1674 | } |
1675 | |
1676 | SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) { |
1677 | SDValue Chain = N->getOperand(Num: 0); |
1678 | EVT VT = N->getValueType(ResNo: 0); |
1679 | unsigned NE = VT.getVectorNumElements(); |
1680 | EVT EltVT = VT.getVectorElementType(); |
1681 | SDLoc dl(N); |
1682 | |
1683 | SmallVector<SDValue, 8> Scalars; |
1684 | SmallVector<SDValue, 4> Operands(N->getNumOperands()); |
1685 | |
1686 | // If ResNE is 0, fully unroll the vector op. |
1687 | if (ResNE == 0) |
1688 | ResNE = NE; |
1689 | else if (NE > ResNE) |
1690 | NE = ResNE; |
1691 | |
1692 | //The results of each unrolled operation, including the chain. |
1693 | EVT ChainVTs[] = {EltVT, MVT::Other}; |
1694 | SmallVector<SDValue, 8> Chains; |
1695 | |
1696 | unsigned i; |
1697 | for (i = 0; i != NE; ++i) { |
1698 | Operands[0] = Chain; |
1699 | for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) { |
1700 | SDValue Operand = N->getOperand(Num: j); |
1701 | EVT OperandVT = Operand.getValueType(); |
1702 | if (OperandVT.isVector()) { |
1703 | EVT OperandEltVT = OperandVT.getVectorElementType(); |
1704 | Operands[j] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OperandEltVT, |
1705 | N1: Operand, N2: DAG.getVectorIdxConstant(Val: i, DL: dl)); |
1706 | } else { |
1707 | Operands[j] = Operand; |
1708 | } |
1709 | } |
1710 | SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands); |
1711 | Scalar.getNode()->setFlags(N->getFlags()); |
1712 | |
1713 | //Add in the scalar as well as its chain value to the |
1714 | //result vectors. |
1715 | Scalars.push_back(Elt: Scalar); |
1716 | Chains.push_back(Elt: Scalar.getValue(R: 1)); |
1717 | } |
1718 | |
1719 | for (; i < ResNE; ++i) |
1720 | Scalars.push_back(Elt: DAG.getUNDEF(VT: EltVT)); |
1721 | |
1722 | // Build a new factor node to connect the chain back together. |
1723 | Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); |
1724 | ReplaceValueWith(From: SDValue(N, 1), To: Chain); |
1725 | |
1726 | // Create a new BUILD_VECTOR node |
1727 | EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE); |
1728 | return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars); |
1729 | } |
1730 | |
1731 | void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo, |
1732 | SDValue &Lo, SDValue &Hi) { |
1733 | SDLoc dl(N); |
1734 | EVT ResVT = N->getValueType(ResNo: 0); |
1735 | EVT OvVT = N->getValueType(ResNo: 1); |
1736 | EVT LoResVT, HiResVT, LoOvVT, HiOvVT; |
1737 | std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT); |
1738 | std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT); |
1739 | |
1740 | SDValue LoLHS, HiLHS, LoRHS, HiRHS; |
1741 | if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) { |
1742 | GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS); |
1743 | GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS); |
1744 | } else { |
1745 | std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0); |
1746 | std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1); |
1747 | } |
1748 | |
1749 | unsigned Opcode = N->getOpcode(); |
1750 | SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT); |
1751 | SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT); |
1752 | SDNode *LoNode = DAG.getNode(Opcode, DL: dl, VTList: LoVTs, N1: LoLHS, N2: LoRHS).getNode(); |
1753 | SDNode *HiNode = DAG.getNode(Opcode, DL: dl, VTList: HiVTs, N1: HiLHS, N2: HiRHS).getNode(); |
1754 | LoNode->setFlags(N->getFlags()); |
1755 | HiNode->setFlags(N->getFlags()); |
1756 | |
1757 | Lo = SDValue(LoNode, ResNo); |
1758 | Hi = SDValue(HiNode, ResNo); |
1759 | |
1760 | // Replace the other vector result not being explicitly split here. |
1761 | unsigned OtherNo = 1 - ResNo; |
1762 | EVT OtherVT = N->getValueType(ResNo: OtherNo); |
1763 | if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) { |
1764 | SetSplitVector(Op: SDValue(N, OtherNo), |
1765 | Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo)); |
1766 | } else { |
1767 | SDValue OtherVal = DAG.getNode( |
1768 | Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, |
1769 | N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo)); |
1770 | ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal); |
1771 | } |
1772 | } |
1773 | |
1774 | void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, |
1775 | SDValue &Hi) { |
1776 | SDValue Vec = N->getOperand(Num: 0); |
1777 | SDValue Elt = N->getOperand(Num: 1); |
1778 | SDValue Idx = N->getOperand(Num: 2); |
1779 | SDLoc dl(N); |
1780 | GetSplitVector(Op: Vec, Lo, Hi); |
1781 | |
1782 | if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) { |
1783 | unsigned IdxVal = CIdx->getZExtValue(); |
1784 | unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements(); |
1785 | if (IdxVal < LoNumElts) { |
1786 | Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, |
1787 | VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx); |
1788 | return; |
1789 | } else if (!Vec.getValueType().isScalableVector()) { |
1790 | Hi = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: Elt, |
1791 | N3: DAG.getVectorIdxConstant(Val: IdxVal - LoNumElts, DL: dl)); |
1792 | return; |
1793 | } |
1794 | } |
1795 | |
1796 | // See if the target wants to custom expand this node. |
1797 | if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true)) |
1798 | return; |
1799 | |
1800 | // Make the vector elements byte-addressable if they aren't already. |
1801 | EVT VecVT = Vec.getValueType(); |
1802 | EVT EltVT = VecVT.getVectorElementType(); |
1803 | if (VecVT.getScalarSizeInBits() < 8) { |
1804 | EltVT = MVT::i8; |
1805 | VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, |
1806 | EC: VecVT.getVectorElementCount()); |
1807 | Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec); |
1808 | // Extend the element type to match if needed. |
1809 | if (EltVT.bitsGT(VT: Elt.getValueType())) |
1810 | Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt); |
1811 | } |
1812 | |
1813 | // Spill the vector to the stack. |
1814 | // In cases where the vector is illegal it will be broken down into parts |
1815 | // and stored in parts - we should use the alignment for the smallest part. |
1816 | Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false); |
1817 | SDValue StackPtr = |
1818 | DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign); |
1819 | auto &MF = DAG.getMachineFunction(); |
1820 | auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex(); |
1821 | auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex); |
1822 | |
1823 | SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo, |
1824 | Alignment: SmallestAlign); |
1825 | |
1826 | // Store the new element. This may be larger than the vector element type, |
1827 | // so use a truncating store. |
1828 | SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx); |
1829 | Store = DAG.getTruncStore( |
1830 | Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT, |
1831 | Alignment: commonAlignment(A: SmallestAlign, |
1832 | Offset: EltVT.getFixedSizeInBits() / 8)); |
1833 | |
1834 | EVT LoVT, HiVT; |
1835 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT); |
1836 | |
1837 | // Load the Lo part from the stack slot. |
1838 | Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign); |
1839 | |
1840 | // Increment the pointer to the other part. |
1841 | auto Load = cast<LoadSDNode>(Val&: Lo); |
1842 | MachinePointerInfo MPI = Load->getPointerInfo(); |
1843 | IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr); |
1844 | |
1845 | Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign); |
1846 | |
1847 | // If we adjusted the original type, we need to truncate the results. |
1848 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
1849 | if (LoVT != Lo.getValueType()) |
1850 | Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo); |
1851 | if (HiVT != Hi.getValueType()) |
1852 | Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi); |
1853 | } |
1854 | |
1855 | void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo, |
1856 | SDValue &Hi) { |
1857 | EVT LoVT, HiVT; |
1858 | SDLoc dl(N); |
1859 | assert(N->getValueType(0).isScalableVector() && |
1860 | "Only scalable vectors are supported for STEP_VECTOR" ); |
1861 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
1862 | SDValue Step = N->getOperand(Num: 0); |
1863 | |
1864 | Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step); |
1865 | |
1866 | // Hi = Lo + (EltCnt * Step) |
1867 | EVT EltVT = Step.getValueType(); |
1868 | APInt StepVal = Step->getAsAPIntVal(); |
1869 | SDValue StartOfHi = |
1870 | DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements()); |
1871 | StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType()); |
1872 | StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi); |
1873 | |
1874 | Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step); |
1875 | Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi); |
1876 | } |
1877 | |
1878 | void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo, |
1879 | SDValue &Hi) { |
1880 | EVT LoVT, HiVT; |
1881 | SDLoc dl(N); |
1882 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
1883 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0)); |
1884 | if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) { |
1885 | Hi = DAG.getUNDEF(VT: HiVT); |
1886 | } else { |
1887 | assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode" ); |
1888 | Hi = Lo; |
1889 | } |
1890 | } |
1891 | |
1892 | void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, |
1893 | SDValue &Hi) { |
1894 | assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!" ); |
1895 | EVT LoVT, HiVT; |
1896 | SDLoc dl(LD); |
1897 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0)); |
1898 | |
1899 | ISD::LoadExtType ExtType = LD->getExtensionType(); |
1900 | SDValue Ch = LD->getChain(); |
1901 | SDValue Ptr = LD->getBasePtr(); |
1902 | SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType()); |
1903 | EVT MemoryVT = LD->getMemoryVT(); |
1904 | MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); |
1905 | AAMDNodes AAInfo = LD->getAAInfo(); |
1906 | |
1907 | EVT LoMemVT, HiMemVT; |
1908 | std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT); |
1909 | |
1910 | if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) { |
1911 | SDValue Value, NewChain; |
1912 | std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG); |
1913 | std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl); |
1914 | ReplaceValueWith(From: SDValue(LD, 1), To: NewChain); |
1915 | return; |
1916 | } |
1917 | |
1918 | Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset, |
1919 | PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getOriginalAlign(), |
1920 | MMOFlags, AAInfo); |
1921 | |
1922 | MachinePointerInfo MPI; |
1923 | IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr); |
1924 | |
1925 | Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI, |
1926 | MemVT: HiMemVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo); |
1927 | |
1928 | // Build a factor node to remember that this load is independent of the |
1929 | // other one. |
1930 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1), |
1931 | Hi.getValue(R: 1)); |
1932 | |
1933 | // Legalize the chain result - switch anything that used the old chain to |
1934 | // use the new one. |
1935 | ReplaceValueWith(From: SDValue(LD, 1), To: Ch); |
1936 | } |
1937 | |
1938 | void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo, |
1939 | SDValue &Hi) { |
1940 | assert(LD->isUnindexed() && "Indexed VP load during type legalization!" ); |
1941 | EVT LoVT, HiVT; |
1942 | SDLoc dl(LD); |
1943 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0)); |
1944 | |
1945 | ISD::LoadExtType ExtType = LD->getExtensionType(); |
1946 | SDValue Ch = LD->getChain(); |
1947 | SDValue Ptr = LD->getBasePtr(); |
1948 | SDValue Offset = LD->getOffset(); |
1949 | assert(Offset.isUndef() && "Unexpected indexed variable-length load offset" ); |
1950 | Align Alignment = LD->getOriginalAlign(); |
1951 | SDValue Mask = LD->getMask(); |
1952 | SDValue EVL = LD->getVectorLength(); |
1953 | EVT MemoryVT = LD->getMemoryVT(); |
1954 | |
1955 | EVT LoMemVT, HiMemVT; |
1956 | bool HiIsEmpty = false; |
1957 | std::tie(args&: LoMemVT, args&: HiMemVT) = |
1958 | DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty); |
1959 | |
1960 | // Split Mask operand |
1961 | SDValue MaskLo, MaskHi; |
1962 | if (Mask.getOpcode() == ISD::SETCC) { |
1963 | SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi); |
1964 | } else { |
1965 | if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector) |
1966 | GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi); |
1967 | else |
1968 | std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl); |
1969 | } |
1970 | |
1971 | // Split EVL operand |
1972 | SDValue EVLLo, EVLHi; |
1973 | std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl); |
1974 | |
1975 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( |
1976 | PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad, |
1977 | Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(), |
1978 | Ranges: LD->getRanges()); |
1979 | |
1980 | Lo = |
1981 | DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset, |
1982 | Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad()); |
1983 | |
1984 | if (HiIsEmpty) { |
1985 | // The hi vp_load has zero storage size. We therefore simply set it to |
1986 | // the low vp_load and rely on subsequent removal from the chain. |
1987 | Hi = Lo; |
1988 | } else { |
1989 | // Generate hi vp_load. |
1990 | Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG, |
1991 | IsCompressedMemory: LD->isExpandingLoad()); |
1992 | |
1993 | MachinePointerInfo MPI; |
1994 | if (LoMemVT.isScalableVector()) |
1995 | MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace()); |
1996 | else |
1997 | MPI = LD->getPointerInfo().getWithOffset( |
1998 | O: LoMemVT.getStoreSize().getFixedValue()); |
1999 | |
2000 | MMO = DAG.getMachineFunction().getMachineMemOperand( |
2001 | PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(), |
2002 | BaseAlignment: Alignment, AAInfo: LD->getAAInfo(), Ranges: LD->getRanges()); |
2003 | |
2004 | Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr, |
2005 | Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO, |
2006 | IsExpanding: LD->isExpandingLoad()); |
2007 | } |
2008 | |
2009 | // Build a factor node to remember that this load is independent of the |
2010 | // other one. |
2011 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1), |
2012 | Hi.getValue(R: 1)); |
2013 | |
2014 | // Legalize the chain result - switch anything that used the old chain to |
2015 | // use the new one. |
2016 | ReplaceValueWith(From: SDValue(LD, 1), To: Ch); |
2017 | } |
2018 | |
2019 | void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD, |
2020 | SDValue &Lo, SDValue &Hi) { |
2021 | assert(SLD->isUnindexed() && |
2022 | "Indexed VP strided load during type legalization!" ); |
2023 | assert(SLD->getOffset().isUndef() && |
2024 | "Unexpected indexed variable-length load offset" ); |
2025 | |
2026 | SDLoc DL(SLD); |
2027 | |
2028 | EVT LoVT, HiVT; |
2029 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0)); |
2030 | |
2031 | EVT LoMemVT, HiMemVT; |
2032 | bool HiIsEmpty = false; |
2033 | std::tie(args&: LoMemVT, args&: HiMemVT) = |
2034 | DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty); |
2035 | |
2036 | SDValue Mask = SLD->getMask(); |
2037 | SDValue LoMask, HiMask; |
2038 | if (Mask.getOpcode() == ISD::SETCC) { |
2039 | SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask); |
2040 | } else { |
2041 | if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector) |
2042 | GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask); |
2043 | else |
2044 | std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL); |
2045 | } |
2046 | |
2047 | SDValue LoEVL, HiEVL; |
2048 | std::tie(args&: LoEVL, args&: HiEVL) = |
2049 | DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL); |
2050 | |
2051 | // Generate the low vp_strided_load |
2052 | Lo = DAG.getStridedLoadVP( |
2053 | AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL, |
2054 | Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(), |
2055 | Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad()); |
2056 | |
2057 | if (HiIsEmpty) { |
2058 | // The high vp_strided_load has zero storage size. We therefore simply set |
2059 | // it to the low vp_strided_load and rely on subsequent removal from the |
2060 | // chain. |
2061 | Hi = Lo; |
2062 | } else { |
2063 | // Generate the high vp_strided_load. |
2064 | // To calculate the high base address, we need to sum to the low base |
2065 | // address stride number of bytes for each element already loaded by low, |
2066 | // that is: Ptr = Ptr + (LoEVL * Stride) |
2067 | EVT PtrVT = SLD->getBasePtr().getValueType(); |
2068 | SDValue Increment = |
2069 | DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL, |
2070 | N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT)); |
2071 | SDValue Ptr = |
2072 | DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment); |
2073 | |
2074 | Align Alignment = SLD->getOriginalAlign(); |
2075 | if (LoMemVT.isScalableVector()) |
2076 | Alignment = commonAlignment( |
2077 | A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8); |
2078 | |
2079 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( |
2080 | PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()), |
2081 | F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(), |
2082 | BaseAlignment: Alignment, AAInfo: SLD->getAAInfo(), Ranges: SLD->getRanges()); |
2083 | |
2084 | Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), |
2085 | VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(), |
2086 | Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO, |
2087 | IsExpanding: SLD->isExpandingLoad()); |
2088 | } |
2089 | |
2090 | // Build a factor node to remember that this load is independent of the |
2091 | // other one. |
2092 | SDValue Ch = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(R: 1), |
2093 | Hi.getValue(R: 1)); |
2094 | |
2095 | // Legalize the chain result - switch anything that used the old chain to |
2096 | // use the new one. |
2097 | ReplaceValueWith(From: SDValue(SLD, 1), To: Ch); |
2098 | } |
2099 | |
2100 | void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, |
2101 | SDValue &Lo, SDValue &Hi) { |
2102 | assert(MLD->isUnindexed() && "Indexed masked load during type legalization!" ); |
2103 | EVT LoVT, HiVT; |
2104 | SDLoc dl(MLD); |
2105 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0)); |
2106 | |
2107 | SDValue Ch = MLD->getChain(); |
2108 | SDValue Ptr = MLD->getBasePtr(); |
2109 | SDValue Offset = MLD->getOffset(); |
2110 | assert(Offset.isUndef() && "Unexpected indexed masked load offset" ); |
2111 | SDValue Mask = MLD->getMask(); |
2112 | SDValue PassThru = MLD->getPassThru(); |
2113 | Align Alignment = MLD->getOriginalAlign(); |
2114 | ISD::LoadExtType ExtType = MLD->getExtensionType(); |
2115 | |
2116 | // Split Mask operand |
2117 | SDValue MaskLo, MaskHi; |
2118 | if (Mask.getOpcode() == ISD::SETCC) { |
2119 | SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi); |
2120 | } else { |
2121 | if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector) |
2122 | GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi); |
2123 | else |
2124 | std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl); |
2125 | } |
2126 | |
2127 | EVT MemoryVT = MLD->getMemoryVT(); |
2128 | EVT LoMemVT, HiMemVT; |
2129 | bool HiIsEmpty = false; |
2130 | std::tie(args&: LoMemVT, args&: HiMemVT) = |
2131 | DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty); |
2132 | |
2133 | SDValue PassThruLo, PassThruHi; |
2134 | if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector) |
2135 | GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi); |
2136 | else |
2137 | std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl); |
2138 | |
2139 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( |
2140 | PtrInfo: MLD->getPointerInfo(), F: MachineMemOperand::MOLoad, |
2141 | Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: MLD->getAAInfo(), |
2142 | Ranges: MLD->getRanges()); |
2143 | |
2144 | Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT, |
2145 | MMO, AM: MLD->getAddressingMode(), ExtType, |
2146 | IsExpanding: MLD->isExpandingLoad()); |
2147 | |
2148 | if (HiIsEmpty) { |
2149 | // The hi masked load has zero storage size. We therefore simply set it to |
2150 | // the low masked load and rely on subsequent removal from the chain. |
2151 | Hi = Lo; |
2152 | } else { |
2153 | // Generate hi masked load. |
2154 | Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG, |
2155 | IsCompressedMemory: MLD->isExpandingLoad()); |
2156 | |
2157 | MachinePointerInfo MPI; |
2158 | if (LoMemVT.isScalableVector()) |
2159 | MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace()); |
2160 | else |
2161 | MPI = MLD->getPointerInfo().getWithOffset( |
2162 | O: LoMemVT.getStoreSize().getFixedValue()); |
2163 | |
2164 | MMO = DAG.getMachineFunction().getMachineMemOperand( |
2165 | PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(), |
2166 | BaseAlignment: Alignment, AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges()); |
2167 | |
2168 | Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi, |
2169 | MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType, |
2170 | IsExpanding: MLD->isExpandingLoad()); |
2171 | } |
2172 | |
2173 | // Build a factor node to remember that this load is independent of the |
2174 | // other one. |
2175 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1), |
2176 | Hi.getValue(R: 1)); |
2177 | |
2178 | // Legalize the chain result - switch anything that used the old chain to |
2179 | // use the new one. |
2180 | ReplaceValueWith(From: SDValue(MLD, 1), To: Ch); |
2181 | |
2182 | } |
2183 | |
2184 | void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo, |
2185 | SDValue &Hi, bool SplitSETCC) { |
2186 | EVT LoVT, HiVT; |
2187 | SDLoc dl(N); |
2188 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
2189 | |
2190 | SDValue Ch = N->getChain(); |
2191 | SDValue Ptr = N->getBasePtr(); |
2192 | struct Operands { |
2193 | SDValue Mask; |
2194 | SDValue Index; |
2195 | SDValue Scale; |
2196 | } Ops = [&]() -> Operands { |
2197 | if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) { |
2198 | return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()}; |
2199 | } |
2200 | auto *VPSC = cast<VPGatherSDNode>(Val: N); |
2201 | return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()}; |
2202 | }(); |
2203 | |
2204 | EVT MemoryVT = N->getMemoryVT(); |
2205 | Align Alignment = N->getOriginalAlign(); |
2206 | |
2207 | // Split Mask operand |
2208 | SDValue MaskLo, MaskHi; |
2209 | if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) { |
2210 | SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi); |
2211 | } else { |
2212 | std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl); |
2213 | } |
2214 | |
2215 | EVT LoMemVT, HiMemVT; |
2216 | // Split MemoryVT |
2217 | std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT); |
2218 | |
2219 | SDValue IndexHi, IndexLo; |
2220 | if (getTypeAction(VT: Ops.Index.getValueType()) == |
2221 | TargetLowering::TypeSplitVector) |
2222 | GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi); |
2223 | else |
2224 | std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl); |
2225 | |
2226 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( |
2227 | PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOLoad, |
2228 | Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(), |
2229 | Ranges: N->getRanges()); |
2230 | |
2231 | if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) { |
2232 | SDValue PassThru = MGT->getPassThru(); |
2233 | SDValue PassThruLo, PassThruHi; |
2234 | if (getTypeAction(VT: PassThru.getValueType()) == |
2235 | TargetLowering::TypeSplitVector) |
2236 | GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi); |
2237 | else |
2238 | std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl); |
2239 | |
2240 | ISD::LoadExtType ExtType = MGT->getExtensionType(); |
2241 | ISD::MemIndexType IndexTy = MGT->getIndexType(); |
2242 | |
2243 | SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale}; |
2244 | Lo = DAG.getMaskedGather(VTs: DAG.getVTList(LoVT, MVT::Other), MemVT: LoMemVT, dl, |
2245 | Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType); |
2246 | |
2247 | SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale}; |
2248 | Hi = DAG.getMaskedGather(VTs: DAG.getVTList(HiVT, MVT::Other), MemVT: HiMemVT, dl, |
2249 | Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType); |
2250 | } else { |
2251 | auto *VPGT = cast<VPGatherSDNode>(Val: N); |
2252 | SDValue EVLLo, EVLHi; |
2253 | std::tie(args&: EVLLo, args&: EVLHi) = |
2254 | DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl); |
2255 | |
2256 | SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo}; |
2257 | Lo = DAG.getGatherVP(VTs: DAG.getVTList(LoVT, MVT::Other), VT: LoMemVT, dl, Ops: OpsLo, |
2258 | MMO, IndexType: VPGT->getIndexType()); |
2259 | |
2260 | SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi}; |
2261 | Hi = DAG.getGatherVP(VTs: DAG.getVTList(HiVT, MVT::Other), VT: HiMemVT, dl, Ops: OpsHi, |
2262 | MMO, IndexType: VPGT->getIndexType()); |
2263 | } |
2264 | |
2265 | // Build a factor node to remember that this load is independent of the |
2266 | // other one. |
2267 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1), |
2268 | Hi.getValue(R: 1)); |
2269 | |
2270 | // Legalize the chain result - switch anything that used the old chain to |
2271 | // use the new one. |
2272 | ReplaceValueWith(From: SDValue(N, 1), To: Ch); |
2273 | } |
2274 | |
2275 | void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { |
2276 | assert(N->getValueType(0).isVector() && |
2277 | N->getOperand(0).getValueType().isVector() && |
2278 | "Operand types must be vectors" ); |
2279 | |
2280 | EVT LoVT, HiVT; |
2281 | SDLoc DL(N); |
2282 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
2283 | |
2284 | // If the input also splits, handle it directly. Otherwise split it by hand. |
2285 | SDValue LL, LH, RL, RH; |
2286 | if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) == |
2287 | TargetLowering::TypeSplitVector) |
2288 | GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH); |
2289 | else |
2290 | std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0); |
2291 | |
2292 | if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) == |
2293 | TargetLowering::TypeSplitVector) |
2294 | GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH); |
2295 | else |
2296 | std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1); |
2297 | |
2298 | if (N->getOpcode() == ISD::SETCC) { |
2299 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2)); |
2300 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2)); |
2301 | } else { |
2302 | assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode" ); |
2303 | SDValue MaskLo, MaskHi, EVLLo, EVLHi; |
2304 | std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3)); |
2305 | std::tie(args&: EVLLo, args&: EVLHi) = |
2306 | DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL); |
2307 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2), N4: MaskLo, |
2308 | N5: EVLLo); |
2309 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2), N4: MaskHi, |
2310 | N5: EVLHi); |
2311 | } |
2312 | } |
2313 | |
2314 | void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, |
2315 | SDValue &Hi) { |
2316 | // Get the dest types - they may not match the input types, e.g. int_to_fp. |
2317 | EVT LoVT, HiVT; |
2318 | SDLoc dl(N); |
2319 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
2320 | |
2321 | // If the input also splits, handle it directly for a compile time speedup. |
2322 | // Otherwise split it by hand. |
2323 | EVT InVT = N->getOperand(Num: 0).getValueType(); |
2324 | if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) |
2325 | GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi); |
2326 | else |
2327 | std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0); |
2328 | |
2329 | const SDNodeFlags Flags = N->getFlags(); |
2330 | unsigned Opcode = N->getOpcode(); |
2331 | if (N->getNumOperands() <= 2) { |
2332 | if (Opcode == ISD::FP_ROUND) { |
2333 | Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags); |
2334 | Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags); |
2335 | } else { |
2336 | Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags); |
2337 | Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags); |
2338 | } |
2339 | return; |
2340 | } |
2341 | |
2342 | assert(N->getNumOperands() == 3 && "Unexpected number of operands!" ); |
2343 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
2344 | |
2345 | SDValue MaskLo, MaskHi; |
2346 | std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1)); |
2347 | |
2348 | SDValue EVLLo, EVLHi; |
2349 | std::tie(args&: EVLLo, args&: EVLHi) = |
2350 | DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl); |
2351 | |
2352 | Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo}, Flags); |
2353 | Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi}, Flags); |
2354 | } |
2355 | |
2356 | void DAGTypeLegalizer::SplitVecRes_FFREXP(SDNode *N, unsigned ResNo, |
2357 | SDValue &Lo, SDValue &Hi) { |
2358 | SDLoc dl(N); |
2359 | auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
2360 | auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1)); |
2361 | |
2362 | // If the input also splits, handle it directly for a compile time speedup. |
2363 | // Otherwise split it by hand. |
2364 | EVT InVT = N->getOperand(Num: 0).getValueType(); |
2365 | if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) |
2366 | GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi); |
2367 | else |
2368 | std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0); |
2369 | |
2370 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo); |
2371 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi); |
2372 | Lo->setFlags(N->getFlags()); |
2373 | Hi->setFlags(N->getFlags()); |
2374 | |
2375 | SDNode *HiNode = Hi.getNode(); |
2376 | SDNode *LoNode = Lo.getNode(); |
2377 | |
2378 | // Replace the other vector result not being explicitly split here. |
2379 | unsigned OtherNo = 1 - ResNo; |
2380 | EVT OtherVT = N->getValueType(ResNo: OtherNo); |
2381 | if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) { |
2382 | SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo), |
2383 | Hi: SDValue(HiNode, OtherNo)); |
2384 | } else { |
2385 | SDValue OtherVal = |
2386 | DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo), |
2387 | N2: SDValue(HiNode, OtherNo)); |
2388 | ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal); |
2389 | } |
2390 | } |
2391 | |
2392 | void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, |
2393 | SDValue &Hi) { |
2394 | SDLoc dl(N); |
2395 | EVT SrcVT = N->getOperand(Num: 0).getValueType(); |
2396 | EVT DestVT = N->getValueType(ResNo: 0); |
2397 | EVT LoVT, HiVT; |
2398 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT); |
2399 | |
2400 | // We can do better than a generic split operation if the extend is doing |
2401 | // more than just doubling the width of the elements and the following are |
2402 | // true: |
2403 | // - The number of vector elements is even, |
2404 | // - the source type is legal, |
2405 | // - the type of a split source is illegal, |
2406 | // - the type of an extended (by doubling element size) source is legal, and |
2407 | // - the type of that extended source when split is legal. |
2408 | // |
2409 | // This won't necessarily completely legalize the operation, but it will |
2410 | // more effectively move in the right direction and prevent falling down |
2411 | // to scalarization in many cases due to the input vector being split too |
2412 | // far. |
2413 | if (SrcVT.getVectorElementCount().isKnownEven() && |
2414 | SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) { |
2415 | LLVMContext &Ctx = *DAG.getContext(); |
2416 | EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx); |
2417 | EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx); |
2418 | |
2419 | EVT SplitLoVT, SplitHiVT; |
2420 | std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT); |
2421 | if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) && |
2422 | TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) { |
2423 | LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:" ; |
2424 | N->dump(&DAG); dbgs() << "\n" ); |
2425 | if (!N->isVPOpcode()) { |
2426 | // Extend the source vector by one step. |
2427 | SDValue NewSrc = |
2428 | DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0)); |
2429 | // Get the low and high halves of the new, extended one step, vector. |
2430 | std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl); |
2431 | // Extend those vector halves the rest of the way. |
2432 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo); |
2433 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi); |
2434 | return; |
2435 | } |
2436 | |
2437 | // Extend the source vector by one step. |
2438 | SDValue NewSrc = |
2439 | DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, N1: N->getOperand(Num: 0), |
2440 | N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2)); |
2441 | // Get the low and high halves of the new, extended one step, vector. |
2442 | std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl); |
2443 | |
2444 | SDValue MaskLo, MaskHi; |
2445 | std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1)); |
2446 | |
2447 | SDValue EVLLo, EVLHi; |
2448 | std::tie(args&: EVLLo, args&: EVLHi) = |
2449 | DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl); |
2450 | // Extend those vector halves the rest of the way. |
2451 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo}); |
2452 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi}); |
2453 | return; |
2454 | } |
2455 | } |
2456 | // Fall back to the generic unary operator splitting otherwise. |
2457 | SplitVecRes_UnaryOp(N, Lo, Hi); |
2458 | } |
2459 | |
2460 | void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, |
2461 | SDValue &Lo, SDValue &Hi) { |
2462 | // The low and high parts of the original input give four input vectors. |
2463 | SDValue Inputs[4]; |
2464 | SDLoc DL(N); |
2465 | GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]); |
2466 | GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]); |
2467 | EVT NewVT = Inputs[0].getValueType(); |
2468 | unsigned NewElts = NewVT.getVectorNumElements(); |
2469 | |
2470 | auto &&IsConstant = [](const SDValue &N) { |
2471 | APInt SplatValue; |
2472 | return N.getResNo() == 0 && |
2473 | (ISD::isConstantSplatVector(N: N.getNode(), SplatValue) || |
2474 | ISD::isBuildVectorOfConstantSDNodes(N: N.getNode())); |
2475 | }; |
2476 | auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1, |
2477 | SDValue &Input2, |
2478 | ArrayRef<int> Mask) { |
2479 | assert(Input1->getOpcode() == ISD::BUILD_VECTOR && |
2480 | Input2->getOpcode() == ISD::BUILD_VECTOR && |
2481 | "Expected build vector node." ); |
2482 | EVT EltVT = NewVT.getVectorElementType(); |
2483 | SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(VT: EltVT)); |
2484 | for (unsigned I = 0; I < NewElts; ++I) { |
2485 | if (Mask[I] == PoisonMaskElem) |
2486 | continue; |
2487 | unsigned Idx = Mask[I]; |
2488 | if (Idx >= NewElts) |
2489 | Ops[I] = Input2.getOperand(i: Idx - NewElts); |
2490 | else |
2491 | Ops[I] = Input1.getOperand(i: Idx); |
2492 | // Make the type of all elements the same as the element type. |
2493 | if (Ops[I].getValueType().bitsGT(VT: EltVT)) |
2494 | Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]); |
2495 | } |
2496 | return DAG.getBuildVector(VT: NewVT, DL, Ops); |
2497 | }; |
2498 | |
2499 | // If Lo or Hi uses elements from at most two of the four input vectors, then |
2500 | // express it as a vector shuffle of those two inputs. Otherwise extract the |
2501 | // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. |
2502 | SmallVector<int> OrigMask(N->getMask()); |
2503 | // Try to pack incoming shuffles/inputs. |
2504 | auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts, |
2505 | &DL](SmallVectorImpl<int> &Mask) { |
2506 | // Check if all inputs are shuffles of the same operands or non-shuffles. |
2507 | MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs; |
2508 | for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) { |
2509 | SDValue Input = Inputs[Idx]; |
2510 | auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode()); |
2511 | if (!Shuffle || |
2512 | Input.getOperand(i: 0).getValueType() != Input.getValueType()) |
2513 | continue; |
2514 | ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))] |
2515 | .push_back(Elt: Idx); |
2516 | ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))] |
2517 | .push_back(Elt: Idx); |
2518 | } |
2519 | for (auto &P : ShufflesIdxs) { |
2520 | if (P.second.size() < 2) |
2521 | continue; |
2522 | // Use shuffles operands instead of shuffles themselves. |
2523 | // 1. Adjust mask. |
2524 | for (int &Idx : Mask) { |
2525 | if (Idx == PoisonMaskElem) |
2526 | continue; |
2527 | unsigned SrcRegIdx = Idx / NewElts; |
2528 | if (Inputs[SrcRegIdx].isUndef()) { |
2529 | Idx = PoisonMaskElem; |
2530 | continue; |
2531 | } |
2532 | auto *Shuffle = |
2533 | dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode()); |
2534 | if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx)) |
2535 | continue; |
2536 | int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts); |
2537 | if (MaskElt == PoisonMaskElem) { |
2538 | Idx = PoisonMaskElem; |
2539 | continue; |
2540 | } |
2541 | Idx = MaskElt % NewElts + |
2542 | P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first |
2543 | ? 0 |
2544 | : 1] * |
2545 | NewElts; |
2546 | } |
2547 | // 2. Update inputs. |
2548 | Inputs[P.second[0]] = P.first.first; |
2549 | Inputs[P.second[1]] = P.first.second; |
2550 | // Clear the pair data. |
2551 | P.second.clear(); |
2552 | ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear(); |
2553 | } |
2554 | // Check if any concat_vectors can be simplified. |
2555 | SmallBitVector UsedSubVector(2 * std::size(Inputs)); |
2556 | for (int &Idx : Mask) { |
2557 | if (Idx == PoisonMaskElem) |
2558 | continue; |
2559 | unsigned SrcRegIdx = Idx / NewElts; |
2560 | if (Inputs[SrcRegIdx].isUndef()) { |
2561 | Idx = PoisonMaskElem; |
2562 | continue; |
2563 | } |
2564 | TargetLowering::LegalizeTypeAction TypeAction = |
2565 | getTypeAction(VT: Inputs[SrcRegIdx].getValueType()); |
2566 | if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS && |
2567 | Inputs[SrcRegIdx].getNumOperands() == 2 && |
2568 | !Inputs[SrcRegIdx].getOperand(i: 1).isUndef() && |
2569 | (TypeAction == TargetLowering::TypeLegal || |
2570 | TypeAction == TargetLowering::TypeWidenVector)) |
2571 | UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2)); |
2572 | } |
2573 | if (UsedSubVector.count() > 1) { |
2574 | SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs; |
2575 | for (unsigned I = 0; I < std::size(Inputs); ++I) { |
2576 | if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1)) |
2577 | continue; |
2578 | if (Pairs.empty() || Pairs.back().size() == 2) |
2579 | Pairs.emplace_back(); |
2580 | if (UsedSubVector.test(Idx: 2 * I)) { |
2581 | Pairs.back().emplace_back(Args&: I, Args: 0); |
2582 | } else { |
2583 | assert(UsedSubVector.test(2 * I + 1) && |
2584 | "Expected to be used one of the subvectors." ); |
2585 | Pairs.back().emplace_back(Args&: I, Args: 1); |
2586 | } |
2587 | } |
2588 | if (!Pairs.empty() && Pairs.front().size() > 1) { |
2589 | // Adjust mask. |
2590 | for (int &Idx : Mask) { |
2591 | if (Idx == PoisonMaskElem) |
2592 | continue; |
2593 | unsigned SrcRegIdx = Idx / NewElts; |
2594 | auto *It = find_if( |
2595 | Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) { |
2596 | return Idxs.front().first == SrcRegIdx || |
2597 | Idxs.back().first == SrcRegIdx; |
2598 | }); |
2599 | if (It == Pairs.end()) |
2600 | continue; |
2601 | Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) + |
2602 | (SrcRegIdx == It->front().first ? 0 : (NewElts / 2)); |
2603 | } |
2604 | // Adjust inputs. |
2605 | for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) { |
2606 | Inputs[Idxs.front().first] = DAG.getNode( |
2607 | Opcode: ISD::CONCAT_VECTORS, DL, |
2608 | VT: Inputs[Idxs.front().first].getValueType(), |
2609 | N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second), |
2610 | N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second)); |
2611 | } |
2612 | } |
2613 | } |
2614 | bool Changed; |
2615 | do { |
2616 | // Try to remove extra shuffles (except broadcasts) and shuffles with the |
2617 | // reused operands. |
2618 | Changed = false; |
2619 | for (unsigned I = 0; I < std::size(Inputs); ++I) { |
2620 | auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode()); |
2621 | if (!Shuffle) |
2622 | continue; |
2623 | if (Shuffle->getOperand(Num: 0).getValueType() != NewVT) |
2624 | continue; |
2625 | int Op = -1; |
2626 | if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() && |
2627 | !Shuffle->isSplat()) { |
2628 | Op = 0; |
2629 | } else if (!Inputs[I].hasOneUse() && |
2630 | !Shuffle->getOperand(Num: 1).isUndef()) { |
2631 | // Find the only used operand, if possible. |
2632 | for (int &Idx : Mask) { |
2633 | if (Idx == PoisonMaskElem) |
2634 | continue; |
2635 | unsigned SrcRegIdx = Idx / NewElts; |
2636 | if (SrcRegIdx != I) |
2637 | continue; |
2638 | int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts); |
2639 | if (MaskElt == PoisonMaskElem) { |
2640 | Idx = PoisonMaskElem; |
2641 | continue; |
2642 | } |
2643 | int OpIdx = MaskElt / NewElts; |
2644 | if (Op == -1) { |
2645 | Op = OpIdx; |
2646 | continue; |
2647 | } |
2648 | if (Op != OpIdx) { |
2649 | Op = -1; |
2650 | break; |
2651 | } |
2652 | } |
2653 | } |
2654 | if (Op < 0) { |
2655 | // Try to check if one of the shuffle operands is used already. |
2656 | for (int OpIdx = 0; OpIdx < 2; ++OpIdx) { |
2657 | if (Shuffle->getOperand(Num: OpIdx).isUndef()) |
2658 | continue; |
2659 | auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx)); |
2660 | if (It == std::end(arr&: Inputs)) |
2661 | continue; |
2662 | int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It); |
2663 | // Found that operand is used already. |
2664 | // 1. Fix the mask for the reused operand. |
2665 | for (int &Idx : Mask) { |
2666 | if (Idx == PoisonMaskElem) |
2667 | continue; |
2668 | unsigned SrcRegIdx = Idx / NewElts; |
2669 | if (SrcRegIdx != I) |
2670 | continue; |
2671 | int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts); |
2672 | if (MaskElt == PoisonMaskElem) { |
2673 | Idx = PoisonMaskElem; |
2674 | continue; |
2675 | } |
2676 | int MaskIdx = MaskElt / NewElts; |
2677 | if (OpIdx == MaskIdx) |
2678 | Idx = MaskElt % NewElts + FoundOp * NewElts; |
2679 | } |
2680 | // 2. Set Op to the unused OpIdx. |
2681 | Op = (OpIdx + 1) % 2; |
2682 | break; |
2683 | } |
2684 | } |
2685 | if (Op >= 0) { |
2686 | Changed = true; |
2687 | Inputs[I] = Shuffle->getOperand(Num: Op); |
2688 | // Adjust mask. |
2689 | for (int &Idx : Mask) { |
2690 | if (Idx == PoisonMaskElem) |
2691 | continue; |
2692 | unsigned SrcRegIdx = Idx / NewElts; |
2693 | if (SrcRegIdx != I) |
2694 | continue; |
2695 | int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts); |
2696 | int OpIdx = MaskElt / NewElts; |
2697 | if (OpIdx != Op) |
2698 | continue; |
2699 | Idx = MaskElt % NewElts + SrcRegIdx * NewElts; |
2700 | } |
2701 | } |
2702 | } |
2703 | } while (Changed); |
2704 | }; |
2705 | TryPeekThroughShufflesInputs(OrigMask); |
2706 | // Proces unique inputs. |
2707 | auto &&MakeUniqueInputs = [&Inputs, &IsConstant, |
2708 | NewElts](SmallVectorImpl<int> &Mask) { |
2709 | SetVector<SDValue> UniqueInputs; |
2710 | SetVector<SDValue> UniqueConstantInputs; |
2711 | for (const auto &I : Inputs) { |
2712 | if (IsConstant(I)) |
2713 | UniqueConstantInputs.insert(X: I); |
2714 | else if (!I.isUndef()) |
2715 | UniqueInputs.insert(X: I); |
2716 | } |
2717 | // Adjust mask in case of reused inputs. Also, need to insert constant |
2718 | // inputs at first, otherwise it affects the final outcome. |
2719 | if (UniqueInputs.size() != std::size(Inputs)) { |
2720 | auto &&UniqueVec = UniqueInputs.takeVector(); |
2721 | auto &&UniqueConstantVec = UniqueConstantInputs.takeVector(); |
2722 | unsigned ConstNum = UniqueConstantVec.size(); |
2723 | for (int &Idx : Mask) { |
2724 | if (Idx == PoisonMaskElem) |
2725 | continue; |
2726 | unsigned SrcRegIdx = Idx / NewElts; |
2727 | if (Inputs[SrcRegIdx].isUndef()) { |
2728 | Idx = PoisonMaskElem; |
2729 | continue; |
2730 | } |
2731 | const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]); |
2732 | if (It != UniqueConstantVec.end()) { |
2733 | Idx = (Idx % NewElts) + |
2734 | NewElts * std::distance(first: UniqueConstantVec.begin(), last: It); |
2735 | assert(Idx >= 0 && "Expected defined mask idx." ); |
2736 | continue; |
2737 | } |
2738 | const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]); |
2739 | assert(RegIt != UniqueVec.end() && "Cannot find non-const value." ); |
2740 | Idx = (Idx % NewElts) + |
2741 | NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum); |
2742 | assert(Idx >= 0 && "Expected defined mask idx." ); |
2743 | } |
2744 | copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs)); |
2745 | copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum)); |
2746 | } |
2747 | }; |
2748 | MakeUniqueInputs(OrigMask); |
2749 | SDValue OrigInputs[4]; |
2750 | copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs)); |
2751 | for (unsigned High = 0; High < 2; ++High) { |
2752 | SDValue &Output = High ? Hi : Lo; |
2753 | |
2754 | // Build a shuffle mask for the output, discovering on the fly which |
2755 | // input vectors to use as shuffle operands. |
2756 | unsigned FirstMaskIdx = High * NewElts; |
2757 | SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem); |
2758 | copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin()); |
2759 | assert(!Output && "Expected default initialized initial value." ); |
2760 | TryPeekThroughShufflesInputs(Mask); |
2761 | MakeUniqueInputs(Mask); |
2762 | SDValue TmpInputs[4]; |
2763 | copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs)); |
2764 | // Track changes in the output registers. |
2765 | int UsedIdx = -1; |
2766 | bool SecondIteration = false; |
2767 | auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) { |
2768 | if (UsedIdx < 0) { |
2769 | UsedIdx = Idx; |
2770 | return false; |
2771 | } |
2772 | if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx) |
2773 | SecondIteration = true; |
2774 | return SecondIteration; |
2775 | }; |
2776 | processShuffleMasks( |
2777 | Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs), |
2778 | /*NumOfUsedRegs=*/1, |
2779 | NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(VT: NewVT); }, |
2780 | SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs, |
2781 | &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) { |
2782 | if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR) |
2783 | Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask); |
2784 | else |
2785 | Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx], |
2786 | N2: DAG.getUNDEF(VT: NewVT), Mask); |
2787 | Inputs[Idx] = Output; |
2788 | }, |
2789 | ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs, |
2790 | &TmpInputs, |
2791 | &BuildVector](ArrayRef<int> Mask, unsigned Idx1, unsigned Idx2) { |
2792 | if (AccumulateResults(Idx1)) { |
2793 | if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR && |
2794 | Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR) |
2795 | Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask); |
2796 | else |
2797 | Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1], |
2798 | N2: Inputs[Idx2], Mask); |
2799 | } else { |
2800 | if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR && |
2801 | TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR) |
2802 | Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask); |
2803 | else |
2804 | Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1], |
2805 | N2: TmpInputs[Idx2], Mask); |
2806 | } |
2807 | Inputs[Idx1] = Output; |
2808 | }); |
2809 | copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs)); |
2810 | } |
2811 | } |
2812 | |
2813 | void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) { |
2814 | EVT OVT = N->getValueType(ResNo: 0); |
2815 | EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext()); |
2816 | SDValue Chain = N->getOperand(Num: 0); |
2817 | SDValue Ptr = N->getOperand(Num: 1); |
2818 | SDValue SV = N->getOperand(Num: 2); |
2819 | SDLoc dl(N); |
2820 | |
2821 | const Align Alignment = |
2822 | DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext())); |
2823 | |
2824 | Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value()); |
2825 | Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value()); |
2826 | Chain = Hi.getValue(R: 1); |
2827 | |
2828 | // Modified the chain - switch anything that used the old chain to use |
2829 | // the new one. |
2830 | ReplaceValueWith(From: SDValue(N, 1), To: Chain); |
2831 | } |
2832 | |
2833 | void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, |
2834 | SDValue &Hi) { |
2835 | EVT DstVTLo, DstVTHi; |
2836 | std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
2837 | SDLoc dl(N); |
2838 | |
2839 | SDValue SrcLo, SrcHi; |
2840 | EVT SrcVT = N->getOperand(Num: 0).getValueType(); |
2841 | if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector) |
2842 | GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi); |
2843 | else |
2844 | std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0); |
2845 | |
2846 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1)); |
2847 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1)); |
2848 | } |
2849 | |
2850 | void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo, |
2851 | SDValue &Hi) { |
2852 | SDValue InLo, InHi; |
2853 | GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi); |
2854 | SDLoc DL(N); |
2855 | |
2856 | Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi); |
2857 | Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo); |
2858 | } |
2859 | |
2860 | void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo, |
2861 | SDValue &Hi) { |
2862 | EVT VT = N->getValueType(ResNo: 0); |
2863 | SDLoc DL(N); |
2864 | |
2865 | EVT LoVT, HiVT; |
2866 | std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT); |
2867 | |
2868 | SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG); |
2869 | Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Expanded, |
2870 | N2: DAG.getVectorIdxConstant(Val: 0, DL)); |
2871 | Hi = |
2872 | DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Expanded, |
2873 | N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL)); |
2874 | } |
2875 | |
2876 | void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo, |
2877 | SDValue &Hi) { |
2878 | EVT VT = N->getValueType(ResNo: 0); |
2879 | SDValue Val = N->getOperand(Num: 0); |
2880 | SDValue Mask = N->getOperand(Num: 1); |
2881 | SDValue EVL = N->getOperand(Num: 2); |
2882 | SDLoc DL(N); |
2883 | |
2884 | // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD. |
2885 | Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false); |
2886 | |
2887 | EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(), |
2888 | EC: VT.getVectorElementCount()); |
2889 | SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment); |
2890 | EVT PtrVT = StackPtr.getValueType(); |
2891 | auto &MF = DAG.getMachineFunction(); |
2892 | auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex(); |
2893 | auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex); |
2894 | |
2895 | MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand( |
2896 | PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(), |
2897 | BaseAlignment: Alignment); |
2898 | MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand( |
2899 | PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(), |
2900 | BaseAlignment: Alignment); |
2901 | |
2902 | unsigned EltWidth = VT.getScalarSizeInBits() / 8; |
2903 | SDValue NumElemMinus1 = |
2904 | DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT), |
2905 | N2: DAG.getConstant(Val: 1, DL, VT: PtrVT)); |
2906 | SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1, |
2907 | N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT)); |
2908 | SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset); |
2909 | SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT); |
2910 | |
2911 | SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT); |
2912 | SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr, |
2913 | Offset: DAG.getUNDEF(VT: PtrVT), Stride, Mask: TrueMask, |
2914 | EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED); |
2915 | |
2916 | SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO); |
2917 | |
2918 | auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT); |
2919 | Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load, |
2920 | N2: DAG.getVectorIdxConstant(Val: 0, DL)); |
2921 | Hi = |
2922 | DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load, |
2923 | N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL)); |
2924 | } |
2925 | |
2926 | void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) { |
2927 | |
2928 | SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi; |
2929 | GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi); |
2930 | GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi); |
2931 | EVT VT = Op0Lo.getValueType(); |
2932 | SDLoc DL(N); |
2933 | SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, |
2934 | VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Op0Lo, N2: Op0Hi); |
2935 | SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, |
2936 | VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Op1Lo, N2: Op1Hi); |
2937 | |
2938 | SetSplitVector(Op: SDValue(N, 0), Lo: ResLo.getValue(R: 0), Hi: ResHi.getValue(R: 0)); |
2939 | SetSplitVector(Op: SDValue(N, 1), Lo: ResLo.getValue(R: 1), Hi: ResHi.getValue(R: 1)); |
2940 | } |
2941 | |
2942 | void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) { |
2943 | SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi; |
2944 | GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi); |
2945 | GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi); |
2946 | EVT VT = Op0Lo.getValueType(); |
2947 | SDLoc DL(N); |
2948 | SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, |
2949 | VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Op0Lo, N2: Op1Lo), |
2950 | DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, |
2951 | VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Op0Hi, N2: Op1Hi)}; |
2952 | |
2953 | SetSplitVector(Op: SDValue(N, 0), Lo: Res[0].getValue(R: 0), Hi: Res[0].getValue(R: 1)); |
2954 | SetSplitVector(Op: SDValue(N, 1), Lo: Res[1].getValue(R: 0), Hi: Res[1].getValue(R: 1)); |
2955 | } |
2956 | |
2957 | //===----------------------------------------------------------------------===// |
2958 | // Operand Vector Splitting |
2959 | //===----------------------------------------------------------------------===// |
2960 | |
2961 | /// This method is called when the specified operand of the specified node is |
2962 | /// found to need vector splitting. At this point, all of the result types of |
2963 | /// the node are known to be legal, but other operands of the node may need |
2964 | /// legalization as well as the specified one. |
2965 | bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { |
2966 | LLVM_DEBUG(dbgs() << "Split node operand: " ; N->dump(&DAG)); |
2967 | SDValue Res = SDValue(); |
2968 | |
2969 | // See if the target wants to custom split this node. |
2970 | if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) |
2971 | return false; |
2972 | |
2973 | switch (N->getOpcode()) { |
2974 | default: |
2975 | #ifndef NDEBUG |
2976 | dbgs() << "SplitVectorOperand Op #" << OpNo << ": " ; |
2977 | N->dump(G: &DAG); |
2978 | dbgs() << "\n" ; |
2979 | #endif |
2980 | report_fatal_error(reason: "Do not know how to split this operator's " |
2981 | "operand!\n" ); |
2982 | |
2983 | case ISD::VP_SETCC: |
2984 | case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; |
2985 | case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; |
2986 | case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; |
2987 | case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break; |
2988 | case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; |
2989 | case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; |
2990 | case ISD::VP_TRUNCATE: |
2991 | case ISD::TRUNCATE: |
2992 | Res = SplitVecOp_TruncateHelper(N); |
2993 | break; |
2994 | case ISD::STRICT_FP_ROUND: |
2995 | case ISD::VP_FP_ROUND: |
2996 | case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; |
2997 | case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break; |
2998 | case ISD::STORE: |
2999 | Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo); |
3000 | break; |
3001 | case ISD::VP_STORE: |
3002 | Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo); |
3003 | break; |
3004 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
3005 | Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo); |
3006 | break; |
3007 | case ISD::MSTORE: |
3008 | Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo); |
3009 | break; |
3010 | case ISD::MSCATTER: |
3011 | case ISD::VP_SCATTER: |
3012 | Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo); |
3013 | break; |
3014 | case ISD::MGATHER: |
3015 | case ISD::VP_GATHER: |
3016 | Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo); |
3017 | break; |
3018 | case ISD::VSELECT: |
3019 | Res = SplitVecOp_VSELECT(N, OpNo); |
3020 | break; |
3021 | case ISD::STRICT_SINT_TO_FP: |
3022 | case ISD::STRICT_UINT_TO_FP: |
3023 | case ISD::SINT_TO_FP: |
3024 | case ISD::UINT_TO_FP: |
3025 | case ISD::VP_SINT_TO_FP: |
3026 | case ISD::VP_UINT_TO_FP: |
3027 | if (N->getValueType(ResNo: 0).bitsLT( |
3028 | VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType())) |
3029 | Res = SplitVecOp_TruncateHelper(N); |
3030 | else |
3031 | Res = SplitVecOp_UnaryOp(N); |
3032 | break; |
3033 | case ISD::FP_TO_SINT_SAT: |
3034 | case ISD::FP_TO_UINT_SAT: |
3035 | Res = SplitVecOp_FP_TO_XINT_SAT(N); |
3036 | break; |
3037 | case ISD::FP_TO_SINT: |
3038 | case ISD::FP_TO_UINT: |
3039 | case ISD::VP_FP_TO_SINT: |
3040 | case ISD::VP_FP_TO_UINT: |
3041 | case ISD::STRICT_FP_TO_SINT: |
3042 | case ISD::STRICT_FP_TO_UINT: |
3043 | case ISD::STRICT_FP_EXTEND: |
3044 | case ISD::FP_EXTEND: |
3045 | case ISD::SIGN_EXTEND: |
3046 | case ISD::ZERO_EXTEND: |
3047 | case ISD::ANY_EXTEND: |
3048 | case ISD::FTRUNC: |
3049 | case ISD::LRINT: |
3050 | case ISD::LLRINT: |
3051 | Res = SplitVecOp_UnaryOp(N); |
3052 | break; |
3053 | case ISD::FLDEXP: |
3054 | Res = SplitVecOp_FPOpDifferentTypes(N); |
3055 | break; |
3056 | |
3057 | case ISD::ANY_EXTEND_VECTOR_INREG: |
3058 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
3059 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
3060 | Res = SplitVecOp_ExtVecInRegOp(N); |
3061 | break; |
3062 | |
3063 | case ISD::VECREDUCE_FADD: |
3064 | case ISD::VECREDUCE_FMUL: |
3065 | case ISD::VECREDUCE_ADD: |
3066 | case ISD::VECREDUCE_MUL: |
3067 | case ISD::VECREDUCE_AND: |
3068 | case ISD::VECREDUCE_OR: |
3069 | case ISD::VECREDUCE_XOR: |
3070 | case ISD::VECREDUCE_SMAX: |
3071 | case ISD::VECREDUCE_SMIN: |
3072 | case ISD::VECREDUCE_UMAX: |
3073 | case ISD::VECREDUCE_UMIN: |
3074 | case ISD::VECREDUCE_FMAX: |
3075 | case ISD::VECREDUCE_FMIN: |
3076 | case ISD::VECREDUCE_FMAXIMUM: |
3077 | case ISD::VECREDUCE_FMINIMUM: |
3078 | Res = SplitVecOp_VECREDUCE(N, OpNo); |
3079 | break; |
3080 | case ISD::VECREDUCE_SEQ_FADD: |
3081 | case ISD::VECREDUCE_SEQ_FMUL: |
3082 | Res = SplitVecOp_VECREDUCE_SEQ(N); |
3083 | break; |
3084 | case ISD::VP_REDUCE_FADD: |
3085 | case ISD::VP_REDUCE_SEQ_FADD: |
3086 | case ISD::VP_REDUCE_FMUL: |
3087 | case ISD::VP_REDUCE_SEQ_FMUL: |
3088 | case ISD::VP_REDUCE_ADD: |
3089 | case ISD::VP_REDUCE_MUL: |
3090 | case ISD::VP_REDUCE_AND: |
3091 | case ISD::VP_REDUCE_OR: |
3092 | case ISD::VP_REDUCE_XOR: |
3093 | case ISD::VP_REDUCE_SMAX: |
3094 | case ISD::VP_REDUCE_SMIN: |
3095 | case ISD::VP_REDUCE_UMAX: |
3096 | case ISD::VP_REDUCE_UMIN: |
3097 | case ISD::VP_REDUCE_FMAX: |
3098 | case ISD::VP_REDUCE_FMIN: |
3099 | Res = SplitVecOp_VP_REDUCE(N, OpNo); |
3100 | break; |
3101 | } |
3102 | |
3103 | // If the result is null, the sub-method took care of registering results etc. |
3104 | if (!Res.getNode()) return false; |
3105 | |
3106 | // If the result is N, the sub-method updated N in place. Tell the legalizer |
3107 | // core about this. |
3108 | if (Res.getNode() == N) |
3109 | return true; |
3110 | |
3111 | if (N->isStrictFPOpcode()) |
3112 | assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 && |
3113 | "Invalid operand expansion" ); |
3114 | else |
3115 | assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && |
3116 | "Invalid operand expansion" ); |
3117 | |
3118 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
3119 | return false; |
3120 | } |
3121 | |
3122 | SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { |
3123 | // The only possibility for an illegal operand is the mask, since result type |
3124 | // legalization would have handled this node already otherwise. |
3125 | assert(OpNo == 0 && "Illegal operand must be mask" ); |
3126 | |
3127 | SDValue Mask = N->getOperand(Num: 0); |
3128 | SDValue Src0 = N->getOperand(Num: 1); |
3129 | SDValue Src1 = N->getOperand(Num: 2); |
3130 | EVT Src0VT = Src0.getValueType(); |
3131 | SDLoc DL(N); |
3132 | assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?" ); |
3133 | |
3134 | SDValue Lo, Hi; |
3135 | GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi); |
3136 | assert(Lo.getValueType() == Hi.getValueType() && |
3137 | "Lo and Hi have differing types" ); |
3138 | |
3139 | EVT LoOpVT, HiOpVT; |
3140 | std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT); |
3141 | assert(LoOpVT == HiOpVT && "Asymmetric vector split?" ); |
3142 | |
3143 | SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask; |
3144 | std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL); |
3145 | std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL); |
3146 | std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL); |
3147 | |
3148 | SDValue LoSelect = |
3149 | DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1); |
3150 | SDValue HiSelect = |
3151 | DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1); |
3152 | |
3153 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect); |
3154 | } |
3155 | |
3156 | SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) { |
3157 | EVT ResVT = N->getValueType(ResNo: 0); |
3158 | SDValue Lo, Hi; |
3159 | SDLoc dl(N); |
3160 | |
3161 | SDValue VecOp = N->getOperand(Num: OpNo); |
3162 | EVT VecVT = VecOp.getValueType(); |
3163 | assert(VecVT.isVector() && "Can only split reduce vector operand" ); |
3164 | GetSplitVector(Op: VecOp, Lo, Hi); |
3165 | EVT LoOpVT, HiOpVT; |
3166 | std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT); |
3167 | |
3168 | // Use the appropriate scalar instruction on the split subvectors before |
3169 | // reducing the now partially reduced smaller vector. |
3170 | unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode()); |
3171 | SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags()); |
3172 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags()); |
3173 | } |
3174 | |
3175 | SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) { |
3176 | EVT ResVT = N->getValueType(ResNo: 0); |
3177 | SDValue Lo, Hi; |
3178 | SDLoc dl(N); |
3179 | |
3180 | SDValue AccOp = N->getOperand(Num: 0); |
3181 | SDValue VecOp = N->getOperand(Num: 1); |
3182 | SDNodeFlags Flags = N->getFlags(); |
3183 | |
3184 | EVT VecVT = VecOp.getValueType(); |
3185 | assert(VecVT.isVector() && "Can only split reduce vector operand" ); |
3186 | GetSplitVector(Op: VecOp, Lo, Hi); |
3187 | EVT LoOpVT, HiOpVT; |
3188 | std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT); |
3189 | |
3190 | // Reduce low half. |
3191 | SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags); |
3192 | |
3193 | // Reduce high half, using low half result as initial value. |
3194 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags); |
3195 | } |
3196 | |
3197 | SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) { |
3198 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
3199 | assert(OpNo == 1 && "Can only split reduce vector operand" ); |
3200 | |
3201 | unsigned Opc = N->getOpcode(); |
3202 | EVT ResVT = N->getValueType(ResNo: 0); |
3203 | SDValue Lo, Hi; |
3204 | SDLoc dl(N); |
3205 | |
3206 | SDValue VecOp = N->getOperand(Num: OpNo); |
3207 | EVT VecVT = VecOp.getValueType(); |
3208 | assert(VecVT.isVector() && "Can only split reduce vector operand" ); |
3209 | GetSplitVector(Op: VecOp, Lo, Hi); |
3210 | |
3211 | SDValue MaskLo, MaskHi; |
3212 | std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2)); |
3213 | |
3214 | SDValue EVLLo, EVLHi; |
3215 | std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl); |
3216 | |
3217 | const SDNodeFlags Flags = N->getFlags(); |
3218 | |
3219 | SDValue ResLo = |
3220 | DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags); |
3221 | return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags); |
3222 | } |
3223 | |
3224 | SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { |
3225 | // The result has a legal vector type, but the input needs splitting. |
3226 | EVT ResVT = N->getValueType(ResNo: 0); |
3227 | SDValue Lo, Hi; |
3228 | SDLoc dl(N); |
3229 | GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi); |
3230 | EVT InVT = Lo.getValueType(); |
3231 | |
3232 | EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(), |
3233 | EC: InVT.getVectorElementCount()); |
3234 | |
3235 | if (N->isStrictFPOpcode()) { |
3236 | Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, |
3237 | { N->getOperand(0), Lo }); |
3238 | Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, |
3239 | { N->getOperand(0), Hi }); |
3240 | |
3241 | // Build a factor node to remember that this operation is independent |
3242 | // of the other one. |
3243 | SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1), |
3244 | Hi.getValue(R: 1)); |
3245 | |
3246 | // Legalize the chain result - switch anything that used the old chain to |
3247 | // use the new one. |
3248 | ReplaceValueWith(From: SDValue(N, 1), To: Ch); |
3249 | } else if (N->getNumOperands() == 3) { |
3250 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
3251 | SDValue MaskLo, MaskHi, EVLLo, EVLHi; |
3252 | std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1)); |
3253 | std::tie(args&: EVLLo, args&: EVLHi) = |
3254 | DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl); |
3255 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo); |
3256 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi); |
3257 | } else { |
3258 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo); |
3259 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi); |
3260 | } |
3261 | |
3262 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi); |
3263 | } |
3264 | |
3265 | SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { |
3266 | // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will |
3267 | // end up being split all the way down to individual components. Convert the |
3268 | // split pieces into integers and reassemble. |
3269 | EVT ResVT = N->getValueType(ResNo: 0); |
3270 | SDValue Lo, Hi; |
3271 | GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi); |
3272 | SDLoc dl(N); |
3273 | |
3274 | if (ResVT.isScalableVector()) { |
3275 | auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: ResVT); |
3276 | Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo); |
3277 | Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi); |
3278 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi); |
3279 | } |
3280 | |
3281 | Lo = BitConvertToInteger(Op: Lo); |
3282 | Hi = BitConvertToInteger(Op: Hi); |
3283 | |
3284 | if (DAG.getDataLayout().isBigEndian()) |
3285 | std::swap(a&: Lo, b&: Hi); |
3286 | |
3287 | return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResVT, Operand: JoinIntegers(Lo, Hi)); |
3288 | } |
3289 | |
3290 | SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N, |
3291 | unsigned OpNo) { |
3292 | assert(OpNo == 1 && "Invalid OpNo; can only split SubVec." ); |
3293 | // We know that the result type is legal. |
3294 | EVT ResVT = N->getValueType(ResNo: 0); |
3295 | |
3296 | SDValue Vec = N->getOperand(Num: 0); |
3297 | SDValue SubVec = N->getOperand(Num: 1); |
3298 | SDValue Idx = N->getOperand(Num: 2); |
3299 | SDLoc dl(N); |
3300 | |
3301 | SDValue Lo, Hi; |
3302 | GetSplitVector(Op: SubVec, Lo, Hi); |
3303 | |
3304 | uint64_t IdxVal = Idx->getAsZExtVal(); |
3305 | uint64_t LoElts = Lo.getValueType().getVectorMinNumElements(); |
3306 | |
3307 | SDValue FirstInsertion = |
3308 | DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx); |
3309 | SDValue SecondInsertion = |
3310 | DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi, |
3311 | N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl)); |
3312 | |
3313 | return SecondInsertion; |
3314 | } |
3315 | |
3316 | SDValue DAGTypeLegalizer::(SDNode *N) { |
3317 | // We know that the extracted result type is legal. |
3318 | EVT SubVT = N->getValueType(ResNo: 0); |
3319 | SDValue Idx = N->getOperand(Num: 1); |
3320 | SDLoc dl(N); |
3321 | SDValue Lo, Hi; |
3322 | |
3323 | GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi); |
3324 | |
3325 | uint64_t LoEltsMin = Lo.getValueType().getVectorMinNumElements(); |
3326 | uint64_t IdxVal = Idx->getAsZExtVal(); |
3327 | |
3328 | if (IdxVal < LoEltsMin) { |
3329 | assert(IdxVal + SubVT.getVectorMinNumElements() <= LoEltsMin && |
3330 | "Extracted subvector crosses vector split!" ); |
3331 | return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx); |
3332 | } else if (SubVT.isScalableVector() == |
3333 | N->getOperand(Num: 0).getValueType().isScalableVector()) |
3334 | return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Hi, |
3335 | N2: DAG.getVectorIdxConstant(Val: IdxVal - LoEltsMin, DL: dl)); |
3336 | |
3337 | // After this point the DAG node only permits extracting fixed-width |
3338 | // subvectors from scalable vectors. |
3339 | assert(SubVT.isFixedLengthVector() && |
3340 | "Extracting scalable subvector from fixed-width unsupported" ); |
3341 | |
3342 | // If the element type is i1 and we're not promoting the result, then we may |
3343 | // end up loading the wrong data since the bits are packed tightly into |
3344 | // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal) |
3345 | // type at index 4, then we will load a byte starting at index 0. |
3346 | if (SubVT.getScalarType() == MVT::i1) |
3347 | report_fatal_error(reason: "Don't know how to extract fixed-width predicate " |
3348 | "subvector from a scalable predicate vector" ); |
3349 | |
3350 | // Spill the vector to the stack. We should use the alignment for |
3351 | // the smallest part. |
3352 | SDValue Vec = N->getOperand(Num: 0); |
3353 | EVT VecVT = Vec.getValueType(); |
3354 | Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false); |
3355 | SDValue StackPtr = |
3356 | DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign); |
3357 | auto &MF = DAG.getMachineFunction(); |
3358 | auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex(); |
3359 | auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex); |
3360 | |
3361 | SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo, |
3362 | Alignment: SmallestAlign); |
3363 | |
3364 | // Extract the subvector by loading the correct part. |
3365 | StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx); |
3366 | |
3367 | return DAG.getLoad( |
3368 | VT: SubVT, dl, Chain: Store, Ptr: StackPtr, |
3369 | PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction())); |
3370 | } |
3371 | |
3372 | SDValue DAGTypeLegalizer::(SDNode *N) { |
3373 | SDValue Vec = N->getOperand(Num: 0); |
3374 | SDValue Idx = N->getOperand(Num: 1); |
3375 | EVT VecVT = Vec.getValueType(); |
3376 | |
3377 | if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) { |
3378 | uint64_t IdxVal = Index->getZExtValue(); |
3379 | |
3380 | SDValue Lo, Hi; |
3381 | GetSplitVector(Op: Vec, Lo, Hi); |
3382 | |
3383 | uint64_t LoElts = Lo.getValueType().getVectorMinNumElements(); |
3384 | |
3385 | if (IdxVal < LoElts) |
3386 | return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0); |
3387 | else if (!Vec.getValueType().isScalableVector()) |
3388 | return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi, |
3389 | Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N), |
3390 | VT: Idx.getValueType())), 0); |
3391 | } |
3392 | |
3393 | // See if the target wants to custom expand this node. |
3394 | if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true)) |
3395 | return SDValue(); |
3396 | |
3397 | // Make the vector elements byte-addressable if they aren't already. |
3398 | SDLoc dl(N); |
3399 | EVT EltVT = VecVT.getVectorElementType(); |
3400 | if (VecVT.getScalarSizeInBits() < 8) { |
3401 | EltVT = MVT::i8; |
3402 | VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, |
3403 | EC: VecVT.getVectorElementCount()); |
3404 | Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec); |
3405 | } |
3406 | |
3407 | // Store the vector to the stack. |
3408 | // In cases where the vector is illegal it will be broken down into parts |
3409 | // and stored in parts - we should use the alignment for the smallest part. |
3410 | Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false); |
3411 | SDValue StackPtr = |
3412 | DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign); |
3413 | auto &MF = DAG.getMachineFunction(); |
3414 | auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex(); |
3415 | auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex); |
3416 | SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo, |
3417 | Alignment: SmallestAlign); |
3418 | |
3419 | // Load back the required element. |
3420 | StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx); |
3421 | |
3422 | // FIXME: This is to handle i1 vectors with elements promoted to i8. |
3423 | // i1 vector handling needs general improvement. |
3424 | if (N->getValueType(ResNo: 0).bitsLT(VT: EltVT)) { |
3425 | SDValue Load = DAG.getLoad(VT: EltVT, dl, Chain: Store, Ptr: StackPtr, |
3426 | PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction())); |
3427 | return DAG.getZExtOrTrunc(Op: Load, DL: dl, VT: N->getValueType(ResNo: 0)); |
3428 | } |
3429 | |
3430 | return DAG.getExtLoad( |
3431 | ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr, |
3432 | PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT, |
3433 | Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8)); |
3434 | } |
3435 | |
3436 | SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) { |
3437 | SDValue Lo, Hi; |
3438 | |
3439 | // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so |
3440 | // splitting the result has the same effect as splitting the input operand. |
3441 | SplitVecRes_ExtVecInRegOp(N, Lo, Hi); |
3442 | |
3443 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi); |
3444 | } |
3445 | |
3446 | SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) { |
3447 | (void)OpNo; |
3448 | SDValue Lo, Hi; |
3449 | SplitVecRes_Gather(N, Lo, Hi); |
3450 | |
3451 | SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi); |
3452 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
3453 | return SDValue(); |
3454 | } |
3455 | |
3456 | SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) { |
3457 | assert(N->isUnindexed() && "Indexed vp_store of vector?" ); |
3458 | SDValue Ch = N->getChain(); |
3459 | SDValue Ptr = N->getBasePtr(); |
3460 | SDValue Offset = N->getOffset(); |
3461 | assert(Offset.isUndef() && "Unexpected VP store offset" ); |
3462 | SDValue Mask = N->getMask(); |
3463 | SDValue EVL = N->getVectorLength(); |
3464 | SDValue Data = N->getValue(); |
3465 | Align Alignment = N->getOriginalAlign(); |
3466 | SDLoc DL(N); |
3467 | |
3468 | SDValue DataLo, DataHi; |
3469 | if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector) |
3470 | // Split Data operand |
3471 | GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi); |
3472 | else |
3473 | std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL); |
3474 | |
3475 | // Split Mask operand |
3476 | SDValue MaskLo, MaskHi; |
3477 | if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) { |
3478 | SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi); |
3479 | } else { |
3480 | if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector) |
3481 | GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi); |
3482 | else |
3483 | std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL); |
3484 | } |
3485 | |
3486 | EVT MemoryVT = N->getMemoryVT(); |
3487 | EVT LoMemVT, HiMemVT; |
3488 | bool HiIsEmpty = false; |
3489 | std::tie(args&: LoMemVT, args&: HiMemVT) = |
3490 | DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty); |
3491 | |
3492 | // Split EVL |
3493 | SDValue EVLLo, EVLHi; |
3494 | std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL); |
3495 | |
3496 | SDValue Lo, Hi; |
3497 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( |
3498 | PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore, |
3499 | Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(), |
3500 | Ranges: N->getRanges()); |
3501 | |
3502 | Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, |
3503 | AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), |
3504 | IsCompressing: N->isCompressingStore()); |
3505 | |
3506 | // If the hi vp_store has zero storage size, only the lo vp_store is needed. |
3507 | if (HiIsEmpty) |
3508 | return Lo; |
3509 | |
3510 | Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG, |
3511 | IsCompressedMemory: N->isCompressingStore()); |
3512 | |
3513 | MachinePointerInfo MPI; |
3514 | if (LoMemVT.isScalableVector()) { |
3515 | Alignment = commonAlignment(A: Alignment, |
3516 | Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8); |
3517 | MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace()); |
3518 | } else |
3519 | MPI = N->getPointerInfo().getWithOffset( |
3520 | O: LoMemVT.getStoreSize().getFixedValue()); |
3521 | |
3522 | MMO = DAG.getMachineFunction().getMachineMemOperand( |
3523 | PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(), |
3524 | BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges()); |
3525 | |
3526 | Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO, |
3527 | AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), |
3528 | IsCompressing: N->isCompressingStore()); |
3529 | |
3530 | // Build a factor node to remember that this store is independent of the |
3531 | // other one. |
3532 | return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); |
3533 | } |
3534 | |
3535 | SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N, |
3536 | unsigned OpNo) { |
3537 | assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?" ); |
3538 | assert(N->getOffset().isUndef() && "Unexpected VP strided store offset" ); |
3539 | |
3540 | SDLoc DL(N); |
3541 | |
3542 | SDValue Data = N->getValue(); |
3543 | SDValue LoData, HiData; |
3544 | if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector) |
3545 | GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData); |
3546 | else |
3547 | std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL); |
3548 | |
3549 | EVT LoMemVT, HiMemVT; |
3550 | bool HiIsEmpty = false; |
3551 | std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs( |
3552 | VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty); |
3553 | |
3554 | SDValue Mask = N->getMask(); |
3555 | SDValue LoMask, HiMask; |
3556 | if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) |
3557 | SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask); |
3558 | else if (getTypeAction(VT: Mask.getValueType()) == |
3559 | TargetLowering::TypeSplitVector) |
3560 | GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask); |
3561 | else |
3562 | std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL); |
3563 | |
3564 | SDValue LoEVL, HiEVL; |
3565 | std::tie(args&: LoEVL, args&: HiEVL) = |
3566 | DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL); |
3567 | |
3568 | // Generate the low vp_strided_store |
3569 | SDValue Lo = DAG.getStridedStoreVP( |
3570 | Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(), |
3571 | Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(), |
3572 | AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore()); |
3573 | |
3574 | // If the high vp_strided_store has zero storage size, only the low |
3575 | // vp_strided_store is needed. |
3576 | if (HiIsEmpty) |
3577 | return Lo; |
3578 | |
3579 | // Generate the high vp_strided_store. |
3580 | // To calculate the high base address, we need to sum to the low base |
3581 | // address stride number of bytes for each element already stored by low, |
3582 | // that is: Ptr = Ptr + (LoEVL * Stride) |
3583 | EVT PtrVT = N->getBasePtr().getValueType(); |
3584 | SDValue Increment = |
3585 | DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL, |
3586 | N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT)); |
3587 | SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment); |
3588 | |
3589 | Align Alignment = N->getOriginalAlign(); |
3590 | if (LoMemVT.isScalableVector()) |
3591 | Alignment = commonAlignment(A: Alignment, |
3592 | Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8); |
3593 | |
3594 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( |
3595 | PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()), |
3596 | F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(), |
3597 | BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges()); |
3598 | |
3599 | SDValue Hi = DAG.getStridedStoreVP( |
3600 | Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask, |
3601 | EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), |
3602 | IsCompressing: N->isCompressingStore()); |
3603 | |
3604 | // Build a factor node to remember that this store is independent of the |
3605 | // other one. |
3606 | return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); |
3607 | } |
3608 | |
3609 | SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N, |
3610 | unsigned OpNo) { |
3611 | assert(N->isUnindexed() && "Indexed masked store of vector?" ); |
3612 | SDValue Ch = N->getChain(); |
3613 | SDValue Ptr = N->getBasePtr(); |
3614 | SDValue Offset = N->getOffset(); |
3615 | assert(Offset.isUndef() && "Unexpected indexed masked store offset" ); |
3616 | SDValue Mask = N->getMask(); |
3617 | SDValue Data = N->getValue(); |
3618 | Align Alignment = N->getOriginalAlign(); |
3619 | SDLoc DL(N); |
3620 | |
3621 | SDValue DataLo, DataHi; |
3622 | if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector) |
3623 | // Split Data operand |
3624 | GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi); |
3625 | else |
3626 | std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL); |
3627 | |
3628 | // Split Mask operand |
3629 | SDValue MaskLo, MaskHi; |
3630 | if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) { |
3631 | SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi); |
3632 | } else { |
3633 | if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector) |
3634 | GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi); |
3635 | else |
3636 | std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL); |
3637 | } |
3638 | |
3639 | EVT MemoryVT = N->getMemoryVT(); |
3640 | EVT LoMemVT, HiMemVT; |
3641 | bool HiIsEmpty = false; |
3642 | std::tie(args&: LoMemVT, args&: HiMemVT) = |
3643 | DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty); |
3644 | |
3645 | SDValue Lo, Hi, Res; |
3646 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( |
3647 | PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore, |
3648 | Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(), |
3649 | Ranges: N->getRanges()); |
3650 | |
3651 | Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO, |
3652 | AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), |
3653 | IsCompressing: N->isCompressingStore()); |
3654 | |
3655 | if (HiIsEmpty) { |
3656 | // The hi masked store has zero storage size. |
3657 | // Only the lo masked store is needed. |
3658 | Res = Lo; |
3659 | } else { |
3660 | |
3661 | Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG, |
3662 | IsCompressedMemory: N->isCompressingStore()); |
3663 | |
3664 | MachinePointerInfo MPI; |
3665 | if (LoMemVT.isScalableVector()) { |
3666 | Alignment = commonAlignment( |
3667 | A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8); |
3668 | MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace()); |
3669 | } else |
3670 | MPI = N->getPointerInfo().getWithOffset( |
3671 | O: LoMemVT.getStoreSize().getFixedValue()); |
3672 | |
3673 | MMO = DAG.getMachineFunction().getMachineMemOperand( |
3674 | PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(), |
3675 | BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges()); |
3676 | |
3677 | Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO, |
3678 | AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), |
3679 | IsCompressing: N->isCompressingStore()); |
3680 | |
3681 | // Build a factor node to remember that this store is independent of the |
3682 | // other one. |
3683 | Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); |
3684 | } |
3685 | |
3686 | return Res; |
3687 | } |
3688 | |
3689 | SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) { |
3690 | SDValue Ch = N->getChain(); |
3691 | SDValue Ptr = N->getBasePtr(); |
3692 | EVT MemoryVT = N->getMemoryVT(); |
3693 | Align Alignment = N->getOriginalAlign(); |
3694 | SDLoc DL(N); |
3695 | struct Operands { |
3696 | SDValue Mask; |
3697 | SDValue Index; |
3698 | SDValue Scale; |
3699 | SDValue Data; |
3700 | } Ops = [&]() -> Operands { |
3701 | if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) { |
3702 | return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(), |
3703 | .Data: MSC->getValue()}; |
3704 | } |
3705 | auto *VPSC = cast<VPScatterSDNode>(Val: N); |
3706 | return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(), |
3707 | .Data: VPSC->getValue()}; |
3708 | }(); |
3709 | // Split all operands |
3710 | |
3711 | EVT LoMemVT, HiMemVT; |
3712 | std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT); |
3713 | |
3714 | SDValue DataLo, DataHi; |
3715 | if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector) |
3716 | // Split Data operand |
3717 | GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi); |
3718 | else |
3719 | std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL); |
3720 | |
3721 | // Split Mask operand |
3722 | SDValue MaskLo, MaskHi; |
3723 | if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) { |
3724 | SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi); |
3725 | } else { |
3726 | std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL); |
3727 | } |
3728 | |
3729 | SDValue IndexHi, IndexLo; |
3730 | if (getTypeAction(VT: Ops.Index.getValueType()) == |
3731 | TargetLowering::TypeSplitVector) |
3732 | GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi); |
3733 | else |
3734 | std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL); |
3735 | |
3736 | SDValue Lo; |
3737 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( |
3738 | PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore, |
3739 | Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(), |
3740 | Ranges: N->getRanges()); |
3741 | |
3742 | if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) { |
3743 | SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale}; |
3744 | Lo = |
3745 | DAG.getMaskedScatter(VTs: DAG.getVTList(MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO, |
3746 | IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore()); |
3747 | |
3748 | // The order of the Scatter operation after split is well defined. The "Hi" |
3749 | // part comes after the "Lo". So these two operations should be chained one |
3750 | // after another. |
3751 | SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale}; |
3752 | return DAG.getMaskedScatter(VTs: DAG.getVTList(MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi, |
3753 | MMO, IndexType: MSC->getIndexType(), |
3754 | IsTruncating: MSC->isTruncatingStore()); |
3755 | } |
3756 | auto *VPSC = cast<VPScatterSDNode>(Val: N); |
3757 | SDValue EVLLo, EVLHi; |
3758 | std::tie(args&: EVLLo, args&: EVLHi) = |
3759 | DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL); |
3760 | |
3761 | SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo}; |
3762 | Lo = DAG.getScatterVP(VTs: DAG.getVTList(MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO, |
3763 | IndexType: VPSC->getIndexType()); |
3764 | |
3765 | // The order of the Scatter operation after split is well defined. The "Hi" |
3766 | // part comes after the "Lo". So these two operations should be chained one |
3767 | // after another. |
3768 | SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi}; |
3769 | return DAG.getScatterVP(VTs: DAG.getVTList(MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO, |
3770 | IndexType: VPSC->getIndexType()); |
3771 | } |
3772 | |
3773 | SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { |
3774 | assert(N->isUnindexed() && "Indexed store of vector?" ); |
3775 | assert(OpNo == 1 && "Can only split the stored value" ); |
3776 | SDLoc DL(N); |
3777 | |
3778 | bool isTruncating = N->isTruncatingStore(); |
3779 | SDValue Ch = N->getChain(); |
3780 | SDValue Ptr = N->getBasePtr(); |
3781 | EVT MemoryVT = N->getMemoryVT(); |
3782 | Align Alignment = N->getOriginalAlign(); |
3783 | MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); |
3784 | AAMDNodes AAInfo = N->getAAInfo(); |
3785 | SDValue Lo, Hi; |
3786 | GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi); |
3787 | |
3788 | EVT LoMemVT, HiMemVT; |
3789 | std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT); |
3790 | |
3791 | // Scalarize if the split halves are not byte-sized. |
3792 | if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) |
3793 | return TLI.scalarizeVectorStore(ST: N, DAG); |
3794 | |
3795 | if (isTruncating) |
3796 | Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT, |
3797 | Alignment, MMOFlags, AAInfo); |
3798 | else |
3799 | Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags, |
3800 | AAInfo); |
3801 | |
3802 | MachinePointerInfo MPI; |
3803 | IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr); |
3804 | |
3805 | if (isTruncating) |
3806 | Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, |
3807 | SVT: HiMemVT, Alignment, MMOFlags, AAInfo); |
3808 | else |
3809 | Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, AAInfo); |
3810 | |
3811 | return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); |
3812 | } |
3813 | |
3814 | SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { |
3815 | SDLoc DL(N); |
3816 | |
3817 | // The input operands all must have the same type, and we know the result |
3818 | // type is valid. Convert this to a buildvector which extracts all the |
3819 | // input elements. |
3820 | // TODO: If the input elements are power-two vectors, we could convert this to |
3821 | // a new CONCAT_VECTORS node with elements that are half-wide. |
3822 | SmallVector<SDValue, 32> Elts; |
3823 | EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType(); |
3824 | for (const SDValue &Op : N->op_values()) { |
3825 | for (unsigned i = 0, e = Op.getValueType().getVectorNumElements(); |
3826 | i != e; ++i) { |
3827 | Elts.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Op, |
3828 | N2: DAG.getVectorIdxConstant(Val: i, DL))); |
3829 | } |
3830 | } |
3831 | |
3832 | return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts); |
3833 | } |
3834 | |
3835 | SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) { |
3836 | // The result type is legal, but the input type is illegal. If splitting |
3837 | // ends up with the result type of each half still being legal, just |
3838 | // do that. If, however, that would result in an illegal result type, |
3839 | // we can try to get more clever with power-two vectors. Specifically, |
3840 | // split the input type, but also widen the result element size, then |
3841 | // concatenate the halves and truncate again. For example, consider a target |
3842 | // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit |
3843 | // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do: |
3844 | // %inlo = v4i32 extract_subvector %in, 0 |
3845 | // %inhi = v4i32 extract_subvector %in, 4 |
3846 | // %lo16 = v4i16 trunc v4i32 %inlo |
3847 | // %hi16 = v4i16 trunc v4i32 %inhi |
3848 | // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16 |
3849 | // %res = v8i8 trunc v8i16 %in16 |
3850 | // |
3851 | // Without this transform, the original truncate would end up being |
3852 | // scalarized, which is pretty much always a last resort. |
3853 | unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; |
3854 | SDValue InVec = N->getOperand(Num: OpNo); |
3855 | EVT InVT = InVec->getValueType(ResNo: 0); |
3856 | EVT OutVT = N->getValueType(ResNo: 0); |
3857 | ElementCount NumElements = OutVT.getVectorElementCount(); |
3858 | bool IsFloat = OutVT.isFloatingPoint(); |
3859 | |
3860 | unsigned InElementSize = InVT.getScalarSizeInBits(); |
3861 | unsigned OutElementSize = OutVT.getScalarSizeInBits(); |
3862 | |
3863 | // Determine the split output VT. If its legal we can just split dirctly. |
3864 | EVT LoOutVT, HiOutVT; |
3865 | std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT); |
3866 | assert(LoOutVT == HiOutVT && "Unequal split?" ); |
3867 | |
3868 | // If the input elements are only 1/2 the width of the result elements, |
3869 | // just use the normal splitting. Our trick only work if there's room |
3870 | // to split more than once. |
3871 | if (isTypeLegal(VT: LoOutVT) || |
3872 | InElementSize <= OutElementSize * 2) |
3873 | return SplitVecOp_UnaryOp(N); |
3874 | SDLoc DL(N); |
3875 | |
3876 | // Don't touch if this will be scalarized. |
3877 | EVT FinalVT = InVT; |
3878 | while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector) |
3879 | FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext()); |
3880 | |
3881 | if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector) |
3882 | return SplitVecOp_UnaryOp(N); |
3883 | |
3884 | // Get the split input vector. |
3885 | SDValue InLoVec, InHiVec; |
3886 | GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec); |
3887 | |
3888 | // Truncate them to 1/2 the element size. |
3889 | // |
3890 | // This assumes the number of elements is a power of two; any vector that |
3891 | // isn't should be widened, not split. |
3892 | EVT HalfElementVT = IsFloat ? |
3893 | EVT::getFloatingPointVT(BitWidth: InElementSize/2) : |
3894 | EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2); |
3895 | EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, |
3896 | EC: NumElements.divideCoefficientBy(RHS: 2)); |
3897 | |
3898 | SDValue HalfLo; |
3899 | SDValue HalfHi; |
3900 | SDValue Chain; |
3901 | if (N->isStrictFPOpcode()) { |
3902 | HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other}, |
3903 | {N->getOperand(0), InLoVec}); |
3904 | HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other}, |
3905 | {N->getOperand(0), InHiVec}); |
3906 | // Legalize the chain result - switch anything that used the old chain to |
3907 | // use the new one. |
3908 | Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(R: 1), |
3909 | HalfHi.getValue(R: 1)); |
3910 | } else { |
3911 | HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec); |
3912 | HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec); |
3913 | } |
3914 | |
3915 | // Concatenate them to get the full intermediate truncation result. |
3916 | EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements); |
3917 | SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo, |
3918 | N2: HalfHi); |
3919 | // Now finish up by truncating all the way down to the original result |
3920 | // type. This should normally be something that ends up being legal directly, |
3921 | // but in theory if a target has very wide vectors and an annoyingly |
3922 | // restricted set of legal types, this split can chain to build things up. |
3923 | |
3924 | if (N->isStrictFPOpcode()) { |
3925 | SDValue Res = DAG.getNode( |
3926 | ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other}, |
3927 | {Chain, InterVec, |
3928 | DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))}); |
3929 | // Relink the chain |
3930 | ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1)); |
3931 | return Res; |
3932 | } |
3933 | |
3934 | return IsFloat |
3935 | ? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec, |
3936 | N2: DAG.getTargetConstant( |
3937 | Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))) |
3938 | : DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec); |
3939 | } |
3940 | |
3941 | SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { |
3942 | assert(N->getValueType(0).isVector() && |
3943 | N->getOperand(0).getValueType().isVector() && |
3944 | "Operand types must be vectors" ); |
3945 | // The result has a legal vector type, but the input needs splitting. |
3946 | SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; |
3947 | SDLoc DL(N); |
3948 | GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Lo0, Hi&: Hi0); |
3949 | GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Lo1, Hi&: Hi1); |
3950 | auto PartEltCnt = Lo0.getValueType().getVectorElementCount(); |
3951 | |
3952 | LLVMContext &Context = *DAG.getContext(); |
3953 | EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt); |
3954 | EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2); |
3955 | |
3956 | if (N->getOpcode() == ISD::SETCC) { |
3957 | LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2)); |
3958 | HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2)); |
3959 | } else { |
3960 | assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode" ); |
3961 | SDValue MaskLo, MaskHi, EVLLo, EVLHi; |
3962 | std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3)); |
3963 | std::tie(args&: EVLLo, args&: EVLHi) = |
3964 | DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL); |
3965 | LoRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, |
3966 | N3: N->getOperand(Num: 2), N4: MaskLo, N5: EVLLo); |
3967 | HiRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, |
3968 | N3: N->getOperand(Num: 2), N4: MaskHi, N5: EVLHi); |
3969 | } |
3970 | SDValue Con = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WideResVT, N1: LoRes, N2: HiRes); |
3971 | |
3972 | EVT OpVT = N->getOperand(Num: 0).getValueType(); |
3973 | ISD::NodeType ExtendCode = |
3974 | TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT)); |
3975 | return DAG.getNode(Opcode: ExtendCode, DL, VT: N->getValueType(ResNo: 0), Operand: Con); |
3976 | } |
3977 | |
3978 | |
3979 | SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { |
3980 | // The result has a legal vector type, but the input needs splitting. |
3981 | EVT ResVT = N->getValueType(ResNo: 0); |
3982 | SDValue Lo, Hi; |
3983 | SDLoc DL(N); |
3984 | GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi); |
3985 | EVT InVT = Lo.getValueType(); |
3986 | |
3987 | EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(), |
3988 | EC: InVT.getVectorElementCount()); |
3989 | |
3990 | if (N->isStrictFPOpcode()) { |
3991 | Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, |
3992 | { N->getOperand(0), Lo, N->getOperand(2) }); |
3993 | Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, |
3994 | { N->getOperand(0), Hi, N->getOperand(2) }); |
3995 | // Legalize the chain result - switch anything that used the old chain to |
3996 | // use the new one. |
3997 | SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, |
3998 | Lo.getValue(1), Hi.getValue(1)); |
3999 | ReplaceValueWith(From: SDValue(N, 1), To: NewChain); |
4000 | } else if (N->getOpcode() == ISD::VP_FP_ROUND) { |
4001 | SDValue MaskLo, MaskHi, EVLLo, EVLHi; |
4002 | std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1)); |
4003 | std::tie(args&: EVLLo, args&: EVLHi) = |
4004 | DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL); |
4005 | Lo = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo); |
4006 | Hi = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi); |
4007 | } else { |
4008 | Lo = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1)); |
4009 | Hi = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1)); |
4010 | } |
4011 | |
4012 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi); |
4013 | } |
4014 | |
4015 | // Split a vector type in an FP binary operation where the second operand has a |
4016 | // different type from the first. |
4017 | // |
4018 | // The result (and the first input) has a legal vector type, but the second |
4019 | // input needs splitting. |
4020 | SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) { |
4021 | SDLoc DL(N); |
4022 | |
4023 | EVT LHSLoVT, LHSHiVT; |
4024 | std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0)); |
4025 | |
4026 | if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT)) |
4027 | return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements()); |
4028 | |
4029 | SDValue LHSLo, LHSHi; |
4030 | std::tie(args&: LHSLo, args&: LHSHi) = |
4031 | DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT); |
4032 | |
4033 | SDValue RHSLo, RHSHi; |
4034 | std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL); |
4035 | |
4036 | SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo); |
4037 | SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi); |
4038 | |
4039 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi); |
4040 | } |
4041 | |
4042 | SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) { |
4043 | EVT ResVT = N->getValueType(ResNo: 0); |
4044 | SDValue Lo, Hi; |
4045 | SDLoc dl(N); |
4046 | GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi); |
4047 | EVT InVT = Lo.getValueType(); |
4048 | |
4049 | EVT NewResVT = |
4050 | EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(), |
4051 | EC: InVT.getVectorElementCount()); |
4052 | |
4053 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1)); |
4054 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1)); |
4055 | |
4056 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi); |
4057 | } |
4058 | |
4059 | //===----------------------------------------------------------------------===// |
4060 | // Result Vector Widening |
4061 | //===----------------------------------------------------------------------===// |
4062 | |
4063 | void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { |
4064 | LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": " ; N->dump(&DAG)); |
4065 | |
4066 | // See if the target wants to custom widen this node. |
4067 | if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo))) |
4068 | return; |
4069 | |
4070 | SDValue Res = SDValue(); |
4071 | |
4072 | auto unrollExpandedOp = [&]() { |
4073 | // We're going to widen this vector op to a legal type by padding with undef |
4074 | // elements. If the wide vector op is eventually going to be expanded to |
4075 | // scalar libcalls, then unroll into scalar ops now to avoid unnecessary |
4076 | // libcalls on the undef elements. |
4077 | EVT VT = N->getValueType(ResNo: 0); |
4078 | EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
4079 | if (!TLI.isOperationLegalOrCustom(Op: N->getOpcode(), VT: WideVecVT) && |
4080 | TLI.isOperationExpand(Op: N->getOpcode(), VT: VT.getScalarType())) { |
4081 | Res = DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements()); |
4082 | return true; |
4083 | } |
4084 | return false; |
4085 | }; |
4086 | |
4087 | switch (N->getOpcode()) { |
4088 | default: |
4089 | #ifndef NDEBUG |
4090 | dbgs() << "WidenVectorResult #" << ResNo << ": " ; |
4091 | N->dump(G: &DAG); |
4092 | dbgs() << "\n" ; |
4093 | #endif |
4094 | report_fatal_error(reason: "Do not know how to widen the result of this operator!" ); |
4095 | |
4096 | case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; |
4097 | case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break; |
4098 | case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; |
4099 | case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; |
4100 | case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; |
4101 | case ISD::INSERT_SUBVECTOR: |
4102 | Res = WidenVecRes_INSERT_SUBVECTOR(N); |
4103 | break; |
4104 | case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; |
4105 | case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; |
4106 | case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; |
4107 | case ISD::STEP_VECTOR: |
4108 | case ISD::SPLAT_VECTOR: |
4109 | case ISD::SCALAR_TO_VECTOR: |
4110 | Res = WidenVecRes_ScalarOp(N); |
4111 | break; |
4112 | case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; |
4113 | case ISD::VSELECT: |
4114 | case ISD::SELECT: |
4115 | case ISD::VP_SELECT: |
4116 | case ISD::VP_MERGE: |
4117 | Res = WidenVecRes_Select(N); |
4118 | break; |
4119 | case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; |
4120 | case ISD::VP_SETCC: |
4121 | case ISD::SETCC: Res = WidenVecRes_SETCC(N); break; |
4122 | case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; |
4123 | case ISD::VECTOR_SHUFFLE: |
4124 | Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N)); |
4125 | break; |
4126 | case ISD::VP_LOAD: |
4127 | Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N)); |
4128 | break; |
4129 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: |
4130 | Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N)); |
4131 | break; |
4132 | case ISD::MLOAD: |
4133 | Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N)); |
4134 | break; |
4135 | case ISD::MGATHER: |
4136 | Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N)); |
4137 | break; |
4138 | case ISD::VP_GATHER: |
4139 | Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N)); |
4140 | break; |
4141 | case ISD::VECTOR_REVERSE: |
4142 | Res = WidenVecRes_VECTOR_REVERSE(N); |
4143 | break; |
4144 | |
4145 | case ISD::ADD: case ISD::VP_ADD: |
4146 | case ISD::AND: case ISD::VP_AND: |
4147 | case ISD::MUL: case ISD::VP_MUL: |
4148 | case ISD::MULHS: |
4149 | case ISD::MULHU: |
4150 | case ISD::OR: case ISD::VP_OR: |
4151 | case ISD::SUB: case ISD::VP_SUB: |
4152 | case ISD::XOR: case ISD::VP_XOR: |
4153 | case ISD::SHL: case ISD::VP_SHL: |
4154 | case ISD::SRA: case ISD::VP_ASHR: |
4155 | case ISD::SRL: case ISD::VP_LSHR: |
4156 | case ISD::FMINNUM: case ISD::VP_FMINNUM: |
4157 | case ISD::FMAXNUM: case ISD::VP_FMAXNUM: |
4158 | case ISD::FMINIMUM: |
4159 | case ISD::VP_FMINIMUM: |
4160 | case ISD::FMAXIMUM: |
4161 | case ISD::VP_FMAXIMUM: |
4162 | case ISD::SMIN: case ISD::VP_SMIN: |
4163 | case ISD::SMAX: case ISD::VP_SMAX: |
4164 | case ISD::UMIN: case ISD::VP_UMIN: |
4165 | case ISD::UMAX: case ISD::VP_UMAX: |
4166 | case ISD::UADDSAT: case ISD::VP_UADDSAT: |
4167 | case ISD::SADDSAT: case ISD::VP_SADDSAT: |
4168 | case ISD::USUBSAT: case ISD::VP_USUBSAT: |
4169 | case ISD::SSUBSAT: case ISD::VP_SSUBSAT: |
4170 | case ISD::SSHLSAT: |
4171 | case ISD::USHLSAT: |
4172 | case ISD::ROTL: |
4173 | case ISD::ROTR: |
4174 | case ISD::AVGFLOORS: |
4175 | case ISD::AVGFLOORU: |
4176 | case ISD::AVGCEILS: |
4177 | case ISD::AVGCEILU: |
4178 | // Vector-predicated binary op widening. Note that -- unlike the |
4179 | // unpredicated versions -- we don't have to worry about trapping on |
4180 | // operations like UDIV, FADD, etc., as we pass on the original vector |
4181 | // length parameter. This means the widened elements containing garbage |
4182 | // aren't active. |
4183 | case ISD::VP_SDIV: |
4184 | case ISD::VP_UDIV: |
4185 | case ISD::VP_SREM: |
4186 | case ISD::VP_UREM: |
4187 | case ISD::VP_FADD: |
4188 | case ISD::VP_FSUB: |
4189 | case ISD::VP_FMUL: |
4190 | case ISD::VP_FDIV: |
4191 | case ISD::VP_FREM: |
4192 | case ISD::VP_FCOPYSIGN: |
4193 | Res = WidenVecRes_Binary(N); |
4194 | break; |
4195 | |
4196 | case ISD::FPOW: |
4197 | case ISD::FREM: |
4198 | if (unrollExpandedOp()) |
4199 | break; |
4200 | // If the target has custom/legal support for the scalar FP intrinsic ops |
4201 | // (they are probably not destined to become libcalls), then widen those |
4202 | // like any other binary ops. |
4203 | [[fallthrough]]; |
4204 | |
4205 | case ISD::FADD: |
4206 | case ISD::FMUL: |
4207 | case ISD::FSUB: |
4208 | case ISD::FDIV: |
4209 | case ISD::SDIV: |
4210 | case ISD::UDIV: |
4211 | case ISD::SREM: |
4212 | case ISD::UREM: |
4213 | Res = WidenVecRes_BinaryCanTrap(N); |
4214 | break; |
4215 | |
4216 | case ISD::SMULFIX: |
4217 | case ISD::SMULFIXSAT: |
4218 | case ISD::UMULFIX: |
4219 | case ISD::UMULFIXSAT: |
4220 | // These are binary operations, but with an extra operand that shouldn't |
4221 | // be widened (the scale). |
4222 | Res = WidenVecRes_BinaryWithExtraScalarOp(N); |
4223 | break; |
4224 | |
4225 | #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ |
4226 | case ISD::STRICT_##DAGN: |
4227 | #include "llvm/IR/ConstrainedOps.def" |
4228 | Res = WidenVecRes_StrictFP(N); |
4229 | break; |
4230 | |
4231 | case ISD::UADDO: |
4232 | case ISD::SADDO: |
4233 | case ISD::USUBO: |
4234 | case ISD::SSUBO: |
4235 | case ISD::UMULO: |
4236 | case ISD::SMULO: |
4237 | Res = WidenVecRes_OverflowOp(N, ResNo); |
4238 | break; |
4239 | |
4240 | case ISD::FCOPYSIGN: |
4241 | Res = WidenVecRes_FCOPYSIGN(N); |
4242 | break; |
4243 | |
4244 | case ISD::IS_FPCLASS: |
4245 | case ISD::FPTRUNC_ROUND: |
4246 | Res = WidenVecRes_UnarySameEltsWithScalarArg(N); |
4247 | break; |
4248 | |
4249 | case ISD::FLDEXP: |
4250 | case ISD::FPOWI: |
4251 | if (!unrollExpandedOp()) |
4252 | Res = WidenVecRes_ExpOp(N); |
4253 | break; |
4254 | |
4255 | case ISD::ANY_EXTEND_VECTOR_INREG: |
4256 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
4257 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
4258 | Res = WidenVecRes_EXTEND_VECTOR_INREG(N); |
4259 | break; |
4260 | |
4261 | case ISD::ANY_EXTEND: |
4262 | case ISD::FP_EXTEND: |
4263 | case ISD::VP_FP_EXTEND: |
4264 | case ISD::FP_ROUND: |
4265 | case ISD::VP_FP_ROUND: |
4266 | case ISD::FP_TO_SINT: |
4267 | case ISD::VP_FP_TO_SINT: |
4268 | case ISD::FP_TO_UINT: |
4269 | case ISD::VP_FP_TO_UINT: |
4270 | case ISD::SIGN_EXTEND: |
4271 | case ISD::VP_SIGN_EXTEND: |
4272 | case ISD::SINT_TO_FP: |
4273 | case ISD::VP_SINT_TO_FP: |
4274 | case ISD::VP_TRUNCATE: |
4275 | case ISD::TRUNCATE: |
4276 | case ISD::UINT_TO_FP: |
4277 | case ISD::VP_UINT_TO_FP: |
4278 | case ISD::ZERO_EXTEND: |
4279 | case ISD::VP_ZERO_EXTEND: |
4280 | Res = WidenVecRes_Convert(N); |
4281 | break; |
4282 | |
4283 | case ISD::FP_TO_SINT_SAT: |
4284 | case ISD::FP_TO_UINT_SAT: |
4285 | Res = WidenVecRes_FP_TO_XINT_SAT(N); |
4286 | break; |
4287 | |
4288 | case ISD::LRINT: |
4289 | case ISD::LLRINT: |
4290 | case ISD::VP_LRINT: |
4291 | case ISD::VP_LLRINT: |
4292 | Res = WidenVecRes_XRINT(N); |
4293 | break; |
4294 | |
4295 | case ISD::FABS: |
4296 | case ISD::FCEIL: |
4297 | case ISD::FCOS: |
4298 | case ISD::FEXP: |
4299 | case ISD::FEXP2: |
4300 | case ISD::FEXP10: |
4301 | case ISD::FFLOOR: |
4302 | case ISD::FLOG: |
4303 | case ISD::FLOG10: |
4304 | case ISD::FLOG2: |
4305 | case ISD::FNEARBYINT: |
4306 | case ISD::FRINT: |
4307 | case ISD::FROUND: |
4308 | case ISD::FROUNDEVEN: |
4309 | case ISD::FSIN: |
4310 | case ISD::FSQRT: |
4311 | case ISD::FTRUNC: |
4312 | if (unrollExpandedOp()) |
4313 | break; |
4314 | // If the target has custom/legal support for the scalar FP intrinsic ops |
4315 | // (they are probably not destined to become libcalls), then widen those |
4316 | // like any other unary ops. |
4317 | [[fallthrough]]; |
4318 | |
4319 | case ISD::ABS: |
4320 | case ISD::VP_ABS: |
4321 | case ISD::BITREVERSE: |
4322 | case ISD::VP_BITREVERSE: |
4323 | case ISD::BSWAP: |
4324 | case ISD::VP_BSWAP: |
4325 | case ISD::CTLZ: |
4326 | case ISD::VP_CTLZ: |
4327 | case ISD::CTLZ_ZERO_UNDEF: |
4328 | case ISD::VP_CTLZ_ZERO_UNDEF: |
4329 | case ISD::CTPOP: |
4330 | case ISD::VP_CTPOP: |
4331 | case ISD::CTTZ: |
4332 | case ISD::VP_CTTZ: |
4333 | case ISD::CTTZ_ZERO_UNDEF: |
4334 | case ISD::VP_CTTZ_ZERO_UNDEF: |
4335 | case ISD::FNEG: case ISD::VP_FNEG: |
4336 | case ISD::VP_FABS: |
4337 | case ISD::VP_SQRT: |
4338 | case ISD::VP_FCEIL: |
4339 | case ISD::VP_FFLOOR: |
4340 | case ISD::VP_FRINT: |
4341 | case ISD::VP_FNEARBYINT: |
4342 | case ISD::VP_FROUND: |
4343 | case ISD::VP_FROUNDEVEN: |
4344 | case ISD::VP_FROUNDTOZERO: |
4345 | case ISD::FREEZE: |
4346 | case ISD::ARITH_FENCE: |
4347 | case ISD::FCANONICALIZE: |
4348 | Res = WidenVecRes_Unary(N); |
4349 | break; |
4350 | case ISD::FMA: case ISD::VP_FMA: |
4351 | case ISD::FSHL: |
4352 | case ISD::VP_FSHL: |
4353 | case ISD::FSHR: |
4354 | case ISD::VP_FSHR: |
4355 | Res = WidenVecRes_Ternary(N); |
4356 | break; |
4357 | } |
4358 | |
4359 | // If Res is null, the sub-method took care of registering the result. |
4360 | if (Res.getNode()) |
4361 | SetWidenedVector(Op: SDValue(N, ResNo), Result: Res); |
4362 | } |
4363 | |
4364 | SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { |
4365 | // Ternary op widening. |
4366 | SDLoc dl(N); |
4367 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4368 | SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0)); |
4369 | SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1)); |
4370 | SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2)); |
4371 | if (N->getNumOperands() == 3) |
4372 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3); |
4373 | |
4374 | assert(N->getNumOperands() == 5 && "Unexpected number of operands!" ); |
4375 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
4376 | |
4377 | SDValue Mask = |
4378 | GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount()); |
4379 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, |
4380 | Ops: {InOp1, InOp2, InOp3, Mask, N->getOperand(Num: 4)}); |
4381 | } |
4382 | |
4383 | SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { |
4384 | // Binary op widening. |
4385 | SDLoc dl(N); |
4386 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4387 | SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0)); |
4388 | SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1)); |
4389 | if (N->getNumOperands() == 2) |
4390 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, |
4391 | Flags: N->getFlags()); |
4392 | |
4393 | assert(N->getNumOperands() == 4 && "Unexpected number of operands!" ); |
4394 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
4395 | |
4396 | SDValue Mask = |
4397 | GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount()); |
4398 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, |
4399 | Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags()); |
4400 | } |
4401 | |
4402 | SDValue DAGTypeLegalizer::(SDNode *N) { |
4403 | // Binary op widening, but with an extra operand that shouldn't be widened. |
4404 | SDLoc dl(N); |
4405 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4406 | SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0)); |
4407 | SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1)); |
4408 | SDValue InOp3 = N->getOperand(Num: 2); |
4409 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3, |
4410 | Flags: N->getFlags()); |
4411 | } |
4412 | |
4413 | // Given a vector of operations that have been broken up to widen, see |
4414 | // if we can collect them together into the next widest legal VT. This |
4415 | // implementation is trap-safe. |
4416 | static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI, |
4417 | SmallVectorImpl<SDValue> &ConcatOps, |
4418 | unsigned ConcatEnd, EVT VT, EVT MaxVT, |
4419 | EVT WidenVT) { |
4420 | // Check to see if we have a single operation with the widen type. |
4421 | if (ConcatEnd == 1) { |
4422 | VT = ConcatOps[0].getValueType(); |
4423 | if (VT == WidenVT) |
4424 | return ConcatOps[0]; |
4425 | } |
4426 | |
4427 | SDLoc dl(ConcatOps[0]); |
4428 | EVT WidenEltVT = WidenVT.getVectorElementType(); |
4429 | |
4430 | // while (Some element of ConcatOps is not of type MaxVT) { |
4431 | // From the end of ConcatOps, collect elements of the same type and put |
4432 | // them into an op of the next larger supported type |
4433 | // } |
4434 | while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { |
4435 | int Idx = ConcatEnd - 1; |
4436 | VT = ConcatOps[Idx--].getValueType(); |
4437 | while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) |
4438 | Idx--; |
4439 | |
4440 | int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1; |
4441 | EVT NextVT; |
4442 | do { |
4443 | NextSize *= 2; |
4444 | NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize); |
4445 | } while (!TLI.isTypeLegal(VT: NextVT)); |
4446 | |
4447 | if (!VT.isVector()) { |
4448 | // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT |
4449 | SDValue VecOp = DAG.getUNDEF(VT: NextVT); |
4450 | unsigned NumToInsert = ConcatEnd - Idx - 1; |
4451 | for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) { |
4452 | VecOp = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: NextVT, N1: VecOp, |
4453 | N2: ConcatOps[OpIdx], N3: DAG.getVectorIdxConstant(Val: i, DL: dl)); |
4454 | } |
4455 | ConcatOps[Idx+1] = VecOp; |
4456 | ConcatEnd = Idx + 2; |
4457 | } else { |
4458 | // Vector type, create a CONCAT_VECTORS of type NextVT |
4459 | SDValue undefVec = DAG.getUNDEF(VT); |
4460 | unsigned OpsToConcat = NextSize/VT.getVectorNumElements(); |
4461 | SmallVector<SDValue, 16> SubConcatOps(OpsToConcat); |
4462 | unsigned RealVals = ConcatEnd - Idx - 1; |
4463 | unsigned SubConcatEnd = 0; |
4464 | unsigned SubConcatIdx = Idx + 1; |
4465 | while (SubConcatEnd < RealVals) |
4466 | SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx]; |
4467 | while (SubConcatEnd < OpsToConcat) |
4468 | SubConcatOps[SubConcatEnd++] = undefVec; |
4469 | ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, |
4470 | VT: NextVT, Ops: SubConcatOps); |
4471 | ConcatEnd = SubConcatIdx + 1; |
4472 | } |
4473 | } |
4474 | |
4475 | // Check to see if we have a single operation with the widen type. |
4476 | if (ConcatEnd == 1) { |
4477 | VT = ConcatOps[0].getValueType(); |
4478 | if (VT == WidenVT) |
4479 | return ConcatOps[0]; |
4480 | } |
4481 | |
4482 | // add undefs of size MaxVT until ConcatOps grows to length of WidenVT |
4483 | unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements(); |
4484 | if (NumOps != ConcatEnd ) { |
4485 | SDValue UndefVal = DAG.getUNDEF(VT: MaxVT); |
4486 | for (unsigned j = ConcatEnd; j < NumOps; ++j) |
4487 | ConcatOps[j] = UndefVal; |
4488 | } |
4489 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, |
4490 | Ops: ArrayRef(ConcatOps.data(), NumOps)); |
4491 | } |
4492 | |
4493 | SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) { |
4494 | // Binary op widening for operations that can trap. |
4495 | unsigned Opcode = N->getOpcode(); |
4496 | SDLoc dl(N); |
4497 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4498 | EVT WidenEltVT = WidenVT.getVectorElementType(); |
4499 | EVT VT = WidenVT; |
4500 | unsigned NumElts = VT.getVectorMinNumElements(); |
4501 | const SDNodeFlags Flags = N->getFlags(); |
4502 | while (!TLI.isTypeLegal(VT) && NumElts != 1) { |
4503 | NumElts = NumElts / 2; |
4504 | VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts); |
4505 | } |
4506 | |
4507 | if (NumElts != 1 && !TLI.canOpTrap(Op: N->getOpcode(), VT)) { |
4508 | // Operation doesn't trap so just widen as normal. |
4509 | SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0)); |
4510 | SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1)); |
4511 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags); |
4512 | } |
4513 | |
4514 | // FIXME: Improve support for scalable vectors. |
4515 | assert(!VT.isScalableVector() && "Scalable vectors not handled yet." ); |
4516 | |
4517 | // No legal vector version so unroll the vector operation and then widen. |
4518 | if (NumElts == 1) |
4519 | return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements()); |
4520 | |
4521 | // Since the operation can trap, apply operation on the original vector. |
4522 | EVT MaxVT = VT; |
4523 | SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0)); |
4524 | SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1)); |
4525 | unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements(); |
4526 | |
4527 | SmallVector<SDValue, 16> ConcatOps(CurNumElts); |
4528 | unsigned ConcatEnd = 0; // Current ConcatOps index. |
4529 | int Idx = 0; // Current Idx into input vectors. |
4530 | |
4531 | // NumElts := greatest legal vector size (at most WidenVT) |
4532 | // while (orig. vector has unhandled elements) { |
4533 | // take munches of size NumElts from the beginning and add to ConcatOps |
4534 | // NumElts := next smaller supported vector size or 1 |
4535 | // } |
4536 | while (CurNumElts != 0) { |
4537 | while (CurNumElts >= NumElts) { |
4538 | SDValue EOp1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: InOp1, |
4539 | N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
4540 | SDValue EOp2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: InOp2, |
4541 | N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
4542 | ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags); |
4543 | Idx += NumElts; |
4544 | CurNumElts -= NumElts; |
4545 | } |
4546 | do { |
4547 | NumElts = NumElts / 2; |
4548 | VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts); |
4549 | } while (!TLI.isTypeLegal(VT) && NumElts != 1); |
4550 | |
4551 | if (NumElts == 1) { |
4552 | for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { |
4553 | SDValue EOp1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: WidenEltVT, |
4554 | N1: InOp1, N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
4555 | SDValue EOp2 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: WidenEltVT, |
4556 | N1: InOp2, N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
4557 | ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT: WidenEltVT, |
4558 | N1: EOp1, N2: EOp2, Flags); |
4559 | } |
4560 | CurNumElts = 0; |
4561 | } |
4562 | } |
4563 | |
4564 | return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT); |
4565 | } |
4566 | |
4567 | SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) { |
4568 | switch (N->getOpcode()) { |
4569 | case ISD::STRICT_FSETCC: |
4570 | case ISD::STRICT_FSETCCS: |
4571 | return WidenVecRes_STRICT_FSETCC(N); |
4572 | case ISD::STRICT_FP_EXTEND: |
4573 | case ISD::STRICT_FP_ROUND: |
4574 | case ISD::STRICT_FP_TO_SINT: |
4575 | case ISD::STRICT_FP_TO_UINT: |
4576 | case ISD::STRICT_SINT_TO_FP: |
4577 | case ISD::STRICT_UINT_TO_FP: |
4578 | return WidenVecRes_Convert_StrictFP(N); |
4579 | default: |
4580 | break; |
4581 | } |
4582 | |
4583 | // StrictFP op widening for operations that can trap. |
4584 | unsigned NumOpers = N->getNumOperands(); |
4585 | unsigned Opcode = N->getOpcode(); |
4586 | SDLoc dl(N); |
4587 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4588 | EVT WidenEltVT = WidenVT.getVectorElementType(); |
4589 | EVT VT = WidenVT; |
4590 | unsigned NumElts = VT.getVectorNumElements(); |
4591 | while (!TLI.isTypeLegal(VT) && NumElts != 1) { |
4592 | NumElts = NumElts / 2; |
4593 | VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts); |
4594 | } |
4595 | |
4596 | // No legal vector version so unroll the vector operation and then widen. |
4597 | if (NumElts == 1) |
4598 | return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements()); |
4599 | |
4600 | // Since the operation can trap, apply operation on the original vector. |
4601 | EVT MaxVT = VT; |
4602 | SmallVector<SDValue, 4> InOps; |
4603 | unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements(); |
4604 | |
4605 | SmallVector<SDValue, 16> ConcatOps(CurNumElts); |
4606 | SmallVector<SDValue, 16> Chains; |
4607 | unsigned ConcatEnd = 0; // Current ConcatOps index. |
4608 | int Idx = 0; // Current Idx into input vectors. |
4609 | |
4610 | // The Chain is the first operand. |
4611 | InOps.push_back(Elt: N->getOperand(Num: 0)); |
4612 | |
4613 | // Now process the remaining operands. |
4614 | for (unsigned i = 1; i < NumOpers; ++i) { |
4615 | SDValue Oper = N->getOperand(Num: i); |
4616 | |
4617 | EVT OpVT = Oper.getValueType(); |
4618 | if (OpVT.isVector()) { |
4619 | if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector) |
4620 | Oper = GetWidenedVector(Op: Oper); |
4621 | else { |
4622 | EVT WideOpVT = |
4623 | EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(), |
4624 | EC: WidenVT.getVectorElementCount()); |
4625 | Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT, |
4626 | N1: DAG.getUNDEF(VT: WideOpVT), N2: Oper, |
4627 | N3: DAG.getVectorIdxConstant(Val: 0, DL: dl)); |
4628 | } |
4629 | } |
4630 | |
4631 | InOps.push_back(Elt: Oper); |
4632 | } |
4633 | |
4634 | // NumElts := greatest legal vector size (at most WidenVT) |
4635 | // while (orig. vector has unhandled elements) { |
4636 | // take munches of size NumElts from the beginning and add to ConcatOps |
4637 | // NumElts := next smaller supported vector size or 1 |
4638 | // } |
4639 | while (CurNumElts != 0) { |
4640 | while (CurNumElts >= NumElts) { |
4641 | SmallVector<SDValue, 4> EOps; |
4642 | |
4643 | for (unsigned i = 0; i < NumOpers; ++i) { |
4644 | SDValue Op = InOps[i]; |
4645 | |
4646 | EVT OpVT = Op.getValueType(); |
4647 | if (OpVT.isVector()) { |
4648 | EVT = |
4649 | EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(), |
4650 | EC: VT.getVectorElementCount()); |
4651 | Op = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: OpExtractVT, N1: Op, |
4652 | N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
4653 | } |
4654 | |
4655 | EOps.push_back(Elt: Op); |
4656 | } |
4657 | |
4658 | EVT OperVT[] = {VT, MVT::Other}; |
4659 | SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps); |
4660 | ConcatOps[ConcatEnd++] = Oper; |
4661 | Chains.push_back(Elt: Oper.getValue(R: 1)); |
4662 | Idx += NumElts; |
4663 | CurNumElts -= NumElts; |
4664 | } |
4665 | do { |
4666 | NumElts = NumElts / 2; |
4667 | VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts); |
4668 | } while (!TLI.isTypeLegal(VT) && NumElts != 1); |
4669 | |
4670 | if (NumElts == 1) { |
4671 | for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { |
4672 | SmallVector<SDValue, 4> EOps; |
4673 | |
4674 | for (unsigned i = 0; i < NumOpers; ++i) { |
4675 | SDValue Op = InOps[i]; |
4676 | |
4677 | EVT OpVT = Op.getValueType(); |
4678 | if (OpVT.isVector()) |
4679 | Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, |
4680 | VT: OpVT.getVectorElementType(), N1: Op, |
4681 | N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
4682 | |
4683 | EOps.push_back(Elt: Op); |
4684 | } |
4685 | |
4686 | EVT WidenVT[] = {WidenEltVT, MVT::Other}; |
4687 | SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps); |
4688 | ConcatOps[ConcatEnd++] = Oper; |
4689 | Chains.push_back(Elt: Oper.getValue(R: 1)); |
4690 | } |
4691 | CurNumElts = 0; |
4692 | } |
4693 | } |
4694 | |
4695 | // Build a factor node to remember all the Ops that have been created. |
4696 | SDValue NewChain; |
4697 | if (Chains.size() == 1) |
4698 | NewChain = Chains[0]; |
4699 | else |
4700 | NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); |
4701 | ReplaceValueWith(From: SDValue(N, 1), To: NewChain); |
4702 | |
4703 | return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT); |
4704 | } |
4705 | |
4706 | SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) { |
4707 | SDLoc DL(N); |
4708 | EVT ResVT = N->getValueType(ResNo: 0); |
4709 | EVT OvVT = N->getValueType(ResNo: 1); |
4710 | EVT WideResVT, WideOvVT; |
4711 | SDValue WideLHS, WideRHS; |
4712 | |
4713 | // TODO: This might result in a widen/split loop. |
4714 | if (ResNo == 0) { |
4715 | WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT); |
4716 | WideOvVT = EVT::getVectorVT( |
4717 | Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(), |
4718 | NumElements: WideResVT.getVectorNumElements()); |
4719 | |
4720 | WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0)); |
4721 | WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1)); |
4722 | } else { |
4723 | WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT); |
4724 | WideResVT = EVT::getVectorVT( |
4725 | Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(), |
4726 | NumElements: WideOvVT.getVectorNumElements()); |
4727 | |
4728 | SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL); |
4729 | WideLHS = DAG.getNode( |
4730 | Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: DAG.getUNDEF(VT: WideResVT), |
4731 | N2: N->getOperand(Num: 0), N3: Zero); |
4732 | WideRHS = DAG.getNode( |
4733 | Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: DAG.getUNDEF(VT: WideResVT), |
4734 | N2: N->getOperand(Num: 1), N3: Zero); |
4735 | } |
4736 | |
4737 | SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT); |
4738 | SDNode *WideNode = DAG.getNode( |
4739 | Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode(); |
4740 | |
4741 | // Replace the other vector result not being explicitly widened here. |
4742 | unsigned OtherNo = 1 - ResNo; |
4743 | EVT OtherVT = N->getValueType(ResNo: OtherNo); |
4744 | if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) { |
4745 | SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo)); |
4746 | } else { |
4747 | SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL); |
4748 | SDValue OtherVal = DAG.getNode( |
4749 | Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero); |
4750 | ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal); |
4751 | } |
4752 | |
4753 | return SDValue(WideNode, ResNo); |
4754 | } |
4755 | |
4756 | SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { |
4757 | LLVMContext &Ctx = *DAG.getContext(); |
4758 | SDValue InOp = N->getOperand(Num: 0); |
4759 | SDLoc DL(N); |
4760 | |
4761 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0)); |
4762 | ElementCount WidenEC = WidenVT.getVectorElementCount(); |
4763 | |
4764 | EVT InVT = InOp.getValueType(); |
4765 | |
4766 | unsigned Opcode = N->getOpcode(); |
4767 | const SDNodeFlags Flags = N->getFlags(); |
4768 | |
4769 | // Handle the case of ZERO_EXTEND where the promoted InVT element size does |
4770 | // not equal that of WidenVT. |
4771 | if (N->getOpcode() == ISD::ZERO_EXTEND && |
4772 | getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger && |
4773 | TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() != |
4774 | WidenVT.getScalarSizeInBits()) { |
4775 | InOp = ZExtPromotedInteger(Op: InOp); |
4776 | InVT = InOp.getValueType(); |
4777 | if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits()) |
4778 | Opcode = ISD::TRUNCATE; |
4779 | } |
4780 | |
4781 | EVT InEltVT = InVT.getVectorElementType(); |
4782 | EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC); |
4783 | ElementCount InVTEC = InVT.getVectorElementCount(); |
4784 | |
4785 | if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) { |
4786 | InOp = GetWidenedVector(Op: N->getOperand(Num: 0)); |
4787 | InVT = InOp.getValueType(); |
4788 | InVTEC = InVT.getVectorElementCount(); |
4789 | if (InVTEC == WidenEC) { |
4790 | if (N->getNumOperands() == 1) |
4791 | return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp); |
4792 | if (N->getNumOperands() == 3) { |
4793 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
4794 | SDValue Mask = |
4795 | GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount()); |
4796 | return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: Mask, N3: N->getOperand(Num: 2)); |
4797 | } |
4798 | return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: N->getOperand(Num: 1), Flags); |
4799 | } |
4800 | if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) { |
4801 | // If both input and result vector types are of same width, extend |
4802 | // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which |
4803 | // accepts fewer elements in the result than in the input. |
4804 | if (Opcode == ISD::ANY_EXTEND) |
4805 | return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp); |
4806 | if (Opcode == ISD::SIGN_EXTEND) |
4807 | return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp); |
4808 | if (Opcode == ISD::ZERO_EXTEND) |
4809 | return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp); |
4810 | } |
4811 | } |
4812 | |
4813 | if (TLI.isTypeLegal(VT: InWidenVT)) { |
4814 | // Because the result and the input are different vector types, widening |
4815 | // the result could create a legal type but widening the input might make |
4816 | // it an illegal type that might lead to repeatedly splitting the input |
4817 | // and then widening it. To avoid this, we widen the input only if |
4818 | // it results in a legal type. |
4819 | if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) { |
4820 | // Widen the input and call convert on the widened input vector. |
4821 | unsigned NumConcat = |
4822 | WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue(); |
4823 | SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(VT: InVT)); |
4824 | Ops[0] = InOp; |
4825 | SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops); |
4826 | if (N->getNumOperands() == 1) |
4827 | return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVec); |
4828 | return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVec, N2: N->getOperand(Num: 1), Flags); |
4829 | } |
4830 | |
4831 | if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) { |
4832 | SDValue InVal = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: InWidenVT, N1: InOp, |
4833 | N2: DAG.getVectorIdxConstant(Val: 0, DL)); |
4834 | // Extract the input and convert the shorten input vector. |
4835 | if (N->getNumOperands() == 1) |
4836 | return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVal); |
4837 | return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVal, N2: N->getOperand(Num: 1), Flags); |
4838 | } |
4839 | } |
4840 | |
4841 | // Otherwise unroll into some nasty scalar code and rebuild the vector. |
4842 | EVT EltVT = WidenVT.getVectorElementType(); |
4843 | SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getUNDEF(VT: EltVT)); |
4844 | // Use the original element count so we don't do more scalar opts than |
4845 | // necessary. |
4846 | unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements(); |
4847 | for (unsigned i=0; i < MinElts; ++i) { |
4848 | SDValue Val = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: InEltVT, N1: InOp, |
4849 | N2: DAG.getVectorIdxConstant(Val: i, DL)); |
4850 | if (N->getNumOperands() == 1) |
4851 | Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, Operand: Val); |
4852 | else |
4853 | Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, N1: Val, N2: N->getOperand(Num: 1), Flags); |
4854 | } |
4855 | |
4856 | return DAG.getBuildVector(VT: WidenVT, DL, Ops); |
4857 | } |
4858 | |
4859 | SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) { |
4860 | SDLoc dl(N); |
4861 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4862 | ElementCount WidenNumElts = WidenVT.getVectorElementCount(); |
4863 | |
4864 | SDValue Src = N->getOperand(Num: 0); |
4865 | EVT SrcVT = Src.getValueType(); |
4866 | |
4867 | // Also widen the input. |
4868 | if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) { |
4869 | Src = GetWidenedVector(Op: Src); |
4870 | SrcVT = Src.getValueType(); |
4871 | } |
4872 | |
4873 | // Input and output not widened to the same size, give up. |
4874 | if (WidenNumElts != SrcVT.getVectorElementCount()) |
4875 | return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue()); |
4876 | |
4877 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1)); |
4878 | } |
4879 | |
4880 | SDValue DAGTypeLegalizer::WidenVecRes_XRINT(SDNode *N) { |
4881 | SDLoc dl(N); |
4882 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4883 | ElementCount WidenNumElts = WidenVT.getVectorElementCount(); |
4884 | |
4885 | SDValue Src = N->getOperand(Num: 0); |
4886 | EVT SrcVT = Src.getValueType(); |
4887 | |
4888 | // Also widen the input. |
4889 | if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) { |
4890 | Src = GetWidenedVector(Op: Src); |
4891 | SrcVT = Src.getValueType(); |
4892 | } |
4893 | |
4894 | // Input and output not widened to the same size, give up. |
4895 | if (WidenNumElts != SrcVT.getVectorElementCount()) |
4896 | return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue()); |
4897 | |
4898 | if (N->getNumOperands() == 1) |
4899 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src); |
4900 | |
4901 | assert(N->getNumOperands() == 3 && "Unexpected number of operands!" ); |
4902 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
4903 | |
4904 | SDValue Mask = |
4905 | GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount()); |
4906 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: Mask, N3: N->getOperand(Num: 2)); |
4907 | } |
4908 | |
4909 | SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) { |
4910 | SDValue InOp = N->getOperand(Num: 1); |
4911 | SDLoc DL(N); |
4912 | SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); |
4913 | |
4914 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4915 | unsigned WidenNumElts = WidenVT.getVectorNumElements(); |
4916 | |
4917 | EVT InVT = InOp.getValueType(); |
4918 | EVT InEltVT = InVT.getVectorElementType(); |
4919 | |
4920 | unsigned Opcode = N->getOpcode(); |
4921 | |
4922 | // FIXME: Optimizations need to be implemented here. |
4923 | |
4924 | // Otherwise unroll into some nasty scalar code and rebuild the vector. |
4925 | EVT EltVT = WidenVT.getVectorElementType(); |
4926 | std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}}; |
4927 | SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(VT: EltVT)); |
4928 | SmallVector<SDValue, 32> OpChains; |
4929 | // Use the original element count so we don't do more scalar opts than |
4930 | // necessary. |
4931 | unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements(); |
4932 | for (unsigned i=0; i < MinElts; ++i) { |
4933 | NewOps[1] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: InEltVT, N1: InOp, |
4934 | N2: DAG.getVectorIdxConstant(Val: i, DL)); |
4935 | Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps); |
4936 | OpChains.push_back(Elt: Ops[i].getValue(R: 1)); |
4937 | } |
4938 | SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains); |
4939 | ReplaceValueWith(From: SDValue(N, 1), To: NewChain); |
4940 | |
4941 | return DAG.getBuildVector(VT: WidenVT, DL, Ops); |
4942 | } |
4943 | |
4944 | SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) { |
4945 | unsigned Opcode = N->getOpcode(); |
4946 | SDValue InOp = N->getOperand(Num: 0); |
4947 | SDLoc DL(N); |
4948 | |
4949 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4950 | EVT WidenSVT = WidenVT.getVectorElementType(); |
4951 | unsigned WidenNumElts = WidenVT.getVectorNumElements(); |
4952 | |
4953 | EVT InVT = InOp.getValueType(); |
4954 | EVT InSVT = InVT.getVectorElementType(); |
4955 | unsigned InVTNumElts = InVT.getVectorNumElements(); |
4956 | |
4957 | if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) { |
4958 | InOp = GetWidenedVector(Op: InOp); |
4959 | InVT = InOp.getValueType(); |
4960 | if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) { |
4961 | switch (Opcode) { |
4962 | case ISD::ANY_EXTEND_VECTOR_INREG: |
4963 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
4964 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
4965 | return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp); |
4966 | } |
4967 | } |
4968 | } |
4969 | |
4970 | // Unroll, extend the scalars and rebuild the vector. |
4971 | SmallVector<SDValue, 16> Ops; |
4972 | for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) { |
4973 | SDValue Val = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: InSVT, N1: InOp, |
4974 | N2: DAG.getVectorIdxConstant(Val: i, DL)); |
4975 | switch (Opcode) { |
4976 | case ISD::ANY_EXTEND_VECTOR_INREG: |
4977 | Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val); |
4978 | break; |
4979 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
4980 | Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val); |
4981 | break; |
4982 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
4983 | Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val); |
4984 | break; |
4985 | default: |
4986 | llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected" ); |
4987 | } |
4988 | Ops.push_back(Elt: Val); |
4989 | } |
4990 | |
4991 | while (Ops.size() != WidenNumElts) |
4992 | Ops.push_back(Elt: DAG.getUNDEF(VT: WidenSVT)); |
4993 | |
4994 | return DAG.getBuildVector(VT: WidenVT, DL, Ops); |
4995 | } |
4996 | |
4997 | SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) { |
4998 | // If this is an FCOPYSIGN with same input types, we can treat it as a |
4999 | // normal (can trap) binary op. |
5000 | if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType()) |
5001 | return WidenVecRes_BinaryCanTrap(N); |
5002 | |
5003 | // If the types are different, fall back to unrolling. |
5004 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5005 | return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements()); |
5006 | } |
5007 | |
5008 | /// Result and first source operand are different scalar types, but must have |
5009 | /// the same number of elements. There is an additional control argument which |
5010 | /// should be passed through unchanged. |
5011 | SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) { |
5012 | SDValue FpValue = N->getOperand(Num: 0); |
5013 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5014 | if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector) |
5015 | return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements()); |
5016 | SDValue Arg = GetWidenedVector(Op: FpValue); |
5017 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)}, |
5018 | Flags: N->getFlags()); |
5019 | } |
5020 | |
5021 | SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) { |
5022 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5023 | SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0)); |
5024 | SDValue RHS = N->getOperand(Num: 1); |
5025 | SDValue ExpOp = RHS.getValueType().isVector() ? GetWidenedVector(Op: RHS) : RHS; |
5026 | |
5027 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp); |
5028 | } |
5029 | |
5030 | SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { |
5031 | // Unary op widening. |
5032 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5033 | SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0)); |
5034 | if (N->getNumOperands() == 1) |
5035 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags()); |
5036 | |
5037 | assert(N->getNumOperands() == 3 && "Unexpected number of operands!" ); |
5038 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
5039 | |
5040 | SDValue Mask = |
5041 | GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount()); |
5042 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, |
5043 | Ops: {InOp, Mask, N->getOperand(Num: 2)}); |
5044 | } |
5045 | |
5046 | SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { |
5047 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5048 | EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
5049 | VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT() |
5050 | .getVectorElementType(), |
5051 | NumElements: WidenVT.getVectorNumElements()); |
5052 | SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0)); |
5053 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), |
5054 | VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT)); |
5055 | } |
5056 | |
5057 | SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { |
5058 | SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); |
5059 | return GetWidenedVector(Op: WidenVec); |
5060 | } |
5061 | |
5062 | SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { |
5063 | SDValue InOp = N->getOperand(Num: 0); |
5064 | EVT InVT = InOp.getValueType(); |
5065 | EVT VT = N->getValueType(ResNo: 0); |
5066 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
5067 | SDLoc dl(N); |
5068 | |
5069 | switch (getTypeAction(VT: InVT)) { |
5070 | case TargetLowering::TypeLegal: |
5071 | break; |
5072 | case TargetLowering::TypeScalarizeScalableVector: |
5073 | report_fatal_error(reason: "Scalarization of scalable vectors is not supported." ); |
5074 | case TargetLowering::TypePromoteInteger: { |
5075 | // If the incoming type is a vector that is being promoted, then |
5076 | // we know that the elements are arranged differently and that we |
5077 | // must perform the conversion using a stack slot. |
5078 | if (InVT.isVector()) |
5079 | break; |
5080 | |
5081 | // If the InOp is promoted to the same size, convert it. Otherwise, |
5082 | // fall out of the switch and widen the promoted input. |
5083 | SDValue NInOp = GetPromotedInteger(Op: InOp); |
5084 | EVT NInVT = NInOp.getValueType(); |
5085 | if (WidenVT.bitsEq(VT: NInVT)) { |
5086 | // For big endian targets we need to shift the input integer or the |
5087 | // interesting bits will end up at the wrong place. |
5088 | if (DAG.getDataLayout().isBigEndian()) { |
5089 | unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); |
5090 | EVT ShiftAmtTy = TLI.getShiftAmountTy(LHSTy: NInVT, DL: DAG.getDataLayout()); |
5091 | assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!" ); |
5092 | NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp, |
5093 | N2: DAG.getConstant(Val: ShiftAmt, DL: dl, VT: ShiftAmtTy)); |
5094 | } |
5095 | return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp); |
5096 | } |
5097 | InOp = NInOp; |
5098 | InVT = NInVT; |
5099 | break; |
5100 | } |
5101 | case TargetLowering::TypeSoftenFloat: |
5102 | case TargetLowering::TypePromoteFloat: |
5103 | case TargetLowering::TypeSoftPromoteHalf: |
5104 | case TargetLowering::TypeExpandInteger: |
5105 | case TargetLowering::TypeExpandFloat: |
5106 | case TargetLowering::TypeScalarizeVector: |
5107 | case TargetLowering::TypeSplitVector: |
5108 | break; |
5109 | case TargetLowering::TypeWidenVector: |
5110 | // If the InOp is widened to the same size, convert it. Otherwise, fall |
5111 | // out of the switch and widen the widened input. |
5112 | InOp = GetWidenedVector(Op: InOp); |
5113 | InVT = InOp.getValueType(); |
5114 | if (WidenVT.bitsEq(VT: InVT)) |
5115 | // The input widens to the same size. Convert to the widen value. |
5116 | return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp); |
5117 | break; |
5118 | } |
5119 | |
5120 | unsigned WidenSize = WidenVT.getSizeInBits(); |
5121 | unsigned InSize = InVT.getSizeInBits(); |
5122 | unsigned InScalarSize = InVT.getScalarSizeInBits(); |
5123 | // x86mmx is not an acceptable vector element type, so don't try. |
5124 | if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) { |
5125 | // Determine new input vector type. The new input vector type will use |
5126 | // the same element type (if its a vector) or use the input type as a |
5127 | // vector. It is the same size as the type to widen to. |
5128 | EVT NewInVT; |
5129 | unsigned NewNumParts = WidenSize / InSize; |
5130 | if (InVT.isVector()) { |
5131 | EVT InEltVT = InVT.getVectorElementType(); |
5132 | NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT, |
5133 | NumElements: WidenSize / InEltVT.getSizeInBits()); |
5134 | } else { |
5135 | // For big endian systems, using the promoted input scalar type |
5136 | // to produce the scalar_to_vector would put the desired bits into |
5137 | // the least significant byte(s) of the wider element zero. This |
5138 | // will mean that the users of the result vector are using incorrect |
5139 | // bits. Use the original input type instead. Although either input |
5140 | // type can be used on little endian systems, for consistency we |
5141 | // use the original type there as well. |
5142 | EVT OrigInVT = N->getOperand(Num: 0).getValueType(); |
5143 | NewNumParts = WidenSize / OrigInVT.getSizeInBits(); |
5144 | NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts); |
5145 | } |
5146 | |
5147 | if (TLI.isTypeLegal(VT: NewInVT)) { |
5148 | SDValue NewVec; |
5149 | if (InVT.isVector()) { |
5150 | // Because the result and the input are different vector types, widening |
5151 | // the result could create a legal type but widening the input might |
5152 | // make it an illegal type that might lead to repeatedly splitting the |
5153 | // input and then widening it. To avoid this, we widen the input only if |
5154 | // it results in a legal type. |
5155 | if (WidenSize % InSize == 0) { |
5156 | SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getUNDEF(VT: InVT)); |
5157 | Ops[0] = InOp; |
5158 | |
5159 | NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops); |
5160 | } else { |
5161 | SmallVector<SDValue, 16> Ops; |
5162 | DAG.ExtractVectorElements(Op: InOp, Args&: Ops); |
5163 | Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(), |
5164 | Elt: DAG.getUNDEF(VT: InVT.getVectorElementType())); |
5165 | |
5166 | NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops); |
5167 | } |
5168 | } else { |
5169 | NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp); |
5170 | } |
5171 | return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec); |
5172 | } |
5173 | } |
5174 | |
5175 | return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT); |
5176 | } |
5177 | |
5178 | SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { |
5179 | SDLoc dl(N); |
5180 | // Build a vector with undefined for the new nodes. |
5181 | EVT VT = N->getValueType(ResNo: 0); |
5182 | |
5183 | // Integer BUILD_VECTOR operands may be larger than the node's vector element |
5184 | // type. The UNDEFs need to have the same type as the existing operands. |
5185 | EVT EltVT = N->getOperand(Num: 0).getValueType(); |
5186 | unsigned NumElts = VT.getVectorNumElements(); |
5187 | |
5188 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
5189 | unsigned WidenNumElts = WidenVT.getVectorNumElements(); |
5190 | |
5191 | SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); |
5192 | assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!" ); |
5193 | NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getUNDEF(VT: EltVT)); |
5194 | |
5195 | return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps); |
5196 | } |
5197 | |
5198 | SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { |
5199 | EVT InVT = N->getOperand(Num: 0).getValueType(); |
5200 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5201 | SDLoc dl(N); |
5202 | unsigned NumOperands = N->getNumOperands(); |
5203 | |
5204 | bool InputWidened = false; // Indicates we need to widen the input. |
5205 | if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) { |
5206 | unsigned WidenNumElts = WidenVT.getVectorMinNumElements(); |
5207 | unsigned NumInElts = InVT.getVectorMinNumElements(); |
5208 | if (WidenNumElts % NumInElts == 0) { |
5209 | // Add undef vectors to widen to correct length. |
5210 | unsigned NumConcat = WidenNumElts / NumInElts; |
5211 | SDValue UndefVal = DAG.getUNDEF(VT: InVT); |
5212 | SmallVector<SDValue, 16> Ops(NumConcat); |
5213 | for (unsigned i=0; i < NumOperands; ++i) |
5214 | Ops[i] = N->getOperand(Num: i); |
5215 | for (unsigned i = NumOperands; i != NumConcat; ++i) |
5216 | Ops[i] = UndefVal; |
5217 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops); |
5218 | } |
5219 | } else { |
5220 | InputWidened = true; |
5221 | if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) { |
5222 | // The inputs and the result are widen to the same value. |
5223 | unsigned i; |
5224 | for (i=1; i < NumOperands; ++i) |
5225 | if (!N->getOperand(Num: i).isUndef()) |
5226 | break; |
5227 | |
5228 | if (i == NumOperands) |
5229 | // Everything but the first operand is an UNDEF so just return the |
5230 | // widened first operand. |
5231 | return GetWidenedVector(Op: N->getOperand(Num: 0)); |
5232 | |
5233 | if (NumOperands == 2) { |
5234 | assert(!WidenVT.isScalableVector() && |
5235 | "Cannot use vector shuffles to widen CONCAT_VECTOR result" ); |
5236 | unsigned WidenNumElts = WidenVT.getVectorNumElements(); |
5237 | unsigned NumInElts = InVT.getVectorNumElements(); |
5238 | |
5239 | // Replace concat of two operands with a shuffle. |
5240 | SmallVector<int, 16> MaskOps(WidenNumElts, -1); |
5241 | for (unsigned i = 0; i < NumInElts; ++i) { |
5242 | MaskOps[i] = i; |
5243 | MaskOps[i + NumInElts] = i + WidenNumElts; |
5244 | } |
5245 | return DAG.getVectorShuffle(VT: WidenVT, dl, |
5246 | N1: GetWidenedVector(Op: N->getOperand(Num: 0)), |
5247 | N2: GetWidenedVector(Op: N->getOperand(Num: 1)), |
5248 | Mask: MaskOps); |
5249 | } |
5250 | } |
5251 | } |
5252 | |
5253 | assert(!WidenVT.isScalableVector() && |
5254 | "Cannot use build vectors to widen CONCAT_VECTOR result" ); |
5255 | unsigned WidenNumElts = WidenVT.getVectorNumElements(); |
5256 | unsigned NumInElts = InVT.getVectorNumElements(); |
5257 | |
5258 | // Fall back to use extracts and build vector. |
5259 | EVT EltVT = WidenVT.getVectorElementType(); |
5260 | SmallVector<SDValue, 16> Ops(WidenNumElts); |
5261 | unsigned Idx = 0; |
5262 | for (unsigned i=0; i < NumOperands; ++i) { |
5263 | SDValue InOp = N->getOperand(Num: i); |
5264 | if (InputWidened) |
5265 | InOp = GetWidenedVector(Op: InOp); |
5266 | for (unsigned j = 0; j < NumInElts; ++j) |
5267 | Ops[Idx++] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: InOp, |
5268 | N2: DAG.getVectorIdxConstant(Val: j, DL: dl)); |
5269 | } |
5270 | SDValue UndefVal = DAG.getUNDEF(VT: EltVT); |
5271 | for (; Idx < WidenNumElts; ++Idx) |
5272 | Ops[Idx] = UndefVal; |
5273 | return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops); |
5274 | } |
5275 | |
5276 | SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) { |
5277 | EVT VT = N->getValueType(ResNo: 0); |
5278 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
5279 | SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0)); |
5280 | SDValue InOp2 = N->getOperand(Num: 1); |
5281 | SDValue Idx = N->getOperand(Num: 2); |
5282 | SDLoc dl(N); |
5283 | return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx); |
5284 | } |
5285 | |
5286 | SDValue DAGTypeLegalizer::(SDNode *N) { |
5287 | EVT VT = N->getValueType(ResNo: 0); |
5288 | EVT EltVT = VT.getVectorElementType(); |
5289 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
5290 | SDValue InOp = N->getOperand(Num: 0); |
5291 | SDValue Idx = N->getOperand(Num: 1); |
5292 | SDLoc dl(N); |
5293 | |
5294 | auto InOpTypeAction = getTypeAction(VT: InOp.getValueType()); |
5295 | if (InOpTypeAction == TargetLowering::TypeWidenVector) |
5296 | InOp = GetWidenedVector(Op: InOp); |
5297 | |
5298 | EVT InVT = InOp.getValueType(); |
5299 | |
5300 | // Check if we can just return the input vector after widening. |
5301 | uint64_t IdxVal = Idx->getAsZExtVal(); |
5302 | if (IdxVal == 0 && InVT == WidenVT) |
5303 | return InOp; |
5304 | |
5305 | // Check if we can extract from the vector. |
5306 | unsigned WidenNumElts = WidenVT.getVectorMinNumElements(); |
5307 | unsigned InNumElts = InVT.getVectorMinNumElements(); |
5308 | unsigned VTNumElts = VT.getVectorMinNumElements(); |
5309 | assert(IdxVal % VTNumElts == 0 && |
5310 | "Expected Idx to be a multiple of subvector minimum vector length" ); |
5311 | if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) |
5312 | return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx); |
5313 | |
5314 | if (VT.isScalableVector()) { |
5315 | // Try to split the operation up into smaller extracts and concat the |
5316 | // results together, e.g. |
5317 | // nxv6i64 extract_subvector(nxv12i64, 6) |
5318 | // <-> |
5319 | // nxv8i64 concat( |
5320 | // nxv2i64 extract_subvector(nxv16i64, 6) |
5321 | // nxv2i64 extract_subvector(nxv16i64, 8) |
5322 | // nxv2i64 extract_subvector(nxv16i64, 10) |
5323 | // undef) |
5324 | unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts); |
5325 | assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken " |
5326 | "down type's element count" ); |
5327 | EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, |
5328 | EC: ElementCount::getScalable(MinVal: GCD)); |
5329 | // Avoid recursion around e.g. nxv1i8. |
5330 | if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) { |
5331 | SmallVector<SDValue> Parts; |
5332 | unsigned I = 0; |
5333 | for (; I < VTNumElts / GCD; ++I) |
5334 | Parts.push_back( |
5335 | Elt: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: PartVT, N1: InOp, |
5336 | N2: DAG.getVectorIdxConstant(Val: IdxVal + I * GCD, DL: dl))); |
5337 | for (; I < WidenNumElts / GCD; ++I) |
5338 | Parts.push_back(Elt: DAG.getUNDEF(VT: PartVT)); |
5339 | |
5340 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts); |
5341 | } |
5342 | |
5343 | report_fatal_error(reason: "Don't know how to widen the result of " |
5344 | "EXTRACT_SUBVECTOR for scalable vectors" ); |
5345 | } |
5346 | |
5347 | // We could try widening the input to the right length but for now, extract |
5348 | // the original elements, fill the rest with undefs and build a vector. |
5349 | SmallVector<SDValue, 16> Ops(WidenNumElts); |
5350 | unsigned i; |
5351 | for (i = 0; i < VTNumElts; ++i) |
5352 | Ops[i] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: InOp, |
5353 | N2: DAG.getVectorIdxConstant(Val: IdxVal + i, DL: dl)); |
5354 | |
5355 | SDValue UndefVal = DAG.getUNDEF(VT: EltVT); |
5356 | for (; i < WidenNumElts; ++i) |
5357 | Ops[i] = UndefVal; |
5358 | return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops); |
5359 | } |
5360 | |
5361 | SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) { |
5362 | SDValue InOp = ModifyToType( |
5363 | InOp: N->getOperand(Num: 0), |
5364 | NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true); |
5365 | return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp, |
5366 | N2: N->getOperand(Num: 1)); |
5367 | } |
5368 | |
5369 | SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { |
5370 | SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0)); |
5371 | return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), |
5372 | VT: InOp.getValueType(), N1: InOp, |
5373 | N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2)); |
5374 | } |
5375 | |
5376 | SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { |
5377 | LoadSDNode *LD = cast<LoadSDNode>(Val: N); |
5378 | ISD::LoadExtType ExtType = LD->getExtensionType(); |
5379 | |
5380 | // A vector must always be stored in memory as-is, i.e. without any padding |
5381 | // between the elements, since various code depend on it, e.g. in the |
5382 | // handling of a bitcast of a vector type to int, which may be done with a |
5383 | // vector store followed by an integer load. A vector that does not have |
5384 | // elements that are byte-sized must therefore be stored as an integer |
5385 | // built out of the extracted vector elements. |
5386 | if (!LD->getMemoryVT().isByteSized()) { |
5387 | SDValue Value, NewChain; |
5388 | std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG); |
5389 | ReplaceValueWith(From: SDValue(LD, 0), To: Value); |
5390 | ReplaceValueWith(From: SDValue(LD, 1), To: NewChain); |
5391 | return SDValue(); |
5392 | } |
5393 | |
5394 | // Generate a vector-predicated load if it is custom/legal on the target. To |
5395 | // avoid possible recursion, only do this if the widened mask type is legal. |
5396 | // FIXME: Not all targets may support EVL in VP_LOAD. These will have been |
5397 | // removed from the IR by the ExpandVectorPredication pass but we're |
5398 | // reintroducing them here. |
5399 | EVT LdVT = LD->getMemoryVT(); |
5400 | EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LdVT); |
5401 | EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, |
5402 | WideVT.getVectorElementCount()); |
5403 | if (ExtType == ISD::NON_EXTLOAD && |
5404 | TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) && |
5405 | TLI.isTypeLegal(VT: WideMaskVT)) { |
5406 | SDLoc DL(N); |
5407 | SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT); |
5408 | SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(), |
5409 | EC: LdVT.getVectorElementCount()); |
5410 | const auto *MMO = LD->getMemOperand(); |
5411 | SDValue NewLoad = |
5412 | DAG.getLoadVP(VT: WideVT, dl: DL, Chain: LD->getChain(), Ptr: LD->getBasePtr(), Mask, EVL, |
5413 | PtrInfo: MMO->getPointerInfo(), Alignment: MMO->getAlign(), MMOFlags: MMO->getFlags(), |
5414 | AAInfo: MMO->getAAInfo()); |
5415 | |
5416 | // Modified the chain - switch anything that used the old chain to use |
5417 | // the new one. |
5418 | ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1)); |
5419 | |
5420 | return NewLoad; |
5421 | } |
5422 | |
5423 | SDValue Result; |
5424 | SmallVector<SDValue, 16> LdChain; // Chain for the series of load |
5425 | if (ExtType != ISD::NON_EXTLOAD) |
5426 | Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); |
5427 | else |
5428 | Result = GenWidenVectorLoads(LdChain, LD); |
5429 | |
5430 | if (Result) { |
5431 | // If we generate a single load, we can use that for the chain. Otherwise, |
5432 | // build a factor node to remember the multiple loads are independent and |
5433 | // chain to that. |
5434 | SDValue NewChain; |
5435 | if (LdChain.size() == 1) |
5436 | NewChain = LdChain[0]; |
5437 | else |
5438 | NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain); |
5439 | |
5440 | // Modified the chain - switch anything that used the old chain to use |
5441 | // the new one. |
5442 | ReplaceValueWith(From: SDValue(N, 1), To: NewChain); |
5443 | |
5444 | return Result; |
5445 | } |
5446 | |
5447 | report_fatal_error(reason: "Unable to widen vector load" ); |
5448 | } |
5449 | |
5450 | SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) { |
5451 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5452 | SDValue Mask = N->getMask(); |
5453 | SDValue EVL = N->getVectorLength(); |
5454 | ISD::LoadExtType ExtType = N->getExtensionType(); |
5455 | SDLoc dl(N); |
5456 | |
5457 | // The mask should be widened as well |
5458 | assert(getTypeAction(Mask.getValueType()) == |
5459 | TargetLowering::TypeWidenVector && |
5460 | "Unable to widen binary VP op" ); |
5461 | Mask = GetWidenedVector(Op: Mask); |
5462 | assert(Mask.getValueType().getVectorElementCount() == |
5463 | TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType()) |
5464 | .getVectorElementCount() && |
5465 | "Unable to widen vector load" ); |
5466 | |
5467 | SDValue Res = |
5468 | DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(), |
5469 | Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL, |
5470 | MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad()); |
5471 | // Legalize the chain result - switch anything that used the old chain to |
5472 | // use the new one. |
5473 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
5474 | return Res; |
5475 | } |
5476 | |
5477 | SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) { |
5478 | SDLoc DL(N); |
5479 | |
5480 | // The mask should be widened as well |
5481 | SDValue Mask = N->getMask(); |
5482 | assert(getTypeAction(Mask.getValueType()) == |
5483 | TargetLowering::TypeWidenVector && |
5484 | "Unable to widen VP strided load" ); |
5485 | Mask = GetWidenedVector(Op: Mask); |
5486 | |
5487 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5488 | assert(Mask.getValueType().getVectorElementCount() == |
5489 | WidenVT.getVectorElementCount() && |
5490 | "Data and mask vectors should have the same number of elements" ); |
5491 | |
5492 | SDValue Res = DAG.getStridedLoadVP( |
5493 | AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(), |
5494 | Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask, |
5495 | EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), |
5496 | IsExpanding: N->isExpandingLoad()); |
5497 | |
5498 | // Legalize the chain result - switch anything that used the old chain to |
5499 | // use the new one. |
5500 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
5501 | return Res; |
5502 | } |
5503 | |
5504 | SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) { |
5505 | |
5506 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: N->getValueType(ResNo: 0)); |
5507 | SDValue Mask = N->getMask(); |
5508 | EVT MaskVT = Mask.getValueType(); |
5509 | SDValue PassThru = GetWidenedVector(Op: N->getPassThru()); |
5510 | ISD::LoadExtType ExtType = N->getExtensionType(); |
5511 | SDLoc dl(N); |
5512 | |
5513 | // The mask should be widened as well |
5514 | EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
5515 | VT: MaskVT.getVectorElementType(), |
5516 | NumElements: WidenVT.getVectorNumElements()); |
5517 | Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true); |
5518 | |
5519 | SDValue Res = DAG.getMaskedLoad( |
5520 | VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask, |
5521 | Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(), |
5522 | ExtType, IsExpanding: N->isExpandingLoad()); |
5523 | // Legalize the chain result - switch anything that used the old chain to |
5524 | // use the new one. |
5525 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
5526 | return Res; |
5527 | } |
5528 | |
5529 | SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) { |
5530 | |
5531 | EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5532 | SDValue Mask = N->getMask(); |
5533 | EVT MaskVT = Mask.getValueType(); |
5534 | SDValue PassThru = GetWidenedVector(Op: N->getPassThru()); |
5535 | SDValue Scale = N->getScale(); |
5536 | unsigned NumElts = WideVT.getVectorNumElements(); |
5537 | SDLoc dl(N); |
5538 | |
5539 | // The mask should be widened as well |
5540 | EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
5541 | VT: MaskVT.getVectorElementType(), |
5542 | NumElements: WideVT.getVectorNumElements()); |
5543 | Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true); |
5544 | |
5545 | // Widen the Index operand |
5546 | SDValue Index = N->getIndex(); |
5547 | EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
5548 | VT: Index.getValueType().getScalarType(), |
5549 | NumElements: NumElts); |
5550 | Index = ModifyToType(InOp: Index, NVT: WideIndexVT); |
5551 | SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index, |
5552 | Scale }; |
5553 | |
5554 | // Widen the MemoryType |
5555 | EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
5556 | VT: N->getMemoryVT().getScalarType(), NumElements: NumElts); |
5557 | SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other), |
5558 | WideMemVT, dl, Ops, N->getMemOperand(), |
5559 | N->getIndexType(), N->getExtensionType()); |
5560 | |
5561 | // Legalize the chain result - switch anything that used the old chain to |
5562 | // use the new one. |
5563 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
5564 | return Res; |
5565 | } |
5566 | |
5567 | SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) { |
5568 | EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5569 | SDValue Mask = N->getMask(); |
5570 | SDValue Scale = N->getScale(); |
5571 | ElementCount WideEC = WideVT.getVectorElementCount(); |
5572 | SDLoc dl(N); |
5573 | |
5574 | SDValue Index = GetWidenedVector(Op: N->getIndex()); |
5575 | EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
5576 | VT: N->getMemoryVT().getScalarType(), EC: WideEC); |
5577 | Mask = GetWidenedMask(Mask, EC: WideEC); |
5578 | |
5579 | SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale, |
5580 | Mask, N->getVectorLength()}; |
5581 | SDValue Res = DAG.getGatherVP(DAG.getVTList(WideVT, MVT::Other), WideMemVT, |
5582 | dl, Ops, N->getMemOperand(), N->getIndexType()); |
5583 | |
5584 | // Legalize the chain result - switch anything that used the old chain to |
5585 | // use the new one. |
5586 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
5587 | return Res; |
5588 | } |
5589 | |
5590 | SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) { |
5591 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5592 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0)); |
5593 | } |
5594 | |
5595 | // Return true is this is a SETCC node or a strict version of it. |
5596 | static inline bool isSETCCOp(unsigned Opcode) { |
5597 | switch (Opcode) { |
5598 | case ISD::SETCC: |
5599 | case ISD::STRICT_FSETCC: |
5600 | case ISD::STRICT_FSETCCS: |
5601 | return true; |
5602 | } |
5603 | return false; |
5604 | } |
5605 | |
5606 | // Return true if this is a node that could have two SETCCs as operands. |
5607 | static inline bool isLogicalMaskOp(unsigned Opcode) { |
5608 | switch (Opcode) { |
5609 | case ISD::AND: |
5610 | case ISD::OR: |
5611 | case ISD::XOR: |
5612 | return true; |
5613 | } |
5614 | return false; |
5615 | } |
5616 | |
5617 | // If N is a SETCC or a strict variant of it, return the type |
5618 | // of the compare operands. |
5619 | static inline EVT getSETCCOperandType(SDValue N) { |
5620 | unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; |
5621 | return N->getOperand(Num: OpNo).getValueType(); |
5622 | } |
5623 | |
5624 | // This is used just for the assert in convertMask(). Check that this either |
5625 | // a SETCC or a previously handled SETCC by convertMask(). |
5626 | #ifndef NDEBUG |
5627 | static inline bool isSETCCorConvertedSETCC(SDValue N) { |
5628 | if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) |
5629 | N = N.getOperand(i: 0); |
5630 | else if (N.getOpcode() == ISD::CONCAT_VECTORS) { |
5631 | for (unsigned i = 1; i < N->getNumOperands(); ++i) |
5632 | if (!N->getOperand(Num: i)->isUndef()) |
5633 | return false; |
5634 | N = N.getOperand(i: 0); |
5635 | } |
5636 | |
5637 | if (N.getOpcode() == ISD::TRUNCATE) |
5638 | N = N.getOperand(i: 0); |
5639 | else if (N.getOpcode() == ISD::SIGN_EXTEND) |
5640 | N = N.getOperand(i: 0); |
5641 | |
5642 | if (isLogicalMaskOp(Opcode: N.getOpcode())) |
5643 | return isSETCCorConvertedSETCC(N: N.getOperand(i: 0)) && |
5644 | isSETCCorConvertedSETCC(N: N.getOperand(i: 1)); |
5645 | |
5646 | return (isSETCCOp(Opcode: N.getOpcode()) || |
5647 | ISD::isBuildVectorOfConstantSDNodes(N: N.getNode())); |
5648 | } |
5649 | #endif |
5650 | |
5651 | // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT |
5652 | // to ToMaskVT if needed with vector extension or truncation. |
5653 | SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT, |
5654 | EVT ToMaskVT) { |
5655 | // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled. |
5656 | // FIXME: This code seems to be too restrictive, we might consider |
5657 | // generalizing it or dropping it. |
5658 | assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument." ); |
5659 | |
5660 | // Make a new Mask node, with a legal result VT. |
5661 | SDValue Mask; |
5662 | SmallVector<SDValue, 4> Ops; |
5663 | for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i) |
5664 | Ops.push_back(Elt: InMask->getOperand(Num: i)); |
5665 | if (InMask->isStrictFPOpcode()) { |
5666 | Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), |
5667 | { MaskVT, MVT::Other }, Ops); |
5668 | ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1)); |
5669 | } |
5670 | else |
5671 | Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops); |
5672 | |
5673 | // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign |
5674 | // extend or truncate is needed. |
5675 | LLVMContext &Ctx = *DAG.getContext(); |
5676 | unsigned MaskScalarBits = MaskVT.getScalarSizeInBits(); |
5677 | unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits(); |
5678 | if (MaskScalarBits < ToMaskScalBits) { |
5679 | EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(), |
5680 | NumElements: MaskVT.getVectorNumElements()); |
5681 | Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask); |
5682 | } else if (MaskScalarBits > ToMaskScalBits) { |
5683 | EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(), |
5684 | NumElements: MaskVT.getVectorNumElements()); |
5685 | Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask); |
5686 | } |
5687 | |
5688 | assert(Mask->getValueType(0).getScalarSizeInBits() == |
5689 | ToMaskVT.getScalarSizeInBits() && |
5690 | "Mask should have the right element size by now." ); |
5691 | |
5692 | // Adjust Mask to the right number of elements. |
5693 | unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements(); |
5694 | if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) { |
5695 | SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(Mask)); |
5696 | Mask = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(Mask), VT: ToMaskVT, N1: Mask, |
5697 | N2: ZeroIdx); |
5698 | } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) { |
5699 | unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls); |
5700 | EVT SubVT = Mask->getValueType(ResNo: 0); |
5701 | SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(VT: SubVT)); |
5702 | SubOps[0] = Mask; |
5703 | Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps); |
5704 | } |
5705 | |
5706 | assert((Mask->getValueType(0) == ToMaskVT) && |
5707 | "A mask of ToMaskVT should have been produced by now." ); |
5708 | |
5709 | return Mask; |
5710 | } |
5711 | |
5712 | // This method tries to handle some special cases for the vselect mask |
5713 | // and if needed adjusting the mask vector type to match that of the VSELECT. |
5714 | // Without it, many cases end up with scalarization of the SETCC, with many |
5715 | // unnecessary instructions. |
5716 | SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) { |
5717 | LLVMContext &Ctx = *DAG.getContext(); |
5718 | SDValue Cond = N->getOperand(Num: 0); |
5719 | |
5720 | if (N->getOpcode() != ISD::VSELECT) |
5721 | return SDValue(); |
5722 | |
5723 | if (!isSETCCOp(Opcode: Cond->getOpcode()) && !isLogicalMaskOp(Opcode: Cond->getOpcode())) |
5724 | return SDValue(); |
5725 | |
5726 | // If this is a splitted VSELECT that was previously already handled, do |
5727 | // nothing. |
5728 | EVT CondVT = Cond->getValueType(ResNo: 0); |
5729 | if (CondVT.getScalarSizeInBits() != 1) |
5730 | return SDValue(); |
5731 | |
5732 | EVT VSelVT = N->getValueType(ResNo: 0); |
5733 | |
5734 | // This method can't handle scalable vector types. |
5735 | // FIXME: This support could be added in the future. |
5736 | if (VSelVT.isScalableVector()) |
5737 | return SDValue(); |
5738 | |
5739 | // Only handle vector types which are a power of 2. |
5740 | if (!isPowerOf2_64(Value: VSelVT.getSizeInBits())) |
5741 | return SDValue(); |
5742 | |
5743 | // Don't touch if this will be scalarized. |
5744 | EVT FinalVT = VSelVT; |
5745 | while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector) |
5746 | FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx); |
5747 | |
5748 | if (FinalVT.getVectorNumElements() == 1) |
5749 | return SDValue(); |
5750 | |
5751 | // If there is support for an i1 vector mask, don't touch. |
5752 | if (isSETCCOp(Opcode: Cond.getOpcode())) { |
5753 | EVT SetCCOpVT = getSETCCOperandType(N: Cond); |
5754 | while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal) |
5755 | SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT); |
5756 | EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT); |
5757 | if (SetCCResVT.getScalarSizeInBits() == 1) |
5758 | return SDValue(); |
5759 | } else if (CondVT.getScalarType() == MVT::i1) { |
5760 | // If there is support for an i1 vector mask (or only scalar i1 conditions), |
5761 | // don't touch. |
5762 | while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal) |
5763 | CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT); |
5764 | |
5765 | if (CondVT.getScalarType() == MVT::i1) |
5766 | return SDValue(); |
5767 | } |
5768 | |
5769 | // Widen the vselect result type if needed. |
5770 | if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector) |
5771 | VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT); |
5772 | |
5773 | // The mask of the VSELECT should have integer elements. |
5774 | EVT ToMaskVT = VSelVT; |
5775 | if (!ToMaskVT.getScalarType().isInteger()) |
5776 | ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger(); |
5777 | |
5778 | SDValue Mask; |
5779 | if (isSETCCOp(Opcode: Cond->getOpcode())) { |
5780 | EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: Cond)); |
5781 | Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT); |
5782 | } else if (isLogicalMaskOp(Opcode: Cond->getOpcode()) && |
5783 | isSETCCOp(Opcode: Cond->getOperand(Num: 0).getOpcode()) && |
5784 | isSETCCOp(Opcode: Cond->getOperand(Num: 1).getOpcode())) { |
5785 | // Cond is (AND/OR/XOR (SETCC, SETCC)) |
5786 | SDValue SETCC0 = Cond->getOperand(Num: 0); |
5787 | SDValue SETCC1 = Cond->getOperand(Num: 1); |
5788 | EVT VT0 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC0)); |
5789 | EVT VT1 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC1)); |
5790 | unsigned ScalarBits0 = VT0.getScalarSizeInBits(); |
5791 | unsigned ScalarBits1 = VT1.getScalarSizeInBits(); |
5792 | unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits(); |
5793 | EVT MaskVT; |
5794 | // If the two SETCCs have different VTs, either extend/truncate one of |
5795 | // them to the other "towards" ToMaskVT, or truncate one and extend the |
5796 | // other to ToMaskVT. |
5797 | if (ScalarBits0 != ScalarBits1) { |
5798 | EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1); |
5799 | EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0); |
5800 | if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits()) |
5801 | MaskVT = WideVT; |
5802 | else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits()) |
5803 | MaskVT = NarrowVT; |
5804 | else |
5805 | MaskVT = ToMaskVT; |
5806 | } else |
5807 | // If the two SETCCs have the same VT, don't change it. |
5808 | MaskVT = VT0; |
5809 | |
5810 | // Make new SETCCs and logical nodes. |
5811 | SETCC0 = convertMask(InMask: SETCC0, MaskVT: VT0, ToMaskVT: MaskVT); |
5812 | SETCC1 = convertMask(InMask: SETCC1, MaskVT: VT1, ToMaskVT: MaskVT); |
5813 | Cond = DAG.getNode(Opcode: Cond->getOpcode(), DL: SDLoc(Cond), VT: MaskVT, N1: SETCC0, N2: SETCC1); |
5814 | |
5815 | // Convert the logical op for VSELECT if needed. |
5816 | Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT); |
5817 | } else |
5818 | return SDValue(); |
5819 | |
5820 | return Mask; |
5821 | } |
5822 | |
5823 | SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) { |
5824 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5825 | ElementCount WidenEC = WidenVT.getVectorElementCount(); |
5826 | |
5827 | SDValue Cond1 = N->getOperand(Num: 0); |
5828 | EVT CondVT = Cond1.getValueType(); |
5829 | unsigned Opcode = N->getOpcode(); |
5830 | if (CondVT.isVector()) { |
5831 | if (SDValue WideCond = WidenVSELECTMask(N)) { |
5832 | SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1)); |
5833 | SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2)); |
5834 | assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); |
5835 | return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2); |
5836 | } |
5837 | |
5838 | EVT CondEltVT = CondVT.getVectorElementType(); |
5839 | EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC); |
5840 | if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector) |
5841 | Cond1 = GetWidenedVector(Op: Cond1); |
5842 | |
5843 | // If we have to split the condition there is no point in widening the |
5844 | // select. This would result in an cycle of widening the select -> |
5845 | // widening the condition operand -> splitting the condition operand -> |
5846 | // splitting the select -> widening the select. Instead split this select |
5847 | // further and widen the resulting type. |
5848 | if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) { |
5849 | SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0); |
5850 | SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT); |
5851 | return Res; |
5852 | } |
5853 | |
5854 | if (Cond1.getValueType() != CondWidenVT) |
5855 | Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT); |
5856 | } |
5857 | |
5858 | SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1)); |
5859 | SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2)); |
5860 | assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); |
5861 | if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE) |
5862 | return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2, |
5863 | N4: N->getOperand(Num: 3)); |
5864 | return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2); |
5865 | } |
5866 | |
5867 | SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { |
5868 | SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2)); |
5869 | SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3)); |
5870 | return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), |
5871 | VT: InOp1.getValueType(), N1: N->getOperand(Num: 0), |
5872 | N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4)); |
5873 | } |
5874 | |
5875 | SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { |
5876 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5877 | return DAG.getUNDEF(VT: WidenVT); |
5878 | } |
5879 | |
5880 | SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { |
5881 | EVT VT = N->getValueType(ResNo: 0); |
5882 | SDLoc dl(N); |
5883 | |
5884 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
5885 | unsigned NumElts = VT.getVectorNumElements(); |
5886 | unsigned WidenNumElts = WidenVT.getVectorNumElements(); |
5887 | |
5888 | SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0)); |
5889 | SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1)); |
5890 | |
5891 | // Adjust mask based on new input vector length. |
5892 | SmallVector<int, 16> NewMask; |
5893 | for (unsigned i = 0; i != NumElts; ++i) { |
5894 | int Idx = N->getMaskElt(Idx: i); |
5895 | if (Idx < (int)NumElts) |
5896 | NewMask.push_back(Elt: Idx); |
5897 | else |
5898 | NewMask.push_back(Elt: Idx - NumElts + WidenNumElts); |
5899 | } |
5900 | for (unsigned i = NumElts; i != WidenNumElts; ++i) |
5901 | NewMask.push_back(Elt: -1); |
5902 | return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask); |
5903 | } |
5904 | |
5905 | SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) { |
5906 | EVT VT = N->getValueType(ResNo: 0); |
5907 | EVT EltVT = VT.getVectorElementType(); |
5908 | SDLoc dl(N); |
5909 | |
5910 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
5911 | SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0)); |
5912 | assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type" ); |
5913 | |
5914 | SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue); |
5915 | unsigned WidenNumElts = WidenVT.getVectorMinNumElements(); |
5916 | unsigned VTNumElts = VT.getVectorMinNumElements(); |
5917 | unsigned IdxVal = WidenNumElts - VTNumElts; |
5918 | |
5919 | if (VT.isScalableVector()) { |
5920 | // Try to split the 'Widen ReverseVal' into smaller extracts and concat the |
5921 | // results together, e.g.(nxv6i64 -> nxv8i64) |
5922 | // nxv8i64 vector_reverse |
5923 | // <-> |
5924 | // nxv8i64 concat( |
5925 | // nxv2i64 extract_subvector(nxv8i64, 2) |
5926 | // nxv2i64 extract_subvector(nxv8i64, 4) |
5927 | // nxv2i64 extract_subvector(nxv8i64, 6) |
5928 | // nxv2i64 undef) |
5929 | |
5930 | unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts); |
5931 | EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, |
5932 | EC: ElementCount::getScalable(MinVal: GCD)); |
5933 | assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken " |
5934 | "down type's element count" ); |
5935 | SmallVector<SDValue> Parts; |
5936 | unsigned i = 0; |
5937 | for (; i < VTNumElts / GCD; ++i) |
5938 | Parts.push_back( |
5939 | Elt: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: PartVT, N1: ReverseVal, |
5940 | N2: DAG.getVectorIdxConstant(Val: IdxVal + i * GCD, DL: dl))); |
5941 | for (; i < WidenNumElts / GCD; ++i) |
5942 | Parts.push_back(Elt: DAG.getUNDEF(VT: PartVT)); |
5943 | |
5944 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts); |
5945 | } |
5946 | |
5947 | // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for |
5948 | // fixed-vectors. |
5949 | SmallVector<int, 16> Mask; |
5950 | for (unsigned i = 0; i != VTNumElts; ++i) { |
5951 | Mask.push_back(Elt: IdxVal + i); |
5952 | } |
5953 | for (unsigned i = VTNumElts; i != WidenNumElts; ++i) |
5954 | Mask.push_back(Elt: -1); |
5955 | |
5956 | return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getUNDEF(VT: WidenVT), |
5957 | Mask); |
5958 | } |
5959 | |
5960 | SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { |
5961 | assert(N->getValueType(0).isVector() && |
5962 | N->getOperand(0).getValueType().isVector() && |
5963 | "Operands must be vectors" ); |
5964 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5965 | ElementCount WidenEC = WidenVT.getVectorElementCount(); |
5966 | |
5967 | SDValue InOp1 = N->getOperand(Num: 0); |
5968 | EVT InVT = InOp1.getValueType(); |
5969 | assert(InVT.isVector() && "can not widen non-vector type" ); |
5970 | EVT WidenInVT = |
5971 | EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC); |
5972 | |
5973 | // The input and output types often differ here, and it could be that while |
5974 | // we'd prefer to widen the result type, the input operands have been split. |
5975 | // In this case, we also need to split the result of this node as well. |
5976 | if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) { |
5977 | SDValue SplitVSetCC = SplitVecOp_VSETCC(N); |
5978 | SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT); |
5979 | return Res; |
5980 | } |
5981 | |
5982 | // If the inputs also widen, handle them directly. Otherwise widen by hand. |
5983 | SDValue InOp2 = N->getOperand(Num: 1); |
5984 | if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) { |
5985 | InOp1 = GetWidenedVector(Op: InOp1); |
5986 | InOp2 = GetWidenedVector(Op: InOp2); |
5987 | } else { |
5988 | InOp1 = DAG.WidenVector(N: InOp1, DL: SDLoc(N)); |
5989 | InOp2 = DAG.WidenVector(N: InOp2, DL: SDLoc(N)); |
5990 | } |
5991 | |
5992 | // Assume that the input and output will be widen appropriately. If not, |
5993 | // we will have to unroll it at some point. |
5994 | assert(InOp1.getValueType() == WidenInVT && |
5995 | InOp2.getValueType() == WidenInVT && |
5996 | "Input not widened to expected type!" ); |
5997 | (void)WidenInVT; |
5998 | if (N->getOpcode() == ISD::VP_SETCC) { |
5999 | SDValue Mask = |
6000 | GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount()); |
6001 | return DAG.getNode(Opcode: ISD::VP_SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2, |
6002 | N3: N->getOperand(Num: 2), N4: Mask, N5: N->getOperand(Num: 4)); |
6003 | } |
6004 | return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2, |
6005 | N3: N->getOperand(Num: 2)); |
6006 | } |
6007 | |
6008 | SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) { |
6009 | assert(N->getValueType(0).isVector() && |
6010 | N->getOperand(1).getValueType().isVector() && |
6011 | "Operands must be vectors" ); |
6012 | EVT VT = N->getValueType(ResNo: 0); |
6013 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
6014 | unsigned WidenNumElts = WidenVT.getVectorNumElements(); |
6015 | unsigned NumElts = VT.getVectorNumElements(); |
6016 | EVT EltVT = VT.getVectorElementType(); |
6017 | |
6018 | SDLoc dl(N); |
6019 | SDValue Chain = N->getOperand(Num: 0); |
6020 | SDValue LHS = N->getOperand(Num: 1); |
6021 | SDValue RHS = N->getOperand(Num: 2); |
6022 | SDValue CC = N->getOperand(Num: 3); |
6023 | EVT TmpEltVT = LHS.getValueType().getVectorElementType(); |
6024 | |
6025 | // Fully unroll and reassemble. |
6026 | SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(VT: EltVT)); |
6027 | SmallVector<SDValue, 8> Chains(NumElts); |
6028 | for (unsigned i = 0; i != NumElts; ++i) { |
6029 | SDValue LHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: LHS, |
6030 | N2: DAG.getVectorIdxConstant(Val: i, DL: dl)); |
6031 | SDValue RHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: RHS, |
6032 | N2: DAG.getVectorIdxConstant(Val: i, DL: dl)); |
6033 | |
6034 | Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other}, |
6035 | {Chain, LHSElem, RHSElem, CC}); |
6036 | Chains[i] = Scalars[i].getValue(R: 1); |
6037 | Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i], |
6038 | LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT), |
6039 | RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT)); |
6040 | } |
6041 | |
6042 | SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); |
6043 | ReplaceValueWith(From: SDValue(N, 1), To: NewChain); |
6044 | |
6045 | return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars); |
6046 | } |
6047 | |
6048 | //===----------------------------------------------------------------------===// |
6049 | // Widen Vector Operand |
6050 | //===----------------------------------------------------------------------===// |
6051 | bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { |
6052 | LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": " ; N->dump(&DAG)); |
6053 | SDValue Res = SDValue(); |
6054 | |
6055 | // See if the target wants to custom widen this node. |
6056 | if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) |
6057 | return false; |
6058 | |
6059 | switch (N->getOpcode()) { |
6060 | default: |
6061 | #ifndef NDEBUG |
6062 | dbgs() << "WidenVectorOperand op #" << OpNo << ": " ; |
6063 | N->dump(G: &DAG); |
6064 | dbgs() << "\n" ; |
6065 | #endif |
6066 | report_fatal_error(reason: "Do not know how to widen this operator's operand!" ); |
6067 | |
6068 | case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break; |
6069 | case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; |
6070 | case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break; |
6071 | case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; |
6072 | case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; |
6073 | case ISD::STORE: Res = WidenVecOp_STORE(N); break; |
6074 | case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break; |
6075 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
6076 | Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo); |
6077 | break; |
6078 | case ISD::ANY_EXTEND_VECTOR_INREG: |
6079 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
6080 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
6081 | Res = WidenVecOp_EXTEND_VECTOR_INREG(N); |
6082 | break; |
6083 | case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break; |
6084 | case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break; |
6085 | case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break; |
6086 | case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break; |
6087 | case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; |
6088 | case ISD::STRICT_FSETCC: |
6089 | case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break; |
6090 | case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break; |
6091 | case ISD::FLDEXP: |
6092 | case ISD::FCOPYSIGN: |
6093 | case ISD::LRINT: |
6094 | case ISD::LLRINT: |
6095 | Res = WidenVecOp_UnrollVectorOp(N); |
6096 | break; |
6097 | case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break; |
6098 | |
6099 | case ISD::ANY_EXTEND: |
6100 | case ISD::SIGN_EXTEND: |
6101 | case ISD::ZERO_EXTEND: |
6102 | Res = WidenVecOp_EXTEND(N); |
6103 | break; |
6104 | |
6105 | case ISD::FP_EXTEND: |
6106 | case ISD::STRICT_FP_EXTEND: |
6107 | case ISD::FP_ROUND: |
6108 | case ISD::STRICT_FP_ROUND: |
6109 | case ISD::FP_TO_SINT: |
6110 | case ISD::STRICT_FP_TO_SINT: |
6111 | case ISD::FP_TO_UINT: |
6112 | case ISD::STRICT_FP_TO_UINT: |
6113 | case ISD::SINT_TO_FP: |
6114 | case ISD::STRICT_SINT_TO_FP: |
6115 | case ISD::UINT_TO_FP: |
6116 | case ISD::STRICT_UINT_TO_FP: |
6117 | case ISD::TRUNCATE: |
6118 | Res = WidenVecOp_Convert(N); |
6119 | break; |
6120 | |
6121 | case ISD::FP_TO_SINT_SAT: |
6122 | case ISD::FP_TO_UINT_SAT: |
6123 | Res = WidenVecOp_FP_TO_XINT_SAT(N); |
6124 | break; |
6125 | |
6126 | case ISD::VECREDUCE_FADD: |
6127 | case ISD::VECREDUCE_FMUL: |
6128 | case ISD::VECREDUCE_ADD: |
6129 | case ISD::VECREDUCE_MUL: |
6130 | case ISD::VECREDUCE_AND: |
6131 | case ISD::VECREDUCE_OR: |
6132 | case ISD::VECREDUCE_XOR: |
6133 | case ISD::VECREDUCE_SMAX: |
6134 | case ISD::VECREDUCE_SMIN: |
6135 | case ISD::VECREDUCE_UMAX: |
6136 | case ISD::VECREDUCE_UMIN: |
6137 | case ISD::VECREDUCE_FMAX: |
6138 | case ISD::VECREDUCE_FMIN: |
6139 | case ISD::VECREDUCE_FMAXIMUM: |
6140 | case ISD::VECREDUCE_FMINIMUM: |
6141 | Res = WidenVecOp_VECREDUCE(N); |
6142 | break; |
6143 | case ISD::VECREDUCE_SEQ_FADD: |
6144 | case ISD::VECREDUCE_SEQ_FMUL: |
6145 | Res = WidenVecOp_VECREDUCE_SEQ(N); |
6146 | break; |
6147 | case ISD::VP_REDUCE_FADD: |
6148 | case ISD::VP_REDUCE_SEQ_FADD: |
6149 | case ISD::VP_REDUCE_FMUL: |
6150 | case ISD::VP_REDUCE_SEQ_FMUL: |
6151 | case ISD::VP_REDUCE_ADD: |
6152 | case ISD::VP_REDUCE_MUL: |
6153 | case ISD::VP_REDUCE_AND: |
6154 | case ISD::VP_REDUCE_OR: |
6155 | case ISD::VP_REDUCE_XOR: |
6156 | case ISD::VP_REDUCE_SMAX: |
6157 | case ISD::VP_REDUCE_SMIN: |
6158 | case ISD::VP_REDUCE_UMAX: |
6159 | case ISD::VP_REDUCE_UMIN: |
6160 | case ISD::VP_REDUCE_FMAX: |
6161 | case ISD::VP_REDUCE_FMIN: |
6162 | Res = WidenVecOp_VP_REDUCE(N); |
6163 | break; |
6164 | } |
6165 | |
6166 | // If Res is null, the sub-method took care of registering the result. |
6167 | if (!Res.getNode()) return false; |
6168 | |
6169 | // If the result is N, the sub-method updated N in place. Tell the legalizer |
6170 | // core about this. |
6171 | if (Res.getNode() == N) |
6172 | return true; |
6173 | |
6174 | |
6175 | if (N->isStrictFPOpcode()) |
6176 | assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 && |
6177 | "Invalid operand expansion" ); |
6178 | else |
6179 | assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && |
6180 | "Invalid operand expansion" ); |
6181 | |
6182 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
6183 | return false; |
6184 | } |
6185 | |
6186 | SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) { |
6187 | SDLoc DL(N); |
6188 | EVT VT = N->getValueType(ResNo: 0); |
6189 | |
6190 | SDValue InOp = N->getOperand(Num: 0); |
6191 | assert(getTypeAction(InOp.getValueType()) == |
6192 | TargetLowering::TypeWidenVector && |
6193 | "Unexpected type action" ); |
6194 | InOp = GetWidenedVector(Op: InOp); |
6195 | assert(VT.getVectorNumElements() < |
6196 | InOp.getValueType().getVectorNumElements() && |
6197 | "Input wasn't widened!" ); |
6198 | |
6199 | // We may need to further widen the operand until it has the same total |
6200 | // vector size as the result. |
6201 | EVT InVT = InOp.getValueType(); |
6202 | if (InVT.getSizeInBits() != VT.getSizeInBits()) { |
6203 | EVT InEltVT = InVT.getVectorElementType(); |
6204 | for (EVT FixedVT : MVT::vector_valuetypes()) { |
6205 | EVT FixedEltVT = FixedVT.getVectorElementType(); |
6206 | if (TLI.isTypeLegal(FixedVT) && |
6207 | FixedVT.getSizeInBits() == VT.getSizeInBits() && |
6208 | FixedEltVT == InEltVT) { |
6209 | assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() && |
6210 | "Not enough elements in the fixed type for the operand!" ); |
6211 | assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() && |
6212 | "We can't have the same type as we started with!" ); |
6213 | if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements()) |
6214 | InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT, |
6215 | DAG.getUNDEF(FixedVT), InOp, |
6216 | DAG.getVectorIdxConstant(0, DL)); |
6217 | else |
6218 | InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp, |
6219 | DAG.getVectorIdxConstant(0, DL)); |
6220 | break; |
6221 | } |
6222 | } |
6223 | InVT = InOp.getValueType(); |
6224 | if (InVT.getSizeInBits() != VT.getSizeInBits()) |
6225 | // We couldn't find a legal vector type that was a widening of the input |
6226 | // and could be extended in-register to the result type, so we have to |
6227 | // scalarize. |
6228 | return WidenVecOp_Convert(N); |
6229 | } |
6230 | |
6231 | // Use special DAG nodes to represent the operation of extending the |
6232 | // low lanes. |
6233 | switch (N->getOpcode()) { |
6234 | default: |
6235 | llvm_unreachable("Extend legalization on extend operation!" ); |
6236 | case ISD::ANY_EXTEND: |
6237 | return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp); |
6238 | case ISD::SIGN_EXTEND: |
6239 | return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp); |
6240 | case ISD::ZERO_EXTEND: |
6241 | return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp); |
6242 | } |
6243 | } |
6244 | |
6245 | SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) { |
6246 | // The result (and first input) is legal, but the second input is illegal. |
6247 | // We can't do much to fix that, so just unroll and let the extracts off of |
6248 | // the second input be widened as needed later. |
6249 | return DAG.UnrollVectorOp(N); |
6250 | } |
6251 | |
6252 | SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) { |
6253 | SDLoc DL(N); |
6254 | EVT ResultVT = N->getValueType(ResNo: 0); |
6255 | SDValue Test = N->getOperand(Num: 1); |
6256 | SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0)); |
6257 | |
6258 | // Process this node similarly to SETCC. |
6259 | EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType()); |
6260 | if (ResultVT.getScalarType() == MVT::i1) |
6261 | WideResultVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, |
6262 | WideResultVT.getVectorNumElements()); |
6263 | |
6264 | SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT, |
6265 | Ops: {WideArg, Test}, Flags: N->getFlags()); |
6266 | |
6267 | // Extract the needed results from the result vector. |
6268 | EVT ResVT = |
6269 | EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(), |
6270 | NumElements: ResultVT.getVectorNumElements()); |
6271 | SDValue CC = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: ResVT, N1: WideNode, |
6272 | N2: DAG.getVectorIdxConstant(Val: 0, DL)); |
6273 | |
6274 | EVT OpVT = N->getOperand(Num: 0).getValueType(); |
6275 | ISD::NodeType ExtendCode = |
6276 | TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT)); |
6277 | return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC); |
6278 | } |
6279 | |
6280 | SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { |
6281 | // Since the result is legal and the input is illegal. |
6282 | EVT VT = N->getValueType(ResNo: 0); |
6283 | EVT EltVT = VT.getVectorElementType(); |
6284 | SDLoc dl(N); |
6285 | SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0); |
6286 | assert(getTypeAction(InOp.getValueType()) == |
6287 | TargetLowering::TypeWidenVector && |
6288 | "Unexpected type action" ); |
6289 | InOp = GetWidenedVector(Op: InOp); |
6290 | EVT InVT = InOp.getValueType(); |
6291 | unsigned Opcode = N->getOpcode(); |
6292 | |
6293 | // See if a widened result type would be legal, if so widen the node. |
6294 | // FIXME: This isn't safe for StrictFP. Other optimization here is needed. |
6295 | EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, |
6296 | EC: InVT.getVectorElementCount()); |
6297 | if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) { |
6298 | SDValue Res; |
6299 | if (N->isStrictFPOpcode()) { |
6300 | if (Opcode == ISD::STRICT_FP_ROUND) |
6301 | Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other }, |
6302 | { N->getOperand(0), InOp, N->getOperand(2) }); |
6303 | else |
6304 | Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other }, |
6305 | { N->getOperand(0), InOp }); |
6306 | // Legalize the chain result - switch anything that used the old chain to |
6307 | // use the new one. |
6308 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
6309 | } else { |
6310 | if (Opcode == ISD::FP_ROUND) |
6311 | Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, N1: InOp, N2: N->getOperand(Num: 1)); |
6312 | else |
6313 | Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, Operand: InOp); |
6314 | } |
6315 | return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: Res, |
6316 | N2: DAG.getVectorIdxConstant(Val: 0, DL: dl)); |
6317 | } |
6318 | |
6319 | EVT InEltVT = InVT.getVectorElementType(); |
6320 | |
6321 | // Unroll the convert into some scalar code and create a nasty build vector. |
6322 | unsigned NumElts = VT.getVectorNumElements(); |
6323 | SmallVector<SDValue, 16> Ops(NumElts); |
6324 | if (N->isStrictFPOpcode()) { |
6325 | SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); |
6326 | SmallVector<SDValue, 32> OpChains; |
6327 | for (unsigned i=0; i < NumElts; ++i) { |
6328 | NewOps[1] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: InEltVT, N1: InOp, |
6329 | N2: DAG.getVectorIdxConstant(Val: i, DL: dl)); |
6330 | Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps); |
6331 | OpChains.push_back(Elt: Ops[i].getValue(R: 1)); |
6332 | } |
6333 | SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains); |
6334 | ReplaceValueWith(From: SDValue(N, 1), To: NewChain); |
6335 | } else { |
6336 | for (unsigned i = 0; i < NumElts; ++i) |
6337 | Ops[i] = DAG.getNode(Opcode, DL: dl, VT: EltVT, |
6338 | Operand: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: InEltVT, |
6339 | N1: InOp, N2: DAG.getVectorIdxConstant(Val: i, DL: dl))); |
6340 | } |
6341 | |
6342 | return DAG.getBuildVector(VT, DL: dl, Ops); |
6343 | } |
6344 | |
6345 | SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) { |
6346 | EVT DstVT = N->getValueType(ResNo: 0); |
6347 | SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0)); |
6348 | EVT SrcVT = Src.getValueType(); |
6349 | ElementCount WideNumElts = SrcVT.getVectorElementCount(); |
6350 | SDLoc dl(N); |
6351 | |
6352 | // See if a widened result type would be legal, if so widen the node. |
6353 | EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
6354 | VT: DstVT.getVectorElementType(), EC: WideNumElts); |
6355 | if (TLI.isTypeLegal(VT: WideDstVT)) { |
6356 | SDValue Res = |
6357 | DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1)); |
6358 | return DAG.getNode( |
6359 | Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res, |
6360 | N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()))); |
6361 | } |
6362 | |
6363 | // Give up and unroll. |
6364 | return DAG.UnrollVectorOp(N); |
6365 | } |
6366 | |
6367 | SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { |
6368 | EVT VT = N->getValueType(ResNo: 0); |
6369 | SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0)); |
6370 | EVT InWidenVT = InOp.getValueType(); |
6371 | SDLoc dl(N); |
6372 | |
6373 | // Check if we can convert between two legal vector types and extract. |
6374 | TypeSize InWidenSize = InWidenVT.getSizeInBits(); |
6375 | TypeSize Size = VT.getSizeInBits(); |
6376 | // x86mmx is not an acceptable vector element type, so don't try. |
6377 | if (!VT.isVector() && VT != MVT::x86mmx && |
6378 | InWidenSize.hasKnownScalarFactor(Size)) { |
6379 | unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size); |
6380 | EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts); |
6381 | if (TLI.isTypeLegal(VT: NewVT)) { |
6382 | SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp); |
6383 | return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT, N1: BitOp, |
6384 | N2: DAG.getVectorIdxConstant(Val: 0, DL: dl)); |
6385 | } |
6386 | } |
6387 | |
6388 | // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened |
6389 | // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not, |
6390 | // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids |
6391 | // having to copy via memory. |
6392 | if (VT.isVector()) { |
6393 | EVT EltVT = VT.getVectorElementType(); |
6394 | unsigned EltSize = EltVT.getFixedSizeInBits(); |
6395 | if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) { |
6396 | ElementCount NewNumElts = |
6397 | (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits()) |
6398 | .divideCoefficientBy(RHS: EltSize); |
6399 | EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts); |
6400 | if (TLI.isTypeLegal(VT: NewVT)) { |
6401 | SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp); |
6402 | return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: BitOp, |
6403 | N2: DAG.getVectorIdxConstant(Val: 0, DL: dl)); |
6404 | } |
6405 | } |
6406 | } |
6407 | |
6408 | return CreateStackStoreLoad(Op: InOp, DestVT: VT); |
6409 | } |
6410 | |
6411 | SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { |
6412 | EVT VT = N->getValueType(ResNo: 0); |
6413 | EVT EltVT = VT.getVectorElementType(); |
6414 | EVT InVT = N->getOperand(Num: 0).getValueType(); |
6415 | SDLoc dl(N); |
6416 | |
6417 | // If the widen width for this operand is the same as the width of the concat |
6418 | // and all but the first operand is undef, just use the widened operand. |
6419 | unsigned NumOperands = N->getNumOperands(); |
6420 | if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) { |
6421 | unsigned i; |
6422 | for (i = 1; i < NumOperands; ++i) |
6423 | if (!N->getOperand(Num: i).isUndef()) |
6424 | break; |
6425 | |
6426 | if (i == NumOperands) |
6427 | return GetWidenedVector(Op: N->getOperand(Num: 0)); |
6428 | } |
6429 | |
6430 | // Otherwise, fall back to a nasty build vector. |
6431 | unsigned NumElts = VT.getVectorNumElements(); |
6432 | SmallVector<SDValue, 16> Ops(NumElts); |
6433 | |
6434 | unsigned NumInElts = InVT.getVectorNumElements(); |
6435 | |
6436 | unsigned Idx = 0; |
6437 | for (unsigned i=0; i < NumOperands; ++i) { |
6438 | SDValue InOp = N->getOperand(Num: i); |
6439 | assert(getTypeAction(InOp.getValueType()) == |
6440 | TargetLowering::TypeWidenVector && |
6441 | "Unexpected type action" ); |
6442 | InOp = GetWidenedVector(Op: InOp); |
6443 | for (unsigned j = 0; j < NumInElts; ++j) |
6444 | Ops[Idx++] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: InOp, |
6445 | N2: DAG.getVectorIdxConstant(Val: j, DL: dl)); |
6446 | } |
6447 | return DAG.getBuildVector(VT, DL: dl, Ops); |
6448 | } |
6449 | |
6450 | SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) { |
6451 | EVT VT = N->getValueType(ResNo: 0); |
6452 | SDValue SubVec = N->getOperand(Num: 1); |
6453 | SDValue InVec = N->getOperand(Num: 0); |
6454 | |
6455 | if (getTypeAction(VT: SubVec.getValueType()) == TargetLowering::TypeWidenVector) |
6456 | SubVec = GetWidenedVector(Op: SubVec); |
6457 | |
6458 | EVT SubVT = SubVec.getValueType(); |
6459 | |
6460 | // Whether or not all the elements of the widened SubVec will be inserted into |
6461 | // valid indices of VT. |
6462 | bool IndicesValid = false; |
6463 | // If we statically know that VT can fit SubVT, the indices are valid. |
6464 | if (VT.knownBitsGE(VT: SubVT)) |
6465 | IndicesValid = true; |
6466 | else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) { |
6467 | // Otherwise, if we're inserting a fixed vector into a scalable vector and |
6468 | // we know the minimum vscale we can work out if it's valid ourselves. |
6469 | Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute( |
6470 | Attribute::VScaleRange); |
6471 | if (Attr.isValid()) { |
6472 | unsigned VScaleMin = Attr.getVScaleRangeMin(); |
6473 | if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >= |
6474 | SubVT.getFixedSizeInBits()) |
6475 | IndicesValid = true; |
6476 | } |
6477 | } |
6478 | |
6479 | // We need to make sure that the indices are still valid, otherwise we might |
6480 | // widen what was previously well-defined to something undefined. |
6481 | if (IndicesValid && InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0) |
6482 | return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT, N1: InVec, N2: SubVec, |
6483 | N3: N->getOperand(Num: 2)); |
6484 | |
6485 | report_fatal_error(reason: "Don't know how to widen the operands for " |
6486 | "INSERT_SUBVECTOR" ); |
6487 | } |
6488 | |
6489 | SDValue DAGTypeLegalizer::(SDNode *N) { |
6490 | SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0)); |
6491 | return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N), |
6492 | VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1)); |
6493 | } |
6494 | |
6495 | SDValue DAGTypeLegalizer::(SDNode *N) { |
6496 | SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0)); |
6497 | return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N), |
6498 | VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1)); |
6499 | } |
6500 | |
6501 | SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) { |
6502 | SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0)); |
6503 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: InOp); |
6504 | } |
6505 | |
6506 | SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { |
6507 | // We have to widen the value, but we want only to store the original |
6508 | // vector type. |
6509 | StoreSDNode *ST = cast<StoreSDNode>(Val: N); |
6510 | |
6511 | if (!ST->getMemoryVT().getScalarType().isByteSized()) |
6512 | return TLI.scalarizeVectorStore(ST, DAG); |
6513 | |
6514 | if (ST->isTruncatingStore()) |
6515 | return TLI.scalarizeVectorStore(ST, DAG); |
6516 | |
6517 | // Generate a vector-predicated store if it is custom/legal on the target. |
6518 | // To avoid possible recursion, only do this if the widened mask type is |
6519 | // legal. |
6520 | // FIXME: Not all targets may support EVL in VP_STORE. These will have been |
6521 | // removed from the IR by the ExpandVectorPredication pass but we're |
6522 | // reintroducing them here. |
6523 | SDValue StVal = ST->getValue(); |
6524 | EVT StVT = StVal.getValueType(); |
6525 | EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT); |
6526 | EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, |
6527 | WideVT.getVectorElementCount()); |
6528 | |
6529 | if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) && |
6530 | TLI.isTypeLegal(VT: WideMaskVT)) { |
6531 | // Widen the value. |
6532 | SDLoc DL(N); |
6533 | StVal = GetWidenedVector(Op: StVal); |
6534 | SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT); |
6535 | SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(), |
6536 | EC: StVT.getVectorElementCount()); |
6537 | return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(), |
6538 | Offset: DAG.getUNDEF(VT: ST->getBasePtr().getValueType()), Mask, |
6539 | EVL, MemVT: StVT, MMO: ST->getMemOperand(), |
6540 | AM: ST->getAddressingMode()); |
6541 | } |
6542 | |
6543 | SmallVector<SDValue, 16> StChain; |
6544 | if (GenWidenVectorStores(StChain, ST)) { |
6545 | if (StChain.size() == 1) |
6546 | return StChain[0]; |
6547 | |
6548 | return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain); |
6549 | } |
6550 | |
6551 | report_fatal_error(reason: "Unable to widen vector store" ); |
6552 | } |
6553 | |
6554 | SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) { |
6555 | assert((OpNo == 1 || OpNo == 3) && |
6556 | "Can widen only data or mask operand of vp_store" ); |
6557 | VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N); |
6558 | SDValue Mask = ST->getMask(); |
6559 | SDValue StVal = ST->getValue(); |
6560 | SDLoc dl(N); |
6561 | |
6562 | if (OpNo == 1) { |
6563 | // Widen the value. |
6564 | StVal = GetWidenedVector(Op: StVal); |
6565 | |
6566 | // We only handle the case where the mask needs widening to an |
6567 | // identically-sized type as the vector inputs. |
6568 | assert(getTypeAction(Mask.getValueType()) == |
6569 | TargetLowering::TypeWidenVector && |
6570 | "Unable to widen VP store" ); |
6571 | Mask = GetWidenedVector(Op: Mask); |
6572 | } else { |
6573 | Mask = GetWidenedVector(Op: Mask); |
6574 | |
6575 | // We only handle the case where the stored value needs widening to an |
6576 | // identically-sized type as the mask. |
6577 | assert(getTypeAction(StVal.getValueType()) == |
6578 | TargetLowering::TypeWidenVector && |
6579 | "Unable to widen VP store" ); |
6580 | StVal = GetWidenedVector(Op: StVal); |
6581 | } |
6582 | |
6583 | assert(Mask.getValueType().getVectorElementCount() == |
6584 | StVal.getValueType().getVectorElementCount() && |
6585 | "Mask and data vectors should have the same number of elements" ); |
6586 | return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(), |
6587 | Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(), |
6588 | MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(), |
6589 | AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(), |
6590 | IsCompressing: ST->isCompressingStore()); |
6591 | } |
6592 | |
6593 | SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N, |
6594 | unsigned OpNo) { |
6595 | assert((OpNo == 1 || OpNo == 4) && |
6596 | "Can widen only data or mask operand of vp_strided_store" ); |
6597 | VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N); |
6598 | SDValue Mask = SST->getMask(); |
6599 | SDValue StVal = SST->getValue(); |
6600 | SDLoc DL(N); |
6601 | |
6602 | if (OpNo == 1) |
6603 | assert(getTypeAction(Mask.getValueType()) == |
6604 | TargetLowering::TypeWidenVector && |
6605 | "Unable to widen VP strided store" ); |
6606 | else |
6607 | assert(getTypeAction(StVal.getValueType()) == |
6608 | TargetLowering::TypeWidenVector && |
6609 | "Unable to widen VP strided store" ); |
6610 | |
6611 | StVal = GetWidenedVector(Op: StVal); |
6612 | Mask = GetWidenedVector(Op: Mask); |
6613 | |
6614 | assert(StVal.getValueType().getVectorElementCount() == |
6615 | Mask.getValueType().getVectorElementCount() && |
6616 | "Data and mask vectors should have the same number of elements" ); |
6617 | |
6618 | return DAG.getStridedStoreVP( |
6619 | Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(), |
6620 | Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(), |
6621 | MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(), |
6622 | IsCompressing: SST->isCompressingStore()); |
6623 | } |
6624 | |
6625 | SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) { |
6626 | assert((OpNo == 1 || OpNo == 4) && |
6627 | "Can widen only data or mask operand of mstore" ); |
6628 | MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N); |
6629 | SDValue Mask = MST->getMask(); |
6630 | EVT MaskVT = Mask.getValueType(); |
6631 | SDValue StVal = MST->getValue(); |
6632 | SDLoc dl(N); |
6633 | |
6634 | if (OpNo == 1) { |
6635 | // Widen the value. |
6636 | StVal = GetWidenedVector(Op: StVal); |
6637 | |
6638 | // The mask should be widened as well. |
6639 | EVT WideVT = StVal.getValueType(); |
6640 | EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
6641 | VT: MaskVT.getVectorElementType(), |
6642 | NumElements: WideVT.getVectorNumElements()); |
6643 | Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true); |
6644 | } else { |
6645 | // Widen the mask. |
6646 | EVT WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT); |
6647 | Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true); |
6648 | |
6649 | EVT ValueVT = StVal.getValueType(); |
6650 | EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
6651 | VT: ValueVT.getVectorElementType(), |
6652 | NumElements: WideMaskVT.getVectorNumElements()); |
6653 | StVal = ModifyToType(InOp: StVal, NVT: WideVT); |
6654 | } |
6655 | |
6656 | assert(Mask.getValueType().getVectorNumElements() == |
6657 | StVal.getValueType().getVectorNumElements() && |
6658 | "Mask and data vectors should have the same number of elements" ); |
6659 | return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(), |
6660 | Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(), |
6661 | MMO: MST->getMemOperand(), AM: MST->getAddressingMode(), |
6662 | IsTruncating: false, IsCompressing: MST->isCompressingStore()); |
6663 | } |
6664 | |
6665 | SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) { |
6666 | assert(OpNo == 4 && "Can widen only the index of mgather" ); |
6667 | auto *MG = cast<MaskedGatherSDNode>(Val: N); |
6668 | SDValue DataOp = MG->getPassThru(); |
6669 | SDValue Mask = MG->getMask(); |
6670 | SDValue Scale = MG->getScale(); |
6671 | |
6672 | // Just widen the index. It's allowed to have extra elements. |
6673 | SDValue Index = GetWidenedVector(Op: MG->getIndex()); |
6674 | |
6675 | SDLoc dl(N); |
6676 | SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index, |
6677 | Scale}; |
6678 | SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops, |
6679 | MMO: MG->getMemOperand(), IndexType: MG->getIndexType(), |
6680 | ExtTy: MG->getExtensionType()); |
6681 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
6682 | ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0)); |
6683 | return SDValue(); |
6684 | } |
6685 | |
6686 | SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) { |
6687 | MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N); |
6688 | SDValue DataOp = MSC->getValue(); |
6689 | SDValue Mask = MSC->getMask(); |
6690 | SDValue Index = MSC->getIndex(); |
6691 | SDValue Scale = MSC->getScale(); |
6692 | EVT WideMemVT = MSC->getMemoryVT(); |
6693 | |
6694 | if (OpNo == 1) { |
6695 | DataOp = GetWidenedVector(Op: DataOp); |
6696 | unsigned NumElts = DataOp.getValueType().getVectorNumElements(); |
6697 | |
6698 | // Widen index. |
6699 | EVT IndexVT = Index.getValueType(); |
6700 | EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
6701 | VT: IndexVT.getVectorElementType(), NumElements: NumElts); |
6702 | Index = ModifyToType(InOp: Index, NVT: WideIndexVT); |
6703 | |
6704 | // The mask should be widened as well. |
6705 | EVT MaskVT = Mask.getValueType(); |
6706 | EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
6707 | VT: MaskVT.getVectorElementType(), NumElements: NumElts); |
6708 | Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true); |
6709 | |
6710 | // Widen the MemoryType |
6711 | WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
6712 | VT: MSC->getMemoryVT().getScalarType(), NumElements: NumElts); |
6713 | } else if (OpNo == 4) { |
6714 | // Just widen the index. It's allowed to have extra elements. |
6715 | Index = GetWidenedVector(Op: Index); |
6716 | } else |
6717 | llvm_unreachable("Can't widen this operand of mscatter" ); |
6718 | |
6719 | SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index, |
6720 | Scale}; |
6721 | return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N), |
6722 | Ops, MSC->getMemOperand(), MSC->getIndexType(), |
6723 | MSC->isTruncatingStore()); |
6724 | } |
6725 | |
6726 | SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) { |
6727 | VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N); |
6728 | SDValue DataOp = VPSC->getValue(); |
6729 | SDValue Mask = VPSC->getMask(); |
6730 | SDValue Index = VPSC->getIndex(); |
6731 | SDValue Scale = VPSC->getScale(); |
6732 | EVT WideMemVT = VPSC->getMemoryVT(); |
6733 | |
6734 | if (OpNo == 1) { |
6735 | DataOp = GetWidenedVector(Op: DataOp); |
6736 | Index = GetWidenedVector(Op: Index); |
6737 | const auto WideEC = DataOp.getValueType().getVectorElementCount(); |
6738 | Mask = GetWidenedMask(Mask, EC: WideEC); |
6739 | WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
6740 | VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC); |
6741 | } else if (OpNo == 3) { |
6742 | // Just widen the index. It's allowed to have extra elements. |
6743 | Index = GetWidenedVector(Op: Index); |
6744 | } else |
6745 | llvm_unreachable("Can't widen this operand of VP_SCATTER" ); |
6746 | |
6747 | SDValue Ops[] = { |
6748 | VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask, |
6749 | VPSC->getVectorLength()}; |
6750 | return DAG.getScatterVP(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N), Ops, |
6751 | VPSC->getMemOperand(), VPSC->getIndexType()); |
6752 | } |
6753 | |
6754 | SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { |
6755 | SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0)); |
6756 | SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1)); |
6757 | SDLoc dl(N); |
6758 | EVT VT = N->getValueType(ResNo: 0); |
6759 | |
6760 | // WARNING: In this code we widen the compare instruction with garbage. |
6761 | // This garbage may contain denormal floats which may be slow. Is this a real |
6762 | // concern ? Should we zero the unused lanes if this is a float compare ? |
6763 | |
6764 | // Get a new SETCC node to compare the newly widened operands. |
6765 | // Only some of the compared elements are legal. |
6766 | EVT SVT = getSetCCResultType(VT: InOp0.getValueType()); |
6767 | // The result type is legal, if its vXi1, keep vXi1 for the new SETCC. |
6768 | if (VT.getScalarType() == MVT::i1) |
6769 | SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, |
6770 | SVT.getVectorElementCount()); |
6771 | |
6772 | SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), |
6773 | VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2)); |
6774 | |
6775 | // Extract the needed results from the result vector. |
6776 | EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
6777 | VT: SVT.getVectorElementType(), |
6778 | EC: VT.getVectorElementCount()); |
6779 | SDValue CC = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: ResVT, N1: WideSETCC, |
6780 | N2: DAG.getVectorIdxConstant(Val: 0, DL: dl)); |
6781 | |
6782 | EVT OpVT = N->getOperand(Num: 0).getValueType(); |
6783 | ISD::NodeType ExtendCode = |
6784 | TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT)); |
6785 | return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC); |
6786 | } |
6787 | |
6788 | SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) { |
6789 | SDValue Chain = N->getOperand(Num: 0); |
6790 | SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1)); |
6791 | SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2)); |
6792 | SDValue CC = N->getOperand(Num: 3); |
6793 | SDLoc dl(N); |
6794 | |
6795 | EVT VT = N->getValueType(ResNo: 0); |
6796 | EVT EltVT = VT.getVectorElementType(); |
6797 | EVT TmpEltVT = LHS.getValueType().getVectorElementType(); |
6798 | unsigned NumElts = VT.getVectorNumElements(); |
6799 | |
6800 | // Unroll into a build vector. |
6801 | SmallVector<SDValue, 8> Scalars(NumElts); |
6802 | SmallVector<SDValue, 8> Chains(NumElts); |
6803 | |
6804 | for (unsigned i = 0; i != NumElts; ++i) { |
6805 | SDValue LHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: LHS, |
6806 | N2: DAG.getVectorIdxConstant(Val: i, DL: dl)); |
6807 | SDValue RHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: RHS, |
6808 | N2: DAG.getVectorIdxConstant(Val: i, DL: dl)); |
6809 | |
6810 | Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other}, |
6811 | {Chain, LHSElem, RHSElem, CC}); |
6812 | Chains[i] = Scalars[i].getValue(R: 1); |
6813 | Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i], |
6814 | LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT), |
6815 | RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT)); |
6816 | } |
6817 | |
6818 | SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); |
6819 | ReplaceValueWith(From: SDValue(N, 1), To: NewChain); |
6820 | |
6821 | return DAG.getBuildVector(VT, DL: dl, Ops: Scalars); |
6822 | } |
6823 | |
6824 | SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) { |
6825 | SDLoc dl(N); |
6826 | SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0)); |
6827 | EVT OrigVT = N->getOperand(Num: 0).getValueType(); |
6828 | EVT WideVT = Op.getValueType(); |
6829 | EVT ElemVT = OrigVT.getVectorElementType(); |
6830 | SDNodeFlags Flags = N->getFlags(); |
6831 | |
6832 | unsigned Opc = N->getOpcode(); |
6833 | unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc); |
6834 | SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags); |
6835 | assert(NeutralElem && "Neutral element must exist" ); |
6836 | |
6837 | // Pad the vector with the neutral element. |
6838 | unsigned OrigElts = OrigVT.getVectorMinNumElements(); |
6839 | unsigned WideElts = WideVT.getVectorMinNumElements(); |
6840 | |
6841 | if (WideVT.isScalableVector()) { |
6842 | unsigned GCD = std::gcd(m: OrigElts, n: WideElts); |
6843 | EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT, |
6844 | EC: ElementCount::getScalable(MinVal: GCD)); |
6845 | SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem); |
6846 | for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD) |
6847 | Op = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideVT, N1: Op, N2: SplatNeutral, |
6848 | N3: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
6849 | return DAG.getNode(Opcode: Opc, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Op, Flags); |
6850 | } |
6851 | |
6852 | for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) |
6853 | Op = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: WideVT, N1: Op, N2: NeutralElem, |
6854 | N3: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
6855 | |
6856 | return DAG.getNode(Opcode: Opc, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Op, Flags); |
6857 | } |
6858 | |
6859 | SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) { |
6860 | SDLoc dl(N); |
6861 | SDValue AccOp = N->getOperand(Num: 0); |
6862 | SDValue VecOp = N->getOperand(Num: 1); |
6863 | SDValue Op = GetWidenedVector(Op: VecOp); |
6864 | |
6865 | EVT OrigVT = VecOp.getValueType(); |
6866 | EVT WideVT = Op.getValueType(); |
6867 | EVT ElemVT = OrigVT.getVectorElementType(); |
6868 | SDNodeFlags Flags = N->getFlags(); |
6869 | |
6870 | unsigned Opc = N->getOpcode(); |
6871 | unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc); |
6872 | SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags); |
6873 | |
6874 | // Pad the vector with the neutral element. |
6875 | unsigned OrigElts = OrigVT.getVectorMinNumElements(); |
6876 | unsigned WideElts = WideVT.getVectorMinNumElements(); |
6877 | |
6878 | if (WideVT.isScalableVector()) { |
6879 | unsigned GCD = std::gcd(m: OrigElts, n: WideElts); |
6880 | EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT, |
6881 | EC: ElementCount::getScalable(MinVal: GCD)); |
6882 | SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem); |
6883 | for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD) |
6884 | Op = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideVT, N1: Op, N2: SplatNeutral, |
6885 | N3: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
6886 | return DAG.getNode(Opcode: Opc, DL: dl, VT: N->getValueType(ResNo: 0), N1: AccOp, N2: Op, Flags); |
6887 | } |
6888 | |
6889 | for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) |
6890 | Op = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: WideVT, N1: Op, N2: NeutralElem, |
6891 | N3: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
6892 | |
6893 | return DAG.getNode(Opcode: Opc, DL: dl, VT: N->getValueType(ResNo: 0), N1: AccOp, N2: Op, Flags); |
6894 | } |
6895 | |
6896 | SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) { |
6897 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
6898 | |
6899 | SDLoc dl(N); |
6900 | SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1)); |
6901 | SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2), |
6902 | EC: Op.getValueType().getVectorElementCount()); |
6903 | |
6904 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), |
6905 | Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)}, |
6906 | Flags: N->getFlags()); |
6907 | } |
6908 | |
6909 | SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) { |
6910 | // This only gets called in the case that the left and right inputs and |
6911 | // result are of a legal odd vector type, and the condition is illegal i1 of |
6912 | // the same odd width that needs widening. |
6913 | EVT VT = N->getValueType(ResNo: 0); |
6914 | assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT)); |
6915 | |
6916 | SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0)); |
6917 | SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N)); |
6918 | SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N)); |
6919 | SDLoc DL(N); |
6920 | |
6921 | SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond, |
6922 | N2: LeftIn, N3: RightIn); |
6923 | return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Select, |
6924 | N2: DAG.getVectorIdxConstant(Val: 0, DL)); |
6925 | } |
6926 | |
6927 | //===----------------------------------------------------------------------===// |
6928 | // Vector Widening Utilities |
6929 | //===----------------------------------------------------------------------===// |
6930 | |
6931 | // Utility function to find the type to chop up a widen vector for load/store |
6932 | // TLI: Target lowering used to determine legal types. |
6933 | // Width: Width left need to load/store. |
6934 | // WidenVT: The widen vector type to load to/store from |
6935 | // Align: If 0, don't allow use of a wider type |
6936 | // WidenEx: If Align is not 0, the amount additional we can load/store from. |
6937 | |
6938 | static std::optional<EVT> findMemType(SelectionDAG &DAG, |
6939 | const TargetLowering &TLI, unsigned Width, |
6940 | EVT WidenVT, unsigned Align = 0, |
6941 | unsigned WidenEx = 0) { |
6942 | EVT WidenEltVT = WidenVT.getVectorElementType(); |
6943 | const bool Scalable = WidenVT.isScalableVector(); |
6944 | unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue(); |
6945 | unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); |
6946 | unsigned AlignInBits = Align*8; |
6947 | |
6948 | // If we have one element to load/store, return it. |
6949 | EVT RetVT = WidenEltVT; |
6950 | if (!Scalable && Width == WidenEltWidth) |
6951 | return RetVT; |
6952 | |
6953 | // Don't bother looking for an integer type if the vector is scalable, skip |
6954 | // to vector types. |
6955 | if (!Scalable) { |
6956 | // See if there is larger legal integer than the element type to load/store. |
6957 | for (EVT MemVT : reverse(MVT::integer_valuetypes())) { |
6958 | unsigned MemVTWidth = MemVT.getSizeInBits(); |
6959 | if (MemVT.getSizeInBits() <= WidenEltWidth) |
6960 | break; |
6961 | auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT); |
6962 | if ((Action == TargetLowering::TypeLegal || |
6963 | Action == TargetLowering::TypePromoteInteger) && |
6964 | (WidenWidth % MemVTWidth) == 0 && |
6965 | isPowerOf2_32(WidenWidth / MemVTWidth) && |
6966 | (MemVTWidth <= Width || |
6967 | (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { |
6968 | if (MemVTWidth == WidenWidth) |
6969 | return MemVT; |
6970 | RetVT = MemVT; |
6971 | break; |
6972 | } |
6973 | } |
6974 | } |
6975 | |
6976 | // See if there is a larger vector type to load/store that has the same vector |
6977 | // element type and is evenly divisible with the WidenVT. |
6978 | for (EVT MemVT : reverse(MVT::vector_valuetypes())) { |
6979 | // Skip vector MVTs which don't match the scalable property of WidenVT. |
6980 | if (Scalable != MemVT.isScalableVector()) |
6981 | continue; |
6982 | unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue(); |
6983 | auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT); |
6984 | if ((Action == TargetLowering::TypeLegal || |
6985 | Action == TargetLowering::TypePromoteInteger) && |
6986 | WidenEltVT == MemVT.getVectorElementType() && |
6987 | (WidenWidth % MemVTWidth) == 0 && |
6988 | isPowerOf2_32(WidenWidth / MemVTWidth) && |
6989 | (MemVTWidth <= Width || |
6990 | (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { |
6991 | if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT) |
6992 | return MemVT; |
6993 | } |
6994 | } |
6995 | |
6996 | // Using element-wise loads and stores for widening operations is not |
6997 | // supported for scalable vectors |
6998 | if (Scalable) |
6999 | return std::nullopt; |
7000 | |
7001 | return RetVT; |
7002 | } |
7003 | |
7004 | // Builds a vector type from scalar loads |
7005 | // VecTy: Resulting Vector type |
7006 | // LDOps: Load operators to build a vector type |
7007 | // [Start,End) the list of loads to use. |
7008 | static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, |
7009 | SmallVectorImpl<SDValue> &LdOps, |
7010 | unsigned Start, unsigned End) { |
7011 | SDLoc dl(LdOps[Start]); |
7012 | EVT LdTy = LdOps[Start].getValueType(); |
7013 | unsigned Width = VecTy.getSizeInBits(); |
7014 | unsigned NumElts = Width / LdTy.getSizeInBits(); |
7015 | EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts); |
7016 | |
7017 | unsigned Idx = 1; |
7018 | SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]); |
7019 | |
7020 | for (unsigned i = Start + 1; i != End; ++i) { |
7021 | EVT NewLdTy = LdOps[i].getValueType(); |
7022 | if (NewLdTy != LdTy) { |
7023 | NumElts = Width / NewLdTy.getSizeInBits(); |
7024 | NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts); |
7025 | VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp); |
7026 | // Readjust position and vector position based on new load type. |
7027 | Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); |
7028 | LdTy = NewLdTy; |
7029 | } |
7030 | VecOp = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: NewVecVT, N1: VecOp, N2: LdOps[i], |
7031 | N3: DAG.getVectorIdxConstant(Val: Idx++, DL: dl)); |
7032 | } |
7033 | return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp); |
7034 | } |
7035 | |
7036 | SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain, |
7037 | LoadSDNode *LD) { |
7038 | // The strategy assumes that we can efficiently load power-of-two widths. |
7039 | // The routine chops the vector into the largest vector loads with the same |
7040 | // element type or scalar loads and then recombines it to the widen vector |
7041 | // type. |
7042 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0)); |
7043 | EVT LdVT = LD->getMemoryVT(); |
7044 | SDLoc dl(LD); |
7045 | assert(LdVT.isVector() && WidenVT.isVector()); |
7046 | assert(LdVT.isScalableVector() == WidenVT.isScalableVector()); |
7047 | assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); |
7048 | |
7049 | // Load information |
7050 | SDValue Chain = LD->getChain(); |
7051 | SDValue BasePtr = LD->getBasePtr(); |
7052 | MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); |
7053 | AAMDNodes AAInfo = LD->getAAInfo(); |
7054 | |
7055 | TypeSize LdWidth = LdVT.getSizeInBits(); |
7056 | TypeSize WidenWidth = WidenVT.getSizeInBits(); |
7057 | TypeSize WidthDiff = WidenWidth - LdWidth; |
7058 | // Allow wider loads if they are sufficiently aligned to avoid memory faults |
7059 | // and if the original load is simple. |
7060 | unsigned LdAlign = |
7061 | (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value(); |
7062 | |
7063 | // Find the vector type that can load from. |
7064 | std::optional<EVT> FirstVT = |
7065 | findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign, |
7066 | WidenEx: WidthDiff.getKnownMinValue()); |
7067 | |
7068 | if (!FirstVT) |
7069 | return SDValue(); |
7070 | |
7071 | SmallVector<EVT, 8> MemVTs; |
7072 | TypeSize FirstVTWidth = FirstVT->getSizeInBits(); |
7073 | |
7074 | // Unless we're able to load in one instruction we must work out how to load |
7075 | // the remainder. |
7076 | if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) { |
7077 | std::optional<EVT> NewVT = FirstVT; |
7078 | TypeSize RemainingWidth = LdWidth; |
7079 | TypeSize NewVTWidth = FirstVTWidth; |
7080 | do { |
7081 | RemainingWidth -= NewVTWidth; |
7082 | if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) { |
7083 | // The current type we are using is too large. Find a better size. |
7084 | NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(), |
7085 | WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue()); |
7086 | if (!NewVT) |
7087 | return SDValue(); |
7088 | NewVTWidth = NewVT->getSizeInBits(); |
7089 | } |
7090 | MemVTs.push_back(Elt: *NewVT); |
7091 | } while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth)); |
7092 | } |
7093 | |
7094 | SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(), |
7095 | Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo); |
7096 | LdChain.push_back(Elt: LdOp.getValue(R: 1)); |
7097 | |
7098 | // Check if we can load the element with one instruction. |
7099 | if (MemVTs.empty()) { |
7100 | assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth)); |
7101 | if (!FirstVT->isVector()) { |
7102 | unsigned NumElts = |
7103 | WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue(); |
7104 | EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: *FirstVT, NumElements: NumElts); |
7105 | SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp); |
7106 | return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp); |
7107 | } |
7108 | if (FirstVT == WidenVT) |
7109 | return LdOp; |
7110 | |
7111 | // TODO: We don't currently have any tests that exercise this code path. |
7112 | assert(WidenWidth.getFixedValue() % FirstVTWidth.getFixedValue() == 0); |
7113 | unsigned NumConcat = |
7114 | WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue(); |
7115 | SmallVector<SDValue, 16> ConcatOps(NumConcat); |
7116 | SDValue UndefVal = DAG.getUNDEF(VT: *FirstVT); |
7117 | ConcatOps[0] = LdOp; |
7118 | for (unsigned i = 1; i != NumConcat; ++i) |
7119 | ConcatOps[i] = UndefVal; |
7120 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: ConcatOps); |
7121 | } |
7122 | |
7123 | // Load vector by using multiple loads from largest vector to scalar. |
7124 | SmallVector<SDValue, 16> LdOps; |
7125 | LdOps.push_back(Elt: LdOp); |
7126 | |
7127 | uint64_t ScaledOffset = 0; |
7128 | MachinePointerInfo MPI = LD->getPointerInfo(); |
7129 | |
7130 | // First incremement past the first load. |
7131 | IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr, |
7132 | ScaledOffset: &ScaledOffset); |
7133 | |
7134 | for (EVT MemVT : MemVTs) { |
7135 | Align NewAlign = ScaledOffset == 0 |
7136 | ? LD->getOriginalAlign() |
7137 | : commonAlignment(A: LD->getAlign(), Offset: ScaledOffset); |
7138 | SDValue L = |
7139 | DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags, AAInfo); |
7140 | |
7141 | LdOps.push_back(Elt: L); |
7142 | LdChain.push_back(Elt: L.getValue(R: 1)); |
7143 | IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset); |
7144 | } |
7145 | |
7146 | // Build the vector from the load operations. |
7147 | unsigned End = LdOps.size(); |
7148 | if (!LdOps[0].getValueType().isVector()) |
7149 | // All the loads are scalar loads. |
7150 | return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End); |
7151 | |
7152 | // If the load contains vectors, build the vector using concat vector. |
7153 | // All of the vectors used to load are power-of-2, and the scalar loads can be |
7154 | // combined to make a power-of-2 vector. |
7155 | SmallVector<SDValue, 16> ConcatOps(End); |
7156 | int i = End - 1; |
7157 | int Idx = End; |
7158 | EVT LdTy = LdOps[i].getValueType(); |
7159 | // First, combine the scalar loads to a vector. |
7160 | if (!LdTy.isVector()) { |
7161 | for (--i; i >= 0; --i) { |
7162 | LdTy = LdOps[i].getValueType(); |
7163 | if (LdTy.isVector()) |
7164 | break; |
7165 | } |
7166 | ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End); |
7167 | } |
7168 | |
7169 | ConcatOps[--Idx] = LdOps[i]; |
7170 | for (--i; i >= 0; --i) { |
7171 | EVT NewLdTy = LdOps[i].getValueType(); |
7172 | if (NewLdTy != LdTy) { |
7173 | // Create a larger vector. |
7174 | TypeSize LdTySize = LdTy.getSizeInBits(); |
7175 | TypeSize NewLdTySize = NewLdTy.getSizeInBits(); |
7176 | assert(NewLdTySize.isScalable() == LdTySize.isScalable() && |
7177 | NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue())); |
7178 | unsigned NumOps = |
7179 | NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue(); |
7180 | SmallVector<SDValue, 16> WidenOps(NumOps); |
7181 | unsigned j = 0; |
7182 | for (; j != End-Idx; ++j) |
7183 | WidenOps[j] = ConcatOps[Idx+j]; |
7184 | for (; j != NumOps; ++j) |
7185 | WidenOps[j] = DAG.getUNDEF(VT: LdTy); |
7186 | |
7187 | ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy, |
7188 | Ops: WidenOps); |
7189 | Idx = End - 1; |
7190 | LdTy = NewLdTy; |
7191 | } |
7192 | ConcatOps[--Idx] = LdOps[i]; |
7193 | } |
7194 | |
7195 | if (WidenWidth == LdTy.getSizeInBits() * (End - Idx)) |
7196 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, |
7197 | Ops: ArrayRef(&ConcatOps[Idx], End - Idx)); |
7198 | |
7199 | // We need to fill the rest with undefs to build the vector. |
7200 | unsigned NumOps = |
7201 | WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue(); |
7202 | SmallVector<SDValue, 16> WidenOps(NumOps); |
7203 | SDValue UndefVal = DAG.getUNDEF(VT: LdTy); |
7204 | { |
7205 | unsigned i = 0; |
7206 | for (; i != End-Idx; ++i) |
7207 | WidenOps[i] = ConcatOps[Idx+i]; |
7208 | for (; i != NumOps; ++i) |
7209 | WidenOps[i] = UndefVal; |
7210 | } |
7211 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps); |
7212 | } |
7213 | |
7214 | SDValue |
7215 | DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain, |
7216 | LoadSDNode *LD, |
7217 | ISD::LoadExtType ExtType) { |
7218 | // For extension loads, it may not be more efficient to chop up the vector |
7219 | // and then extend it. Instead, we unroll the load and build a new vector. |
7220 | EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0)); |
7221 | EVT LdVT = LD->getMemoryVT(); |
7222 | SDLoc dl(LD); |
7223 | assert(LdVT.isVector() && WidenVT.isVector()); |
7224 | assert(LdVT.isScalableVector() == WidenVT.isScalableVector()); |
7225 | |
7226 | // Load information |
7227 | SDValue Chain = LD->getChain(); |
7228 | SDValue BasePtr = LD->getBasePtr(); |
7229 | MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); |
7230 | AAMDNodes AAInfo = LD->getAAInfo(); |
7231 | |
7232 | if (LdVT.isScalableVector()) |
7233 | report_fatal_error(reason: "Generating widen scalable extending vector loads is " |
7234 | "not yet supported" ); |
7235 | |
7236 | EVT EltVT = WidenVT.getVectorElementType(); |
7237 | EVT LdEltVT = LdVT.getVectorElementType(); |
7238 | unsigned NumElts = LdVT.getVectorNumElements(); |
7239 | |
7240 | // Load each element and widen. |
7241 | unsigned WidenNumElts = WidenVT.getVectorNumElements(); |
7242 | SmallVector<SDValue, 16> Ops(WidenNumElts); |
7243 | unsigned Increment = LdEltVT.getSizeInBits() / 8; |
7244 | Ops[0] = |
7245 | DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(), |
7246 | MemVT: LdEltVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo); |
7247 | LdChain.push_back(Elt: Ops[0].getValue(R: 1)); |
7248 | unsigned i = 0, Offset = Increment; |
7249 | for (i=1; i < NumElts; ++i, Offset += Increment) { |
7250 | SDValue NewBasePtr = |
7251 | DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset)); |
7252 | Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr, |
7253 | PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT, |
7254 | Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo); |
7255 | LdChain.push_back(Elt: Ops[i].getValue(R: 1)); |
7256 | } |
7257 | |
7258 | // Fill the rest with undefs. |
7259 | SDValue UndefVal = DAG.getUNDEF(VT: EltVT); |
7260 | for (; i != WidenNumElts; ++i) |
7261 | Ops[i] = UndefVal; |
7262 | |
7263 | return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops); |
7264 | } |
7265 | |
7266 | bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, |
7267 | StoreSDNode *ST) { |
7268 | // The strategy assumes that we can efficiently store power-of-two widths. |
7269 | // The routine chops the vector into the largest vector stores with the same |
7270 | // element type or scalar stores. |
7271 | SDValue Chain = ST->getChain(); |
7272 | SDValue BasePtr = ST->getBasePtr(); |
7273 | MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); |
7274 | AAMDNodes AAInfo = ST->getAAInfo(); |
7275 | SDValue ValOp = GetWidenedVector(Op: ST->getValue()); |
7276 | SDLoc dl(ST); |
7277 | |
7278 | EVT StVT = ST->getMemoryVT(); |
7279 | TypeSize StWidth = StVT.getSizeInBits(); |
7280 | EVT ValVT = ValOp.getValueType(); |
7281 | TypeSize ValWidth = ValVT.getSizeInBits(); |
7282 | EVT ValEltVT = ValVT.getVectorElementType(); |
7283 | unsigned ValEltWidth = ValEltVT.getFixedSizeInBits(); |
7284 | assert(StVT.getVectorElementType() == ValEltVT); |
7285 | assert(StVT.isScalableVector() == ValVT.isScalableVector() && |
7286 | "Mismatch between store and value types" ); |
7287 | |
7288 | int Idx = 0; // current index to store |
7289 | |
7290 | MachinePointerInfo MPI = ST->getPointerInfo(); |
7291 | uint64_t ScaledOffset = 0; |
7292 | |
7293 | // A breakdown of how to widen this vector store. Each element of the vector |
7294 | // is a memory VT combined with the number of times it is to be stored to, |
7295 | // e,g., v5i32 -> {{v2i32,2},{i32,1}} |
7296 | SmallVector<std::pair<EVT, unsigned>, 4> MemVTs; |
7297 | |
7298 | while (StWidth.isNonZero()) { |
7299 | // Find the largest vector type we can store with. |
7300 | std::optional<EVT> NewVT = |
7301 | findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT); |
7302 | if (!NewVT) |
7303 | return false; |
7304 | MemVTs.push_back(Elt: {*NewVT, 0}); |
7305 | TypeSize NewVTWidth = NewVT->getSizeInBits(); |
7306 | |
7307 | do { |
7308 | StWidth -= NewVTWidth; |
7309 | MemVTs.back().second++; |
7310 | } while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth)); |
7311 | } |
7312 | |
7313 | for (const auto &Pair : MemVTs) { |
7314 | EVT NewVT = Pair.first; |
7315 | unsigned Count = Pair.second; |
7316 | TypeSize NewVTWidth = NewVT.getSizeInBits(); |
7317 | |
7318 | if (NewVT.isVector()) { |
7319 | unsigned NumVTElts = NewVT.getVectorMinNumElements(); |
7320 | do { |
7321 | Align NewAlign = ScaledOffset == 0 |
7322 | ? ST->getOriginalAlign() |
7323 | : commonAlignment(A: ST->getAlign(), Offset: ScaledOffset); |
7324 | SDValue EOp = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: NewVT, N1: ValOp, |
7325 | N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
7326 | SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, |
7327 | MMOFlags, AAInfo); |
7328 | StChain.push_back(Elt: PartStore); |
7329 | |
7330 | Idx += NumVTElts; |
7331 | IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr, |
7332 | ScaledOffset: &ScaledOffset); |
7333 | } while (--Count); |
7334 | } else { |
7335 | // Cast the vector to the scalar type we can store. |
7336 | unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue(); |
7337 | EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts); |
7338 | SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp); |
7339 | // Readjust index position based on new vector type. |
7340 | Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue(); |
7341 | do { |
7342 | SDValue EOp = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: NewVT, N1: VecOp, |
7343 | N2: DAG.getVectorIdxConstant(Val: Idx++, DL: dl)); |
7344 | SDValue PartStore = |
7345 | DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: ST->getOriginalAlign(), |
7346 | MMOFlags, AAInfo); |
7347 | StChain.push_back(Elt: PartStore); |
7348 | |
7349 | IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr); |
7350 | } while (--Count); |
7351 | // Restore index back to be relative to the original widen element type. |
7352 | Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth; |
7353 | } |
7354 | } |
7355 | |
7356 | return true; |
7357 | } |
7358 | |
7359 | /// Modifies a vector input (widen or narrows) to a vector of NVT. The |
7360 | /// input vector must have the same element type as NVT. |
7361 | /// FillWithZeroes specifies that the vector should be widened with zeroes. |
7362 | SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT, |
7363 | bool FillWithZeroes) { |
7364 | // Note that InOp might have been widened so it might already have |
7365 | // the right width or it might need be narrowed. |
7366 | EVT InVT = InOp.getValueType(); |
7367 | assert(InVT.getVectorElementType() == NVT.getVectorElementType() && |
7368 | "input and widen element type must match" ); |
7369 | assert(InVT.isScalableVector() == NVT.isScalableVector() && |
7370 | "cannot modify scalable vectors in this way" ); |
7371 | SDLoc dl(InOp); |
7372 | |
7373 | // Check if InOp already has the right width. |
7374 | if (InVT == NVT) |
7375 | return InOp; |
7376 | |
7377 | ElementCount InEC = InVT.getVectorElementCount(); |
7378 | ElementCount WidenEC = NVT.getVectorElementCount(); |
7379 | if (WidenEC.hasKnownScalarFactor(RHS: InEC)) { |
7380 | unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC); |
7381 | SmallVector<SDValue, 16> Ops(NumConcat); |
7382 | SDValue FillVal = FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) : |
7383 | DAG.getUNDEF(VT: InVT); |
7384 | Ops[0] = InOp; |
7385 | for (unsigned i = 1; i != NumConcat; ++i) |
7386 | Ops[i] = FillVal; |
7387 | |
7388 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops); |
7389 | } |
7390 | |
7391 | if (InEC.hasKnownScalarFactor(RHS: WidenEC)) |
7392 | return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: NVT, N1: InOp, |
7393 | N2: DAG.getVectorIdxConstant(Val: 0, DL: dl)); |
7394 | |
7395 | assert(!InVT.isScalableVector() && !NVT.isScalableVector() && |
7396 | "Scalable vectors should have been handled already." ); |
7397 | |
7398 | unsigned InNumElts = InEC.getFixedValue(); |
7399 | unsigned WidenNumElts = WidenEC.getFixedValue(); |
7400 | |
7401 | // Fall back to extract and build (+ mask, if padding with zeros). |
7402 | SmallVector<SDValue, 16> Ops(WidenNumElts); |
7403 | EVT EltVT = NVT.getVectorElementType(); |
7404 | unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts); |
7405 | unsigned Idx; |
7406 | for (Idx = 0; Idx < MinNumElts; ++Idx) |
7407 | Ops[Idx] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: InOp, |
7408 | N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
7409 | |
7410 | SDValue UndefVal = DAG.getUNDEF(VT: EltVT); |
7411 | for (; Idx < WidenNumElts; ++Idx) |
7412 | Ops[Idx] = UndefVal; |
7413 | |
7414 | SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops); |
7415 | if (!FillWithZeroes) |
7416 | return Widened; |
7417 | |
7418 | assert(NVT.isInteger() && |
7419 | "We expect to never want to FillWithZeroes for non-integral types." ); |
7420 | |
7421 | SmallVector<SDValue, 16> MaskOps; |
7422 | MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT)); |
7423 | MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT)); |
7424 | |
7425 | return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened, |
7426 | N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps)); |
7427 | } |
7428 | |