1	/* Internal functions.
2	Copyright (C) 2011-2025 Free Software Foundation, Inc.
3
4	This file is part of GCC.
5
6	GCC is free software; you can redistribute it and/or modify it under
7	the terms of the GNU General Public License as published by the Free
8	Software Foundation; either version 3, or (at your option) any later
9	version.
10
11	GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12	WARRANTY; without even the implied warranty of MERCHANTABILITY or
13	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14	for more details.
15
16	You should have received a copy of the GNU General Public License
17	along with GCC; see the file COPYING3. If not see
18	<http://www.gnu.org/licenses/>. */
19
20	#include "config.h"
21	#define INCLUDE_MEMORY
22	#include "system.h"
23	#include "coretypes.h"
24	#include "backend.h"
25	#include "target.h"
26	#include "rtl.h"
27	#include "tree.h"
28	#include "gimple.h"
29	#include "predict.h"
30	#include "stringpool.h"
31	#include "tree-vrp.h"
32	#include "tree-ssanames.h"
33	#include "expmed.h"
34	#include "memmodel.h"
35	#include "optabs.h"
36	#include "emit-rtl.h"
37	#include "diagnostic-core.h"
38	#include "fold-const.h"
39	#include "internal-fn.h"
40	#include "stor-layout.h"
41	#include "dojump.h"
42	#include "expr.h"
43	#include "stringpool.h"
44	#include "attribs.h"
45	#include "asan.h"
46	#include "ubsan.h"
47	#include "recog.h"
48	#include "builtins.h"
49	#include "optabs-tree.h"
50	#include "gimple-ssa.h"
51	#include "tree-phinodes.h"
52	#include "ssa-iterators.h"
53	#include "explow.h"
54	#include "rtl-iter.h"
55	#include "gimple-range.h"
56	#include "fold-const-call.h"
57	#include "tree-ssa-live.h"
58	#include "tree-outof-ssa.h"
59	#include "gcc-urlifier.h"
60
61	/* For lang_hooks.types.type_for_mode. */
62	#include "langhooks.h"
63
64	/* The names of each internal function, indexed by function number. */
65	const char *const internal_fn_name_array[] = {
66	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
67	#include "internal-fn.def"
68	"<invalid-fn>"
69	};
70
71	/* The ECF_* flags of each internal function, indexed by function number. */
72	const int internal_fn_flags_array[] = {
73	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
74	#include "internal-fn.def"
75	0
76	};
77
78	/* Return the internal function called NAME, or IFN_LAST if there's
79	no such function. */
80
81	internal_fn
82	lookup_internal_fn (const char *name)
83	{
84	typedef hash_map<nofree_string_hash, internal_fn> name_to_fn_map_type;
85	static name_to_fn_map_type *name_to_fn_map;
86
87	if (!name_to_fn_map)
88	{
89	name_to_fn_map = new name_to_fn_map_type (IFN_LAST);
90	for (unsigned int i = 0; i < IFN_LAST; ++i)
91	name_to_fn_map->put (k: internal_fn_name (fn: internal_fn (i)),
92	v: internal_fn (i));
93	}
94	internal_fn *entry = name_to_fn_map->get (k: name);
95	return entry ? *entry : IFN_LAST;
96	}
97
98	/* Geven an internal_fn IFN that is a widening function, return its
99	corresponding LO and HI internal_fns. */
100
101	extern void
102	lookup_hilo_internal_fn (internal_fn ifn, internal_fn *lo, internal_fn *hi)
103	{
104	gcc_assert (widening_fn_p (ifn));
105
106	switch (ifn)
107	{
108	default:
109	gcc_unreachable ();
110	#define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
111	#define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
112	case IFN_##NAME: \
113	*lo = internal_fn (IFN_##NAME##_LO); \
114	*hi = internal_fn (IFN_##NAME##_HI); \
115	break;
116	#include "internal-fn.def"
117	}
118	}
119
120	/* Given an internal_fn IFN that is a widening function, return its
121	corresponding _EVEN and _ODD internal_fns in *EVEN and *ODD. */
122
123	extern void
124	lookup_evenodd_internal_fn (internal_fn ifn, internal_fn *even,
125	internal_fn *odd)
126	{
127	gcc_assert (widening_fn_p (ifn));
128
129	switch (ifn)
130	{
131	default:
132	gcc_unreachable ();
133	#define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
134	#define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
135	case IFN_##NAME: \
136	*even = internal_fn (IFN_##NAME##_EVEN); \
137	*odd = internal_fn (IFN_##NAME##_ODD); \
138	break;
139	#include "internal-fn.def"
140	}
141	}
142
143
144	/* Fnspec of each internal function, indexed by function number. */
145	const_tree internal_fn_fnspec_array[IFN_LAST + 1];
146
147	void
148	init_internal_fns ()
149	{
150	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
151	if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
152	build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
153	#include "internal-fn.def"
154	internal_fn_fnspec_array[IFN_LAST] = 0;
155	}
156
157	/* Create static initializers for the information returned by
158	direct_internal_fn. */
159	#define not_direct { -2, -2, false }
160	#define mask_load_direct { -1, 2, false }
161	#define load_lanes_direct { -1, -1, false }
162	#define mask_load_lanes_direct { -1, -1, false }
163	#define gather_load_direct { 3, 1, false }
164	#define strided_load_direct { -1, -1, false }
165	#define len_load_direct { -1, -1, false }
166	#define mask_len_load_direct { -1, 4, false }
167	#define mask_store_direct { 3, 2, false }
168	#define store_lanes_direct { 0, 0, false }
169	#define mask_store_lanes_direct { 0, 0, false }
170	#define vec_cond_mask_direct { 1, 0, false }
171	#define vec_cond_mask_len_direct { 1, 1, false }
172	#define vec_cond_direct { 2, 0, false }
173	#define scatter_store_direct { 3, 1, false }
174	#define strided_store_direct { 1, 1, false }
175	#define len_store_direct { 3, 3, false }
176	#define mask_len_store_direct { 4, 5, false }
177	#define vec_set_direct { 3, 3, false }
178	#define vec_extract_direct { 0, -1, false }
179	#define unary_direct { 0, 0, true }
180	#define unary_convert_direct { -1, 0, true }
181	#define binary_direct { 0, 0, true }
182	#define ternary_direct { 0, 0, true }
183	#define cond_unary_direct { 1, 1, true }
184	#define cond_binary_direct { 1, 1, true }
185	#define cond_ternary_direct { 1, 1, true }
186	#define cond_len_unary_direct { 1, 1, true }
187	#define cond_len_binary_direct { 1, 1, true }
188	#define cond_len_ternary_direct { 1, 1, true }
189	#define while_direct { 0, 2, false }
190	#define fold_extract_direct { 2, 2, false }
191	#define fold_len_extract_direct { 2, 2, false }
192	#define fold_left_direct { 1, 1, false }
193	#define mask_fold_left_direct { 1, 1, false }
194	#define mask_len_fold_left_direct { 1, 1, false }
195	#define check_ptrs_direct { 0, 0, false }
196	#define crc_direct { 1, -1, true }
197
198	const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
199	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
200	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
201	#define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
202	UNSIGNED_OPTAB, TYPE) TYPE##_direct,
203	#include "internal-fn.def"
204	not_direct
205	};
206
207	/* Like create_output_operand, but for callers that will use
208	assign_call_lhs afterwards. */
209
210	static void
211	create_call_lhs_operand (expand_operand *op, rtx lhs_rtx, machine_mode mode)
212	{
213	/* Do not assign directly to a promoted subreg, since there is no
214	guarantee that the instruction will leave the upper bits of the
215	register in the state required by SUBREG_PROMOTED_SIGN. */
216	rtx dest = lhs_rtx;
217	if (dest && GET_CODE (dest) == SUBREG && SUBREG_PROMOTED_VAR_P (dest))
218	dest = NULL_RTX;
219	create_output_operand (op, x: dest, mode);
220	}
221
222	/* Move the result of an expanded instruction into the lhs of a gimple call.
223	LHS is the lhs of the call, LHS_RTX is its expanded form, and OP is the
224	result of the expanded instruction. OP should have been set up by
225	create_call_lhs_operand. */
226
227	static void
228	assign_call_lhs (tree lhs, rtx lhs_rtx, expand_operand *op)
229	{
230	if (rtx_equal_p (lhs_rtx, op->value))
231	return;
232
233	/* If the return value has an integral type, convert the instruction
234	result to that type. This is useful for things that return an
235	int regardless of the size of the input. If the instruction result
236	is smaller than required, assume that it is signed.
237
238	If the return value has a nonintegral type, its mode must match
239	the instruction result. */
240	if (GET_CODE (lhs_rtx) == SUBREG && SUBREG_PROMOTED_VAR_P (lhs_rtx))
241	{
242	/* If this is a scalar in a register that is stored in a wider
243	mode than the declared mode, compute the result into its
244	declared mode and then convert to the wider mode. */
245	gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
246	rtx tmp = convert_to_mode (GET_MODE (lhs_rtx), op->value, 0);
247	convert_move (SUBREG_REG (lhs_rtx), tmp,
248	SUBREG_PROMOTED_SIGN (lhs_rtx));
249	}
250	else if (GET_MODE (lhs_rtx) == GET_MODE (op->value))
251	emit_move_insn (lhs_rtx, op->value);
252	else
253	{
254	gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
255	convert_move (lhs_rtx, op->value, 0);
256	}
257	}
258
259	/* Expand STMT using instruction ICODE. The instruction has NOUTPUTS
260	output operands and NINPUTS input operands, where NOUTPUTS is either
261	0 or 1. The output operand (if any) comes first, followed by the
262	NINPUTS input operands. */
263
264	static void
265	expand_fn_using_insn (gcall *stmt, insn_code icode, unsigned int noutputs,
266	unsigned int ninputs)
267	{
268	gcc_assert (icode != CODE_FOR_nothing);
269
270	expand_operand *ops = XALLOCAVEC (expand_operand, noutputs + ninputs);
271	unsigned int opno = 0;
272	rtx lhs_rtx = NULL_RTX;
273	tree lhs = gimple_call_lhs (gs: stmt);
274
275	if (noutputs)
276	{
277	gcc_assert (noutputs == 1);
278	if (lhs)
279	lhs_rtx = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
280	create_call_lhs_operand (op: &ops[opno], lhs_rtx,
281	mode: insn_data[icode].operand[opno].mode);
282	opno += 1;
283	}
284	else
285	gcc_assert (!lhs);
286
287	for (unsigned int i = 0; i < ninputs; ++i)
288	{
289	tree rhs = gimple_call_arg (gs: stmt, index: i);
290	tree rhs_type = TREE_TYPE (rhs);
291	rtx rhs_rtx = expand_normal (exp: rhs);
292	if (INTEGRAL_TYPE_P (rhs_type))
293	create_convert_operand_from (op: &ops[opno], value: rhs_rtx,
294	TYPE_MODE (rhs_type),
295	TYPE_UNSIGNED (rhs_type));
296	else if (TREE_CODE (rhs) == SSA_NAME
297	&& SSA_NAME_IS_DEFAULT_DEF (rhs)
298	&& VAR_P (SSA_NAME_VAR (rhs)))
299	create_undefined_input_operand (op: &ops[opno], TYPE_MODE (rhs_type));
300	else if (VECTOR_BOOLEAN_TYPE_P (rhs_type)
301	&& SCALAR_INT_MODE_P (TYPE_MODE (rhs_type))
302	&& maybe_ne (a: GET_MODE_PRECISION (TYPE_MODE (rhs_type)),
303	b: TYPE_VECTOR_SUBPARTS (node: rhs_type).to_constant ()))
304	{
305	/* Ensure that the vector bitmasks do not have excess bits. */
306	int nunits = TYPE_VECTOR_SUBPARTS (node: rhs_type).to_constant ();
307	rtx tmp = expand_binop (TYPE_MODE (rhs_type), and_optab, rhs_rtx,
308	GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
309	NULL_RTX, true, OPTAB_WIDEN);
310	create_input_operand (op: &ops[opno], value: tmp, TYPE_MODE (rhs_type));
311	}
312	else
313	create_input_operand (op: &ops[opno], value: rhs_rtx, TYPE_MODE (rhs_type));
314	opno += 1;
315	}
316
317	gcc_assert (opno == noutputs + ninputs);
318	expand_insn (icode, nops: opno, ops);
319	if (lhs_rtx)
320	assign_call_lhs (lhs, lhs_rtx, op: &ops[0]);
321	}
322
323	/* ARRAY_TYPE is an array of vector modes. Return the associated insn
324	for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
325
326	static enum insn_code
327	get_multi_vector_move (tree array_type, convert_optab optab)
328	{
329	machine_mode imode;
330	machine_mode vmode;
331
332	gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
333	imode = TYPE_MODE (array_type);
334	vmode = TYPE_MODE (TREE_TYPE (array_type));
335
336	return convert_optab_handler (op: optab, to_mode: imode, from_mode: vmode);
337	}
338
339	/* Add mask, else, and len arguments according to the STMT. */
340
341	static unsigned int
342	add_mask_else_and_len_args (expand_operand *ops, unsigned int opno, gcall *stmt)
343	{
344	internal_fn ifn = gimple_call_internal_fn (gs: stmt);
345	int len_index = internal_fn_len_index (ifn);
346	/* BIAS is always consecutive next of LEN. */
347	int bias_index = len_index + 1;
348	int mask_index = internal_fn_mask_index (ifn);
349
350	/* The order of arguments is always {mask, else, len, bias}. */
351	if (mask_index >= 0)
352	{
353	tree mask = gimple_call_arg (gs: stmt, index: mask_index);
354	rtx mask_rtx = expand_normal (exp: mask);
355
356	tree mask_type = TREE_TYPE (mask);
357	if (VECTOR_BOOLEAN_TYPE_P (mask_type)
358	&& SCALAR_INT_MODE_P (TYPE_MODE (mask_type))
359	&& maybe_ne (a: GET_MODE_PRECISION (TYPE_MODE (mask_type)),
360	b: TYPE_VECTOR_SUBPARTS (node: mask_type).to_constant ()))
361	{
362	/* Ensure that the vector bitmasks do not have excess bits. */
363	int nunits = TYPE_VECTOR_SUBPARTS (node: mask_type).to_constant ();
364	mask_rtx = expand_binop (TYPE_MODE (mask_type), and_optab, mask_rtx,
365	GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
366	NULL_RTX, true, OPTAB_WIDEN);
367	}
368
369	create_input_operand (op: &ops[opno++], value: mask_rtx,
370	TYPE_MODE (TREE_TYPE (mask)));
371	}
372
373	int els_index = internal_fn_else_index (ifn);
374	if (els_index >= 0)
375	{
376	tree els = gimple_call_arg (gs: stmt, index: els_index);
377	tree els_type = TREE_TYPE (els);
378	if (TREE_CODE (els) == SSA_NAME
379	&& SSA_NAME_IS_DEFAULT_DEF (els)
380	&& VAR_P (SSA_NAME_VAR (els)))
381	create_undefined_input_operand (op: &ops[opno++], TYPE_MODE (els_type));
382	else
383	{
384	rtx els_rtx = expand_normal (exp: els);
385	create_input_operand (op: &ops[opno++], value: els_rtx, TYPE_MODE (els_type));
386	}
387	}
388	if (len_index >= 0)
389	{
390	tree len = gimple_call_arg (gs: stmt, index: len_index);
391	rtx len_rtx = expand_normal (exp: len);
392	create_convert_operand_from (op: &ops[opno++], value: len_rtx,
393	TYPE_MODE (TREE_TYPE (len)),
394	TYPE_UNSIGNED (TREE_TYPE (len)));
395	tree biast = gimple_call_arg (gs: stmt, index: bias_index);
396	rtx bias = expand_normal (exp: biast);
397	create_input_operand (op: &ops[opno++], value: bias, QImode);
398	}
399	return opno;
400	}
401
402	/* Expand LOAD_LANES call STMT using optab OPTAB. */
403
404	static void
405	expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
406	{
407	class expand_operand ops[2];
408	tree type, lhs, rhs;
409	rtx target, mem;
410
411	lhs = gimple_call_lhs (gs: stmt);
412	rhs = gimple_call_arg (gs: stmt, index: 0);
413	type = TREE_TYPE (lhs);
414
415	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
416	mem = expand_normal (exp: rhs);
417
418	gcc_assert (MEM_P (mem));
419	PUT_MODE (x: mem, TYPE_MODE (type));
420
421	create_call_lhs_operand (op: &ops[0], lhs_rtx: target, TYPE_MODE (type));
422	create_fixed_operand (op: &ops[1], x: mem);
423	expand_insn (icode: get_multi_vector_move (array_type: type, optab), nops: 2, ops);
424	assign_call_lhs (lhs, lhs_rtx: target, op: &ops[0]);
425	}
426
427	/* Expand STORE_LANES call STMT using optab OPTAB. */
428
429	static void
430	expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
431	{
432	class expand_operand ops[2];
433	tree type, lhs, rhs;
434	rtx target, reg;
435
436	lhs = gimple_call_lhs (gs: stmt);
437	rhs = gimple_call_arg (gs: stmt, index: 0);
438	type = TREE_TYPE (rhs);
439
440	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
441	reg = expand_normal (exp: rhs);
442
443	gcc_assert (MEM_P (target));
444	PUT_MODE (x: target, TYPE_MODE (type));
445
446	create_fixed_operand (op: &ops[0], x: target);
447	create_input_operand (op: &ops[1], value: reg, TYPE_MODE (type));
448	expand_insn (icode: get_multi_vector_move (array_type: type, optab), nops: 2, ops);
449	}
450
451	static void
452	expand_ANNOTATE (internal_fn, gcall *)
453	{
454	gcc_unreachable ();
455	}
456
457	/* This should get expanded in omp_device_lower pass. */
458
459	static void
460	expand_GOMP_USE_SIMT (internal_fn, gcall *)
461	{
462	gcc_unreachable ();
463	}
464
465	/* This should get expanded in omp_device_lower pass. */
466
467	static void
468	expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
469	{
470	gcc_unreachable ();
471	}
472
473	/* Allocate per-lane storage and begin non-uniform execution region. */
474
475	static void
476	expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
477	{
478	rtx target;
479	tree lhs = gimple_call_lhs (gs: stmt);
480	if (lhs)
481	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
482	else
483	target = gen_reg_rtx (Pmode);
484	rtx size = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
485	rtx align = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
486	class expand_operand ops[3];
487	create_call_lhs_operand (op: &ops[0], lhs_rtx: target, Pmode);
488	create_input_operand (op: &ops[1], value: size, Pmode);
489	create_input_operand (op: &ops[2], value: align, Pmode);
490	gcc_assert (targetm.have_omp_simt_enter ());
491	expand_insn (icode: targetm.code_for_omp_simt_enter, nops: 3, ops);
492	assign_call_lhs (lhs, lhs_rtx: target, op: &ops[0]);
493	}
494
495	/* Deallocate per-lane storage and leave non-uniform execution region. */
496
497	static void
498	expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
499	{
500	gcc_checking_assert (!gimple_call_lhs (stmt));
501	rtx arg = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
502	class expand_operand ops[1];
503	create_input_operand (op: &ops[0], value: arg, Pmode);
504	gcc_assert (targetm.have_omp_simt_exit ());
505	expand_insn (icode: targetm.code_for_omp_simt_exit, nops: 1, ops);
506	}
507
508	/* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
509	without SIMT execution this should be expanded in omp_device_lower pass. */
510
511	static void
512	expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
513	{
514	tree lhs = gimple_call_lhs (gs: stmt);
515	if (!lhs)
516	return;
517
518	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
519	gcc_assert (targetm.have_omp_simt_lane ());
520	emit_insn (targetm.gen_omp_simt_lane (target));
521	}
522
523	/* This should get expanded in omp_device_lower pass. */
524
525	static void
526	expand_GOMP_SIMT_VF (internal_fn, gcall *)
527	{
528	gcc_unreachable ();
529	}
530
531	/* This should get expanded in omp_device_lower pass. */
532
533	static void
534	expand_GOMP_MAX_VF (internal_fn, gcall *)
535	{
536	gcc_unreachable ();
537	}
538
539	/* This should get expanded in omp_device_lower pass. */
540
541	static void
542	expand_GOMP_TARGET_REV (internal_fn, gcall *)
543	{
544	gcc_unreachable ();
545	}
546
547	/* Lane index of the first SIMT lane that supplies a non-zero argument.
548	This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
549	lane that executed the last iteration for handling OpenMP lastprivate. */
550
551	static void
552	expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
553	{
554	tree lhs = gimple_call_lhs (gs: stmt);
555	if (!lhs)
556	return;
557
558	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
559	rtx cond = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
560	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
561	class expand_operand ops[2];
562	create_call_lhs_operand (op: &ops[0], lhs_rtx: target, mode);
563	create_input_operand (op: &ops[1], value: cond, mode);
564	gcc_assert (targetm.have_omp_simt_last_lane ());
565	expand_insn (icode: targetm.code_for_omp_simt_last_lane, nops: 2, ops);
566	assign_call_lhs (lhs, lhs_rtx: target, op: &ops[0]);
567	}
568
569	/* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
570
571	static void
572	expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
573	{
574	tree lhs = gimple_call_lhs (gs: stmt);
575	if (!lhs)
576	return;
577
578	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
579	rtx ctr = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
580	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
581	class expand_operand ops[2];
582	create_call_lhs_operand (op: &ops[0], lhs_rtx: target, mode);
583	create_input_operand (op: &ops[1], value: ctr, mode);
584	gcc_assert (targetm.have_omp_simt_ordered ());
585	expand_insn (icode: targetm.code_for_omp_simt_ordered, nops: 2, ops);
586	assign_call_lhs (lhs, lhs_rtx: target, op: &ops[0]);
587	}
588
589	/* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
590	any lane supplies a non-zero argument. */
591
592	static void
593	expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
594	{
595	tree lhs = gimple_call_lhs (gs: stmt);
596	if (!lhs)
597	return;
598
599	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
600	rtx cond = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
601	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
602	class expand_operand ops[2];
603	create_call_lhs_operand (op: &ops[0], lhs_rtx: target, mode);
604	create_input_operand (op: &ops[1], value: cond, mode);
605	gcc_assert (targetm.have_omp_simt_vote_any ());
606	expand_insn (icode: targetm.code_for_omp_simt_vote_any, nops: 2, ops);
607	assign_call_lhs (lhs, lhs_rtx: target, op: &ops[0]);
608	}
609
610	/* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
611	is destination lane index XOR given offset. */
612
613	static void
614	expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
615	{
616	tree lhs = gimple_call_lhs (gs: stmt);
617	if (!lhs)
618	return;
619
620	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
621	rtx src = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
622	rtx idx = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
623	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
624	class expand_operand ops[3];
625	create_call_lhs_operand (op: &ops[0], lhs_rtx: target, mode);
626	create_input_operand (op: &ops[1], value: src, mode);
627	create_input_operand (op: &ops[2], value: idx, SImode);
628	gcc_assert (targetm.have_omp_simt_xchg_bfly ());
629	expand_insn (icode: targetm.code_for_omp_simt_xchg_bfly, nops: 3, ops);
630	assign_call_lhs (lhs, lhs_rtx: target, op: &ops[0]);
631	}
632
633	/* Exchange between SIMT lanes according to given source lane index. */
634
635	static void
636	expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
637	{
638	tree lhs = gimple_call_lhs (gs: stmt);
639	if (!lhs)
640	return;
641
642	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
643	rtx src = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
644	rtx idx = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
645	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
646	class expand_operand ops[3];
647	create_call_lhs_operand (op: &ops[0], lhs_rtx: target, mode);
648	create_input_operand (op: &ops[1], value: src, mode);
649	create_input_operand (op: &ops[2], value: idx, SImode);
650	gcc_assert (targetm.have_omp_simt_xchg_idx ());
651	expand_insn (icode: targetm.code_for_omp_simt_xchg_idx, nops: 3, ops);
652	assign_call_lhs (lhs, lhs_rtx: target, op: &ops[0]);
653	}
654
655	/* This should get expanded in adjust_simduid_builtins. */
656
657	static void
658	expand_GOMP_SIMD_LANE (internal_fn, gcall *)
659	{
660	gcc_unreachable ();
661	}
662
663	/* This should get expanded in adjust_simduid_builtins. */
664
665	static void
666	expand_GOMP_SIMD_VF (internal_fn, gcall *)
667	{
668	gcc_unreachable ();
669	}
670
671	/* This should get expanded in adjust_simduid_builtins. */
672
673	static void
674	expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
675	{
676	gcc_unreachable ();
677	}
678
679	/* This should get expanded in adjust_simduid_builtins. */
680
681	static void
682	expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
683	{
684	gcc_unreachable ();
685	}
686
687	/* This should get expanded in adjust_simduid_builtins. */
688
689	static void
690	expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
691	{
692	gcc_unreachable ();
693	}
694
695	/* This should get expanded in gimplify_omp_dispatch. */
696
697	static void
698	expand_GOMP_DISPATCH (internal_fn, gcall *)
699	{
700	gcc_unreachable ();
701	}
702
703	/* This should get expanded in the sanopt pass. */
704
705	static void
706	expand_UBSAN_NULL (internal_fn, gcall *)
707	{
708	gcc_unreachable ();
709	}
710
711	/* This should get expanded in the sanopt pass. */
712
713	static void
714	expand_UBSAN_BOUNDS (internal_fn, gcall *)
715	{
716	gcc_unreachable ();
717	}
718
719	/* This should get expanded in the sanopt pass. */
720
721	static void
722	expand_UBSAN_VPTR (internal_fn, gcall *)
723	{
724	gcc_unreachable ();
725	}
726
727	/* This should get expanded in the sanopt pass. */
728
729	static void
730	expand_UBSAN_PTR (internal_fn, gcall *)
731	{
732	gcc_unreachable ();
733	}
734
735	/* This should get expanded in the sanopt pass. */
736
737	static void
738	expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
739	{
740	gcc_unreachable ();
741	}
742
743	/* This should get expanded in the sanopt pass. */
744
745	static void
746	expand_HWASAN_CHECK (internal_fn, gcall *)
747	{
748	gcc_unreachable ();
749	}
750
751	/* For hwasan stack tagging:
752	Clear tags on the dynamically allocated space.
753	For use after an object dynamically allocated on the stack goes out of
754	scope. */
755	static void
756	expand_HWASAN_ALLOCA_UNPOISON (internal_fn, gcall *gc)
757	{
758	gcc_assert (Pmode == ptr_mode);
759	tree restored_position = gimple_call_arg (gs: gc, index: 0);
760	rtx restored_rtx = expand_expr (exp: restored_position, NULL_RTX, VOIDmode,
761	modifier: EXPAND_NORMAL);
762	rtx func = init_one_libfunc ("__hwasan_tag_memory");
763	rtx off = expand_simple_binop (Pmode, MINUS, restored_rtx,
764	stack_pointer_rtx, NULL_RTX, 0,
765	OPTAB_WIDEN);
766	emit_library_call_value (fun: func, NULL_RTX, fn_type: LCT_NORMAL, VOIDmode,
767	virtual_stack_dynamic_rtx, Pmode,
768	HWASAN_STACK_BACKGROUND, QImode,
769	arg3: off, Pmode);
770	}
771
772	/* For hwasan stack tagging:
773	Return a tag to be used for a dynamic allocation. */
774	static void
775	expand_HWASAN_CHOOSE_TAG (internal_fn, gcall *gc)
776	{
777	tree tag = gimple_call_lhs (gs: gc);
778	rtx target = expand_expr (exp: tag, NULL_RTX, VOIDmode, modifier: EXPAND_NORMAL);
779	machine_mode mode = GET_MODE (target);
780	gcc_assert (mode == QImode);
781
782	rtx base_tag = targetm.memtag.extract_tag (hwasan_frame_base (), NULL_RTX);
783	gcc_assert (base_tag);
784	rtx tag_offset = gen_int_mode (hwasan_current_frame_tag (), QImode);
785	rtx chosen_tag = expand_simple_binop (QImode, PLUS, base_tag, tag_offset,
786	target, /* unsignedp = */1,
787	OPTAB_WIDEN);
788	chosen_tag = hwasan_truncate_to_tag_size (chosen_tag, target);
789
790	/* Really need to put the tag into the `target` RTX. */
791	if (chosen_tag != target)
792	{
793	rtx temp = chosen_tag;
794	gcc_assert (GET_MODE (chosen_tag) == mode);
795	emit_move_insn (target, temp);
796	}
797
798	hwasan_increment_frame_tag ();
799	}
800
801	/* For hwasan stack tagging:
802	Tag a region of space in the shadow stack according to the base pointer of
803	an object on the stack. N.b. the length provided in the internal call is
804	required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
805	static void
806	expand_HWASAN_MARK (internal_fn, gcall *gc)
807	{
808	gcc_assert (ptr_mode == Pmode);
809	HOST_WIDE_INT flag = tree_to_shwi (gimple_call_arg (gs: gc, index: 0));
810	bool is_poison = ((asan_mark_flags)flag) == ASAN_MARK_POISON;
811
812	tree base = gimple_call_arg (gs: gc, index: 1);
813	gcc_checking_assert (TREE_CODE (base) == ADDR_EXPR);
814	rtx base_rtx = expand_normal (exp: base);
815
816	rtx tag = is_poison ? HWASAN_STACK_BACKGROUND
817	: targetm.memtag.extract_tag (base_rtx, NULL_RTX);
818	rtx address = targetm.memtag.untagged_pointer (base_rtx, NULL_RTX);
819
820	tree len = gimple_call_arg (gs: gc, index: 2);
821	rtx r_len = expand_normal (exp: len);
822
823	rtx func = init_one_libfunc ("__hwasan_tag_memory");
824	emit_library_call (fun: func, fn_type: LCT_NORMAL, VOIDmode, arg1: address, Pmode,
825	arg2: tag, QImode, arg3: r_len, Pmode);
826	}
827
828	/* For hwasan stack tagging:
829	Store a tag into a pointer. */
830	static void
831	expand_HWASAN_SET_TAG (internal_fn, gcall *gc)
832	{
833	gcc_assert (ptr_mode == Pmode);
834	tree g_target = gimple_call_lhs (gs: gc);
835	tree g_ptr = gimple_call_arg (gs: gc, index: 0);
836	tree g_tag = gimple_call_arg (gs: gc, index: 1);
837
838	rtx ptr = expand_normal (exp: g_ptr);
839	rtx tag = expand_expr (exp: g_tag, NULL_RTX, QImode, modifier: EXPAND_NORMAL);
840	rtx target = expand_normal (exp: g_target);
841
842	rtx untagged = targetm.memtag.untagged_pointer (ptr, target);
843	rtx tagged_value = targetm.memtag.set_tag (untagged, tag, target);
844	if (tagged_value != target)
845	emit_move_insn (target, tagged_value);
846	}
847
848	/* This should get expanded in the sanopt pass. */
849
850	static void
851	expand_ASAN_CHECK (internal_fn, gcall *)
852	{
853	gcc_unreachable ();
854	}
855
856	/* This should get expanded in the sanopt pass. */
857
858	static void
859	expand_ASAN_MARK (internal_fn, gcall *)
860	{
861	gcc_unreachable ();
862	}
863
864	/* This should get expanded in the sanopt pass. */
865
866	static void
867	expand_ASAN_POISON (internal_fn, gcall *)
868	{
869	gcc_unreachable ();
870	}
871
872	/* This should get expanded in the sanopt pass. */
873
874	static void
875	expand_ASAN_POISON_USE (internal_fn, gcall *)
876	{
877	gcc_unreachable ();
878	}
879
880	/* This should get expanded in the tsan pass. */
881
882	static void
883	expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
884	{
885	gcc_unreachable ();
886	}
887
888	/* This should get expanded in the lower pass. */
889
890	static void
891	expand_FALLTHROUGH (internal_fn, gcall *call)
892	{
893	auto_urlify_attributes sentinel;
894	error_at (gimple_location (g: call),
895	"invalid use of attribute %<fallthrough%>");
896	}
897
898	/* Return minimum precision needed to represent all values
899	of ARG in SIGNed integral type. */
900
901	static int
902	get_min_precision (tree arg, signop sign)
903	{
904	int prec = TYPE_PRECISION (TREE_TYPE (arg));
905	int cnt = 0;
906	signop orig_sign = sign;
907	if (TREE_CODE (arg) == INTEGER_CST)
908	{
909	int p;
910	if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
911	{
912	widest_int w = wi::to_widest (t: arg);
913	w = wi::ext (x: w, offset: prec, sgn: sign);
914	p = wi::min_precision (x: w, sgn: sign);
915	}
916	else
917	p = wi::min_precision (x: wi::to_wide (t: arg), sgn: sign);
918	return MIN (p, prec);
919	}
920	while (CONVERT_EXPR_P (arg)
921	&& INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
922	&& TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
923	{
924	arg = TREE_OPERAND (arg, 0);
925	if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
926	{
927	if (TYPE_UNSIGNED (TREE_TYPE (arg)))
928	sign = UNSIGNED;
929	else if (sign == UNSIGNED
930	&& (get_range_pos_neg (arg,
931	currently_expanding_gimple_stmt)
932	!= 1))
933	return prec + (orig_sign != sign);
934	prec = TYPE_PRECISION (TREE_TYPE (arg));
935	}
936	if (++cnt > 30)
937	return prec + (orig_sign != sign);
938	}
939	if (CONVERT_EXPR_P (arg)
940	&& INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
941	&& TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) > prec)
942	{
943	/* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
944	If y_2's min precision is smaller than prec, return that. */
945	int oprec = get_min_precision (TREE_OPERAND (arg, 0), sign);
946	if (oprec < prec)
947	return oprec + (orig_sign != sign);
948	}
949	if (TREE_CODE (arg) != SSA_NAME)
950	return prec + (orig_sign != sign);
951	int_range_max r;
952	gimple *cg = currently_expanding_gimple_stmt;
953	while (!get_range_query (cfun)->range_of_expr (r, expr: arg, cg)
954	|| r.varying_p ()
955	|| r.undefined_p ())
956	{
957	gimple *g = SSA_NAME_DEF_STMT (arg);
958	if (is_gimple_assign (gs: g)
959	&& CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
960	{
961	tree t = gimple_assign_rhs1 (gs: g);
962	if (INTEGRAL_TYPE_P (TREE_TYPE (t))
963	&& TYPE_PRECISION (TREE_TYPE (t)) <= prec)
964	{
965	arg = t;
966	if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
967	{
968	if (TYPE_UNSIGNED (TREE_TYPE (arg)))
969	sign = UNSIGNED;
970	else if (sign == UNSIGNED
971	&& get_range_pos_neg (arg, g) != 1)
972	return prec + (orig_sign != sign);
973	prec = TYPE_PRECISION (TREE_TYPE (arg));
974	}
975	if (++cnt > 30)
976	return prec + (orig_sign != sign);
977	continue;
978	}
979	}
980	return prec + (orig_sign != sign);
981	}
982	if (sign == TYPE_SIGN (TREE_TYPE (arg)))
983	{
984	int p1 = wi::min_precision (x: r.lower_bound (), sgn: sign);
985	int p2 = wi::min_precision (x: r.upper_bound (), sgn: sign);
986	p1 = MAX (p1, p2);
987	prec = MIN (prec, p1);
988	}
989	else if (sign == UNSIGNED && !wi::neg_p (x: r.lower_bound (), sgn: SIGNED))
990	{
991	int p = wi::min_precision (x: r.upper_bound (), sgn: UNSIGNED);
992	prec = MIN (prec, p);
993	}
994	return prec + (orig_sign != sign);
995	}
996
997	/* Helper for expand_*_overflow. Set the __imag__ part to true
998	(1 except for signed:1 type, in which case store -1). */
999
1000	static void
1001	expand_arith_set_overflow (tree lhs, rtx target)
1002	{
1003	if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs))) == 1
1004	&& !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))))
1005	write_complex_part (target, constm1_rtx, true, false);
1006	else
1007	write_complex_part (target, const1_rtx, true, false);
1008	}
1009
1010	/* Helper for expand_*_overflow. Store RES into the __real__ part
1011	of TARGET. If RES has larger MODE than __real__ part of TARGET,
1012	set the __imag__ part to 1 if RES doesn't fit into it. Similarly
1013	if LHS has smaller precision than its mode. */
1014
1015	static void
1016	expand_arith_overflow_result_store (tree lhs, rtx target,
1017	scalar_int_mode mode, rtx res)
1018	{
1019	scalar_int_mode tgtmode
1020	= as_a <scalar_int_mode> (GET_MODE_INNER (GET_MODE (target)));
1021	rtx lres = res;
1022	if (tgtmode != mode)
1023	{
1024	rtx_code_label *done_label = gen_label_rtx ();
1025	int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
1026	lres = convert_modes (mode: tgtmode, oldmode: mode, x: res, unsignedp: uns);
1027	gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
1028	do_compare_rtx_and_jump (res, convert_modes (mode, oldmode: tgtmode, x: lres, unsignedp: uns),
1029	EQ, true, mode, NULL_RTX, NULL, done_label,
1030	profile_probability::very_likely ());
1031	expand_arith_set_overflow (lhs, target);
1032	emit_label (done_label);
1033	}
1034	int prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
1035	int tgtprec = GET_MODE_PRECISION (mode: tgtmode);
1036	if (prec < tgtprec)
1037	{
1038	rtx_code_label *done_label = gen_label_rtx ();
1039	int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
1040	res = lres;
1041	if (uns)
1042	{
1043	rtx mask
1044	= immed_wide_int_const (wi::shifted_mask (start: 0, width: prec, negate_p: false, precision: tgtprec),
1045	tgtmode);
1046	lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
1047	true, OPTAB_LIB_WIDEN);
1048	}
1049	else
1050	{
1051	lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
1052	NULL_RTX, 1);
1053	lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
1054	NULL_RTX, 0);
1055	}
1056	do_compare_rtx_and_jump (res, lres,
1057	EQ, true, tgtmode, NULL_RTX, NULL, done_label,
1058	profile_probability::very_likely ());
1059	expand_arith_set_overflow (lhs, target);
1060	emit_label (done_label);
1061	}
1062	write_complex_part (target, lres, false, false);
1063	}
1064
1065	/* Helper for expand_*_overflow. Store RES into TARGET. */
1066
1067	static void
1068	expand_ubsan_result_store (tree lhs, rtx target, scalar_int_mode mode,
1069	rtx res, rtx_code_label *do_error)
1070	{
1071	if (TREE_CODE (TREE_TYPE (lhs)) == BITINT_TYPE
1072	&& TYPE_PRECISION (TREE_TYPE (lhs)) < GET_MODE_PRECISION (mode))
1073	{
1074	int uns = TYPE_UNSIGNED (TREE_TYPE (lhs));
1075	int prec = TYPE_PRECISION (TREE_TYPE (lhs));
1076	int tgtprec = GET_MODE_PRECISION (mode);
1077	rtx resc = gen_reg_rtx (mode), lres;
1078	emit_move_insn (resc, res);
1079	if (uns)
1080	{
1081	rtx mask
1082	= immed_wide_int_const (wi::shifted_mask (start: 0, width: prec, negate_p: false, precision: tgtprec),
1083	mode);
1084	lres = expand_simple_binop (mode, AND, res, mask, NULL_RTX,
1085	true, OPTAB_LIB_WIDEN);
1086	}
1087	else
1088	{
1089	lres = expand_shift (LSHIFT_EXPR, mode, res, tgtprec - prec,
1090	NULL_RTX, 1);
1091	lres = expand_shift (RSHIFT_EXPR, mode, lres, tgtprec - prec,
1092	NULL_RTX, 0);
1093	}
1094	if (lres != res)
1095	emit_move_insn (res, lres);
1096	do_compare_rtx_and_jump (res, resc,
1097	NE, true, mode, NULL_RTX, NULL, do_error,
1098	profile_probability::very_unlikely ());
1099	}
1100	if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
1101	/* If this is a scalar in a register that is stored in a wider mode
1102	than the declared mode, compute the result into its declared mode
1103	and then convert to the wider mode. Our value is the computed
1104	expression. */
1105	convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
1106	else
1107	emit_move_insn (target, res);
1108	}
1109
1110	/* Add sub/add overflow checking to the statement STMT.
1111	CODE says whether the operation is +, or -. */
1112
1113	void
1114	expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
1115	tree arg0, tree arg1, bool unsr_p, bool uns0_p,
1116	bool uns1_p, bool is_ubsan, tree *datap)
1117	{
1118	rtx res, target = NULL_RTX;
1119	tree fn;
1120	rtx_code_label *done_label = gen_label_rtx ();
1121	rtx_code_label *do_error = gen_label_rtx ();
1122	do_pending_stack_adjust ();
1123	rtx op0 = expand_normal (exp: arg0);
1124	rtx op1 = expand_normal (exp: arg1);
1125	scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
1126	int prec = GET_MODE_PRECISION (mode);
1127	rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1128	bool do_xor = false;
1129
1130	if (is_ubsan)
1131	gcc_assert (!unsr_p && !uns0_p && !uns1_p);
1132
1133	if (lhs)
1134	{
1135	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
1136	if (!is_ubsan)
1137	write_complex_part (target, const0_rtx, true, false);
1138	}
1139
1140	/* We assume both operands and result have the same precision
1141	here (GET_MODE_BITSIZE (mode)), S stands for signed type
1142	with that precision, U for unsigned type with that precision,
1143	sgn for unsigned most significant bit in that precision.
1144	s1 is signed first operand, u1 is unsigned first operand,
1145	s2 is signed second operand, u2 is unsigned second operand,
1146	sr is signed result, ur is unsigned result and the following
1147	rules say how to compute result (which is always result of
1148	the operands as if both were unsigned, cast to the right
1149	signedness) and how to compute whether operation overflowed.
1150
1151	s1 + s2 -> sr
1152	res = (S) ((U) s1 + (U) s2)
1153	ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
1154	s1 - s2 -> sr
1155	res = (S) ((U) s1 - (U) s2)
1156	ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
1157	u1 + u2 -> ur
1158	res = u1 + u2
1159	ovf = res < u1 (or jump on carry, but RTL opts will handle it)
1160	u1 - u2 -> ur
1161	res = u1 - u2
1162	ovf = res > u1 (or jump on carry, but RTL opts will handle it)
1163	s1 + u2 -> sr
1164	res = (S) ((U) s1 + u2)
1165	ovf = ((U) res ^ sgn) < u2
1166	s1 + u2 -> ur
1167	t1 = (S) (u2 ^ sgn)
1168	t2 = s1 + t1
1169	res = (U) t2 ^ sgn
1170	ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
1171	s1 - u2 -> sr
1172	res = (S) ((U) s1 - u2)
1173	ovf = u2 > ((U) s1 ^ sgn)
1174	s1 - u2 -> ur
1175	res = (U) s1 - u2
1176	ovf = s1 < 0 || u2 > (U) s1
1177	u1 - s2 -> sr
1178	res = u1 - (U) s2
1179	ovf = u1 >= ((U) s2 ^ sgn)
1180	u1 - s2 -> ur
1181	t1 = u1 ^ sgn
1182	t2 = t1 - (U) s2
1183	res = t2 ^ sgn
1184	ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
1185	s1 + s2 -> ur
1186	res = (U) s1 + (U) s2
1187	ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
1188	u1 + u2 -> sr
1189	res = (S) (u1 + u2)
1190	ovf = (U) res < u2 || res < 0
1191	u1 - u2 -> sr
1192	res = (S) (u1 - u2)
1193	ovf = u1 >= u2 ? res < 0 : res >= 0
1194	s1 - s2 -> ur
1195	res = (U) s1 - (U) s2
1196	ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
1197
1198	if (code == PLUS_EXPR && uns0_p && !uns1_p)
1199	{
1200	/* PLUS_EXPR is commutative, if operand signedness differs,
1201	canonicalize to the first operand being signed and second
1202	unsigned to simplify following code. */
1203	std::swap (a&: op0, b&: op1);
1204	std::swap (a&: arg0, b&: arg1);
1205	uns0_p = false;
1206	uns1_p = true;
1207	}
1208
1209	/* u1 +- u2 -> ur */
1210	if (uns0_p && uns1_p && unsr_p)
1211	{
1212	insn_code icode = optab_handler (op: code == PLUS_EXPR ? uaddv4_optab
1213	: usubv4_optab, mode);
1214	if (icode != CODE_FOR_nothing)
1215	{
1216	class expand_operand ops[4];
1217	rtx_insn *last = get_last_insn ();
1218
1219	res = gen_reg_rtx (mode);
1220	create_output_operand (op: &ops[0], x: res, mode);
1221	create_input_operand (op: &ops[1], value: op0, mode);
1222	create_input_operand (op: &ops[2], value: op1, mode);
1223	create_fixed_operand (op: &ops[3], x: do_error);
1224	if (maybe_expand_insn (icode, nops: 4, ops))
1225	{
1226	last = get_last_insn ();
1227	if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1228	&& JUMP_P (last)
1229	&& any_condjump_p (last)
1230	&& !find_reg_note (last, REG_BR_PROB, 0))
1231	add_reg_br_prob_note (last,
1232	profile_probability::very_unlikely ());
1233	emit_jump (done_label);
1234	goto do_error_label;
1235	}
1236
1237	delete_insns_since (last);
1238	}
1239
1240	/* Compute the operation. On RTL level, the addition is always
1241	unsigned. */
1242	res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1243	op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1244	rtx tem = op0;
1245	/* For PLUS_EXPR, the operation is commutative, so we can pick
1246	operand to compare against. For prec <= BITS_PER_WORD, I think
1247	preferring REG operand is better over CONST_INT, because
1248	the CONST_INT might enlarge the instruction or CSE would need
1249	to figure out we'd already loaded it into a register before.
1250	For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
1251	as then the multi-word comparison can be perhaps simplified. */
1252	if (code == PLUS_EXPR
1253	&& (prec <= BITS_PER_WORD
1254	? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
1255	: CONST_SCALAR_INT_P (op1)))
1256	tem = op1;
1257	do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
1258	true, mode, NULL_RTX, NULL, done_label,
1259	profile_probability::very_likely ());
1260	goto do_error_label;
1261	}
1262
1263	/* s1 +- u2 -> sr */
1264	if (!uns0_p && uns1_p && !unsr_p)
1265	{
1266	/* Compute the operation. On RTL level, the addition is always
1267	unsigned. */
1268	res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1269	op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1270	rtx tem = expand_binop (mode, add_optab,
1271	code == PLUS_EXPR ? res : op0, sgn,
1272	NULL_RTX, false, OPTAB_LIB_WIDEN);
1273	do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
1274	done_label, profile_probability::very_likely ());
1275	goto do_error_label;
1276	}
1277
1278	/* s1 + u2 -> ur */
1279	if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
1280	{
1281	op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
1282	OPTAB_LIB_WIDEN);
1283	/* As we've changed op1, we have to avoid using the value range
1284	for the original argument. */
1285	arg1 = error_mark_node;
1286	do_xor = true;
1287	goto do_signed;
1288	}
1289
1290	/* u1 - s2 -> ur */
1291	if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
1292	{
1293	op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
1294	OPTAB_LIB_WIDEN);
1295	/* As we've changed op0, we have to avoid using the value range
1296	for the original argument. */
1297	arg0 = error_mark_node;
1298	do_xor = true;
1299	goto do_signed;
1300	}
1301
1302	/* s1 - u2 -> ur */
1303	if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
1304	{
1305	/* Compute the operation. On RTL level, the addition is always
1306	unsigned. */
1307	res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1308	OPTAB_LIB_WIDEN);
1309	int pos_neg = get_range_pos_neg (arg0, currently_expanding_gimple_stmt);
1310	if (pos_neg == 2)
1311	/* If ARG0 is known to be always negative, this is always overflow. */
1312	emit_jump (do_error);
1313	else if (pos_neg == 3)
1314	/* If ARG0 is not known to be always positive, check at runtime. */
1315	do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
1316	NULL, do_error, profile_probability::very_unlikely ());
1317	do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
1318	done_label, profile_probability::very_likely ());
1319	goto do_error_label;
1320	}
1321
1322	/* u1 - s2 -> sr */
1323	if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
1324	{
1325	/* Compute the operation. On RTL level, the addition is always
1326	unsigned. */
1327	res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1328	OPTAB_LIB_WIDEN);
1329	rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
1330	OPTAB_LIB_WIDEN);
1331	do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
1332	done_label, profile_probability::very_likely ());
1333	goto do_error_label;
1334	}
1335
1336	/* u1 + u2 -> sr */
1337	if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
1338	{
1339	/* Compute the operation. On RTL level, the addition is always
1340	unsigned. */
1341	res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
1342	OPTAB_LIB_WIDEN);
1343	do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1344	NULL, do_error, profile_probability::very_unlikely ());
1345	rtx tem = op1;
1346	/* The operation is commutative, so we can pick operand to compare
1347	against. For prec <= BITS_PER_WORD, I think preferring REG operand
1348	is better over CONST_INT, because the CONST_INT might enlarge the
1349	instruction or CSE would need to figure out we'd already loaded it
1350	into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1351	might be more beneficial, as then the multi-word comparison can be
1352	perhaps simplified. */
1353	if (prec <= BITS_PER_WORD
1354	? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
1355	: CONST_SCALAR_INT_P (op0))
1356	tem = op0;
1357	do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
1358	done_label, profile_probability::very_likely ());
1359	goto do_error_label;
1360	}
1361
1362	/* s1 +- s2 -> ur */
1363	if (!uns0_p && !uns1_p && unsr_p)
1364	{
1365	/* Compute the operation. On RTL level, the addition is always
1366	unsigned. */
1367	res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1368	op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1369	int pos_neg = get_range_pos_neg (arg1, currently_expanding_gimple_stmt);
1370	if (code == PLUS_EXPR)
1371	{
1372	int pos_neg0 = get_range_pos_neg (arg0,
1373	currently_expanding_gimple_stmt);
1374	if (pos_neg0 != 3 && pos_neg == 3)
1375	{
1376	std::swap (a&: op0, b&: op1);
1377	pos_neg = pos_neg0;
1378	}
1379	}
1380	rtx tem;
1381	if (pos_neg != 3)
1382	{
1383	tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
1384	? and_optab : ior_optab,
1385	op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
1386	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
1387	NULL, done_label, profile_probability::very_likely ());
1388	}
1389	else
1390	{
1391	rtx_code_label *do_ior_label = gen_label_rtx ();
1392	do_compare_rtx_and_jump (op1, const0_rtx,
1393	code == MINUS_EXPR ? GE : LT, false, mode,
1394	NULL_RTX, NULL, do_ior_label,
1395	profile_probability::even ());
1396	tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
1397	OPTAB_LIB_WIDEN);
1398	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1399	NULL, done_label, profile_probability::very_likely ());
1400	emit_jump (do_error);
1401	emit_label (do_ior_label);
1402	tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
1403	OPTAB_LIB_WIDEN);
1404	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1405	NULL, done_label, profile_probability::very_likely ());
1406	}
1407	goto do_error_label;
1408	}
1409
1410	/* u1 - u2 -> sr */
1411	if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
1412	{
1413	/* Compute the operation. On RTL level, the addition is always
1414	unsigned. */
1415	res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1416	OPTAB_LIB_WIDEN);
1417	rtx_code_label *op0_geu_op1 = gen_label_rtx ();
1418	do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
1419	op0_geu_op1, profile_probability::even ());
1420	do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1421	NULL, done_label, profile_probability::very_likely ());
1422	emit_jump (do_error);
1423	emit_label (op0_geu_op1);
1424	do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
1425	NULL, done_label, profile_probability::very_likely ());
1426	goto do_error_label;
1427	}
1428
1429	gcc_assert (!uns0_p && !uns1_p && !unsr_p);
1430
1431	/* s1 +- s2 -> sr */
1432	do_signed:
1433	{
1434	insn_code icode = optab_handler (op: code == PLUS_EXPR ? addv4_optab
1435	: subv4_optab, mode);
1436	if (icode != CODE_FOR_nothing)
1437	{
1438	class expand_operand ops[4];
1439	rtx_insn *last = get_last_insn ();
1440
1441	res = gen_reg_rtx (mode);
1442	create_output_operand (op: &ops[0], x: res, mode);
1443	create_input_operand (op: &ops[1], value: op0, mode);
1444	create_input_operand (op: &ops[2], value: op1, mode);
1445	create_fixed_operand (op: &ops[3], x: do_error);
1446	if (maybe_expand_insn (icode, nops: 4, ops))
1447	{
1448	last = get_last_insn ();
1449	if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1450	&& JUMP_P (last)
1451	&& any_condjump_p (last)
1452	&& !find_reg_note (last, REG_BR_PROB, 0))
1453	add_reg_br_prob_note (last,
1454	profile_probability::very_unlikely ());
1455	emit_jump (done_label);
1456	goto do_error_label;
1457	}
1458
1459	delete_insns_since (last);
1460	}
1461
1462	/* Compute the operation. On RTL level, the addition is always
1463	unsigned. */
1464	res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1465	op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1466
1467	/* If we can prove that one of the arguments (for MINUS_EXPR only
1468	the second operand, as subtraction is not commutative) is always
1469	non-negative or always negative, we can do just one comparison
1470	and conditional jump. */
1471	int pos_neg = get_range_pos_neg (arg1, currently_expanding_gimple_stmt);
1472	if (code == PLUS_EXPR)
1473	{
1474	int pos_neg0 = get_range_pos_neg (arg0,
1475	currently_expanding_gimple_stmt);
1476	if (pos_neg0 != 3 && pos_neg == 3)
1477	{
1478	std::swap (a&: op0, b&: op1);
1479	pos_neg = pos_neg0;
1480	}
1481	}
1482
1483	/* Addition overflows if and only if the two operands have the same sign,
1484	and the result has the opposite sign. Subtraction overflows if and
1485	only if the two operands have opposite sign, and the subtrahend has
1486	the same sign as the result. Here 0 is counted as positive. */
1487	if (pos_neg == 3)
1488	{
1489	/* Compute op0 ^ op1 (operands have opposite sign). */
1490	rtx op_xor = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1491	OPTAB_LIB_WIDEN);
1492
1493	/* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1494	rtx res_xor = expand_binop (mode, xor_optab, res, op1, NULL_RTX, false,
1495	OPTAB_LIB_WIDEN);
1496
1497	rtx tem;
1498	if (code == PLUS_EXPR)
1499	{
1500	/* Compute (res ^ op1) & ~(op0 ^ op1). */
1501	tem = expand_unop (mode, one_cmpl_optab, op_xor, NULL_RTX, false);
1502	tem = expand_binop (mode, and_optab, res_xor, tem, NULL_RTX, false,
1503	OPTAB_LIB_WIDEN);
1504	}
1505	else
1506	{
1507	/* Compute (op0 ^ op1) & ~(res ^ op1). */
1508	tem = expand_unop (mode, one_cmpl_optab, res_xor, NULL_RTX, false);
1509	tem = expand_binop (mode, and_optab, op_xor, tem, NULL_RTX, false,
1510	OPTAB_LIB_WIDEN);
1511	}
1512
1513	/* No overflow if the result has bit sign cleared. */
1514	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1515	NULL, done_label, profile_probability::very_likely ());
1516	}
1517
1518	/* Compare the result of the operation with the first operand.
1519	No overflow for addition if second operand is positive and result
1520	is larger or second operand is negative and result is smaller.
1521	Likewise for subtraction with sign of second operand flipped. */
1522	else
1523	do_compare_rtx_and_jump (res, op0,
1524	(pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE,
1525	false, mode, NULL_RTX, NULL, done_label,
1526	profile_probability::very_likely ());
1527	}
1528
1529	do_error_label:
1530	emit_label (do_error);
1531	if (is_ubsan)
1532	{
1533	/* Expand the ubsan builtin call. */
1534	push_temp_slots ();
1535	fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
1536	arg0, arg1, datap);
1537	expand_normal (exp: fn);
1538	pop_temp_slots ();
1539	do_pending_stack_adjust ();
1540	}
1541	else if (lhs)
1542	expand_arith_set_overflow (lhs, target);
1543
1544	/* We're done. */
1545	emit_label (done_label);
1546
1547	if (lhs)
1548	{
1549	if (is_ubsan)
1550	expand_ubsan_result_store (lhs, target, mode, res, do_error);
1551	else
1552	{
1553	if (do_xor)
1554	res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
1555	OPTAB_LIB_WIDEN);
1556
1557	expand_arith_overflow_result_store (lhs, target, mode, res);
1558	}
1559	}
1560	}
1561
1562	/* Add negate overflow checking to the statement STMT. */
1563
1564	static void
1565	expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan,
1566	tree *datap)
1567	{
1568	rtx res, op1;
1569	tree fn;
1570	rtx_code_label *done_label, *do_error;
1571	rtx target = NULL_RTX;
1572
1573	done_label = gen_label_rtx ();
1574	do_error = gen_label_rtx ();
1575
1576	do_pending_stack_adjust ();
1577	op1 = expand_normal (exp: arg1);
1578
1579	scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1));
1580	if (lhs)
1581	{
1582	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
1583	if (!is_ubsan)
1584	write_complex_part (target, const0_rtx, true, false);
1585	}
1586
1587	enum insn_code icode = optab_handler (op: negv3_optab, mode);
1588	if (icode != CODE_FOR_nothing)
1589	{
1590	class expand_operand ops[3];
1591	rtx_insn *last = get_last_insn ();
1592
1593	res = gen_reg_rtx (mode);
1594	create_output_operand (op: &ops[0], x: res, mode);
1595	create_input_operand (op: &ops[1], value: op1, mode);
1596	create_fixed_operand (op: &ops[2], x: do_error);
1597	if (maybe_expand_insn (icode, nops: 3, ops))
1598	{
1599	last = get_last_insn ();
1600	if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1601	&& JUMP_P (last)
1602	&& any_condjump_p (last)
1603	&& !find_reg_note (last, REG_BR_PROB, 0))
1604	add_reg_br_prob_note (last,
1605	profile_probability::very_unlikely ());
1606	emit_jump (done_label);
1607	}
1608	else
1609	{
1610	delete_insns_since (last);
1611	icode = CODE_FOR_nothing;
1612	}
1613	}
1614
1615	if (icode == CODE_FOR_nothing)
1616	{
1617	/* Compute the operation. On RTL level, the addition is always
1618	unsigned. */
1619	res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1620
1621	/* Compare the operand with the most negative value. */
1622	rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
1623	do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
1624	done_label, profile_probability::very_likely ());
1625	}
1626
1627	emit_label (do_error);
1628	if (is_ubsan)
1629	{
1630	/* Expand the ubsan builtin call. */
1631	push_temp_slots ();
1632	fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
1633	arg1, NULL_TREE, datap);
1634	expand_normal (exp: fn);
1635	pop_temp_slots ();
1636	do_pending_stack_adjust ();
1637	}
1638	else if (lhs)
1639	expand_arith_set_overflow (lhs, target);
1640
1641	/* We're done. */
1642	emit_label (done_label);
1643
1644	if (lhs)
1645	{
1646	if (is_ubsan)
1647	expand_ubsan_result_store (lhs, target, mode, res, do_error);
1648	else
1649	expand_arith_overflow_result_store (lhs, target, mode, res);
1650	}
1651	}
1652
1653	/* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1654	mode MODE can be expanded without using a libcall. */
1655
1656	static bool
1657	can_widen_mult_without_libcall (scalar_int_mode wmode, scalar_int_mode mode,
1658	rtx op0, rtx op1, bool uns)
1659	{
1660	if (find_widening_optab_handler (umul_widen_optab, wmode, mode)
1661	!= CODE_FOR_nothing)
1662	return true;
1663
1664	if (find_widening_optab_handler (smul_widen_optab, wmode, mode)
1665	!= CODE_FOR_nothing)
1666	return true;
1667
1668	rtx_insn *last = get_last_insn ();
1669	if (CONSTANT_P (op0))
1670	op0 = convert_modes (mode: wmode, oldmode: mode, x: op0, unsignedp: uns);
1671	else
1672	op0 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 1);
1673	if (CONSTANT_P (op1))
1674	op1 = convert_modes (mode: wmode, oldmode: mode, x: op1, unsignedp: uns);
1675	else
1676	op1 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 2);
1677	rtx ret = expand_mult (wmode, op0, op1, NULL_RTX, uns, true);
1678	delete_insns_since (last);
1679	return ret != NULL_RTX;
1680	}
1681
1682	/* Add mul overflow checking to the statement STMT. */
1683
1684	static void
1685	expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
1686	bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan,
1687	tree *datap)
1688	{
1689	rtx res, op0, op1;
1690	tree fn, type;
1691	rtx_code_label *done_label, *do_error;
1692	rtx target = NULL_RTX;
1693	signop sign;
1694	enum insn_code icode;
1695	int save_flag_trapv = flag_trapv;
1696
1697	/* We don't want any __mulv?i3 etc. calls from the expansion of
1698	these internal functions, so disable -ftrapv temporarily. */
1699	flag_trapv = 0;
1700	done_label = gen_label_rtx ();
1701	do_error = gen_label_rtx ();
1702
1703	do_pending_stack_adjust ();
1704	op0 = expand_normal (exp: arg0);
1705	op1 = expand_normal (exp: arg1);
1706
1707	scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
1708	bool uns = unsr_p;
1709	if (lhs)
1710	{
1711	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
1712	if (!is_ubsan)
1713	write_complex_part (target, const0_rtx, true, false);
1714	}
1715
1716	if (is_ubsan)
1717	gcc_assert (!unsr_p && !uns0_p && !uns1_p);
1718
1719	/* We assume both operands and result have the same precision
1720	here (GET_MODE_BITSIZE (mode)), S stands for signed type
1721	with that precision, U for unsigned type with that precision,
1722	sgn for unsigned most significant bit in that precision.
1723	s1 is signed first operand, u1 is unsigned first operand,
1724	s2 is signed second operand, u2 is unsigned second operand,
1725	sr is signed result, ur is unsigned result and the following
1726	rules say how to compute result (which is always result of
1727	the operands as if both were unsigned, cast to the right
1728	signedness) and how to compute whether operation overflowed.
1729	main_ovf (false) stands for jump on signed multiplication
1730	overflow or the main algorithm with uns == false.
1731	main_ovf (true) stands for jump on unsigned multiplication
1732	overflow or the main algorithm with uns == true.
1733
1734	s1 * s2 -> sr
1735	res = (S) ((U) s1 * (U) s2)
1736	ovf = main_ovf (false)
1737	u1 * u2 -> ur
1738	res = u1 * u2
1739	ovf = main_ovf (true)
1740	s1 * u2 -> ur
1741	res = (U) s1 * u2
1742	ovf = (s1 < 0 && u2) || main_ovf (true)
1743	u1 * u2 -> sr
1744	res = (S) (u1 * u2)
1745	ovf = res < 0 || main_ovf (true)
1746	s1 * u2 -> sr
1747	res = (S) ((U) s1 * u2)
1748	ovf = (S) u2 >= 0 ? main_ovf (false)
1749	: (s1 != 0 && (s1 != -1 || u2 != (U) res))
1750	s1 * s2 -> ur
1751	t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1752	t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1753	res = t1 * t2
1754	ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1755
1756	if (uns0_p && !uns1_p)
1757	{
1758	/* Multiplication is commutative, if operand signedness differs,
1759	canonicalize to the first operand being signed and second
1760	unsigned to simplify following code. */
1761	std::swap (a&: op0, b&: op1);
1762	std::swap (a&: arg0, b&: arg1);
1763	uns0_p = false;
1764	uns1_p = true;
1765	}
1766
1767	int pos_neg0 = get_range_pos_neg (arg0, currently_expanding_gimple_stmt);
1768	int pos_neg1 = get_range_pos_neg (arg1, currently_expanding_gimple_stmt);
1769	/* Unsigned types with smaller than mode precision, even if they have most
1770	significant bit set, are still zero-extended. */
1771	if (uns0_p && TYPE_PRECISION (TREE_TYPE (arg0)) < GET_MODE_PRECISION (mode))
1772	pos_neg0 = 1;
1773	if (uns1_p && TYPE_PRECISION (TREE_TYPE (arg1)) < GET_MODE_PRECISION (mode))
1774	pos_neg1 = 1;
1775
1776	/* s1 * u2 -> ur */
1777	if (!uns0_p && uns1_p && unsr_p)
1778	{
1779	switch (pos_neg0)
1780	{
1781	case 1:
1782	/* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1783	goto do_main;
1784	case 2:
1785	/* If s1 is negative, avoid the main code, just multiply and
1786	signal overflow if op1 is not 0. */
1787	struct separate_ops ops;
1788	ops.code = MULT_EXPR;
1789	ops.type = TREE_TYPE (arg1);
1790	ops.op0 = make_tree (ops.type, op0);
1791	ops.op1 = make_tree (ops.type, op1);
1792	ops.op2 = NULL_TREE;
1793	ops.location = loc;
1794	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1795	do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1796	NULL, done_label, profile_probability::very_likely ());
1797	goto do_error_label;
1798	case 3:
1799	if (get_min_precision (arg: arg1, sign: UNSIGNED)
1800	+ get_min_precision (arg: arg0, sign: SIGNED) <= GET_MODE_PRECISION (mode))
1801	{
1802	/* If the first operand is sign extended from narrower type, the
1803	second operand is zero extended from narrower type and
1804	the sum of the two precisions is smaller or equal to the
1805	result precision: if the first argument is at runtime
1806	non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1807	and there will be no overflow, if the first argument is
1808	negative and the second argument zero, the result will be
1809	0 and there will be no overflow, if the first argument is
1810	negative and the second argument positive, the result when
1811	treated as signed will be negative (minimum -0x7f80 or
1812	-0x7f..f80..0) there will be always overflow. So, do
1813	res = (U) (s1 * u2)
1814	ovf = (S) res < 0 */
1815	struct separate_ops ops;
1816	ops.code = MULT_EXPR;
1817	ops.type
1818	= build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1819	1);
1820	ops.op0 = make_tree (ops.type, op0);
1821	ops.op1 = make_tree (ops.type, op1);
1822	ops.op2 = NULL_TREE;
1823	ops.location = loc;
1824	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1825	do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1826	mode, NULL_RTX, NULL, done_label,
1827	profile_probability::very_likely ());
1828	goto do_error_label;
1829	}
1830	rtx_code_label *do_main_label;
1831	do_main_label = gen_label_rtx ();
1832	do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
1833	NULL, do_main_label, profile_probability::very_likely ());
1834	do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1835	NULL, do_main_label, profile_probability::very_likely ());
1836	expand_arith_set_overflow (lhs, target);
1837	emit_label (do_main_label);
1838	goto do_main;
1839	default:
1840	gcc_unreachable ();
1841	}
1842	}
1843
1844	/* u1 * u2 -> sr */
1845	if (uns0_p && uns1_p && !unsr_p)
1846	{
1847	if ((pos_neg0 | pos_neg1) == 1)
1848	{
1849	/* If both arguments are zero extended from narrower types,
1850	the MSB will be clear on both and so we can pretend it is
1851	a normal s1 * s2 -> sr multiplication. */
1852	uns0_p = false;
1853	uns1_p = false;
1854	}
1855	else
1856	uns = true;
1857	/* Rest of handling of this case after res is computed. */
1858	goto do_main;
1859	}
1860
1861	/* s1 * u2 -> sr */
1862	if (!uns0_p && uns1_p && !unsr_p)
1863	{
1864	switch (pos_neg1)
1865	{
1866	case 1:
1867	goto do_main;
1868	case 2:
1869	/* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1870	avoid the main code, just multiply and signal overflow
1871	unless 0 * u2 or -1 * ((U) Smin). */
1872	struct separate_ops ops;
1873	ops.code = MULT_EXPR;
1874	ops.type = TREE_TYPE (arg1);
1875	ops.op0 = make_tree (ops.type, op0);
1876	ops.op1 = make_tree (ops.type, op1);
1877	ops.op2 = NULL_TREE;
1878	ops.location = loc;
1879	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1880	do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1881	NULL, done_label, profile_probability::very_likely ());
1882	do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
1883	NULL, do_error, profile_probability::very_unlikely ());
1884	int prec;
1885	prec = GET_MODE_PRECISION (mode);
1886	rtx sgn;
1887	sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1888	do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
1889	NULL, done_label, profile_probability::very_likely ());
1890	goto do_error_label;
1891	case 3:
1892	/* Rest of handling of this case after res is computed. */
1893	goto do_main;
1894	default:
1895	gcc_unreachable ();
1896	}
1897	}
1898
1899	/* s1 * s2 -> ur */
1900	if (!uns0_p && !uns1_p && unsr_p)
1901	{
1902	rtx tem;
1903	switch (pos_neg0 | pos_neg1)
1904	{
1905	case 1: /* Both operands known to be non-negative. */
1906	goto do_main;
1907	case 2: /* Both operands known to be negative. */
1908	op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
1909	op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1910	/* Avoid looking at arg0/arg1 ranges, as we've changed
1911	the arguments. */
1912	arg0 = error_mark_node;
1913	arg1 = error_mark_node;
1914	goto do_main;
1915	case 3:
1916	if ((pos_neg0 ^ pos_neg1) == 3)
1917	{
1918	/* If one operand is known to be negative and the other
1919	non-negative, this overflows always, unless the non-negative
1920	one is 0. Just do normal multiply and set overflow
1921	unless one of the operands is 0. */
1922	struct separate_ops ops;
1923	ops.code = MULT_EXPR;
1924	ops.type
1925	= build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1926	1);
1927	ops.op0 = make_tree (ops.type, op0);
1928	ops.op1 = make_tree (ops.type, op1);
1929	ops.op2 = NULL_TREE;
1930	ops.location = loc;
1931	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1932	do_compare_rtx_and_jump (pos_neg0 == 1 ? op0 : op1, const0_rtx, EQ,
1933	true, mode, NULL_RTX, NULL, done_label,
1934	profile_probability::very_likely ());
1935	goto do_error_label;
1936	}
1937	if (get_min_precision (arg: arg0, sign: SIGNED)
1938	+ get_min_precision (arg: arg1, sign: SIGNED) <= GET_MODE_PRECISION (mode))
1939	{
1940	/* If both operands are sign extended from narrower types and
1941	the sum of the two precisions is smaller or equal to the
1942	result precision: if both arguments are at runtime
1943	non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1944	and there will be no overflow, if both arguments are negative,
1945	maximum result will be 0x40..00 and there will be no overflow
1946	either, if one argument is positive and the other argument
1947	negative, the result when treated as signed will be negative
1948	and there will be always overflow, and if one argument is
1949	zero and the other negative the result will be zero and no
1950	overflow. So, do
1951	res = (U) (s1 * s2)
1952	ovf = (S) res < 0 */
1953	struct separate_ops ops;
1954	ops.code = MULT_EXPR;
1955	ops.type
1956	= build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1957	1);
1958	ops.op0 = make_tree (ops.type, op0);
1959	ops.op1 = make_tree (ops.type, op1);
1960	ops.op2 = NULL_TREE;
1961	ops.location = loc;
1962	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1963	do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1964	mode, NULL_RTX, NULL, done_label,
1965	profile_probability::very_likely ());
1966	goto do_error_label;
1967	}
1968	/* The general case, do all the needed comparisons at runtime. */
1969	rtx_code_label *do_main_label, *after_negate_label;
1970	rtx rop0, rop1;
1971	rop0 = gen_reg_rtx (mode);
1972	rop1 = gen_reg_rtx (mode);
1973	emit_move_insn (rop0, op0);
1974	emit_move_insn (rop1, op1);
1975	op0 = rop0;
1976	op1 = rop1;
1977	do_main_label = gen_label_rtx ();
1978	after_negate_label = gen_label_rtx ();
1979	tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1980	OPTAB_LIB_WIDEN);
1981	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1982	NULL, after_negate_label, profile_probability::very_likely ());
1983	/* Both arguments negative here, negate them and continue with
1984	normal unsigned overflow checking multiplication. */
1985	emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
1986	NULL_RTX, false));
1987	emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
1988	NULL_RTX, false));
1989	/* Avoid looking at arg0/arg1 ranges, as we might have changed
1990	the arguments. */
1991	arg0 = error_mark_node;
1992	arg1 = error_mark_node;
1993	emit_jump (do_main_label);
1994	emit_label (after_negate_label);
1995	tem = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1996	OPTAB_LIB_WIDEN);
1997	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1998	NULL, do_main_label,
1999	profile_probability::very_likely ());
2000	/* One argument is negative here, the other positive. This
2001	overflows always, unless one of the arguments is 0. But
2002	if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
2003	is, thus we can keep do_main code oring in overflow as is. */
2004	if (pos_neg0 != 2)
2005	do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
2006	NULL, do_main_label,
2007	profile_probability::very_unlikely ());
2008	if (pos_neg1 != 2)
2009	do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
2010	NULL, do_main_label,
2011	profile_probability::very_unlikely ());
2012	expand_arith_set_overflow (lhs, target);
2013	emit_label (do_main_label);
2014	goto do_main;
2015	default:
2016	gcc_unreachable ();
2017	}
2018	}
2019
2020	do_main:
2021	type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
2022	sign = uns ? UNSIGNED : SIGNED;
2023	icode = optab_handler (op: uns ? umulv4_optab : mulv4_optab, mode);
2024	if (uns
2025	&& (integer_pow2p (arg0) || integer_pow2p (arg1))
2026	&& (optimize_insn_for_speed_p () || icode == CODE_FOR_nothing))
2027	{
2028	/* Optimize unsigned multiplication by power of 2 constant
2029	using 2 shifts, one for result, one to extract the shifted
2030	out bits to see if they are all zero.
2031	Don't do this if optimizing for size and we have umulv4_optab,
2032	in that case assume multiplication will be shorter.
2033	This is heuristics based on the single target that provides
2034	umulv4 right now (i?86/x86_64), if further targets add it, this
2035	might need to be revisited.
2036	Cases where both operands are constant should be folded already
2037	during GIMPLE, and cases where one operand is constant but not
2038	power of 2 are questionable, either the WIDEN_MULT_EXPR case
2039	below can be done without multiplication, just by shifts and adds,
2040	or we'd need to divide the result (and hope it actually doesn't
2041	really divide nor multiply) and compare the result of the division
2042	with the original operand. */
2043	rtx opn0 = op0;
2044	rtx opn1 = op1;
2045	tree argn0 = arg0;
2046	tree argn1 = arg1;
2047	if (integer_pow2p (arg0))
2048	{
2049	std::swap (a&: opn0, b&: opn1);
2050	std::swap (a&: argn0, b&: argn1);
2051	}
2052	int cnt = tree_log2 (argn1);
2053	if (cnt >= 0 && cnt < GET_MODE_PRECISION (mode))
2054	{
2055	rtx upper = const0_rtx;
2056	res = expand_shift (LSHIFT_EXPR, mode, opn0, cnt, NULL_RTX, uns);
2057	if (cnt != 0)
2058	upper = expand_shift (RSHIFT_EXPR, mode, opn0,
2059	GET_MODE_PRECISION (mode) - cnt,
2060	NULL_RTX, uns);
2061	do_compare_rtx_and_jump (upper, const0_rtx, EQ, true, mode,
2062	NULL_RTX, NULL, done_label,
2063	profile_probability::very_likely ());
2064	goto do_error_label;
2065	}
2066	}
2067	if (icode != CODE_FOR_nothing)
2068	{
2069	class expand_operand ops[4];
2070	rtx_insn *last = get_last_insn ();
2071
2072	res = gen_reg_rtx (mode);
2073	create_output_operand (op: &ops[0], x: res, mode);
2074	create_input_operand (op: &ops[1], value: op0, mode);
2075	create_input_operand (op: &ops[2], value: op1, mode);
2076	create_fixed_operand (op: &ops[3], x: do_error);
2077	if (maybe_expand_insn (icode, nops: 4, ops))
2078	{
2079	last = get_last_insn ();
2080	if (profile_status_for_fn (cfun) != PROFILE_ABSENT
2081	&& JUMP_P (last)
2082	&& any_condjump_p (last)
2083	&& !find_reg_note (last, REG_BR_PROB, 0))
2084	add_reg_br_prob_note (last,
2085	profile_probability::very_unlikely ());
2086	emit_jump (done_label);
2087	}
2088	else
2089	{
2090	delete_insns_since (last);
2091	icode = CODE_FOR_nothing;
2092	}
2093	}
2094
2095	if (icode == CODE_FOR_nothing)
2096	{
2097	struct separate_ops ops;
2098	int prec = GET_MODE_PRECISION (mode);
2099	scalar_int_mode hmode, wmode;
2100	ops.op0 = make_tree (type, op0);
2101	ops.op1 = make_tree (type, op1);
2102	ops.op2 = NULL_TREE;
2103	ops.location = loc;
2104
2105	/* Optimize unsigned overflow check where we don't use the
2106	multiplication result, just whether overflow happened.
2107	If we can do MULT_HIGHPART_EXPR, that followed by
2108	comparison of the result against zero is cheapest.
2109	We'll still compute res, but it should be DCEd later. */
2110	use_operand_p use;
2111	gimple *use_stmt;
2112	if (!is_ubsan
2113	&& lhs
2114	&& uns
2115	&& !(uns0_p && uns1_p && !unsr_p)
2116	&& can_mult_highpart_p (mode, uns) == 1
2117	&& single_imm_use (var: lhs, use_p: &use, stmt: &use_stmt)
2118	&& is_gimple_assign (gs: use_stmt)
2119	&& gimple_assign_rhs_code (gs: use_stmt) == IMAGPART_EXPR)
2120	goto highpart;
2121
2122	if (GET_MODE_2XWIDER_MODE (m: mode).exists (mode: &wmode)
2123	&& targetm.scalar_mode_supported_p (wmode)
2124	&& can_widen_mult_without_libcall (wmode, mode, op0, op1, uns))
2125	{
2126	twoxwider:
2127	ops.code = WIDEN_MULT_EXPR;
2128	ops.type
2129	= build_nonstandard_integer_type (GET_MODE_PRECISION (mode: wmode), uns);
2130
2131	res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
2132	rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
2133	NULL_RTX, uns);
2134	hipart = convert_modes (mode, oldmode: wmode, x: hipart, unsignedp: uns);
2135	res = convert_modes (mode, oldmode: wmode, x: res, unsignedp: uns);
2136	if (uns)
2137	/* For the unsigned multiplication, there was overflow if
2138	HIPART is non-zero. */
2139	do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
2140	NULL_RTX, NULL, done_label,
2141	profile_probability::very_likely ());
2142	else
2143	{
2144	/* RES is used more than once, place it in a pseudo. */
2145	res = force_reg (mode, res);
2146
2147	rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
2148	NULL_RTX, 0);
2149	/* RES is low half of the double width result, HIPART
2150	the high half. There was overflow if
2151	HIPART is different from RES < 0 ? -1 : 0. */
2152	do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
2153	NULL_RTX, NULL, done_label,
2154	profile_probability::very_likely ());
2155	}
2156	}
2157	else if (can_mult_highpart_p (mode, uns) == 1)
2158	{
2159	highpart:
2160	ops.code = MULT_HIGHPART_EXPR;
2161	ops.type = type;
2162
2163	rtx hipart = expand_expr_real_2 (&ops, NULL_RTX, mode,
2164	EXPAND_NORMAL);
2165	ops.code = MULT_EXPR;
2166	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2167	if (uns)
2168	/* For the unsigned multiplication, there was overflow if
2169	HIPART is non-zero. */
2170	do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
2171	NULL_RTX, NULL, done_label,
2172	profile_probability::very_likely ());
2173	else
2174	{
2175	rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
2176	NULL_RTX, 0);
2177	/* RES is low half of the double width result, HIPART
2178	the high half. There was overflow if
2179	HIPART is different from RES < 0 ? -1 : 0. */
2180	do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
2181	NULL_RTX, NULL, done_label,
2182	profile_probability::very_likely ());
2183	}
2184
2185	}
2186	else if (int_mode_for_size (size: prec / 2, limit: 1).exists (mode: &hmode)
2187	&& 2 * GET_MODE_PRECISION (mode: hmode) == prec)
2188	{
2189	rtx_code_label *large_op0 = gen_label_rtx ();
2190	rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
2191	rtx_code_label *one_small_one_large = gen_label_rtx ();
2192	rtx_code_label *both_ops_large = gen_label_rtx ();
2193	rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
2194	rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
2195	rtx_code_label *do_overflow = gen_label_rtx ();
2196	rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
2197
2198	unsigned int hprec = GET_MODE_PRECISION (mode: hmode);
2199	rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
2200	NULL_RTX, uns);
2201	hipart0 = convert_modes (mode: hmode, oldmode: mode, x: hipart0, unsignedp: uns);
2202	rtx lopart0 = convert_modes (mode: hmode, oldmode: mode, x: op0, unsignedp: uns);
2203	rtx signbit0 = const0_rtx;
2204	if (!uns)
2205	signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
2206	NULL_RTX, 0);
2207	rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
2208	NULL_RTX, uns);
2209	hipart1 = convert_modes (mode: hmode, oldmode: mode, x: hipart1, unsignedp: uns);
2210	rtx lopart1 = convert_modes (mode: hmode, oldmode: mode, x: op1, unsignedp: uns);
2211	rtx signbit1 = const0_rtx;
2212	if (!uns)
2213	signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
2214	NULL_RTX, 0);
2215
2216	res = gen_reg_rtx (mode);
2217
2218	/* True if op0 resp. op1 are known to be in the range of
2219	halfstype. */
2220	bool op0_small_p = false;
2221	bool op1_small_p = false;
2222	/* True if op0 resp. op1 are known to have all zeros or all ones
2223	in the upper half of bits, but are not known to be
2224	op{0,1}_small_p. */
2225	bool op0_medium_p = false;
2226	bool op1_medium_p = false;
2227	/* -1 if op{0,1} is known to be negative, 0 if it is known to be
2228	nonnegative, 1 if unknown. */
2229	int op0_sign = 1;
2230	int op1_sign = 1;
2231
2232	if (pos_neg0 == 1)
2233	op0_sign = 0;
2234	else if (pos_neg0 == 2)
2235	op0_sign = -1;
2236	if (pos_neg1 == 1)
2237	op1_sign = 0;
2238	else if (pos_neg1 == 2)
2239	op1_sign = -1;
2240
2241	unsigned int mprec0 = prec;
2242	if (arg0 != error_mark_node)
2243	mprec0 = get_min_precision (arg: arg0, sign);
2244	if (mprec0 <= hprec)
2245	op0_small_p = true;
2246	else if (!uns && mprec0 <= hprec + 1)
2247	op0_medium_p = true;
2248	unsigned int mprec1 = prec;
2249	if (arg1 != error_mark_node)
2250	mprec1 = get_min_precision (arg: arg1, sign);
2251	if (mprec1 <= hprec)
2252	op1_small_p = true;
2253	else if (!uns && mprec1 <= hprec + 1)
2254	op1_medium_p = true;
2255
2256	int smaller_sign = 1;
2257	int larger_sign = 1;
2258	if (op0_small_p)
2259	{
2260	smaller_sign = op0_sign;
2261	larger_sign = op1_sign;
2262	}
2263	else if (op1_small_p)
2264	{
2265	smaller_sign = op1_sign;
2266	larger_sign = op0_sign;
2267	}
2268	else if (op0_sign == op1_sign)
2269	{
2270	smaller_sign = op0_sign;
2271	larger_sign = op0_sign;
2272	}
2273
2274	if (!op0_small_p)
2275	do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
2276	NULL_RTX, NULL, large_op0,
2277	profile_probability::unlikely ());
2278
2279	if (!op1_small_p)
2280	do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
2281	NULL_RTX, NULL, small_op0_large_op1,
2282	profile_probability::unlikely ());
2283
2284	/* If both op0 and op1 are sign (!uns) or zero (uns) extended from
2285	hmode to mode, the multiplication will never overflow. We can
2286	do just one hmode x hmode => mode widening multiplication. */
2287	tree halfstype = build_nonstandard_integer_type (hprec, uns);
2288	ops.op0 = make_tree (halfstype, lopart0);
2289	ops.op1 = make_tree (halfstype, lopart1);
2290	ops.code = WIDEN_MULT_EXPR;
2291	ops.type = type;
2292	rtx thisres
2293	= expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2294	emit_move_insn (res, thisres);
2295	emit_jump (done_label);
2296
2297	emit_label (small_op0_large_op1);
2298
2299	/* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
2300	but op1 is not, just swap the arguments and handle it as op1
2301	sign/zero extended, op0 not. */
2302	rtx larger = gen_reg_rtx (mode);
2303	rtx hipart = gen_reg_rtx (hmode);
2304	rtx lopart = gen_reg_rtx (hmode);
2305	emit_move_insn (larger, op1);
2306	emit_move_insn (hipart, hipart1);
2307	emit_move_insn (lopart, lopart0);
2308	emit_jump (one_small_one_large);
2309
2310	emit_label (large_op0);
2311
2312	if (!op1_small_p)
2313	do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
2314	NULL_RTX, NULL, both_ops_large,
2315	profile_probability::unlikely ());
2316
2317	/* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2318	but op0 is not, prepare larger, hipart and lopart pseudos and
2319	handle it together with small_op0_large_op1. */
2320	emit_move_insn (larger, op0);
2321	emit_move_insn (hipart, hipart0);
2322	emit_move_insn (lopart, lopart1);
2323
2324	emit_label (one_small_one_large);
2325
2326	/* lopart is the low part of the operand that is sign extended
2327	to mode, larger is the other operand, hipart is the
2328	high part of larger and lopart0 and lopart1 are the low parts
2329	of both operands.
2330	We perform lopart0 * lopart1 and lopart * hipart widening
2331	multiplications. */
2332	tree halfutype = build_nonstandard_integer_type (hprec, 1);
2333	ops.op0 = make_tree (halfutype, lopart0);
2334	ops.op1 = make_tree (halfutype, lopart1);
2335	rtx lo0xlo1
2336	= expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2337
2338	ops.op0 = make_tree (halfutype, lopart);
2339	ops.op1 = make_tree (halfutype, hipart);
2340	rtx loxhi = gen_reg_rtx (mode);
2341	rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2342	emit_move_insn (loxhi, tem);
2343
2344	if (!uns)
2345	{
2346	/* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2347	if (larger_sign == 0)
2348	emit_jump (after_hipart_neg);
2349	else if (larger_sign != -1)
2350	do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
2351	NULL_RTX, NULL, after_hipart_neg,
2352	profile_probability::even ());
2353
2354	tem = convert_modes (mode, oldmode: hmode, x: lopart, unsignedp: 1);
2355	tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
2356	tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
2357	1, OPTAB_WIDEN);
2358	emit_move_insn (loxhi, tem);
2359
2360	emit_label (after_hipart_neg);
2361
2362	/* if (lopart < 0) loxhi -= larger; */
2363	if (smaller_sign == 0)
2364	emit_jump (after_lopart_neg);
2365	else if (smaller_sign != -1)
2366	do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
2367	NULL_RTX, NULL, after_lopart_neg,
2368	profile_probability::even ());
2369
2370	tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
2371	1, OPTAB_WIDEN);
2372	emit_move_insn (loxhi, tem);
2373
2374	emit_label (after_lopart_neg);
2375	}
2376
2377	/* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2378	tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
2379	tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
2380	1, OPTAB_WIDEN);
2381	emit_move_insn (loxhi, tem);
2382
2383	/* if (loxhi >> (bitsize / 2)
2384	== (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2385	if (loxhi >> (bitsize / 2) == 0 (if uns). */
2386	rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
2387	NULL_RTX, 0);
2388	hipartloxhi = convert_modes (mode: hmode, oldmode: mode, x: hipartloxhi, unsignedp: 0);
2389	rtx signbitloxhi = const0_rtx;
2390	if (!uns)
2391	signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
2392	convert_modes (mode: hmode, oldmode: mode,
2393	x: loxhi, unsignedp: 0),
2394	hprec - 1, NULL_RTX, 0);
2395
2396	do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
2397	NULL_RTX, NULL, do_overflow,
2398	profile_probability::very_unlikely ());
2399
2400	/* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2401	rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
2402	NULL_RTX, 1);
2403	tem = convert_modes (mode, oldmode: hmode,
2404	x: convert_modes (mode: hmode, oldmode: mode, x: lo0xlo1, unsignedp: 1), unsignedp: 1);
2405
2406	tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
2407	1, OPTAB_WIDEN);
2408	if (tem != res)
2409	emit_move_insn (res, tem);
2410	emit_jump (done_label);
2411
2412	emit_label (both_ops_large);
2413
2414	/* If both operands are large (not sign (!uns) or zero (uns)
2415	extended from hmode), then perform the full multiplication
2416	which will be the result of the operation.
2417	The only cases which don't overflow are for signed multiplication
2418	some cases where both hipart0 and highpart1 are 0 or -1.
2419	For unsigned multiplication when high parts are both non-zero
2420	this overflows always. */
2421	ops.code = MULT_EXPR;
2422	ops.op0 = make_tree (type, op0);
2423	ops.op1 = make_tree (type, op1);
2424	tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2425	emit_move_insn (res, tem);
2426
2427	if (!uns)
2428	{
2429	if (!op0_medium_p)
2430	{
2431	tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
2432	NULL_RTX, 1, OPTAB_WIDEN);
2433	do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2434	NULL_RTX, NULL, do_error,
2435	profile_probability::very_unlikely ());
2436	}
2437
2438	if (!op1_medium_p)
2439	{
2440	tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
2441	NULL_RTX, 1, OPTAB_WIDEN);
2442	do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2443	NULL_RTX, NULL, do_error,
2444	profile_probability::very_unlikely ());
2445	}
2446
2447	/* At this point hipart{0,1} are both in [-1, 0]. If they are
2448	the same, overflow happened if res is non-positive, if they
2449	are different, overflow happened if res is positive. */
2450	if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
2451	emit_jump (hipart_different);
2452	else if (op0_sign == 1 || op1_sign == 1)
2453	do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
2454	NULL_RTX, NULL, hipart_different,
2455	profile_probability::even ());
2456
2457	do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
2458	NULL_RTX, NULL, do_error,
2459	profile_probability::very_unlikely ());
2460	emit_jump (done_label);
2461
2462	emit_label (hipart_different);
2463
2464	do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
2465	NULL_RTX, NULL, do_error,
2466	profile_probability::very_unlikely ());
2467	emit_jump (done_label);
2468	}
2469
2470	emit_label (do_overflow);
2471
2472	/* Overflow, do full multiplication and fallthru into do_error. */
2473	ops.op0 = make_tree (type, op0);
2474	ops.op1 = make_tree (type, op1);
2475	tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2476	emit_move_insn (res, tem);
2477	}
2478	else if (GET_MODE_2XWIDER_MODE (m: mode).exists (mode: &wmode)
2479	&& targetm.scalar_mode_supported_p (wmode))
2480	/* Even emitting a libcall is better than not detecting overflow
2481	at all. */
2482	goto twoxwider;
2483	else
2484	{
2485	gcc_assert (!is_ubsan);
2486	ops.code = MULT_EXPR;
2487	ops.type = type;
2488	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2489	emit_jump (done_label);
2490	}
2491	}
2492
2493	do_error_label:
2494	emit_label (do_error);
2495	if (is_ubsan)
2496	{
2497	/* Expand the ubsan builtin call. */
2498	push_temp_slots ();
2499	fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
2500	arg0, arg1, datap);
2501	expand_normal (exp: fn);
2502	pop_temp_slots ();
2503	do_pending_stack_adjust ();
2504	}
2505	else if (lhs)
2506	expand_arith_set_overflow (lhs, target);
2507
2508	/* We're done. */
2509	emit_label (done_label);
2510
2511	/* u1 * u2 -> sr */
2512	if (uns0_p && uns1_p && !unsr_p)
2513	{
2514	rtx_code_label *all_done_label = gen_label_rtx ();
2515	do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
2516	NULL, all_done_label, profile_probability::very_likely ());
2517	expand_arith_set_overflow (lhs, target);
2518	emit_label (all_done_label);
2519	}
2520
2521	/* s1 * u2 -> sr */
2522	if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
2523	{
2524	rtx_code_label *all_done_label = gen_label_rtx ();
2525	rtx_code_label *set_noovf = gen_label_rtx ();
2526	do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
2527	NULL, all_done_label, profile_probability::very_likely ());
2528	expand_arith_set_overflow (lhs, target);
2529	do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
2530	NULL, set_noovf, profile_probability::very_likely ());
2531	do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
2532	NULL, all_done_label, profile_probability::very_unlikely ());
2533	do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
2534	all_done_label, profile_probability::very_unlikely ());
2535	emit_label (set_noovf);
2536	write_complex_part (target, const0_rtx, true, false);
2537	emit_label (all_done_label);
2538	}
2539
2540	if (lhs)
2541	{
2542	if (is_ubsan)
2543	expand_ubsan_result_store (lhs, target, mode, res, do_error);
2544	else
2545	expand_arith_overflow_result_store (lhs, target, mode, res);
2546	}
2547	flag_trapv = save_flag_trapv;
2548	}
2549
2550	/* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2551
2552	static void
2553	expand_vector_ubsan_overflow (location_t loc, enum tree_code code, tree lhs,
2554	tree arg0, tree arg1)
2555	{
2556	poly_uint64 cnt = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
2557	rtx_code_label *loop_lab = NULL;
2558	rtx cntvar = NULL_RTX;
2559	tree cntv = NULL_TREE;
2560	tree eltype = TREE_TYPE (TREE_TYPE (arg0));
2561	tree sz = TYPE_SIZE (eltype);
2562	tree data = NULL_TREE;
2563	tree resv = NULL_TREE;
2564	rtx lhsr = NULL_RTX;
2565	rtx resvr = NULL_RTX;
2566	unsigned HOST_WIDE_INT const_cnt = 0;
2567	bool use_loop_p = (!cnt.is_constant (const_value: &const_cnt) || const_cnt > 4);
2568	int save_flag_trapv = flag_trapv;
2569
2570	/* We don't want any __mulv?i3 etc. calls from the expansion of
2571	these internal functions, so disable -ftrapv temporarily. */
2572	flag_trapv = 0;
2573	if (lhs)
2574	{
2575	optab op;
2576	lhsr = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
2577	if (!VECTOR_MODE_P (GET_MODE (lhsr))
2578	|| (op = optab_for_tree_code (code, TREE_TYPE (arg0),
2579	optab_default)) == unknown_optab
2580	|| (optab_handler (op, TYPE_MODE (TREE_TYPE (arg0)))
2581	== CODE_FOR_nothing))
2582	{
2583	if (MEM_P (lhsr))
2584	resv = make_tree (TREE_TYPE (lhs), lhsr);
2585	else
2586	{
2587	resvr = assign_temp (TREE_TYPE (lhs), 1, 1);
2588	resv = make_tree (TREE_TYPE (lhs), resvr);
2589	}
2590	}
2591	}
2592	if (use_loop_p)
2593	{
2594	do_pending_stack_adjust ();
2595	loop_lab = gen_label_rtx ();
2596	cntvar = gen_reg_rtx (TYPE_MODE (sizetype));
2597	cntv = make_tree (sizetype, cntvar);
2598	emit_move_insn (cntvar, const0_rtx);
2599	emit_label (loop_lab);
2600	}
2601	if (TREE_CODE (arg0) != VECTOR_CST)
2602	{
2603	rtx arg0r = expand_normal (exp: arg0);
2604	arg0 = make_tree (TREE_TYPE (arg0), arg0r);
2605	}
2606	if (TREE_CODE (arg1) != VECTOR_CST)
2607	{
2608	rtx arg1r = expand_normal (exp: arg1);
2609	arg1 = make_tree (TREE_TYPE (arg1), arg1r);
2610	}
2611	for (unsigned int i = 0; i < (use_loop_p ? 1 : const_cnt); i++)
2612	{
2613	tree op0, op1, res = NULL_TREE;
2614	if (use_loop_p)
2615	{
2616	tree atype = build_array_type_nelts (eltype, cnt);
2617	op0 = uniform_vector_p (arg0);
2618	if (op0 == NULL_TREE)
2619	{
2620	op0 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg0);
2621	op0 = build4_loc (loc, code: ARRAY_REF, type: eltype, arg0: op0, arg1: cntv,
2622	NULL_TREE, NULL_TREE);
2623	}
2624	op1 = uniform_vector_p (arg1);
2625	if (op1 == NULL_TREE)
2626	{
2627	op1 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg1);
2628	op1 = build4_loc (loc, code: ARRAY_REF, type: eltype, arg0: op1, arg1: cntv,
2629	NULL_TREE, NULL_TREE);
2630	}
2631	if (resv)
2632	{
2633	res = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, resv);
2634	res = build4_loc (loc, code: ARRAY_REF, type: eltype, arg0: res, arg1: cntv,
2635	NULL_TREE, NULL_TREE);
2636	}
2637	}
2638	else
2639	{
2640	tree bitpos = bitsize_int (tree_to_uhwi (sz) * i);
2641	op0 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg0, sz, bitpos);
2642	op1 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg1, sz, bitpos);
2643	if (resv)
2644	res = fold_build3_loc (loc, BIT_FIELD_REF, eltype, resv, sz,
2645	bitpos);
2646	}
2647	switch (code)
2648	{
2649	case PLUS_EXPR:
2650	expand_addsub_overflow (loc, code: PLUS_EXPR, lhs: res, arg0: op0, arg1: op1,
2651	unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true, datap: &data);
2652	break;
2653	case MINUS_EXPR:
2654	if (use_loop_p ? integer_zerop (arg0) : integer_zerop (op0))
2655	expand_neg_overflow (loc, lhs: res, arg1: op1, is_ubsan: true, datap: &data);
2656	else
2657	expand_addsub_overflow (loc, code: MINUS_EXPR, lhs: res, arg0: op0, arg1: op1,
2658	unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true, datap: &data);
2659	break;
2660	case MULT_EXPR:
2661	expand_mul_overflow (loc, lhs: res, arg0: op0, arg1: op1, unsr_p: false, uns0_p: false, uns1_p: false,
2662	is_ubsan: true, datap: &data);
2663	break;
2664	default:
2665	gcc_unreachable ();
2666	}
2667	}
2668	if (use_loop_p)
2669	{
2670	struct separate_ops ops;
2671	ops.code = PLUS_EXPR;
2672	ops.type = TREE_TYPE (cntv);
2673	ops.op0 = cntv;
2674	ops.op1 = build_int_cst (TREE_TYPE (cntv), 1);
2675	ops.op2 = NULL_TREE;
2676	ops.location = loc;
2677	rtx ret = expand_expr_real_2 (&ops, cntvar, TYPE_MODE (sizetype),
2678	EXPAND_NORMAL);
2679	if (ret != cntvar)
2680	emit_move_insn (cntvar, ret);
2681	rtx cntrtx = gen_int_mode (cnt, TYPE_MODE (sizetype));
2682	do_compare_rtx_and_jump (cntvar, cntrtx, NE, false,
2683	TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab,
2684	profile_probability::very_likely ());
2685	}
2686	if (lhs && resv == NULL_TREE)
2687	{
2688	struct separate_ops ops;
2689	ops.code = code;
2690	ops.type = TREE_TYPE (arg0);
2691	ops.op0 = arg0;
2692	ops.op1 = arg1;
2693	ops.op2 = NULL_TREE;
2694	ops.location = loc;
2695	rtx ret = expand_expr_real_2 (&ops, lhsr, TYPE_MODE (TREE_TYPE (arg0)),
2696	EXPAND_NORMAL);
2697	if (ret != lhsr)
2698	emit_move_insn (lhsr, ret);
2699	}
2700	else if (resvr)
2701	emit_move_insn (lhsr, resvr);
2702	flag_trapv = save_flag_trapv;
2703	}
2704
2705	/* Expand UBSAN_CHECK_ADD call STMT. */
2706
2707	static void
2708	expand_UBSAN_CHECK_ADD (internal_fn, gcall *stmt)
2709	{
2710	location_t loc = gimple_location (g: stmt);
2711	tree lhs = gimple_call_lhs (gs: stmt);
2712	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
2713	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
2714	if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2715	expand_vector_ubsan_overflow (loc, code: PLUS_EXPR, lhs, arg0, arg1);
2716	else
2717	expand_addsub_overflow (loc, code: PLUS_EXPR, lhs, arg0, arg1,
2718	unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true, NULL);
2719	}
2720
2721	/* Expand UBSAN_CHECK_SUB call STMT. */
2722
2723	static void
2724	expand_UBSAN_CHECK_SUB (internal_fn, gcall *stmt)
2725	{
2726	location_t loc = gimple_location (g: stmt);
2727	tree lhs = gimple_call_lhs (gs: stmt);
2728	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
2729	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
2730	if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2731	expand_vector_ubsan_overflow (loc, code: MINUS_EXPR, lhs, arg0, arg1);
2732	else if (integer_zerop (arg0))
2733	expand_neg_overflow (loc, lhs, arg1, is_ubsan: true, NULL);
2734	else
2735	expand_addsub_overflow (loc, code: MINUS_EXPR, lhs, arg0, arg1,
2736	unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true, NULL);
2737	}
2738
2739	/* Expand UBSAN_CHECK_MUL call STMT. */
2740
2741	static void
2742	expand_UBSAN_CHECK_MUL (internal_fn, gcall *stmt)
2743	{
2744	location_t loc = gimple_location (g: stmt);
2745	tree lhs = gimple_call_lhs (gs: stmt);
2746	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
2747	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
2748	if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2749	expand_vector_ubsan_overflow (loc, code: MULT_EXPR, lhs, arg0, arg1);
2750	else
2751	expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p: false, uns0_p: false, uns1_p: false, is_ubsan: true,
2752	NULL);
2753	}
2754
2755	/* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2756
2757	static void
2758	expand_arith_overflow (enum tree_code code, gimple *stmt)
2759	{
2760	tree lhs = gimple_call_lhs (gs: stmt);
2761	if (lhs == NULL_TREE)
2762	return;
2763	tree arg0 = gimple_call_arg (gs: stmt, index: 0);
2764	tree arg1 = gimple_call_arg (gs: stmt, index: 1);
2765	tree type = TREE_TYPE (TREE_TYPE (lhs));
2766	int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
2767	int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
2768	int unsr_p = TYPE_UNSIGNED (type);
2769	int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
2770	int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
2771	int precres = TYPE_PRECISION (type);
2772	location_t loc = gimple_location (g: stmt);
2773	if (!uns0_p && get_range_pos_neg (arg0, stmt) == 1)
2774	uns0_p = true;
2775	if (!uns1_p && get_range_pos_neg (arg1, stmt) == 1)
2776	uns1_p = true;
2777	int pr = get_min_precision (arg: arg0, sign: uns0_p ? UNSIGNED : SIGNED);
2778	prec0 = MIN (prec0, pr);
2779	pr = get_min_precision (arg: arg1, sign: uns1_p ? UNSIGNED : SIGNED);
2780	prec1 = MIN (prec1, pr);
2781	int save_flag_trapv = flag_trapv;
2782
2783	/* We don't want any __mulv?i3 etc. calls from the expansion of
2784	these internal functions, so disable -ftrapv temporarily. */
2785	flag_trapv = 0;
2786	/* If uns0_p && uns1_p, precop is minimum needed precision
2787	of unsigned type to hold the exact result, otherwise
2788	precop is minimum needed precision of signed type to
2789	hold the exact result. */
2790	int precop;
2791	if (code == MULT_EXPR)
2792	precop = prec0 + prec1 + (uns0_p != uns1_p);
2793	else
2794	{
2795	if (uns0_p == uns1_p)
2796	precop = MAX (prec0, prec1) + 1;
2797	else if (uns0_p)
2798	precop = MAX (prec0 + 1, prec1) + 1;
2799	else
2800	precop = MAX (prec0, prec1 + 1) + 1;
2801	}
2802	int orig_precres = precres;
2803
2804	do
2805	{
2806	if ((uns0_p && uns1_p)
2807	? ((precop + !unsr_p) <= precres
2808	/* u1 - u2 -> ur can overflow, no matter what precision
2809	the result has. */
2810	&& (code != MINUS_EXPR || !unsr_p))
2811	: (!unsr_p && precop <= precres))
2812	{
2813	/* The infinity precision result will always fit into result. */
2814	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
2815	write_complex_part (target, const0_rtx, true, false);
2816	scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
2817	struct separate_ops ops;
2818	ops.code = code;
2819	ops.type = type;
2820	ops.op0 = fold_convert_loc (loc, type, arg0);
2821	ops.op1 = fold_convert_loc (loc, type, arg1);
2822	ops.op2 = NULL_TREE;
2823	ops.location = loc;
2824	rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2825	expand_arith_overflow_result_store (lhs, target, mode, res: tem);
2826	flag_trapv = save_flag_trapv;
2827	return;
2828	}
2829
2830	/* For operations with low precision, if target doesn't have them, start
2831	with precres widening right away, otherwise do it only if the most
2832	simple cases can't be used. */
2833	const int min_precision = targetm.min_arithmetic_precision ();
2834	if (orig_precres == precres && precres < min_precision)
2835	;
2836	else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
2837	&& prec1 <= precres)
2838	|| ((!uns0_p || !uns1_p) && !unsr_p
2839	&& prec0 + uns0_p <= precres
2840	&& prec1 + uns1_p <= precres))
2841	{
2842	arg0 = fold_convert_loc (loc, type, arg0);
2843	arg1 = fold_convert_loc (loc, type, arg1);
2844	switch (code)
2845	{
2846	case MINUS_EXPR:
2847	if (integer_zerop (arg0) && !unsr_p)
2848	{
2849	expand_neg_overflow (loc, lhs, arg1, is_ubsan: false, NULL);
2850	flag_trapv = save_flag_trapv;
2851	return;
2852	}
2853	/* FALLTHRU */
2854	case PLUS_EXPR:
2855	expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2856	uns0_p: unsr_p, uns1_p: unsr_p, is_ubsan: false, NULL);
2857	flag_trapv = save_flag_trapv;
2858	return;
2859	case MULT_EXPR:
2860	expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2861	uns0_p: unsr_p, uns1_p: unsr_p, is_ubsan: false, NULL);
2862	flag_trapv = save_flag_trapv;
2863	return;
2864	default:
2865	gcc_unreachable ();
2866	}
2867	}
2868
2869	/* For sub-word operations, retry with a wider type first. */
2870	if (orig_precres == precres && precop <= BITS_PER_WORD)
2871	{
2872	int p = MAX (min_precision, precop);
2873	scalar_int_mode m = smallest_int_mode_for_size (size: p).require ();
2874	tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (mode: m),
2875	uns0_p && uns1_p
2876	&& unsr_p);
2877	p = TYPE_PRECISION (optype);
2878	if (p > precres)
2879	{
2880	precres = p;
2881	unsr_p = TYPE_UNSIGNED (optype);
2882	type = optype;
2883	continue;
2884	}
2885	}
2886
2887	if (prec0 <= precres && prec1 <= precres)
2888	{
2889	tree types[2];
2890	if (unsr_p)
2891	{
2892	types[0] = build_nonstandard_integer_type (precres, 0);
2893	types[1] = type;
2894	}
2895	else
2896	{
2897	types[0] = type;
2898	types[1] = build_nonstandard_integer_type (precres, 1);
2899	}
2900	arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
2901	arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
2902	if (code != MULT_EXPR)
2903	expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2904	uns0_p, uns1_p, is_ubsan: false, NULL);
2905	else
2906	expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2907	uns0_p, uns1_p, is_ubsan: false, NULL);
2908	flag_trapv = save_flag_trapv;
2909	return;
2910	}
2911
2912	/* Retry with a wider type. */
2913	if (orig_precres == precres)
2914	{
2915	int p = MAX (prec0, prec1);
2916	scalar_int_mode m = smallest_int_mode_for_size (size: p).require ();
2917	tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (mode: m),
2918	uns0_p && uns1_p
2919	&& unsr_p);
2920	p = TYPE_PRECISION (optype);
2921	if (p > precres)
2922	{
2923	precres = p;
2924	unsr_p = TYPE_UNSIGNED (optype);
2925	type = optype;
2926	continue;
2927	}
2928	}
2929
2930	gcc_unreachable ();
2931	}
2932	while (1);
2933	}
2934
2935	/* Expand ADD_OVERFLOW STMT. */
2936
2937	static void
2938	expand_ADD_OVERFLOW (internal_fn, gcall *stmt)
2939	{
2940	expand_arith_overflow (code: PLUS_EXPR, stmt);
2941	}
2942
2943	/* Expand SUB_OVERFLOW STMT. */
2944
2945	static void
2946	expand_SUB_OVERFLOW (internal_fn, gcall *stmt)
2947	{
2948	expand_arith_overflow (code: MINUS_EXPR, stmt);
2949	}
2950
2951	/* Expand MUL_OVERFLOW STMT. */
2952
2953	static void
2954	expand_MUL_OVERFLOW (internal_fn, gcall *stmt)
2955	{
2956	expand_arith_overflow (code: MULT_EXPR, stmt);
2957	}
2958
2959	/* Expand UADDC STMT. */
2960
2961	static void
2962	expand_UADDC (internal_fn ifn, gcall *stmt)
2963	{
2964	tree lhs = gimple_call_lhs (gs: stmt);
2965	tree arg1 = gimple_call_arg (gs: stmt, index: 0);
2966	tree arg2 = gimple_call_arg (gs: stmt, index: 1);
2967	tree arg3 = gimple_call_arg (gs: stmt, index: 2);
2968	tree type = TREE_TYPE (arg1);
2969	machine_mode mode = TYPE_MODE (type);
2970	insn_code icode = optab_handler (op: ifn == IFN_UADDC
2971	? uaddc5_optab : usubc5_optab, mode);
2972	rtx op1 = expand_normal (exp: arg1);
2973	rtx op2 = expand_normal (exp: arg2);
2974	rtx op3 = expand_normal (exp: arg3);
2975	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
2976	rtx re = gen_reg_rtx (mode);
2977	rtx im = gen_reg_rtx (mode);
2978	class expand_operand ops[5];
2979	create_output_operand (op: &ops[0], x: re, mode);
2980	create_output_operand (op: &ops[1], x: im, mode);
2981	create_input_operand (op: &ops[2], value: op1, mode);
2982	create_input_operand (op: &ops[3], value: op2, mode);
2983	create_input_operand (op: &ops[4], value: op3, mode);
2984	expand_insn (icode, nops: 5, ops);
2985	write_complex_part (target, re, false, false);
2986	write_complex_part (target, im, true, false);
2987	}
2988
2989	/* Expand USUBC STMT. */
2990
2991	static void
2992	expand_USUBC (internal_fn ifn, gcall *stmt)
2993	{
2994	expand_UADDC (ifn, stmt);
2995	}
2996
2997	/* This should get folded in tree-vectorizer.cc. */
2998
2999	static void
3000	expand_LOOP_VECTORIZED (internal_fn, gcall *)
3001	{
3002	gcc_unreachable ();
3003	}
3004
3005	/* This should get folded in tree-vectorizer.cc. */
3006
3007	static void
3008	expand_LOOP_DIST_ALIAS (internal_fn, gcall *)
3009	{
3010	gcc_unreachable ();
3011	}
3012
3013	/* Return a memory reference of type TYPE for argument INDEX of STMT.
3014	Use argument INDEX + 1 to derive the second (TBAA) operand. */
3015
3016	static tree
3017	expand_call_mem_ref (tree type, gcall *stmt, int index)
3018	{
3019	tree addr = gimple_call_arg (gs: stmt, index);
3020	tree alias_ptr_type = TREE_TYPE (gimple_call_arg (stmt, index + 1));
3021	unsigned int align = tree_to_shwi (gimple_call_arg (gs: stmt, index: index + 1));
3022	if (TYPE_ALIGN (type) != align)
3023	type = build_aligned_type (type, align);
3024
3025	tree tmp = addr;
3026	if (TREE_CODE (tmp) == SSA_NAME)
3027	{
3028	gimple *def = get_gimple_for_ssa_name (exp: tmp);
3029	if (def && gimple_assign_single_p (gs: def))
3030	tmp = gimple_assign_rhs1 (gs: def);
3031	}
3032
3033	if (TREE_CODE (tmp) == ADDR_EXPR)
3034	{
3035	tree mem = TREE_OPERAND (tmp, 0);
3036	if (TREE_CODE (mem) == TARGET_MEM_REF
3037	&& types_compatible_p (TREE_TYPE (mem), type2: type))
3038	{
3039	tree offset = TMR_OFFSET (mem);
3040	if (type != TREE_TYPE (mem)
3041	|| alias_ptr_type != TREE_TYPE (offset)
3042	|| !integer_zerop (offset))
3043	{
3044	mem = copy_node (mem);
3045	TMR_OFFSET (mem) = wide_int_to_tree (type: alias_ptr_type,
3046	cst: wi::to_poly_wide (t: offset));
3047	TREE_TYPE (mem) = type;
3048	}
3049	return mem;
3050	}
3051	}
3052
3053	return fold_build2 (MEM_REF, type, addr, build_int_cst (alias_ptr_type, 0));
3054	}
3055
3056	/* Expand MASK_LOAD{,_LANES}, MASK_LEN_LOAD or LEN_LOAD call STMT using optab
3057	* OPTAB. */
3058
3059	static void
3060	expand_partial_load_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab)
3061	{
3062	int i = 0;
3063	class expand_operand ops[6];
3064	tree type, lhs, rhs, maskt;
3065	rtx mem, target;
3066	insn_code icode;
3067
3068	maskt = gimple_call_arg (gs: stmt, index: internal_fn_mask_index (ifn));
3069	lhs = gimple_call_lhs (gs: stmt);
3070	if (lhs == NULL_TREE)
3071	return;
3072	type = TREE_TYPE (lhs);
3073	rhs = expand_call_mem_ref (type, stmt, index: 0);
3074
3075	if (optab == vec_mask_load_lanes_optab
3076	|| optab == vec_mask_len_load_lanes_optab)
3077	icode = get_multi_vector_move (array_type: type, optab);
3078	else if (optab == len_load_optab)
3079	icode = direct_optab_handler (op: optab, TYPE_MODE (type));
3080	else
3081	icode = convert_optab_handler (op: optab, TYPE_MODE (type),
3082	TYPE_MODE (TREE_TYPE (maskt)));
3083
3084	mem = expand_expr (exp: rhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3085	gcc_assert (MEM_P (mem));
3086	/* The built MEM_REF does not accurately reflect that the load
3087	is only partial. Clear it. */
3088	set_mem_expr (mem, NULL_TREE);
3089	clear_mem_offset (mem);
3090	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3091	create_call_lhs_operand (op: &ops[i++], lhs_rtx: target, TYPE_MODE (type));
3092	create_fixed_operand (op: &ops[i++], x: mem);
3093	i = add_mask_else_and_len_args (ops, opno: i, stmt);
3094	expand_insn (icode, nops: i, ops);
3095
3096	assign_call_lhs (lhs, lhs_rtx: target, op: &ops[0]);
3097	}
3098
3099	#define expand_mask_load_optab_fn expand_partial_load_optab_fn
3100	#define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
3101	#define expand_len_load_optab_fn expand_partial_load_optab_fn
3102	#define expand_mask_len_load_optab_fn expand_partial_load_optab_fn
3103
3104	/* Expand MASK_STORE{,_LANES}, MASK_LEN_STORE or LEN_STORE call STMT using optab
3105	* OPTAB. */
3106
3107	static void
3108	expand_partial_store_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab)
3109	{
3110	int i = 0;
3111	class expand_operand ops[5];
3112	tree type, lhs, rhs, maskt;
3113	rtx mem, reg;
3114	insn_code icode;
3115
3116	maskt = gimple_call_arg (gs: stmt, index: internal_fn_mask_index (ifn));
3117	rhs = gimple_call_arg (gs: stmt, index: internal_fn_stored_value_index (ifn));
3118	type = TREE_TYPE (rhs);
3119	lhs = expand_call_mem_ref (type, stmt, index: 0);
3120
3121	if (optab == vec_mask_store_lanes_optab
3122	|| optab == vec_mask_len_store_lanes_optab)
3123	icode = get_multi_vector_move (array_type: type, optab);
3124	else if (optab == len_store_optab)
3125	icode = direct_optab_handler (op: optab, TYPE_MODE (type));
3126	else
3127	icode = convert_optab_handler (op: optab, TYPE_MODE (type),
3128	TYPE_MODE (TREE_TYPE (maskt)));
3129
3130	mem = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3131	gcc_assert (MEM_P (mem));
3132	/* The built MEM_REF does not accurately reflect that the store
3133	is only partial. Clear it. */
3134	set_mem_expr (mem, NULL_TREE);
3135	clear_mem_offset (mem);
3136	reg = expand_normal (exp: rhs);
3137	create_fixed_operand (op: &ops[i++], x: mem);
3138	create_input_operand (op: &ops[i++], value: reg, TYPE_MODE (type));
3139	i = add_mask_else_and_len_args (ops, opno: i, stmt);
3140	expand_insn (icode, nops: i, ops);
3141	}
3142
3143	#define expand_mask_store_optab_fn expand_partial_store_optab_fn
3144	#define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
3145	#define expand_len_store_optab_fn expand_partial_store_optab_fn
3146	#define expand_mask_len_store_optab_fn expand_partial_store_optab_fn
3147
3148	/* Expand VCOND_MASK optab internal function.
3149	The expansion of STMT happens based on OPTAB table associated. */
3150
3151	static void
3152	expand_vec_cond_mask_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3153	{
3154	class expand_operand ops[4];
3155
3156	tree lhs = gimple_call_lhs (gs: stmt);
3157	tree op0 = gimple_call_arg (gs: stmt, index: 0);
3158	tree op1 = gimple_call_arg (gs: stmt, index: 1);
3159	tree op2 = gimple_call_arg (gs: stmt, index: 2);
3160	tree vec_cond_type = TREE_TYPE (lhs);
3161
3162	machine_mode mode = TYPE_MODE (vec_cond_type);
3163	machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
3164	enum insn_code icode = convert_optab_handler (op: optab, to_mode: mode, from_mode: mask_mode);
3165	rtx mask, rtx_op1, rtx_op2;
3166
3167	gcc_assert (icode != CODE_FOR_nothing);
3168
3169	mask = expand_normal (exp: op0);
3170	rtx_op1 = expand_normal (exp: op1);
3171	rtx_op2 = expand_normal (exp: op2);
3172
3173	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3174	create_call_lhs_operand (op: &ops[0], lhs_rtx: target, mode);
3175	create_input_operand (op: &ops[1], value: rtx_op1, mode);
3176	create_input_operand (op: &ops[2], value: rtx_op2, mode);
3177	create_input_operand (op: &ops[3], value: mask, mode: mask_mode);
3178	expand_insn (icode, nops: 4, ops);
3179	assign_call_lhs (lhs, lhs_rtx: target, op: &ops[0]);
3180	}
3181
3182	/* Expand VEC_SET internal functions. */
3183
3184	static void
3185	expand_vec_set_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3186	{
3187	tree lhs = gimple_call_lhs (gs: stmt);
3188	tree op0 = gimple_call_arg (gs: stmt, index: 0);
3189	tree op1 = gimple_call_arg (gs: stmt, index: 1);
3190	tree op2 = gimple_call_arg (gs: stmt, index: 2);
3191	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3192	rtx src = expand_normal (exp: op0);
3193
3194	machine_mode outermode = TYPE_MODE (TREE_TYPE (op0));
3195	scalar_mode innermode = GET_MODE_INNER (outermode);
3196
3197	rtx value = expand_normal (exp: op1);
3198	rtx pos = expand_normal (exp: op2);
3199
3200	class expand_operand ops[3];
3201	enum insn_code icode = optab_handler (op: optab, mode: outermode);
3202
3203	if (icode != CODE_FOR_nothing)
3204	{
3205	rtx temp = gen_reg_rtx (outermode);
3206	emit_move_insn (temp, src);
3207
3208	create_fixed_operand (op: &ops[0], x: temp);
3209	create_input_operand (op: &ops[1], value, mode: innermode);
3210	create_convert_operand_from (op: &ops[2], value: pos, TYPE_MODE (TREE_TYPE (op2)),
3211	unsigned_p: true);
3212	if (maybe_expand_insn (icode, nops: 3, ops))
3213	{
3214	emit_move_insn (target, temp);
3215	return;
3216	}
3217	}
3218	gcc_unreachable ();
3219	}
3220
3221	static void
3222	expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
3223	{
3224	}
3225
3226	static void
3227	expand_BUILTIN_EXPECT (internal_fn, gcall *stmt)
3228	{
3229	/* When guessing was done, the hints should be already stripped away. */
3230	gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
3231
3232	rtx target;
3233	tree lhs = gimple_call_lhs (gs: stmt);
3234	if (lhs)
3235	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3236	else
3237	target = const0_rtx;
3238	rtx val = expand_expr (exp: gimple_call_arg (gs: stmt, index: 0), target, VOIDmode, modifier: EXPAND_NORMAL);
3239	if (lhs && val != target)
3240	emit_move_insn (target, val);
3241	}
3242
3243	/* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
3244	should never be called. */
3245
3246	static void
3247	expand_VA_ARG (internal_fn, gcall *)
3248	{
3249	gcc_unreachable ();
3250	}
3251
3252	/* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
3253	dummy function should never be called. */
3254
3255	static void
3256	expand_VEC_CONVERT (internal_fn, gcall *)
3257	{
3258	gcc_unreachable ();
3259	}
3260
3261	/* Expand IFN_RAWMEMCHR internal function. */
3262
3263	void
3264	expand_RAWMEMCHR (internal_fn, gcall *stmt)
3265	{
3266	expand_operand ops[3];
3267
3268	tree lhs = gimple_call_lhs (gs: stmt);
3269	if (!lhs)
3270	return;
3271	machine_mode lhs_mode = TYPE_MODE (TREE_TYPE (lhs));
3272	rtx lhs_rtx = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3273	create_call_lhs_operand (op: &ops[0], lhs_rtx, mode: lhs_mode);
3274
3275	tree mem = gimple_call_arg (gs: stmt, index: 0);
3276	rtx mem_rtx = get_memory_rtx (exp: mem, NULL);
3277	create_fixed_operand (op: &ops[1], x: mem_rtx);
3278
3279	tree pattern = gimple_call_arg (gs: stmt, index: 1);
3280	machine_mode mode = TYPE_MODE (TREE_TYPE (pattern));
3281	rtx pattern_rtx = expand_normal (exp: pattern);
3282	create_input_operand (op: &ops[2], value: pattern_rtx, mode);
3283
3284	insn_code icode = direct_optab_handler (op: rawmemchr_optab, mode);
3285
3286	expand_insn (icode, nops: 3, ops);
3287	assign_call_lhs (lhs, lhs_rtx, op: &ops[0]);
3288	}
3289
3290	/* Expand the IFN_UNIQUE function according to its first argument. */
3291
3292	static void
3293	expand_UNIQUE (internal_fn, gcall *stmt)
3294	{
3295	rtx pattern = NULL_RTX;
3296	enum ifn_unique_kind kind
3297	= (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
3298
3299	switch (kind)
3300	{
3301	default:
3302	gcc_unreachable ();
3303
3304	case IFN_UNIQUE_UNSPEC:
3305	if (targetm.have_unique ())
3306	pattern = targetm.gen_unique ();
3307	break;
3308
3309	case IFN_UNIQUE_OACC_FORK:
3310	case IFN_UNIQUE_OACC_JOIN:
3311	if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
3312	{
3313	tree lhs = gimple_call_lhs (gs: stmt);
3314	rtx target = const0_rtx;
3315
3316	if (lhs)
3317	target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3318
3319	rtx data_dep = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
3320	rtx axis = expand_normal (exp: gimple_call_arg (gs: stmt, index: 2));
3321
3322	if (kind == IFN_UNIQUE_OACC_FORK)
3323	pattern = targetm.gen_oacc_fork (target, data_dep, axis);
3324	else
3325	pattern = targetm.gen_oacc_join (target, data_dep, axis);
3326	}
3327	else
3328	gcc_unreachable ();
3329	break;
3330	}
3331
3332	if (pattern)
3333	emit_insn (pattern);
3334	}
3335
3336	/* Expand the IFN_DEFERRED_INIT function:
3337	LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
3338
3339	Initialize the LHS with zero/pattern according to its second argument
3340	INIT_TYPE:
3341	if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
3342	if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
3343	to initialize;
3344	The LHS variable is initialized including paddings.
3345	The reasons to choose 0xFE for pattern initialization are:
3346	1. It is a non-canonical virtual address on x86_64, and at the
3347	high end of the i386 kernel address space.
3348	2. It is a very large float value (-1.694739530317379e+38).
3349	3. It is also an unusual number for integers. */
3350	#define INIT_PATTERN_VALUE 0xFE
3351	static void
3352	expand_DEFERRED_INIT (internal_fn, gcall *stmt)
3353	{
3354	tree lhs = gimple_call_lhs (gs: stmt);
3355	tree var_size = gimple_call_arg (gs: stmt, index: 0);
3356	enum auto_init_type init_type
3357	= (enum auto_init_type) TREE_INT_CST_LOW (gimple_call_arg (stmt, 1));
3358	bool reg_lhs = true;
3359
3360	tree var_type = TREE_TYPE (lhs);
3361	gcc_assert (init_type > AUTO_INIT_UNINITIALIZED);
3362
3363	if (TREE_CODE (lhs) == SSA_NAME)
3364	reg_lhs = true;
3365	else
3366	{
3367	tree lhs_base = lhs;
3368	while (handled_component_p (t: lhs_base))
3369	lhs_base = TREE_OPERAND (lhs_base, 0);
3370	reg_lhs = (mem_ref_refers_to_non_mem_p (lhs_base)
3371	|| non_mem_decl_p (lhs_base));
3372	/* If this expands to a register and the underlying decl is wrapped in
3373	a MEM_REF that just serves as an access type change expose the decl
3374	if it is of correct size. This avoids a situation as in PR103271
3375	if the target does not support a direct move to the registers mode. */
3376	if (reg_lhs
3377	&& TREE_CODE (lhs_base) == MEM_REF
3378	&& TREE_CODE (TREE_OPERAND (lhs_base, 0)) == ADDR_EXPR
3379	&& DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))
3380	&& integer_zerop (TREE_OPERAND (lhs_base, 1))
3381	&& tree_fits_uhwi_p (var_size)
3382	&& tree_int_cst_equal
3383	(var_size,
3384	DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))))
3385	{
3386	lhs = TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0);
3387	var_type = TREE_TYPE (lhs);
3388	}
3389	}
3390
3391	if (!reg_lhs)
3392	{
3393	/* If the variable is not in register, expand to a memset
3394	to initialize it. */
3395	mark_addressable (lhs);
3396	tree var_addr = build_fold_addr_expr (lhs);
3397
3398	tree value = (init_type == AUTO_INIT_PATTERN)
3399	? build_int_cst (integer_type_node,
3400	INIT_PATTERN_VALUE)
3401	: integer_zero_node;
3402	tree m_call = build_call_expr (builtin_decl_implicit (fncode: BUILT_IN_MEMSET),
3403	3, var_addr, value, var_size);
3404	/* Expand this memset call. */
3405	expand_builtin_memset (m_call, NULL_RTX, TYPE_MODE (var_type));
3406	}
3407	else
3408	{
3409	/* If this variable is in a register use expand_assignment.
3410	For boolean scalars force zero-init. */
3411	tree init;
3412	scalar_int_mode var_mode;
3413	if (TREE_CODE (TREE_TYPE (lhs)) != BOOLEAN_TYPE
3414	&& tree_fits_uhwi_p (var_size)
3415	&& (init_type == AUTO_INIT_PATTERN
3416	|| !is_gimple_reg_type (type: var_type))
3417	&& int_mode_for_size (size: tree_to_uhwi (var_size) * BITS_PER_UNIT,
3418	limit: 0).exists (mode: &var_mode)
3419	&& have_insn_for (SET, var_mode))
3420	{
3421	unsigned HOST_WIDE_INT total_bytes = tree_to_uhwi (var_size);
3422	unsigned char *buf = XALLOCAVEC (unsigned char, total_bytes);
3423	memset (s: buf, c: (init_type == AUTO_INIT_PATTERN
3424	? INIT_PATTERN_VALUE : 0), n: total_bytes);
3425	tree itype = build_nonstandard_integer_type
3426	(total_bytes * BITS_PER_UNIT, 1);
3427	wide_int w = wi::from_buffer (buf, total_bytes);
3428	init = wide_int_to_tree (type: itype, cst: w);
3429	/* Pun the LHS to make sure its type has constant size
3430	unless it is an SSA name where that's already known. */
3431	if (TREE_CODE (lhs) != SSA_NAME)
3432	lhs = build1 (VIEW_CONVERT_EXPR, itype, lhs);
3433	else
3434	init = fold_build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), init);
3435	}
3436	else
3437	/* Use zero-init also for variable-length sizes. */
3438	init = build_zero_cst (var_type);
3439
3440	expand_assignment (lhs, init, false);
3441	}
3442	}
3443
3444	/* Expand the IFN_ACCESS_WITH_SIZE function:
3445	ACCESS_WITH_SIZE (REF_TO_OBJ, REF_TO_SIZE, CLASS_OF_SIZE,
3446	TYPE_OF_SIZE, ACCESS_MODE)
3447	which returns the REF_TO_OBJ same as the 1st argument;
3448
3449	1st argument REF_TO_OBJ: The reference to the object;
3450	2nd argument REF_TO_SIZE: The reference to the size of the object,
3451	3rd argument CLASS_OF_SIZE: The size referenced by the REF_TO_SIZE represents
3452	0: the number of bytes.
3453	1: the number of the elements of the object type;
3454	4th argument TYPE_OF_SIZE: A constant 0 with its TYPE being the same as the TYPE
3455	of the object referenced by REF_TO_SIZE
3456	5th argument ACCESS_MODE:
3457	-1: Unknown access semantics
3458	0: none
3459	1: read_only
3460	2: write_only
3461	3: read_write
3462	6th argument: A constant 0 with the pointer TYPE to the original flexible
3463	array type.
3464
3465	Both the return type and the type of the first argument of this
3466	function have been converted from the incomplete array type to
3467	the corresponding pointer type.
3468
3469	For each call to a .ACCESS_WITH_SIZE, replace it with its 1st argument. */
3470
3471	static void
3472	expand_ACCESS_WITH_SIZE (internal_fn, gcall *stmt)
3473	{
3474	tree lhs = gimple_call_lhs (gs: stmt);
3475	tree ref_to_obj = gimple_call_arg (gs: stmt, index: 0);
3476	if (lhs)
3477	expand_assignment (lhs, ref_to_obj, false);
3478	}
3479
3480	/* The size of an OpenACC compute dimension. */
3481
3482	static void
3483	expand_GOACC_DIM_SIZE (internal_fn, gcall *stmt)
3484	{
3485	tree lhs = gimple_call_lhs (gs: stmt);
3486
3487	if (!lhs)
3488	return;
3489
3490	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3491	if (targetm.have_oacc_dim_size ())
3492	{
3493	rtx dim = expand_expr (exp: gimple_call_arg (gs: stmt, index: 0), NULL_RTX,
3494	VOIDmode, modifier: EXPAND_NORMAL);
3495	emit_insn (targetm.gen_oacc_dim_size (target, dim));
3496	}
3497	else
3498	emit_move_insn (target, GEN_INT (1));
3499	}
3500
3501	/* The position of an OpenACC execution engine along one compute axis. */
3502
3503	static void
3504	expand_GOACC_DIM_POS (internal_fn, gcall *stmt)
3505	{
3506	tree lhs = gimple_call_lhs (gs: stmt);
3507
3508	if (!lhs)
3509	return;
3510
3511	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3512	if (targetm.have_oacc_dim_pos ())
3513	{
3514	rtx dim = expand_expr (exp: gimple_call_arg (gs: stmt, index: 0), NULL_RTX,
3515	VOIDmode, modifier: EXPAND_NORMAL);
3516	emit_insn (targetm.gen_oacc_dim_pos (target, dim));
3517	}
3518	else
3519	emit_move_insn (target, const0_rtx);
3520	}
3521
3522	/* This is expanded by oacc_device_lower pass. */
3523
3524	static void
3525	expand_GOACC_LOOP (internal_fn, gcall *)
3526	{
3527	gcc_unreachable ();
3528	}
3529
3530	/* This is expanded by oacc_device_lower pass. */
3531
3532	static void
3533	expand_GOACC_REDUCTION (internal_fn, gcall *)
3534	{
3535	gcc_unreachable ();
3536	}
3537
3538	/* This is expanded by oacc_device_lower pass. */
3539
3540	static void
3541	expand_GOACC_TILE (internal_fn, gcall *)
3542	{
3543	gcc_unreachable ();
3544	}
3545
3546	/* Set errno to EDOM. */
3547
3548	static void
3549	expand_SET_EDOM (internal_fn, gcall *)
3550	{
3551	#ifdef TARGET_EDOM
3552	#ifdef GEN_ERRNO_RTX
3553	rtx errno_rtx = GEN_ERRNO_RTX;
3554	#else
3555	rtx errno_rtx = gen_rtx_MEM (word_mode, gen_rtx_SYMBOL_REF (Pmode, "errno"));
3556	#endif
3557	emit_move_insn (errno_rtx,
3558	gen_int_mode (TARGET_EDOM, GET_MODE (errno_rtx)));
3559	#else
3560	gcc_unreachable ();
3561	#endif
3562	}
3563
3564	/* Expand atomic bit test and set. */
3565
3566	static void
3567	expand_ATOMIC_BIT_TEST_AND_SET (internal_fn, gcall *call)
3568	{
3569	expand_ifn_atomic_bit_test_and (call);
3570	}
3571
3572	/* Expand atomic bit test and complement. */
3573
3574	static void
3575	expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn, gcall *call)
3576	{
3577	expand_ifn_atomic_bit_test_and (call);
3578	}
3579
3580	/* Expand atomic bit test and reset. */
3581
3582	static void
3583	expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn, gcall *call)
3584	{
3585	expand_ifn_atomic_bit_test_and (call);
3586	}
3587
3588	/* Expand atomic bit test and set. */
3589
3590	static void
3591	expand_ATOMIC_COMPARE_EXCHANGE (internal_fn, gcall *call)
3592	{
3593	expand_ifn_atomic_compare_exchange (call);
3594	}
3595
3596	/* Expand atomic add fetch and cmp with 0. */
3597
3598	static void
3599	expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn, gcall *call)
3600	{
3601	expand_ifn_atomic_op_fetch_cmp_0 (call);
3602	}
3603
3604	/* Expand atomic sub fetch and cmp with 0. */
3605
3606	static void
3607	expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn, gcall *call)
3608	{
3609	expand_ifn_atomic_op_fetch_cmp_0 (call);
3610	}
3611
3612	/* Expand atomic and fetch and cmp with 0. */
3613
3614	static void
3615	expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn, gcall *call)
3616	{
3617	expand_ifn_atomic_op_fetch_cmp_0 (call);
3618	}
3619
3620	/* Expand atomic or fetch and cmp with 0. */
3621
3622	static void
3623	expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn, gcall *call)
3624	{
3625	expand_ifn_atomic_op_fetch_cmp_0 (call);
3626	}
3627
3628	/* Expand atomic xor fetch and cmp with 0. */
3629
3630	static void
3631	expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn, gcall *call)
3632	{
3633	expand_ifn_atomic_op_fetch_cmp_0 (call);
3634	}
3635
3636	/* Expand LAUNDER to assignment, lhs = arg0. */
3637
3638	static void
3639	expand_LAUNDER (internal_fn, gcall *call)
3640	{
3641	tree lhs = gimple_call_lhs (gs: call);
3642
3643	if (!lhs)
3644	return;
3645
3646	expand_assignment (lhs, gimple_call_arg (gs: call, index: 0), false);
3647	}
3648
3649	/* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3650
3651	static void
3652	expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3653	{
3654	internal_fn ifn = gimple_call_internal_fn (gs: stmt);
3655	int rhs_index = internal_fn_stored_value_index (ifn);
3656	tree base = gimple_call_arg (gs: stmt, index: 0);
3657	tree offset = gimple_call_arg (gs: stmt, index: 1);
3658	tree scale = gimple_call_arg (gs: stmt, index: 2);
3659	tree rhs = gimple_call_arg (gs: stmt, index: rhs_index);
3660
3661	rtx base_rtx = expand_normal (exp: base);
3662	rtx offset_rtx = expand_normal (exp: offset);
3663	HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3664	rtx rhs_rtx = expand_normal (exp: rhs);
3665
3666	class expand_operand ops[8];
3667	int i = 0;
3668	create_address_operand (op: &ops[i++], value: base_rtx);
3669	create_input_operand (op: &ops[i++], value: offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3670	create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3671	create_integer_operand (&ops[i++], scale_int);
3672	create_input_operand (op: &ops[i++], value: rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
3673	i = add_mask_else_and_len_args (ops, opno: i, stmt);
3674
3675	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (TREE_TYPE (rhs)),
3676	TYPE_MODE (TREE_TYPE (offset)));
3677	expand_insn (icode, nops: i, ops);
3678	}
3679
3680	/* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3681
3682	static void
3683	expand_gather_load_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3684	{
3685	tree lhs = gimple_call_lhs (gs: stmt);
3686	tree base = gimple_call_arg (gs: stmt, index: 0);
3687	tree offset = gimple_call_arg (gs: stmt, index: 1);
3688	tree scale = gimple_call_arg (gs: stmt, index: 2);
3689
3690	rtx lhs_rtx = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3691	rtx base_rtx = expand_normal (exp: base);
3692	rtx offset_rtx = expand_normal (exp: offset);
3693	HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3694
3695	int i = 0;
3696	class expand_operand ops[9];
3697	create_call_lhs_operand (op: &ops[i++], lhs_rtx, TYPE_MODE (TREE_TYPE (lhs)));
3698	create_address_operand (op: &ops[i++], value: base_rtx);
3699	create_input_operand (op: &ops[i++], value: offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3700	create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3701	create_integer_operand (&ops[i++], scale_int);
3702	i = add_mask_else_and_len_args (ops, opno: i, stmt);
3703	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (TREE_TYPE (lhs)),
3704	TYPE_MODE (TREE_TYPE (offset)));
3705	expand_insn (icode, nops: i, ops);
3706	assign_call_lhs (lhs, lhs_rtx, op: &ops[0]);
3707	}
3708
3709	/* Expand MASK_LEN_STRIDED_LOAD call CALL by optab OPTAB. */
3710
3711	static void
3712	expand_strided_load_optab_fn (ATTRIBUTE_UNUSED internal_fn, gcall *stmt,
3713	direct_optab optab)
3714	{
3715	tree lhs = gimple_call_lhs (gs: stmt);
3716	tree base = gimple_call_arg (gs: stmt, index: 0);
3717	tree stride = gimple_call_arg (gs: stmt, index: 1);
3718
3719	rtx lhs_rtx = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3720	rtx base_rtx = expand_normal (exp: base);
3721	rtx stride_rtx = expand_normal (exp: stride);
3722
3723	unsigned i = 0;
3724	class expand_operand ops[7];
3725	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
3726
3727	create_output_operand (op: &ops[i++], x: lhs_rtx, mode);
3728	create_address_operand (op: &ops[i++], value: base_rtx);
3729	create_address_operand (op: &ops[i++], value: stride_rtx);
3730
3731	i = add_mask_else_and_len_args (ops, opno: i, stmt);
3732	expand_insn (icode: direct_optab_handler (op: optab, mode), nops: i, ops);
3733
3734	if (!rtx_equal_p (lhs_rtx, ops[0].value))
3735	emit_move_insn (lhs_rtx, ops[0].value);
3736	}
3737
3738	/* Expand MASK_LEN_STRIDED_STORE call CALL by optab OPTAB. */
3739
3740	static void
3741	expand_strided_store_optab_fn (ATTRIBUTE_UNUSED internal_fn, gcall *stmt,
3742	direct_optab optab)
3743	{
3744	internal_fn fn = gimple_call_internal_fn (gs: stmt);
3745	int rhs_index = internal_fn_stored_value_index (fn);
3746
3747	tree base = gimple_call_arg (gs: stmt, index: 0);
3748	tree stride = gimple_call_arg (gs: stmt, index: 1);
3749	tree rhs = gimple_call_arg (gs: stmt, index: rhs_index);
3750
3751	rtx base_rtx = expand_normal (exp: base);
3752	rtx stride_rtx = expand_normal (exp: stride);
3753	rtx rhs_rtx = expand_normal (exp: rhs);
3754
3755	unsigned i = 0;
3756	class expand_operand ops[6];
3757	machine_mode mode = TYPE_MODE (TREE_TYPE (rhs));
3758
3759	create_address_operand (op: &ops[i++], value: base_rtx);
3760	create_address_operand (op: &ops[i++], value: stride_rtx);
3761	create_input_operand (op: &ops[i++], value: rhs_rtx, mode);
3762
3763	i = add_mask_else_and_len_args (ops, opno: i, stmt);
3764	expand_insn (icode: direct_optab_handler (op: optab, mode), nops: i, ops);
3765	}
3766
3767	/* Helper for expand_DIVMOD. Return true if the sequence starting with
3768	INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3769
3770	static bool
3771	contains_call_div_mod (rtx_insn *insn)
3772	{
3773	subrtx_iterator::array_type array;
3774	for (; insn; insn = NEXT_INSN (insn))
3775	if (CALL_P (insn))
3776	return true;
3777	else if (INSN_P (insn))
3778	FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
3779	switch (GET_CODE (*iter))
3780	{
3781	case CALL:
3782	case DIV:
3783	case UDIV:
3784	case MOD:
3785	case UMOD:
3786	return true;
3787	default:
3788	break;
3789	}
3790	return false;
3791	}
3792
3793	/* Expand DIVMOD() using:
3794	a) optab handler for udivmod/sdivmod if it is available.
3795	b) If optab_handler doesn't exist, generate call to
3796	target-specific divmod libfunc. */
3797
3798	static void
3799	expand_DIVMOD (internal_fn, gcall *call_stmt)
3800	{
3801	tree lhs = gimple_call_lhs (gs: call_stmt);
3802	tree arg0 = gimple_call_arg (gs: call_stmt, index: 0);
3803	tree arg1 = gimple_call_arg (gs: call_stmt, index: 1);
3804
3805	gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
3806	tree type = TREE_TYPE (TREE_TYPE (lhs));
3807	machine_mode mode = TYPE_MODE (type);
3808	bool unsignedp = TYPE_UNSIGNED (type);
3809	optab tab = (unsignedp) ? udivmod_optab : sdivmod_optab;
3810
3811	rtx op0 = expand_normal (exp: arg0);
3812	rtx op1 = expand_normal (exp: arg1);
3813	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3814
3815	rtx quotient = NULL_RTX, remainder = NULL_RTX;
3816	rtx_insn *insns = NULL;
3817
3818	if (TREE_CODE (arg1) == INTEGER_CST)
3819	{
3820	/* For DIVMOD by integral constants, there could be efficient code
3821	expanded inline e.g. using shifts and plus/minus. Try to expand
3822	the division and modulo and if it emits any library calls or any
3823	{,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3824	divmod libcall. */
3825	scalar_int_mode int_mode;
3826	if (remainder == NULL_RTX
3827	&& optimize
3828	&& CONST_INT_P (op1)
3829	&& !pow2p_hwi (INTVAL (op1))
3830	&& is_int_mode (TYPE_MODE (type), int_mode: &int_mode)
3831	&& GET_MODE_SIZE (mode: int_mode) == 2 * UNITS_PER_WORD
3832	&& optab_handler (op: and_optab, mode: word_mode) != CODE_FOR_nothing
3833	&& optab_handler (op: add_optab, mode: word_mode) != CODE_FOR_nothing
3834	&& optimize_insn_for_speed_p ())
3835	{
3836	rtx_insn *last = get_last_insn ();
3837	remainder = NULL_RTX;
3838	quotient = expand_doubleword_divmod (int_mode, op0, op1, &remainder,
3839	TYPE_UNSIGNED (type));
3840	if (quotient != NULL_RTX)
3841	{
3842	if (optab_handler (op: mov_optab, mode: int_mode) != CODE_FOR_nothing)
3843	{
3844	rtx_insn *move = emit_move_insn (quotient, quotient);
3845	set_dst_reg_note (move, REG_EQUAL,
3846	gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3847	? UDIV : DIV, int_mode,
3848	copy_rtx (op0), op1),
3849	quotient);
3850	move = emit_move_insn (remainder, remainder);
3851	set_dst_reg_note (move, REG_EQUAL,
3852	gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3853	? UMOD : MOD, int_mode,
3854	copy_rtx (op0), op1),
3855	quotient);
3856	}
3857	}
3858	else
3859	delete_insns_since (last);
3860	}
3861
3862	if (remainder == NULL_RTX)
3863	{
3864	struct separate_ops ops;
3865	ops.code = TRUNC_DIV_EXPR;
3866	ops.type = type;
3867	ops.op0 = make_tree (ops.type, op0);
3868	ops.op1 = arg1;
3869	ops.op2 = NULL_TREE;
3870	ops.location = gimple_location (g: call_stmt);
3871	start_sequence ();
3872	quotient = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
3873	if (contains_call_div_mod (insn: get_insns ()))
3874	quotient = NULL_RTX;
3875	else
3876	{
3877	ops.code = TRUNC_MOD_EXPR;
3878	remainder = expand_expr_real_2 (&ops, NULL_RTX, mode,
3879	EXPAND_NORMAL);
3880	if (contains_call_div_mod (insn: get_insns ()))
3881	remainder = NULL_RTX;
3882	}
3883	if (remainder)
3884	insns = get_insns ();
3885	end_sequence ();
3886	}
3887	}
3888
3889	if (remainder)
3890	emit_insn (insns);
3891
3892	/* Check if optab_handler exists for divmod_optab for given mode. */
3893	else if (optab_handler (op: tab, mode) != CODE_FOR_nothing)
3894	{
3895	quotient = gen_reg_rtx (mode);
3896	remainder = gen_reg_rtx (mode);
3897	expand_twoval_binop (tab, op0, op1, quotient, remainder, unsignedp);
3898	}
3899
3900	/* Generate call to divmod libfunc if it exists. */
3901	else if (rtx libfunc = optab_libfunc (tab, mode))
3902	targetm.expand_divmod_libfunc (libfunc, mode, op0, op1,
3903	&quotient, &remainder);
3904
3905	else
3906	gcc_unreachable ();
3907
3908	/* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3909	expand_expr (exp: build2 (COMPLEX_EXPR, TREE_TYPE (lhs),
3910	make_tree (TREE_TYPE (arg0), quotient),
3911	make_tree (TREE_TYPE (arg1), remainder)),
3912	target, VOIDmode, modifier: EXPAND_NORMAL);
3913	}
3914
3915	/* Expand a NOP. */
3916
3917	static void
3918	expand_NOP (internal_fn, gcall *)
3919	{
3920	/* Nothing. But it shouldn't really prevail. */
3921	}
3922
3923	/* Coroutines, all should have been processed at this stage. */
3924
3925	static void
3926	expand_CO_FRAME (internal_fn, gcall *)
3927	{
3928	gcc_unreachable ();
3929	}
3930
3931	static void
3932	expand_CO_YIELD (internal_fn, gcall *)
3933	{
3934	gcc_unreachable ();
3935	}
3936
3937	static void
3938	expand_CO_SUSPN (internal_fn, gcall *)
3939	{
3940	gcc_unreachable ();
3941	}
3942
3943	static void
3944	expand_CO_ACTOR (internal_fn, gcall *)
3945	{
3946	gcc_unreachable ();
3947	}
3948
3949	/* Expand a call to FN using the operands in STMT. FN has a single
3950	output operand and NARGS input operands. */
3951
3952	static void
3953	expand_direct_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab,
3954	unsigned int nargs)
3955	{
3956	tree_pair types = direct_internal_fn_types (fn, stmt);
3957	insn_code icode = direct_optab_handler (op: optab, TYPE_MODE (types.first));
3958	expand_fn_using_insn (stmt, icode, noutputs: 1, ninputs: nargs);
3959	}
3960
3961	/* Expand WHILE_ULT call STMT using optab OPTAB. */
3962
3963	static void
3964	expand_while_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3965	{
3966	expand_operand ops[4];
3967	tree rhs_type[2];
3968
3969	tree lhs = gimple_call_lhs (gs: stmt);
3970	tree lhs_type = TREE_TYPE (lhs);
3971	rtx lhs_rtx = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
3972	create_call_lhs_operand (op: &ops[0], lhs_rtx, TYPE_MODE (lhs_type));
3973
3974	for (unsigned int i = 0; i < 2; ++i)
3975	{
3976	tree rhs = gimple_call_arg (gs: stmt, index: i);
3977	rhs_type[i] = TREE_TYPE (rhs);
3978	rtx rhs_rtx = expand_normal (exp: rhs);
3979	create_input_operand (op: &ops[i + 1], value: rhs_rtx, TYPE_MODE (rhs_type[i]));
3980	}
3981
3982	int opcnt;
3983	if (!VECTOR_MODE_P (TYPE_MODE (lhs_type)))
3984	{
3985	/* When the mask is an integer mode the exact vector length may not
3986	be clear to the backend, so we pass it in operand[3].
3987	Use the vector in arg2 for the most reliable intended size. */
3988	tree type = TREE_TYPE (gimple_call_arg (stmt, 2));
3989	create_integer_operand (&ops[3], TYPE_VECTOR_SUBPARTS (node: type));
3990	opcnt = 4;
3991	}
3992	else
3993	/* The mask has a vector type so the length operand is unnecessary. */
3994	opcnt = 3;
3995
3996	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (rhs_type[0]),
3997	TYPE_MODE (lhs_type));
3998
3999	expand_insn (icode, nops: opcnt, ops);
4000	assign_call_lhs (lhs, lhs_rtx, op: &ops[0]);
4001	}
4002
4003	/* Expand a call to a convert-like optab using the operands in STMT.
4004	FN has a single output operand and NARGS input operands. */
4005
4006	static void
4007	expand_convert_optab_fn (internal_fn fn, gcall *stmt, convert_optab optab,
4008	unsigned int nargs)
4009	{
4010	tree_pair types = direct_internal_fn_types (fn, stmt);
4011	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (types.first),
4012	TYPE_MODE (types.second));
4013	expand_fn_using_insn (stmt, icode, noutputs: 1, ninputs: nargs);
4014	}
4015
4016	/* Expand CRC call STMT. */
4017
4018	static void
4019	expand_crc_optab_fn (internal_fn fn, gcall *stmt, convert_optab optab)
4020	{
4021	tree lhs = gimple_call_lhs (gs: stmt);
4022	tree rhs1 = gimple_call_arg (gs: stmt, index: 0); // crc
4023	tree rhs2 = gimple_call_arg (gs: stmt, index: 1); // data
4024	tree rhs3 = gimple_call_arg (gs: stmt, index: 2); // polynomial
4025
4026	tree result_type = TREE_TYPE (lhs);
4027	tree data_type = TREE_TYPE (rhs2);
4028
4029	gcc_assert (TYPE_MODE (result_type) >= TYPE_MODE (data_type));
4030
4031	rtx dest = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
4032	rtx crc = expand_normal (exp: rhs1);
4033	rtx data = expand_normal (exp: rhs2);
4034	gcc_assert (TREE_CODE (rhs3) == INTEGER_CST);
4035	rtx polynomial = gen_rtx_CONST_INT (TYPE_MODE (result_type),
4036	TREE_INT_CST_LOW (rhs3));
4037
4038	/* Use target specific expansion if it exists.
4039	Otherwise, generate table-based CRC. */
4040	if (direct_internal_fn_supported_p (fn, tree_pair (data_type, result_type),
4041	OPTIMIZE_FOR_SPEED))
4042	{
4043	class expand_operand ops[4];
4044
4045	if (dump_file && (dump_flags & TDF_DETAILS))
4046	{
4047	fprintf (stream: dump_file,
4048	format: ";; using optab for crc_%u_polynomial_"
4049	HOST_WIDE_INT_PRINT_HEX "\n",
4050	GET_MODE_BITSIZE (GET_MODE (dest)).to_constant (),
4051	TREE_INT_CST_LOW (rhs3));
4052	}
4053
4054	create_call_lhs_operand (op: &ops[0], lhs_rtx: dest, TYPE_MODE (result_type));
4055	create_input_operand (op: &ops[1], value: crc, TYPE_MODE (result_type));
4056	create_input_operand (op: &ops[2], value: data, TYPE_MODE (data_type));
4057	create_input_operand (op: &ops[3], value: polynomial, TYPE_MODE (result_type));
4058	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (data_type),
4059	TYPE_MODE (result_type));
4060	expand_insn (icode, nops: 4, ops);
4061	assign_call_lhs (lhs, lhs_rtx: dest, op: &ops[0]);
4062	}
4063	else
4064	{
4065	/* We're bypassing all the operand conversions that are done in the
4066	case when we get an icode, operands and pass that off to expand_insn.
4067
4068	That path has special case handling for promoted return values which
4069	we must emulate here (is the same kind of special treatment ever
4070	needed for input arguments here?).
4071
4072	In particular we do not want to store directly into a promoted
4073	SUBREG destination, instead store into a suitably sized pseudo. */
4074	rtx orig_dest = dest;
4075	if (SUBREG_P (dest) && SUBREG_PROMOTED_VAR_P (dest))
4076	dest = gen_reg_rtx (GET_MODE (dest));
4077
4078	/* If it's IFN_CRC generate bit-forward CRC. */
4079	if (fn == IFN_CRC)
4080	expand_crc_table_based (dest, crc, data, polynomial,
4081	TYPE_MODE (data_type));
4082	else
4083	/* If it's IFN_CRC_REV generate bit-reversed CRC. */
4084	expand_reversed_crc_table_based (dest, crc, data, polynomial,
4085	TYPE_MODE (data_type),
4086	generate_reflecting_code_standard);
4087
4088	/* Now get the return value where it needs to be, taking care to
4089	ensure it's promoted appropriately if the ABI demands it.
4090
4091	Re-use assign_call_lhs to handle the details. */
4092	class expand_operand ops[4];
4093	create_call_lhs_operand (op: &ops[0], lhs_rtx: dest, TYPE_MODE (result_type));
4094	ops[0].value = dest;
4095	assign_call_lhs (lhs, lhs_rtx: orig_dest, op: &ops[0]);
4096	}
4097	}
4098
4099	/* Expanders for optabs that can use expand_direct_optab_fn. */
4100
4101	#define expand_unary_optab_fn(FN, STMT, OPTAB) \
4102	expand_direct_optab_fn (FN, STMT, OPTAB, 1)
4103
4104	#define expand_binary_optab_fn(FN, STMT, OPTAB) \
4105	expand_direct_optab_fn (FN, STMT, OPTAB, 2)
4106
4107	#define expand_ternary_optab_fn(FN, STMT, OPTAB) \
4108	expand_direct_optab_fn (FN, STMT, OPTAB, 3)
4109
4110	#define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
4111	expand_direct_optab_fn (FN, STMT, OPTAB, 3)
4112
4113	#define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
4114	expand_direct_optab_fn (FN, STMT, OPTAB, 4)
4115
4116	#define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
4117	expand_direct_optab_fn (FN, STMT, OPTAB, 5)
4118
4119	#define expand_cond_len_unary_optab_fn(FN, STMT, OPTAB) \
4120	expand_direct_optab_fn (FN, STMT, OPTAB, 5)
4121
4122	#define expand_cond_len_binary_optab_fn(FN, STMT, OPTAB) \
4123	expand_direct_optab_fn (FN, STMT, OPTAB, 6)
4124
4125	#define expand_cond_len_ternary_optab_fn(FN, STMT, OPTAB) \
4126	expand_direct_optab_fn (FN, STMT, OPTAB, 7)
4127
4128	#define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
4129	expand_direct_optab_fn (FN, STMT, OPTAB, 3)
4130
4131	#define expand_fold_len_extract_optab_fn(FN, STMT, OPTAB) \
4132	expand_direct_optab_fn (FN, STMT, OPTAB, 5)
4133
4134	#define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
4135	expand_direct_optab_fn (FN, STMT, OPTAB, 2)
4136
4137	#define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
4138	expand_direct_optab_fn (FN, STMT, OPTAB, 3)
4139
4140	#define expand_mask_len_fold_left_optab_fn(FN, STMT, OPTAB) \
4141	expand_direct_optab_fn (FN, STMT, OPTAB, 5)
4142
4143	#define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
4144	expand_direct_optab_fn (FN, STMT, OPTAB, 4)
4145
4146	/* Expanders for optabs that can use expand_convert_optab_fn. */
4147
4148	#define expand_unary_convert_optab_fn(FN, STMT, OPTAB) \
4149	expand_convert_optab_fn (FN, STMT, OPTAB, 1)
4150
4151	#define expand_vec_extract_optab_fn(FN, STMT, OPTAB) \
4152	expand_convert_optab_fn (FN, STMT, OPTAB, 2)
4153
4154	/* RETURN_TYPE and ARGS are a return type and argument list that are
4155	in principle compatible with FN (which satisfies direct_internal_fn_p).
4156	Return the types that should be used to determine whether the
4157	target supports FN. */
4158
4159	tree_pair
4160	direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
4161	{
4162	const direct_internal_fn_info &info = direct_internal_fn (fn);
4163	tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
4164	tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
4165	return tree_pair (type0, type1);
4166	}
4167
4168	/* CALL is a call whose return type and arguments are in principle
4169	compatible with FN (which satisfies direct_internal_fn_p). Return the
4170	types that should be used to determine whether the target supports FN. */
4171
4172	tree_pair
4173	direct_internal_fn_types (internal_fn fn, gcall *call)
4174	{
4175	const direct_internal_fn_info &info = direct_internal_fn (fn);
4176	tree op0 = (info.type0 < 0
4177	? gimple_call_lhs (gs: call)
4178	: gimple_call_arg (gs: call, index: info.type0));
4179	tree op1 = (info.type1 < 0
4180	? gimple_call_lhs (gs: call)
4181	: gimple_call_arg (gs: call, index: info.type1));
4182	return tree_pair (TREE_TYPE (op0), TREE_TYPE (op1));
4183	}
4184
4185	/* Return true if OPTAB is supported for TYPES (whose modes should be
4186	the same) when the optimization type is OPT_TYPE. Used for simple
4187	direct optabs. */
4188
4189	static bool
4190	direct_optab_supported_p (direct_optab optab, tree_pair types,
4191	optimization_type opt_type)
4192	{
4193	machine_mode mode = TYPE_MODE (types.first);
4194	gcc_checking_assert (mode == TYPE_MODE (types.second));
4195	return direct_optab_handler (optab, mode, opt_type) != CODE_FOR_nothing;
4196	}
4197
4198	/* Return true if OPTAB is supported for TYPES, where the first type
4199	is the destination and the second type is the source. Used for
4200	convert optabs. */
4201
4202	static bool
4203	convert_optab_supported_p (convert_optab optab, tree_pair types,
4204	optimization_type opt_type)
4205	{
4206	return (convert_optab_handler (optab, TYPE_MODE (types.first),
4207	TYPE_MODE (types.second), opt_type)
4208	!= CODE_FOR_nothing);
4209	}
4210
4211	/* Return true if load/store lanes optab OPTAB is supported for
4212	array type TYPES.first when the optimization type is OPT_TYPE. */
4213
4214	static bool
4215	multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
4216	optimization_type opt_type)
4217	{
4218	gcc_assert (TREE_CODE (types.first) == ARRAY_TYPE);
4219	machine_mode imode = TYPE_MODE (types.first);
4220	machine_mode vmode = TYPE_MODE (TREE_TYPE (types.first));
4221	return (convert_optab_handler (optab, imode, vmode, opt_type)
4222	!= CODE_FOR_nothing);
4223	}
4224
4225	#define direct_unary_optab_supported_p direct_optab_supported_p
4226	#define direct_unary_convert_optab_supported_p convert_optab_supported_p
4227	#define direct_binary_optab_supported_p direct_optab_supported_p
4228	#define direct_ternary_optab_supported_p direct_optab_supported_p
4229	#define direct_cond_unary_optab_supported_p direct_optab_supported_p
4230	#define direct_cond_binary_optab_supported_p direct_optab_supported_p
4231	#define direct_cond_ternary_optab_supported_p direct_optab_supported_p
4232	#define direct_cond_len_unary_optab_supported_p direct_optab_supported_p
4233	#define direct_cond_len_binary_optab_supported_p direct_optab_supported_p
4234	#define direct_cond_len_ternary_optab_supported_p direct_optab_supported_p
4235	#define direct_crc_optab_supported_p convert_optab_supported_p
4236	#define direct_mask_load_optab_supported_p convert_optab_supported_p
4237	#define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
4238	#define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
4239	#define direct_gather_load_optab_supported_p convert_optab_supported_p
4240	#define direct_strided_load_optab_supported_p direct_optab_supported_p
4241	#define direct_len_load_optab_supported_p direct_optab_supported_p
4242	#define direct_mask_len_load_optab_supported_p convert_optab_supported_p
4243	#define direct_mask_store_optab_supported_p convert_optab_supported_p
4244	#define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
4245	#define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
4246	#define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
4247	#define direct_vec_cond_optab_supported_p convert_optab_supported_p
4248	#define direct_scatter_store_optab_supported_p convert_optab_supported_p
4249	#define direct_strided_store_optab_supported_p direct_optab_supported_p
4250	#define direct_len_store_optab_supported_p direct_optab_supported_p
4251	#define direct_mask_len_store_optab_supported_p convert_optab_supported_p
4252	#define direct_while_optab_supported_p convert_optab_supported_p
4253	#define direct_fold_extract_optab_supported_p direct_optab_supported_p
4254	#define direct_fold_len_extract_optab_supported_p direct_optab_supported_p
4255	#define direct_fold_left_optab_supported_p direct_optab_supported_p
4256	#define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
4257	#define direct_mask_len_fold_left_optab_supported_p direct_optab_supported_p
4258	#define direct_check_ptrs_optab_supported_p direct_optab_supported_p
4259	#define direct_vec_set_optab_supported_p direct_optab_supported_p
4260	#define direct_vec_extract_optab_supported_p convert_optab_supported_p
4261
4262	/* Return the optab used by internal function FN. */
4263
4264	optab
4265	direct_internal_fn_optab (internal_fn fn, tree_pair types)
4266	{
4267	switch (fn)
4268	{
4269	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4270	case IFN_##CODE: break;
4271	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4272	case IFN_##CODE: return OPTAB##_optab;
4273	#define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4274	UNSIGNED_OPTAB, TYPE) \
4275	case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
4276	? UNSIGNED_OPTAB ## _optab \
4277	: SIGNED_OPTAB ## _optab);
4278	#include "internal-fn.def"
4279
4280	case IFN_LAST:
4281	break;
4282	}
4283	gcc_unreachable ();
4284	}
4285
4286	/* Return the optab used by internal function FN. */
4287
4288	static optab
4289	direct_internal_fn_optab (internal_fn fn)
4290	{
4291	switch (fn)
4292	{
4293	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4294	case IFN_##CODE: break;
4295	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4296	case IFN_##CODE: return OPTAB##_optab;
4297	#include "internal-fn.def"
4298
4299	case IFN_LAST:
4300	break;
4301	}
4302	gcc_unreachable ();
4303	}
4304
4305	/* Return true if TYPE's mode has the same format as TYPE, and if there is
4306	a 1:1 correspondence between the values that the mode can store and the
4307	values that the type can store. */
4308
4309	static bool
4310	type_strictly_matches_mode_p (const_tree type)
4311	{
4312	/* The masked vector operations have both vector data operands and vector
4313	boolean operands. The vector data operands are expected to have a vector
4314	mode, but the vector boolean operands can be an integer mode rather than
4315	a vector mode, depending on how TARGET_VECTORIZE_GET_MASK_MODE is
4316	defined. PR116103. */
4317	if (VECTOR_BOOLEAN_TYPE_P (type)
4318	&& SCALAR_INT_MODE_P (TYPE_MODE (type))
4319	&& TYPE_PRECISION (TREE_TYPE (type)) == 1)
4320	return true;
4321
4322	if (VECTOR_TYPE_P (type))
4323	return VECTOR_MODE_P (TYPE_MODE (type));
4324
4325	if (INTEGRAL_TYPE_P (type))
4326	return type_has_mode_precision_p (t: type);
4327
4328	if (SCALAR_FLOAT_TYPE_P (type) || COMPLEX_FLOAT_TYPE_P (type))
4329	return true;
4330
4331	return false;
4332	}
4333
4334	/* Returns true if both types of TYPE_PAIR strictly match their modes,
4335	else returns false. */
4336
4337	static bool
4338	type_pair_strictly_matches_mode_p (tree_pair type_pair)
4339	{
4340	return type_strictly_matches_mode_p (type: type_pair.first)
4341	&& type_strictly_matches_mode_p (type: type_pair.second);
4342	}
4343
4344	/* Return true if FN is supported for the types in TYPES when the
4345	optimization type is OPT_TYPE. The types are those associated with
4346	the "type0" and "type1" fields of FN's direct_internal_fn_info
4347	structure. */
4348
4349	bool
4350	direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
4351	optimization_type opt_type)
4352	{
4353	if (!type_pair_strictly_matches_mode_p (type_pair: types))
4354	return false;
4355
4356	switch (fn)
4357	{
4358	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4359	case IFN_##CODE: break;
4360	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4361	case IFN_##CODE: \
4362	return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
4363	opt_type);
4364	#define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4365	UNSIGNED_OPTAB, TYPE) \
4366	case IFN_##CODE: \
4367	{ \
4368	optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
4369	? UNSIGNED_OPTAB ## _optab \
4370	: SIGNED_OPTAB ## _optab); \
4371	return direct_##TYPE##_optab_supported_p (which_optab, types, \
4372	opt_type); \
4373	}
4374	#include "internal-fn.def"
4375
4376	case IFN_LAST:
4377	break;
4378	}
4379	gcc_unreachable ();
4380	}
4381
4382	/* Return true if FN is supported for type TYPE when the optimization
4383	type is OPT_TYPE. The caller knows that the "type0" and "type1"
4384	fields of FN's direct_internal_fn_info structure are the same. */
4385
4386	bool
4387	direct_internal_fn_supported_p (internal_fn fn, tree type,
4388	optimization_type opt_type)
4389	{
4390	const direct_internal_fn_info &info = direct_internal_fn (fn);
4391	gcc_checking_assert (info.type0 == info.type1);
4392	return direct_internal_fn_supported_p (fn, types: tree_pair (type, type), opt_type);
4393	}
4394
4395	/* Return true if the STMT is supported when the optimization type is OPT_TYPE,
4396	given that STMT is a call to a direct internal function. */
4397
4398	bool
4399	direct_internal_fn_supported_p (gcall *stmt, optimization_type opt_type)
4400	{
4401	internal_fn fn = gimple_call_internal_fn (gs: stmt);
4402	tree_pair types = direct_internal_fn_types (fn, call: stmt);
4403	return direct_internal_fn_supported_p (fn, types, opt_type);
4404	}
4405
4406	/* Return true if FN is a binary operation and if FN is commutative. */
4407
4408	bool
4409	commutative_binary_fn_p (internal_fn fn)
4410	{
4411	switch (fn)
4412	{
4413	case IFN_AVG_FLOOR:
4414	case IFN_AVG_CEIL:
4415	case IFN_MULH:
4416	case IFN_MULHS:
4417	case IFN_MULHRS:
4418	case IFN_FMIN:
4419	case IFN_FMAX:
4420	case IFN_COMPLEX_MUL:
4421	case IFN_UBSAN_CHECK_ADD:
4422	case IFN_UBSAN_CHECK_MUL:
4423	case IFN_ADD_OVERFLOW:
4424	case IFN_MUL_OVERFLOW:
4425	case IFN_SAT_ADD:
4426	case IFN_VEC_WIDEN_PLUS:
4427	case IFN_VEC_WIDEN_PLUS_LO:
4428	case IFN_VEC_WIDEN_PLUS_HI:
4429	case IFN_VEC_WIDEN_PLUS_EVEN:
4430	case IFN_VEC_WIDEN_PLUS_ODD:
4431	return true;
4432
4433	default:
4434	return false;
4435	}
4436	}
4437
4438	/* Return true if FN is a ternary operation and if its first two arguments
4439	are commutative. */
4440
4441	bool
4442	commutative_ternary_fn_p (internal_fn fn)
4443	{
4444	switch (fn)
4445	{
4446	case IFN_FMA:
4447	case IFN_FMS:
4448	case IFN_FNMA:
4449	case IFN_FNMS:
4450	case IFN_UADDC:
4451	return true;
4452
4453	default:
4454	return false;
4455	}
4456	}
4457
4458	/* Return true if FN is an associative binary operation. */
4459
4460	bool
4461	associative_binary_fn_p (internal_fn fn)
4462	{
4463	switch (fn)
4464	{
4465	case IFN_FMIN:
4466	case IFN_FMAX:
4467	return true;
4468
4469	default:
4470	return false;
4471	}
4472	}
4473
4474	/* If FN is commutative in two consecutive arguments, return the
4475	index of the first, otherwise return -1. */
4476
4477	int
4478	first_commutative_argument (internal_fn fn)
4479	{
4480	switch (fn)
4481	{
4482	case IFN_COND_ADD:
4483	case IFN_COND_MUL:
4484	case IFN_COND_MIN:
4485	case IFN_COND_MAX:
4486	case IFN_COND_FMIN:
4487	case IFN_COND_FMAX:
4488	case IFN_COND_AND:
4489	case IFN_COND_IOR:
4490	case IFN_COND_XOR:
4491	case IFN_COND_FMA:
4492	case IFN_COND_FMS:
4493	case IFN_COND_FNMA:
4494	case IFN_COND_FNMS:
4495	case IFN_COND_LEN_ADD:
4496	case IFN_COND_LEN_MUL:
4497	case IFN_COND_LEN_MIN:
4498	case IFN_COND_LEN_MAX:
4499	case IFN_COND_LEN_FMIN:
4500	case IFN_COND_LEN_FMAX:
4501	case IFN_COND_LEN_AND:
4502	case IFN_COND_LEN_IOR:
4503	case IFN_COND_LEN_XOR:
4504	case IFN_COND_LEN_FMA:
4505	case IFN_COND_LEN_FMS:
4506	case IFN_COND_LEN_FNMA:
4507	case IFN_COND_LEN_FNMS:
4508	return 1;
4509
4510	default:
4511	if (commutative_binary_fn_p (fn)
4512	|| commutative_ternary_fn_p (fn))
4513	return 0;
4514	return -1;
4515	}
4516	}
4517
4518	/* Return true if this CODE describes an internal_fn that returns a vector with
4519	elements twice as wide as the element size of the input vectors. */
4520
4521	bool
4522	widening_fn_p (code_helper code)
4523	{
4524	if (!code.is_fn_code ())
4525	return false;
4526
4527	if (!internal_fn_p (code: (combined_fn) code))
4528	return false;
4529
4530	internal_fn fn = as_internal_fn (code: (combined_fn) code);
4531	switch (fn)
4532	{
4533	#define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
4534	case IFN_##NAME: \
4535	case IFN_##NAME##_HI: \
4536	case IFN_##NAME##_LO: \
4537	case IFN_##NAME##_EVEN: \
4538	case IFN_##NAME##_ODD: \
4539	return true;
4540	#include "internal-fn.def"
4541
4542	default:
4543	return false;
4544	}
4545	}
4546
4547	/* Return true if IFN_SET_EDOM is supported. */
4548
4549	bool
4550	set_edom_supported_p (void)
4551	{
4552	#ifdef TARGET_EDOM
4553	return true;
4554	#else
4555	return false;
4556	#endif
4557	}
4558
4559	#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4560	static void \
4561	expand_##CODE (internal_fn fn, gcall *stmt) \
4562	{ \
4563	expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
4564	}
4565	#define DEF_INTERNAL_INT_EXT_FN(CODE, FLAGS, OPTAB, TYPE)
4566	#define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4567	UNSIGNED_OPTAB, TYPE) \
4568	static void \
4569	expand_##CODE (internal_fn fn, gcall *stmt) \
4570	{ \
4571	tree_pair types = direct_internal_fn_types (fn, stmt); \
4572	optab which_optab = direct_internal_fn_optab (fn, types); \
4573	expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
4574	}
4575	#include "internal-fn.def"
4576
4577	/* Routines to expand each internal function, indexed by function number.
4578	Each routine has the prototype:
4579
4580	expand_<NAME> (gcall *stmt)
4581
4582	where STMT is the statement that performs the call. */
4583	static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
4584
4585	#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
4586	#include "internal-fn.def"
4587	0
4588	};
4589
4590	/* Invoke T(CODE, SUFFIX) for each conditional function IFN_COND_##SUFFIX
4591	that maps to a tree code CODE. There is also an IFN_COND_LEN_##SUFFIX
4592	for each such IFN_COND_##SUFFIX. */
4593	#define FOR_EACH_CODE_MAPPING(T) \
4594	T (PLUS_EXPR, ADD) \
4595	T (MINUS_EXPR, SUB) \
4596	T (MULT_EXPR, MUL) \
4597	T (TRUNC_DIV_EXPR, DIV) \
4598	T (TRUNC_MOD_EXPR, MOD) \
4599	T (RDIV_EXPR, RDIV) \
4600	T (MIN_EXPR, MIN) \
4601	T (MAX_EXPR, MAX) \
4602	T (BIT_AND_EXPR, AND) \
4603	T (BIT_IOR_EXPR, IOR) \
4604	T (BIT_XOR_EXPR, XOR) \
4605	T (LSHIFT_EXPR, SHL) \
4606	T (RSHIFT_EXPR, SHR) \
4607	T (NEGATE_EXPR, NEG)
4608
4609	/* Return a function that only performs CODE when a certain condition is met
4610	and that uses a given fallback value otherwise. For example, if CODE is
4611	a binary operation associated with conditional function FN:
4612
4613	LHS = FN (COND, A, B, ELSE)
4614
4615	is equivalent to the C expression:
4616
4617	LHS = COND ? A CODE B : ELSE;
4618
4619	operating elementwise if the operands are vectors.
4620
4621	Return IFN_LAST if no such function exists. */
4622
4623	internal_fn
4624	get_conditional_internal_fn (tree_code code)
4625	{
4626	switch (code)
4627	{
4628	#define CASE(CODE, IFN) case CODE: return IFN_COND_##IFN;
4629	FOR_EACH_CODE_MAPPING(CASE)
4630	#undef CASE
4631	default:
4632	return IFN_LAST;
4633	}
4634	}
4635
4636	/* If IFN implements the conditional form of a tree code, return that
4637	tree code, otherwise return ERROR_MARK. */
4638
4639	tree_code
4640	conditional_internal_fn_code (internal_fn ifn)
4641	{
4642	switch (ifn)
4643	{
4644	#define CASE(CODE, IFN) \
4645	case IFN_COND_##IFN: \
4646	case IFN_COND_LEN_##IFN: \
4647	return CODE;
4648	FOR_EACH_CODE_MAPPING (CASE)
4649	#undef CASE
4650	default:
4651	return ERROR_MARK;
4652	}
4653	}
4654
4655	/* Like get_conditional_internal_fn, but return a function that
4656	additionally restricts the operation to the leading elements
4657	of a vector. The number of elements to process is given by a length
4658	and bias pair, as for IFN_LOAD_LEN. The values of the remaining
4659	elements are taken from the fallback ("else") argument.
4660
4661	For example, if CODE is a binary operation associated with FN:
4662
4663	LHS = FN (COND, A, B, ELSE, LEN, BIAS)
4664
4665	is equivalent to the C code:
4666
4667	for (int i = 0; i < NUNITS; i++)
4668	{
4669	if (i < LEN + BIAS && COND[i])
4670	LHS[i] = A[i] CODE B[i];
4671	else
4672	LHS[i] = ELSE[i];
4673	}
4674	*/
4675
4676	internal_fn
4677	get_conditional_len_internal_fn (tree_code code)
4678	{
4679	switch (code)
4680	{
4681	#define CASE(CODE, IFN) case CODE: return IFN_COND_LEN_##IFN;
4682	FOR_EACH_CODE_MAPPING(CASE)
4683	#undef CASE
4684	default:
4685	return IFN_LAST;
4686	}
4687	}
4688
4689	/* Invoke T(IFN) for each internal function IFN that also has an
4690	IFN_COND_* form. */
4691	#define FOR_EACH_COND_FN_PAIR(T) \
4692	T (FMAX) \
4693	T (FMIN) \
4694	T (FMA) \
4695	T (FMS) \
4696	T (FNMA) \
4697	T (FNMS)
4698
4699	/* Return a function that only performs internal function FN when a
4700	certain condition is met and that uses a given fallback value otherwise.
4701	In other words, the returned function FN' is such that:
4702
4703	LHS = FN' (COND, A1, ... An, ELSE)
4704
4705	is equivalent to the C expression:
4706
4707	LHS = COND ? FN (A1, ..., An) : ELSE;
4708
4709	operating elementwise if the operands are vectors.
4710
4711	Return IFN_LAST if no such function exists. */
4712
4713	internal_fn
4714	get_conditional_internal_fn (internal_fn fn)
4715	{
4716	switch (fn)
4717	{
4718	#define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
4719	FOR_EACH_COND_FN_PAIR(CASE)
4720	#undef CASE
4721	default:
4722	return IFN_LAST;
4723	}
4724	}
4725
4726	/* If there exists an internal function like IFN that operates on vectors,
4727	but with additional length and bias parameters, return the internal_fn
4728	for that function, otherwise return IFN_LAST. */
4729	internal_fn
4730	get_len_internal_fn (internal_fn fn)
4731	{
4732	switch (fn)
4733	{
4734	#define DEF_INTERNAL_COND_FN(NAME, ...) \
4735	case IFN_COND_##NAME: \
4736	return IFN_COND_LEN_##NAME;
4737	#define DEF_INTERNAL_SIGNED_COND_FN(NAME, ...) \
4738	case IFN_COND_##NAME: \
4739	return IFN_COND_LEN_##NAME;
4740	#include "internal-fn.def"
4741	default:
4742	break;
4743	}
4744
4745	switch (fn)
4746	{
4747	case IFN_MASK_LOAD:
4748	return IFN_MASK_LEN_LOAD;
4749	case IFN_MASK_LOAD_LANES:
4750	return IFN_MASK_LEN_LOAD_LANES;
4751	case IFN_MASK_GATHER_LOAD:
4752	return IFN_MASK_LEN_GATHER_LOAD;
4753	default:
4754	return IFN_LAST;
4755	}
4756	}
4757
4758	/* If IFN implements the conditional form of an unconditional internal
4759	function, return that unconditional function, otherwise return IFN_LAST. */
4760
4761	internal_fn
4762	get_unconditional_internal_fn (internal_fn ifn)
4763	{
4764	switch (ifn)
4765	{
4766	#define CASE(NAME) \
4767	case IFN_COND_##NAME: \
4768	case IFN_COND_LEN_##NAME: \
4769	return IFN_##NAME;
4770	FOR_EACH_COND_FN_PAIR (CASE)
4771	#undef CASE
4772	default:
4773	return IFN_LAST;
4774	}
4775	}
4776
4777	/* Return true if STMT can be interpreted as a conditional tree code
4778	operation of the form:
4779
4780	LHS = COND ? OP (RHS1, ...) : ELSE;
4781
4782	operating elementwise if the operands are vectors. This includes
4783	the case of an all-true COND, so that the operation always happens.
4784
4785	There is an alternative approach to interpret the STMT when the operands
4786	are vectors which is the operation predicated by both conditional mask
4787	and loop control length, the equivalent C code:
4788
4789	for (int i = 0; i < NUNTIS; i++)
4790	{
4791	if (i < LEN + BIAS && COND[i])
4792	LHS[i] = A[i] CODE B[i];
4793	else
4794	LHS[i] = ELSE[i];
4795	}
4796
4797	When returning true, set:
4798
4799	- *COND_OUT to the condition COND, or to NULL_TREE if the condition
4800	is known to be all-true
4801	- *CODE_OUT to the tree code
4802	- OPS[I] to operand I of *CODE_OUT
4803	- *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
4804	condition is known to be all true.
4805	- *LEN to the len argument if it COND_LEN_* operations or to NULL_TREE.
4806	- *BIAS to the bias argument if it COND_LEN_* operations or to NULL_TREE. */
4807
4808	bool
4809	can_interpret_as_conditional_op_p (gimple *stmt, tree *cond_out,
4810	tree_code *code_out,
4811	tree (&ops)[3], tree *else_out,
4812	tree *len, tree *bias)
4813	{
4814	*len = NULL_TREE;
4815	*bias = NULL_TREE;
4816	if (gassign *assign = dyn_cast <gassign *> (p: stmt))
4817	{
4818	*cond_out = NULL_TREE;
4819	*code_out = gimple_assign_rhs_code (gs: assign);
4820	ops[0] = gimple_assign_rhs1 (gs: assign);
4821	ops[1] = gimple_assign_rhs2 (gs: assign);
4822	ops[2] = gimple_assign_rhs3 (gs: assign);
4823	*else_out = NULL_TREE;
4824	return true;
4825	}
4826	if (gcall *call = dyn_cast <gcall *> (p: stmt))
4827	if (gimple_call_internal_p (gs: call))
4828	{
4829	internal_fn ifn = gimple_call_internal_fn (gs: call);
4830	tree_code code = conditional_internal_fn_code (ifn);
4831	int len_index = internal_fn_len_index (ifn);
4832	int cond_nargs = len_index >= 0 ? 4 : 2;
4833	if (code != ERROR_MARK)
4834	{
4835	*cond_out = gimple_call_arg (gs: call, index: 0);
4836	*code_out = code;
4837	unsigned int nops = gimple_call_num_args (gs: call) - cond_nargs;
4838	for (unsigned int i = 0; i < 3; ++i)
4839	ops[i] = i < nops ? gimple_call_arg (gs: call, index: i + 1) : NULL_TREE;
4840	*else_out = gimple_call_arg (gs: call, index: nops + 1);
4841	if (len_index < 0)
4842	{
4843	if (integer_truep (*cond_out))
4844	{
4845	*cond_out = NULL_TREE;
4846	*else_out = NULL_TREE;
4847	}
4848	}
4849	else
4850	{
4851	*len = gimple_call_arg (gs: call, index: len_index);
4852	*bias = gimple_call_arg (gs: call, index: len_index + 1);
4853	}
4854	return true;
4855	}
4856	}
4857	return false;
4858	}
4859
4860	/* Return true if IFN is some form of load from memory. */
4861
4862	bool
4863	internal_load_fn_p (internal_fn fn)
4864	{
4865	switch (fn)
4866	{
4867	case IFN_MASK_LOAD:
4868	case IFN_LOAD_LANES:
4869	case IFN_MASK_LOAD_LANES:
4870	case IFN_MASK_LEN_LOAD_LANES:
4871	case IFN_GATHER_LOAD:
4872	case IFN_MASK_GATHER_LOAD:
4873	case IFN_MASK_LEN_GATHER_LOAD:
4874	case IFN_LEN_LOAD:
4875	case IFN_MASK_LEN_LOAD:
4876	return true;
4877
4878	default:
4879	return false;
4880	}
4881	}
4882
4883	/* Return true if IFN is some form of store to memory. */
4884
4885	bool
4886	internal_store_fn_p (internal_fn fn)
4887	{
4888	switch (fn)
4889	{
4890	case IFN_MASK_STORE:
4891	case IFN_STORE_LANES:
4892	case IFN_MASK_STORE_LANES:
4893	case IFN_MASK_LEN_STORE_LANES:
4894	case IFN_SCATTER_STORE:
4895	case IFN_MASK_SCATTER_STORE:
4896	case IFN_MASK_LEN_SCATTER_STORE:
4897	case IFN_LEN_STORE:
4898	case IFN_MASK_LEN_STORE:
4899	return true;
4900
4901	default:
4902	return false;
4903	}
4904	}
4905
4906	/* Return true if IFN is some form of gather load or scatter store. */
4907
4908	bool
4909	internal_gather_scatter_fn_p (internal_fn fn)
4910	{
4911	switch (fn)
4912	{
4913	case IFN_GATHER_LOAD:
4914	case IFN_MASK_GATHER_LOAD:
4915	case IFN_MASK_LEN_GATHER_LOAD:
4916	case IFN_SCATTER_STORE:
4917	case IFN_MASK_SCATTER_STORE:
4918	case IFN_MASK_LEN_SCATTER_STORE:
4919	return true;
4920
4921	default:
4922	return false;
4923	}
4924	}
4925
4926	/* If FN takes a vector len argument, return the index of that argument,
4927	otherwise return -1. */
4928
4929	int
4930	internal_fn_len_index (internal_fn fn)
4931	{
4932	switch (fn)
4933	{
4934	case IFN_LEN_LOAD:
4935	case IFN_LEN_STORE:
4936	return 2;
4937
4938	case IFN_MASK_LEN_SCATTER_STORE:
4939	case IFN_MASK_LEN_STRIDED_LOAD:
4940	return 5;
4941
4942	case IFN_MASK_LEN_GATHER_LOAD:
4943	return 6;
4944
4945	case IFN_COND_LEN_FMA:
4946	case IFN_COND_LEN_FMS:
4947	case IFN_COND_LEN_FNMA:
4948	case IFN_COND_LEN_FNMS:
4949	return 5;
4950
4951	case IFN_COND_LEN_ADD:
4952	case IFN_COND_LEN_SUB:
4953	case IFN_COND_LEN_MUL:
4954	case IFN_COND_LEN_DIV:
4955	case IFN_COND_LEN_MOD:
4956	case IFN_COND_LEN_RDIV:
4957	case IFN_COND_LEN_MIN:
4958	case IFN_COND_LEN_MAX:
4959	case IFN_COND_LEN_FMIN:
4960	case IFN_COND_LEN_FMAX:
4961	case IFN_COND_LEN_AND:
4962	case IFN_COND_LEN_IOR:
4963	case IFN_COND_LEN_XOR:
4964	case IFN_COND_LEN_SHL:
4965	case IFN_COND_LEN_SHR:
4966	case IFN_MASK_LEN_STRIDED_STORE:
4967	return 4;
4968
4969	case IFN_COND_LEN_NEG:
4970	case IFN_MASK_LEN_STORE:
4971	case IFN_MASK_LEN_STORE_LANES:
4972	case IFN_VCOND_MASK_LEN:
4973	return 3;
4974
4975	case IFN_MASK_LEN_LOAD:
4976	case IFN_MASK_LEN_LOAD_LANES:
4977	return 4;
4978
4979	default:
4980	return -1;
4981	}
4982	}
4983
4984	/* If FN is an IFN_COND_* or IFN_COND_LEN_* function, return the index of the
4985	argument that is used when the condition is false. Return -1 otherwise. */
4986
4987	int
4988	internal_fn_else_index (internal_fn fn)
4989	{
4990	switch (fn)
4991	{
4992	case IFN_COND_NEG:
4993	case IFN_COND_NOT:
4994	case IFN_COND_LEN_NEG:
4995	case IFN_COND_LEN_NOT:
4996	return 2;
4997
4998	case IFN_COND_ADD:
4999	case IFN_COND_SUB:
5000	case IFN_COND_MUL:
5001	case IFN_COND_DIV:
5002	case IFN_COND_MOD:
5003	case IFN_COND_MIN:
5004	case IFN_COND_MAX:
5005	case IFN_COND_FMIN:
5006	case IFN_COND_FMAX:
5007	case IFN_COND_AND:
5008	case IFN_COND_IOR:
5009	case IFN_COND_XOR:
5010	case IFN_COND_SHL:
5011	case IFN_COND_SHR:
5012	case IFN_COND_LEN_ADD:
5013	case IFN_COND_LEN_SUB:
5014	case IFN_COND_LEN_MUL:
5015	case IFN_COND_LEN_DIV:
5016	case IFN_COND_LEN_MOD:
5017	case IFN_COND_LEN_MIN:
5018	case IFN_COND_LEN_MAX:
5019	case IFN_COND_LEN_FMIN:
5020	case IFN_COND_LEN_FMAX:
5021	case IFN_COND_LEN_AND:
5022	case IFN_COND_LEN_IOR:
5023	case IFN_COND_LEN_XOR:
5024	case IFN_COND_LEN_SHL:
5025	case IFN_COND_LEN_SHR:
5026	return 3;
5027
5028	case IFN_MASK_LOAD:
5029	case IFN_MASK_LEN_LOAD:
5030	case IFN_MASK_LOAD_LANES:
5031	case IFN_MASK_LEN_LOAD_LANES:
5032	return 3;
5033
5034	case IFN_COND_FMA:
5035	case IFN_COND_FMS:
5036	case IFN_COND_FNMA:
5037	case IFN_COND_FNMS:
5038	case IFN_COND_LEN_FMA:
5039	case IFN_COND_LEN_FMS:
5040	case IFN_COND_LEN_FNMA:
5041	case IFN_COND_LEN_FNMS:
5042	case IFN_MASK_LEN_STRIDED_LOAD:
5043	return 4;
5044
5045	case IFN_MASK_GATHER_LOAD:
5046	case IFN_MASK_LEN_GATHER_LOAD:
5047	return 5;
5048
5049	default:
5050	return -1;
5051	}
5052
5053	return -1;
5054	}
5055
5056	/* If FN takes a vector mask argument, return the index of that argument,
5057	otherwise return -1. */
5058
5059	int
5060	internal_fn_mask_index (internal_fn fn)
5061	{
5062	switch (fn)
5063	{
5064	case IFN_MASK_LOAD:
5065	case IFN_MASK_LOAD_LANES:
5066	case IFN_MASK_LEN_LOAD_LANES:
5067	case IFN_MASK_STORE:
5068	case IFN_MASK_STORE_LANES:
5069	case IFN_MASK_LEN_STORE_LANES:
5070	case IFN_MASK_LEN_LOAD:
5071	case IFN_MASK_LEN_STORE:
5072	return 2;
5073
5074	case IFN_MASK_LEN_STRIDED_LOAD:
5075	case IFN_MASK_LEN_STRIDED_STORE:
5076	return 3;
5077
5078	case IFN_MASK_GATHER_LOAD:
5079	case IFN_MASK_SCATTER_STORE:
5080	case IFN_MASK_LEN_GATHER_LOAD:
5081	case IFN_MASK_LEN_SCATTER_STORE:
5082	return 4;
5083
5084	case IFN_VCOND_MASK:
5085	case IFN_VCOND_MASK_LEN:
5086	return 0;
5087
5088	default:
5089	return (conditional_internal_fn_code (ifn: fn) != ERROR_MARK
5090	|| get_unconditional_internal_fn (ifn: fn) != IFN_LAST ? 0 : -1);
5091	}
5092	}
5093
5094	/* If FN takes a value that should be stored to memory, return the index
5095	of that argument, otherwise return -1. */
5096
5097	int
5098	internal_fn_stored_value_index (internal_fn fn)
5099	{
5100	switch (fn)
5101	{
5102	case IFN_MASK_LEN_STRIDED_STORE:
5103	return 2;
5104
5105	case IFN_MASK_STORE:
5106	case IFN_MASK_STORE_LANES:
5107	case IFN_SCATTER_STORE:
5108	case IFN_MASK_SCATTER_STORE:
5109	case IFN_MASK_LEN_SCATTER_STORE:
5110	return 3;
5111
5112	case IFN_LEN_STORE:
5113	return 4;
5114
5115	case IFN_MASK_LEN_STORE:
5116	case IFN_MASK_LEN_STORE_LANES:
5117	return 5;
5118
5119	default:
5120	return -1;
5121	}
5122	}
5123
5124
5125	/* Store all supported else values for the optab referred to by ICODE
5126	in ELSE_VALS. The index of the else operand must be specified in
5127	ELSE_INDEX. */
5128
5129	void
5130	get_supported_else_vals (enum insn_code icode, unsigned else_index,
5131	vec<int> &else_vals)
5132	{
5133	const struct insn_data_d *data = &insn_data[icode];
5134	if ((char)else_index >= data->n_operands)
5135	return;
5136
5137	machine_mode else_mode = data->operand[else_index].mode;
5138
5139	else_vals.truncate (size: 0);
5140
5141	/* For now we only support else values of 0, -1, and "undefined". */
5142	if (insn_operand_matches (icode, opno: else_index, CONST0_RTX (else_mode)))
5143	else_vals.safe_push (MASK_LOAD_ELSE_ZERO);
5144
5145	if (insn_operand_matches (icode, opno: else_index, gen_rtx_SCRATCH (else_mode)))
5146	else_vals.safe_push (MASK_LOAD_ELSE_UNDEFINED);
5147
5148	if (GET_MODE_CLASS (else_mode) == MODE_VECTOR_INT
5149	&& insn_operand_matches (icode, opno: else_index, CONSTM1_RTX (else_mode)))
5150	else_vals.safe_push (MASK_LOAD_ELSE_M1);
5151	}
5152
5153	/* Return true if the else value ELSE_VAL (one of MASK_LOAD_ELSE_ZERO,
5154	MASK_LOAD_ELSE_M1, and MASK_LOAD_ELSE_UNDEFINED) is valid fo the optab
5155	referred to by ICODE. The index of the else operand must be specified
5156	in ELSE_INDEX. */
5157
5158	bool
5159	supported_else_val_p (enum insn_code icode, unsigned else_index, int else_val)
5160	{
5161	if (else_val != MASK_LOAD_ELSE_ZERO && else_val != MASK_LOAD_ELSE_M1
5162	&& else_val != MASK_LOAD_ELSE_UNDEFINED)
5163	gcc_unreachable ();
5164
5165	auto_vec<int> else_vals;
5166	get_supported_else_vals (icode, else_index, else_vals);
5167	return else_vals.contains (search: else_val);
5168	}
5169
5170	/* Return true if the target supports gather load or scatter store function
5171	IFN. For loads, VECTOR_TYPE is the vector type of the load result,
5172	while for stores it is the vector type of the stored data argument.
5173	MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
5174	or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
5175	offset from the shared base address of each loaded or stored element.
5176	SCALE is the amount by which these offsets should be multiplied
5177	*after* they have been extended to address width.
5178	If the target supports the gather load the supported else values
5179	will be added to the vector ELSVAL points to if it is nonzero. */
5180
5181	bool
5182	internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
5183	tree memory_element_type,
5184	tree offset_vector_type, int scale,
5185	vec<int> *elsvals)
5186	{
5187	if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)),
5188	TYPE_SIZE (memory_element_type)))
5189	return false;
5190	if (maybe_ne (a: TYPE_VECTOR_SUBPARTS (node: vector_type),
5191	b: TYPE_VECTOR_SUBPARTS (node: offset_vector_type)))
5192	return false;
5193	optab optab = direct_internal_fn_optab (fn: ifn);
5194	insn_code icode = convert_optab_handler (op: optab, TYPE_MODE (vector_type),
5195	TYPE_MODE (offset_vector_type));
5196	int output_ops = internal_load_fn_p (fn: ifn) ? 1 : 0;
5197	bool unsigned_p = TYPE_UNSIGNED (TREE_TYPE (offset_vector_type));
5198	bool ok = icode != CODE_FOR_nothing
5199	&& insn_operand_matches (icode, opno: 2 + output_ops, GEN_INT (unsigned_p))
5200	&& insn_operand_matches (icode, opno: 3 + output_ops, GEN_INT (scale));
5201
5202	/* For gather the optab's operand indices do not match the IFN's because
5203	the latter does not have the extension operand (operand 3). It is
5204	implicitly added during expansion so we use the IFN's else index + 1.
5205	*/
5206	if (ok && elsvals)
5207	get_supported_else_vals
5208	(icode, else_index: internal_fn_else_index (fn: IFN_MASK_GATHER_LOAD) + 1, else_vals&: *elsvals);
5209
5210	return ok;
5211	}
5212
5213	/* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
5214	for pointers of type TYPE when the accesses have LENGTH bytes and their
5215	common byte alignment is ALIGN. */
5216
5217	bool
5218	internal_check_ptrs_fn_supported_p (internal_fn ifn, tree type,
5219	poly_uint64 length, unsigned int align)
5220	{
5221	machine_mode mode = TYPE_MODE (type);
5222	optab optab = direct_internal_fn_optab (fn: ifn);
5223	insn_code icode = direct_optab_handler (op: optab, mode);
5224	if (icode == CODE_FOR_nothing)
5225	return false;
5226	rtx length_rtx = immed_wide_int_const (length, mode);
5227	return (insn_operand_matches (icode, opno: 3, operand: length_rtx)
5228	&& insn_operand_matches (icode, opno: 4, GEN_INT (align)));
5229	}
5230
5231	/* Return the supported bias for IFN which is either IFN_{LEN_,MASK_LEN_,}LOAD
5232	or IFN_{LEN_,MASK_LEN_,}STORE. For now we only support the biases of 0 and
5233	-1 (in case 0 is not an allowable length for {len_,mask_len_}load or
5234	{len_,mask_len_}store). If none of the biases match what the backend
5235	provides, return VECT_PARTIAL_BIAS_UNSUPPORTED. */
5236
5237	signed char
5238	internal_len_load_store_bias (internal_fn ifn, machine_mode mode)
5239	{
5240	optab optab = direct_internal_fn_optab (fn: ifn);
5241	insn_code icode = direct_optab_handler (op: optab, mode);
5242	int bias_no = 3;
5243
5244	if (icode == CODE_FOR_nothing)
5245	{
5246	machine_mode mask_mode;
5247	if (!targetm.vectorize.get_mask_mode (mode).exists (mode: &mask_mode))
5248	return VECT_PARTIAL_BIAS_UNSUPPORTED;
5249	if (ifn == IFN_LEN_LOAD)
5250	{
5251	/* Try MASK_LEN_LOAD. */
5252	optab = direct_internal_fn_optab (fn: IFN_MASK_LEN_LOAD);
5253	}
5254	else
5255	{
5256	/* Try MASK_LEN_STORE. */
5257	optab = direct_internal_fn_optab (fn: IFN_MASK_LEN_STORE);
5258	}
5259	icode = convert_optab_handler (op: optab, to_mode: mode, from_mode: mask_mode);
5260	bias_no = 4;
5261	}
5262
5263	if (icode != CODE_FOR_nothing)
5264	{
5265	/* For now we only support biases of 0 or -1. Try both of them. */
5266	if (insn_operand_matches (icode, opno: bias_no, GEN_INT (0)))
5267	return 0;
5268	if (insn_operand_matches (icode, opno: bias_no, GEN_INT (-1)))
5269	return -1;
5270	}
5271
5272	return VECT_PARTIAL_BIAS_UNSUPPORTED;
5273	}
5274
5275	/* Expand STMT as though it were a call to internal function FN. */
5276
5277	void
5278	expand_internal_call (internal_fn fn, gcall *stmt)
5279	{
5280	internal_fn_expanders[fn] (fn, stmt);
5281	}
5282
5283	/* Expand STMT, which is a call to internal function FN. */
5284
5285	void
5286	expand_internal_call (gcall *stmt)
5287	{
5288	expand_internal_call (fn: gimple_call_internal_fn (gs: stmt), stmt);
5289	}
5290
5291	/* If TYPE is a vector type, return true if IFN is a direct internal
5292	function that is supported for that type. If TYPE is a scalar type,
5293	return true if IFN is a direct internal function that is supported for
5294	the target's preferred vector version of TYPE. */
5295
5296	bool
5297	vectorized_internal_fn_supported_p (internal_fn ifn, tree type)
5298	{
5299	if (VECTOR_MODE_P (TYPE_MODE (type)))
5300	return direct_internal_fn_supported_p (fn: ifn, type, opt_type: OPTIMIZE_FOR_SPEED);
5301
5302	scalar_mode smode;
5303	if (VECTOR_TYPE_P (type)
5304	|| !is_a <scalar_mode> (TYPE_MODE (type), result: &smode))
5305	return false;
5306
5307	machine_mode vmode = targetm.vectorize.preferred_simd_mode (smode);
5308	if (VECTOR_MODE_P (vmode))
5309	{
5310	tree vectype = build_vector_type_for_mode (type, vmode);
5311	if (direct_internal_fn_supported_p (fn: ifn, type: vectype, opt_type: OPTIMIZE_FOR_SPEED))
5312	return true;
5313	}
5314
5315	auto_vector_modes vector_modes;
5316	targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
5317	for (machine_mode base_mode : vector_modes)
5318	if (related_vector_mode (base_mode, smode).exists (mode: &vmode))
5319	{
5320	tree vectype = build_vector_type_for_mode (type, vmode);
5321	if (direct_internal_fn_supported_p (fn: ifn, type: vectype, opt_type: OPTIMIZE_FOR_SPEED))
5322	return true;
5323	}
5324
5325	return false;
5326	}
5327
5328	void
5329	expand_SHUFFLEVECTOR (internal_fn, gcall *)
5330	{
5331	gcc_unreachable ();
5332	}
5333
5334	void
5335	expand_PHI (internal_fn, gcall *)
5336	{
5337	gcc_unreachable ();
5338	}
5339
5340	void
5341	expand_SPACESHIP (internal_fn, gcall *stmt)
5342	{
5343	tree lhs = gimple_call_lhs (gs: stmt);
5344	tree rhs1 = gimple_call_arg (gs: stmt, index: 0);
5345	tree rhs2 = gimple_call_arg (gs: stmt, index: 1);
5346	tree rhs3 = gimple_call_arg (gs: stmt, index: 2);
5347	tree type = TREE_TYPE (rhs1);
5348
5349	do_pending_stack_adjust ();
5350
5351	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
5352	rtx op1 = expand_normal (exp: rhs1);
5353	rtx op2 = expand_normal (exp: rhs2);
5354	rtx op3 = expand_normal (exp: rhs3);
5355
5356	class expand_operand ops[4];
5357	create_call_lhs_operand (op: &ops[0], lhs_rtx: target, TYPE_MODE (TREE_TYPE (lhs)));
5358	create_input_operand (op: &ops[1], value: op1, TYPE_MODE (type));
5359	create_input_operand (op: &ops[2], value: op2, TYPE_MODE (type));
5360	create_input_operand (op: &ops[3], value: op3, TYPE_MODE (TREE_TYPE (rhs3)));
5361	insn_code icode = optab_handler (op: spaceship_optab, TYPE_MODE (type));
5362	expand_insn (icode, nops: 4, ops);
5363	assign_call_lhs (lhs, lhs_rtx: target, op: &ops[0]);
5364	}
5365
5366	void
5367	expand_ASSUME (internal_fn, gcall *)
5368	{
5369	}
5370
5371	void
5372	expand_MASK_CALL (internal_fn, gcall *)
5373	{
5374	/* This IFN should only exist between ifcvt and vect passes. */
5375	gcc_unreachable ();
5376	}
5377
5378	void
5379	expand_MULBITINT (internal_fn, gcall *stmt)
5380	{
5381	rtx_mode_t args[6];
5382	for (int i = 0; i < 6; i++)
5383	args[i] = rtx_mode_t (expand_normal (exp: gimple_call_arg (gs: stmt, index: i)),
5384	(i & 1) ? SImode : ptr_mode);
5385	rtx fun = init_one_libfunc ("__mulbitint3");
5386	emit_library_call_value_1 (0, fun, NULL_RTX, LCT_NORMAL, VOIDmode, 6, args);
5387	}
5388
5389	void
5390	expand_DIVMODBITINT (internal_fn, gcall *stmt)
5391	{
5392	rtx_mode_t args[8];
5393	for (int i = 0; i < 8; i++)
5394	args[i] = rtx_mode_t (expand_normal (exp: gimple_call_arg (gs: stmt, index: i)),
5395	(i & 1) ? SImode : ptr_mode);
5396	rtx fun = init_one_libfunc ("__divmodbitint4");
5397	emit_library_call_value_1 (0, fun, NULL_RTX, LCT_NORMAL, VOIDmode, 8, args);
5398	}
5399
5400	void
5401	expand_FLOATTOBITINT (internal_fn, gcall *stmt)
5402	{
5403	machine_mode mode = TYPE_MODE (TREE_TYPE (gimple_call_arg (stmt, 2)));
5404	rtx arg0 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
5405	rtx arg1 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
5406	rtx arg2 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 2));
5407	const char *mname = GET_MODE_NAME (mode);
5408	unsigned mname_len = strlen (s: mname);
5409	int len = 12 + mname_len;
5410	if (DECIMAL_FLOAT_MODE_P (mode))
5411	len += 4;
5412	char *libfunc_name = XALLOCAVEC (char, len);
5413	char *p = libfunc_name;
5414	const char *q;
5415	if (DECIMAL_FLOAT_MODE_P (mode))
5416	{
5417	#if ENABLE_DECIMAL_BID_FORMAT
5418	memcpy (dest: p, src: "__bid_fix", n: 9);
5419	#else
5420	memcpy (p, "__dpd_fix", 9);
5421	#endif
5422	p += 9;
5423	}
5424	else
5425	{
5426	memcpy (dest: p, src: "__fix", n: 5);
5427	p += 5;
5428	}
5429	for (q = mname; *q; q++)
5430	*p++ = TOLOWER (*q);
5431	memcpy (dest: p, src: "bitint", n: 7);
5432	rtx fun = init_one_libfunc (libfunc_name);
5433	emit_library_call (fun, fn_type: LCT_NORMAL, VOIDmode, arg1: arg0, arg1_mode: ptr_mode, arg2: arg1,
5434	SImode, arg3: arg2, arg3_mode: mode);
5435	}
5436
5437	void
5438	expand_BITINTTOFLOAT (internal_fn, gcall *stmt)
5439	{
5440	tree lhs = gimple_call_lhs (gs: stmt);
5441	if (!lhs)
5442	return;
5443	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
5444	rtx arg0 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 0));
5445	rtx arg1 = expand_normal (exp: gimple_call_arg (gs: stmt, index: 1));
5446	const char *mname = GET_MODE_NAME (mode);
5447	unsigned mname_len = strlen (s: mname);
5448	int len = 14 + mname_len;
5449	if (DECIMAL_FLOAT_MODE_P (mode))
5450	len += 4;
5451	char *libfunc_name = XALLOCAVEC (char, len);
5452	char *p = libfunc_name;
5453	const char *q;
5454	if (DECIMAL_FLOAT_MODE_P (mode))
5455	{
5456	#if ENABLE_DECIMAL_BID_FORMAT
5457	memcpy (dest: p, src: "__bid_floatbitint", n: 17);
5458	#else
5459	memcpy (p, "__dpd_floatbitint", 17);
5460	#endif
5461	p += 17;
5462	}
5463	else
5464	{
5465	memcpy (dest: p, src: "__floatbitint", n: 13);
5466	p += 13;
5467	}
5468	for (q = mname; *q; q++)
5469	*p++ = TOLOWER (*q);
5470	*p = '\0';
5471	rtx fun = init_one_libfunc (libfunc_name);
5472	rtx target = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
5473	rtx val = emit_library_call_value (fun, value: target, fn_type: LCT_PURE, outmode: mode,
5474	arg1: arg0, arg1_mode: ptr_mode, arg2: arg1, SImode);
5475	if (val != target)
5476	emit_move_insn (target, val);
5477	}
5478
5479	static bool
5480	expand_bitquery (internal_fn fn, gcall *stmt)
5481	{
5482	tree lhs = gimple_call_lhs (gs: stmt);
5483	if (lhs == NULL_TREE)
5484	return false;
5485	tree arg = gimple_call_arg (gs: stmt, index: 0);
5486	if (TREE_CODE (arg) == INTEGER_CST)
5487	{
5488	tree ret = fold_const_call (as_combined_fn (fn), TREE_TYPE (arg), arg);
5489	gcc_checking_assert (ret && TREE_CODE (ret) == INTEGER_CST);
5490	expand_assignment (lhs, ret, false);
5491	return false;
5492	}
5493	return true;
5494	}
5495
5496	void
5497	expand_CLRSB (internal_fn fn, gcall *stmt)
5498	{
5499	if (expand_bitquery (fn, stmt))
5500	expand_unary_optab_fn (fn, stmt, clrsb_optab);
5501	}
5502
5503	void
5504	expand_CLZ (internal_fn fn, gcall *stmt)
5505	{
5506	if (expand_bitquery (fn, stmt))
5507	expand_unary_optab_fn (fn, stmt, clz_optab);
5508	}
5509
5510	void
5511	expand_CTZ (internal_fn fn, gcall *stmt)
5512	{
5513	if (expand_bitquery (fn, stmt))
5514	expand_unary_optab_fn (fn, stmt, ctz_optab);
5515	}
5516
5517	void
5518	expand_FFS (internal_fn fn, gcall *stmt)
5519	{
5520	if (expand_bitquery (fn, stmt))
5521	expand_unary_optab_fn (fn, stmt, ffs_optab);
5522	}
5523
5524	void
5525	expand_PARITY (internal_fn fn, gcall *stmt)
5526	{
5527	if (expand_bitquery (fn, stmt))
5528	expand_unary_optab_fn (fn, stmt, parity_optab);
5529	}
5530
5531	void
5532	expand_POPCOUNT (internal_fn fn, gcall *stmt)
5533	{
5534	if (!expand_bitquery (fn, stmt))
5535	return;
5536	if (gimple_call_num_args (gs: stmt) == 1)
5537	{
5538	expand_unary_optab_fn (fn, stmt, popcount_optab);
5539	return;
5540	}
5541	/* If .POPCOUNT call has 2 arguments, match_single_bit_test marked it
5542	because the result is only used in an equality comparison against 1.
5543	Use rtx costs in that case to determine if .POPCOUNT (arg) == 1
5544	or (arg ^ (arg - 1)) > arg - 1 is cheaper.
5545	If .POPCOUNT second argument is 0, we additionally know that arg
5546	is non-zero, so use arg & (arg - 1) == 0 instead.
5547	If .POPCOUNT second argument is -1, the comparison was either `<= 1`
5548	or `> 1`. */
5549	bool speed_p = optimize_insn_for_speed_p ();
5550	tree lhs = gimple_call_lhs (gs: stmt);
5551	tree arg = gimple_call_arg (gs: stmt, index: 0);
5552	bool nonzero_arg = integer_zerop (gimple_call_arg (gs: stmt, index: 1));
5553	bool was_le = integer_minus_onep (gimple_call_arg (gs: stmt, index: 1));
5554	if (was_le)
5555	nonzero_arg = true;
5556	tree type = TREE_TYPE (arg);
5557	machine_mode mode = TYPE_MODE (type);
5558	machine_mode lhsmode = TYPE_MODE (TREE_TYPE (lhs));
5559	do_pending_stack_adjust ();
5560	start_sequence ();
5561	expand_unary_optab_fn (fn, stmt, popcount_optab);
5562	rtx_insn *popcount_insns = end_sequence ();
5563	start_sequence ();
5564	rtx plhs = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
5565	rtx pcmp = emit_store_flag (NULL_RTX, EQ, plhs, const1_rtx, lhsmode, 0, 0);
5566	if (pcmp == NULL_RTX)
5567	{
5568	fail:
5569	end_sequence ();
5570	emit_insn (popcount_insns);
5571	return;
5572	}
5573	rtx_insn *popcount_cmp_insns = end_sequence ();
5574	start_sequence ();
5575	rtx op0 = expand_normal (exp: arg);
5576	rtx argm1 = expand_simple_binop (mode, PLUS, op0, constm1_rtx, NULL_RTX,
5577	1, OPTAB_WIDEN);
5578	if (argm1 == NULL_RTX)
5579	goto fail;
5580	rtx argxorargm1 = expand_simple_binop (mode, nonzero_arg ? AND : XOR, op0,
5581	argm1, NULL_RTX, 1, OPTAB_WIDEN);
5582	if (argxorargm1 == NULL_RTX)
5583	goto fail;
5584	rtx cmp;
5585	if (nonzero_arg)
5586	cmp = emit_store_flag (NULL_RTX, EQ, argxorargm1, const0_rtx, mode, 1, 1);
5587	else
5588	cmp = emit_store_flag (NULL_RTX, GTU, argxorargm1, argm1, mode, 1, 1);
5589	if (cmp == NULL_RTX)
5590	goto fail;
5591	rtx_insn *cmp_insns = end_sequence ();
5592	unsigned popcount_cost = (seq_cost (popcount_insns, speed_p)
5593	+ seq_cost (popcount_cmp_insns, speed_p));
5594	unsigned cmp_cost = seq_cost (cmp_insns, speed_p);
5595
5596	if (dump_file && (dump_flags & TDF_DETAILS))
5597	fprintf(stream: dump_file, format: "popcount == 1: popcount cost: %u; cmp cost: %u\n",
5598	popcount_cost, cmp_cost);
5599
5600	if (popcount_cost <= cmp_cost)
5601	emit_insn (popcount_insns);
5602	else
5603	{
5604	start_sequence ();
5605	emit_insn (cmp_insns);
5606	plhs = expand_expr (exp: lhs, NULL_RTX, VOIDmode, modifier: EXPAND_WRITE);
5607	if (GET_MODE (cmp) != GET_MODE (plhs))
5608	cmp = convert_to_mode (GET_MODE (plhs), cmp, 1);
5609	/* For `<= 1`, we need to produce `2 - cmp` or `cmp ? 1 : 2` as that
5610	then gets compared against 1 and we need the false case to be 2. */
5611	if (was_le)
5612	{
5613	cmp = expand_simple_binop (GET_MODE (cmp), MINUS, const2_rtx,
5614	cmp, NULL_RTX, 1, OPTAB_WIDEN);
5615	if (!cmp)
5616	goto fail;
5617	}
5618	emit_move_insn (plhs, cmp);
5619	rtx_insn *all_insns = end_sequence ();
5620	emit_insn (all_insns);
5621	}
5622	}
5623