1/* Machine-dependent ELF dynamic relocation inline functions. x86-64 version.
2 Copyright (C) 2001-2024 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#ifndef dl_machine_h
20#define dl_machine_h
21
22#define ELF_MACHINE_NAME "x86_64"
23
24#include <assert.h>
25#include <stdint.h>
26#include <sys/param.h>
27#include <sysdep.h>
28#include <tls.h>
29#include <dl-tlsdesc.h>
30#include <dl-static-tls.h>
31#include <dl-machine-rel.h>
32#include <isa-level.h>
33#ifdef __CET__
34# include <dl-cet.h>
35#else
36# define RTLD_START_ENABLE_X86_FEATURES
37#endif
38
39/* Translate a processor specific dynamic tag to the index in l_info array. */
40#define DT_X86_64(x) (DT_X86_64_##x - DT_LOPROC + DT_NUM)
41
42/* Return nonzero iff ELF header is compatible with the running host. */
43static inline int __attribute__ ((unused))
44elf_machine_matches_host (const ElfW(Ehdr) *ehdr)
45{
46 return ehdr->e_machine == EM_X86_64;
47}
48
49
50/* Return the run-time load address of the shared object. */
51static inline ElfW(Addr) __attribute__ ((unused))
52elf_machine_load_address (void)
53{
54 extern const ElfW(Ehdr) __ehdr_start attribute_hidden;
55 return (ElfW(Addr)) &__ehdr_start;
56}
57
58/* Return the link-time address of _DYNAMIC. */
59static inline ElfW(Addr) __attribute__ ((unused))
60elf_machine_dynamic (void)
61{
62 extern ElfW(Dyn) _DYNAMIC[] attribute_hidden;
63 return (ElfW(Addr)) _DYNAMIC - elf_machine_load_address ();
64}
65
66/* Set up the loaded object described by L so its unrelocated PLT
67 entries will jump to the on-demand fixup code in dl-runtime.c. */
68
69static inline int __attribute__ ((unused, always_inline))
70elf_machine_runtime_setup (struct link_map *l, struct r_scope_elem *scope[],
71 int lazy, int profile)
72{
73 Elf64_Addr *got;
74 extern void _dl_runtime_profile_sse (ElfW(Word)) attribute_hidden;
75 extern void _dl_runtime_profile_avx (ElfW(Word)) attribute_hidden;
76 extern void _dl_runtime_profile_avx512 (ElfW(Word)) attribute_hidden;
77
78 if (l->l_info[DT_JMPREL] && lazy)
79 {
80 /* The GOT entries for functions in the PLT have not yet been filled
81 in. Their initial contents will arrange when called to push an
82 offset into the .rel.plt section, push _GLOBAL_OFFSET_TABLE_[1],
83 and then jump to _GLOBAL_OFFSET_TABLE_[2]. */
84 got = (Elf64_Addr *) D_PTR (l, l_info[DT_PLTGOT]);
85 /* If a library is prelinked but we have to relocate anyway,
86 we have to be able to undo the prelinking of .got.plt.
87 The prelinker saved us here address of .plt + 0x16. */
88 if (got[1])
89 {
90 l->l_mach.plt = got[1] + l->l_addr;
91 l->l_mach.gotplt = (ElfW(Addr)) &got[3];
92 }
93 /* Identify this shared object. */
94 *(ElfW(Addr) *) (got + 1) = (ElfW(Addr)) l;
95
96#ifdef SHARED
97 /* The got[2] entry contains the address of a function which gets
98 called to get the address of a so far unresolved function and
99 jump to it. The profiling extension of the dynamic linker allows
100 to intercept the calls to collect information. In this case we
101 don't store the address in the GOT so that all future calls also
102 end in this function. */
103 if (__glibc_unlikely (profile))
104 {
105 const struct cpu_features* cpu_features = __get_cpu_features ();
106 if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512F))
107 *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_profile_avx512;
108 else if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX))
109 *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_profile_avx;
110 else
111 *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_profile_sse;
112
113 if (GLRO(dl_profile) != NULL
114 && _dl_name_match_p (GLRO(dl_profile), map: l))
115 /* This is the object we are looking for. Say that we really
116 want profiling and the timers are started. */
117 GL(dl_profile_map) = l;
118 }
119 else
120#endif
121 {
122 /* This function will get called to fix up the GOT entry
123 indicated by the offset on the stack, and then jump to
124 the resolved address. */
125 *(ElfW(Addr) *) (got + 2)
126 = (ElfW(Addr)) GLRO(dl_x86_64_runtime_resolve);
127 }
128 }
129
130 return lazy;
131}
132
133/* Initial entry point code for the dynamic linker.
134 The C function `_dl_start' is the real entry point;
135 its return value is the user program's entry point. */
136#define RTLD_START asm ("\n\
137.text\n\
138 .align 16\n\
139.globl _start\n\
140.globl _dl_start_user\n\
141_start:\n\
142 movq %rsp, %rdi\n\
143 call _dl_start\n\
144_dl_start_user:\n\
145 # Save the user entry point address in %r12.\n\
146 movq %rax, %r12\n\
147 # Save %rsp value in %r13.\n\
148 movq %rsp, %r13\n\
149"\
150 RTLD_START_ENABLE_X86_FEATURES \
151"\
152 # Read the original argument count.\n\
153 movq (%rsp), %rdx\n\
154 # Call _dl_init (struct link_map *main_map, int argc, char **argv, char **env)\n\
155 # argc -> rsi\n\
156 movq %rdx, %rsi\n\
157 # And align stack for the _dl_init call. \n\
158 andq $-16, %rsp\n\
159 # _dl_loaded -> rdi\n\
160 movq _rtld_local(%rip), %rdi\n\
161 # env -> rcx\n\
162 leaq 16(%r13,%rdx,8), %rcx\n\
163 # argv -> rdx\n\
164 leaq 8(%r13), %rdx\n\
165 # Clear %rbp to mark outermost frame obviously even for constructors.\n\
166 xorl %ebp, %ebp\n\
167 # Call the function to run the initializers.\n\
168 call _dl_init\n\
169 # Pass our finalizer function to the user in %rdx, as per ELF ABI.\n\
170 leaq _dl_fini(%rip), %rdx\n\
171 # And make sure %rsp points to argc stored on the stack.\n\
172 movq %r13, %rsp\n\
173 # Jump to the user's entry point.\n\
174 jmp *%r12\n\
175.previous\n\
176");
177
178/* ELF_RTYPE_CLASS_PLT iff TYPE describes relocation of a PLT entry or
179 TLS variable, so undefined references should not be allowed to
180 define the value.
181 ELF_RTYPE_CLASS_COPY iff TYPE should not be allowed to resolve to one
182 of the main executable's symbols, as for a COPY reloc. */
183#define elf_machine_type_class(type) \
184 ((((type) == R_X86_64_JUMP_SLOT \
185 || (type) == R_X86_64_DTPMOD64 \
186 || (type) == R_X86_64_DTPOFF64 \
187 || (type) == R_X86_64_TPOFF64 \
188 || (type) == R_X86_64_TLSDESC) \
189 * ELF_RTYPE_CLASS_PLT) \
190 | (((type) == R_X86_64_COPY) * ELF_RTYPE_CLASS_COPY))
191
192/* A reloc type used for ld.so cmdline arg lookups to reject PLT entries. */
193#define ELF_MACHINE_JMP_SLOT R_X86_64_JUMP_SLOT
194
195/* The relative ifunc relocation. */
196// XXX This is a work-around for a broken linker. Remove!
197#define ELF_MACHINE_IRELATIVE R_X86_64_IRELATIVE
198
199/* We define an initialization function. This is called very early in
200 _dl_sysdep_start. */
201#define DL_PLATFORM_INIT dl_platform_init ()
202
203static inline void __attribute__ ((unused))
204dl_platform_init (void)
205{
206#if IS_IN (rtld)
207 /* _dl_x86_init_cpu_features is a wrapper for init_cpu_features which
208 has been called early from __libc_start_main in static executable. */
209 _dl_x86_init_cpu_features ();
210#else
211 if (GLRO(dl_platform) != NULL && *GLRO(dl_platform) == '\0')
212 /* Avoid an empty string which would disturb us. */
213 GLRO(dl_platform) = NULL;
214#endif
215}
216
217static inline ElfW(Addr)
218elf_machine_fixup_plt (struct link_map *map, lookup_t t,
219 const ElfW(Sym) *refsym, const ElfW(Sym) *sym,
220 const ElfW(Rela) *reloc,
221 ElfW(Addr) *reloc_addr, ElfW(Addr) value)
222{
223 return *reloc_addr = value;
224}
225
226/* Return the final value of a PLT relocation. On x86-64 the
227 JUMP_SLOT relocation ignores the addend. */
228static inline ElfW(Addr)
229elf_machine_plt_value (struct link_map *map, const ElfW(Rela) *reloc,
230 ElfW(Addr) value)
231{
232 return value;
233}
234
235
236/* Names of the architecture-specific auditing callback functions. */
237#define ARCH_LA_PLTENTER x86_64_gnu_pltenter
238#define ARCH_LA_PLTEXIT x86_64_gnu_pltexit
239
240#endif /* !dl_machine_h */
241
242#ifdef RESOLVE_MAP
243
244/* Perform the relocation specified by RELOC and SYM (which is fully resolved).
245 MAP is the object containing the reloc. */
246
247static inline void __attribute__((always_inline))
248elf_machine_rela(struct link_map *map, struct r_scope_elem *scope[],
249 const ElfW(Rela) *reloc, const ElfW(Sym) *sym,
250 const struct r_found_version *version,
251 void *const reloc_addr_arg, int skip_ifunc) {
252 ElfW(Addr) *const reloc_addr = reloc_addr_arg;
253 const unsigned long int r_type = ELFW(R_TYPE) (reloc->r_info);
254
255# if !defined RTLD_BOOTSTRAP
256 if (__glibc_unlikely (r_type == R_X86_64_RELATIVE))
257 *reloc_addr = map->l_addr + reloc->r_addend;
258 else
259# endif
260# if !defined RTLD_BOOTSTRAP
261 /* l_addr + r_addend may be > 0xffffffff and R_X86_64_RELATIVE64
262 relocation updates the whole 64-bit entry. */
263 if (__glibc_unlikely (r_type == R_X86_64_RELATIVE64))
264 *(Elf64_Addr *) reloc_addr = (Elf64_Addr) map->l_addr + reloc->r_addend;
265 else
266# endif
267 if (__glibc_unlikely (r_type == R_X86_64_NONE))
268 return;
269 else
270 {
271# ifndef RTLD_BOOTSTRAP
272 const ElfW(Sym) *const refsym = sym;
273# endif
274 struct link_map *sym_map = RESOLVE_MAP (map, scope, &sym, version,
275 r_type);
276 ElfW(Addr) value = SYMBOL_ADDRESS (sym_map, sym, true);
277
278 if (sym != NULL
279 && __glibc_unlikely (ELFW(ST_TYPE) (sym->st_info) == STT_GNU_IFUNC)
280 && __glibc_likely (sym->st_shndx != SHN_UNDEF)
281 && __glibc_likely (!skip_ifunc))
282 {
283# ifndef RTLD_BOOTSTRAP
284 if (sym_map != map
285 && !sym_map->l_relocated)
286 {
287 const char *strtab
288 = (const char *) D_PTR (map, l_info[DT_STRTAB]);
289 if (sym_map->l_type == lt_executable)
290 _dl_fatal_printf ("\
291%s: IFUNC symbol '%s' referenced in '%s' is defined in the executable \
292and creates an unsatisfiable circular dependency.\n",
293 RTLD_PROGNAME, strtab + refsym->st_name,
294 map->l_name);
295 else
296 _dl_error_printf ("\
297%s: Relink `%s' with `%s' for IFUNC symbol `%s'\n",
298 RTLD_PROGNAME, map->l_name,
299 sym_map->l_name,
300 strtab + refsym->st_name);
301 }
302# endif
303 value = ((ElfW(Addr) (*) (void)) value) ();
304 }
305
306 switch (r_type)
307 {
308 case R_X86_64_JUMP_SLOT:
309 map->l_has_jump_slot_reloc = true;
310 /* fallthrough */
311 case R_X86_64_GLOB_DAT:
312 *reloc_addr = value;
313 break;
314
315# ifndef RTLD_BOOTSTRAP
316# ifdef __ILP32__
317 case R_X86_64_SIZE64:
318 /* Set to symbol size plus addend. */
319 *(Elf64_Addr *) (uintptr_t) reloc_addr
320 = (Elf64_Addr) sym->st_size + reloc->r_addend;
321 break;
322
323 case R_X86_64_SIZE32:
324# else
325 case R_X86_64_SIZE64:
326# endif
327 /* Set to symbol size plus addend. */
328 value = sym->st_size;
329 *reloc_addr = value + reloc->r_addend;
330 break;
331
332 case R_X86_64_DTPMOD64:
333 /* Get the information from the link map returned by the
334 resolve function. */
335 if (sym_map != NULL)
336 *reloc_addr = sym_map->l_tls_modid;
337 break;
338 case R_X86_64_DTPOFF64:
339 /* During relocation all TLS symbols are defined and used.
340 Therefore the offset is already correct. */
341 if (sym != NULL)
342 {
343 value = sym->st_value + reloc->r_addend;
344# ifdef __ILP32__
345 /* This relocation type computes a signed offset that is
346 usually negative. The symbol and addend values are 32
347 bits but the GOT entry is 64 bits wide and the whole
348 64-bit entry is used as a signed quantity, so we need
349 to sign-extend the computed value to 64 bits. */
350 *(Elf64_Sxword *) reloc_addr = (Elf64_Sxword) (Elf32_Sword) value;
351# else
352 *reloc_addr = value;
353# endif
354 }
355 break;
356 case R_X86_64_TLSDESC:
357 {
358 struct tlsdesc volatile *td =
359 (struct tlsdesc volatile *)reloc_addr;
360
361 if (! sym)
362 {
363 td->arg = (void*)reloc->r_addend;
364 td->entry = _dl_tlsdesc_undefweak;
365 }
366 else
367 {
368# ifndef SHARED
369 CHECK_STATIC_TLS (map, sym_map);
370# else
371 if (!TRY_STATIC_TLS (map, sym_map))
372 {
373 td->arg = _dl_make_tlsdesc_dynamic
374 (sym_map, sym->st_value + reloc->r_addend);
375 td->entry = GLRO(dl_x86_tlsdesc_dynamic);
376 }
377 else
378# endif
379 {
380 td->arg = (void*)(sym->st_value - sym_map->l_tls_offset
381 + reloc->r_addend);
382 td->entry = _dl_tlsdesc_return;
383 }
384 }
385 break;
386 }
387 case R_X86_64_TPOFF64:
388 /* The offset is negative, forward from the thread pointer. */
389 if (sym != NULL)
390 {
391 CHECK_STATIC_TLS (map, sym_map);
392 /* We know the offset of the object the symbol is contained in.
393 It is a negative value which will be added to the
394 thread pointer. */
395 value = (sym->st_value + reloc->r_addend
396 - sym_map->l_tls_offset);
397# ifdef __ILP32__
398 /* The symbol and addend values are 32 bits but the GOT
399 entry is 64 bits wide and the whole 64-bit entry is used
400 as a signed quantity, so we need to sign-extend the
401 computed value to 64 bits. */
402 *(Elf64_Sxword *) reloc_addr = (Elf64_Sxword) (Elf32_Sword) value;
403# else
404 *reloc_addr = value;
405# endif
406 }
407 break;
408
409 case R_X86_64_64:
410 /* value + r_addend may be > 0xffffffff and R_X86_64_64
411 relocation updates the whole 64-bit entry. */
412 *(Elf64_Addr *) reloc_addr = (Elf64_Addr) value + reloc->r_addend;
413 break;
414# ifndef __ILP32__
415 case R_X86_64_SIZE32:
416 /* Set to symbol size plus addend. */
417 value = sym->st_size;
418# endif
419 /* Fall through. */
420 case R_X86_64_32:
421 value += reloc->r_addend;
422 *(unsigned int *) reloc_addr = value;
423
424 const char *fmt;
425 if (__glibc_unlikely (value > UINT_MAX))
426 {
427 const char *strtab;
428
429 fmt = "\
430%s: Symbol `%s' causes overflow in R_X86_64_32 relocation\n";
431 print_err:
432 strtab = (const char *) D_PTR (map, l_info[DT_STRTAB]);
433
434 _dl_error_printf (fmt, RTLD_PROGNAME, strtab + refsym->st_name);
435 }
436 break;
437 /* Not needed for dl-conflict.c. */
438 case R_X86_64_PC32:
439 value += reloc->r_addend - (ElfW(Addr)) reloc_addr;
440 *(unsigned int *) reloc_addr = value;
441 if (__glibc_unlikely (value != (int) value))
442 {
443 fmt = "\
444%s: Symbol `%s' causes overflow in R_X86_64_PC32 relocation\n";
445 goto print_err;
446 }
447 break;
448 case R_X86_64_COPY:
449 if (sym == NULL)
450 /* This can happen in trace mode if an object could not be
451 found. */
452 break;
453 memcpy (reloc_addr_arg, (void *) value,
454 MIN (sym->st_size, refsym->st_size));
455 if (__glibc_unlikely (sym->st_size > refsym->st_size)
456 || (__glibc_unlikely (sym->st_size < refsym->st_size)
457 && GLRO(dl_verbose)))
458 {
459 fmt = "\
460%s: Symbol `%s' has different size in shared object, consider re-linking\n";
461 goto print_err;
462 }
463 break;
464 case R_X86_64_IRELATIVE:
465 value = map->l_addr + reloc->r_addend;
466 if (__glibc_likely (!skip_ifunc))
467 value = ((ElfW(Addr) (*) (void)) value) ();
468 *reloc_addr = value;
469 break;
470 default:
471 _dl_reloc_bad_type (map, r_type, 0);
472 break;
473# endif /* !RTLD_BOOTSTRAP */
474 }
475 }
476}
477
478static inline void
479__attribute ((always_inline))
480elf_machine_rela_relative (ElfW(Addr) l_addr, const ElfW(Rela) *reloc,
481 void *const reloc_addr_arg)
482{
483 ElfW(Addr) *const reloc_addr = reloc_addr_arg;
484#if !defined RTLD_BOOTSTRAP
485 /* l_addr + r_addend may be > 0xffffffff and R_X86_64_RELATIVE64
486 relocation updates the whole 64-bit entry. */
487 if (__glibc_unlikely (ELFW(R_TYPE) (reloc->r_info) == R_X86_64_RELATIVE64))
488 *(Elf64_Addr *) reloc_addr = (Elf64_Addr) l_addr + reloc->r_addend;
489 else
490#endif
491 {
492 assert (ELFW(R_TYPE) (reloc->r_info) == R_X86_64_RELATIVE);
493 *reloc_addr = l_addr + reloc->r_addend;
494 }
495}
496
497static inline void
498__attribute ((always_inline))
499elf_machine_lazy_rel (struct link_map *map, struct r_scope_elem *scope[],
500 ElfW(Addr) l_addr, const ElfW(Rela) *reloc,
501 int skip_ifunc)
502{
503 ElfW(Addr) *const reloc_addr = (void *) (l_addr + reloc->r_offset);
504 const unsigned long int r_type = ELFW(R_TYPE) (reloc->r_info);
505
506 /* Check for unexpected PLT reloc type. */
507 if (__glibc_likely (r_type == R_X86_64_JUMP_SLOT))
508 {
509 /* Prelink has been deprecated. */
510 if (__glibc_likely (map->l_mach.plt == 0))
511 *reloc_addr += l_addr;
512 else
513 *reloc_addr =
514 map->l_mach.plt
515 + (((ElfW(Addr)) reloc_addr) - map->l_mach.gotplt) * 2;
516 }
517 else if (__glibc_likely (r_type == R_X86_64_TLSDESC))
518 {
519 const Elf_Symndx symndx = ELFW (R_SYM) (reloc->r_info);
520 const ElfW (Sym) *symtab = (const void *)D_PTR (map, l_info[DT_SYMTAB]);
521 const ElfW (Sym) *sym = &symtab[symndx];
522 const struct r_found_version *version = NULL;
523
524 if (map->l_info[VERSYMIDX (DT_VERSYM)] != NULL)
525 {
526 const ElfW (Half) *vernum =
527 (const void *)D_PTR (map, l_info[VERSYMIDX (DT_VERSYM)]);
528 version = &map->l_versions[vernum[symndx] & 0x7fff];
529 }
530
531 /* Always initialize TLS descriptors completely at load time, in
532 case static TLS is allocated for it that requires locking. */
533 elf_machine_rela (map, scope, reloc, sym, version, reloc_addr, skip_ifunc);
534 }
535 else if (__glibc_unlikely (r_type == R_X86_64_IRELATIVE))
536 {
537 ElfW(Addr) value = map->l_addr + reloc->r_addend;
538 if (__glibc_likely (!skip_ifunc))
539 value = ((ElfW(Addr) (*) (void)) value) ();
540 *reloc_addr = value;
541 }
542 else
543 _dl_reloc_bad_type (map, r_type, 1);
544}
545
546#endif /* RESOLVE_MAP */
547
548#if !defined ELF_DYNAMIC_AFTER_RELOC && !defined RTLD_BOOTSTRAP \
549 && defined SHARED
550# define ELF_DYNAMIC_AFTER_RELOC(map, lazy) \
551 x86_64_dynamic_after_reloc (map, (lazy))
552
553# define JMP32_INSN_OPCODE 0xe9
554# define JMP32_INSN_SIZE 5
555# define JMPABS_INSN_OPCODE 0xa100d5
556# define JMPABS_INSN_SIZE 11
557# define INT3_INSN_OPCODE 0xcc
558
559static const char *
560x86_64_reloc_symbol_name (struct link_map *map, const ElfW(Rela) *reloc)
561{
562 const ElfW(Sym) *const symtab
563 = (const void *) map->l_info[DT_SYMTAB]->d_un.d_ptr;
564 const ElfW(Sym) *const refsym = &symtab[ELFW (R_SYM) (reloc->r_info)];
565 const char *strtab = (const char *) map->l_info[DT_STRTAB]->d_un.d_ptr;
566 return strtab + refsym->st_name;
567}
568
569static void
570x86_64_rewrite_plt (struct link_map *map, ElfW(Addr) plt_rewrite)
571{
572 ElfW(Addr) l_addr = map->l_addr;
573 ElfW(Addr) pltent = map->l_info[DT_X86_64 (PLTENT)]->d_un.d_val;
574 ElfW(Addr) start = map->l_info[DT_JMPREL]->d_un.d_ptr;
575 ElfW(Addr) size = map->l_info[DT_PLTRELSZ]->d_un.d_val;
576 const ElfW(Rela) *reloc = (const void *) start;
577 const ElfW(Rela) *reloc_end = (const void *) (start + size);
578
579# ifdef __CET__
580 bool ibt_enabled_p = dl_cet_ibt_enabled ();
581# else
582 bool ibt_enabled_p = false;
583# endif
584
585 if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES))
586 _dl_debug_printf (fmt: "\nchanging PLT in '%s' to direct branch\n",
587 DSO_FILENAME (map->l_name));
588
589 for (; reloc < reloc_end; reloc++)
590 if (ELFW(R_TYPE) (reloc->r_info) == R_X86_64_JUMP_SLOT)
591 {
592 /* Get the value from the GOT entry. */
593 ElfW(Addr) value = *(ElfW(Addr) *) (l_addr + reloc->r_offset);
594
595 /* Get the corresponding PLT entry from r_addend. */
596 ElfW(Addr) branch_start = l_addr + reloc->r_addend;
597 /* Skip ENDBR64 if IBT isn't enabled. */
598 if (!ibt_enabled_p)
599 branch_start = ALIGN_DOWN (branch_start, pltent);
600 /* Get the displacement from the branch target. NB: We must use
601 64-bit integer on x32 to avoid overflow. */
602 uint64_t disp = (uint64_t) value - branch_start - JMP32_INSN_SIZE;
603 ElfW(Addr) plt_end;
604 ElfW(Addr) pad;
605
606 plt_end = (branch_start | (pltent - 1)) + 1;
607
608 /* Update the PLT entry. */
609 if (((uint64_t) disp + (uint64_t) ((uint32_t) INT32_MIN))
610 <= (uint64_t) UINT32_MAX)
611 {
612 pad = branch_start + JMP32_INSN_SIZE;
613
614 if (__glibc_unlikely (pad > plt_end))
615 continue;
616
617 /* If the target branch can be reached with a direct branch,
618 rewrite the PLT entry with a direct branch. */
619 if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_BINDINGS))
620 {
621 const char *sym_name = x86_64_reloc_symbol_name (map,
622 reloc);
623 _dl_debug_printf (fmt: "changing '%s' PLT entry in '%s' to "
624 "direct branch\n", sym_name,
625 DSO_FILENAME (map->l_name));
626 }
627
628 /* Write out direct branch. */
629 *(uint8_t *) branch_start = JMP32_INSN_OPCODE;
630 *(uint32_t *) (branch_start + 1) = disp;
631 }
632 else
633 {
634 if (GL(dl_x86_feature_control).plt_rewrite
635 != plt_rewrite_jmpabs)
636 {
637 if (__glibc_unlikely (GLRO(dl_debug_mask)
638 & DL_DEBUG_BINDINGS))
639 {
640 const char *sym_name
641 = x86_64_reloc_symbol_name (map, reloc);
642 _dl_debug_printf (fmt: "skipping '%s' PLT entry in '%s'\n",
643 sym_name,
644 DSO_FILENAME (map->l_name));
645 }
646 continue;
647 }
648
649 pad = branch_start + JMPABS_INSN_SIZE;
650
651 if (__glibc_unlikely (pad > plt_end))
652 continue;
653
654 /* Rewrite the PLT entry with JMPABS. */
655 if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_BINDINGS))
656 {
657 const char *sym_name = x86_64_reloc_symbol_name (map,
658 reloc);
659 _dl_debug_printf (fmt: "changing '%s' PLT entry in '%s' to "
660 "JMPABS\n", sym_name,
661 DSO_FILENAME (map->l_name));
662 }
663
664 /* "jmpabs $target" for 64-bit displacement. NB: JMPABS has
665 a 3-byte opcode + 64bit address. There is a 1-byte overlap
666 between 4-byte write and 8-byte write. */
667 *(uint32_t *) (branch_start) = JMPABS_INSN_OPCODE;
668 *(uint64_t *) (branch_start + 3) = value;
669 }
670
671 /* Fill the unused part of the PLT entry with INT3. */
672 for (; pad < plt_end; pad++)
673 *(uint8_t *) pad = INT3_INSN_OPCODE;
674 }
675}
676
677static inline void
678x86_64_rewrite_plt_in_place (struct link_map *map)
679{
680 /* Adjust DT_X86_64_PLT address and DT_X86_64_PLTSZ values. */
681 ElfW(Addr) plt = (map->l_info[DT_X86_64 (PLT)]->d_un.d_ptr
682 + map->l_addr);
683 size_t pagesize = GLRO(dl_pagesize);
684 ElfW(Addr) plt_aligned = ALIGN_DOWN (plt, pagesize);
685 size_t pltsz = (map->l_info[DT_X86_64 (PLTSZ)]->d_un.d_val
686 + plt - plt_aligned);
687
688 if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES))
689 _dl_debug_printf (fmt: "\nchanging PLT in '%s' to writable\n",
690 DSO_FILENAME (map->l_name));
691
692 if (__glibc_unlikely (__mprotect ((void *) plt_aligned, pltsz,
693 PROT_WRITE | PROT_READ) < 0))
694 {
695 if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES))
696 _dl_debug_printf (fmt: "\nfailed to change PLT in '%s' to writable\n",
697 DSO_FILENAME (map->l_name));
698 return;
699 }
700
701 x86_64_rewrite_plt (map, plt_rewrite: plt_aligned);
702
703 if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES))
704 _dl_debug_printf (fmt: "\nchanging PLT in '%s' back to read-only\n",
705 DSO_FILENAME (map->l_name));
706
707 if (__glibc_unlikely (__mprotect ((void *) plt_aligned, pltsz,
708 PROT_EXEC | PROT_READ) < 0))
709 _dl_signal_error (0, DSO_FILENAME (map->l_name), NULL,
710 "failed to change PLT back to read-only");
711}
712
713/* Rewrite PLT entries to direct branch if possible. */
714
715static inline void
716x86_64_dynamic_after_reloc (struct link_map *map, int lazy)
717{
718 /* Ignore DT_X86_64_PLT if the lazy binding is enabled. */
719 if (lazy != 0)
720 return;
721
722 /* Ignore DT_X86_64_PLT if PLT rewrite isn't enabled. */
723 if (__glibc_likely (GL(dl_x86_feature_control).plt_rewrite
724 == plt_rewrite_none))
725 return;
726
727 if (__glibc_likely (map->l_info[DT_X86_64 (PLT)] == NULL))
728 return;
729
730 /* Ignore DT_X86_64_PLT if there is no R_X86_64_JUMP_SLOT. */
731 if (map->l_has_jump_slot_reloc == 0)
732 return;
733
734 /* Ignore DT_X86_64_PLT if
735 1. DT_JMPREL isn't available or its value is 0.
736 2. DT_PLTRELSZ is 0.
737 3. DT_X86_64_PLTENT isn't available or its value is smaller than
738 16 bytes.
739 4. DT_X86_64_PLTSZ isn't available or its value is smaller than
740 DT_X86_64_PLTENT's value or isn't a multiple of DT_X86_64_PLTENT's
741 value. */
742 if (map->l_info[DT_JMPREL] == NULL
743 || map->l_info[DT_JMPREL]->d_un.d_ptr == 0
744 || map->l_info[DT_PLTRELSZ]->d_un.d_val == 0
745 || map->l_info[DT_X86_64 (PLTSZ)] == NULL
746 || map->l_info[DT_X86_64 (PLTENT)] == NULL
747 || map->l_info[DT_X86_64 (PLTENT)]->d_un.d_val < 16
748 || (map->l_info[DT_X86_64 (PLTSZ)]->d_un.d_val
749 < map->l_info[DT_X86_64 (PLTENT)]->d_un.d_val)
750 || (map->l_info[DT_X86_64 (PLTSZ)]->d_un.d_val
751 % map->l_info[DT_X86_64 (PLTENT)]->d_un.d_val) != 0)
752 return;
753
754 x86_64_rewrite_plt_in_place (map);
755}
756#endif
757

source code of glibc/sysdeps/x86_64/dl-machine.h