[OSELAS.Toolchain] Aligment trap in dynamic linker

[Ladislav Michl-2]

I'm not sure if it belongs here, but lets give it a try...

I built ARM EABI toolchain using OSELAS.Toolchain
arm-v4t-linux-gnueabi_gcc-4.4.0_glibc-2.9_binutils-2.19.1_kernel-2.6.29-sanitized.ptxconfig
and run freshly built userland with 2.6.32-rc5 kernel on OMAP5910 (ARM925)
based board. Running every single binary triggers alignment trap.
Here's illustrative example (after boot):

# cat /proc/cpu/alignment
User:           201
System:         0
Skipped:        0
Half:           0
Word:           0
Multi:          0
User faults:    0 (ignored)
# echo 3 > /proc/cpu/alignment
# cat /proc/cpu/alignment
Alignment trap: cat (236) PC=0x4000aba8 Instr=0xe7951002 Address=0x4022b9a7 FSR 0x001
Alignment trap: cat (236) PC=0x4000abb4 Instr=0xe7851002 Address=0x4022b9a7 FSR 0x811
Alignment trap: cat (236) PC=0x4000aba8 Instr=0xe7951002 Address=0x4022bffa FSR 0x001
Alignment trap: cat (236) PC=0x4000abb4 Instr=0xe7851002 Address=0x4022bffa FSR 0x801
User:           205
System:         0
Skipped:        0
Half:           0
Word:           4
Multi:          0
User faults:    3 (fixup+warn)

Looking at process map shows that 0x4000aba8 belongs to /lib/ld-2.9.so address
space and objdump finds out (somehow long, but I do not want to cut is out of
context):

void
_dl_relocate_object (struct link_map *l, struct r_scope_elem *scope[],
    aad4: e50bc094 str ip, [fp, #-148]
elf_dynamic_do_rel (struct link_map *map,
                    ElfW(Addr) reladdr, ElfW(Addr) relsize,
                    int lazy)
{
  const ElfW(Rel) *r = (const void *) reladdr;
  const ElfW(Rel) *end = (const void *) (reladdr + relsize);
    aad8: e0828003 add r8, r2, r3

#if (!defined DO_RELA || !defined ELF_MACHINE_PLT_REL) && !defined RTLD_BOOTSTRAP
  /* We never bind lazily during ld.so bootstrap.  Unfortunately gcc is
     not clever enough to see through all the function calls to realize
     that.  */
  if (lazy)
    aadc: 0a00001d beq ab58 <_dl_relocate_object+0x1dc>
    {
      /* Doing lazy PLT relocations; they need very little info.  */
      for (; r < end; ++r)
    aae0: e1530008 cmp r3, r8
    aae4: 2a00000e bcs ab24 <_dl_relocate_object+0x1a8>
__attribute__ ((always_inline))
elf_machine_lazy_rel (struct link_map *map,
                      Elf32_Addr l_addr, const Elf32_Rel *reloc)
{
  Elf32_Addr *const reloc_addr = (void *) (l_addr + reloc->r_offset);
  const unsigned int r_type = ELF32_R_TYPE (reloc->r_info);
    aae8: e5d31004 ldrb r1, [r3, #4]
  /* Check for unexpected PLT reloc type.  */
  if (__builtin_expect (r_type == R_ARM_JUMP_SLOT, 1))
    aaec: e3510016 cmp r1, #22 ; 0x16
auto inline void
__attribute__ ((always_inline))
elf_machine_lazy_rel (struct link_map *map,
                      Elf32_Addr l_addr, const Elf32_Rel *reloc)
{
  Elf32_Addr *const reloc_addr = (void *) (l_addr + reloc->r_offset);
    aaf0: e5932000 ldr r2, [r3]
  const unsigned int r_type = ELF32_R_TYPE (reloc->r_info);
  /* Check for unexpected PLT reloc type.  */
  if (__builtin_expect (r_type == R_ARM_JUMP_SLOT, 1))
    aaf4: 1a000369 bne b8a0 <_dl_relocate_object+0xf24>
    {
      if (__builtin_expect (map->l_mach.plt, 0) == 0)
    aaf8: e594120c ldr r1, [r4, #524]
auto inline void
__attribute__ ((always_inline))
elf_machine_lazy_rel (struct link_map *map,
                      Elf32_Addr l_addr, const Elf32_Rel *reloc)
{
  Elf32_Addr *const reloc_addr = (void *) (l_addr + reloc->r_offset);
    aafc: e0852002 add r2, r5, r2
  const unsigned int r_type = ELF32_R_TYPE (reloc->r_info);
  /* Check for unexpected PLT reloc type.  */
  if (__builtin_expect (r_type == R_ARM_JUMP_SLOT, 1))
    {
      if (__builtin_expect (map->l_mach.plt, 0) == 0)
    ab00: e3510000 cmp r1, #0 ; 0x0
        *reloc_addr += l_addr;
    ab04: 05921000 ldreq r1, [r2]
    ab08: e2833008 add r3, r3, #8 ; 0x8
    ab0c: 00811005 addeq r1, r1, r5
    ab10: 05821000 streq r1, [r2]
      else
        *reloc_addr = map->l_mach.plt;
    ab14: 15821000 strne r1, [r2]
    ab18: e1580003 cmp r8, r3
    ab1c: 8afffff1 bhi aae8 <_dl_relocate_object+0x16c>
                               (void *) (l_addr + r->r_offset));
            }
        }
#ifndef RTLD_BOOTSTRAP
      else
        for (; r < end; ++r)
    ab20: e5945000 ldr r5, [r4]
    ab24: e51b1064 ldr r1, [fp, #-100]
             l->l_lookup_cache.value = _lr; }))      \
     : l)

#include "dynamic-link.h"

    ELF_DYNAMIC_RELOCATE (l, lazy, consider_profiling);
    ab28: e51b2094 ldr r2, [fp, #-148]
    ab2c: e281100c add r1, r1, #12 ; 0xc
    ab30: e1510002 cmp r1, r2
    ab34: e50b1064 str r1, [fp, #-100]
    ab38: 0a000212 beq b388 <_dl_relocate_object+0xa0c>

#if (!defined DO_RELA || !defined ELF_MACHINE_PLT_REL) && !defined RTLD_BOOTSTRAP
  /* We never bind lazily during ld.so bootstrap.  Unfortunately gcc is
     not clever enough to see through all the function calls to realize
     that.  */
  if (lazy)
    ab3c: e51b3064 ldr r3, [fp, #-100]
    ab40: e511200c ldr r2, [r1, #-12]
    ab44: e5131008 ldr r1, [r3, #-8]
    ab48: e5133010 ldr r3, [r3, #-16]
    ab4c: e3510000 cmp r1, #0 ; 0x0
elf_dynamic_do_rel (struct link_map *map,
                    ElfW(Addr) reladdr, ElfW(Addr) relsize,
                    int lazy)
{
  const ElfW(Rel) *r = (const void *) reladdr;
  const ElfW(Rel) *end = (const void *) (reladdr + relsize);
    ab50: e0828003 add r8, r2, r3

#if (!defined DO_RELA || !defined ELF_MACHINE_PLT_REL) && !defined RTLD_BOOTSTRAP
  /* We never bind lazily during ld.so bootstrap.  Unfortunately gcc is
     not clever enough to see through all the function calls to realize
     that.  */
  if (lazy)
    ab54: 1affffe1 bne aae0 <_dl_relocate_object+0x164>
    }
  else
#endif
    {
      const ElfW(Sym) *const symtab =
        (const void *) D_PTR (map, l_info[DT_SYMTAB]);
    ab58: e5941038 ldr r1, [r4, #56]
      ElfW(Word) nrelative = (map->l_info[RELCOUNT_IDX] == NULL
    ab5c: e59460bc ldr r6, [r4, #188]
    }
  else
#endif
    {
      const ElfW(Sym) *const symtab =
        (const void *) D_PTR (map, l_info[DT_SYMTAB]);
    ab60: e5911004 ldr r1, [r1, #4]
      ElfW(Word) nrelative = (map->l_info[RELCOUNT_IDX] == NULL
    ab64: e3560000 cmp r6, #0 ; 0x0
    }
  else
#endif
    {
      const ElfW(Sym) *const symtab =
        (const void *) D_PTR (map, l_info[DT_SYMTAB]);
    ab68: e50b1068 str r1, [fp, #-104]
      ElfW(Word) nrelative = (map->l_info[RELCOUNT_IDX] == NULL
    ab6c: 0a000004 beq ab84 <_dl_relocate_object+0x208>
    ab70: e5961004 ldr r1, [r6, #4]
    ab74: e1a021a2 lsr r2, r2, #3
    ab78: e1520001 cmp r2, r1
    ab7c: 21a02001 movcs r2, r1
    ab80: e1a06182 lsl r6, r2, #3
         _dl_rtld_map, which is incompatible with a weak decl in the same
         file.  */
# ifndef SHARED
      weak_extern (GL(dl_rtld_map));
# endif
      if (map != &GL(dl_rtld_map)) /* Already done in rtld itself.  */
    ab84: e51bc08c ldr ip, [fp, #-140]
    ab88: e154000c cmp r4, ip
      const ElfW(Sym) *const symtab =
        (const void *) D_PTR (map, l_info[DT_SYMTAB]);
      ElfW(Word) nrelative = (map->l_info[RELCOUNT_IDX] == NULL
                              ? 0 : map->l_info[RELCOUNT_IDX]->d_un.d_val);
      const ElfW(Rel) *relative = r;
      r = r + MIN (nrelative, relsize / sizeof (ElfW(Rel)));
    ab8c: e0836006 add r6, r3, r6
         _dl_rtld_map, which is incompatible with a weak decl in the same
         file.  */
# ifndef SHARED
      weak_extern (GL(dl_rtld_map));
# endif
      if (map != &GL(dl_rtld_map)) /* Already done in rtld itself.  */
    ab90: 0a000009 beq abbc <_dl_relocate_object+0x240>
        /* Rela platforms get the offset from r_addend and this must
           be copied in the relocation address.  Therefore we can skip
           the relative relocations only if this is for rel
           relocations or rela relocations if they are computed as
           memory_loc += l_addr...  */
        if (l_addr != 0)
    ab94: e3550000 cmp r5, #0 ; 0x0
    ab98: 0a000007 beq abbc <_dl_relocate_object+0x240>
# else
        /* ...or we know the object has been prelinked.  */
        if (l_addr != 0 || ! map->l_info[VALIDX(DT_GNU_PRELINKED)])
# endif
#endif
          for (; relative < r; ++relative)
    ab9c: e1530006 cmp r3, r6
    aba0: 2a000005 bcs abbc <_dl_relocate_object+0x240>
auto inline void
__attribute__ ((always_inline))
elf_machine_rel_relative (Elf32_Addr l_addr, const Elf32_Rel *reloc,
                          void *const reloc_addr_arg)
{
  Elf32_Addr *const reloc_addr = reloc_addr_arg;
    aba4: e4932008 ldr r2, [r3], #8
  *reloc_addr += l_addr;
    aba8: e7951002 ldr r1, [r5, r2]
    abac: e1560003 cmp r6, r3
    abb0: e0811005 add r1, r1, r5
    abb4: e7851002 str r1, [r5, r2]
    abb8: 8afffff9 bhi aba4 <_dl_relocate_object+0x228>

#ifdef RTLD_BOOTSTRAP
      /* The dynamic linker always uses versioning.  */
      assert (map->l_info[VERSYMIDX (DT_VERSYM)] != NULL);
#else
      if (map->l_info[VERSYMIDX (DT_VERSYM)])
    abbc: e59430e4 ldr r3, [r4, #228]
    abc0: e3530000 cmp r3, #0 ; 0x0
    abc4: 0a00012b beq b078 <_dl_relocate_object+0x6fc>
#endif

... etc ...

So every dynamically linked binary will trigger it. I'm used to set
/proc/cpu/alignment to be 4, to catch all programs which slows down system
and this problem prevents me from doing so. Btw, why is default value 0?
Silently ignoring problem sounds like strange approach.

Anyone ever hit this?

Thanks,
        ladis

--
For unsubscribe information see http://sourceware.org/lists.html#faq

[martinwguy]

On 11/2/09, Ladislav Michl <Ladislav.Michl@...> wrote:
>  I built ARM EABI toolchain using OSELAS.Toolchain
>  arm-v4t-linux-gnueabi_gcc-4.4.0_glibc-2.9_binutils-2.19.1_kernel-2.6.29-sanitized.ptxconfig
>  and run freshly built userland with 2.6.32-rc5 kernel on OMAP5910 (ARM925)
>  based board. Running every single binary triggers alignment trap.

>  Looking at process map shows that 0x4000aba8 belongs to /lib/ld-2.9.so address

The current Debian testing release (squeeze) has a similar symptom,
but only on C++ binaries as far as I know. It looks like a new bug in
libc or the GCC startup code.
Here is how far we got http://bugs.debian.org/548842

    M

--
For unsubscribe information see http://sourceware.org/lists.html#faq

[Ladislav Michl-2]

On Tue, Nov 03, 2009 at 11:53:07AM +0000, Martin Guy wrote:

> On 11/2/09, Ladislav Michl <Ladislav.Michl@...> wrote:
> >  I built ARM EABI toolchain using OSELAS.Toolchain
> >  arm-v4t-linux-gnueabi_gcc-4.4.0_glibc-2.9_binutils-2.19.1_kernel-2.6.29-sanitized.ptxconfig
> >  and run freshly built userland with 2.6.32-rc5 kernel on OMAP5910 (ARM925)
> >  based board. Running every single binary triggers alignment trap.
>
> >  Looking at process map shows that 0x4000aba8 belongs to /lib/ld-2.9.so address
>
> The current Debian testing release (squeeze) has a similar symptom,
> but only on C++ binaries as far as I know. It looks like a new bug in
> libc or the GCC startup code.
> Here is how far we got http://bugs.debian.org/548842

Just FYI, I found similar one here
http://www.opensubscriber.com/message/fedora-devel-list@.../5545902.html
but it seems that fix didn't went in, so right solution is elsewhere.

However, I blindly modified it for arm and unaligned access went away...

--- glibc-ports-2.9/sysdeps/arm/dl-machine.h.orig 2009-11-03 22:03:57.000000000 +0100
+++ glibc-ports-2.9/sysdeps/arm/dl-machine.h 2009-11-03 22:11:45.000000000 +0100
@@ -568,13 +568,22 @@
 }
 # endif
 
+union arm_unaligned_data {
+  Elf32_Addr l_addr;
+} __attribute__ ((packed));
+
 auto inline void
 __attribute__ ((always_inline))
 elf_machine_rel_relative (Elf32_Addr l_addr, const Elf32_Rel *reloc,
   void *const reloc_addr_arg)
 {
-  Elf32_Addr *const reloc_addr = reloc_addr_arg;
-  *reloc_addr += l_addr;
+  if (((long)reloc_addr_arg) & 0x3) {
+    union arm_unaligned_data *const lpdata = reloc_addr_arg;
+    lpdata->l_addr += l_addr;
+  } else {
+    Elf32_Addr *const reloc_addr = reloc_addr_arg;
+    *reloc_addr += l_addr;
+  }
 }
 
 # ifndef RTLD_BOOTSTRAP


--
For unsubscribe information see http://sourceware.org/lists.html#faq

[Ladislav Michl-2]

For the archives: gcc-4.4.1 emits eh_frame section by default which
a) makes binaries bigger
b) produces said unaligned relocation
(http://sourceware.org/ml/libc-ports/2009-11/msg00084.html)

As I do not preted I understand all gcc's bells and whistles, I used
gcc-4.3.4, which does not suffer from this problem. Patch adding gcc-4.3.4
into OSELAS.Toolchain will be released tomorrow.

Best regards,
        ladis

--
For unsubscribe information see http://sourceware.org/lists.html#faq