occa info stalls on ARM64 when built with g++ 9.3.0

Open tcew opened this issue 5 years ago • 1 comments

This report was precipitated by bug report from libCEED.

When occa is built using g++ version 9.3.0, the occa info command hangs.

Running valgrind revealed the following critical memory errors. This suggests that there is a memory issue with string handling in occa in general that only shows up badly on ARM64. This kind of one-platform error suggests a bad memory handling issue.

valgrind ./bin/occa  info
==23270== Memcheck, a memory error detector
==23270== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al.
==23270== Using Valgrind-3.13.0 and LibVEX; rerun with -h for copyright info
==23270== Command: ./bin/occa info
==23270== 
==23270== Conditional jump or move depends on uninitialised value(s)
==23270==    at 0x4C4693C: std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct(unsigned long, char) (in /usr/lib/aarch64-linux-gnu/libstdc++.so.6.0.28)
==23270==    by 0x4944F73: occa::styling::section::toString[abi:cxx11](int, int, int, int, bool) const (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x4945A63: occa::styling::table::toString[abi:cxx11](int) const (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x4945B5B: occa::styling::operator<<(std::ostream&, occa::styling::table const&) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x495C26F: occa::io::output& occa::io::output::operator<< <occa::styling::table>(occa::styling::table const&) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x495B1DB: occa::printModeInfo() (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x10D81B: runInfo(occa::json const&) (in /home/ubuntu/Work/git/occa/bin/occa)
==23270==    by 0x491B99F: occa::cli::command::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, occa::cli::command*) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x491B9BB: occa::cli::command::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, occa::cli::command*) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x491BF43: occa::cli::command::run(int, char const**) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x10CB9F: main (in /home/ubuntu/Work/git/occa/bin/occa)
==23270== 
==23270== Conditional jump or move depends on uninitialised value(s)
==23270==    at 0x4C4689C: std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_create(unsigned long&, unsigned long) (in /usr/lib/aarch64-linux-gnu/libstdc++.so.6.0.28)
==23270==    by 0x4C46987: std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct(unsigned long, char) (in /usr/lib/aarch64-linux-gnu/libstdc++.so.6.0.28)
==23270==    by 0x4944F73: occa::styling::section::toString[abi:cxx11](int, int, int, int, bool) const (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x4945A63: occa::styling::table::toString[abi:cxx11](int) const (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x4945B5B: occa::styling::operator<<(std::ostream&, occa::styling::table const&) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x495C26F: occa::io::output& occa::io::output::operator<< <occa::styling::table>(occa::styling::table const&) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x495B1DB: occa::printModeInfo() (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x10D81B: runInfo(occa::json const&) (in /home/ubuntu/Work/git/occa/bin/occa)
==23270==    by 0x491B99F: occa::cli::command::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, occa::cli::command*) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x491B9BB: occa::cli::command::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, occa::cli::command*) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x491BF43: occa::cli::command::run(int, char const**) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x10CB9F: main (in /home/ubuntu/Work/git/occa/bin/occa)
==23270== 
==23270== Conditional jump or move depends on uninitialised value(s)
==23270==    at 0x4C468A4: std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_create(unsigned long&, unsigned long) (in /usr/lib/aarch64-linux-gnu/libstdc++.so.6.0.28)
==23270==    by 0x4C46987: std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct(unsigned long, char) (in /usr/lib/aarch64-linux-gnu/libstdc++.so.6.0.28)
==23270==    by 0x4944F73: occa::styling::section::toString[abi:cxx11](int, int, int, int, bool) const (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x4945A63: occa::styling::table::toString[abi:cxx11](int) const (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x4945B5B: occa::styling::operator<<(std::ostream&, occa::styling::table const&) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x495C26F: occa::io::output& occa::io::output::operator<< <occa::styling::table>(occa::styling::table const&) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x495B1DB: occa::printModeInfo() (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x10D81B: runInfo(occa::json const&) (in /home/ubuntu/Work/git/occa/bin/occa)
==23270==    by 0x491B99F: occa::cli::command::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, occa::cli::command*) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x491B9BB: occa::cli::command::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, occa::cli::command*) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x491BF43: occa::cli::command::run(int, char const**) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x10CB9F: main (in /home/ubuntu/Work/git/occa/bin/occa)
==23270== 
==23270== Conditional jump or move depends on uninitialised value(s)
==23270==    at 0x4C468B0: std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_create(unsigned long&, unsigned long) (in /usr/lib/aarch64-linux-gnu/libstdc++.so.6.0.28)
==23270==    by 0x4C46987: std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct(unsigned long, char) (in /usr/lib/aarch64-linux-gnu/libstdc++.so.6.0.28)
==23270==    by 0x4944F73: occa::styling::section::toString[abi:cxx11](int, int, int, int, bool) const (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x4945A63: occa::styling::table::toString[abi:cxx11](int) const (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x4945B5B: occa::styling::operator<<(std::ostream&, occa::styling::table const&) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x495C26F: occa::io::output& occa::io::output::operator<< <occa::styling::table>(occa::styling::table const&) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x495B1DB: occa::printModeInfo() (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x10D81B: runInfo(occa::json const&) (in /home/ubuntu/Work/git/occa/bin/occa)
==23270==    by 0x491B99F: occa::cli::command::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, occa::cli::command*) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x491B9BB: occa::cli::command::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, occa::cli::command*) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x491BF43: occa::cli::command::run(int, char const**) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x10CB9F: main (in /home/ubuntu/Work/git/occa/bin/occa)
==23270== 
==23270== Conditional jump or move depends on uninitialised value(s)
==23270==    at 0x48451D0: operator new(unsigned long) (in /usr/lib/valgrind/vgpreload_memcheck-arm64-linux.so)
==23270== 
==23270== Warning: set address range perms: large range [0x59c05040, 0xbf32bf86) (undefined)
==23270== Conditional jump or move depends on uninitialised value(s)
==23270==    at 0x4C46948: std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct(unsigned long, char) (in /usr/lib/aarch64-linux-gnu/libstdc++.so.6.0.28)
==23270==    by 0x4944F73: occa::styling::section::toString[abi:cxx11](int, int, int, int, bool) const (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x4945A63: occa::styling::table::toString[abi:cxx11](int) const (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x4945B5B: occa::styling::operator<<(std::ostream&, occa::styling::table const&) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x495C26F: occa::io::output& occa::io::output::operator<< <occa::styling::table>(occa::styling::table const&) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x495B1DB: occa::printModeInfo() (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x10D81B: runInfo(occa::json const&) (in /home/ubuntu/Work/git/occa/bin/occa)
==23270==    by 0x491B99F: occa::cli::command::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, occa::cli::command*) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x491B9BB: occa::cli::command::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, occa::cli::command*) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x491BF43: occa::cli::command::run(int, char const**) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x10CB9F: main (in /home/ubuntu/Work/git/occa/bin/occa)
==23270== 
==23270== Conditional jump or move depends on uninitialised value(s)
==23270==    at 0x4C46950: std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct(unsigned long, char) (in /usr/lib/aarch64-linux-gnu/libstdc++.so.6.0.28)
==23270==    by 0x4944F73: occa::styling::section::toString[abi:cxx11](int, int, int, int, bool) const (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x4945A63: occa::styling::table::toString[abi:cxx11](int) const (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x4945B5B: occa::styling::operator<<(std::ostream&, occa::styling::table const&) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x495C26F: occa::io::output& occa::io::output::operator<< <occa::styling::table>(occa::styling::table const&) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x495B1DB: occa::printModeInfo() (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x10D81B: runInfo(occa::json const&) (in /home/ubuntu/Work/git/occa/bin/occa)
==23270==    by 0x491B99F: occa::cli::command::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, occa::cli::command*) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x491B9BB: occa::cli::command::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, occa::cli::command*) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x491BF43: occa::cli::command::run(int, char const**) (in /home/ubuntu/Work/git/occa/lib/libocca.so)
==23270==    by 0x10CB9F: main (in /home/ubuntu/Work/git/occa/bin/occa)
==23270== 
==23270== Conditional jump or move depends on uninitialised value(s)
==23270==    at 0x484C578: memset (in /usr/lib/valgrind/vgpreload_memcheck-arm64-linux.so)
==23270== 
==23270== Conditional jump or move depends on uninitialised value(s)
==23270==    at 0x484C5A4: memset (in /usr/lib/valgrind/vgpreload_memcheck-arm64-linux.so)
==23270== 
==23270== 
==23270== More than 10000000 total errors detected.  I'm not reporting any more.
==23270== Final error counts will be inaccurate.  Go fix your program!
==23270== Rerun with --error-limit=no to disable this cutoff.  Note
==23270== that errors may occur in your program without prior warning from
==23270== Valgrind, because errors are no longer being displayed.
==23270== 
==23270== Warning: set address range perms: large range [0x153c1040, 0x253c1041) (undefined)
==23270== Warning: set address range perms: large range [0x253c2040, 0x453c2041) (undefined)
==23270== Warning: set address range perms: large range [0x153c1028, 0x253c1059) (noaccess)
==23270== Warning: set address range perms: large range [0xbf32c040, 0xff32c041) (undefined)
==23270== Warning: set address range perms: large range [0x253c2028, 0x453c2059) (noaccess)
==23270== Warning: set address range perms: large range [0xff32d040, 0x17f32d041) (undefined)
==23270== Warning: set address range perms: large range [0xbf32c028, 0xff32c059) (noaccess)

Aug 17 '20 17:08 tcew

Some more info:

This problem seems to only be exposed with -O3.

I was able to use the valgrind gdb server to track down where the problem occurs first - note the suspicious fieldWidth and valueWidth values.

Program received signal SIGTRAP, Trace/breakpoint trap.
0x0000000004c6493c in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct(unsigned long, char) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
(gdb) where
#0  0x0000000004c6493c in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct(unsigned long, char) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
#1  0x0000000004949184 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string
    (__a=..., __c=61 '=', __n=**1701998405**, this=0x1ffeffe610) at /usr/include/c++/9/ext/new_allocator.h:83
#2  occa::styling::section::toString[abi:cxx11](int, int, int, int, bool) const (this=0x5015d60, 
    indent=indent@entry=4, sectionWidth=sectionWidth@entry=6, fieldWidth=fieldWidth@entry=**1701998403**, 
    valueWidth=valueWidth@entry=**1869440333**, isFirstSection=isFirstSection@entry=true)
    at /home/ubuntu/Work/git/occa/src/tools/styling.cpp:126
#3  0x0000000004949c74 in occa::styling::table::toString[abi:cxx11](int) const (this=0x1ffeffea08, 
    indent=indent@entry=4) at /usr/include/c++/9/bits/stl_vector.h:1058
#4  0x0000000004949d6c in occa::styling::operator<< (out=..., st=...)
    at /home/ubuntu/Work/git/occa/src/tools/styling.cpp:210
#5  0x00000000049627d8 in occa::io::output::operator<< <occa::styling::table> (this=0x4b29df8 <occa::io::stdout>, 
    t=...) at /usr/include/c++/9/bits/basic_string.h:1936
#6  0x00000000049615a4 in occa::printModeInfo () at /home/ubuntu/Work/git/occa/src/core/base.cpp:269
#7  0x000000000010d864 in runInfo (args=...) at /home/ubuntu/Work/git/occa/bin/occa.cpp:282
#8  0x000000000491e8b8 in occa::cli::command::run (this=0x5006470, args=..., parent=<optimized out>)
    at /home/ubuntu/Work/git/occa/src/tools/cli.cpp:810
#9  0x000000000491e8d4 in occa::cli::command::run (this=this@entry=0x127028 <occaCommand>, 
    args=std::vector of length 2, capacity 2 = {...}, parent=parent@entry=0x0)
    at /home/ubuntu/Work/git/occa/src/tools/cli.cpp:816
#10 0x000000000491ee5c in occa::cli::command::run (this=0x127028 <occaCommand>, argc=<optimized out>, 
    argv=<optimized out>) at /home/ubuntu/Work/git/occa/src/tools/cli.cpp:768
--Type <RET> for more, q to quit, c to continue without paging--
#11 0x000000000010cbd0 in main (argc=2, argv=0x1fff000458) at /usr/include/c++/9/ext/new_allocator.h:80
(gdb)

Going up the stack I get:

gdb) up
#2  occa::styling::section::toString[abi:cxx11](int, int, int, int, bool) const (this=0x5015d60, 
    indent=indent@entry=4, sectionWidth=sectionWidth@entry=6, fieldWidth=fieldWidth@entry=1701998403, 
    valueWidth=valueWidth@entry=1869440333, isFirstSection=isFirstSection@entry=true)
    at /home/ubuntu/Work/git/occa/src/tools/styling.cpp:126
126	         << std::string(fieldWidth   + 2, '=') << '+'
(gdb) l 120
115	
116	    std::string section::toString(const int indent,
117	                                  const int sectionWidth,
118	                                  const int fieldWidth,
119	                                  const int valueWidth,
120	                                  const bool isFirstSection) const {
121	      const std::string indentStr(indent, ' ');
122	
123	      std::stringstream ss;
124	      ss << indentStr
(gdb) l
125	         << std::string(sectionWidth + 2, '=') << '+'
126	         << std::string(fieldWidth   + 2, '=') << '+'
127	         << std::string(valueWidth   + 2, '=') << '\n';
128	      const std::string sectionDivider = ss.str();
129	      ss.str("");
130	
131	      ss << indentStr
132	         << std::string(sectionWidth + 2, ' ') << '|'
133	         << std::string(fieldWidth   + 2, '-') << '+'
134	         << std::string(valueWidth   + 2, '-') << '\n';
(gdb) p indent
$1 = 4
(gdb) p sectionWidth 
$2 = 6
(gdb) p fieldWidth 
$3 = 1701998403
(gdb)

So the fieldWidth for this toString call is garbage but the contents of the table seems ok

$8 = {
  sections = std::vector of length 2, capacity 2 = {{
      name = "CPU(s)", 
      groups = std::vector of length 1, capacity 1 = {{
          fields = std::vector of length 8, capacity 8 = {{
              name = "Cores", 
              value = "64"
            }, {
              name = "Memory (RAM)", 
              value = "123.58 GB"
            }, {
              name = "Clock Frequency", 
              value = "0 MHz"
            }, {
              name = "SIMD Instruction Set", 
              value = "N/A"
            }, {
              name = "SIMD Width", 
              value = "32 bits"
            }, {
              name = "L1 Cache Size (d)", 
              value = "   64K"
            }, {
              name = "L2 Cache Size", 
              value = " 1024K"
            }, {
              name = "L3 Cache Size", 
              value = "32768K"
            }}
        }}
    }, {
      name = "", 
      groups = std::vector of length 1, capacity 1 = {{
          fields = std::vector of length 0, capacity 8
        }}
    }}
}

Aug 17 '20 18:08 tcew