pixie icon indicating copy to clipboard operation
pixie copied to clipboard
verified publisher icon pixie-io

Socket tracer fails to start on latest Container OS

Open ddelnano opened this issue 1 year ago • 1 comments

Running Pixie on a recent (v109) COS GKE cluster results in a failed socket tracer.

To Reproduce

  1. Create GKE cluster using COS latest version (v109)
  2. px deploy
  3. Run a pxl script that uses the socket tracer data tables and see the following error and PEM logs

Screen Shot 2024-05-13 at 2 30 44 PM

pem logs 
I20240513 21:23:40.984936  8330 bcc_wrapper.cc:97] Successfully resolved task_struct offsets: {real_start_time=1832, group_leader=1584, exit_code=1416}
I20240513 21:23:40.985049  8330 bcc_wrapper.cc:166] Initializing BPF program ...
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1310:25: error: expected ')'
    if (({ typeof(void  btf_type_tag(user)*const) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_name); _val; }) != NULL) {
                        ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1310:18: note: to match this '('
    if (({ typeof(void  btf_type_tag(user)*const) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_name); _val; }) != NULL) {
                 ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1310:51: error: variable has incomplete type 'typeof(void)' (aka 'void')
    if (({ typeof(void  btf_type_tag(user)*const) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_name); _val; }) != NULL) {
                                                  ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1313:43: error: expected ')'
      connect_args.addr = ({ typeof(void  btf_type_tag(user)*const) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_name); _val; });
                                          ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1313:36: note: to match this '('
      connect_args.addr = ({ typeof(void  btf_type_tag(user)*const) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_name); _val; });
                                   ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1313:69: error: variable has incomplete type 'typeof(void)' (aka 'void')
      connect_args.addr = ({ typeof(void  btf_type_tag(user)*const) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_name); _val; });
                                                                    ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1321:46: error: expected ')'
    write_args.iov = ({ typeof(struct iovec  btf_type_tag(user)*const) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_iov); _val; });
                                             ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1321:31: note: to match this '('
    write_args.iov = ({ typeof(struct iovec  btf_type_tag(user)*const) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_iov); _val; });
                              ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1321:20: error: assigning to 'const struct iovec *' from incompatible type 'typeof(struct iovec)' (aka 'struct iovec')
    write_args.iov = ({ typeof(struct iovec  btf_type_tag(user)*const) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_iov); _val; });
                   ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1434:25: error: expected ')'
    if (({ typeof(void  btf_type_tag(user)*) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_name); _val; }) != NULL) {
                        ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1434:18: note: to match this '('
    if (({ typeof(void  btf_type_tag(user)*) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_name); _val; }) != NULL) {
                 ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1434:46: error: variable has incomplete type 'typeof(void)' (aka 'void')
    if (({ typeof(void  btf_type_tag(user)*) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_name); _val; }) != NULL) {
                                             ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1437:43: error: expected ')'
      connect_args.addr = ({ typeof(void  btf_type_tag(user)*) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_name); _val; });
                                          ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1437:36: note: to match this '('
      connect_args.addr = ({ typeof(void  btf_type_tag(user)*) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_name); _val; });
                                   ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1437:64: error: variable has incomplete type 'typeof(void)' (aka 'void')
      connect_args.addr = ({ typeof(void  btf_type_tag(user)*) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_name); _val; });
                                                               ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1445:45: error: expected ')'
    read_args.iov = ({ typeof(struct iovec  btf_type_tag(user)*) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_iov); _val; });
                                            ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1445:30: note: to match this '('
    read_args.iov = ({ typeof(struct iovec  btf_type_tag(user)*) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_iov); _val; });
                             ^
src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c:1445:19: error: assigning to 'const struct iovec *' from incompatible type 'typeof(struct iovec)' (aka 'struct iovec')
    read_args.iov = ({ typeof(struct iovec  btf_type_tag(user)*) _val; __builtin_memset(&_val, 0, sizeof(_val)); bpf_probe_read(&_val, sizeof(_val), (void *)&msghdr->msg_iov); _val; });
                  ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
12 errors generated.

Expected behavior Pixie's socket tracer should work with latest COS instances.

App information (please complete the following information):

  • Pixie version: 0.14.9
  • K8s cluster version: 1.28.7
  • Node Kernel version: 6.1.58
  • Node OS:
ddelnano@gke-dev-cluster-ddelnano-default-pool-0f4367ef-x20k ~ $ cat /etc/os-release
NAME="Container-Optimized OS"
ID=cos
PRETTY_NAME="Container-Optimized OS from Google"
HOME_URL="https://cloud.google.com/container-optimized-os/docs"
BUG_REPORT_URL="https://cloud.google.com/container-optimized-os/docs/resources/support-policy#contact_us"
GOOGLE_CRASH_ID=Lakitu
KERNEL_COMMIT_ID=a75611e516d2d29d1e04e26e345c3313a5b04b90
GOOGLE_METRICS_PRODUCT_ID=26
VERSION=109
VERSION_ID=109
BUILD_ID=17800.66.78

ddelnano avatar May 13 '24 21:05 ddelnano

I've tracked down the bug to a bcc bug (in combination with llvm).

Screen Shot 2024-05-17 at 1 34 57 PM

Upgrading the pixie BCC fork to include the changes through 0.30.0 results in a functional socket tracer. This also requires upgrading libbpf since we specify libbpf's version separately in our bazel configuration.

More details on the fix can be seen in https://github.com/iovisor/bcc/commit/57ca51bdd9c86870a43b9f76e062b3aede226ca4

ddelnano avatar May 17 '24 20:05 ddelnano

This is a duplicate of #1871.

ddelnano avatar May 20 '24 18:05 ddelnano