[root@compute-0-2 ~]# gdb /opt/torque/sbin/pbs_mom GNU gdb (GDB) Red Hat Enterprise Linux (7.2-50.el6) Copyright (C) 2010 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting instructions, please see: ... Reading symbols from /opt/torque/sbin/pbs_mom...(no debugging symbols found)...done. (gdb) set args -D (gdb) run Starting program: /opt/torque/sbin/pbs_mom -D [Thread debugging using libthread_db enabled] MOM is up [New Thread 0x2aaaab64a700 (LWP 8857)] [New Thread 0x2aaaab84b700 (LWP 8858)] tcp_request:(exit socket 8) processed 1 saving extra job info stdout=0 stderr=0 taskid=1 nodeid=0 ===== MD5 A5C14F7BC421DCB19DE7201B12840F6E Detaching after fork from child process 8859. saving extra job info stdout=51133 stderr=35944 taskid=2 nodeid=0 saving extra job info stdout=51133 stderr=35944 taskid=2 nodeid=0 do_tcp: got an internal task manager request tcp_request:(exit socket 9) processed 1 do_tcp: got an internal task manager request tcp_request:(exit socket 8) processed 1 do_tcp: got an internal task manager request do_tcp: got an internal task manager request do_tcp: got an internal task manager request tcp_request:(exit socket 9) processed 3 tcp_request:(exit socket 8) processed 1 do_tcp: got an internal task manager request tcp_request:(exit socket 10) processed 1 tcp_request:(exit socket 11) processed 1 tcp_request:(exit socket 8) processed 1 tcp_request:(exit socket 11) processed 1 tcp_request:(exit socket 8) processed 1 tcp_request:(exit socket 11) processed 1 tcp_request:(exit socket 8) processed 1 tcp_request:(exit socket 11) processed 1 tcp_request:(exit socket 12) processed 1 tcp_request:(exit socket 8) processed 1 tcp_request:(exit socket 11) processed 1 tcp_request:(exit socket 8) processed 1 do_tcp: got an internal task manager request saving extra job info stdout=51133 stderr=35944 taskid=11 nodeid=0 tcp_request:(exit socket 8) processed 1 tcp_request:(exit socket 10) processed 1 do_tcp: got an internal task manager request saving extra job info stdout=51133 stderr=35944 taskid=12 nodeid=0 tcp_request:(exit socket 9) processed 1 do_tcp: got an internal task manager request tcp_request:(exit socket 8) processed 1 tcp_request:(exit socket 10) processed 1 tcp_request:(exit socket 11) processed 1 do_tcp: got an internal task manager request tcp_request:(exit socket 9) processed 1 tcp_request:(exit socket 8) processed 1 tcp_request:(exit socket 8) processed 1 tcp_request:(exit socket 9) processed 1 scan_for_exiting: sending signal 9, "KILL" to job 13.hpc1.local, reason: local task termination detected Job - 13.hpc1.local Current State EXITING Detaching after fork from child process 8894. *** glibc detected *** /opt/torque/sbin/pbs_mom: double free or corruption (!prev): 0x0000000000c09e50 *** ======= Backtrace: ========= /lib64/libc.so.6[0x3761a75296] /opt/torque/lib64/libtorque.so.2(DIS_tcp_cleanup+0xd)[0x2aaaaaad56dd] /opt/torque/sbin/pbs_mom[0x434729] /opt/torque/sbin/pbs_mom[0x4347ee] /opt/torque/sbin/pbs_mom[0x434980] /opt/torque/sbin/pbs_mom[0x424f79] /opt/torque/sbin/pbs_mom[0x4352da] /opt/torque/lib64/libtorque.so.2(wait_request+0x1fb)[0x2aaaaaae1f5b] /opt/torque/sbin/pbs_mom[0x41cde6] /opt/torque/sbin/pbs_mom[0x41dfb5] /lib64/libc.so.6(__libc_start_main+0xfd)[0x3761a1ecdd] /opt/torque/sbin/pbs_mom[0x406c89] ======= Memory map: ======== 00400000-0045e000 r-xp 00000000 08:01 533765 /opt/torque/sbin/pbs_mom 0065d000-00664000 rw-p 0005d000 08:01 533765 /opt/torque/sbin/pbs_mom 00664000-00faa000 rw-p 00000000 00:00 0 [heap] 3761600000-3761620000 r-xp 00000000 08:01 410899 /lib64/ld-2.12.so 376181f000-3761820000 r--p 0001f000 08:01 410899 /lib64/ld-2.12.so 3761820000-3761821000 rw-p 00020000 08:01 410899 /lib64/ld-2.12.so 3761821000-3761822000 rw-p 00000000 00:00 0 3761a00000-3761b86000 r-xp 00000000 08:01 410900 /lib64/libc-2.12.so 3761b86000-3761d86000 ---p 00186000 08:01 410900 /lib64/libc-2.12.so 3761d86000-3761d8a000 r--p 00186000 08:01 410900 /lib64/libc-2.12.so 3761d8a000-3761d8b000 rw-p 0018a000 08:01 410900 /lib64/libc-2.12.so 3761d8b000-3761d90000 rw-p 00000000 00:00 0 3761e00000-3761e02000 r-xp 00000000 08:01 410901 /lib64/libdl-2.12.so 3761e02000-3762002000 ---p 00002000 08:01 410901 /lib64/libdl-2.12.so 3762002000-3762003000 r--p 00002000 08:01 410901 /lib64/libdl-2.12.so 3762003000-3762004000 rw-p 00003000 08:01 410901 /lib64/libdl-2.12.so 3762200000-3762217000 r-xp 00000000 08:01 410905 /lib64/libpthread-2.12.so 3762217000-3762416000 ---p 00017000 08:01 410905 /lib64/libpthread-2.12.so 3762416000-3762417000 r--p 00016000 08:01 410905 /lib64/libpthread-2.12.so 3762417000-3762418000 rw-p 00017000 08:01 410905 /lib64/libpthread-2.12.so 3762418000-376241c000 rw-p 00000000 00:00 0 3762a00000-3762a15000 r-xp 00000000 08:01 410490 /lib64/libz.so.1.2.3 3762a15000-3762c14000 ---p 00015000 08:01 410490 /lib64/libz.so.1.2.3 3762c14000-3762c15000 r--p 00014000 08:01 410490 /lib64/libz.so.1.2.3 3762c15000-3762c16000 rw-p 00015000 08:01 410490 /lib64/libz.so.1.2.3 3762e00000-3762e07000 r-xp 00000000 08:01 410542 /lib64/librt-2.12.so 3762e07000-3763006000 ---p 00007000 08:01 410542 /lib64/librt-2.12.so 3763006000-3763007000 r--p 00006000 08:01 410542 /lib64/librt-2.12.so 3763007000-3763008000 rw-p 00007000 08:01 410542 /lib64/librt-2.12.so 3763200000-3763202000 r-xp 00000000 08:01 410903 /lib64/libutil-2.12.so 3763202000-3763401000 ---p 00002000 08:01 410903 /lib64/libutil-2.12.so 3763401000-3763402000 r--p 00001000 08:01 410903 /lib64/libutil-2.12.so 3763402000-3763403000 rw-p 00002000 08:01 410903 /lib64/libutil-2.12.so 3764200000-3764216000 r-xp 00000000 08:01 410921 /lib64/libresolv-2.12.so 3764216000-3764416000 ---p 00016000 08:01 410921 /lib64/libresolv-2.12.so 3764416000-3764417000 r--p 00016000 08:01 410921 /lib64/libresolv-2.12.so 3764417000-3764418000 rw-p 00017000 08:01 410921 /lib64/libresolv-2.12.so 3764418000-376441a000 rw-p 00000000 00:00 0 3764600000-3764616000 r-xp 00000000 08:01 410494 /lib64/libgcc_s-4.4.6-20110824.so.1 3764616000-3764815000 ---p 00016000 08:01 410494 /lib64/libgcc_s-4.4.6-20110824.so.1 3764815000-3764816000 rw-p 00015000 08:01 410494 /lib64/libgcc_s-4.4.6-20110824.so.1 3765a00000-3765a03000 r-xp 00000000 08:01 410723 /lib64/libcom_err.so.2.1 3765a03000-3765c02000 ---p 00003000 08:01 410723 /lib64/libcom_err.so.2.1 3765c02000-3765c03000 r--p 00002000 08:01 410723 /lib64/libcom_err.so.2.1 3765c03000-3765c04000 rw-p 00003000 08:01 410723 /lib64/libcom_err.so.2.1 3767200000-3767373000 r-xp 00000000 08:01 790426 /usr/lib64/libcrypto.so.1.0.0 3767373000-3767573000 ---p 00173000 08:01 790426 /usr/lib64/libcrypto.so.1.0.0 3767573000-376758c000 r--p 00173000 08:01 790426 /usr/lib64/libcrypto.so.1.0.0 376758c000-3767596000 rw-p 0018c000 08:01 790426 /usr/lib64/libcrypto.so.1.0.0 3767596000-376759a000 rw-p 00000000 00:00 0 3767a00000-3767ad4000 r-xp 00000000 08:01 410935 /lib64/libkrb5.so.3.3 3767ad4000-3767cd4000 ---p 000d4000 08:01 410935 /lib64/libkrb5.so.3.3 3767cd4000-3767cdd000 r--p 000d4000 08:01 410935 /lib64/libkrb5.so.3.3 3767cdd000-3767cdf000 rw-p 000dd000 08:01 410935 /lib64/libkrb5.so.3.3 3767e00000-3767e2a000 r-xp 00000000 08:01 410934 /lib64/libk5crypto.so.3.1 3767e2a000-3768029000 ---p 0002a000 08:01 410934 /lib64/libk5crypto.so.3.1 3768029000-376802b000 r--p 00029000 08:01 410934 /lib64/libk5crypto.so.3.1 376802b000-376802c000 rw-p 0002b000 08:01 410934 /lib64/libk5crypto.so.3.1 3768200000-376820a000 r-xp 00000000 08:01 410933 /lib64/libkrb5support.so.0.1 376820a000-3768409000 ---p 0000a000 08:01 410933 /lib64/libkrb5support.so.0.1 3768409000-376840a000 r--p 00009000 08:01 410933 /lib64/libkrb5support.so.0.1 376840a000-376840b000 rw-p 0000a000 08:01 410933 /lib64/libkrb5support.so.0.1 3768600000-3768602000 r-xp 00000000 08:01 410511 /lib64/libkeyutils-1.2.so 3768602000-3768801000 ---p 00002000 08:01 410511 /lib64/libkeyutils-1.2.so 3768801000-3768802000 rw-p 00001000 08:01 410511 /lib64/libkeyutils-1.2.so 3768a00000-3768a3f000 r-xp 00000000 08:01 410936 /lib64/libgssapi_krb5.so.2.2 3768a3f000-3768c3f000 ---p 0003f000 08:01 410936 /lib64/libgssapi_krb5.so.2.2 3768c3f000-3768c40000 r--p 0003f000 08:01 410936 /lib64/libgssapi_krb5.so.2.2 3768c40000-3768c42000 rw-p 00040000 08:01 410936 /lib64/libgssapi_krb5.so.2.2 3768e00000-3768e53000 r-xp 00000000 08:01 792030 /usr/lib64/libssl.so.1.0.0 3768e53000-3769053000 ---p 00053000 08:01 792030 /usr/lib64/libssl.so.1.0.0 3769053000-3769056000 r--p 00053000 08:01 792030 /usr/lib64/libssl.so.1.0.0 3769056000-376905b000 rw-p 00056000 08:01 792030 /usr/lib64/libssl.so.1.0.0 2aaaaaaab000-2aaaaaaac000 r-xp 00000000 00:00 0 [vdso] 2aaaaaaac000-2aaaaaaaf000 rw-p 00000000 00:00 0 2aaaaaab8000-2aaaaaaf2000 r-xp 00000000 08:01 526316 /opt/torque/lib64/libtorque.so.2.0.0 2aaaaaaf2000-2aaaaacf2000 ---p 0003a000 08:01 526316 /opt/torque/lib64/libtorque.so.2.0.0 2aaaaacf2000-2aaaaacf4000 rw-p 0003a000 08:01 526316 /opt/torque/lib64/libtorque.so.2.0.0 2aaaaacf4000-2aaaaae08000 rw-p 00000000 00:00 0 2aaaaae08000-2aaaaae25000 r-xp 00000000 08:01 410919 /lib64/libselinux.so.1 2aaaaae25000-2aaaab024000 ---p 0001d000 08:01 410919 /lib64/libselinux.so.1 2aaaab024000-2aaaab025000 r--p 0001c000 08:01 410919 /lib64/libselinux.so.1 2aaaab025000-2aaaab026000 rw-p 0001d000 08:01 410919 /lib64/libselinux.so.1 2aaaab026000-2aaaab02b000 rw-p 00000000 00:00 0 2aaaab02b000-2aaaab037000 r-xp 00000000 08:01 410430 /lib64/libnss_files-2.12.so 2aaaab037000-2aaaab237000 ---p 0000c000 08:01 410430 /lib64/libnss_files-2.12.so 2aaaab237000-2aaaab238000 r--p 0000c000 08:01 410430 /lib64/libnss_files-2.12.so 2aaaab238000-2aaaab239000 rw-p 0000d000 08:01 410430 /lib64/libnss_files-2.12.so 2aaaab244000-2aaaab249000 r-xp 00000000 08:01 410428 /lib64/libnss_dns-2.12.so 2aaaab249000-2aaaab448000 ---p 00005000 08:01 410428 /lib64/libnss_dns-2.12.so 2aaaab448000-2aaaab449000 r--p 00004000 08:01 410428 /lib64/libnss_dns-2.12.so 2aaaab449000-2aaaab44a000 rw-p 00005000 08:01 410428 /lib64/libnss_dns-2.12.so 2aaaab44a000-2aaaab44b000 ---p 00000000 00:00 0 2aaaab44b000-2aaaab64b000 rw-p 00000000 00:00 0 2aaaab64b000-2aaaab64c000 ---p 00000000 00:00 0 2aaaab64c000-2aaaab84d000 rw-p 00000000 00:00 0 7ffffffbb000-7ffffffff000 rw-p 00000000 00:00 0 [stack] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] Program received signal SIGABRT, Aborted. 0x0000003761a32885 in raise () from /lib64/libc.so.6 Missing separate debuginfos, use: debuginfo-install torque-4.0.1-1.x86_64 (gdb) bt #0 0x0000003761a32885 in raise () from /lib64/libc.so.6 #1 0x0000003761a34065 in abort () from /lib64/libc.so.6 #2 0x0000003761a6f977 in __libc_message () from /lib64/libc.so.6 #3 0x0000003761a75296 in malloc_printerr () from /lib64/libc.so.6 #4 0x00002aaaaaad56dd in DIS_tcp_cleanup (chan=0x8ca040) at ../Libifl/tcp_dis.c:722 #5 0x0000000000434729 in tasks_free () #6 0x00000000004347ee in mom_job_free () #7 0x0000000000434980 in job_purge () #8 0x0000000000424f79 in req_deletejob () #9 0x00000000004352da in mom_process_request () #10 0x00002aaaaaae1f5b in wait_request (waittime=, SState=0x0) at ../Libnet/net_server.c:574 #11 0x000000000041cde6 in main_loop () #12 0x000000000041dfb5 in main () (gdb)