[torqueusers] MPI job not able to run after upgraded to 4.1.5.1

Steven Lo slo at cacr.caltech.edu
Tue Oct 15 16:04:34 MDT 2013


Hi,

We just upgraded our Torque server to 4.1.5.1 and we are having trouble 
of running a simple MPI
program with just 2 nodes:

/* C Example of hello_world with 4 basic mpi calls */
#include <stdio.h>
#include <mpi.h>


int main (argc, argv)
      int argc;
      char *argv[];
{
   int rank, size;

   MPI_Init (&argc, &argv);    /* starts MPI */
   MPI_Comm_rank (MPI_COMM_WORLD, &rank);    /* get current process id */
   MPI_Comm_size (MPI_COMM_WORLD, &size);    /* get number of processes */
   printf( "Hello world from process %d of %d\n", rank, size );
   MPI_Finalize();
   return 0;
}


 From the 'strace' (output attached), it looks like the pbs_mom (on 
172.18.1.188) is not able to
communicate properly with the sister node (172.18.1.172):

     shc172 - daemon did not report back when launched



The 'qstat -n' shows the job is queued properly:

shc188: qstat -n

maitre:
Req'd    Req'd      Elap
Job ID               Username    Queue    Jobname          SessID NDS   
TSK    Memory   Time   S   Time
-------------------- ----------- -------- ---------------- ------ ----- 
------ ------ -------- - --------
444347.maitre        sharon      weekdayQ STDIN 0     2      2    --  
00:30:00 R      --
    shc188/0+shc172/0



The section in the strace which puzzle us is the following:

socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 12
setsockopt(12, SOL_SOCKET, SO_LINGER, {onoff=1, linger=5}, 8) = 0
connect(12, {sa_family=AF_INET, sin_port=htons(15003), 
sin_addr=inet_addr("127.0.0.1")}, 16) = 0
mmap(NULL, 528384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 
0) = 0x2b3a50008000
write(12, "+2+12+13444344.maitre2+32F8908E4"..., 66) = 66
poll([{fd=12, events=POLLIN|POLLHUP}], 1, 2147483647) = 1 ([{fd=12, 
revents=POLLIN}])
fcntl(12, F_GETFL)                      = 0x2 (flags O_RDWR)
read(12, "", 262144)                    = 0
close(12)                               = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, 
events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) 
= 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, 
events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) 
= 0 (Timeout)
sched_yield()                           = 0
     .
     .
     .



I don't think the firewall is an issue since we are allowing all packets 
from/to nodes in the private network.


Your suggestion on how to debug is much appreciated.


Thanks.

Steven.


-------------- next part --------------
execve("/usr/local/openmpi-1.3.3-gcc/bin/mpirun", ["mpirun", "-np", "2", "./hello"], [/* 79 vars */]) = 0
brk(0)                                  = 0x10785000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3cfaf4e000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3cfaf4f000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/tls/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vampirtrace-5.8.2/lib/tls/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/tls/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vampirtrace-5.8.2/lib/tls", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vampirtrace-5.8.2/lib/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vampirtrace-5.8.2/lib", {st_mode=S_IFDIR|S_ISGID|0755, st_size=4096, ...}) = 0
open("/usr/local/openmpi-1.4.3-gcc/lib/tls/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/openmpi-1.4.3-gcc/lib/tls/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/tls/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/openmpi-1.4.3-gcc/lib/tls", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/openmpi-1.4.3-gcc/lib/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/openmpi-1.4.3-gcc/lib", {st_mode=S_IFDIR|S_ISGID|0755, st_size=8192, ...}) = 0
open("/usr/local/vtk-5.0.3/lib/tls/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vtk-5.0.3/lib/tls/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/tls/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vtk-5.0.3/lib/tls", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vtk-5.0.3/lib/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vtk-5.0.3/lib", {st_mode=S_IFDIR|S_ISGID|0755, st_size=20480, ...}) = 0
open("/modules/mpi/tls/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/modules/mpi/tls/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/modules/mpi/tls/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/modules/mpi/tls", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/modules/mpi/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/modules/mpi/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/modules/mpi/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/modules/mpi", 0x7fffa5e6a5f0)    = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/tls/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/home/sharon/local/lib/tls/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/tls/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/home/sharon/local/lib/tls", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/home/sharon/local/lib/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/home/sharon/local/lib", {st_mode=S_IFDIR|0755, st_size=8192, ...}) = 0
open("/usr/local/Intel-11.1/lib/intel64/tls/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/Intel-11.1/lib/intel64/tls/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/tls/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/Intel-11.1/lib/intel64/tls", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/Intel-11.1/lib/intel64/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/Intel-11.1/lib/intel64", {st_mode=S_IFDIR|S_ISGID|0755, st_size=8192, ...}) = 0
open("/usr/local/papi-4.1.3/lib/tls/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/papi-4.1.3/lib/tls/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/tls/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/papi-4.1.3/lib/tls", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/x86_64/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/papi-4.1.3/lib/x86_64", 0x7fffa5e6a5f0) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/libtermcap.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/papi-4.1.3/lib", {st_mode=S_IFDIR|S_ISGID|0755, st_size=4096, ...}) = 0
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=103318, ...}) = 0
mmap(NULL, 103318, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2b3cfaf50000
close(3)                                = 0
open("/lib64/libtermcap.so.2", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\20\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=13576, ...}) = 0
mmap(NULL, 2108944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3cfb14f000
mprotect(0x2b3cfb152000, 2093056, PROT_NONE) = 0
mmap(0x2b3cfb351000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x2b3cfb351000
close(3)                                = 0
open("/usr/local/vampirtrace-5.8.2/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libdl.so.2", O_RDONLY)     = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\16\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=20424, ...}) = 0
mmap(NULL, 2109696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3cfb352000
mprotect(0x2b3cfb354000, 2097152, PROT_NONE) = 0
mmap(0x2b3cfb554000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x2b3cfb554000
close(3)                                = 0
open("/usr/local/vampirtrace-5.8.2/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libc.so.6", O_RDONLY)      = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\332\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1716720, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3cfb556000
mmap(NULL, 3502424, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3cfb557000
mprotect(0x2b3cfb6a5000, 2097152, PROT_NONE) = 0
mmap(0x2b3cfb8a5000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14e000) = 0x2b3cfb8a5000
mmap(0x2b3cfb8aa000, 16728, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2b3cfb8aa000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3cfb8af000
arch_prctl(ARCH_SET_FS, 0x2b3cfb8af6e0) = 0
mprotect(0x2b3cfb8a5000, 16384, PROT_READ) = 0
mprotect(0x2b3cfb554000, 4096, PROT_READ) = 0
mprotect(0x2b3cfb14d000, 4096, PROT_READ) = 0
munmap(0x2b3cfaf50000, 103318)          = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
open("/dev/tty", O_RDWR|O_NONBLOCK)     = 3
close(3)                                = 0
brk(0)                                  = 0x10785000
brk(0x107a6000)                         = 0x107a6000
open("/usr/lib/locale/locale-archive", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=56418112, ...}) = 0
mmap(NULL, 56418112, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2b3cfb8b0000
close(3)                                = 0
getuid()                                = 12012
getgid()                                = 25721
geteuid()                               = 12012
getegid()                               = 25721
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
open("/proc/meminfo", O_RDONLY)         = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3cfee7e000
read(3, "MemTotal:     16441180 kB\nMemFre"..., 1024) = 777
close(3)                                = 0
munmap(0x2b3cfee7e000, 4096)            = 0
rt_sigaction(SIGCHLD, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGCHLD, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, 8) = 0
rt_sigaction(SIGINT, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGINT, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, 8) = 0
rt_sigaction(SIGQUIT, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGQUIT, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, 8) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
rt_sigaction(SIGQUIT, {SIG_IGN, [], SA_RESTORER, 0x2b3cfb5872d0}, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, 8) = 0
uname({sys="Linux", node="shc188", ...}) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
open("/usr/lib64/gconv/gconv-modules.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=25464, ...}) = 0
mmap(NULL, 25464, PROT_READ, MAP_SHARED, 3, 0) = 0x2b3cfee7e000
close(3)                                = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
stat("/home/sharon/basic", {st_mode=S_IFDIR|0755, st_size=20480, ...}) = 0
stat(".", {st_mode=S_IFDIR|0755, st_size=20480, ...}) = 0
getpid()                                = 8193
getppid()                               = 8192
getpgrp()                               = 8192
rt_sigaction(SIGCHLD, {0x436080, [], SA_RESTORER, 0x2b3cfb5872d0}, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, 8) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
open("/usr/local/openmpi-1.3.3-gcc/bin/mpirun", O_RDONLY) = 3
ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffa5e6adf0) = -1 ENOTTY (Inappropriate ioctl for device)
lseek(3, 0, SEEK_CUR)                   = 0
read(3, "#!/bin/sh\n\necho \"openmpi (v1.3.3"..., 80) = 80
lseek(3, 0, SEEK_SET)                   = 0
getrlimit(RLIMIT_NOFILE, {rlim_cur=1024, rlim_max=1024}) = 0
dup2(3, 255)                            = 255
close(3)                                = 0
fcntl(255, F_SETFD, FD_CLOEXEC)         = 0
fcntl(255, F_GETFL)                     = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(255, {st_mode=S_IFREG|0755, st_size=182, ...}) = 0
lseek(255, 0, SEEK_CUR)                 = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
read(255, "#!/bin/sh\n\necho \"openmpi (v1.3.3"..., 182) = 182
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
pipe([3, 4])                            = 0
rt_sigprocmask(SIG_BLOCK, [INT CHLD], [CHLD], 8) = 0
lseek(255, -52, SEEK_CUR)               = 130
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x2b3cfb8af770) = 8194
rt_sigprocmask(SIG_SETMASK, [CHLD], NULL, 8) = 0
close(4)                                = 0
close(4)                                = -1 EBADF (Bad file descriptor)
rt_sigprocmask(SIG_BLOCK, [INT CHLD], [CHLD], 8) = 0
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x2b3cfb8af770) = 8196
rt_sigprocmask(SIG_SETMASK, [CHLD], NULL, 8) = 0
close(3)                                = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [CHLD], 8) = 0
rt_sigprocmask(SIG_SETMASK, [CHLD], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [CHLD], 8) = 0
rt_sigaction(SIGINT, {0x436c60, [], SA_RESTORER, 0x2b3cfb5872d0}, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, 8) = 0
wait4(-1, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0, NULL) = 8194
wait4(-1, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0, NULL) = 8196
rt_sigprocmask(SIG_SETMASK, [CHLD], NULL, 8) = 0
rt_sigaction(SIGINT, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, {0x436c60, [], SA_RESTORER, 0x2b3cfb5872d0}, 8) = 0
close(3)                                = -1 EBADF (Bad file descriptor)
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
--- SIGCHLD (Child exited) @ 0 (0) ---
wait4(-1, 0x7fffa5e6aac4, WNOHANG, NULL) = -1 ECHILD (No child processes)
rt_sigreturn(0xffffffffffffffff)        = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
read(255, "\nexec /usr/local/openmpi-1.3.3-g"..., 182) = 52
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
rt_sigaction(SIGINT, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, 8) = 0
rt_sigaction(SIGQUIT, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, {SIG_IGN, [], SA_RESTORER, 0x2b3cfb5872d0}, 8) = 0
rt_sigaction(SIGCHLD, {SIG_DFL, [], SA_RESTORER, 0x2b3cfb5872d0}, {0x436080, [], SA_RESTORER, 0x2b3cfb5872d0}, 8) = 0
execve("/usr/local/openmpi-1.3.3-gcc/bin/orterun", ["/usr/local/openmpi-1.3.3-gcc/bin"..., "-np", "2", "./hello"], [/* 77 vars */]) = 0
brk(0)                                  = 0x1584000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4c451000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4c452000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/tls/x86_64/libopen-rte.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/openmpi-1.3.3-gcc/lib/tls/x86_64", 0x7fff6b789510) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/tls/libopen-rte.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/openmpi-1.3.3-gcc/lib/tls", 0x7fff6b789510) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/x86_64/libopen-rte.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/openmpi-1.3.3-gcc/lib/x86_64", 0x7fff6b789510) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/libopen-rte.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\315\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=334139, ...}) = 0
mmap(NULL, 2384128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3a4c652000
mprotect(0x2b3a4c694000, 2097152, PROT_NONE) = 0
mmap(0x2b3a4c894000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x42000) = 0x2b3a4c894000
mmap(0x2b3a4c897000, 4352, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2b3a4c897000
close(3)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/libopen-pal.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \335\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=388592, ...}) = 0
mmap(NULL, 2559944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3a4c899000
mprotect(0x2b3a4c8e5000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4cae4000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4b000) = 0x2b3a4cae4000
mmap(0x2b3a4cae7000, 143304, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2b3a4cae7000
close(3)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/tls/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vampirtrace-5.8.2/lib/tls/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vampirtrace-5.8.2/lib/tls", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vampirtrace-5.8.2/lib/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vampirtrace-5.8.2/lib", {st_mode=S_IFDIR|S_ISGID|0755, st_size=4096, ...}) = 0
open("/usr/local/openmpi-1.4.3-gcc/lib/tls/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/openmpi-1.4.3-gcc/lib/tls/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/openmpi-1.4.3-gcc/lib/tls", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/openmpi-1.4.3-gcc/lib/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/openmpi-1.4.3-gcc/lib", {st_mode=S_IFDIR|S_ISGID|0755, st_size=8192, ...}) = 0
open("/usr/local/vtk-5.0.3/lib/tls/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vtk-5.0.3/lib/tls/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vtk-5.0.3/lib/tls", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vtk-5.0.3/lib/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/vtk-5.0.3/lib", {st_mode=S_IFDIR|S_ISGID|0755, st_size=20480, ...}) = 0
open("/modules/mpi/tls/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/modules/mpi/tls/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/modules/mpi/tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/modules/mpi/tls", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/modules/mpi/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/modules/mpi/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/modules/mpi/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/modules/mpi", 0x7fff6b7894b0)    = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/tls/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/home/sharon/local/lib/tls/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/home/sharon/local/lib/tls", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/home/sharon/local/lib/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/home/sharon/local/lib", {st_mode=S_IFDIR|0755, st_size=8192, ...}) = 0
open("/usr/local/Intel-11.1/lib/intel64/tls/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/Intel-11.1/lib/intel64/tls/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/Intel-11.1/lib/intel64/tls", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/Intel-11.1/lib/intel64/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/Intel-11.1/lib/intel64", {st_mode=S_IFDIR|S_ISGID|0755, st_size=8192, ...}) = 0
open("/usr/local/papi-4.1.3/lib/tls/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/papi-4.1.3/lib/tls/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/papi-4.1.3/lib/tls", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/papi-4.1.3/lib/x86_64", 0x7fff6b7894b0) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/papi-4.1.3/lib", {st_mode=S_IFDIR|S_ISGID|0755, st_size=4096, ...}) = 0
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=103318, ...}) = 0
mmap(NULL, 103318, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2b3a4cb0a000
close(3)                                = 0
open("/lib64/libdl.so.2", O_RDONLY)     = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\16\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=20424, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4cb24000
mmap(NULL, 2109696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3a4cb25000
mprotect(0x2b3a4cb27000, 2097152, PROT_NONE) = 0
mmap(0x2b3a4cd27000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x2b3a4cd27000
close(3)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/libnsl.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/libnsl.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/libnsl.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/libnsl.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/libnsl.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/libnsl.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/libnsl.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libnsl.so.1", O_RDONLY)    = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240@\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=111480, ...}) = 0
mmap(NULL, 2194096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3a4cd29000
mprotect(0x2b3a4cd3e000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4cf3d000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x2b3a4cf3d000
mmap(0x2b3a4cf3f000, 6832, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2b3a4cf3f000
close(3)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/libutil.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/libutil.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/libutil.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/libutil.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/libutil.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/libutil.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/libutil.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libutil.so.1", O_RDONLY)   = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\16\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=15280, ...}) = 0
mmap(NULL, 2105616, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3a4cf41000
mprotect(0x2b3a4cf43000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4d142000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x2b3a4d142000
close(3)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libm.so.6", O_RDONLY)      = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`>\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=611880, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4d144000
mmap(NULL, 2629848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3a4d145000
mprotect(0x2b3a4d1c7000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4d3c6000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x81000) = 0x2b3a4d3c6000
close(3)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libpthread.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240W\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=142696, ...}) = 0
mmap(NULL, 2204528, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3a4d3c8000
mprotect(0x2b3a4d3de000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4d5dd000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x2b3a4d5dd000
mmap(0x2b3a4d5df000, 13168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2b3a4d5df000
close(3)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libc.so.6", O_RDONLY)      = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\332\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1716720, ...}) = 0
mmap(NULL, 3502424, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3a4d5e3000
mprotect(0x2b3a4d731000, 2097152, PROT_NONE) = 0
mmap(0x2b3a4d931000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14e000) = 0x2b3a4d931000
mmap(0x2b3a4d936000, 16728, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2b3a4d936000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4d93b000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4d93c000
arch_prctl(ARCH_SET_FS, 0x2b3a4d93c380) = 0
mprotect(0x2b3a4d931000, 16384, PROT_READ) = 0
mprotect(0x2b3a4d5dd000, 4096, PROT_READ) = 0
mprotect(0x2b3a4d3c6000, 4096, PROT_READ) = 0
mprotect(0x2b3a4d142000, 4096, PROT_READ) = 0
mprotect(0x2b3a4cf3d000, 4096, PROT_READ) = 0
mprotect(0x2b3a4cd27000, 4096, PROT_READ) = 0
mprotect(0x2b3a4c650000, 4096, PROT_READ) = 0
munmap(0x2b3a4cb0a000, 103318)          = 0
set_tid_address(0x2b3a4d93c410)         = 8193
set_robust_list(0x2b3a4d93c420, 0x18)   = 0
futex(0x7fff6b78a03c, FUTEX_WAKE_PRIVATE, 1) = 0
rt_sigaction(SIGRTMIN, {0x2b3a4d3cd380, [], SA_RESTORER|SA_SIGINFO, 0x2b3a4d3d6b10}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x2b3a4d3cd2b0, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x2b3a4d3d6b10}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=10240*1024, rlim_max=RLIM_INFINITY}) = 0
stat("/sys/class/infiniband", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/class/infiniband", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
brk(0)                                  = 0x1584000
brk(0x15a5000)                          = 0x15a5000
uname({sys="Linux", node="shc188", ...}) = 0
getcwd("/home/sharon/basic", 4096)      = 19
open("/usr/local/openmpi-1.3.3-gcc/etc/openmpi-mca-params.conf", O_RDONLY) = 3
ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fff6b789a90) = -1 ENOTTY (Inappropriate ioctl for device)
fstat(3, {st_mode=S_IFREG|0644, st_size=2812, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4cb0a000
read(3, "#\n# Copyright (c) 2004-2005 The "..., 8192) = 2812
read(3, "", 4096)                       = 0
read(3, "", 8192)                       = 0
ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fff6b789a60) = -1 ENOTTY (Inappropriate ioctl for device)
close(3)                                = 0
munmap(0x2b3a4cb0a000, 4096)            = 0
open("/home/sharon/.openmpi/mca-params.conf", O_RDONLY) = 3
ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fff6b789a90) = -1 ENOTTY (Inappropriate ioctl for device)
fstat(3, {st_mode=S_IFREG|0644, st_size=81, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4cb0a000
read(3, "# btl_openib_ib_timeout=15\n# btl"..., 8192) = 81
read(3, "", 4096)                       = 0
read(3, "", 8192)                       = 0
ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fff6b789a60) = -1 ENOTTY (Inappropriate ioctl for device)
close(3)                                = 0
munmap(0x2b3a4cb0a000, 4096)            = 0
uname({sys="Linux", node="shc188", ...}) = 0
rt_sigaction(SIGABRT, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGABRT, {0x2b3a4c8c85a0, [], SA_RESTORER|SA_RESETHAND|SA_SIGINFO, 0x2b3a4d3d6b10}, NULL, 8) = 0
rt_sigaction(SIGBUS, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGBUS, {0x2b3a4c8c85a0, [], SA_RESTORER|SA_RESETHAND|SA_SIGINFO, 0x2b3a4d3d6b10}, NULL, 8) = 0
rt_sigaction(SIGFPE, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGFPE, {0x2b3a4c8c85a0, [], SA_RESTORER|SA_RESETHAND|SA_SIGINFO, 0x2b3a4d3d6b10}, NULL, 8) = 0
rt_sigaction(SIGSEGV, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGSEGV, {0x2b3a4c8c85a0, [], SA_RESTORER|SA_RESETHAND|SA_SIGINFO, 0x2b3a4d3d6b10}, NULL, 8) = 0
getrlimit(RLIMIT_NOFILE, {rlim_cur=1024, rlim_max=1024}) = 0
setrlimit(RLIMIT_NOFILE, {rlim_cur=1024, rlim_max=1024}) = 0
getrlimit(RLIMIT_NPROC, {rlim_cur=128424, rlim_max=128424}) = 0
setrlimit(RLIMIT_NPROC, {rlim_cur=128424, rlim_max=128424}) = 0
getrlimit(RLIMIT_FSIZE, {rlim_cur=RLIM_INFINITY, rlim_max=RLIM_INFINITY}) = 0
setrlimit(RLIMIT_FSIZE, {rlim_cur=RLIM_INFINITY, rlim_max=RLIM_INFINITY}) = 0
getcwd("/home/sharon/basic", 4097)      = 19
uname({sys="Linux", node="shc188", ...}) = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
getdents(3, /* 128 entries */, 32768)   = 5248
getdents(3, /* 0 entries */, 32768)     = 0
close(3)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_paffinity_linux.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_paffinity_linux.la", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0755, st_size=1025, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4cb0a000
read(3, "# mca_paffinity_linux.la - a lib"..., 4096) = 1025
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x2b3a4cb0a000, 4096)            = 0
futex(0x2b3a4cd280ec, FUTEX_WAKE_PRIVATE, 2147483647) = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_paffinity_linux.so", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=25896, ...}) = 0
mmap(NULL, 2115040, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3a4d93d000
mprotect(0x2b3a4d941000, 2097152, PROT_NONE) = 0
mmap(0x2b3a4db41000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x2b3a4db41000
close(3)                                = 0
sched_getaffinity(0, 128,  { ff, 0, 0, 0 }) = 32
sched_setaffinity(0, 32,  { 0, 0, 0, 0 }) = -1 EFAULT (Bad address)
access("/sys/devices/system/cpu", R_OK|X_OK) = 0
open("/sys/devices/system/cpu", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
getdents(3, /* 11 entries */, 32768)    = 288
getdents(3, /* 0 entries */, 32768)     = 0
close(3)                                = 0
open("/sys/devices/system/cpu/cpu0/online", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/sys/devices/system/cpu/cpu0/topology/core_id", O_RDONLY) = 3
read(3, "0\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu0/topology/physical_package_id", O_RDONLY) = 3
read(3, "0\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu1/online", O_RDONLY) = -1 EACCES (Permission denied)
open("/sys/devices/system/cpu/cpu1/topology/core_id", O_RDONLY) = 3
read(3, "1\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu1/topology/physical_package_id", O_RDONLY) = 3
read(3, "0\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu2/online", O_RDONLY) = -1 EACCES (Permission denied)
open("/sys/devices/system/cpu/cpu2/topology/core_id", O_RDONLY) = 3
read(3, "2\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu2/topology/physical_package_id", O_RDONLY) = 3
read(3, "0\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu3/online", O_RDONLY) = -1 EACCES (Permission denied)
open("/sys/devices/system/cpu/cpu3/topology/core_id", O_RDONLY) = 3
read(3, "3\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu3/topology/physical_package_id", O_RDONLY) = 3
read(3, "0\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu4/online", O_RDONLY) = -1 EACCES (Permission denied)
open("/sys/devices/system/cpu/cpu4/topology/core_id", O_RDONLY) = 3
read(3, "0\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu4/topology/physical_package_id", O_RDONLY) = 3
read(3, "1\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu5/online", O_RDONLY) = -1 EACCES (Permission denied)
open("/sys/devices/system/cpu/cpu5/topology/core_id", O_RDONLY) = 3
read(3, "1\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu5/topology/physical_package_id", O_RDONLY) = 3
read(3, "1\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu6/online", O_RDONLY) = -1 EACCES (Permission denied)
open("/sys/devices/system/cpu/cpu6/topology/core_id", O_RDONLY) = 3
read(3, "2\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu6/topology/physical_package_id", O_RDONLY) = 3
read(3, "1\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu7/online", O_RDONLY) = -1 EACCES (Permission denied)
open("/sys/devices/system/cpu/cpu7/topology/core_id", O_RDONLY) = 3
read(3, "3\n", 7)                       = 2
close(3)                                = 0
open("/sys/devices/system/cpu/cpu7/topology/physical_package_id", O_RDONLY) = 3
read(3, "1\n", 7)                       = 2
close(3)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
getdents(3, /* 128 entries */, 32768)   = 5248
getdents(3, /* 0 entries */, 32768)     = 0
close(3)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
getdents(3, /* 128 entries */, 32768)   = 5248
getdents(3, /* 0 entries */, 32768)     = 0
close(3)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
mmap(NULL, 4198400, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4db42000
mremap(0x2b3a4db42000, 4198400, 4198400, MREMAP_MAYMOVE) = 0x2b3a4db42000
munmap(0x2b3a4db42000, 4198400)         = 0
mmap(NULL, 1052672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4db42000
munmap(0x2b3a4db42000, 1052672)         = 0
mmap(NULL, 4096, PROT_READ, MAP_FILE|MAP_ANONYMOUS, -1, 0) = -1 EINVAL (Invalid argument)
munmap(0xffffffffffffffff, 4096)        = -1 EINVAL (Invalid argument)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
getdents(3, /* 128 entries */, 32768)   = 5248
getdents(3, /* 0 entries */, 32768)     = 0
close(3)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
getdents(3, /* 128 entries */, 32768)   = 5248
getdents(3, /* 0 entries */, 32768)     = 0
close(3)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
getdents(3, /* 128 entries */, 32768)   = 5248
getdents(3, /* 0 entries */, 32768)     = 0
close(3)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/proc/cpuinfo", O_RDONLY)         = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4db43000
lseek(3, 0, SEEK_SET)                   = 0
read(3, "processor\t: 0\nvendor_id\t: Authen"..., 1024) = 1024
read(3, "u cores\t: 4\napicid\t\t: 1\nfpu\t\t: y"..., 1024) = 1024
read(3, " pdpe1gb rdtscp lm 3dnowext 3dno"..., 1024) = 1024
read(3, "4K pages\nclflush size\t: 64\ncache"..., 1024) = 1024
read(3, "\t: 4\nmodel name\t: Quad-Core AMD "..., 1024) = 1024
read(3, "level\t: 5\nwp\t\t: yes\nflags\t\t: fpu"..., 1024) = 1024
read(3, "op_tsc pni cx16 popcnt lahf_lm c"..., 1024) = 312
read(3, "", 1024)                       = 0
lseek(3, 0, SEEK_SET)                   = 0
read(3, "processor\t: 0\nvendor_id\t: Authen"..., 1024) = 1024
close(3)                                = 0
munmap(0x2b3a4db43000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
getdents(3, /* 128 entries */, 32768)   = 5248
getdents(3, /* 0 entries */, 32768)     = 0
close(3)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_carto_auto_detect.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_carto_auto_detect.la", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0755, st_size=1037, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4db42000
read(3, "# mca_carto_auto_detect.la - a l"..., 4096) = 1037
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x2b3a4db42000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_carto_auto_detect.so", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\10\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=8864, ...}) = 0
mmap(NULL, 2101136, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3a4db42000
mprotect(0x2b3a4db43000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4dd42000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0x2b3a4dd42000
close(3)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_carto_file.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_carto_file.la", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0755, st_size=995, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4dd43000
read(3, "# mca_carto_file.la - a libtool "..., 4096) = 995
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x2b3a4dd43000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_carto_file.so", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\20\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=23640, ...}) = 0
mmap(NULL, 2113336, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2b3a4dd43000
mprotect(0x2b3a4dd47000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4df46000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x2b3a4df46000
close(3)                                = 0
munmap(0x2b3a4dd43000, 2113336)         = 0
socketpair(PF_FILE, SOCK_STREAM, 0, [3, 4]) = 0
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
fcntl(4, F_SETFD, FD_CLOEXEC)           = 0
fcntl(3, F_SETFL, O_RDONLY|O_NONBLOCK)  = 0
uname({sys="Linux", node="shc188", ...}) = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 5
fcntl(5, F_SETFD, FD_CLOEXEC)           = 0
getdents(5, /* 128 entries */, 32768)   = 5248
getdents(5, /* 0 entries */, 32768)     = 0
close(5)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_env.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_env.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=977, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4dd43000
read(5, "# mca_ess_env.la - a libtool lib"..., 4096) = 977
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4dd43000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_env.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\16\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=15232, ...}) = 0
mmap(NULL, 2106776, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4dd43000
mprotect(0x2b3a4dd45000, 2097152, PROT_NONE) = 0
mmap(0x2b3a4df45000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x2000) = 0x2b3a4df45000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_hnp.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_hnp.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=977, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4df46000
read(5, "# mca_ess_hnp.la - a libtool lib"..., 4096) = 977
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4df46000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_hnp.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\34\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=21458, ...}) = 0
mmap(NULL, 2110944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4df46000
mprotect(0x2b3a4df49000, 2097152, PROT_NONE) = 0
mmap(0x2b3a4e149000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x3000) = 0x2b3a4e149000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_singleton.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_singleton.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=1013, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4e14a000
read(5, "# mca_ess_singleton.la - a libto"..., 4096) = 1013
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4e14a000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_singleton.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\22\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=16823, ...}) = 0
mmap(NULL, 2107640, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4e14a000
mprotect(0x2b3a4e14d000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4e34c000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x2000) = 0x2b3a4e34c000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_slurm.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_slurm.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=989, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4e34d000
read(5, "# mca_ess_slurm.la - a libtool l"..., 4096) = 989
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4e34d000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_slurm.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\20\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=15542, ...}) = 0
mmap(NULL, 2106808, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4e34d000
mprotect(0x2b3a4e34f000, 2097152, PROT_NONE) = 0
mmap(0x2b3a4e54f000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x2000) = 0x2b3a4e54f000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_tool.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_tool.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=983, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4e550000
read(5, "# mca_ess_tool.la - a libtool li"..., 4096) = 983
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4e550000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ess_tool.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\t\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=10023, ...}) = 0
mmap(NULL, 2102304, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4e550000
mprotect(0x2b3a4e551000, 2097152, PROT_NONE) = 0
mmap(0x2b3a4e751000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x1000) = 0x2b3a4e751000
close(5)                                = 0
munmap(0x2b3a4dd43000, 2106776)         = 0
munmap(0x2b3a4e14a000, 2107640)         = 0
munmap(0x2b3a4e34d000, 2106808)         = 0
munmap(0x2b3a4e550000, 2102304)         = 0
rt_sigaction(SIGCHLD, {0x2b3a4c8b5dc0, [CHLD], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, {SIG_DFL, [], 0}, 8) = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 5
fcntl(5, F_SETFD, FD_CLOEXEC)           = 0
getdents(5, /* 128 entries */, 32768)   = 5248
getdents(5, /* 0 entries */, 32768)     = 0
close(5)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_plm_rsh.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_plm_rsh.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=977, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4dd43000
read(5, "# mca_plm_rsh.la - a libtool lib"..., 4096) = 977
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4dd43000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_plm_rsh.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@$\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=35962, ...}) = 0
mmap(NULL, 2123840, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4e14a000
mprotect(0x2b3a4e150000, 2097152, PROT_NONE) = 0
mmap(0x2b3a4e350000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x6000) = 0x2b3a4e350000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_plm_slurm.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_plm_slurm.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=989, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4e351000
read(5, "# mca_plm_slurm.la - a libtool l"..., 4096) = 989
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4e351000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_plm_slurm.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\26\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=18625, ...}) = 0
mmap(NULL, 2108632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4e351000
mprotect(0x2b3a4e354000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4e553000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x2000) = 0x2b3a4e553000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_plm_tm.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_plm_tm.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=1042, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4e554000
read(5, "# mca_plm_tm.la - a libtool libr"..., 4096) = 1042
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4e554000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_plm_tm.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260\25\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=17671, ...}) = 0
mmap(NULL, 2108056, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4e554000
mprotect(0x2b3a4e557000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4e756000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x2000) = 0x2b3a4e756000
close(5)                                = 0
open("/usr/local/torque-2.3.3/lib/tls/x86_64/libtorque.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/torque-2.3.3/lib/tls/x86_64", 0x7fff6b784dc0) = -1 ENOENT (No such file or directory)
open("/usr/local/torque-2.3.3/lib/tls/libtorque.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/torque-2.3.3/lib/tls", 0x7fff6b784dc0) = -1 ENOENT (No such file or directory)
open("/usr/local/torque-2.3.3/lib/x86_64/libtorque.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/torque-2.3.3/lib/x86_64", 0x7fff6b784dc0) = -1 ENOENT (No such file or directory)
open("/usr/local/torque-2.3.3/lib/libtorque.so.2", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\257\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=615539, ...}) = 0
mmap(NULL, 2485824, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4e757000
mprotect(0x2b3a4e788000, 2097152, PROT_NONE) = 0
mmap(0x2b3a4e988000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x31000) = 0x2b3a4e988000
mmap(0x2b3a4e98b000, 175680, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2b3a4e98b000
close(5)                                = 0
getcwd("/home/sharon/basic", 4097)      = 19
stat("/opt/torque/bin/ssh", 0x7fff6b788b50) = -1 ENOENT (No such file or directory)
stat("/usr/local/python-2.6.4/bin/ssh", 0x7fff6b788b50) = -1 ENOENT (No such file or directory)
stat("/usr/local/openmpi-1.3.3-gcc/bin/ssh", 0x7fff6b788b50) = -1 ENOENT (No such file or directory)
stat("/bin/ssh", 0x7fff6b788b50)        = -1 ENOENT (No such file or directory)
stat("/usr/bin/ssh", {st_mode=S_IFREG|0755, st_size=301296, ...}) = 0
stat("//usr/bin/ssh", {st_mode=S_IFREG|0755, st_size=301296, ...}) = 0
munmap(0x2b3a4e14a000, 2123840)         = 0
munmap(0x2b3a4e351000, 2108632)         = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 5
fcntl(5, F_SETFD, FD_CLOEXEC)           = 0
brk(0x15c9000)                          = 0x15c9000
getdents(5, /* 128 entries */, 32768)   = 5248
getdents(5, /* 0 entries */, 32768)     = 0
brk(0x15c1000)                          = 0x15c1000
close(5)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_rml_oob.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_rml_oob.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=977, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4e14a000
read(5, "# mca_rml_oob.la - a libtool lib"..., 4096) = 977
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4e14a000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_rml_oob.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\23\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=27352, ...}) = 0
mmap(NULL, 2116736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4e14a000
mprotect(0x2b3a4e14f000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4e34e000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x4000) = 0x2b3a4e34e000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 5
fcntl(5, F_SETFD, FD_CLOEXEC)           = 0
getdents(5, /* 128 entries */, 32768)   = 5248
getdents(5, /* 0 entries */, 32768)     = 0
close(5)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_oob_tcp.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_oob_tcp.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=977, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4e34f000
read(5, "# mca_oob_tcp.la - a libtool lib"..., 4096) = 977
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4e34f000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_oob_tcp.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 %\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=68239, ...}) = 0
mmap(NULL, 2153368, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4e9b6000
mprotect(0x2b3a4e9c3000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4ebc2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0xc000) = 0x2b3a4ebc2000
close(5)                                = 0
socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 5
ioctl(5, SIOCGIFCONF, {120, {{"lo", {AF_INET, inet_addr("127.0.0.1")}}, {"eth1", {AF_INET, inet_addr("172.18.1.188")}}, {"ib0", {AF_INET, inet_addr("192.168.100.188")}}}}) = 0
ioctl(5, SIOCGIFCONF, {120, {{"lo", {AF_INET, inet_addr("127.0.0.1")}}, {"eth1", {AF_INET, inet_addr("172.18.1.188")}}, {"ib0", {AF_INET, inet_addr("192.168.100.188")}}}}) = 0
ioctl(5, SIOCGIFFLAGS, {ifr_name="lo", ifr_flags=IFF_UP|IFF_LOOPBACK|IFF_RUNNING}) = 0
ioctl(5, SIOCGIFINDEX, {ifr_name="lo", ifr_index=1}) = 0
ioctl(5, SIOCGIFADDR, {ifr_name="lo", ifr_addr={AF_INET, inet_addr("127.0.0.1")}}) = 0
ioctl(5, SIOCGIFNETMASK, {ifr_name="lo", ifr_netmask={AF_INET, inet_addr("255.0.0.0")}}) = 0
ioctl(5, SIOCGIFFLAGS, {ifr_name="eth1", ifr_flags=IFF_UP|IFF_BROADCAST|IFF_RUNNING|IFF_MULTICAST}) = 0
ioctl(5, SIOCGIFINDEX, {ifr_name="eth1", ifr_index=3}) = 0
ioctl(5, SIOCGIFADDR, {ifr_name="eth1", ifr_addr={AF_INET, inet_addr("172.18.1.188")}}) = 0
ioctl(5, SIOCGIFNETMASK, {ifr_name="eth1", ifr_netmask={AF_INET, inet_addr("255.240.0.0")}}) = 0
ioctl(5, SIOCGIFFLAGS, {ifr_name="ib0", ifr_flags=IFF_UP|IFF_BROADCAST|IFF_RUNNING|IFF_MULTICAST}) = 0
ioctl(5, SIOCGIFINDEX, {ifr_name="ib0", ifr_index=5}) = 0
ioctl(5, SIOCGIFADDR, {ifr_name="ib0", ifr_addr={AF_INET, inet_addr("192.168.100.188")}}) = 0
ioctl(5, SIOCGIFNETMASK, {ifr_name="ib0", ifr_netmask={AF_INET, inet_addr("255.255.255.0")}}) = 0
close(5)                                = 0
open("/proc/net/if_inet6", O_RDONLY)    = 5
fstat(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4ebc4000
read(5, "fe800000000000000202c902002a9cf5"..., 1024) = 162
read(5, "", 1024)                       = 0
read(5, "", 1024)                       = 0
close(5)                                = 0
munmap(0x2b3a4ebc4000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 5
fcntl(5, F_SETFD, FD_CLOEXEC)           = 0
getdents(5, /* 128 entries */, 32768)   = 5248
getdents(5, /* 0 entries */, 32768)     = 0
close(5)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_routed_binomial.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_routed_binomial.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=1025, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4ebc4000
read(5, "# mca_routed_binomial.la - a lib"..., 4096) = 1025
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4ebc4000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_routed_binomial.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\25\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=22323, ...}) = 0
mmap(NULL, 2112648, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4ebc4000
mprotect(0x2b3a4ebc8000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4edc7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x3000) = 0x2b3a4edc7000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_routed_direct.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_routed_direct.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=1013, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4edc8000
read(5, "# mca_routed_direct.la - a libto"..., 4096) = 1013
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4edc8000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_routed_direct.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\21\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=18260, ...}) = 0
mmap(NULL, 2109112, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4edc8000
mprotect(0x2b3a4edcb000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4efca000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x2000) = 0x2b3a4efca000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_routed_linear.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_routed_linear.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=1013, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4efcb000
read(5, "# mca_routed_linear.la - a libto"..., 4096) = 1013
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4efcb000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_routed_linear.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\22\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=20345, ...}) = 0
mmap(NULL, 2111208, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4efcb000
mprotect(0x2b3a4efce000, 2097152, PROT_NONE) = 0
mmap(0x2b3a4f1ce000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x3000) = 0x2b3a4f1ce000
close(5)                                = 0
munmap(0x2b3a4edc8000, 2109112)         = 0
munmap(0x2b3a4efcb000, 2111208)         = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 5
fcntl(5, F_SETFD, FD_CLOEXEC)           = 0
getdents(5, /* 128 entries */, 32768)   = 5248
getdents(5, /* 0 entries */, 32768)     = 0
close(5)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_grpcomm_bad.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_grpcomm_bad.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=1001, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4edc8000
read(5, "# mca_grpcomm_bad.la - a libtool"..., 4096) = 1001
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4edc8000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_grpcomm_bad.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\r\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=16129, ...}) = 0
mmap(NULL, 2107352, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4edc8000
mprotect(0x2b3a4edcb000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4efca000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x2000) = 0x2b3a4efca000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_grpcomm_basic.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_grpcomm_basic.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=1013, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4efcb000
read(5, "# mca_grpcomm_basic.la - a libto"..., 4096) = 1013
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4efcb000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_grpcomm_basic.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260\r\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=15950, ...}) = 0
mmap(NULL, 2107192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4efcb000
mprotect(0x2b3a4efce000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4f1cd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x2000) = 0x2b3a4f1cd000
close(5)                                = 0
munmap(0x2b3a4efcb000, 2107192)         = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 5
fcntl(5, F_SETFD, FD_CLOEXEC)           = 0
brk(0x15e5000)                          = 0x15e5000
getdents(5, /* 128 entries */, 32768)   = 5248
getdents(5, /* 0 entries */, 32768)     = 0
brk(0x15dd000)                          = 0x15dd000
close(5)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ras_slurm.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ras_slurm.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=989, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4efcb000
read(5, "# mca_ras_slurm.la - a libtool l"..., 4096) = 989
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4efcb000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ras_slurm.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\v\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=13042, ...}) = 0
mmap(NULL, 2104944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4efcb000
mprotect(0x2b3a4efcd000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4f1cc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x1000) = 0x2b3a4f1cc000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ras_tm.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ras_tm.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=1042, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4f1cd000
read(5, "# mca_ras_tm.la - a libtool libr"..., 4096) = 1042
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4f1cd000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_ras_tm.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\v\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=10831, ...}) = 0
mmap(NULL, 2102920, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4f1cd000
mprotect(0x2b3a4f1cf000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4f3ce000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x1000) = 0x2b3a4f3ce000
close(5)                                = 0
munmap(0x2b3a4efcb000, 2104944)         = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 5
fcntl(5, F_SETFD, FD_CLOEXEC)           = 0
getdents(5, /* 128 entries */, 32768)   = 5248
getdents(5, /* 0 entries */, 32768)     = 0
close(5)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_rmaps_rank_file.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_rmaps_rank_file.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=1025, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4efcb000
read(5, "# mca_rmaps_rank_file.la - a lib"..., 4096) = 1025
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4efcb000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_rmaps_rank_file.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\17\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=27184, ...}) = 0
mmap(NULL, 2116704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4f3cf000
mprotect(0x2b3a4f3d4000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4f5d3000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x4000) = 0x2b3a4f5d3000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_rmaps_round_robin.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_rmaps_round_robin.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=1037, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4f5d4000
read(5, "# mca_rmaps_round_robin.la - a l"..., 4096) = 1037
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4f5d4000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_rmaps_round_robin.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\10\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=11379, ...}) = 0
mmap(NULL, 2103728, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4f5d4000
mprotect(0x2b3a4f5d6000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4f7d5000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x1000) = 0x2b3a4f7d5000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_rmaps_seq.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_rmaps_seq.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=989, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4f7d6000
read(5, "# mca_rmaps_seq.la - a libtool l"..., 4096) = 989
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4f7d6000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_rmaps_seq.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\t\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=10928, ...}) = 0
mmap(NULL, 2103232, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4f7d6000
mprotect(0x2b3a4f7d8000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4f9d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x1000) = 0x2b3a4f9d7000
close(5)                                = 0
munmap(0x2b3a4f3cf000, 2116704)         = 0
munmap(0x2b3a4f7d6000, 2103232)         = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 5
fcntl(5, F_SETFD, FD_CLOEXEC)           = 0
getdents(5, /* 128 entries */, 32768)   = 5248
getdents(5, /* 0 entries */, 32768)     = 0
close(5)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_errmgr_default.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_errmgr_default.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=1019, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4f3cf000
read(5, "# mca_errmgr_default.la - a libt"..., 4096) = 1019
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4f3cf000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_errmgr_default.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\7\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=8811, ...}) = 0
mmap(NULL, 2101136, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4f3cf000
mprotect(0x2b3a4f3d0000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4f5cf000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0) = 0x2b3a4f5cf000
close(5)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 5
fcntl(5, F_SETFD, FD_CLOEXEC)           = 0
getdents(5, /* 128 entries */, 32768)   = 5248
getdents(5, /* 0 entries */, 32768)     = 0
close(5)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_odls_default.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_odls_default.la", O_RDONLY) = 5
fstat(5, {st_mode=S_IFREG|0755, st_size=1007, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4f5d0000
read(5, "# mca_odls_default.la - a libtoo"..., 4096) = 1007
read(5, "", 4096)                       = 0
close(5)                                = 0
munmap(0x2b3a4f5d0000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_odls_default.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260\17\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=13790, ...}) = 0
mmap(NULL, 2104712, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b3a4f7d6000
mprotect(0x2b3a4f7d8000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4f9d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x1000) = 0x2b3a4f9d7000
close(5)                                = 0
nanosleep({0, 0}, NULL)                 = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 5
setsockopt(5, SOL_TCP, TCP_NODELAY, [1], 4) = 0
setsockopt(5, SOL_SOCKET, SO_SNDBUF, [131072], 4) = 0
setsockopt(5, SOL_SOCKET, SO_RCVBUF, [131072], 4) = 0
setsockopt(5, SOL_SOCKET, SO_REUSEADDR, [0], 4) = 0
bind(5, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("0.0.0.0")}, 16) = 0
getsockname(5, {sa_family=AF_INET, sin_port=htons(52129), sin_addr=inet_addr("0.0.0.0")}, [16]) = 0
listen(5, 128)                          = 0
fcntl(5, F_GETFL)                       = 0x2 (flags O_RDWR)
fcntl(5, F_SETFL, O_RDWR|O_NONBLOCK)    = 0
socket(PF_INET6, SOCK_STREAM, IPPROTO_IP) = 6
setsockopt(6, SOL_TCP, TCP_NODELAY, [1], 4) = 0
setsockopt(6, SOL_SOCKET, SO_SNDBUF, [131072], 4) = 0
setsockopt(6, SOL_SOCKET, SO_RCVBUF, [131072], 4) = 0
setsockopt(6, SOL_IPV6, IPV6_V6ONLY, [0], 4) = 0
setsockopt(6, SOL_SOCKET, SO_REUSEADDR, [0], 4) = 0
bind(6, {sa_family=AF_INET6, sin6_port=htons(0), inet_pton(AF_INET6, "::", &sin6_addr), sin6_flowinfo=0, sin6_scope_id=0}, 28) = 0
getsockname(6, {sa_family=AF_INET6, sin6_port=htons(49199), inet_pton(AF_INET6, "::", &sin6_addr), sin6_flowinfo=0, sin6_scope_id=0}, [28]) = 0
listen(6, 128)                          = 0
fcntl(6, F_GETFL)                       = 0x2 (flags O_RDWR)
fcntl(6, F_SETFL, O_RDWR|O_NONBLOCK)    = 0
getuid()                                = 12012
socket(PF_FILE, SOCK_STREAM, 0)         = 7
fcntl(7, F_SETFL, O_RDWR|O_NONBLOCK)    = 0
connect(7, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(7)                                = 0
socket(PF_FILE, SOCK_STREAM, 0)         = 7
fcntl(7, F_SETFL, O_RDWR|O_NONBLOCK)    = 0
connect(7, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(7)                                = 0
open("/etc/nsswitch.conf", O_RDONLY)    = 7
fstat(7, {st_mode=S_IFREG|0644, st_size=1696, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4f9d8000
read(7, "#\n# /etc/nsswitch.conf\n#\n# An ex"..., 4096) = 1696
read(7, "", 4096)                       = 0
close(7)                                = 0
munmap(0x2b3a4f9d8000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/libnss_files.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vampirtrace-5.8.2/lib/libnss_files.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.4.3-gcc/lib/libnss_files.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/vtk-5.0.3/lib/libnss_files.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/home/sharon/local/lib/libnss_files.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/Intel-11.1/lib/intel64/libnss_files.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/papi-4.1.3/lib/libnss_files.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 7
fstat(7, {st_mode=S_IFREG|0644, st_size=103318, ...}) = 0
mmap(NULL, 103318, PROT_READ, MAP_PRIVATE, 7, 0) = 0x2b3a4f9d8000
close(7)                                = 0
open("/lib64/libnss_files.so.2", O_RDONLY) = 7
read(7, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\37\0\0\0\0\0\0"..., 832) = 832
fstat(7, {st_mode=S_IFREG|0755, st_size=53880, ...}) = 0
mmap(NULL, 2139432, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 7, 0) = 0x2b3a4f9f2000
mprotect(0x2b3a4f9fc000, 2093056, PROT_NONE) = 0
mmap(0x2b3a4fbfb000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 7, 0x9000) = 0x2b3a4fbfb000
close(7)                                = 0
mprotect(0x2b3a4fbfb000, 4096, PROT_READ) = 0
munmap(0x2b3a4f9d8000, 103318)          = 0
open("/etc/passwd", O_RDONLY)           = 7
fcntl(7, F_GETFD)                       = 0
fcntl(7, F_SETFD, FD_CLOEXEC)           = 0
fstat(7, {st_mode=S_IFREG|0644, st_size=15795, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4f9d8000
read(7, "root:x:0:0:root:/root:/bin/bash\n"..., 4096) = 4096
read(7, "akis:/home/pauld:/bin/tcsh\ndmeir"..., 4096) = 4096
close(7)                                = 0
munmap(0x2b3a4f9d8000, 4096)            = 0
stat("/tmp/openmpi-sessions-sharon at shc188_0/24014/0/0", 0x7fff6b789ca0) = -1 ENOENT (No such file or directory)
stat("/tmp/openmpi-sessions-sharon at shc188_0/24014/0/0", 0x7fff6b789c70) = -1 ENOENT (No such file or directory)
mkdir("/tmp/openmpi-sessions-sharon at shc188_0/24014/0/0", 0700) = -1 ENOENT (No such file or directory)
stat("/tmp", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=4096, ...}) = 0
stat("/tmp/openmpi-sessions-sharon at shc188_0", 0x7fff6b789c70) = -1 ENOENT (No such file or directory)
mkdir("/tmp/openmpi-sessions-sharon at shc188_0", 0700) = 0
stat("/tmp/openmpi-sessions-sharon at shc188_0/24014", 0x7fff6b789c70) = -1 ENOENT (No such file or directory)
mkdir("/tmp/openmpi-sessions-sharon at shc188_0/24014", 0700) = 0
stat("/tmp/openmpi-sessions-sharon at shc188_0/24014/0", 0x7fff6b789c70) = -1 ENOENT (No such file or directory)
mkdir("/tmp/openmpi-sessions-sharon at shc188_0/24014/0", 0700) = 0
stat("/tmp/openmpi-sessions-sharon at shc188_0/24014/0/0", 0x7fff6b789c70) = -1 ENOENT (No such file or directory)
mkdir("/tmp/openmpi-sessions-sharon at shc188_0/24014/0/0", 0700) = 0
open("/tmp/openmpi-sessions-sharon at shc188_0/24014/contact.txt", O_WRONLY|O_CREAT|O_TRUNC, 0666) = 7
fstat(7, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4f9d8000
write(7, "1573781504.0;tcp://172.18.1.188:"..., 71) = 71
close(7)                                = 0
munmap(0x2b3a4f9d8000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 7
fcntl(7, F_SETFD, FD_CLOEXEC)           = 0
getdents(7, /* 128 entries */, 32768)   = 5248
getdents(7, /* 0 entries */, 32768)     = 0
close(7)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_iof_hnp.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_iof_hnp.la", O_RDONLY) = 7
fstat(7, {st_mode=S_IFREG|0755, st_size=977, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4f9d8000
read(7, "# mca_iof_hnp.la - a libtool lib"..., 4096) = 977
read(7, "", 4096)                       = 0
close(7)                                = 0
munmap(0x2b3a4f9d8000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_iof_hnp.so", O_RDONLY) = 7
read(7, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\21\0\0\0\0\0\0"..., 832) = 832
fstat(7, {st_mode=S_IFREG|0755, st_size=24894, ...}) = 0
mmap(NULL, 2115144, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 7, 0) = 0x2b3a4fbfd000
mprotect(0x2b3a4fc01000, 2097152, PROT_NONE) = 0
mmap(0x2b3a4fe01000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 7, 0x4000) = 0x2b3a4fe01000
close(7)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_iof_orted.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_iof_orted.la", O_RDONLY) = 7
fstat(7, {st_mode=S_IFREG|0755, st_size=989, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4fe02000
read(7, "# mca_iof_orted.la - a libtool l"..., 4096) = 989
read(7, "", 4096)                       = 0
close(7)                                = 0
munmap(0x2b3a4fe02000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_iof_orted.so", O_RDONLY) = 7
read(7, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\16\0\0\0\0\0\0"..., 832) = 832
fstat(7, {st_mode=S_IFREG|0755, st_size=19906, ...}) = 0
mmap(NULL, 2110824, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 7, 0) = 0x2b3a4fe02000
mprotect(0x2b3a4fe05000, 2097152, PROT_NONE) = 0
mmap(0x2b3a50005000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 7, 0x3000) = 0x2b3a50005000
close(7)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_iof_tool.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_iof_tool.la", O_RDONLY) = 7
fstat(7, {st_mode=S_IFREG|0755, st_size=983, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a50006000
read(7, "# mca_iof_tool.la - a libtool li"..., 4096) = 983
read(7, "", 4096)                       = 0
close(7)                                = 0
munmap(0x2b3a50006000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_iof_tool.so", O_RDONLY) = 7
read(7, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\n\0\0\0\0\0\0"..., 832) = 832
fstat(7, {st_mode=S_IFREG|0755, st_size=12062, ...}) = 0
mmap(NULL, 2103944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 7, 0) = 0x2b3a50006000
mprotect(0x2b3a50008000, 2093056, PROT_NONE) = 0
mmap(0x2b3a50207000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 7, 0x1000) = 0x2b3a50207000
close(7)                                = 0
munmap(0x2b3a4fe02000, 2110824)         = 0
munmap(0x2b3a50006000, 2103944)         = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 7
fcntl(7, F_SETFD, FD_CLOEXEC)           = 0
getdents(7, /* 128 entries */, 32768)   = 5248
getdents(7, /* 0 entries */, 32768)     = 0
close(7)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_filem_rsh.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_filem_rsh.la", O_RDONLY) = 7
fstat(7, {st_mode=S_IFREG|0755, st_size=989, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a4fe02000
read(7, "# mca_filem_rsh.la - a libtool l"..., 4096) = 989
read(7, "", 4096)                       = 0
close(7)                                = 0
munmap(0x2b3a4fe02000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_filem_rsh.so", O_RDONLY) = 7
read(7, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260\25\0\0\0\0\0\0"..., 832) = 832
fstat(7, {st_mode=S_IFREG|0755, st_size=26984, ...}) = 0
mmap(NULL, 2116672, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 7, 0) = 0x2b3a4fe02000
mprotect(0x2b3a4fe07000, 2093056, PROT_NONE) = 0
mmap(0x2b3a50006000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 7, 0x4000) = 0x2b3a50006000
close(7)                                = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 7
fcntl(7, F_SETFD, FD_CLOEXEC)           = 0
getdents(7, /* 128 entries */, 32768)   = 5248
getdents(7, /* 0 entries */, 32768)     = 0
close(7)                                = 0
open("/home/sharon/.openmpi/components", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_notifier_syslog.ompi_info", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_notifier_syslog.la", O_RDONLY) = 7
fstat(7, {st_mode=S_IFREG|0755, st_size=1025, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a50007000
read(7, "# mca_notifier_syslog.la - a lib"..., 4096) = 1025
read(7, "", 4096)                       = 0
close(7)                                = 0
munmap(0x2b3a50007000, 4096)            = 0
open("/usr/local/openmpi-1.3.3-gcc/lib/openmpi/mca_notifier_syslog.so", O_RDONLY) = 7
read(7, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\6\0\0\0\0\0\0"..., 832) = 832
fstat(7, {st_mode=S_IFREG|0755, st_size=8361, ...}) = 0
mmap(NULL, 2100896, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 7, 0) = 0x2b3a50007000
mprotect(0x2b3a50008000, 2093056, PROT_NONE) = 0
mmap(0x2b3a50207000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 7, 0) = 0x2b3a50207000
close(7)                                = 0
munmap(0x2b3a50007000, 2100896)         = 0
pipe([7, 8])                            = 0
pipe([9, 10])                           = 0
rt_sigaction(SIGTERM, {0x2b3a4c8b5dc0, [TERM], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGINT, {0x2b3a4c8b5dc0, [INT], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGUSR1, {0x2b3a4c8b5dc0, [USR1], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGUSR2, {0x2b3a4c8b5dc0, [USR2], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, {SIG_DFL, [], 0}, 8) = 0
stat("/dev/urandom", {st_mode=S_IFCHR|0444, st_rdev=makedev(1, 9), ...}) = 0
open("/dev/urandom", O_RDONLY)          = 11
read(11, "\"\31\6^\233\177\372IZ\374\376\246\201\302\2\362", 16) = 16
close(11)                               = 0
open("/var/spool/torque/aux/444344.maitre", O_RDONLY) = 11
fstat(11, {st_mode=S_IFREG|0644, st_size=14, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a50007000
read(11, "shc188\nshc172\n", 4096)      = 14
read(11, "", 4096)                      = 0
socket(PF_FILE, SOCK_STREAM, 0)         = 12
fcntl(12, F_SETFL, O_RDWR|O_NONBLOCK)   = 0
connect(12, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(12)                               = 0
socket(PF_FILE, SOCK_STREAM, 0)         = 12
fcntl(12, F_SETFL, O_RDWR|O_NONBLOCK)   = 0
connect(12, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(12)                               = 0
open("/etc/resolv.conf", O_RDONLY)      = 12
fstat(12, {st_mode=S_IFREG|0644, st_size=115, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a50008000
read(12, "; generated by /sbin/dhclient-sc"..., 4096) = 115
read(12, "", 4096)                      = 0
close(12)                               = 0
munmap(0x2b3a50008000, 4096)            = 0
open("/etc/hosts", O_RDONLY)            = 12
fcntl(12, F_GETFD)                      = 0
fcntl(12, F_SETFD, FD_CLOEXEC)          = 0
fstat(12, {st_mode=S_IFREG|0644, st_size=20084, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a50008000
read(12, "# Do not remove the following li"..., 4096) = 4096
read(12, "0\n172.18.1.101\tshc101\n172.18.1.1"..., 4096) = 4096
read(12, "59\tshc059-mp\n192.168.8.60\tshc060"..., 4096) = 4096
read(12, ".168.8.228   shc228-mp\n192.168.8"..., 4096) = 4096
read(12, "92.168.100.137\tshc137-ib\n192.168"..., 4096) = 3700
read(12, "", 4096)                      = 0
close(12)                               = 0
munmap(0x2b3a50008000, 4096)            = 0
open("/etc/hosts", O_RDONLY)            = 12
fcntl(12, F_GETFD)                      = 0
fcntl(12, F_SETFD, FD_CLOEXEC)          = 0
fstat(12, {st_mode=S_IFREG|0644, st_size=20084, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a50008000
read(12, "# Do not remove the following li"..., 4096) = 4096
read(12, "0\n172.18.1.101\tshc101\n172.18.1.1"..., 4096) = 4096
close(12)                               = 0
munmap(0x2b3a50008000, 4096)            = 0
open("/etc/host.conf", O_RDONLY)        = 12
fstat(12, {st_mode=S_IFREG|0644, st_size=17, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a50008000
read(12, "order hosts,bind\n", 4096)    = 17
read(12, "", 4096)                      = 0
close(12)                               = 0
munmap(0x2b3a50008000, 4096)            = 0
futex(0x2b3a4d938d44, FUTEX_WAKE_PRIVATE, 2147483647) = 0
open("/etc/hosts", O_RDONLY)            = 12
fcntl(12, F_GETFD)                      = 0
fcntl(12, F_SETFD, FD_CLOEXEC)          = 0
fstat(12, {st_mode=S_IFREG|0644, st_size=20084, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a50008000
read(12, "# Do not remove the following li"..., 4096) = 4096
close(12)                               = 0
munmap(0x2b3a50008000, 4096)            = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 12
setsockopt(12, SOL_SOCKET, SO_LINGER, {onoff=1, linger=5}, 8) = 0
connect(12, {sa_family=AF_INET, sin_port=htons(15003), sin_addr=inet_addr("127.0.0.1")}, 16) = 0
mmap(NULL, 528384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a50008000
write(12, "+2+12+13444344.maitre2+32F8908E4"..., 66) = 66
poll([{fd=12, events=POLLIN|POLLHUP}], 1, 2147483647) = 1 ([{fd=12, revents=POLLIN}])
fcntl(12, F_GETFL)                      = 0x2 (flags O_RDWR)
read(12, "", 262144)                    = 0
close(12)                               = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 0 (Timeout)
sched_yield()                           = 0
open("/usr/local/openmpi-1.3.3-gcc/share/openmpi/help-plm-base.txt", O_RDONLY) = 12
ioctl(12, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fff6b789a80) = -1 ENOTTY (Inappropriate ioctl for device)
fstat(12, {st_mode=S_IFREG|0644, st_size=3595, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a50089000
read(12, "# -*- text -*-\n#\n# Copyright (c)"..., 8192) = 3595
read(12, "", 4096)                      = 0
close(12)                               = 0
munmap(0x2b3a50089000, 4096)            = 0
write(2, "--------------------------------"..., 586--------------------------------------------------------------------------
A daemon (pid unknown) died unexpectedly on signal 1  while attempting to
launch so we are aborting.

There may be more information reported by the environment (see above).

This may be because the daemon was unable to find all the needed shared
libraries on the remote node. You may set your LD_LIBRARY_PATH to have the
location of the shared libraries on the remote nodes and this will
automatically be forwarded to the remote nodes.
--------------------------------------------------------------------------
) = 586
write(8, "\1\0\0\0", 4)                 = 4
close(8)                                = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 5, 1000) = 1 ([{fd=7, revents=POLLIN|POLLHUP}])
open("/usr/local/openmpi-1.3.3-gcc/share/openmpi/help-orterun.txt", O_RDONLY) = 8
ioctl(8, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fff6b789860) = -1 ENOTTY (Inappropriate ioctl for device)
fstat(8, {st_mode=S_IFREG|0644, st_size=13590, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a50089000
read(8, "# -*- text -*-\n#\n# Copyright (c)"..., 8192) = 8192
close(8)                                = 0
munmap(0x2b3a50089000, 4096)            = 0
write(2, "--------------------------------"..., 250--------------------------------------------------------------------------
orterun noticed that the job aborted, but has no info as to the process
that caused that situation.
--------------------------------------------------------------------------
) = 250
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=9, events=POLLIN}], 4, 0) = 0 (Timeout)
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=9, events=POLLIN}], 4, 2) = 0 (Timeout)
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=9, events=POLLIN}], 4, 1) = 0 (Timeout)
write(10, "\1\0\0\0", 4)                = 4
close(10)                               = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=9, events=POLLIN}], 4, 1000) = 1 ([{fd=9, revents=POLLIN|POLLHUP}])
rt_sigaction(SIGTERM, {SIG_DFL, [TERM], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, {0x2b3a4c8b5dc0, [TERM], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, 8) = 0
rt_sigaction(SIGINT, {SIG_DFL, [INT], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, {0x2b3a4c8b5dc0, [INT], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, 8) = 0
rt_sigaction(SIGUSR1, {SIG_DFL, [USR1], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, {0x2b3a4c8b5dc0, [USR1], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, 8) = 0
rt_sigaction(SIGUSR2, {SIG_DFL, [USR2], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, {0x2b3a4c8b5dc0, [USR2], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, 8) = 0
open("/usr/local/openmpi-1.3.3-gcc/share/openmpi/help-orterun.txt", O_RDONLY) = 8
ioctl(8, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fff6b789750) = -1 ENOTTY (Inappropriate ioctl for device)
fstat(8, {st_mode=S_IFREG|0644, st_size=13590, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b3a50089000
read(8, "# -*- text -*-\n#\n# Copyright (c)"..., 8192) = 8192
read(8, "pe-read-failure]\n%s was unable t"..., 8192) = 5398
read(8, "", 4096)                       = 0
close(8)                                = 0
munmap(0x2b3a50089000, 4096)            = 0
write(2, "--------------------------------"..., 326--------------------------------------------------------------------------
orterun was unable to cleanly terminate the daemons on the nodes shown
below. Additional manual cleanup may be required - please refer to
the "orte-clean" tool for assistance.
--------------------------------------------------------------------------
) = 326
write(2, "\tshc172 - daemon did not report "..., 51	shc172 - daemon did not report back when launched
) = 51
stat("/tmp/openmpi-sessions-sharon at shc188_0/24014", {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
open("/tmp/openmpi-sessions-sharon at shc188_0/24014", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 8
fcntl(8, F_SETFD, FD_CLOEXEC)           = 0
getdents(8, /* 4 entries */, 32768)     = 104
stat("/tmp/openmpi-sessions-sharon at shc188_0/24014/0", {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
open("/tmp/openmpi-sessions-sharon at shc188_0/24014/0", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 10
fcntl(10, F_SETFD, FD_CLOEXEC)          = 0
getdents(10, /* 3 entries */, 32768)    = 72
stat("/tmp/openmpi-sessions-sharon at shc188_0/24014/0/0", {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
open("/tmp/openmpi-sessions-sharon at shc188_0/24014/0/0", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 12
fcntl(12, F_SETFD, FD_CLOEXEC)          = 0
brk(0x1604000)                          = 0x1604000
getdents(12, /* 2 entries */, 32768)    = 48
getdents(12, /* 0 entries */, 32768)    = 0
brk(0x15fc000)                          = 0x15fc000
close(12)                               = 0
open("/tmp/openmpi-sessions-sharon at shc188_0/24014/0/0", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 12
fcntl(12, F_SETFD, FD_CLOEXEC)          = 0
getdents(12, /* 2 entries */, 32768)    = 48
getdents(12, /* 0 entries */, 32768)    = 0
close(12)                               = 0
rmdir("/tmp/openmpi-sessions-sharon at shc188_0/24014/0/0") = 0
getdents(10, /* 0 entries */, 32768)    = 0
brk(0x15f4000)                          = 0x15f4000
close(10)                               = 0
open("/tmp/openmpi-sessions-sharon at shc188_0/24014/0", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 10
fcntl(10, F_SETFD, FD_CLOEXEC)          = 0
getdents(10, /* 2 entries */, 32768)    = 48
getdents(10, /* 0 entries */, 32768)    = 0
close(10)                               = 0
rmdir("/tmp/openmpi-sessions-sharon at shc188_0/24014/0") = 0
unlink("/tmp/openmpi-sessions-sharon at shc188_0/24014/contact.txt") = 0
getdents(8, /* 0 entries */, 32768)     = 0
brk(0x15eb000)                          = 0x15eb000
close(8)                                = 0
open("/tmp/openmpi-sessions-sharon at shc188_0/24014", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 8
fcntl(8, F_SETFD, FD_CLOEXEC)           = 0
getdents(8, /* 2 entries */, 32768)     = 48
getdents(8, /* 0 entries */, 32768)     = 0
close(8)                                = 0
rmdir("/tmp/openmpi-sessions-sharon at shc188_0/24014") = 0
stat("/tmp/openmpi-sessions-sharon at shc188_0", {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
open("/tmp/openmpi-sessions-sharon at shc188_0", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 8
fcntl(8, F_SETFD, FD_CLOEXEC)           = 0
getdents(8, /* 2 entries */, 32768)     = 48
getdents(8, /* 0 entries */, 32768)     = 0
close(8)                                = 0
open("/tmp/openmpi-sessions-sharon at shc188_0", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 8
fcntl(8, F_SETFD, FD_CLOEXEC)           = 0
getdents(8, /* 2 entries */, 32768)     = 48
getdents(8, /* 0 entries */, 32768)     = 0
close(8)                                = 0
rmdir("/tmp/openmpi-sessions-sharon at shc188_0") = 0
open("/tmp/openmpi-sessions-sharon at shc188_0/24014", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
unlink("/openmpi-sessions-sharon at shc188_0/contact.txt") = -1 ENOENT (No such file or directory)
munmap(0x2b3a4fe02000, 2116672)         = 0
munmap(0x2b3a4f7d6000, 2104712)         = 0
rt_sigaction(SIGCHLD, {SIG_DFL, [CHLD], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, {0x2b3a4c8b5dc0, [CHLD], SA_RESTORER|SA_RESTART, 0x2b3a4d6132d0}, 8) = 0
munmap(0x2b3a4fbfd000, 2115144)         = 0
munmap(0x2b3a4f1cd000, 2102920)         = 0
munmap(0x2b3a4f5d4000, 2103728)         = 0
munmap(0x2b3a4e554000, 2108056)         = 0
munmap(0x2b3a4e757000, 2485824)         = 0
munmap(0x2b3a4f3cf000, 2101136)         = 0
munmap(0x2b3a4edc8000, 2107352)         = 0
munmap(0x2b3a4ebc4000, 2112648)         = 0
close(5)                                = 0
close(6)                                = 0
munmap(0x2b3a4e9b6000, 2153368)         = 0
munmap(0x2b3a4e14a000, 2116736)         = 0
stat("/openmpi-sessions-sharon at shc188_0/1573781504/0", 0x7fff6b789850) = -1 ENOENT (No such file or directory)
open("/openmpi-sessions-sharon at shc188_0/1573781504/0", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
stat("/openmpi-sessions-sharon at shc188_0/1573781504", 0x7fff6b789850) = -1 ENOENT (No such file or directory)
open("/openmpi-sessions-sharon at shc188_0/1573781504", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
stat("openmpi-sessions-sharon at shc188_0", 0x7fff6b789850) = -1 ENOENT (No such file or directory)
open("openmpi-sessions-sharon at shc188_0", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
stat("/tmp/openmpi-sessions-sharon at shc188_0", 0x7fff6b789850) = -1 ENOENT (No such file or directory)
open("/tmp/openmpi-sessions-sharon at shc188_0", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
open("/openmpi-sessions-sharon at shc188_0/1573781504/0", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
munmap(0x2b3a4df46000, 2110944)         = 0
munmap(0x2b3a4db42000, 2101136)         = 0
munmap(0x2b3a4d93d000, 2115040)         = 0
exit_group(1)                           = ?


More information about the torqueusers mailing list