Link: API Status Page for Caltech-Test
868922555 LDAS-TEST22936520 mpi::filterLogEntries 22936520.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) }
868922555 LDAS-TEST22936520 mpi::filterLogEntries 22936520.1:progress 100.00%
868922555 LDAS-TEST22936520 mpi::filterLogEntries 22936520.1:projected_ratio 0.00099
868922555 LDAS-TEST22936520 mpi::filterLogEntries 22936520.2:finalizing { totalTime=1.30328 sentObjects=3; Rank0 (inPut: create=0.06249 send=0.00718 ilwd=0.05544; outPut: create=0.01435 send=0.05354 ilwd=0.04320; metadata: (un)pack=0.00019 bcast=0.01305; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01198; max memory=3932160) Rank1 (inPut: create=0.06412 send=0.01038; outPut: create=0.01217 send=0.04261; metadata: (un)pack=0.00016 bcast=0.01398; IS=0.00013; CD=0.02802; AS=0.05947; FO=0.00020; FS=0.00004; LB=0.03424; max memory=5165056) }
868922567 LDAS-TEST22936519 mpi::filterLogEntries 22936519.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) Rank2(metaserver) Rank3(metaserver) }
868922585 LDAS-TEST22936521 mpi::filterLogEntries 22936521.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) }
868922592 LDAS-TEST22936521 mpi::filterLogEntries 22936521.1:progress 100.00%
868922592 LDAS-TEST22936521 mpi::filterLogEntries 22936521.1:projected_ratio 0.00018
868922592 LDAS-TEST22936521 jobstate response: '22936521.1:cont'
868922592 LDAS-TEST22936521 mpi::filterLogEntries 22936521.2:finalizing { totalTime=7.14017 sentObjects=3; Rank0 (inPut: create=0.67621 send=2.45490 ilwd=0.34195; outPut: create=0.06595 send=0.47698 ilwd=0.04341; metadata: (un)pack=0.00012 bcast=0.15848; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.11597; max memory=0) Rank1 (inPut: create=0.82976 send=2.58438; outPut: create=0.01200 send=0.50332; metadata: (un)pack=0.00011 bcast=0.21199; IS=0.00002; CD=0.00000; AS=0.06447; FO=0.00313; FS=0.00000; LB=0.22150; max memory=184320) }
868922632 LDAS-TEST22936519 mpi::filterLogEntries 22936519.1:progress 100.00%
868922632 LDAS-TEST22936519 mpi::filterLogEntries 22936519.1:projected_ratio 0.01865
868922632 LDAS-TEST22936519 jobstate response: '22936519.1:cont'
868922632 LDAS-TEST22936519 mpi::filterLogEntries 22936519.2:progress 100.00%
868922632 LDAS-TEST22936519 mpi::filterLogEntries 22936519.2:projected_ratio 0.03706
868922633 LDAS-TEST22936519 mpi::filterLogEntries 22936519.3:finalizing { totalTime=65.5837 sentObjects=4; Rank0 (inPut: create=0.76921 send=0.94712 ilwd=0.54093; outPut: create=0.02433 send=0.07132 ilwd=0.08615; metadata: (un)pack=0.00031 bcast=0.02649; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.11633; max memory=335872) Rank3 (inPut: create=0.93240 send=0.77273; outPut: create=0.00000 send=0.01191; metadata: (un)pack=0.00011 bcast=0.00449; IS=0.41994; CD=0.23366; AS=1.43757; FO=0.00000; FS=0.00222; LB=0.00000; max memory=729088) Rank1 (inPut: create=0.92149 send=1.05399; outPut: create=0.01180 send=0.03581; metadata: (un)pack=0.00017 bcast=0.01710; IS=0.00013; CD=0.00006; AS=61.24166; FO=0.00004; FS=0.00007; LB=0.02392; max memory=413696) Rank2 (inPut: create=0.69961 send=0.74827; outPut: create=0.02397 send=0.25540; metadata: (un)pack=0.00245 bcast=0.02707; IS=0.43341; CD=0.24180; AS=61.08728; FO=0.00008; FS=0.11775; LB=0.00001; max memory=729088) }
868922657 LDAS-TEST22936522 mpi::filterLogEntries 22936522.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) }
868922657 LDAS-TEST22936522 mpi::filterLogEntries 22936522.1:progress 100.00%
868922657 LDAS-TEST22936522 mpi::filterLogEntries 22936522.1:projected_ratio 0.00000
868922657 LDAS-TEST22936522 mpi::filterLogEntries 22936522.2:finalizing { totalTime=0.342316 sentObjects=2; Rank0 (inPut: create=0.03872 send=0.00069 ilwd=0.03369; outPut: create=0.00000 send=0.01171 ilwd=0.04201; metadata: (un)pack=0.00010 bcast=0.00114; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01195; max memory=0) Rank1 (inPut: create=0.03826 send=0.00143; outPut: create=0.00000 send=0.05061; metadata: (un)pack=0.00016 bcast=0.00196; IS=0.00012; CD=0.00058; AS=0.00013; FO=0.00000; FS=0.00004; LB=0.02410; max memory=651264) }
868922660 LDAS-TEST22936523 mpi::filterLogEntries 22936523.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) }
868922674 LDAS-TEST22936524 mpi::filterLogEntries 22936524.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) }
868922675 LDAS-TEST22936523 mpi::filterLogEntries 22936523.1:progress 68.37%
868922675 LDAS-TEST22936523 mpi::filterLogEntries 22936523.1:projected_ratio 0.03745
868922675 LDAS-TEST22936523 jobstate response: '22936523.1:cont'
868922675 LDAS-TEST22936524 mpi::filterLogEntries 22936524.1:progress 100.00%
868922675 LDAS-TEST22936524 mpi::filterLogEntries 22936524.1:projected_ratio 0.00001
868922675 LDAS-TEST22936524 mpi::filterLogEntries 22936524.2:finalizing { totalTime=1.51605 sentObjects=3; Rank0 (inPut: create=0.12846 send=0.00290 ilwd=0.04098; outPut: create=0.03187 send=0.02474 ilwd=0.04681; metadata: (un)pack=0.00030 bcast=0.11353; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01195; max memory=0) Rank1 (inPut: create=0.12881 send=0.00457; outPut: create=0.01599 send=0.07183; metadata: (un)pack=0.00029 bcast=0.02467; IS=0.00012; CD=0.00301; AS=0.00058; FO=0.00007; FS=0.00004; LB=0.02748; max memory=2060288) }
868922676 LDAS-TEST22936523 mpi::filterLogEntries 22936523.2:progress 100.00%
868922676 LDAS-TEST22936523 mpi::filterLogEntries 22936523.2:projected_ratio 0.03909
868922676 LDAS-TEST22936523 mpi::filterLogEntries 22936523.3:finalizing { totalTime=15.6587 sentObjects=4; Rank0 (inPut: create=0.43303 send=0.16924 ilwd=0.34352; outPut: create=0.10646 send=0.09158 ilwd=0.32196; metadata: (un)pack=0.01197 bcast=0.07634; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.12429; max memory=5664768) Rank1 (inPut: create=0.48656 send=0.14295; outPut: create=0.07099 send=0.10739; metadata: (un)pack=0.01062 bcast=0.04749; IS=0.00010; CD=0.06320; AS=14.07371; FO=0.00210; FS=0.00367; LB=0.04674; max memory=352256) }
868922707 LDAS-TEST22936526 mpi::filterLogEntries 22936526.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) Rank2(metaserver) Rank3(metaserver) }
868922712 LDAS-TEST22936526 mpi::filterLogEntries 22936526.1:progress 0.00%
868922712 LDAS-TEST22936526 mpi::filterLogEntries 22936526.1:projected_ratio 0.00000
868922712 LDAS-TEST22936526 jobstate response: '22936526.1:cont'
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.2:progress 0.00%
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.2:projected_ratio 0.00000
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.3:progress 25.00%
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.3:projected_ratio 0.00026
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.4:progress 50.00%
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.4:projected_ratio 0.00083
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.5:progress 50.00%
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.5:projected_ratio 0.00142
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.6:progress 75.00%
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.6:projected_ratio 0.00146
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.7:progress 75.00%
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.7:projected_ratio 0.00178
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.8:progress 75.00%
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.8:projected_ratio 0.00178
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.9:progress 75.00%
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.9:projected_ratio 0.00178
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.10:progress 75.00%
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.10:projected_ratio 0.00194
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.11:progress 100.00%
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.11:projected_ratio 0.00205
868922713 LDAS-TEST22936526 mpi::filterLogEntries 22936526.12:finalizing { totalTime=5.40555 sentObjects=3; Rank0 (inPut: create=0.43502 send=0.75631 ilwd=0.34174; outPut: create=0.03757 send=0.51859 ilwd=0.04090; metadata: (un)pack=0.00353 bcast=0.03115; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.11682; max memory=1798144) Rank3 (inPut: create=0.52284 send=0.31912; outPut: create=0.00000 send=0.26278; metadata: (un)pack=0.00006 bcast=0.00452; IS=0.00011; CD=1.56953; AS=0.54077; FO=0.00000; FS=0.00474; LB=0.00001; max memory=1110016) Rank2 (inPut: create=0.59804 send=0.59748; outPut: create=0.00000 send=0.13859; metadata: (un)pack=0.00006 bcast=0.00545; IS=0.00012; CD=0.81277; AS=0.70518; FO=0.00000; FS=0.00456; LB=0.00001; max memory=1064960) Rank1 (inPut: create=0.54337 send=0.39428; outPut: create=0.03355 send=0.78271; metadata: (un)pack=0.00297 bcast=0.02987; IS=0.00012; CD=1.38531; AS=0.12220; FO=0.00062; FS=0.00354; LB=0.04753; max memory=344064) }
868922744 LDAS-TEST22936527 mpi::filterLogEntries 22936527.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) Rank2(metaserver) }
868922760 LDAS-TEST22936527 mpi::filterLogEntries 22936527.1:error {Rank 2 failed with error message: ( applySearch function ): Level 1: $Id Status code -1: Recursive error function LALApplySearch, file ../../../../contrib/waveburst/src/Search.c, line 1075 Level 2: $Id Status code -1: Recursive error function LALSlaveWaveburstSearch, file ../../../../contrib/waveburst/src/Search.c, line 210 Level 3: $Id Status code 101: cannot open /dso-test/waveburst/usertag/timeinterval/etg function LALOutputWaveburst, file ../../../../contrib/waveburst/src/Output.c, line 1037 }
868922760 LDAS-TEST22936527 mpi::newJobCallback {Rank 2 failed with error message: ( applySearch function ): Level 1: $Id Status code -1: Recursive error function LALApplySearch, file ../../../../contrib/waveburst/src/Search.c, line 1075 Level 2: $Id Status code -1: Recursive error function LALSlaveWaveburstSearch, file ../../../../contrib/waveburst/src/Search.c, line 210 Level 3: $Id Status code 101: cannot open /dso-test/waveburst/usertag/timeinterval/etg function LALOutputWaveburst, file ../../../../contrib/waveburst/src/Output.c, line 1037 }
868922770 LDAS-TEST22936525 mpi::filterLogEntries 22936525.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) }
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.1:progress 0.00%
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.1:projected_ratio 0.00015
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.2:progress 10.00%
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.2:projected_ratio 0.00030
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.3:progress 20.00%
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.3:projected_ratio 0.00045
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.4:progress 30.00%
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.4:projected_ratio 0.00060
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.5:progress 40.00%
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.5:projected_ratio 0.00076
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.6:progress 50.00%
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.6:projected_ratio 0.00090
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.7:progress 60.00%
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.7:projected_ratio 0.00106
868922773 LDAS-TEST22936525 jobstate response: '22936525.7:cont'
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.8:progress 70.00%
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.8:projected_ratio 0.00121
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.9:progress 80.00%
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.9:projected_ratio 0.00136
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.10:progress 90.00%
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.10:projected_ratio 0.00151
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.11:progress 100.00%
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.11:projected_ratio 0.00151
868922773 LDAS-TEST22936525 mpi::filterLogEntries 22936525.12:finalizing { totalTime=2.99515 sentObjects=3; Rank0 (inPut: create=0.28161 send=0.10305 ilwd=0.22313; outPut: create=0.12458 send=0.45597 ilwd=0.04375; metadata: (un)pack=0.00043 bcast=0.12375; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.22082; max memory=483328) Rank1 (inPut: create=0.31431 send=0.08972; outPut: create=0.12481 send=0.53249; metadata: (un)pack=0.00036 bcast=0.12552; IS=0.01323; CD=0.10606; AS=1.32348; FO=0.00037; FS=0.00260; LB=0.02784; max memory=18403328) }
869268601 LDAS-TEST22992606 mpi::filterLogEntries 22992606.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) Rank2(metaserver) }
869268604 LDAS-TEST22992606 mpi::filterLogEntries 22992606.1:progress 100.00%
869268604 LDAS-TEST22992606 mpi::filterLogEntries 22992606.1:projected_ratio 0.00285
869268604 LDAS-TEST22992606 jobstate response: '22992606.1:cont'
869268604 LDAS-TEST22992606 mpi::filterLogEntries 22992606.2:progress 100.00%
869268604 LDAS-TEST22992606 mpi::filterLogEntries 22992606.2:projected_ratio 0.00443
869268604 LDAS-TEST22992606 mpi::filterLogEntries 22992606.3:finalizing { totalTime=3.68424 sentObjects=3; Rank0 (inPut: create=0.16861 send=0.08945 ilwd=0.12996; outPut: create=0.03211 send=0.07940 ilwd=0.00533; metadata: (un)pack=0.00021 bcast=0.03415; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.12432; max memory=1048576) Rank1 (inPut: create=0.17769 send=0.10310; outPut: create=0.01595 send=0.06494; metadata: (un)pack=0.00014 bcast=0.01889; IS=0.00015; CD=0.00005; AS=1.62302; FO=0.00003; FS=0.00006; LB=0.03193; max memory=3395584) Rank2 (inPut: create=0.18363 send=0.09401; outPut: create=0.03156 send=0.29740; metadata: (un)pack=0.00017 bcast=0.03330; IS=0.15704; CD=0.00010; AS=1.45856; FO=0.00007; FS=0.00142; LB=0.00002; max memory=22573056) }
869268616 LDAS-TEST22992607 mpi::filterLogEntries 22992607.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) Rank2(metaserver) }
869268618 LDAS-TEST22992607 mpi::filterLogEntries 22992607.1:progress 100.00%
869268618 LDAS-TEST22992607 mpi::filterLogEntries 22992607.1:projected_ratio 0.00300
869268618 LDAS-TEST22992607 jobstate response: '22992607.1:cont'
869268619 LDAS-TEST22992607 mpi::filterLogEntries 22992607.2:progress 100.00%
869268619 LDAS-TEST22992607 mpi::filterLogEntries 22992607.2:projected_ratio 0.00467
869268619 LDAS-TEST22992607 mpi::filterLogEntries 22992607.3:finalizing { totalTime=2.66889 sentObjects=3; Rank0 (inPut: create=0.12955 send=0.08737 ilwd=0.10352; outPut: create=0.03212 send=0.07936 ilwd=0.04613; metadata: (un)pack=0.00020 bcast=0.03414; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.12432; max memory=1048576) Rank1 (inPut: create=0.15555 send=0.09650; outPut: create=0.01637 send=0.06292; metadata: (un)pack=0.00015 bcast=0.01826; IS=0.00014; CD=0.00005; AS=1.70164; FO=0.00003; FS=0.00006; LB=0.03189; max memory=3395584) Rank2 (inPut: create=0.14215 send=0.06474; outPut: create=0.03159 send=0.29185; metadata: (un)pack=0.00016 bcast=0.03434; IS=0.11970; CD=0.00010; AS=1.53944; FO=0.00007; FS=0.00141; LB=0.00001; max memory=22573056) }
869268784 LDAS-TEST22992613 mpi::filterLogEntries 22992613.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) Rank2(metaserver) }
869268786 LDAS-TEST22992613 mpi::filterLogEntries 22992613.1:progress 100.00%
869268786 LDAS-TEST22992613 mpi::filterLogEntries 22992613.1:projected_ratio 0.00286
869268786 LDAS-TEST22992613 mpi::filterLogEntries 22992613.2:progress 100.00%
869268786 LDAS-TEST22992613 mpi::filterLogEntries 22992613.2:projected_ratio 0.00433
869268786 LDAS-TEST22992613 jobstate response: '22992613.2:cont'
869268787 LDAS-TEST22992613 mpi::filterLogEntries 22992613.3:finalizing { totalTime=2.51297 sentObjects=3; Rank0 (inPut: create=0.13426 send=0.08620 ilwd=0.10773; outPut: create=0.02417 send=0.07160 ilwd=0.04061; metadata: (un)pack=0.00020 bcast=0.02576; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.11608; max memory=1048576) Rank1 (inPut: create=0.14694 send=0.06466; outPut: create=0.01193 send=0.04793; metadata: (un)pack=0.00015 bcast=0.01454; IS=0.00015; CD=0.00005; AS=1.51146; FO=0.00003; FS=0.00007; LB=0.13577; max memory=3395584) Rank2 (inPut: create=0.16028 send=0.09704; outPut: create=0.02395 send=0.26018; metadata: (un)pack=0.00015 bcast=0.02734; IS=0.12012; CD=0.00010; AS=1.46300; FO=0.00007; FS=0.00142; LB=0.00001; max memory=22573056) }
869268809 LDAS-TEST22992614 mpi::filterLogEntries 22992614.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) Rank2(metaserver) }
869268811 LDAS-TEST22992614 mpi::filterLogEntries 22992614.1:progress 100.00%
869268811 LDAS-TEST22992614 mpi::filterLogEntries 22992614.1:projected_ratio 0.00332
869268812 LDAS-TEST22992614 jobstate response: '22992614.1:cont'
869268813 LDAS-TEST22992614 mpi::filterLogEntries 22992614.2:progress 100.00%
869268813 LDAS-TEST22992614 mpi::filterLogEntries 22992614.2:projected_ratio 0.00679
869268814 LDAS-TEST22992614 mpi::filterLogEntries 22992614.3:finalizing { totalTime=4.44851 sentObjects=4; Rank0 (inPut: create=0.13206 send=0.08425 ilwd=0.10551; outPut: create=0.03619 send=1.61133 ilwd=0.08327; metadata: (un)pack=0.00021 bcast=0.03771; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.23228; max memory=1048576) Rank1 (inPut: create=0.15817 send=0.10587; outPut: create=0.01184 send=0.05391; metadata: (un)pack=0.00014 bcast=0.01537; IS=0.00013; CD=0.00005; AS=3.55868; FO=0.00004; FS=0.00007; LB=0.02397; max memory=3395584) Rank2 (inPut: create=0.15276 send=0.09023; outPut: create=0.03002 send=1.92754; metadata: (un)pack=0.00017 bcast=0.03239; IS=0.12039; CD=0.00011; AS=1.73043; FO=0.00008; FS=0.00143; LB=0.00001; max memory=22573056) }
869268834 LDAS-TEST22992615 mpi::filterLogEntries 22992615.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) Rank2(metaserver) }
869268836 LDAS-TEST22992615 mpi::filterLogEntries 22992615.1:progress 100.00%
869268836 LDAS-TEST22992615 mpi::filterLogEntries 22992615.1:projected_ratio 0.00285
869268836 LDAS-TEST22992615 mpi::filterLogEntries 22992615.2:progress 100.00%
869268836 LDAS-TEST22992615 mpi::filterLogEntries 22992615.2:projected_ratio 0.00433
869268836 LDAS-TEST22992615 jobstate response: '22992615.2:cont'
869268836 LDAS-TEST22992615 mpi::filterLogEntries 22992615.3:finalizing { totalTime=2.51999 sentObjects=3; Rank0 (inPut: create=0.14453 send=0.08820 ilwd=0.10433; outPut: create=0.02411 send=0.05965 ilwd=0.04408; metadata: (un)pack=0.00019 bcast=0.02569; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.11609; max memory=1048576) Rank1 (inPut: create=0.12161 send=0.09640; outPut: create=0.01191 send=0.04743; metadata: (un)pack=0.00015 bcast=0.01443; IS=0.00013; CD=0.00005; AS=1.50922; FO=0.00003; FS=0.00007; LB=0.13195; max memory=3395584) Rank2 (inPut: create=0.15699 send=0.08596; outPut: create=0.02398 send=0.25560; metadata: (un)pack=0.00016 bcast=0.02740; IS=0.12022; CD=0.00010; AS=1.46096; FO=0.00007; FS=0.00141; LB=0.00001; max memory=22573056) }
869268859 LDAS-TEST22992616 mpi::filterLogEntries 22992616.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) Rank2(metaserver) }
869268861 LDAS-TEST22992616 mpi::filterLogEntries 22992616.1:progress 100.00%
869268861 LDAS-TEST22992616 mpi::filterLogEntries 22992616.1:projected_ratio 0.00282
869268861 LDAS-TEST22992616 mpi::filterLogEntries 22992616.2:progress 100.00%
869268861 LDAS-TEST22992616 mpi::filterLogEntries 22992616.2:projected_ratio 0.00428
869268861 LDAS-TEST22992616 jobstate response: '22992616.2:cont'
869268862 LDAS-TEST22992616 mpi::filterLogEntries 22992616.3:finalizing { totalTime=2.47271 sentObjects=3; Rank0 (inPut: create=0.13125 send=0.08734 ilwd=0.10523; outPut: create=0.02412 send=0.05969 ilwd=0.04048; metadata: (un)pack=0.00019 bcast=0.02619; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.11607; max memory=1048576) Rank1 (inPut: create=0.15721 send=0.09680; outPut: create=0.01189 send=0.04957; metadata: (un)pack=0.00016 bcast=0.01378; IS=0.00014; CD=0.00005; AS=1.49357; FO=0.00003; FS=0.00006; LB=0.12041; max memory=3395584) Rank2 (inPut: create=0.14389 send=0.06461; outPut: create=0.02385 send=0.24568; metadata: (un)pack=0.00016 bcast=0.02662; IS=0.12047; CD=0.00010; AS=1.44533; FO=0.00007; FS=0.00142; LB=0.00001; max memory=22573056) }
869268905 LDAS-TEST22992617 mpi::filterLogEntries 22992617.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) }
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.1:progress 0.00%
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.1:projected_ratio 0.00015
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.2:progress 10.00%
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.2:projected_ratio 0.00030
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.3:progress 20.00%
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.3:projected_ratio 0.00045
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.4:progress 30.00%
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.4:projected_ratio 0.00060
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.5:progress 40.00%
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.5:projected_ratio 0.00075
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.6:progress 50.00%
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.6:projected_ratio 0.00090
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.7:progress 60.00%
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.7:projected_ratio 0.00105
869268907 LDAS-TEST22992617 jobstate response: '22992617.7:cont'
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.8:progress 70.00%
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.8:projected_ratio 0.00122
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.9:progress 80.00%
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.9:projected_ratio 0.00137
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.10:progress 90.00%
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.10:projected_ratio 0.00152
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.11:progress 100.00%
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.11:projected_ratio 0.00152
869268907 LDAS-TEST22992617 mpi::filterLogEntries 22992617.12:finalizing { totalTime=2.89835 sentObjects=3; Rank0 (inPut: create=0.28847 send=0.10096 ilwd=0.23109; outPut: create=0.12042 send=0.50375 ilwd=0.04390; metadata: (un)pack=0.00043 bcast=0.11964; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.22249; max memory=483328) Rank1 (inPut: create=0.32116 send=0.08768; outPut: create=0.12110 send=0.49172; metadata: (un)pack=0.00040 bcast=0.12180; IS=0.01324; CD=0.10659; AS=1.33224; FO=0.00035; FS=0.00262; LB=0.02400; max memory=18403328) }
869268937 LDAS-TEST22992618 mpi::filterLogEntries 22992618.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) }
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.1:progress 0.00%
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.1:projected_ratio 0.00015
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.2:progress 10.00%
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.2:projected_ratio 0.00030
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.3:progress 20.00%
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.3:projected_ratio 0.00045
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.4:progress 30.00%
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.4:projected_ratio 0.00060
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.5:progress 40.00%
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.5:projected_ratio 0.00075
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.6:progress 50.00%
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.6:projected_ratio 0.00090
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.7:progress 60.00%
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.7:projected_ratio 0.00105
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.8:progress 70.00%
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.8:projected_ratio 0.00120
869268940 LDAS-TEST22992618 jobstate response: '22992618.8:cont'
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.9:progress 80.00%
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.9:projected_ratio 0.00136
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.10:progress 90.00%
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.10:projected_ratio 0.00151
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.11:progress 100.00%
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.11:projected_ratio 0.00151
869268940 LDAS-TEST22992618 mpi::filterLogEntries 22992618.12:finalizing { totalTime=2.7903 sentObjects=3; Rank0 (inPut: create=0.28322 send=0.10124 ilwd=0.22576; outPut: create=0.12041 send=0.45960 ilwd=0.04314; metadata: (un)pack=0.00042 bcast=0.11958; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.22088; max memory=483328) Rank1 (inPut: create=0.31639 send=0.12543; outPut: create=0.12009 send=0.49985; metadata: (un)pack=0.00036 bcast=0.12084; IS=0.01326; CD=0.10618; AS=1.32237; FO=0.00034; FS=0.00262; LB=0.02397; max memory=18403328) }
869268973 LDAS-TEST22992619 mpi::filterLogEntries 22992619.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) }
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.1:progress 0.00%
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.1:projected_ratio 0.00015
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.2:progress 10.00%
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.2:projected_ratio 0.00030
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.3:progress 20.00%
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.3:projected_ratio 0.00045
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.4:progress 30.00%
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.4:projected_ratio 0.00060
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.5:progress 40.00%
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.5:projected_ratio 0.00075
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.6:progress 50.00%
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.6:projected_ratio 0.00090
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.7:progress 60.00%
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.7:projected_ratio 0.00105
869268975 LDAS-TEST22992619 jobstate response: '22992619.7:cont'
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.8:progress 70.00%
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.8:projected_ratio 0.00122
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.9:progress 80.00%
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.9:projected_ratio 0.00136
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.10:progress 90.00%
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.10:projected_ratio 0.00151
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.11:progress 100.00%
869268975 LDAS-TEST22992619 mpi::filterLogEntries 22992619.11:projected_ratio 0.00151
869268976 LDAS-TEST22992619 mpi::filterLogEntries 22992619.12:finalizing { totalTime=2.90439 sentObjects=3; Rank0 (inPut: create=0.28115 send=0.10096 ilwd=0.22382; outPut: create=0.13240 send=0.45563 ilwd=0.04243; metadata: (un)pack=0.00041 bcast=0.13165; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.22082; max memory=483328) Rank1 (inPut: create=0.31391 send=0.08840; outPut: create=0.12794 send=0.49612; metadata: (un)pack=0.00036 bcast=0.12867; IS=0.01335; CD=0.10638; AS=1.32552; FO=0.00036; FS=0.00262; LB=0.02398; max memory=18403328) }
869269008 LDAS-TEST22992620 mpi::filterLogEntries 22992620.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) }
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.1:progress 0.00%
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.1:projected_ratio 0.00015
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.2:progress 10.00%
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.2:projected_ratio 0.00031
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.3:progress 20.00%
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.3:projected_ratio 0.00046
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.4:progress 30.00%
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.4:projected_ratio 0.00061
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.5:progress 40.00%
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.5:projected_ratio 0.00076
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.6:progress 50.00%
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.6:projected_ratio 0.00090
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.7:progress 60.00%
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.7:projected_ratio 0.00105
869269010 LDAS-TEST22992620 jobstate response: '22992620.7:cont'
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.8:progress 70.00%
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.8:projected_ratio 0.00122
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.9:progress 80.00%
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.9:projected_ratio 0.00137
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.10:progress 90.00%
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.10:projected_ratio 0.00152
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.11:progress 100.00%
869269010 LDAS-TEST22992620 mpi::filterLogEntries 22992620.11:projected_ratio 0.00152
869269011 LDAS-TEST22992620 mpi::filterLogEntries 22992620.12:finalizing { totalTime=2.9162 sentObjects=3; Rank0 (inPut: create=0.28424 send=0.10053 ilwd=0.22691; outPut: create=0.12445 send=0.49553 ilwd=0.04457; metadata: (un)pack=0.00043 bcast=0.12369; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.22242; max memory=483328) Rank1 (inPut: create=0.31252 send=0.08755; outPut: create=0.12883 send=0.53266; metadata: (un)pack=0.00036 bcast=0.12957; IS=0.01323; CD=0.11748; AS=1.33134; FO=0.00037; FS=0.00262; LB=0.02400; max memory=18403328) }
869269045 LDAS-TEST22992621 mpi::filterLogEntries 22992621.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) }
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.1:progress 0.00%
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.1:projected_ratio 0.00016
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.2:progress 10.00%
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.2:projected_ratio 0.00031
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.3:progress 20.00%
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.3:projected_ratio 0.00045
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.4:progress 30.00%
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.4:projected_ratio 0.00060
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.5:progress 40.00%
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.5:projected_ratio 0.00075
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.6:progress 50.00%
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.6:projected_ratio 0.00090
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.7:progress 60.00%
869269047 LDAS-TEST22992621 mpi::filterLogEntries 22992621.7:projected_ratio 0.00105
869269047 LDAS-TEST22992621 jobstate response: '22992621.7:cont'
869269048 LDAS-TEST22992621 mpi::filterLogEntries 22992621.8:progress 70.00%
869269048 LDAS-TEST22992621 mpi::filterLogEntries 22992621.8:projected_ratio 0.00123
869269048 LDAS-TEST22992621 mpi::filterLogEntries 22992621.9:progress 80.00%
869269048 LDAS-TEST22992621 mpi::filterLogEntries 22992621.9:projected_ratio 0.00137
869269048 LDAS-TEST22992621 mpi::filterLogEntries 22992621.10:progress 90.00%
869269048 LDAS-TEST22992621 mpi::filterLogEntries 22992621.10:projected_ratio 0.00152
869269048 LDAS-TEST22992621 mpi::filterLogEntries 22992621.11:progress 100.00%
869269048 LDAS-TEST22992621 mpi::filterLogEntries 22992621.11:projected_ratio 0.00152
869269048 LDAS-TEST22992621 mpi::filterLogEntries 22992621.12:finalizing { totalTime=2.94631 sentObjects=3; Rank0 (inPut: create=0.28334 send=0.10074 ilwd=0.22593; outPut: create=0.16051 send=0.54355 ilwd=0.04390; metadata: (un)pack=0.00041 bcast=0.15974; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.22486; max memory=483328) Rank1 (inPut: create=0.31603 send=0.10848; outPut: create=0.16011 send=0.56411; metadata: (un)pack=0.00036 bcast=0.16081; IS=0.01321; CD=0.10644; AS=1.33402; FO=0.00036; FS=0.00262; LB=0.02801; max memory=18403328) }
869269102 LDAS-TEST22992628 mpi::filterLogEntries 22992628.0:initializing { Master-Rank0(metaserver) Rank1(metaserver) }
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.1:progress 0.00%
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.1:projected_ratio 0.00015
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.2:progress 10.00%
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.2:projected_ratio 0.00030
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.3:progress 20.00%
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.3:projected_ratio 0.00045
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.4:progress 30.00%
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.4:projected_ratio 0.00060
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.5:progress 40.00%
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.5:projected_ratio 0.00075
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.6:progress 50.00%
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.6:projected_ratio 0.00090
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.7:progress 60.00%
869269104 LDAS-TEST22992628 mpi::filterLogEntries 22992628.7:projected_ratio 0.00105
869269104 LDAS-TEST22992628 jobstate response: '22992628.7:cont'
869269105 LDAS-TEST22992628 mpi::filterLogEntries 22992628.8:progress 70.00%
869269105 LDAS-TEST22992628 mpi::filterLogEntries 22992628.8:projected_ratio 0.00123
869269105 LDAS-TEST22992628 mpi::filterLogEntries 22992628.9:progress 80.00%
869269105 LDAS-TEST22992628 mpi::filterLogEntries 22992628.9:projected_ratio 0.00138
869269105 LDAS-TEST22992628 mpi::filterLogEntries 22992628.10:progress 90.00%
869269105 LDAS-TEST22992628 mpi::filterLogEntries 22992628.10:projected_ratio 0.00153
869269105 LDAS-TEST22992628 mpi::filterLogEntries 22992628.11:progress 100.00%
869269105 LDAS-TEST22992628 mpi::filterLogEntries 22992628.11:projected_ratio 0.00153