The mpi API on metaserver at Caltech-Test

Legend:

Normal status or debugging message

Notable condition which may be a non-fatal error

Error condition not fatal to job

Error condition which fatal to job

Notable condition which is not an error

Currently undefined

Condition requires email notification of the responsible administrator of this API

Condition requires phone notification of the responsible administrator of this API

Link: API Status Page for Caltech-Test

941578885 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578885 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578885 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578885 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578885 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578885 CXX_THREAD fork_parent@MountPointStatus.cc end
941578885 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578885 CXX_THREAD fork_parent@MountPointStatus.cc end
941578885 STARTUP execOverload overloaded exec
941578885 STARTUP closeListenSock no cid registered for service 'data'
941578885 STARTUP mpi::init unused data port 10024 closed
941578885 STARTUP mpi::initNodes *** no slave nodes configured on this system ***
941578885 STARTUP mpi::init port 10024 (jobstate) opened on metaserver as sock3
941578885 STARTUP bgLoop Looping process watchlogs started
941578885 STARTUP openListenSock port 10022 (operator) opened on metaserver as sock4
941578885 STARTUP openListenSock port 10023 (emergency) opened on metaserver as sock5
941578885 STARTUP setResourceLimit vmemoryuse=unlimited; datasize=unlimited; core=unlimited; maxproc=32767; descriptors=1024; memorylocked=32768; filesize=unlimited; cputime=unlimited
941578885 STARTUP leakLogger inital size of mpi API: 13524 kB
941578885 STARTUP bgLoop Looping process etchosts started
941578885 STARTUP mpi Trying API
941578885 STARTUP mpi API yes
941578885 STARTUP mpi Trying LDAS_SYSTEM
941578885 STARTUP mpi LDAS_SYSTEM yes
941578885 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578885 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578885 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578885 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578885 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578885 CXX_THREAD fork_parent@MountPointStatus.cc end
941578885 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578885 CXX_THREAD fork_parent@MountPointStatus.cc end
941578885 IDLE bgLoop Looping process statpagefile started
941578885 IDLE bgLoop Looping process killedjobreaper started
941578889 IDLE setFTPandHTTPinfo (::FTPURL 'ftp://131.215.115.235') (::FTPDIR '') (::HTTPURL 'http://131.215.115.235/ldas_outgoing/jobs') (::HTTPDIR '/ldas_outgoing/jobs') (::GRIDFTPURL 'gridftp:/export/grid/ldas') (::GRIDFTPDIR '/export/grid/ldas') (::LDAS_GATEWAY 'ldas-test 131.215.115.235') (::LDAS_SYSTEM 'ldas-test') (::RUNCODE 'LDAS-TEST')
941578890 IDLE setFTPandHTTPinfo (::FTPURL 'ftp://131.215.115.235') (::FTPDIR '') (::HTTPURL 'http://131.215.115.235/ldas_outgoing/jobs') (::HTTPDIR '/ldas_outgoing/jobs') (::GRIDFTPURL 'gridftp:/export/grid/ldas') (::GRIDFTPDIR '/export/grid/ldas') (::LDAS_GATEWAY 'ldas-test 131.215.115.235') (::LDAS_SYSTEM 'ldas-test') (::RUNCODE 'LDAS-TEST')
941578890 STARTUP mpi::killAllMpirun cleaning up for user ldas
941578890 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578890 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578890 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578890 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578890 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578890 CXX_THREAD fork_parent@MountPointStatus.cc end
941578890 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578890 CXX_THREAD fork_parent@MountPointStatus.cc end
941578890 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578890 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578890 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578890 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578890 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578890 CXX_THREAD fork_parent@MountPointStatus.cc end
941578890 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578890 CXX_THREAD fork_parent@MountPointStatus.cc end
941578891 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578891 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578891 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578891 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578891 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578891 CXX_THREAD fork_parent@MountPointStatus.cc end
941578891 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578891 CXX_THREAD fork_parent@MountPointStatus.cc end
941578891 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578891 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578891 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578891 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578891 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578891 CXX_THREAD fork_parent@MountPointStatus.cc end
941578891 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578891 CXX_THREAD fork_parent@MountPointStatus.cc end
941578892 STARTUP mpi::killAllMpirun ran kill 2 times in 1.541 seconds
941578892 STARTUP mpi::prestartLamds running lamboot for user search01
941578892 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578892 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578892 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578892 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578892 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578892 CXX_THREAD fork_parent@MountPointStatus.cc end
941578892 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578892 CXX_THREAD fork_parent@MountPointStatus.cc end
941578892 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578892 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578892 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578892 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578892 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578892 CXX_THREAD fork_parent@MountPointStatus.cc end
941578892 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578892 CXX_THREAD fork_parent@MountPointStatus.cc end
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578893 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578893 CXX_THREAD fork_parent@MountPointStatus.cc end
941578893 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578893 CXX_THREAD fork_parent@MountPointStatus.cc end
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578893 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578893 CXX_THREAD fork_parent@MountPointStatus.cc end
941578893 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578893 CXX_THREAD fork_parent@MountPointStatus.cc end
941578893 STARTUP mpi::prestartLamds running lamboot for user search02
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578893 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578893 CXX_THREAD fork_parent@MountPointStatus.cc end
941578893 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578893 CXX_THREAD fork_parent@MountPointStatus.cc end
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578893 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578893 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578893 CXX_THREAD fork_parent@MountPointStatus.cc end
941578893 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578893 CXX_THREAD fork_parent@MountPointStatus.cc end
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578894 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578894 CXX_THREAD fork_parent@MountPointStatus.cc end
941578894 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578894 CXX_THREAD fork_parent@MountPointStatus.cc end
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578894 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578894 CXX_THREAD fork_parent@MountPointStatus.cc end
941578894 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578894 CXX_THREAD fork_parent@MountPointStatus.cc end
941578894 STARTUP mpi::prestartLamds running lamboot for user search03
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578894 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578894 CXX_THREAD fork_parent@MountPointStatus.cc end
941578894 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578894 CXX_THREAD fork_parent@MountPointStatus.cc end
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578894 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578894 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578894 CXX_THREAD fork_parent@MountPointStatus.cc end
941578894 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578894 CXX_THREAD fork_parent@MountPointStatus.cc end
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578895 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578895 CXX_THREAD fork_parent@MountPointStatus.cc end
941578895 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578895 CXX_THREAD fork_parent@MountPointStatus.cc end
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578895 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578895 CXX_THREAD fork_parent@MountPointStatus.cc end
941578895 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578895 CXX_THREAD fork_parent@MountPointStatus.cc end
941578895 STARTUP mpi::prestartLamds running lamboot for user search04
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578895 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578895 CXX_THREAD fork_parent@MountPointStatus.cc end
941578895 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578895 CXX_THREAD fork_parent@MountPointStatus.cc end
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578895 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578895 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578895 CXX_THREAD fork_parent@MountPointStatus.cc end
941578895 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578895 CXX_THREAD fork_parent@MountPointStatus.cc end
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578896 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578896 CXX_THREAD fork_parent@MountPointStatus.cc end
941578896 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578896 CXX_THREAD fork_parent@MountPointStatus.cc end
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578896 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578896 CXX_THREAD fork_parent@MountPointStatus.cc end
941578896 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578896 CXX_THREAD fork_parent@MountPointStatus.cc end
941578896 STARTUP mpi::prestartLamds running lamboot for user search05
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578896 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578896 CXX_THREAD fork_parent@MountPointStatus.cc end
941578896 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578896 CXX_THREAD fork_parent@MountPointStatus.cc end
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578896 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578896 CXX_THREAD fork_parent@MountPointStatus.cc end
941578896 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578896 CXX_THREAD fork_parent@MountPointStatus.cc end
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578896 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578896 CXX_THREAD fork_parent@MountPointStatus.cc end
941578896 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578896 CXX_THREAD fork_parent@MountPointStatus.cc end
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578896 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578896 CXX_THREAD fork_parent@MountPointStatus.cc end
941578896 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578896 CXX_THREAD fork_parent@MountPointStatus.cc end
941578896 STARTUP mpi::prestartLamds running lamboot for user search06
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578896 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578897 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578897 CXX_THREAD fork_parent@MountPointStatus.cc end
941578897 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578897 CXX_THREAD fork_parent@MountPointStatus.cc end
941578897 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578897 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578897 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578897 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578897 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578897 CXX_THREAD fork_parent@MountPointStatus.cc end
941578897 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578897 CXX_THREAD fork_parent@MountPointStatus.cc end
941578897 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578897 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578897 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578897 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578897 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578897 CXX_THREAD fork_parent@MountPointStatus.cc end
941578897 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578897 CXX_THREAD fork_parent@MountPointStatus.cc end
941578897 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578897 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578897 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941578897 CXX_THREAD fork_prepare@MountPointStatus.cc end
941578897 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578897 CXX_THREAD fork_parent@MountPointStatus.cc end
941578897 CXX_THREAD fork_parent@MountPointStatus.cc begin
941578897 CXX_THREAD fork_parent@MountPointStatus.cc end
941578897 STARTUP mpi::prestartLamds running lamboot for user search07
941578897 CXX_THREAD fork_prepare@MountPointStatus.cc begin