srun --partition milano --reservation=lcls:scaling --account lcls:prjdat21 -n 1 --time=02:00:00 --exclusive --pty /bin/bash

IN OTHER WINDOW:

ssh -Y sdfmilan216
cd LCLS/con-py3

source /sdf/group/lcls/ds/ana/sw/conda1/manage/bin/psconda.sh
source /sdf/group/lcls/ds/ana/sw/conda1/manage/bin/setup_testrel

mpirun -n 1 python  Detector/examples/test-scaling-mpi.py 2
mpirun -n 80 python Detector/examples/test-scaling-mpi.py 2

python Detector/examples/test-scaling-mpi.py -99 # to order results in file for rank number

    ds = MPIDataSource('exp=xcsx1003522:run=33') # 72257 events
    det = Detector('XcsEndstation.0:Jungfrau.1')

    for nevt,evt in enumerate(ds.events()):
        raw = det.raw(evt) # prefetch the data from disk
        if raw is None: continue
        if nevt >= nevents: break
        t0_sec = time()
        calib = det.calib(evt)   ##### TIMING FOR det.calib
        dt_sec = time()-t0_sec

        arrts[nevt] = dt_sec

mpirun -n 1 python Detector/examples/test-scaling-mpi.py 2
figs/fig-mpi-data-1p-v00-sdfmilan216-ncores01-summary.txt
hostname:sdfmilan216 rank:000 cpu:000 cmt:1p-v00 proc time (sec) mean: 0.0430 +/- 0.0013 rms: 0.0014 +/- 0.0009


mpirun -n 80 python Detector/examples/test-scaling-mpi.py 2
python Detector/examples/test-scaling-mpi.py -99
figs/fig-mpi-data-1p-v00-sdfmilan216-ncores80-summary-ordered.txt

hostname:sdfmilan216 rank:000 cpu:000 cmt:1p-v00 proc time (sec) mean: 0.0891 +/- 0.0068 rms: 0.0273 +/- 0.0048
hostname:sdfmilan216 rank:001 cpu:027 cmt:1p-v00 proc time (sec) mean: 0.0689 +/- 0.0027 rms: 0.0048 +/- 0.0019
hostname:sdfmilan216 rank:002 cpu:035 cmt:1p-v00 proc time (sec) mean: 0.0706 +/- 0.0029 rms: 0.0059 +/- 0.0020
hostname:sdfmilan216 rank:003 cpu:055 cmt:1p-v00 proc time (sec) mean: 0.0577 +/- 0.0031 rms: 0.0061 +/- 0.0022
hostname:sdfmilan216 rank:004 cpu:072 cmt:1p-v00 proc time (sec) mean: 0.0883 +/- 0.0072 rms: 0.0235 +/- 0.0051
hostname:sdfmilan216 rank:005 cpu:094 cmt:1p-v00 proc time (sec) mean: 0.0893 +/- 0.0067 rms: 0.0253 +/- 0.0047
hostname:sdfmilan216 rank:006 cpu:103 cmt:1p-v00 proc time (sec) mean: 0.1082 +/- 0.0142 rms: 0.0520 +/- 0.0101
hostname:sdfmilan216 rank:007 cpu:125 cmt:1p-v00 proc time (sec) mean: 0.1131 +/- 0.0161 rms: 0.0583 +/- 0.0114
hostname:sdfmilan216 rank:008 cpu:014 cmt:1p-v00 proc time (sec) mean: 0.0923 +/- 0.0072 rms: 0.0293 +/- 0.0051
hostname:sdfmilan216 rank:009 cpu:028 cmt:1p-v00 proc time (sec) mean: 0.0742 +/- 0.0031 rms: 0.0058 +/- 0.0022
hostname:sdfmilan216 rank:010 cpu:044 cmt:1p-v00 proc time (sec) mean: 0.0711 +/- 0.0040 rms: 0.0099 +/- 0.0028
hostname:sdfmilan216 rank:011 cpu:051 cmt:1p-v00 proc time (sec) mean: 0.0578 +/- 0.0027 rms: 0.0051 +/- 0.0019
hostname:sdfmilan216 rank:012 cpu:067 cmt:1p-v00 proc time (sec) mean: 0.0877 +/- 0.0079 rms: 0.0251 +/- 0.0056
hostname:sdfmilan216 rank:013 cpu:082 cmt:1p-v00 proc time (sec) mean: 0.0859 +/- 0.0061 rms: 0.0228 +/- 0.0043
hostname:sdfmilan216 rank:014 cpu:096 cmt:1p-v00 proc time (sec) mean: 0.1075 +/- 0.0139 rms: 0.0538 +/- 0.0098
hostname:sdfmilan216 rank:015 cpu:112 cmt:1p-v00 proc time (sec) mean: 0.1064 +/- 0.0161 rms: 0.0584 +/- 0.0114
hostname:sdfmilan216 rank:016 cpu:012 cmt:1p-v00 proc time (sec) mean: 0.0895 +/- 0.0062 rms: 0.0249 +/- 0.0044
hostname:sdfmilan216 rank:017 cpu:031 cmt:1p-v00 proc time (sec) mean: 0.0708 +/- 0.0029 rms: 0.0055 +/- 0.0021
hostname:sdfmilan216 rank:018 cpu:032 cmt:1p-v00 proc time (sec) mean: 0.0715 +/- 0.0032 rms: 0.0064 +/- 0.0023
hostname:sdfmilan216 rank:019 cpu:061 cmt:1p-v00 proc time (sec) mean: 0.0571 +/- 0.0027 rms: 0.0052 +/- 0.0019
hostname:sdfmilan216 rank:020 cpu:078 cmt:1p-v00 proc time (sec) mean: 0.0889 +/- 0.0067 rms: 0.0230 +/- 0.0047
hostname:sdfmilan216 rank:021 cpu:092 cmt:1p-v00 proc time (sec) mean: 0.0896 +/- 0.0069 rms: 0.0242 +/- 0.0049
hostname:sdfmilan216 rank:022 cpu:100 cmt:1p-v00 proc time (sec) mean: 0.1095 +/- 0.0146 rms: 0.0554 +/- 0.0103
hostname:sdfmilan216 rank:023 cpu:117 cmt:1p-v00 proc time (sec) mean: 0.1083 +/- 0.0167 rms: 0.0620 +/- 0.0118
hostname:sdfmilan216 rank:024 cpu:006 cmt:1p-v00 proc time (sec) mean: 0.0861 +/- 0.0064 rms: 0.0249 +/- 0.0045
hostname:sdfmilan216 rank:025 cpu:026 cmt:1p-v00 proc time (sec) mean: 0.0760 +/- 0.0029 rms: 0.0053 +/- 0.0021
hostname:sdfmilan216 rank:026 cpu:041 cmt:1p-v00 proc time (sec) mean: 0.0744 +/- 0.0036 rms: 0.0095 +/- 0.0026
hostname:sdfmilan216 rank:027 cpu:058 cmt:1p-v00 proc time (sec) mean: 0.0560 +/- 0.0024 rms: 0.0039 +/- 0.0017
hostname:sdfmilan216 rank:028 cpu:074 cmt:1p-v00 proc time (sec) mean: 0.0881 +/- 0.0078 rms: 0.0271 +/- 0.0055
hostname:sdfmilan216 rank:029 cpu:091 cmt:1p-v00 proc time (sec) mean: 0.0854 +/- 0.0069 rms: 0.0248 +/- 0.0049
hostname:sdfmilan216 rank:030 cpu:104 cmt:1p-v00 proc time (sec) mean: 0.1100 +/- 0.0155 rms: 0.0595 +/- 0.0110
hostname:sdfmilan216 rank:031 cpu:123 cmt:1p-v00 proc time (sec) mean: 0.1081 +/- 0.0140 rms: 0.0544 +/- 0.0099
hostname:sdfmilan216 rank:032 cpu:009 cmt:1p-v00 proc time (sec) mean: 0.0900 +/- 0.0063 rms: 0.0251 +/- 0.0044
hostname:sdfmilan216 rank:033 cpu:029 cmt:1p-v00 proc time (sec) mean: 0.0723 +/- 0.0028 rms: 0.0054 +/- 0.0020
hostname:sdfmilan216 rank:034 cpu:033 cmt:1p-v00 proc time (sec) mean: 0.0693 +/- 0.0035 rms: 0.0087 +/- 0.0025
hostname:sdfmilan216 rank:035 cpu:056 cmt:1p-v00 proc time (sec) mean: 0.0551 +/- 0.0019 rms: 0.0028 +/- 0.0014
hostname:sdfmilan216 rank:036 cpu:069 cmt:1p-v00 proc time (sec) mean: 0.0866 +/- 0.0068 rms: 0.0245 +/- 0.0048
hostname:sdfmilan216 rank:037 cpu:086 cmt:1p-v00 proc time (sec) mean: 0.0879 +/- 0.0071 rms: 0.0250 +/- 0.0050
hostname:sdfmilan216 rank:038 cpu:110 cmt:1p-v00 proc time (sec) mean: 0.1079 +/- 0.0148 rms: 0.0573 +/- 0.0105
hostname:sdfmilan216 rank:039 cpu:116 cmt:1p-v00 proc time (sec) mean: 0.1057 +/- 0.0147 rms: 0.0571 +/- 0.0104
hostname:sdfmilan216 rank:040 cpu:005 cmt:1p-v00 proc time (sec) mean: 0.0886 +/- 0.0068 rms: 0.0271 +/- 0.0048
hostname:sdfmilan216 rank:041 cpu:024 cmt:1p-v00 proc time (sec) mean: 0.0757 +/- 0.0028 rms: 0.0051 +/- 0.0020
hostname:sdfmilan216 rank:042 cpu:046 cmt:1p-v00 proc time (sec) mean: 0.0699 +/- 0.0035 rms: 0.0086 +/- 0.0025
hostname:sdfmilan216 rank:043 cpu:057 cmt:1p-v00 proc time (sec) mean: 0.0554 +/- 0.0023 rms: 0.0035 +/- 0.0016
hostname:sdfmilan216 rank:044 cpu:077 cmt:1p-v00 proc time (sec) mean: 0.0851 +/- 0.0063 rms: 0.0213 +/- 0.0045
hostname:sdfmilan216 rank:045 cpu:089 cmt:1p-v00 proc time (sec) mean: 0.0838 +/- 0.0066 rms: 0.0217 +/- 0.0046
hostname:sdfmilan216 rank:046 cpu:111 cmt:1p-v00 proc time (sec) mean: 0.1077 +/- 0.0146 rms: 0.0562 +/- 0.0103
hostname:sdfmilan216 rank:047 cpu:124 cmt:1p-v00 proc time (sec) mean: 0.1009 +/- 0.0135 rms: 0.0512 +/- 0.0096
hostname:sdfmilan216 rank:048 cpu:004 cmt:1p-v00 proc time (sec) mean: 0.0867 +/- 0.0066 rms: 0.0268 +/- 0.0047
hostname:sdfmilan216 rank:049 cpu:025 cmt:1p-v00 proc time (sec) mean: 0.0764 +/- 0.0028 rms: 0.0048 +/- 0.0020
hostname:sdfmilan216 rank:050 cpu:034 cmt:1p-v00 proc time (sec) mean: 0.0712 +/- 0.0039 rms: 0.0100 +/- 0.0027
hostname:sdfmilan216 rank:051 cpu:050 cmt:1p-v00 proc time (sec) mean: 0.0559 +/- 0.0027 rms: 0.0052 +/- 0.0019
hostname:sdfmilan216 rank:052 cpu:066 cmt:1p-v00 proc time (sec) mean: 0.0880 +/- 0.0069 rms: 0.0236 +/- 0.0049
hostname:sdfmilan216 rank:053 cpu:084 cmt:1p-v00 proc time (sec) mean: 0.0857 +/- 0.0062 rms: 0.0218 +/- 0.0044
hostname:sdfmilan216 rank:054 cpu:107 cmt:1p-v00 proc time (sec) mean: 0.1075 +/- 0.0156 rms: 0.0564 +/- 0.0110
hostname:sdfmilan216 rank:055 cpu:115 cmt:1p-v00 proc time (sec) mean: 0.1042 +/- 0.0156 rms: 0.0575 +/- 0.0110
hostname:sdfmilan216 rank:056 cpu:000 cmt:1p-v00 proc time (sec) mean: 0.0902 +/- 0.0068 rms: 0.0278 +/- 0.0048
hostname:sdfmilan216 rank:057 cpu:030 cmt:1p-v00 proc time (sec) mean: 0.0725 +/- 0.0025 rms: 0.0045 +/- 0.0018
hostname:sdfmilan216 rank:058 cpu:047 cmt:1p-v00 proc time (sec) mean: 0.0726 +/- 0.0035 rms: 0.0086 +/- 0.0025
hostname:sdfmilan216 rank:059 cpu:061 cmt:1p-v00 proc time (sec) mean: 0.0555 +/- 0.0024 rms: 0.0039 +/- 0.0017
hostname:sdfmilan216 rank:060 cpu:076 cmt:1p-v00 proc time (sec) mean: 0.0862 +/- 0.0070 rms: 0.0234 +/- 0.0050
hostname:sdfmilan216 rank:061 cpu:081 cmt:1p-v00 proc time (sec) mean: 0.0853 +/- 0.0060 rms: 0.0212 +/- 0.0042
hostname:sdfmilan216 rank:062 cpu:101 cmt:1p-v00 proc time (sec) mean: 0.1028 +/- 0.0130 rms: 0.0505 +/- 0.0092
hostname:sdfmilan216 rank:063 cpu:122 cmt:1p-v00 proc time (sec) mean: 0.1056 +/- 0.0150 rms: 0.0555 +/- 0.0106
hostname:sdfmilan216 rank:064 cpu:001 cmt:1p-v00 proc time (sec) mean: 0.0940 +/- 0.0067 rms: 0.0286 +/- 0.0047
hostname:sdfmilan216 rank:065 cpu:033 cmt:1p-v00 proc time (sec) mean: 0.0741 +/- 0.0035 rms: 0.0079 +/- 0.0025
hostname:sdfmilan216 rank:066 cpu:040 cmt:1p-v00 proc time (sec) mean: 0.0684 +/- 0.0042 rms: 0.0102 +/- 0.0030
hostname:sdfmilan216 rank:067 cpu:052 cmt:1p-v00 proc time (sec) mean: 0.0574 +/- 0.0028 rms: 0.0051 +/- 0.0020
hostname:sdfmilan216 rank:068 cpu:075 cmt:1p-v00 proc time (sec) mean: 0.0862 +/- 0.0061 rms: 0.0208 +/- 0.0043
hostname:sdfmilan216 rank:069 cpu:093 cmt:1p-v00 proc time (sec) mean: 0.0893 +/- 0.0073 rms: 0.0260 +/- 0.0052
hostname:sdfmilan216 rank:070 cpu:099 cmt:1p-v00 proc time (sec) mean: 0.1050 +/- 0.0138 rms: 0.0555 +/- 0.0098
hostname:sdfmilan216 rank:071 cpu:126 cmt:1p-v00 proc time (sec) mean: 0.1115 +/- 0.0150 rms: 0.0559 +/- 0.0106
hostname:sdfmilan216 rank:072 cpu:007 cmt:1p-v00 proc time (sec) mean: 0.0829 +/- 0.0066 rms: 0.0258 +/- 0.0046
hostname:sdfmilan216 rank:073 cpu:045 cmt:1p-v00 proc time (sec) mean: 0.0728 +/- 0.0036 rms: 0.0092 +/- 0.0025
hostname:sdfmilan216 rank:074 cpu:036 cmt:1p-v00 proc time (sec) mean: 0.0708 +/- 0.0033 rms: 0.0074 +/- 0.0024
hostname:sdfmilan216 rank:075 cpu:063 cmt:1p-v00 proc time (sec) mean: 0.0550 +/- 0.0021 rms: 0.0032 +/- 0.0015
hostname:sdfmilan216 rank:076 cpu:071 cmt:1p-v00 proc time (sec) mean: 0.0882 +/- 0.0071 rms: 0.0259 +/- 0.0050
hostname:sdfmilan216 rank:077 cpu:080 cmt:1p-v00 proc time (sec) mean: 0.0835 +/- 0.0057 rms: 0.0203 +/- 0.0040
hostname:sdfmilan216 rank:078 cpu:108 cmt:1p-v00 proc time (sec) mean: 0.1045 +/- 0.0154 rms: 0.0555 +/- 0.0109
hostname:sdfmilan216 rank:079 cpu:118 cmt:1p-v00 proc time (sec) mean: 0.1002 +/- 0.0135 rms: 0.0533 +/- 0.0095
mean time (sec): 0.0846