Versions Compared

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.
Comment: Migration of unmigrated content due to installation of a new plugin

...

--stream <Stream ID=-1>

Integer stream identifier. Auto assigned if option not specified.

--nStreams <Number of Streams=1>

Number of streams to create, not valid if Stream ID specified

<ac:structured-macro ac:name="unmigrated-wiki-markup" ac:schema -version="1" ac:macro-id="39ad809f-b0a6-4cee-9502-88dc3bc8ede9"><ac:plain-text-body><![CDATA[ --define <name=value>

Define a variable. Syntax is "name=value[,name2=value2,...]"

]]></ac:plain-text-body></ac:structured-macro>

For example to create 10 streams of the EXOMCBackground task defined above, overriding the default value of the MAXEVENTS variable we would use the following command:

...

Code Block
xml
xml
<?xml version="1.0" encoding="UTF-8"?>
<pipeline xmlns="http://glast-ground.slac.stanford.edu/pipeline"
          xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
          xs:schemaLocation="http://glast-ground.slac.stanford.edu/pipeline
          http://srs.slac.stanford.edu/Pipeline-II/schemas/2.0/pipeline.xsd">
  <task name="EXOMCBackground" type="EXO" version="1.11">
    <notation>A generic task for running EXO MC backgrounds</notation>
    <variables>
      <var name="EXODIR">/nfs/slac/g/exo</var>
      <var name="EXOBASE">${EXODIR}/software/builds/trunk</var>
      <var name="BATCHOPTIONS">-R &quot;select[-rhel30] rusage[scratch=1]&quot;</var>
      <var name="CORE_LIMIT">1024</var>
      <var name="MAXEVENTS">10000</var>
      <var name="PRINTMODULO">${MAXEVENTS/100}</var>
      <var name="INITIALSEED">pipeline.stream%100000<>${pipeline.stream%100000}</var>
      <var name="MAXCPU">${MAXEVENTS/10}</var>
      <var name="MAXMEM">1000</var>
      <var name="SOURCE_VOLUME">HFE</var>
      <var name="SOURCE_ION">k</var>
      <var name="OUTPUT_DIR">/nfs/slac/g/exo/exo_data/data/MC/backgrounds/TestBkgdMC/${SOURCE_ION}/${SOURCE_VOLUME}</var>
      <var name="OUTPUT_FORMAT">MC-background-%06d.root</var>
      <var name="OUTPUT_NAME">${format(pipeline.stream,OUTPUT_FORMAT)}</var>
      <var name="OUTPUT_FILE">${OUTPUT_DIR}/${OUTPUT_NAME}</var>
      <var name="DATACAT_DIR">EXO/TestBkgdMC/${SOURCE_ION}</var>
      <var name="DATACAT_GROUP">${SOURCE_VOLUME}</var>
    </variables>

      <process name="runMonteCarlo">
         <job batchOptions="${BATCHOPTIONS}" maxCPU="${MAXCPU}" maxMemory="${MAXMEM}">
             ...
         </job>
      </process>


      <process name="register-ds">
         <notation>Register datasets created in this task</notation>
         <script>
             ...
         </script>

         <depends>
              <after process="runMonteCarlo"/>
         </depends>

     </process>
  </task>
</pipeline>

...

Code Block
xml
xml
titleEXOMCBackground.xml
<?xml version="1.0" encoding="UTF-8"?>
<pipeline xmlns="http://glast-ground.slac.stanford.edu/pipeline"
          xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
          xs:schemaLocation="http://glast-ground.slac.stanford.edu/pipeline
          http://srs.slac.stanford.edu/Pipeline-II/schemas/2.0/pipeline.xsd">
  <task name="EXOMCBackground" type="EXO" version="1.11">
    <notation>A generic task for running EXO MC backgrounds</notation>
    <variables>
      <var name="EXODIR">/nfs/slac/g/exo</var>
      <var name="EXOBASE">${EXODIR}/software/builds/trunk</var>
      <var name="BATCHOPTIONS">-R &quot;select[-rhel30] rusage[scratch=1]&quot;</var>
      <var name="CORE_LIMIT">1024</var>
      <var name="MAXEVENTS">10000</var>
      <var name="PRINTMODULO">${MAXEVENTS/100}</var>
      <var name="INITIALSEED">pipeline.stream%100000<>${pipeline.stream%100000}</var>
      <var name="MAXCPU">${MAXEVENTS/10}</var>
      <var name="MAXMEM">1000</var>
      <var name="SOURCE_VOLUME">HFE</var>
      <var name="SOURCE_ION">k</var>
      <var name="OUTPUT_DIR">/nfs/slac/g/exo/exo_data/data/MC/backgrounds/TestBkgdMC/${SOURCE_ION}/${SOURCE_VOLUME}</var>
      <var name="OUTPUT_FORMAT">MC-background-%06d.root</var>
      <var name="OUTPUT_NAME">${format(pipeline.stream,OUTPUT_FORMAT)}</var>
      <var name="OUTPUT_FILE">${OUTPUT_DIR}/${OUTPUT_NAME}</var>
      <var name="DATACAT_DIR">EXO/TestBkgdMC/${SOURCE_ION}</var>
      <var name="DATACAT_GROUP">${SOURCE_VOLUME}</var>
    </variables>

      <process name="runMonteCarlo">
         <job batchOptions="${BATCHOPTIONS}" maxCPU="${MAXCPU}" maxMemory="${MAXMEM}"><![CDATA[
              ulimit -c ${CORE_LIMIT} # Limit core dumps
              set -e # exit on error

              # Create a scratch area to write the output to
              export SCRATCH_DIR=/scratch/exo/${PIPELINE_PROCESSINSTANCE}
              mkdir -p ${SCRATCH_DIR}
              gotEXIT()
              {
                 rm -rf ${SCRATCH_DIR}
              }
              trap gotEXIT EXIT

              source ${EXOBASE}/setup.sh

              # Create background.exe
              cat > background.exo <<EOF
              use exosim rec toutput
              /exosim/macro background.mac
              /exosim/filter true
              printmodulo ${PRINTMODULO}
              /exosim/initial_seed ${INITIALSEED}
              /exosim/run_number ${PIPELINE_STREAM}
              maxevents ${MAXEVENTS}
              /toutput/file ${SCRATCH_DIR}/output.root
              begin
              exit
              EOF

              case ${SOURCE_ION} in
              k)
                GPS_ION="19 40 0 0"
                ;;
              th)
                GPS_ION="90 232 0 0"
                ;;
              u)
                GPS_ION="92 238 0 0"
                ;;
              *)
                echo "Unknown ION ${SOURCE_ION}"
                exit 1
              esac

              case ${SOURCE_VOLUME} in
              HFE)
                HALFZ=72.5
                RADIUS=75.0
                ;;
              InnnerCryo)
                HALFZ=74.5
                RADIUS=78.0
                ;;
              *)
                echo "Unknown volume ${SOURCE_VOLUME}"
                exit 1
              esac

              # Create background.mac
              cat > background.mac <<EOF
              /digitizer/wireNoise 800.000000
              /digitizer/APDNoise 2000.000000
              /digitizer/LXeEnergyRes 0.015000
              /event/LXeEventsOnly true
              /event/digitizeWires true
              /event/digitizeAPDs true
              /gps/pos/type Volume
              /gps/pos/shape Cylinder
              /gps/pos/halfz ${HALFZ} cm
              /gps/pos/radius ${RADIUS} cm
              /gps/pos/centre 0.0 0.0 0.0 cm
              /gps/pos/confine ${SOURCE_VOLUME}
              /gps/energy 0 keV
              /gps/particle ion
              /gps/ion ${GPS_ION}
              /grdm/analogueMC 1
              EOF

              EXOAnalysis background.exo

              mkdir -p ${OUTPUT_DIR}
              cp -pv ${SCRATCH_DIR}/output.root ${OUTPUT_FILE}
              ]]>
         </job>
      </process>


      <process name="register-ds">
         <notation>Register datasets created in this task</notation>
         <script><![CDATA[
           metaData = {'nGeneratedEvents':MAXEVENTS,'SourceVolume':SOURCE_VOLUME,'SourceIon':SOURCE_ION}
           dsNew = datacatalog.newDataset(OUTPUT_NAME, "root", "EXOROOT", DATACAT_DIR, DATACAT_GROUP, "SLAC", OUTPUT_FILE)
           datacatalog.registerDataset(dsNew, metaData)
           ]]>
         </script>

         <depends>
              <after process="runMonteCarlo"/>
         </depends>

     </process>
  </task>
</pipeline>

Once a new task has been defined it can be uploaded using either the pipeline web interface (on the Admin page) or from the pipeline command on SLAC unix:

Once a new task has been defined it can be uploaded using either the pipeline web interface (on the Admin page) or from the pipeline command on SLAC unix:

No Format
~exodata/pipeline/prod/pipeline load <xml-file>
No Format
~exodata/pipeline/prod/pipeline load <xml-file>

Note that each file to be uploaded must have a unique task name and version number, so when uploading new version of a task it is necessary to increment the version number in the <task> element. Once a task has been uploaded, an associated job will not start until you issue a "createStream" command (see Running Jobs section above).

List of available variables for EXOMCBackground (v1.12 and higher)

...