[MRG] New recipe for DPS

.travis.yml

@@ -18,20 +18,22 @@ install:
   - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then curl --silent http://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -o miniconda.sh; fi
   - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then wget -nv http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; fi
 
-  - bash miniconda.sh -b -p $HOME/miniconda
-  - export PATH="$HOME/miniconda/bin:$PATH"
+  - export SWPATH=/software
+  - sudo mkdir $SWPATH && sudo chown $USER $SWPATH
+  - bash miniconda.sh -b -p $SWPATH/miniconda
+  - export PATH="$SWPATH/miniconda/bin:$PATH"
+  - export ENV_NAME=dps
   - hash -r
   - conda config --set always_yes yes --set changeps1 no
   - conda update -q conda
   - conda install -q psutil
   - conda info -a # Useful for debugging any issues with conda
   - conda config --add channels http://conda.anaconda.org/NLeSC
   - conda config --set show_channel_urls yes
-  - conda create -q -n testenv python=${PYTHON} root=${ROOT} root-numpy numpy matplotlib nose sphinx pytables rootpy
-  - export CONDA_ENV_PATH=$HOME/miniconda/envs/testenv
-  - source activate testenv
+  - conda create -q -n ${ENV_NAME} python=${PYTHON} root=${ROOT} root-numpy numpy matplotlib nose sphinx pytables rootpy
+  - export CONDA_ENV_PATH=$SWPATH/miniconda/envs/${ENV_NAME}
+  - source activate ${ENV_NAME}
   - pip install -U uncertainties
-  - bash ci/build_RooUnfold.sh
 
 script: bash ci/test.sh
 cache: apt

README.md

@@ -4,48 +4,28 @@
 
 Python scripts for the daily tasks in particle physics (*Run 2*) - Daily Python Scripts (aka DPS)
 
-Quick installation recipe:
+If working on soolin (or anywhere where dependencies like ROOT/latex/etc are not available), run it within CMSSW:
 ```
-# get the code from the repository
 git clone https://github.com/BristolTopGroup/DailyPythonScripts
 cd DailyPythonScripts
-
-# for Run 1 code, please checkout the run1 branch
-# git checkout run1
-
-# get submodules:
-git submodule init && git submodule update
-
-# setup run:
-source setup_conda.sh
-
-# setup environment (using virtualenv for python):
-source environment_conda.sh
-
-# make sure matplotlib is up to date (should return 1.3.1 or above):
+source bin/env.sh
+# make sure matplotlib is up to date (should return 1.5.1 or above):
 python -c 'import matplotlib; print matplotlib.__version__'
+# test ROOT and rootpy
+root -l -q
+time python -c "import ROOT; ROOT.TBrowser()"
+time python -c 'import rootpy'
+time python -c 'from ROOT import kTRUE; import rootpy'
 ```
 
-If working on soolin (or anywhere where dependencies like ROOT/latex/etc are not available), run it within CMSSW:
-
-```
-# install CMSSW and setup environment:
-export SCRAM_ARCH=slc6_amd64_gcc491
-scram p -n CMSSW_7_4_7_DPS_ CMSSW_7_4_7
-cd CMSSW_7_4_7_DPS/src/
-cmsenv
-# This version comes with ROOT 6.02/05
-
-# then install DailyPythonScripts according to the recipe above
-
-```
+Setting up conda environment on your own machine, please have a look under the `Conda` section.
 
 ## Dependencies
-[ROOT](http://root.cern.ch) ≥ 6.02
+[ROOT](http://root.cern.ch) ≥ 6.04
 
 [freetype](http://www.freetype.org)
 
-[matplotlib](http://matplotlib.org/) ≥ 1.3
+[matplotlib](http://matplotlib.org/) ≥ 1.5
 
 ## Disclaimer
 All plots/histograms provided by this package are based on either toy MC or simulated events from the CMS experiment.
@@ -58,8 +38,6 @@ data/* - for input/output of ROOT & JSON files (will be created by some scripts)
 
 examples/* - generic examples for available modules
 
-external/* - external projects (virtualenv, RooUnfold, etc)
-
 plots/* - default output folder for plots (will be created by some scripts)
 
 src/* - specific use of available modules
@@ -112,3 +90,37 @@ x_make_control_plots
 x_make_fit_variable_plots
 ```
 (script AN-14-071 runs all of these scripts automatically if you are confident everything will run smoothly(!))
+
+## Conda
+DailyPythonScripts relies on a (mini)conda environment to provide all necessary dependencies. This section describes how to set up this environment from scratch.
+As a first step, download and install conda (you can skip this step if you are using a shared conda install, e.g. on soolin):
+```bash
+wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+export CONDAINSTALL=<path to miniconda install> # e.g. /software/miniconda
+bash miniconda.sh -b -p $CONDAINSTALL # or a different location
+```
+
+Next let us create a new conda environment with some base packages:
+```bash
+export PATH="$CONDAINSTALL/bin:$PATH"
+export ENV_NAME=dps
+export CONDA_ENV_PATH=$CONDAINSTALL/envs/${ENV_NAME}
+conda config --add channels http://conda.anaconda.org/NLeSC
+conda config --set show_channel_urls yes
+conda update -y conda
+conda install -y psutil
+conda create -y -n ${ENV_NAME} python=2.7 root=6 root-numpy numpy matplotlib nose sphinx pytables rootpy
+```
+Then activate the environment and install the remaining dependencies
+```bash
+source activate $ENV_NAME
+# install dependencies that have no conda recipe
+pip install -U uncertainties tabulate
+```
+At this point you should have all necessary dependencies which you can try out with:
+```bash
+root -l -q
+time python -c "import ROOT; ROOT.TBrowser()"
+time python -c 'import rootpy'
+time python -c 'from ROOT import kTRUE; import rootpy'
+```

bin/env.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+drop_from_path()
+{
+   # Assert that we got enough arguments
+   if test $# -ne 2 ; then
+      echo "drop_from_path: needs 2 arguments"
+      return 1
+   fi
+
+   p=$1
+   drop=$2
+
+   newpath=`echo $p | sed -e "s;:${drop}:;:;g" \
+                          -e "s;:${drop};;g"   \
+                          -e "s;${drop}:;;g"   \
+                          -e "s;${drop};;g"`
+}
+
+
+if [ -n "${DPSROOT}" ] ; then
+   old_dpsbase=${DPSROOT}
+fi
+
+
+if [ "x${BASH_ARGV[0]}" = "x" ]; then
+    if [ ! -f bin/env.sh ]; then
+        echo ERROR: must "cd where/DailyPythonScripts/is" before calling ". bin/env.sh" for this version of bash!
+        DPSROOT=; export DPSROOT
+        return 1
+    fi
+    DPSROOT="$PWD"; export DPSROOT
+else
+    # get param to "."
+    envscript=$(dirname ${BASH_ARGV[0]})
+    DPSROOT=$(cd ${envscript}/..;pwd); export DPSROOT
+fi
+
+if [ -n "${old_dpsbase}" ] ; then
+   if [ -n "${PATH}" ]; then
+      drop_from_path "$PATH" ${old_dpsbase}/bin
+      PATH=$newpath
+   fi
+   if [ -n "${PYTHONPATH}" ]; then
+      drop_from_path $PYTHONPATH ${old_dpsbase}
+      PYTHONPATH=$newpath
+   fi
+fi
+
+
+if [ -z "${PATH}" ]; then
+   PATH=$DPSROOT/bin; export PATH
+else
+   PATH=$DPSROOT/bin:$PATH; export PATH
+fi
+
+if [ -z "${PYTHONPATH}" ]; then
+   PYTHONPATH=$DPSROOT/python; export PYTHONPATH
+else
+   PYTHONPATH=$DPSROOT/python:$PYTHONPATH; export PYTHONPATH
+fi
+
+CONDAINSTALL=/software/miniconda
+PATH=$CONDAINSTALL/bin:$PATH; export PATH
+export ENV_NAME=dps-new
+export CONDA_ENV_PATH=$CONDAINSTALL/envs/${ENV_NAME}
+source $CONDAINSTALL/bin/activate ${ENV_NAME}
+
+unset old_dpsbase
+unset envscript

condor/run.sh

@@ -1,31 +1,18 @@
 #!/bin/bash
-cmssw_version=CMSSW_7_4_7
-git_branch=master
-echo "Setting up ${cmssw_version} ..."
-echo "... sourcing CMS default environment from CVMFS"
-source /cvmfs/cms.cern.ch/cmsset_default.sh
-echo "... creating CMSSW project area"
-scramv1 project CMSSW ${cmssw_version}
-cd ${cmssw_version}/src
-eval `scramv1 runtime -sh`
-echo "${cmssw_version} has been set up"
+git clone https://github.com/BristolTopGroup/DailyPythonScripts
+cd DailyPythonScripts
+source bin/env.sh
 
 echo "Setting up DailyPythonScripts from tar file ..."
 echo "... getting ${git_branch} branch"
 >&2 echo "... getting ${git_branch} branch"
 time git clone https://github.com/BristolTopGroup/DailyPythonScripts.git
 cd DailyPythonScripts/
 git checkout ${git_branch}
-echo "... setting up git submodules"
->&2 echo "... setting up git submodules"
-time git submodule init && git submodule update
 echo "... copying dps.tar from hdfs"
 hadoop fs -copyToLocal $5/dps.tar ${_CONDOR_JOB_IWD}/dps.tar
 echo "... extracting ${_CONDOR_JOB_IWD}/dps.tar on top"
 tar -xf ${_CONDOR_JOB_IWD}/dps.tar --overwrite
-echo "... enforcing conda python environment"
-# this is safe, as the dangerous part is only executed on soolin
-source environment_conda.sh
 echo "DailyPythonScripts are set up"
 
 echo "Running payload"

condor/setup_dps_from_tar.sh

@@ -1,14 +1,12 @@
 #!/bin/bash
 echo "Setting up DailyPythonScripts from tar file ..."
-echo "... getting master branch"
-git clone https://github.com/BristolTopGroup/DailyPythonScripts.git
+echo "... getting ${git_branch} branch"
+>&2 echo "... getting ${git_branch} branch"
+time git clone https://github.com/BristolTopGroup/DailyPythonScripts.git
 cd DailyPythonScripts/
-echo "... setting up git submodules"
-git submodule init && git submodule update
-echo "... running setup routine"
-./setup.sh
-echo "... enforcing virtual python environment"
-source environment.sh
+git checkout ${git_branch}
+echo "... copying dps.tar from hdfs"
+hadoop fs -copyToLocal $5/dps.tar ${_CONDOR_JOB_IWD}/dps.tar
 echo "... extracting ${_CONDOR_JOB_IWD}/dps.tar on top"
 tar -xf ${_CONDOR_JOB_IWD}/dps.tar --overwrite
 echo "DailyPythonScripts are set up"

config/RooUnfold_template.py → config/unfold.py

@@ -4,9 +4,7 @@
 @author: kreczko
 '''
 availablemethods = [
-                    'TUnfold',
-                    'RooUnfoldBayes',
-                    'RooUnfoldSvd',
+                    'TUnfold'
                     ]
 
 SVD_k_value = 5