From e42fc0df47ec7970924c3208a84aac57295c24f9 Mon Sep 17 00:00:00 2001
From: Alexx Perloff <Alexx.Perloff@Colorado.edu>
Date: Fri, 25 Jun 2021 18:12:20 -0500
Subject: [PATCH 1/6] Given the presence of builda, the containerize.sh script
 will create an OCI style image containing the current CMSSW release.

---
 containerize/Dockerfile      |  52 ++++++++++
 containerize/cache.json      |  12 +++
 containerize/containerize.sh | 191 +++++++++++++++++++++++++++++++++++
 3 files changed, 255 insertions(+)
 create mode 100644 containerize/Dockerfile
 create mode 100644 containerize/cache.json
 create mode 100755 containerize/containerize.sh

diff --git a/containerize/Dockerfile b/containerize/Dockerfile
new file mode 100644
index 0000000..0545aa2
--- /dev/null
+++ b/containerize/Dockerfile
@@ -0,0 +1,52 @@
+ARG BUILDIMAGE
+FROM ${BUILDIMAGE} as builder
+
+ARG BUILD_DATE
+ARG ANALYSIS_NAME=analysis
+LABEL   org.label-schema.build-date=$BUILD_DATE \
+        org.label-schema.name="$ANALYSIS_NAME Docker image" \
+        org.label-schema.description="Provides a runnable CMSSW images with $ANALYSIS_NAME pre-installed." \
+        org.label-schema.vendor="FNAL"
+
+USER    cmsusr
+WORKDIR /home/cmsusr
+
+ARG CMSSW_VERSION
+ARG TAR
+
+COPY ${TAR} /home/cmsusr/${CMSSW_VERSION}.tar.gz
+RUN pwd && \
+	ls -alh ./ && \
+	source /cvmfs/cms.cern.ch/cmsset_default.sh && \
+	echo "Unpacking ${CMSSW_VERSION}.tar.gz into ${PWD} ..." && \
+	tar -xzf ${CMSSW_VERSION}.tar.gz && \
+	ls -alh ./ && \
+	rm ${CMSSW_VERSION}.tar.gz && \
+	ls -alh ./ && \
+	cd ${CMSSW_VERSION}/src/ && \
+	scram b ProjectRename && \
+	pwd && \
+	ls -alh ./ && \
+	echo "Setting the CMSSW environment ..." && \
+	eval `scramv1 runtime -sh` && \
+	echo "The CMSSW_BASE is ${CMSSW_BASE}" && \
+	cd ${HOME}
+
+# ---
+
+ARG BASEIMAGE
+FROM ${BASEIMAGE}
+
+ARG BUILD_DATE
+ARG ANALYSIS_NAME
+LABEL   org.label-schema.build-date=$BUILD_DATE \
+        org.label-schema.name="$ANALYSIS_NAME Docker image" \
+        org.label-schema.description="Provides a runnable CMSSW images with $ANALYSIS_NAME pre-installed." \
+        org.label-schema.vendor="FNAL"
+
+ARG CMSSW_VERSION
+#ARG USERNAME=cmsuser
+#COPY --from=builder --chown=${USERNAME}:${USERNAME} /home/cmsusr/${CMSSW_VERSION} /home/${USERNAME}/${CMSSW_VERSION}
+COPY --from=builder --chown=cmsusr:cmsusr /home/cmsusr/${CMSSW_VERSION} /home/cmsusr/${CMSSW_VERSION}
+WORKDIR /home/cmsusr
+RUN pwd && ls -alh ./
diff --git a/containerize/cache.json b/containerize/cache.json
new file mode 100644
index 0000000..d558e71
--- /dev/null
+++ b/containerize/cache.json
@@ -0,0 +1,12 @@
+{
+	"Directories" : [
+		{
+			"Path" : "${CMSSW_BASE}/test",
+			"Cache" : 1
+		},
+		{
+			"Path" : "${CMSSW_BASE}/tmp",
+			"Cache" : 1
+		}
+	]
+}
diff --git a/containerize/containerize.sh b/containerize/containerize.sh
new file mode 100755
index 0000000..171f915
--- /dev/null
+++ b/containerize/containerize.sh
@@ -0,0 +1,191 @@
+#!/bin/bash -e
+
+CWD=${PWD}
+SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+BASE=
+KEEPCACHE="--exclude-caches-all"
+CLEAN="false"
+DIR="/home/cmsusr"
+FILE="Dockerfile"
+JSON="${SCRIPTPATH}/cache.json"
+TAG="analysis"
+TAR=""
+VCS="--exclude-vcs"
+XRDIR=""
+
+usage(){
+    EXIT=$1
+
+    echo "containerize.sh [options]"
+    echo ""
+	echo "-b [base]           the base image to use (default = ${BASE})"
+	echo "-c                  keep the cached files when making the tarball (default = False)"
+	echo "-C                  cleanup the temporary files when finished making the image (default = ${ClEAN})"
+    echo "-d [dir]            project installation area inside the container (default = ${DIR})"
+    echo "-f [file]           the Dockerfile to use to build the image (default = ${FILE})"
+    echo "-j [json]           path to the json file containing the path to cache (default = ${JSON})"
+    echo "-t [tag]            the tag to use for the image (default = ${TAG})"
+	echo "-T [tar]            path to an existing tarball to use (default = ${TAR})"
+	echo "-v                  include the vcs directories (default = False)"
+	echo "-x [XRDIR]          the path to the tarball on an XRootD accessible device (default = ${XRDIR})"
+    echo "-h                  display this message and exit"
+	echo
+	echo "Examples:"
+	echo "./containerize.sh -t <tag> -b docker://docker.io/cmscloud/cc7-cms"
+	echo "podman run --rm -it -v /cvmfs/cms.cern.ch/:/cvmfs/cms.cern.ch/:ro 4a73e8eb3bf5f2a3ab84e2eedfb314e3489529e2b02a464cd7eeb0d2782091ff"
+	echo "./containerize.sh -t <tag> -b docker://docker.io/aperloff/cms-cvmfs-docker:light -C -v"
+	echo "podman run --rm -it <tag>"
+
+    exit $EXIT
+}
+
+# process options
+while getopts "b:cCd:f:j:t:T:vx:h" opt; do
+    case "$opt" in
+	b) BASE=$OPTARG
+	;;
+    c) KEEPCACHE=""
+    ;;
+	C) CLEAN="true"
+    ;;
+    d) DIR=$OPTARG
+    ;;
+    f) FILE=$OPTARG
+    ;;
+    j) JSON=$OPTARG
+    ;;
+    t) TAG=$OPTARG
+    ;;
+    T) TAR=$OPTARG
+    ;;
+	v) VCS=""
+    ;;
+    x) XRDIR=$OPTARG
+    ;;
+    h) usage 0
+    ;;
+    esac
+done
+
+dependency_check() {
+	subuid=$(cat /etc/subuid | grep "^`id -u`:")
+	subgid=$(cat /etc/subgid | grep "^`id -u`:")
+	if [ ! command -v buildah &> /dev/null ]; then
+		EXIT=$?
+		echo "Buildah could not be found!"
+		exit $EXIT
+	elif [[ -z "${subuid}" ]] || [[ -z "${subgid}" ]]; then
+		echo "Unable to find a subuid or subgid for id=`id -u` in /etc/subuid and /etc/subgid."
+		echo "Contact user support or your sysadmin for further assistance."
+	fi
+
+	if [ ! command -v podman &> /dev/null ]; then
+		echo "Podman could not be found. While this is not strictly necessary, you will not be able to create a container from the resulting image."
+	fi
+}
+dependency_check
+
+# cache unneeded files
+CACHEDIR="""\
+Signature: 8a477f597d28d172789f06886806bc55\n
+# This file is a cache directory tag.\n
+# For information about cache directory tags, see:\n
+#       http://www.brynosaurus.com/cachedir/
+"""
+IFS=$'\n'
+list_of_cache_files=()
+# https://stackoverflow.com/questions/1955505/parsing-json-with-unix-tools
+for dir in $(jq -r '.Directories[] | .Path + " " + (.Cache|tostring)' ${JSON}); do
+	IFS=' '
+	dirarray=($dir)
+	path=${dirarray[0]}
+	path=`eval echo ${path}`
+	cache=${dirarray[1]}
+	cache_file=${path}/CACHEDIR.TAG
+	if [[ "${cache}" == "1" ]] && [[ ! -f ${cache_file} ]]; then
+		echo -e "Cache ${path}"
+		echo ${CACHEDIR} > ${cache_file}
+		list_of_cache_files=(${list_of_cache_files[@]} ${cache_file})
+	elif [[ "${cache}" == "1" ]] && [[ -f ${cache_file} ]]; then
+		echo -e "Already cached ${path}"
+	elif [[ "${cache}" == "0" ]] && [[ -f ${cache_file} ]]; then
+		echo -e "Uncache ${path}"
+		rm ${cache_file}
+	elif [[ "${cache}" == "0" ]] && [[ ! -f ${cache_file} ]]; then
+		echo "Already uncached ${path}"
+	fi
+done
+
+# tarball of CMSSW area
+if [ -z "${TAR}" ]; then
+	echo -e "Making the ${CMSSW_VERSION} tarball ... "
+	cd ${CMSSW_BASE}/..
+    tar ${KEEPCACHE} ${VCS} -zcf ${CMSSW_VERSION}.tar.gz -C ${CMSSW_BASE}/.. ${CMSSW_VERSION}
+	TAR="${CMSSW_VERSION}.tar.gz"
+fi
+
+# show the tarball
+if [ -e "${TAR}" ]; then
+	ls -lth ${TAR}
+fi
+
+# transfer the tarball if necessary
+XRD_TAR=""
+CMD_CP=""
+CMD_RM=""
+if [[ ! -z "${XRDIR}" ]]; then
+	if [[ "${XRDIR}" == *"root://"* ]]; then
+        CMD_CP="xrdcp"
+		CMD_RM="xrdrm"
+	elif [[ "${XRDIR}" == *"gsiftp://"* ]]; then
+        CMD_CP="env -i X509_USER_PROXY=${X509_USER_PROXY} gfal-copy"
+		CMD_RM="env -i X509_USER_PROXY=${X509_USER_PROXY} gfal-rm"
+	elif [[ -n "${XRDIR}" ]]; then
+        echo "ERROR Unknown transfer protocol for the tarball"
+        exit 1
+	fi
+
+	if [ -n "$XRDIR" ] && [ -n "$CMD" ]; then
+        ${CMD_CP} -f ${TAR} ${XRDIR}/${CMSSW_VERSION}.tar.gz
+		XRD_TAR=${XRDIR}/${CMSSW_VERSION}.tar.gz
+	fi
+
+fi
+
+# select the correct build image based on the SCRAM_ARCH of the CMSSW release
+ARCH=${SCRAM_ARCH%%_*}
+ARCH_VER=${ARCH: -1}
+BUILDIMAGE=docker://docker.io/cmscloud/
+if [[ "${ARCH_VER}" == "5" ]] || [[ "${ARCH_VER}" == "6" ]]; then
+	BUILDIMAGE=${BUILDIMAGE}slc${ARCH_VER}-cms
+elif [[ "${ARCH_VER}" == "7" ]]; then
+	BUILDIMAGE=${BUILDIMAGE}cc${ARCH_VER}-cms
+else
+	echo -e "Unknown CMSSW architecture version (${SCRAM_ARCH} -- > ${ARCH} -- > ${ARCH_VER}). Defaulting to cc7."
+	BUILDIMAGE=${BUILDIMAGE}cc${ARCH_VER}-cms
+fi
+
+# build the image
+echo -e "Building the image ..."
+buildah --root /scratch/containers/`whoami`/ --runroot /scratch/containers/`whoami`/ bud -f ${FILE} -t ${TAG} -v /cvmfs/cms.cern.ch:/cvmfs/cms.cern.ch \
+		--build-arg BUILDIMAGE=${BUILDIMAGE} --build-arg BASEIMAGE=${BASE} --build-arg BUILD_DATE=`date -u +%Y-%m-%d` --build-arg ANALYSIS_NAME=${NAME} \
+		--build-arg CMSSW_VERSION=${CMSSW_VERSION} --build-arg TAR=`realpath --relative-to="${PWD}" "${TAR}"`
+
+# Cleanup the temporary files
+if [[ "${CLEAN}" == "true" ]]; then
+	echo -e "Cleaning the temporary files made while building the images ..."
+	echo -e "\tRemoving the tarball ${TAR} ..."
+	rm ${TAR}
+	if [[ ! -z "${XRDIR}" ]]; then
+		echo -e "\tRemoving the remote tarball ${XRD_TAR} ..."
+		${CMD_RM} ${XRD_TAR}
+	fi
+	for f in "${list_of_cache_files[@]}"; do
+		echo -e "\tRemoving the cache file ${f} ..."
+		rm ${f}
+	done
+fi
+
+# Return to the original working directory
+cd ${CWD}

From 4b173a302e6f842e2b4c54f095dc6f27ae701531 Mon Sep 17 00:00:00 2001
From: Alexx Perloff <Alexx.Perloff@Colorado.edu>
Date: Fri, 25 Jun 2021 18:22:07 -0500
Subject: [PATCH 2/6] Make sure /cvmfs/cms.cern.ch is mounted.

---
 containerize/containerize.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/containerize/containerize.sh b/containerize/containerize.sh
index 171f915..205f4e7 100755
--- a/containerize/containerize.sh
+++ b/containerize/containerize.sh
@@ -37,7 +37,7 @@ usage(){
 	echo "./containerize.sh -t <tag> -b docker://docker.io/aperloff/cms-cvmfs-docker:light -C -v"
 	echo "podman run --rm -it <tag>"
 
-    exit $EXIT
+    exit ${EXIT}
 }
 
 # process options
@@ -74,10 +74,14 @@ dependency_check() {
 	if [ ! command -v buildah &> /dev/null ]; then
 		EXIT=$?
 		echo "Buildah could not be found!"
-		exit $EXIT
+		exit ${EXIT}
 	elif [[ -z "${subuid}" ]] || [[ -z "${subgid}" ]]; then
 		echo "Unable to find a subuid or subgid for id=`id -u` in /etc/subuid and /etc/subgid."
 		echo "Contact user support or your sysadmin for further assistance."
+	elif [[ ! -d /cvmfs/cms.cern.ch ]] || [ ! cvmfs_config probe cms.cern.ch &> /dev/null ]; then
+		EXIT=$?
+		echo "/cvmfs/cms.cern.ch must be mounted on the host to proceed."
+		exit ${EXIT}
 	fi
 
 	if [ ! command -v podman &> /dev/null ]; then

From d3b8e23aae07134fa3d64a19d7f2847384776bbd Mon Sep 17 00:00:00 2001
From: Alexx Perloff <Alexx.Perloff@Colorado.edu>
Date: Fri, 25 Jun 2021 22:57:58 -0500
Subject: [PATCH 3/6] Some updates after review comments.

---
 containerize/Dockerfile      |  47 +++++-----
 containerize/containerize.sh | 177 +++++++++++++++--------------------
 2 files changed, 97 insertions(+), 127 deletions(-)

diff --git a/containerize/Dockerfile b/containerize/Dockerfile
index 0545aa2..b3abe02 100644
--- a/containerize/Dockerfile
+++ b/containerize/Dockerfile
@@ -3,10 +3,10 @@ FROM ${BUILDIMAGE} as builder
 
 ARG BUILD_DATE
 ARG ANALYSIS_NAME=analysis
-LABEL   org.label-schema.build-date=$BUILD_DATE \
-        org.label-schema.name="$ANALYSIS_NAME Docker image" \
-        org.label-schema.description="Provides a runnable CMSSW images with $ANALYSIS_NAME pre-installed." \
-        org.label-schema.vendor="FNAL"
+LABEL org.label-schema.build-date=$BUILD_DATE \
+      org.label-schema.name="$ANALYSIS_NAME Docker image" \
+      org.label-schema.description="Provides a runnable CMSSW image with $ANALYSIS_NAME pre-installed." \
+      org.label-schema.vendor="FNAL"
 
 USER    cmsusr
 WORKDIR /home/cmsusr
@@ -16,21 +16,20 @@ ARG TAR
 
 COPY ${TAR} /home/cmsusr/${CMSSW_VERSION}.tar.gz
 RUN pwd && \
-	ls -alh ./ && \
-	source /cvmfs/cms.cern.ch/cmsset_default.sh && \
-	echo "Unpacking ${CMSSW_VERSION}.tar.gz into ${PWD} ..." && \
-	tar -xzf ${CMSSW_VERSION}.tar.gz && \
-	ls -alh ./ && \
-	rm ${CMSSW_VERSION}.tar.gz && \
-	ls -alh ./ && \
-	cd ${CMSSW_VERSION}/src/ && \
-	scram b ProjectRename && \
-	pwd && \
-	ls -alh ./ && \
-	echo "Setting the CMSSW environment ..." && \
-	eval `scramv1 runtime -sh` && \
-	echo "The CMSSW_BASE is ${CMSSW_BASE}" && \
-	cd ${HOME}
+    ls -alh ./ && \
+    source /cvmfs/cms.cern.ch/cmsset_default.sh && \
+    echo "Unpacking ${CMSSW_VERSION}.tar.gz into ${PWD} ..." && \
+    tar -xzf ${CMSSW_VERSION}.tar.gz && \
+    rm ${CMSSW_VERSION}.tar.gz && \
+    ls -alh ./ && \
+    cd ${CMSSW_VERSION}/src/ && \
+    scram b ProjectRename && \
+    pwd && \
+    ls -alh ./ && \
+    echo "Setting the CMSSW environment ..." && \
+    eval `scramv1 runtime -sh` && \
+    echo "The CMSSW_BASE is ${CMSSW_BASE}" && \
+    cd ${HOME}
 
 # ---
 
@@ -39,14 +38,12 @@ FROM ${BASEIMAGE}
 
 ARG BUILD_DATE
 ARG ANALYSIS_NAME
-LABEL   org.label-schema.build-date=$BUILD_DATE \
-        org.label-schema.name="$ANALYSIS_NAME Docker image" \
-        org.label-schema.description="Provides a runnable CMSSW images with $ANALYSIS_NAME pre-installed." \
-        org.label-schema.vendor="FNAL"
+LABEL org.label-schema.build-date=$BUILD_DATE \
+      org.label-schema.name="$ANALYSIS_NAME Docker image" \
+      org.label-schema.description="Provides a runnable CMSSW image with $ANALYSIS_NAME pre-installed." \
+      org.label-schema.vendor="FNAL"
 
 ARG CMSSW_VERSION
-#ARG USERNAME=cmsuser
-#COPY --from=builder --chown=${USERNAME}:${USERNAME} /home/cmsusr/${CMSSW_VERSION} /home/${USERNAME}/${CMSSW_VERSION}
 COPY --from=builder --chown=cmsusr:cmsusr /home/cmsusr/${CMSSW_VERSION} /home/cmsusr/${CMSSW_VERSION}
 WORKDIR /home/cmsusr
 RUN pwd && ls -alh ./
diff --git a/containerize/containerize.sh b/containerize/containerize.sh
index 205f4e7..b70dcb7 100755
--- a/containerize/containerize.sh
+++ b/containerize/containerize.sh
@@ -12,42 +12,40 @@ JSON="${SCRIPTPATH}/cache.json"
 TAG="analysis"
 TAR=""
 VCS="--exclude-vcs"
-XRDIR=""
 
 usage(){
     EXIT=$1
 
     echo "containerize.sh [options]"
     echo ""
-	echo "-b [base]           the base image to use (default = ${BASE})"
-	echo "-c                  keep the cached files when making the tarball (default = False)"
-	echo "-C                  cleanup the temporary files when finished making the image (default = ${ClEAN})"
+    echo "-b [base]           the base image to use (default = ${BASE})"
+    echo "-c                  keep the cached files when making the tarball (default = False)"
+    echo "-C                  cleanup the temporary files when finished making the image (default = ${CLEAN})"
     echo "-d [dir]            project installation area inside the container (default = ${DIR})"
     echo "-f [file]           the Dockerfile to use to build the image (default = ${FILE})"
     echo "-j [json]           path to the json file containing the path to cache (default = ${JSON})"
     echo "-t [tag]            the tag to use for the image (default = ${TAG})"
-	echo "-T [tar]            path to an existing tarball to use (default = ${TAR})"
-	echo "-v                  include the vcs directories (default = False)"
-	echo "-x [XRDIR]          the path to the tarball on an XRootD accessible device (default = ${XRDIR})"
+    echo "-T [tar]            path to an existing tarball to use (default = ${TAR})"
+    echo "-v                  include the vcs directories (default = False)"
     echo "-h                  display this message and exit"
-	echo
-	echo "Examples:"
-	echo "./containerize.sh -t <tag> -b docker://docker.io/cmscloud/cc7-cms"
-	echo "podman run --rm -it -v /cvmfs/cms.cern.ch/:/cvmfs/cms.cern.ch/:ro 4a73e8eb3bf5f2a3ab84e2eedfb314e3489529e2b02a464cd7eeb0d2782091ff"
-	echo "./containerize.sh -t <tag> -b docker://docker.io/aperloff/cms-cvmfs-docker:light -C -v"
-	echo "podman run --rm -it <tag>"
+    echo
+    echo "Examples:"
+    echo "./containerize.sh -t <name>:<tag> -b docker://docker.io/cmscloud/cc7-cms"
+    echo "podman run --rm -it -v /cvmfs/cms.cern.ch/:/cvmfs/cms.cern.ch/:ro <name>:<tag>"
+    echo "./containerize.sh -t <name>:<tag> -b docker://docker.io/aperloff/cms-cvmfs-docker:light -C -v"
+    echo "podman run --rm -it <name>:<tag>"
 
     exit ${EXIT}
 }
 
 # process options
-while getopts "b:cCd:f:j:t:T:vx:h" opt; do
+while getopts "b:cCd:f:j:t:T:vh" opt; do
     case "$opt" in
-	b) BASE=$OPTARG
-	;;
+    b) BASE=$OPTARG
+    ;;
     c) KEEPCACHE=""
     ;;
-	C) CLEAN="true"
+    C) CLEAN="true"
     ;;
     d) DIR=$OPTARG
     ;;
@@ -59,9 +57,7 @@ while getopts "b:cCd:f:j:t:T:vx:h" opt; do
     ;;
     T) TAR=$OPTARG
     ;;
-	v) VCS=""
-    ;;
-    x) XRDIR=$OPTARG
+    v) VCS=""
     ;;
     h) usage 0
     ;;
@@ -69,92 +65,73 @@ while getopts "b:cCd:f:j:t:T:vx:h" opt; do
 done
 
 dependency_check() {
-	subuid=$(cat /etc/subuid | grep "^`id -u`:")
-	subgid=$(cat /etc/subgid | grep "^`id -u`:")
-	if [ ! command -v buildah &> /dev/null ]; then
-		EXIT=$?
-		echo "Buildah could not be found!"
-		exit ${EXIT}
-	elif [[ -z "${subuid}" ]] || [[ -z "${subgid}" ]]; then
-		echo "Unable to find a subuid or subgid for id=`id -u` in /etc/subuid and /etc/subgid."
-		echo "Contact user support or your sysadmin for further assistance."
-	elif [[ ! -d /cvmfs/cms.cern.ch ]] || [ ! cvmfs_config probe cms.cern.ch &> /dev/null ]; then
-		EXIT=$?
-		echo "/cvmfs/cms.cern.ch must be mounted on the host to proceed."
-		exit ${EXIT}
-	fi
-
-	if [ ! command -v podman &> /dev/null ]; then
-		echo "Podman could not be found. While this is not strictly necessary, you will not be able to create a container from the resulting image."
-	fi
+    subuid=$(cat /etc/subuid | grep "^`id -u`:")
+    subgid=$(cat /etc/subgid | grep "^`id -u`:")
+    if [ ! command -v buildah &> /dev/null ]; then
+        EXIT=$?
+        echo "Buildah could not be found!"
+        exit ${EXIT}
+    elif [ ! command -v jq &> /dev/null ];then
+        EXIT=$?
+        echo "jq could not be found!"
+        exit ${EXIT}
+    elif [[ -z "${subuid}" ]] || [[ -z "${subgid}" ]]; then
+        echo "Unable to find a subuid or subgid for id=`id -u` in /etc/subuid and /etc/subgid."
+        echo "Contact user support or your sysadmin for further assistance."
+    elif [[ ! -d /cvmfs/cms.cern.ch ]] || [ ! cvmfs_config probe cms.cern.ch &> /dev/null ]; then
+        EXIT=$?
+        echo "/cvmfs/cms.cern.ch must be mounted on the host to proceed."
+        exit ${EXIT}
+    fi
+
+    if [ ! command -v podman &> /dev/null ]; then
+        echo "Podman could not be found. While this is not strictly necessary, you will not be able to create a container from the resulting image."
+    fi
 }
 dependency_check
 
 # cache unneeded files
-CACHEDIR="""\
+CACHEDIR=" \
 Signature: 8a477f597d28d172789f06886806bc55\n
 # This file is a cache directory tag.\n
 # For information about cache directory tags, see:\n
 #       http://www.brynosaurus.com/cachedir/
-"""
+"
 IFS=$'\n'
 list_of_cache_files=()
 # https://stackoverflow.com/questions/1955505/parsing-json-with-unix-tools
 for dir in $(jq -r '.Directories[] | .Path + " " + (.Cache|tostring)' ${JSON}); do
-	IFS=' '
-	dirarray=($dir)
-	path=${dirarray[0]}
-	path=`eval echo ${path}`
-	cache=${dirarray[1]}
-	cache_file=${path}/CACHEDIR.TAG
-	if [[ "${cache}" == "1" ]] && [[ ! -f ${cache_file} ]]; then
-		echo -e "Cache ${path}"
-		echo ${CACHEDIR} > ${cache_file}
-		list_of_cache_files=(${list_of_cache_files[@]} ${cache_file})
-	elif [[ "${cache}" == "1" ]] && [[ -f ${cache_file} ]]; then
-		echo -e "Already cached ${path}"
-	elif [[ "${cache}" == "0" ]] && [[ -f ${cache_file} ]]; then
-		echo -e "Uncache ${path}"
-		rm ${cache_file}
-	elif [[ "${cache}" == "0" ]] && [[ ! -f ${cache_file} ]]; then
-		echo "Already uncached ${path}"
-	fi
+    IFS=' '
+    dirarray=($dir)
+    path=${dirarray[0]}
+    path=`eval echo ${path}`
+    cache=${dirarray[1]}
+    cache_file=${path}/CACHEDIR.TAG
+    if [[ "${cache}" == "1" ]] && [[ ! -f ${cache_file} ]]; then
+        echo -e "Cache ${path}"
+        echo ${CACHEDIR} > ${cache_file}
+        list_of_cache_files=(${list_of_cache_files[@]} ${cache_file})
+    elif [[ "${cache}" == "1" ]] && [[ -f ${cache_file} ]]; then
+        echo -e "Already cached ${path}"
+    elif [[ "${cache}" == "0" ]] && [[ -f ${cache_file} ]]; then
+        echo -e "Uncache ${path}"
+        rm ${cache_file}
+    elif [[ "${cache}" == "0" ]] && [[ ! -f ${cache_file} ]]; then
+        echo "Already uncached ${path}"
+    fi
 done
 
 # tarball of CMSSW area
 if [ -z "${TAR}" ]; then
-	echo -e "Making the ${CMSSW_VERSION} tarball ... "
-	cd ${CMSSW_BASE}/..
+    echo -e "Making the ${CMSSW_VERSION} tarball ... "
+    cd ${CMSSW_BASE}/..
     tar ${KEEPCACHE} ${VCS} -zcf ${CMSSW_VERSION}.tar.gz -C ${CMSSW_BASE}/.. ${CMSSW_VERSION}
-	TAR="${CMSSW_VERSION}.tar.gz"
+    TAR="${CMSSW_VERSION}.tar.gz"
 fi
 
 # show the tarball
 if [ -e "${TAR}" ]; then
-	ls -lth ${TAR}
-fi
-
-# transfer the tarball if necessary
-XRD_TAR=""
-CMD_CP=""
-CMD_RM=""
-if [[ ! -z "${XRDIR}" ]]; then
-	if [[ "${XRDIR}" == *"root://"* ]]; then
-        CMD_CP="xrdcp"
-		CMD_RM="xrdrm"
-	elif [[ "${XRDIR}" == *"gsiftp://"* ]]; then
-        CMD_CP="env -i X509_USER_PROXY=${X509_USER_PROXY} gfal-copy"
-		CMD_RM="env -i X509_USER_PROXY=${X509_USER_PROXY} gfal-rm"
-	elif [[ -n "${XRDIR}" ]]; then
-        echo "ERROR Unknown transfer protocol for the tarball"
-        exit 1
-	fi
-
-	if [ -n "$XRDIR" ] && [ -n "$CMD" ]; then
-        ${CMD_CP} -f ${TAR} ${XRDIR}/${CMSSW_VERSION}.tar.gz
-		XRD_TAR=${XRDIR}/${CMSSW_VERSION}.tar.gz
-	fi
-
+    ls -lth ${TAR}
 fi
 
 # select the correct build image based on the SCRAM_ARCH of the CMSSW release
@@ -162,33 +139,29 @@ ARCH=${SCRAM_ARCH%%_*}
 ARCH_VER=${ARCH: -1}
 BUILDIMAGE=docker://docker.io/cmscloud/
 if [[ "${ARCH_VER}" == "5" ]] || [[ "${ARCH_VER}" == "6" ]]; then
-	BUILDIMAGE=${BUILDIMAGE}slc${ARCH_VER}-cms
+    BUILDIMAGE=${BUILDIMAGE}slc${ARCH_VER}-cms
 elif [[ "${ARCH_VER}" == "7" ]]; then
-	BUILDIMAGE=${BUILDIMAGE}cc${ARCH_VER}-cms
+    BUILDIMAGE=${BUILDIMAGE}cc${ARCH_VER}-cms
 else
-	echo -e "Unknown CMSSW architecture version (${SCRAM_ARCH} -- > ${ARCH} -- > ${ARCH_VER}). Defaulting to cc7."
-	BUILDIMAGE=${BUILDIMAGE}cc${ARCH_VER}-cms
+    echo -e "Unknown CMSSW architecture version (${SCRAM_ARCH} -- > ${ARCH} -- > ${ARCH_VER}). Defaulting to cc7."
+    BUILDIMAGE=${BUILDIMAGE}cc${ARCH_VER}-cms
 fi
 
 # build the image
 echo -e "Building the image ..."
 buildah --root /scratch/containers/`whoami`/ --runroot /scratch/containers/`whoami`/ bud -f ${FILE} -t ${TAG} -v /cvmfs/cms.cern.ch:/cvmfs/cms.cern.ch \
-		--build-arg BUILDIMAGE=${BUILDIMAGE} --build-arg BASEIMAGE=${BASE} --build-arg BUILD_DATE=`date -u +%Y-%m-%d` --build-arg ANALYSIS_NAME=${NAME} \
-		--build-arg CMSSW_VERSION=${CMSSW_VERSION} --build-arg TAR=`realpath --relative-to="${PWD}" "${TAR}"`
+        --build-arg BUILDIMAGE=${BUILDIMAGE} --build-arg BASEIMAGE=${BASE} --build-arg BUILD_DATE=`date -u +%Y-%m-%d` --build-arg ANALYSIS_NAME=${NAME} \
+        --build-arg CMSSW_VERSION=${CMSSW_VERSION} --build-arg TAR=`realpath --relative-to="${PWD}" "${TAR}"`
 
 # Cleanup the temporary files
 if [[ "${CLEAN}" == "true" ]]; then
-	echo -e "Cleaning the temporary files made while building the images ..."
-	echo -e "\tRemoving the tarball ${TAR} ..."
-	rm ${TAR}
-	if [[ ! -z "${XRDIR}" ]]; then
-		echo -e "\tRemoving the remote tarball ${XRD_TAR} ..."
-		${CMD_RM} ${XRD_TAR}
-	fi
-	for f in "${list_of_cache_files[@]}"; do
-		echo -e "\tRemoving the cache file ${f} ..."
-		rm ${f}
-	done
+    echo -e "Cleaning the temporary files made while building the images ..."
+    echo -e "\tRemoving the tarball ${TAR} ..."
+    rm ${TAR}
+    for f in "${list_of_cache_files[@]}"; do
+        echo -e "\tRemoving the cache file ${f} ..."
+        rm ${f}
+    done
 fi
 
 # Return to the original working directory

From a4bbe3ee6c3ccb9e57823ae1eb534a070d9be423 Mon Sep 17 00:00:00 2001
From: Alexx Perloff <Alexx.Perloff@Colorado.edu>
Date: Mon, 28 Jun 2021 14:47:59 -0500
Subject: [PATCH 4/6] Add options to change buildah root and runroot
 directories. Also add option to override default username in container and
 some changes to the Dockerfile for sustainability.

---
 containerize/Dockerfile      | 11 ++++++-----
 containerize/containerize.sh | 18 +++++++++++++++---
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/containerize/Dockerfile b/containerize/Dockerfile
index b3abe02..50dd545 100644
--- a/containerize/Dockerfile
+++ b/containerize/Dockerfile
@@ -8,13 +8,13 @@ LABEL org.label-schema.build-date=$BUILD_DATE \
       org.label-schema.description="Provides a runnable CMSSW image with $ANALYSIS_NAME pre-installed." \
       org.label-schema.vendor="FNAL"
 
-USER    cmsusr
-WORKDIR /home/cmsusr
+USER cmsusr
+WORKDIR /home/${USER}
 
 ARG CMSSW_VERSION
 ARG TAR
 
-COPY ${TAR} /home/cmsusr/${CMSSW_VERSION}.tar.gz
+COPY ${TAR} /home/${USER}/${CMSSW_VERSION}.tar.gz
 RUN pwd && \
     ls -alh ./ && \
     source /cvmfs/cms.cern.ch/cmsset_default.sh && \
@@ -44,6 +44,7 @@ LABEL org.label-schema.build-date=$BUILD_DATE \
       org.label-schema.vendor="FNAL"
 
 ARG CMSSW_VERSION
-COPY --from=builder --chown=cmsusr:cmsusr /home/cmsusr/${CMSSW_VERSION} /home/cmsusr/${CMSSW_VERSION}
-WORKDIR /home/cmsusr
+ARG NONPRIVILEGED_USER=cmsusr
+WORKDIR /home/${NONPRIVILEGED_USER}
+COPY --from=builder --chown=cmsusr:cmsusr /home/cmsusr/${CMSSW_VERSION} /home/${NONPRIVILEGED_USER}/${CMSSW_VERSION}
 RUN pwd && ls -alh ./
diff --git a/containerize/containerize.sh b/containerize/containerize.sh
index b70dcb7..d1a640a 100755
--- a/containerize/containerize.sh
+++ b/containerize/containerize.sh
@@ -9,8 +9,10 @@ CLEAN="false"
 DIR="/home/cmsusr"
 FILE="Dockerfile"
 JSON="${SCRIPTPATH}/cache.json"
+ROOT="/scratch/containers/`whoami`/"
 TAG="analysis"
 TAR=""
+USER="cmsusr"
 VCS="--exclude-vcs"
 
 usage(){
@@ -24,8 +26,10 @@ usage(){
     echo "-d [dir]            project installation area inside the container (default = ${DIR})"
     echo "-f [file]           the Dockerfile to use to build the image (default = ${FILE})"
     echo "-j [json]           path to the json file containing the path to cache (default = ${JSON})"
+    echo "-r [root]           change the 'root' and 'runroot' locations for buildah (default = ${ROOT})"
     echo "-t [tag]            the tag to use for the image (default = ${TAG})"
     echo "-T [tar]            path to an existing tarball to use (default = ${TAR})"
+    echo "-u [user]           override the default username in the container (default = ${USER})"
     echo "-v                  include the vcs directories (default = False)"
     echo "-h                  display this message and exit"
     echo
@@ -39,7 +43,7 @@ usage(){
 }
 
 # process options
-while getopts "b:cCd:f:j:t:T:vh" opt; do
+while getopts "b:cCd:f:j:r:t:T:u:vh" opt; do
     case "$opt" in
     b) BASE=$OPTARG
     ;;
@@ -53,10 +57,14 @@ while getopts "b:cCd:f:j:t:T:vh" opt; do
     ;;
     j) JSON=$OPTARG
     ;;
+    r) ROOT=$OPTARG
+    ;;
     t) TAG=$OPTARG
     ;;
     T) TAR=$OPTARG
     ;;
+    u) USER=$OPTARG
+    ;;
     v) VCS=""
     ;;
     h) usage 0
@@ -75,6 +83,10 @@ dependency_check() {
         EXIT=$?
         echo "jq could not be found!"
         exit ${EXIT}
+    elif [[ ! -d ${ROOT} ]]; then
+        EXIT=$?
+        echo "The directory ${ROOT} does not exist and cannot be used for buildah's 'root' or 'runroot' directory."
+        exit ${EXIT}
     elif [[ -z "${subuid}" ]] || [[ -z "${subgid}" ]]; then
         echo "Unable to find a subuid or subgid for id=`id -u` in /etc/subuid and /etc/subgid."
         echo "Contact user support or your sysadmin for further assistance."
@@ -149,9 +161,9 @@ fi
 
 # build the image
 echo -e "Building the image ..."
-buildah --root /scratch/containers/`whoami`/ --runroot /scratch/containers/`whoami`/ bud -f ${FILE} -t ${TAG} -v /cvmfs/cms.cern.ch:/cvmfs/cms.cern.ch \
+buildah --root ${ROOT} --runroot ${ROOT} bud -f ${FILE} -t ${TAG} -v /cvmfs/cms.cern.ch:/cvmfs/cms.cern.ch \
         --build-arg BUILDIMAGE=${BUILDIMAGE} --build-arg BASEIMAGE=${BASE} --build-arg BUILD_DATE=`date -u +%Y-%m-%d` --build-arg ANALYSIS_NAME=${NAME} \
-        --build-arg CMSSW_VERSION=${CMSSW_VERSION} --build-arg TAR=`realpath --relative-to="${PWD}" "${TAR}"`
+        --build-arg CMSSW_VERSION=${CMSSW_VERSION} --build-arg TAR=`realpath --relative-to="${PWD}" "${TAR}"` --build-arg NONPRIVILEGED_USER=${USER}
 
 # Cleanup the temporary files
 if [[ "${CLEAN}" == "true" ]]; then

From d2a42ccf523fbd0987ac1a109640011c4d743293 Mon Sep 17 00:00:00 2001
From: Alexx Perloff <Alexx.Perloff@Colorado.edu>
Date: Mon, 28 Jun 2021 15:08:27 -0500
Subject: [PATCH 5/6] More sustainability changes.

---
 containerize/Dockerfile | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/containerize/Dockerfile b/containerize/Dockerfile
index 50dd545..a6f2712 100644
--- a/containerize/Dockerfile
+++ b/containerize/Dockerfile
@@ -9,12 +9,13 @@ LABEL org.label-schema.build-date=$BUILD_DATE \
       org.label-schema.vendor="FNAL"
 
 USER cmsusr
-WORKDIR /home/${USER}
+ARG USERDIR_BUILD=/home/${USER}
+WORKDIR ${USERDIR_BUILD}
 
 ARG CMSSW_VERSION
 ARG TAR
 
-COPY ${TAR} /home/${USER}/${CMSSW_VERSION}.tar.gz
+COPY ${TAR} ${USERDIR_BUILD}/${CMSSW_VERSION}.tar.gz
 RUN pwd && \
     ls -alh ./ && \
     source /cvmfs/cms.cern.ch/cmsset_default.sh && \
@@ -45,6 +46,7 @@ LABEL org.label-schema.build-date=$BUILD_DATE \
 
 ARG CMSSW_VERSION
 ARG NONPRIVILEGED_USER=cmsusr
-WORKDIR /home/${NONPRIVILEGED_USER}
-COPY --from=builder --chown=cmsusr:cmsusr /home/cmsusr/${CMSSW_VERSION} /home/${NONPRIVILEGED_USER}/${CMSSW_VERSION}
+ARG USERDIR=/home/${NONPRIVILEGED_USER}
+WORKDIR ${USERDIR}
+COPY --from=builder --chown=cmsusr:cmsusr /home/cmsusr/${CMSSW_VERSION} ${USERDIR}/${CMSSW_VERSION}
 RUN pwd && ls -alh ./

From 9431c84ff6d92962f4ddfb102a0a1458635b00d8 Mon Sep 17 00:00:00 2001
From: Alexx Perloff <Alexx.Perloff@Colorado.edu>
Date: Mon, 28 Jun 2021 15:31:48 -0500
Subject: [PATCH 6/6] Partially mitigate #19.

---
 containerize/Dockerfile | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/containerize/Dockerfile b/containerize/Dockerfile
index a6f2712..7bf1d60 100644
--- a/containerize/Dockerfile
+++ b/containerize/Dockerfile
@@ -8,8 +8,9 @@ LABEL org.label-schema.build-date=$BUILD_DATE \
       org.label-schema.description="Provides a runnable CMSSW image with $ANALYSIS_NAME pre-installed." \
       org.label-schema.vendor="FNAL"
 
-USER cmsusr
-ARG USERDIR_BUILD=/home/${USER}
+ARG USER_BUILD=cmsusr
+ARG USERDIR_BUILD=/home/${USER_BUILD}
+USER ${USER_BUILD}
 WORKDIR ${USERDIR_BUILD}
 
 ARG CMSSW_VERSION
@@ -45,8 +46,11 @@ LABEL org.label-schema.build-date=$BUILD_DATE \
       org.label-schema.vendor="FNAL"
 
 ARG CMSSW_VERSION
+ARG USER_BUILD=cmsusr
 ARG NONPRIVILEGED_USER=cmsusr
+ARG USERDIR_BUILD=/home/${USER_BUILD}
 ARG USERDIR=/home/${NONPRIVILEGED_USER}
 WORKDIR ${USERDIR}
-COPY --from=builder --chown=cmsusr:cmsusr /home/cmsusr/${CMSSW_VERSION} ${USERDIR}/${CMSSW_VERSION}
+
+COPY --from=builder --chown=cmsusr:cmsusr ${USERDIR_BUILD}/${CMSSW_VERSION} ${USERDIR}/${CMSSW_VERSION}
 RUN pwd && ls -alh ./