From 5f90dcb24283537d1a9b052b3459333c5b169101 Mon Sep 17 00:00:00 2001
From: Khushnaseeb Ali <khushnaseebali@MacBook-Pro-3.home>
Date: Thu, 21 Jun 2018 00:31:07 -0400
Subject: [PATCH 1/8] First draft for Singularity unpacker

---
 .../Singularity_multiple_runs.py              | 135 ++++++++++++++++++
 1 file changed, 135 insertions(+)
 create mode 100644 reprounzip-singularity/Singularity_multiple_runs.py

diff --git a/reprounzip-singularity/Singularity_multiple_runs.py b/reprounzip-singularity/Singularity_multiple_runs.py
new file mode 100644
index 000000000..709370f67
--- /dev/null
+++ b/reprounzip-singularity/Singularity_multiple_runs.py
@@ -0,0 +1,135 @@
+# Singularity unpacker for reprozip 
+import subprocess
+import os
+import sys
+import re
+from shutil import copyfile
+import yaml
+
+#TODO:
+# 1. Logging error for every bash command
+# 2. Using existing reprozip utility functions for common tasks like copying busy box
+# 3. Comments
+# 4. integrating with exisitng unpacking coding design
+# 5. Improving code structure
+
+def extract_reprozip_file(filename):
+    bashCommand = " tar -xf {}".format(filename)
+    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+    output, error = process.communicate()
+    data_file="DATA.tar.gz"
+    if data_file:
+        bashCommand = " tar -xf {}".format(data_file)
+        process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+        output, error = process.communicate()
+
+args = sys.argv[1:]
+filename, IMAGE_DIR = args
+
+if not os.path.exists(IMAGE_DIR):
+    os.makedirs(IMAGE_DIR)
+os.chdir(IMAGE_DIR)
+filename = "../"+filename
+extract_reprozip_file(filename)
+
+SINGULARITY_DIR = "DATA/.singularity.d" 
+run_env_file="90-environment.sh"
+apps_file="95-apps.sh"
+base_file="99-base.sh"
+ENV_DIR = SINGULARITY_DIR+"/env"
+SHELL_DIR = "DATA/bin"
+
+def make_singularity_directories():
+    if os.path.exists("DATA"):
+        root_path = 'DATA'
+        folders = ['proc','dev','sys']
+        for folder in folders:
+            os.mkdir(os.path.join(root_path,folder))
+        folders = ['actions','libs','env']
+        os.mkdir(SINGULARITY_DIR)
+        for folder in folders:
+            os.mkdir(os.path.join(SINGULARITY_DIR,folder)) 
+        singularoty_files = ["labels.json","runscript","startscript"]
+        for file in singularoty_files:
+            open(os.path.join(SINGULARITY_DIR,file), 'a').close()
+
+make_singularity_directories()
+
+def copy_action_files():
+    for file in os.listdir("../../singularitd_files/actions/"):
+        bashCommand = "cp ../../singularitd_files/actions/{} DATA/.singularity.d/actions/".format(file)
+        process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+        output, error = process.communicate()
+
+copy_action_files()
+
+def write_env_file(env,env_file):
+    env_file = os.path.join(ENV_DIR, env_file)
+    with open(env_file, 'w+') as f:
+        for key,value in env.items():
+            f.write(key+"='"+value+"'\n")
+    bashCommand = "chmod +x {}".format(env_file)
+    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+    output, error = process.communicate()
+
+def make_environment_file():
+    env_files = [run_env_file,apps_file,base_file]
+    for file in env_files:
+        open(os.path.join(ENV_DIR,file), 'a').close()
+    #Write the environment file from config.yml file
+    source_config="METADATA/config.yml"
+    my_dict = yaml.load(open(source_config))
+    runs = my_dict.get('runs')
+    if len(runs) > 1:
+        # create one env file for each run:
+        for run in runs:
+            filename = run['id']+"_env.sh"
+            write_env_file(run.get('environ'), filename)
+    else:
+        print(len(runs))
+        write_env_file(runs[0].get('environ'),run_env_file)
+
+make_environment_file()
+
+def make_runscript():
+    source_config="METADATA/config.yml"
+    my_dict = yaml.load(open(source_config))
+    runs = my_dict['runs']
+    cmd="#!/bin/sh \n"
+    for run in runs:
+        binary = run['binary']
+        workingdir = run ['workingdir']
+        run_file = run['argv'][1]
+        if len(runs)>1:
+            cmd += "source /.singularity.d/env/{}_env.sh \n".format(run['id'])
+        cmd +="cd {0}\nexec {1} {2}\n".format(workingdir,binary,run_file)
+    with open(os.path.join(SINGULARITY_DIR, "runscript"), 'w') as f:
+        f.write(cmd)
+    bashCommand = "chmod +x {}/runscript".format(SINGULARITY_DIR)
+    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+    output, error = process.communicate() 
+
+make_runscript()
+
+
+def copy_busybox():
+    if not os.path.exists(SHELL_DIR):
+    	os.makedirs(SHELL_DIR)
+    if not os.path.isfile(os.path.join(SHELL_DIR,"sh")):
+	print("no sh in bin")
+	bashCommand = "cp  ../bin/sh {}".format(SHELL_DIR)
+	process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+    	output, error = process.communicate()
+
+
+copy_busybox()
+
+def run_singularity_image():
+    home = os.environ['HOME']
+    print(home)
+    bashCommand = "singularity run -C -H {}:/something  DATA".format(home)
+    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+    output, error = process.communicate()
+    print(output)
+
+run_singularity_image()

From 93e221ba7855b80fba27f064fcaccf4fe0493d45 Mon Sep 17 00:00:00 2001
From: Khushnaseeb Ali <khushnaseebali@MacBook-Pro-3.home>
Date: Tue, 3 Jul 2018 13:28:28 -0400
Subject: [PATCH 2/8] Singualrity unpacker in tar image format

---
 reprounzip-singularity/Singularity_tar.py | 148 ++++++++++++++++++++++
 1 file changed, 148 insertions(+)
 create mode 100644 reprounzip-singularity/Singularity_tar.py

diff --git a/reprounzip-singularity/Singularity_tar.py b/reprounzip-singularity/Singularity_tar.py
new file mode 100644
index 000000000..f6edb5813
--- /dev/null
+++ b/reprounzip-singularity/Singularity_tar.py
@@ -0,0 +1,148 @@
+import copy
+import tarfile
+import os
+import sys
+import yaml
+import subprocess
+
+# Open outer tar, the RPZ file
+SINGULARITY_DIR = "../../.singularity.d"
+RUN_ENV_FILE="90-environment.sh"
+ENV_DIR = SINGULARITY_DIR+"/env"
+OVERLAY_IMAGE = "repro_overlay.img"
+IMAGE_TAR_FILE = "new.tar.gz"
+
+
+
+def write_env_file(env,env_file):
+    env_file = os.path.join(ENV_DIR, env_file)
+    with open(env_file, 'w+') as f:
+        for key,value in env.items():
+            f.write(key+"='"+value+"'\n")
+    bashCommand = "chmod +x {}".format(env_file)
+    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+    output, error = process.communicate()
+
+
+def make_environment_file():
+    #Write the environment file from config.yml file
+    source_config="METADATA/config.yml"
+    my_dict = yaml.load(open(source_config))
+    runs = my_dict.get('runs')
+    if len(runs) > 1:
+        # create one env file for each run:
+        for run in runs:
+            filename = run['id']+"_env.sh"
+            write_env_file(run.get('environ'), filename)
+    else:
+        write_env_file(runs[0].get('environ'),RUN_ENV_FILE)
+
+
+def make_runscript():
+    source_config="METADATA/config.yml"
+    my_dict = yaml.load(open(source_config))
+    runs = my_dict['runs']
+    cmd = ''
+    for run in runs:
+        binary = run['binary']
+        workingdir = run ['workingdir']
+        run_file = run['argv'][1]
+        if len(runs)>1:
+            cmd += "source /.singularity.d/env/{}_env.sh \n".format(run['id'])
+        cmd +="cd {0}\n{1} {2}\n".format(workingdir,binary,run_file)
+    with open(os.path.join(SINGULARITY_DIR, "runscript"), 'w') as f:
+        f.write(cmd)
+    bashCommand = "chmod +x {}/runscript".format(SINGULARITY_DIR)
+    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+    output, error = process.communicate()
+
+
+# Check if bin is present in the tar if not add bin and sh
+def copy_busybox(tar):
+    if not "bin" in tar.getnames():
+        print("bin is absent")
+        tar.add("../../bin",arcname="bin")
+    elif not "sh" in tar.getnames():
+        print("bin but no sh")
+        tar.add("../../bin/sh", arcname='bin/sh')
+
+
+def setup_singularity_image(filename):
+	rpz = tarfile.open(filename, 'r:*')
+
+	# Open the inner tar in the original, without extracting it to disk
+	data = rpz.extractfile('DATA.tar.gz')
+	tar = tarfile.open('DATA.tar.gz', fileobj=data)
+
+	# Open the new tar we're writing
+	new = tarfile.open('new.tar.gz', 'w:gz')
+	recursive = True
+	# For each member of the data tar
+	for info in tar.getmembers():
+	    # Make a new TarInfo, removing the DATA/ prefix from the file name
+	    new_info = copy.copy(info)
+	    new_info.name = info.name[5:]
+
+	    if new_info.name:
+	        # Copy the file from the inner tar to the new tar
+	        if new_info.isreg():
+	        	new.addfile(new_info, tar.extractfile(info.name))
+	        	# with tarfile.open(info.name, "rb") as f:
+	        	#     new.addfile(new_info, f)
+
+	        elif new_info.isdir():
+	        	new.addfile(new_info)
+	        	# if recursive:
+	        	#     for f in os.listdir(tar.extractfile(info.name)):
+	        	#     	new.add(os.path.join(info.name, f), os.path.join('', f),recursive)
+	        else:
+	        	new.addfile(new_info)
+	# Add the missing folders - proc,run, sys and temp_home
+	folders = ['proc','dev','sys','temp_home']
+	for folder in folders:
+    	new.add("../../missing_folders/"+folder,folder)
+
+
+    rpz.extractall()
+    make_environment_file()
+	make_runscript()
+	copy_busybox(new)
+    new.add(SINGULARITY_DIR,arcname=".singularity.d")
+	tar.close()
+	data.close()
+	rpz.close()
+	new.close()
+
+
+
+def create_overlay_image(OVERLAY_IMAGE):
+    bashCommand = "singularity image.create {}".format(OVERLAY_IMAGE)
+    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+    output, error = process.communicate()
+    print(output)
+    return error
+
+
+def run_singularity_image(IMAGE_TAR_FILE):
+    home = os.environ['HOME']
+    if not create_overlay_image(OVERLAY_IMAGE):
+        bashCommand = "singularity run  --overlay {0} -C -H {1}:/temp_home {2}".format(OVERLAY_IMAGE,home,IMAGE_TAR_FILE)
+        process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+        output, error = process.communicate()
+        #copy the output from the overlay to some dir and destroy the overlay later
+        print(output)
+
+
+
+
+args = sys.argv[1:]
+filename, IMAGE_DIR = args
+
+if not os.path.exists(IMAGE_DIR):
+    os.makedirs(IMAGE_DIR)
+os.chdir(IMAGE_DIR)
+filename = "../"+filename
+setup_singularity_image(filename)
+run_singularity_image(IMAGE_TAR_FILE)
+
+

From fc0b38bea3df12fe1575a99c6b07bc417817a0ed Mon Sep 17 00:00:00 2001
From: Khushnaseeb Ali <khushnaseebali@172-16-18-73.dynapool.nyu.edu>
Date: Mon, 16 Jul 2018 12:58:20 -0400
Subject: [PATCH 3/8] Unpacker with uplad, download and app functionality

---
 .../Singularity_unpacker.py                   | 236 ++++++++++++++++++
 1 file changed, 236 insertions(+)
 create mode 100644 reprounzip-singularity/Singularity_unpacker.py

diff --git a/reprounzip-singularity/Singularity_unpacker.py b/reprounzip-singularity/Singularity_unpacker.py
new file mode 100644
index 000000000..38480bfbd
--- /dev/null
+++ b/reprounzip-singularity/Singularity_unpacker.py
@@ -0,0 +1,236 @@
+import copy
+import tarfile
+import os
+import sys
+import yaml
+import subprocess
+
+# Open outer tar, the RPZ file
+SINGULARITY_DIR = "../../.singularity.d"
+RUN_ENV_FILE="90-environment.sh"
+ENV_DIR = SINGULARITY_DIR+"/env"
+OVERLAY_IMAGE = "repro_overlay.img"
+IMAGE_TAR_FILE = "new.tar.gz"
+MOUNT_DIR = "/mnt/"
+source_config="METADATA/config.yml"
+APP_DIR = "../../app_structure/sample_app"
+SCIF_DIR = "../../app_structure/scif"
+APP_ENV_DIR = APP_DIR+"/scif/env"
+
+
+def write_env_file(env,env_file,apps=False):
+    if apps:
+        env_file = os.path.join(APP_ENV_DIR, env_file)
+    else:
+        env_file = os.path.join(ENV_DIR, env_file)
+    with open(env_file, 'w+') as f:
+        for key,value in env.items():
+            f.write(key+"='"+value+"'\n")
+    bashCommand = "chmod +x {}".format(env_file)
+    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+    output, error = process.communicate()
+
+  
+
+def make_runscript(workingdir, binary, run_file, apps=False):
+    cmd = "cd {0}\n{1} {2}".format(workingdir, binary, run_file)
+    run_cmd = "run()\n{\n echo \"Nothing to run. Run Apps!!\"\n}\n\n"
+    if apps:
+        with open(os.path.join(APP_DIR+"/scif","runscript"), 'w') as f:
+            f.write(cmd)
+    else:
+        run_cmd = "run()\n"
+        run_cmd += "{\n\t" + cmd + "\n}\n"
+    upload_download_cmd = "upload()\n{\n"
+    upload_download_cmd += "\tcp " + MOUNT_DIR + "\"$1\" " + workingdir + "\n}\n\n"
+    upload_download_cmd += "download()\n{\n"
+    upload_download_cmd += "\tcp {0}/".format(workingdir) + "\"$1\" " + MOUNT_DIR + "\n}\n\n"
+    upload_download_cmd += "if [ \"$#\" -gt 0 ]\nthen \n \tcase \"$1\" in \n\t\t\"upload\"|\"download\")\n\t\t\t\"$1\" \"$2\" \n\t\t;;\n\t\t*)\n\t\techo \"Invalid arguments!!\" >&2\n\t\texit 1\n\t\t;;\n\tesac\nelse\n\trun\nfi"
+    with open(os.path.join(SINGULARITY_DIR, "runscript"), 'w') as f:
+	f.write(run_cmd)
+        f.write(upload_download_cmd)
+
+def make_main_app_base_script(app_name):
+    print("rached 94")
+    cmd =  "SCIF_APPDATA_{0}=/scif/data/{1}\n".format(app_name,app_name)
+    cmd += "SCIF_APPMETA_{0}=/scif/apps/{1}/scif\n".format(app_name,app_name)
+    cmd += "SCIF_APPROOT_{0}=/scif/apps/{1}\n".format(app_name,app_name)
+    cmd += "SCIF_APPBIN_{0}=/scif/apps/{1}/bin\n".format(app_name,app_name)
+    cmd += "SCIF_APPLIB_{0}=/scif/apps/{1}/lib\n".format(app_name,app_name)
+    cmd += "export SCIF_APPDATA_{0} SCIF_APPROOT_{1} SCIF_APPMETA_{2} SCIF_APPBIN_{3} SCIF_APPLIB_{4}\n".format(app_name,app_name,app_name,app_name,app_name)
+    cmd += "SCIF_APPENV_{0}=/scif/apps/{1}/scif/env/90-environment.sh\n".format(app_name,app_name)
+    cmd += "export SCIF_APPENV_{0}\n".format(app_name)
+    cmd += "SCIF_APPLABELS_{0}=/scif/apps/{1}/scif/labels.json\n".format(app_name,app_name)
+    cmd += "export SCIF_APPLABELS_{0}\n".format(app_name)
+    cmd += "SCIF_APPRUN_{0}=/scif/apps/{1}/scif/runscript\n".format(app_name,app_name)
+    cmd += "export SCIF_APPRUN_{0}\n".format(app_name)
+    with open(os.path.join(ENV_DIR, "94-appsbase.sh"), 'a') as f:
+        f.write(cmd)
+
+
+def make_app_specific_base_script(app_name):
+    cmd = "SCIF_APPNAME={}\n".format(app_name)
+    cmd += "SCIF_APPROOT=\"/scif/apps/{}\"\n".format(app_name)
+    cmd += "SCIF_APPMETA=\"/scif/apps/{}/scif\"\n".format(app_name)
+    cmd += "SCIF_DATA=\"/scif/data\"\n"
+    cmd += "SCIF_APPDATA=\"/scif/data/{}\"\n".format(app_name)
+    cmd += "SCIF_APPINPUT=\"/scif/data/{}/input\"\n".format(app_name)
+    cmd += "SCIF_APPOUTPUT=\"/scif/data/{}/output\"\n".format(app_name)
+    cmd += "export SCIF_APPDATA SCIF_APPNAME SCIF_APPROOT SCIF_APPMETA SCIF_APPINPUT SCIF_APPOUTPUT SCIF_DATA\n"
+    with open(os.path.join(APP_ENV_DIR, "01-base.sh"), 'w') as f:
+        f.write(cmd)
+
+# Check if bin is present in the tar if not add bin and sh
+def copy_busybox(tar):
+    if not "bin" in tar.getnames():
+        print("bin is absent")
+        tar.add("../../bin",arcname="bin")
+    elif not "sh" in tar.getnames():
+        print("bin but no sh")
+        tar.add("../../bin/sh", arcname='bin/sh')
+
+
+def deleteContent(fName):
+    with open(fName, "w"):
+        pass
+
+def add_singularity_folder(tar):
+    # check if the runs are multiple or single
+    my_dict = yaml.load(open(source_config))
+    runs = my_dict['runs']
+    copy_busybox(tar)
+    deleteContent(os.path.join(ENV_DIR, "94-appsbase.sh"))
+    if len(runs) > 1:
+	print("multiple runs")
+        # App structure is present
+        # First add the SCIF folder 
+        tar.add(SCIF_DIR, arcname="scif")
+        for run in runs:
+            binary = run['binary']
+            workingdir = run['workingdir']
+            run_file = run['argv'][1]
+            app_name = run['id']
+            make_runscript(workingdir, binary, run_file, apps=True)
+            write_env_file(runs[0].get('environ'), RUN_ENV_FILE, apps=True)
+            make_app_specific_base_script(app_name)
+            make_main_app_base_script(app_name)
+            # add the modified app.
+            tar.add(APP_DIR, arcname="scif/apps/" + app_name)
+
+    else:
+        write_env_file(runs[0].get('environ'),RUN_ENV_FILE)
+        make_runscript(runs[0]['workingdir'], runs[0]['binary'], runs[0]['argv'][1])
+    tar.add(SINGULARITY_DIR, arcname=".singularity.d")
+    
+
+
+def setup(filename):
+    rpz = tarfile.open(filename, 'r:*')
+
+    # Open the inner tar in the original, without extracting it to disk
+    data = rpz.extractfile('DATA.tar.gz')
+    tar = tarfile.open('DATA.tar.gz', fileobj=data)
+
+    # Open the new tar we're writing
+    new = tarfile.open('new.tar.gz', 'w:gz')
+    # For each member of the data tar
+    for info in tar.getmembers():
+        # Make a new TarInfo, removing the DATA/ prefix from the file name
+        new_info = copy.copy(info)
+        new_info.name = info.name[5:]
+        if new_info.name:
+            # Copy the file from the inner tar to the new tar
+            if new_info.isreg():
+                new.addfile(new_info, tar.extractfile(info.name))
+            elif new_info.isdir():
+                new.addfile(new_info)
+            else:
+                new.addfile(new_info)
+    # Add the missing folders - proc,run, sys and temp_home
+    folders = ['proc','dev','sys','temp_home','mnt']
+    for folder in folders:
+        new.add("../../missing_folders/"+folder,folder)
+    print("added missing folder")
+
+    rpz.extractall()
+    add_singularity_folder(new)
+    print("added singularioty folder")
+    tar.close()
+    data.close()
+    rpz.close()
+    new.close()
+
+
+
+def create_overlay_image(OVERLAY_IMAGE):
+    if not os.path.exists(OVERLAY_IMAGE):
+    	bashCommand = "singularity image.create {}".format(OVERLAY_IMAGE)
+    	process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+    	output, error = process.communicate()
+    	print(output)
+    	return error
+    return 0
+
+
+
+def run(IMAGE_TAR_FILE,app):
+    home = os.environ['HOME']
+    if not create_overlay_image(OVERLAY_IMAGE):
+        if app:
+            print("running app:{}!".format(app))
+            bashCommand = "singularity run  --overlay {0} --app {1} -C -H {2}:/temp_home {3}".format(OVERLAY_IMAGE,app,home,IMAGE_TAR_FILE)
+            process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+            output, error = process.communicate()
+	else:
+	    # get the name of all the apps
+            my_dict = yaml.load(open(source_config))
+            runs = my_dict['runs']
+            for run in runs:	
+		app = run['id']    
+            	print("running app:{}!".format(app))
+            	bashCommand = "singularity run  --overlay {0} --app {1} -C -H {2}:/temp_home {3}".format(OVERLAY_IMAGE,app,home,IMAGE_TAR_FILE)
+            	process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+            	output, error = process.communicate()
+
+def download(IMAGE_TAR_FILE,filename):
+    home = os.environ['HOME']
+    current_dir = os.getcwd()
+    bashCommand = "singularity run  -B {0}:{1} --overlay {2} -C -H {3}:/temp_home {4} download {5}".format(current_dir,MOUNT_DIR,OVERLAY_IMAGE, home, IMAGE_TAR_FILE, filename)
+    print(bashCommand)
+    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+    output, error = process.communicate()
+
+
+def upload(IMAGE_TAR_FILE, filename):
+    home = os.environ['HOME']
+    current_dir = os.getcwd()
+    bashCommand = "singularity run  -B {0}:{1} --overlay {2} -C -H {3}:/temp_home {4} upload {5}".format(current_dir,MOUNT_DIR,OVERLAY_IMAGE, home, IMAGE_TAR_FILE, filename)
+    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+    output, error = process.communicate()
+
+
+args = sys.argv[1:]
+cmd = args[0]
+print(cmd)
+if cmd=="setup":
+    rpz_file, IMAGE_DIR = args[1:]
+    if not os.path.exists(IMAGE_DIR):
+        os.makedirs(IMAGE_DIR)
+    os.chdir(IMAGE_DIR)
+    rpz_file = "../" + rpz_file
+    setup(rpz_file)
+elif cmd=="run":
+    IMAGE_DIR = args[1]
+    app = args[2] 
+    os.chdir(IMAGE_DIR)
+    print(app)
+    run(IMAGE_TAR_FILE,app)
+elif cmd in ["upload","download"]:
+    IMAGE_DIR,filename = args[1:]
+    os.chdir(IMAGE_DIR)
+    print(IMAGE_DIR)
+    if not filename:
+        print("output file missing!")
+    globals()[cmd](IMAGE_TAR_FILE,filename)
+
+

From 17ffa505be3a4c92675216bf68a16010a023808c Mon Sep 17 00:00:00 2001
From: Khushnaseeb Ali <khushnaseebali@172-16-19-152.dynapool.nyu.edu>
Date: Mon, 30 Jul 2018 14:05:27 -0400
Subject: [PATCH 4/8] Code review-1  changes

---
 .../.singularity.d/Singularity                |  60 ++++
 .../.singularity.d/actions/exec               |   9 +
 .../.singularity.d/actions/run                |  23 ++
 .../.singularity.d/actions/shell              |  27 ++
 .../.singularity.d/actions/start              |  15 +
 .../.singularity.d/actions/test               |  23 ++
 .../.singularity.d/env/95-apps.sh             |  52 +++
 .../.singularity.d/startscript                |   1 +
 .../Singularity_multiple_runs.py              | 135 --------
 reprounzip-singularity/Singularity_tar.py     | 148 ---------
 .../Singularity_unpacker.py                   | 302 ++++++++++--------
 .../sample_app/scif/Singularity               |   3 +
 .../sample_app/scif/runscript                 |   2 +
 13 files changed, 387 insertions(+), 413 deletions(-)
 create mode 100644 reprounzip-singularity/.singularity.d/Singularity
 create mode 100755 reprounzip-singularity/.singularity.d/actions/exec
 create mode 100755 reprounzip-singularity/.singularity.d/actions/run
 create mode 100755 reprounzip-singularity/.singularity.d/actions/shell
 create mode 100755 reprounzip-singularity/.singularity.d/actions/start
 create mode 100755 reprounzip-singularity/.singularity.d/actions/test
 create mode 100755 reprounzip-singularity/.singularity.d/env/95-apps.sh
 create mode 100755 reprounzip-singularity/.singularity.d/startscript
 delete mode 100644 reprounzip-singularity/Singularity_multiple_runs.py
 delete mode 100644 reprounzip-singularity/Singularity_tar.py
 create mode 100644 reprounzip-singularity/sample_app/scif/Singularity
 create mode 100755 reprounzip-singularity/sample_app/scif/runscript

diff --git a/reprounzip-singularity/.singularity.d/Singularity b/reprounzip-singularity/.singularity.d/Singularity
new file mode 100644
index 000000000..092317621
--- /dev/null
+++ b/reprounzip-singularity/.singularity.d/Singularity
@@ -0,0 +1,60 @@
+Bootstrap: docker
+From: ubuntu:16.04
+
+# =======================
+# global
+# =======================
+%post
+    apt-get -y update
+
+%environment
+    export LC_ALL=C
+    export PATH=/usr/games:$PATH
+
+# =======================
+# fortune
+# =======================
+%appinstall fortune
+    apt-get -y install fortune
+
+%appenv fortune
+    BEST_APP=fortune
+    export BEST_APP
+
+%apphelp fortune
+    fortune is the best app
+
+%apprun fortune
+    fortune "$@"
+
+# =======================
+# cowsay
+# =======================
+%appinstall cowsay
+    apt-get -y install cowsay
+
+%appenv cowsay
+    BEST_APP=cowsay
+    export BEST_APP
+
+%apphelp cowsay
+    cowsay is the best app
+
+%apprun cowsay
+    cowsay "$@"
+
+# =======================
+# lolcat
+# =======================
+%appinstall lolcat
+    apt-get -y install lolcat
+
+%appenv lolcat
+    BEST_APP=lolcat
+    export BEST_APP
+
+%apphelp lolcat
+    lolcat is the best app
+
+%apprun lolcat
+    lolcat "$@"
diff --git a/reprounzip-singularity/.singularity.d/actions/exec b/reprounzip-singularity/.singularity.d/actions/exec
new file mode 100755
index 000000000..b2e51f7a9
--- /dev/null
+++ b/reprounzip-singularity/.singularity.d/actions/exec
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+for script in /.singularity.d/env/*.sh; do
+    if [ -f "$script" ]; then
+        . "$script"
+    fi
+done
+
+exec "$@"
diff --git a/reprounzip-singularity/.singularity.d/actions/run b/reprounzip-singularity/.singularity.d/actions/run
new file mode 100755
index 000000000..71a985b75
--- /dev/null
+++ b/reprounzip-singularity/.singularity.d/actions/run
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+for script in /.singularity.d/env/*.sh; do
+    if [ -f "$script" ]; then
+        . "$script"
+    fi
+done
+
+if test -n "${SINGULARITY_APPNAME:-}"; then
+
+    if test -x "/scif/apps/${SINGULARITY_APPNAME:-}/scif/runscript"; then
+        exec "/scif/apps/${SINGULARITY_APPNAME:-}/scif/runscript" "$@"
+    else
+        echo "No Singularity runscript for contained app: ${SINGULARITY_APPNAME:-}"
+        exit 1
+    fi
+
+elif test -x "/.singularity.d/runscript"; then
+    exec "/.singularity.d/runscript" "$@"
+else
+    echo "No Singularity runscript found, executing /bin/sh"
+    exec /bin/sh "$@"
+fi
diff --git a/reprounzip-singularity/.singularity.d/actions/shell b/reprounzip-singularity/.singularity.d/actions/shell
new file mode 100755
index 000000000..1e78e59fa
--- /dev/null
+++ b/reprounzip-singularity/.singularity.d/actions/shell
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+for script in /.singularity.d/env/*.sh; do
+    if [ -f "$script" ]; then
+        . "$script"
+    fi
+done
+
+if test -n "$SINGULARITY_SHELL" -a -x "$SINGULARITY_SHELL"; then
+    exec $SINGULARITY_SHELL "$@"
+
+    echo "ERROR: Failed running shell as defined by '\$SINGULARITY_SHELL'" 1>&2
+    exit 1
+
+elif test -x /bin/bash; then
+    SHELL=/bin/bash
+    PS1="Singularity $SINGULARITY_CONTAINER:\\w> "
+    export SHELL PS1
+    exec /bin/bash --norc "$@"
+elif test -x /bin/sh; then
+    SHELL=/bin/sh
+    export SHELL
+    exec /bin/sh "$@"
+else
+    echo "ERROR: /bin/sh does not exist in container" 1>&2
+fi
+exit 1
diff --git a/reprounzip-singularity/.singularity.d/actions/start b/reprounzip-singularity/.singularity.d/actions/start
new file mode 100755
index 000000000..577dc01b7
--- /dev/null
+++ b/reprounzip-singularity/.singularity.d/actions/start
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# if we are here start notify PID 1 to continue
+# DON'T REMOVE
+kill -CONT 1
+
+for script in /.singularity.d/env/*.sh; do
+    if [ -f "$script" ]; then
+        . "$script"
+    fi
+done
+
+if test -x "/.singularity.d/startscript"; then
+    exec "/.singularity.d/startscript"
+fi
diff --git a/reprounzip-singularity/.singularity.d/actions/test b/reprounzip-singularity/.singularity.d/actions/test
new file mode 100755
index 000000000..cdab47bc0
--- /dev/null
+++ b/reprounzip-singularity/.singularity.d/actions/test
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+for script in /.singularity.d/env/*.sh; do
+    if [ -f "$script" ]; then
+        . "$script"
+    fi
+done
+
+
+if test -n "${SINGULARITY_APPNAME:-}"; then
+
+    if test -x "/scif/apps/${SINGULARITY_APPNAME:-}/scif/test"; then
+        exec "/scif/apps/${SINGULARITY_APPNAME:-}/scif/test" "$@"
+    else
+        echo "No Singularity tests for contained app: ${SINGULARITY_APPNAME:-}"
+        exit 1
+    fi
+elif test -x "/.singularity.d/test"; then
+    exec "/.singularity.d/test" "$@"
+else
+    echo "No Singularity container test found, executing /bin/sh"
+    exec /bin/sh "$@"
+fi
diff --git a/reprounzip-singularity/.singularity.d/env/95-apps.sh b/reprounzip-singularity/.singularity.d/env/95-apps.sh
new file mode 100755
index 000000000..1c74eb890
--- /dev/null
+++ b/reprounzip-singularity/.singularity.d/env/95-apps.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Copyright (c) 2017-2018, SyLabs, Inc. All rights reserved.
+# Copyright (c) 2017, SingularityWare, LLC. All rights reserved.
+#
+# See the COPYRIGHT.md file at the top-level directory of this distribution and at
+# https://github.com/singularityware/singularity/blob/master/COPYRIGHT.md.
+#
+# This file is part of the Singularity Linux container project. It is subject to the license
+# terms in the LICENSE.md file found in the top-level directory of this distribution and
+# at https://github.com/singularityware/singularity/blob/master/LICENSE.md. No part
+# of Singularity, including this file, may be copied, modified, propagated, or distributed
+# except according to the terms contained in the LICENSE.md file.
+
+
+if test -n "${SINGULARITY_APPNAME:-}"; then
+
+    # The active app should be exported
+    export SINGULARITY_APPNAME
+
+    if test -d "/scif/apps/${SINGULARITY_APPNAME:-}/"; then
+        SCIF_APPS="/scif/apps"
+        SCIF_APPROOT="/scif/apps/${SINGULARITY_APPNAME:-}"
+        export SCIF_APPROOT SCIF_APPS
+        PATH="/scif/apps/${SINGULARITY_APPNAME:-}:$PATH"
+
+        # Automatically add application bin to path
+        if test -d "/scif/apps/${SINGULARITY_APPNAME:-}/bin"; then
+            PATH="/scif/apps/${SINGULARITY_APPNAME:-}/bin:$PATH"
+        fi
+
+        # Automatically add application lib to LD_LIBRARY_PATH
+        if test -d "/scif/apps/${SINGULARITY_APPNAME:-}/lib"; then
+            LD_LIBRARY_PATH="/scif/apps/${SINGULARITY_APPNAME:-}/lib:$LD_LIBRARY_PATH"
+            export LD_LIBRARY_PATH
+        fi
+
+        # Automatically source environment
+        if [ -f "/scif/apps/${SINGULARITY_APPNAME:-}/scif/env/01-base.sh" ]; then
+            . "/scif/apps/${SINGULARITY_APPNAME:-}/scif/env/01-base.sh"
+        fi
+        if [ -f "/scif/apps/${SINGULARITY_APPNAME:-}/scif/env/90-environment.sh" ]; then
+            . "/scif/apps/${SINGULARITY_APPNAME:-}/scif/env/90-environment.sh"
+        fi
+
+        export PATH
+    else
+        echo "Could not locate the container application: ${SINGULARITY_APPNAME}"
+        exit 1
+    fi
+fi
+
diff --git a/reprounzip-singularity/.singularity.d/startscript b/reprounzip-singularity/.singularity.d/startscript
new file mode 100755
index 000000000..1a2485251
--- /dev/null
+++ b/reprounzip-singularity/.singularity.d/startscript
@@ -0,0 +1 @@
+#!/bin/sh
diff --git a/reprounzip-singularity/Singularity_multiple_runs.py b/reprounzip-singularity/Singularity_multiple_runs.py
deleted file mode 100644
index 709370f67..000000000
--- a/reprounzip-singularity/Singularity_multiple_runs.py
+++ /dev/null
@@ -1,135 +0,0 @@
-# Singularity unpacker for reprozip 
-import subprocess
-import os
-import sys
-import re
-from shutil import copyfile
-import yaml
-
-#TODO:
-# 1. Logging error for every bash command
-# 2. Using existing reprozip utility functions for common tasks like copying busy box
-# 3. Comments
-# 4. integrating with exisitng unpacking coding design
-# 5. Improving code structure
-
-def extract_reprozip_file(filename):
-    bashCommand = " tar -xf {}".format(filename)
-    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-    output, error = process.communicate()
-    data_file="DATA.tar.gz"
-    if data_file:
-        bashCommand = " tar -xf {}".format(data_file)
-        process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-        output, error = process.communicate()
-
-args = sys.argv[1:]
-filename, IMAGE_DIR = args
-
-if not os.path.exists(IMAGE_DIR):
-    os.makedirs(IMAGE_DIR)
-os.chdir(IMAGE_DIR)
-filename = "../"+filename
-extract_reprozip_file(filename)
-
-SINGULARITY_DIR = "DATA/.singularity.d" 
-run_env_file="90-environment.sh"
-apps_file="95-apps.sh"
-base_file="99-base.sh"
-ENV_DIR = SINGULARITY_DIR+"/env"
-SHELL_DIR = "DATA/bin"
-
-def make_singularity_directories():
-    if os.path.exists("DATA"):
-        root_path = 'DATA'
-        folders = ['proc','dev','sys']
-        for folder in folders:
-            os.mkdir(os.path.join(root_path,folder))
-        folders = ['actions','libs','env']
-        os.mkdir(SINGULARITY_DIR)
-        for folder in folders:
-            os.mkdir(os.path.join(SINGULARITY_DIR,folder)) 
-        singularoty_files = ["labels.json","runscript","startscript"]
-        for file in singularoty_files:
-            open(os.path.join(SINGULARITY_DIR,file), 'a').close()
-
-make_singularity_directories()
-
-def copy_action_files():
-    for file in os.listdir("../../singularitd_files/actions/"):
-        bashCommand = "cp ../../singularitd_files/actions/{} DATA/.singularity.d/actions/".format(file)
-        process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-        output, error = process.communicate()
-
-copy_action_files()
-
-def write_env_file(env,env_file):
-    env_file = os.path.join(ENV_DIR, env_file)
-    with open(env_file, 'w+') as f:
-        for key,value in env.items():
-            f.write(key+"='"+value+"'\n")
-    bashCommand = "chmod +x {}".format(env_file)
-    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-    output, error = process.communicate()
-
-def make_environment_file():
-    env_files = [run_env_file,apps_file,base_file]
-    for file in env_files:
-        open(os.path.join(ENV_DIR,file), 'a').close()
-    #Write the environment file from config.yml file
-    source_config="METADATA/config.yml"
-    my_dict = yaml.load(open(source_config))
-    runs = my_dict.get('runs')
-    if len(runs) > 1:
-        # create one env file for each run:
-        for run in runs:
-            filename = run['id']+"_env.sh"
-            write_env_file(run.get('environ'), filename)
-    else:
-        print(len(runs))
-        write_env_file(runs[0].get('environ'),run_env_file)
-
-make_environment_file()
-
-def make_runscript():
-    source_config="METADATA/config.yml"
-    my_dict = yaml.load(open(source_config))
-    runs = my_dict['runs']
-    cmd="#!/bin/sh \n"
-    for run in runs:
-        binary = run['binary']
-        workingdir = run ['workingdir']
-        run_file = run['argv'][1]
-        if len(runs)>1:
-            cmd += "source /.singularity.d/env/{}_env.sh \n".format(run['id'])
-        cmd +="cd {0}\nexec {1} {2}\n".format(workingdir,binary,run_file)
-    with open(os.path.join(SINGULARITY_DIR, "runscript"), 'w') as f:
-        f.write(cmd)
-    bashCommand = "chmod +x {}/runscript".format(SINGULARITY_DIR)
-    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-    output, error = process.communicate() 
-
-make_runscript()
-
-
-def copy_busybox():
-    if not os.path.exists(SHELL_DIR):
-    	os.makedirs(SHELL_DIR)
-    if not os.path.isfile(os.path.join(SHELL_DIR,"sh")):
-	print("no sh in bin")
-	bashCommand = "cp  ../bin/sh {}".format(SHELL_DIR)
-	process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-    	output, error = process.communicate()
-
-
-copy_busybox()
-
-def run_singularity_image():
-    home = os.environ['HOME']
-    print(home)
-    bashCommand = "singularity run -C -H {}:/something  DATA".format(home)
-    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-    output, error = process.communicate()
-    print(output)
-
-run_singularity_image()
diff --git a/reprounzip-singularity/Singularity_tar.py b/reprounzip-singularity/Singularity_tar.py
deleted file mode 100644
index f6edb5813..000000000
--- a/reprounzip-singularity/Singularity_tar.py
+++ /dev/null
@@ -1,148 +0,0 @@
-import copy
-import tarfile
-import os
-import sys
-import yaml
-import subprocess
-
-# Open outer tar, the RPZ file
-SINGULARITY_DIR = "../../.singularity.d"
-RUN_ENV_FILE="90-environment.sh"
-ENV_DIR = SINGULARITY_DIR+"/env"
-OVERLAY_IMAGE = "repro_overlay.img"
-IMAGE_TAR_FILE = "new.tar.gz"
-
-
-
-def write_env_file(env,env_file):
-    env_file = os.path.join(ENV_DIR, env_file)
-    with open(env_file, 'w+') as f:
-        for key,value in env.items():
-            f.write(key+"='"+value+"'\n")
-    bashCommand = "chmod +x {}".format(env_file)
-    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-    output, error = process.communicate()
-
-
-def make_environment_file():
-    #Write the environment file from config.yml file
-    source_config="METADATA/config.yml"
-    my_dict = yaml.load(open(source_config))
-    runs = my_dict.get('runs')
-    if len(runs) > 1:
-        # create one env file for each run:
-        for run in runs:
-            filename = run['id']+"_env.sh"
-            write_env_file(run.get('environ'), filename)
-    else:
-        write_env_file(runs[0].get('environ'),RUN_ENV_FILE)
-
-
-def make_runscript():
-    source_config="METADATA/config.yml"
-    my_dict = yaml.load(open(source_config))
-    runs = my_dict['runs']
-    cmd = ''
-    for run in runs:
-        binary = run['binary']
-        workingdir = run ['workingdir']
-        run_file = run['argv'][1]
-        if len(runs)>1:
-            cmd += "source /.singularity.d/env/{}_env.sh \n".format(run['id'])
-        cmd +="cd {0}\n{1} {2}\n".format(workingdir,binary,run_file)
-    with open(os.path.join(SINGULARITY_DIR, "runscript"), 'w') as f:
-        f.write(cmd)
-    bashCommand = "chmod +x {}/runscript".format(SINGULARITY_DIR)
-    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-    output, error = process.communicate()
-
-
-# Check if bin is present in the tar if not add bin and sh
-def copy_busybox(tar):
-    if not "bin" in tar.getnames():
-        print("bin is absent")
-        tar.add("../../bin",arcname="bin")
-    elif not "sh" in tar.getnames():
-        print("bin but no sh")
-        tar.add("../../bin/sh", arcname='bin/sh')
-
-
-def setup_singularity_image(filename):
-	rpz = tarfile.open(filename, 'r:*')
-
-	# Open the inner tar in the original, without extracting it to disk
-	data = rpz.extractfile('DATA.tar.gz')
-	tar = tarfile.open('DATA.tar.gz', fileobj=data)
-
-	# Open the new tar we're writing
-	new = tarfile.open('new.tar.gz', 'w:gz')
-	recursive = True
-	# For each member of the data tar
-	for info in tar.getmembers():
-	    # Make a new TarInfo, removing the DATA/ prefix from the file name
-	    new_info = copy.copy(info)
-	    new_info.name = info.name[5:]
-
-	    if new_info.name:
-	        # Copy the file from the inner tar to the new tar
-	        if new_info.isreg():
-	        	new.addfile(new_info, tar.extractfile(info.name))
-	        	# with tarfile.open(info.name, "rb") as f:
-	        	#     new.addfile(new_info, f)
-
-	        elif new_info.isdir():
-	        	new.addfile(new_info)
-	        	# if recursive:
-	        	#     for f in os.listdir(tar.extractfile(info.name)):
-	        	#     	new.add(os.path.join(info.name, f), os.path.join('', f),recursive)
-	        else:
-	        	new.addfile(new_info)
-	# Add the missing folders - proc,run, sys and temp_home
-	folders = ['proc','dev','sys','temp_home']
-	for folder in folders:
-    	new.add("../../missing_folders/"+folder,folder)
-
-
-    rpz.extractall()
-    make_environment_file()
-	make_runscript()
-	copy_busybox(new)
-    new.add(SINGULARITY_DIR,arcname=".singularity.d")
-	tar.close()
-	data.close()
-	rpz.close()
-	new.close()
-
-
-
-def create_overlay_image(OVERLAY_IMAGE):
-    bashCommand = "singularity image.create {}".format(OVERLAY_IMAGE)
-    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-    output, error = process.communicate()
-    print(output)
-    return error
-
-
-def run_singularity_image(IMAGE_TAR_FILE):
-    home = os.environ['HOME']
-    if not create_overlay_image(OVERLAY_IMAGE):
-        bashCommand = "singularity run  --overlay {0} -C -H {1}:/temp_home {2}".format(OVERLAY_IMAGE,home,IMAGE_TAR_FILE)
-        process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-        output, error = process.communicate()
-        #copy the output from the overlay to some dir and destroy the overlay later
-        print(output)
-
-
-
-
-args = sys.argv[1:]
-filename, IMAGE_DIR = args
-
-if not os.path.exists(IMAGE_DIR):
-    os.makedirs(IMAGE_DIR)
-os.chdir(IMAGE_DIR)
-filename = "../"+filename
-setup_singularity_image(filename)
-run_singularity_image(IMAGE_TAR_FILE)
-
-
diff --git a/reprounzip-singularity/Singularity_unpacker.py b/reprounzip-singularity/Singularity_unpacker.py
index 38480bfbd..a2ceb2f8b 100644
--- a/reprounzip-singularity/Singularity_unpacker.py
+++ b/reprounzip-singularity/Singularity_unpacker.py
@@ -4,54 +4,84 @@
 import sys
 import yaml
 import subprocess
+import io
+import string
 
-# Open outer tar, the RPZ file
-SINGULARITY_DIR = "../../.singularity.d"
-RUN_ENV_FILE="90-environment.sh"
-ENV_DIR = SINGULARITY_DIR+"/env"
-OVERLAY_IMAGE = "repro_overlay.img"
-IMAGE_TAR_FILE = "new.tar.gz"
+SINGULARITY_DIR = "../.singularity.d"
+APP_DIR = "../sample_app"
 MOUNT_DIR = "/mnt/"
-source_config="METADATA/config.yml"
-APP_DIR = "../../app_structure/sample_app"
-SCIF_DIR = "../../app_structure/scif"
-APP_ENV_DIR = APP_DIR+"/scif/env"
+RUN_ENV_FILE = "90-environment.sh"
+APP_BASE_FILE = "01-base.sh"
+MAIN_APP_BASE_FILE = ".singularity.d/env/94-appsbase.sh"
+RUNSCRIPT = "runscript"
+IMAGE_TAR_FILE = "new.tar.gz"
+SOURCE_CONFIG = "METADATA/config.yml"
+OVERLAY_IMAGE = "repro_overlay.img"
 
+safe_shell_chars = set("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                       "abcdefghijklmnopqrstuvwxyz"
+                       "0123456789"
+                       "-+=/:.,%_")
 
-def write_env_file(env,env_file,apps=False):
-    if apps:
-        env_file = os.path.join(APP_ENV_DIR, env_file)
-    else:
-        env_file = os.path.join(ENV_DIR, env_file)
-    with open(env_file, 'w+') as f:
-        for key,value in env.items():
-            f.write(key+"='"+value+"'\n")
-    bashCommand = "chmod +x {}".format(env_file)
-    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-    output, error = process.communicate()
-
-  
-
-def make_runscript(workingdir, binary, run_file, apps=False):
-    cmd = "cd {0}\n{1} {2}".format(workingdir, binary, run_file)
-    run_cmd = "run()\n{\n echo \"Nothing to run. Run Apps!!\"\n}\n\n"
-    if apps:
-        with open(os.path.join(APP_DIR+"/scif","runscript"), 'w') as f:
-            f.write(cmd)
+def sanitize_appname(name):
+    name = name.strip()
+    valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
+    valid_name = ''.join(c for c in name if c in valid_chars)
+    valid_name = valid_name.replace(' ','_')
+    return valid_name
+
+def shell_escape(s):
+    """Given bl"a, returns "bl\\"a".
+    """
+    if isinstance(s, bytes):
+        s = s.decode('utf-8')
+    if not s or any(c not in safe_shell_chars for c in s):
+        return '"%s"' % (s.replace('\\', '\\\\')
+                          .replace('"', '\\"')
+                          .replace('`', '\\`')
+                          .replace('$', '\\$'))
     else:
-        run_cmd = "run()\n"
-        run_cmd += "{\n\t" + cmd + "\n}\n"
+        return s
+
+
+def write_env_file(tar, env, env_file, app_name=None):
+    cmd = ''
+    for key,value in env.items():
+            cmd += shell_escape(key)+ "='" + shell_escape(value)+"'\n"
+   
+    env_file = "scif/apps/"+app_name+"/scif/env/" + env_file
+    cmd = cmd.encode('utf-8')
+    t = tarfile.TarInfo(env_file)
+    t.size = len(cmd)
+    t.mode = 0o755
+    tar.addfile(t, io.BytesIO(cmd))
+        
+def make_app_runscript(tar, cmd, app_name=None):
+    file = "scif/apps/"+app_name+"/scif/"+ RUNSCRIPT
+    cmd = cmd.encode("utf-8")
+    t = tarfile.TarInfo(file)
+    t.mode=0o755
+    t.size = len(cmd)
+    tar.addfile(t, io.BytesIO(cmd))
+
+   
+def make_runscript(tar, cmd, workingdir):
+    run_cmd = "run()\n"
+    run_cmd += "{"+ cmd + "\n}\n\n"
     upload_download_cmd = "upload()\n{\n"
     upload_download_cmd += "\tcp " + MOUNT_DIR + "\"$1\" " + workingdir + "\n}\n\n"
     upload_download_cmd += "download()\n{\n"
     upload_download_cmd += "\tcp {0}/".format(workingdir) + "\"$1\" " + MOUNT_DIR + "\n}\n\n"
     upload_download_cmd += "if [ \"$#\" -gt 0 ]\nthen \n \tcase \"$1\" in \n\t\t\"upload\"|\"download\")\n\t\t\t\"$1\" \"$2\" \n\t\t;;\n\t\t*)\n\t\techo \"Invalid arguments!!\" >&2\n\t\texit 1\n\t\t;;\n\tesac\nelse\n\trun\nfi"
-    with open(os.path.join(SINGULARITY_DIR, "runscript"), 'w') as f:
-	f.write(run_cmd)
-        f.write(upload_download_cmd)
-
-def make_main_app_base_script(app_name):
-    print("rached 94")
+    file = ".singularity.d/"+ RUNSCRIPT
+    run_cmd = run_cmd + upload_download_cmd
+    run_cmd=run_cmd.encode("utf-8")
+    t = tarfile.TarInfo(file)
+    t.size = len(run_cmd)
+    t.mode=0o755
+    tar.addfile(t, io.BytesIO(run_cmd))
+    
+def get_main_app_base_script_content(app_name=None):
     cmd =  "SCIF_APPDATA_{0}=/scif/data/{1}\n".format(app_name,app_name)
     cmd += "SCIF_APPMETA_{0}=/scif/apps/{1}/scif\n".format(app_name,app_name)
     cmd += "SCIF_APPROOT_{0}=/scif/apps/{1}\n".format(app_name,app_name)
@@ -64,11 +94,17 @@ def make_main_app_base_script(app_name):
     cmd += "export SCIF_APPLABELS_{0}\n".format(app_name)
     cmd += "SCIF_APPRUN_{0}=/scif/apps/{1}/scif/runscript\n".format(app_name,app_name)
     cmd += "export SCIF_APPRUN_{0}\n".format(app_name)
-    with open(os.path.join(ENV_DIR, "94-appsbase.sh"), 'a') as f:
-        f.write(cmd)
+    return cmd
+
+
+def make_main_app_base_script(tar, main_app_base_cmd):
+    main_app_base_cmd = main_app_base_cmd.encode('utf-8')
+    t = tarfile.TarInfo(MAIN_APP_BASE_FILE)
+    t.size = len(main_app_base_cmd)
+    tar.addfile(t, io.BytesIO(main_app_base_cmd))
 
 
-def make_app_specific_base_script(app_name):
+def make_app_specific_base_script(tar, file, app_name=None):
     cmd = "SCIF_APPNAME={}\n".format(app_name)
     cmd += "SCIF_APPROOT=\"/scif/apps/{}\"\n".format(app_name)
     cmd += "SCIF_APPMETA=\"/scif/apps/{}/scif\"\n".format(app_name)
@@ -77,60 +113,69 @@ def make_app_specific_base_script(app_name):
     cmd += "SCIF_APPINPUT=\"/scif/data/{}/input\"\n".format(app_name)
     cmd += "SCIF_APPOUTPUT=\"/scif/data/{}/output\"\n".format(app_name)
     cmd += "export SCIF_APPDATA SCIF_APPNAME SCIF_APPROOT SCIF_APPMETA SCIF_APPINPUT SCIF_APPOUTPUT SCIF_DATA\n"
-    with open(os.path.join(APP_ENV_DIR, "01-base.sh"), 'w') as f:
-        f.write(cmd)
+    cmd = cmd.encode("utf-8")
+    file = "scif/apps/" + app_name +"/scif/env/" + file
+    t = tarfile.TarInfo(file)
+    t.size = len(cmd)
+    tar.addfile(t, io.BytesIO(cmd))
 
 # Check if bin is present in the tar if not add bin and sh
 def copy_busybox(tar):
-    if not "bin" in tar.getnames():
-        print("bin is absent")
-        tar.add("../../bin",arcname="bin")
-    elif not "sh" in tar.getnames():
-        print("bin but no sh")
-        tar.add("../../bin/sh", arcname='bin/sh')
-
-
-def deleteContent(fName):
-    with open(fName, "w"):
-        pass
+  tar.add("../bin",arcname="bin")
 
 def add_singularity_folder(tar):
     # check if the runs are multiple or single
-    my_dict = yaml.load(open(source_config))
+    tar.add(SINGULARITY_DIR, arcname=".singularity.d")
+    my_dict = yaml.load(open(SOURCE_CONFIG))
     runs = my_dict['runs']
+    main_app_base_cmd = ''
+    main_run_cmd = ''
     copy_busybox(tar)
-    deleteContent(os.path.join(ENV_DIR, "94-appsbase.sh"))
-    if len(runs) > 1:
-	print("multiple runs")
-        # App structure is present
-        # First add the SCIF folder 
-        tar.add(SCIF_DIR, arcname="scif")
-        for run in runs:
-            binary = run['binary']
-            workingdir = run['workingdir']
-            run_file = run['argv'][1]
-            app_name = run['id']
-            make_runscript(workingdir, binary, run_file, apps=True)
-            write_env_file(runs[0].get('environ'), RUN_ENV_FILE, apps=True)
-            make_app_specific_base_script(app_name)
-            make_main_app_base_script(app_name)
-            # add the modified app.
-            tar.add(APP_DIR, arcname="scif/apps/" + app_name)
+    # Add scif folders - apps and data
+    for folder in ["apps","data"]:
+        new_info = tarfile.TarInfo("scif/"+folder)
+        new_info.type = tarfile.DIRTYPE
+        new_info.mode = 0o755
+        tar.addfile(new_info)
+    for run in runs:
+        binary = run['binary']
+        workingdir = run['workingdir']
+        run_file = run['argv'][1]
+        app_name = run['id']
+        #sanitize the app_name
+        app_name = sanitize_appname(app_name)
+        # add the app
+        tar.add(APP_DIR, arcname="scif/apps/" + app_name)
+        # Make app specific runscript 
+        cmd = "cd {0}\n{1} {2}\n".format(workingdir, binary, run_file)
+        make_app_runscript(tar, cmd, app_name)
+        # Make the environment file
+        write_env_file(tar,run.get('environ'), RUN_ENV_FILE, app_name)
+        # Add the app base script
+        make_app_specific_base_script(tar, APP_BASE_FILE, app_name)
+        # Get the main app base script content
+        main_app_base_cmd += get_main_app_base_script_content(app_name)  
+        # get consolidated app run commands for main runscript
+        main_run_cmd += "\n\techo \"running app: "+ app_name + "\""
+        main_run_cmd += "\n\tsource /scif/apps/"+ app_name + "/scif/env/" + RUN_ENV_FILE 
+        main_run_cmd += "\n\tsource /scif/apps/"+ app_name + "/scif/" + RUNSCRIPT 
+    if main_run_cmd:
+         make_runscript(tar,main_run_cmd,runs[0]['workingdir'])
+    if main_app_base_cmd:
+        make_main_app_base_script(tar,main_app_base_cmd)
 
-    else:
-        write_env_file(runs[0].get('environ'),RUN_ENV_FILE)
-        make_runscript(runs[0]['workingdir'], runs[0]['binary'], runs[0]['argv'][1])
-    tar.add(SINGULARITY_DIR, arcname=".singularity.d")
+def create_overlay_image(OVERLAY_IMAGE):
+    if not os.path.exists(OVERLAY_IMAGE):
+        bashCommand = "singularity image.create {}".format(OVERLAY_IMAGE)
+        process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
+        output, error = process.communicate()
     
-
-
 def setup(filename):
+    # Open outer tar, the RPZ file
     rpz = tarfile.open(filename, 'r:*')
-
     # Open the inner tar in the original, without extracting it to disk
     data = rpz.extractfile('DATA.tar.gz')
     tar = tarfile.open('DATA.tar.gz', fileobj=data)
-
     # Open the new tar we're writing
     new = tarfile.open('new.tar.gz', 'w:gz')
     # For each member of the data tar
@@ -142,76 +187,71 @@ def setup(filename):
             # Copy the file from the inner tar to the new tar
             if new_info.isreg():
                 new.addfile(new_info, tar.extractfile(info.name))
-            elif new_info.isdir():
-                new.addfile(new_info)
             else:
                 new.addfile(new_info)
-    # Add the missing folders - proc,run, sys and temp_home
     folders = ['proc','dev','sys','temp_home','mnt']
     for folder in folders:
-        new.add("../../missing_folders/"+folder,folder)
-    print("added missing folder")
+        new_info = tarfile.TarInfo(folder)
+        new_info.type = tarfile.DIRTYPE
+        new_info.mode=0o755
+        new.addfile(new_info)
 
-    rpz.extractall()
+    rpz.extract("METADATA/config.yml",path="")
     add_singularity_folder(new)
-    print("added singularioty folder")
+    create_overlay_image(OVERLAY_IMAGE)
     tar.close()
     data.close()
     rpz.close()
     new.close()
 
-
-
-def create_overlay_image(OVERLAY_IMAGE):
-    if not os.path.exists(OVERLAY_IMAGE):
-    	bashCommand = "singularity image.create {}".format(OVERLAY_IMAGE)
-    	process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-    	output, error = process.communicate()
-    	print(output)
-    	return error
-    return 0
-
-
-
-def run(IMAGE_TAR_FILE,app):
+def run(IMAGE_TAR_FILE, app):
     home = os.environ['HOME']
-    if not create_overlay_image(OVERLAY_IMAGE):
-        if app:
-            print("running app:{}!".format(app))
-            bashCommand = "singularity run  --overlay {0} --app {1} -C -H {2}:/temp_home {3}".format(OVERLAY_IMAGE,app,home,IMAGE_TAR_FILE)
-            process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-            output, error = process.communicate()
-	else:
-	    # get the name of all the apps
-            my_dict = yaml.load(open(source_config))
-            runs = my_dict['runs']
-            for run in runs:	
-		app = run['id']    
-            	print("running app:{}!".format(app))
-            	bashCommand = "singularity run  --overlay {0} --app {1} -C -H {2}:/temp_home {3}".format(OVERLAY_IMAGE,app,home,IMAGE_TAR_FILE)
-            	process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-            	output, error = process.communicate()
-
-def download(IMAGE_TAR_FILE,filename):
+    if app:
+        print("running app:{}!".format(app))
+        bashCommand = "singularity run  --overlay {0} --app {1} -C -H {2}:/temp_home {3}".format(OVERLAY_IMAGE,app,home,IMAGE_TAR_FILE)
+        try:
+            subprocess.check_call([bashCommand],shell=True)
+        except subprocess.CalledProcessError:
+            print("Error running '{0}'".format(app))
+    else:    
+        bashCommand = "singularity run  --overlay {0} -C -H {1}:/temp_home {2}".format(OVERLAY_IMAGE,home,IMAGE_TAR_FILE)
+        try:
+            subprocess.check_call([bashCommand],shell=True)
+        except subprocess.CalledProcessError:
+            print("Error running image '{0}'".format(IMAGE_TAR_FILE))
+
+def download(IMAGE_TAR_FILE, filename):
     home = os.environ['HOME']
     current_dir = os.getcwd()
     bashCommand = "singularity run  -B {0}:{1} --overlay {2} -C -H {3}:/temp_home {4} download {5}".format(current_dir,MOUNT_DIR,OVERLAY_IMAGE, home, IMAGE_TAR_FILE, filename)
-    print(bashCommand)
-    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-    output, error = process.communicate()
-
+    try:
+        subprocess.check_call([bashCommand],shell=True)
+    except subprocess.CalledProcessError:
+        print("Error downloading '{0}'".format(filename))
 
 def upload(IMAGE_TAR_FILE, filename):
     home = os.environ['HOME']
     current_dir = os.getcwd()
     bashCommand = "singularity run  -B {0}:{1} --overlay {2} -C -H {3}:/temp_home {4} upload {5}".format(current_dir,MOUNT_DIR,OVERLAY_IMAGE, home, IMAGE_TAR_FILE, filename)
-    process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-    output, error = process.communicate()
+    try:
+        subprocess.check_call([bashCommand],shell=True)
+    except subprocess.CalledProcessError:
+        error_out("Error uploading '{0}' to '{1}".format(filename,IMAGE_DIR))
 
+def destroy(IMAGE_DIR):
+    bashCommand = "rm -rf {}".format(IMAGE_DIR)
+    try:
+        subprocess.check_call([bashCommand],shell=True)
+    except subprocess.CalledProcessError:
+        error_out("Error destroying '{0}' ".format(IMAGE_DIR))
 
 args = sys.argv[1:]
 cmd = args[0]
-print(cmd)
+
+if cmd not in ["setup","run","upload","download"]:
+    print("Invalid Commands - only setup/run/download/upload are allowed")
+    exit()
+
 if cmd=="setup":
     rpz_file, IMAGE_DIR = args[1:]
     if not os.path.exists(IMAGE_DIR):
@@ -220,17 +260,19 @@ def upload(IMAGE_TAR_FILE, filename):
     rpz_file = "../" + rpz_file
     setup(rpz_file)
 elif cmd=="run":
+    app = None
     IMAGE_DIR = args[1]
-    app = args[2] 
+    if len(args) == 3:
+        app = args[2]
     os.chdir(IMAGE_DIR)
-    print(app)
     run(IMAGE_TAR_FILE,app)
 elif cmd in ["upload","download"]:
     IMAGE_DIR,filename = args[1:]
     os.chdir(IMAGE_DIR)
-    print(IMAGE_DIR)
     if not filename:
         print("output file missing!")
     globals()[cmd](IMAGE_TAR_FILE,filename)
-
+elif cmd=="destroy":
+     IMAGE_DIR = args[1]
+     destroy(IMAGE_DIR)
 
diff --git a/reprounzip-singularity/sample_app/scif/Singularity b/reprounzip-singularity/sample_app/scif/Singularity
new file mode 100644
index 000000000..a70ebddaa
--- /dev/null
+++ b/reprounzip-singularity/sample_app/scif/Singularity
@@ -0,0 +1,3 @@
+%appinstall cowsay
+apt-get -y install cowsay
+
diff --git a/reprounzip-singularity/sample_app/scif/runscript b/reprounzip-singularity/sample_app/scif/runscript
new file mode 100755
index 000000000..b7a021892
--- /dev/null
+++ b/reprounzip-singularity/sample_app/scif/runscript
@@ -0,0 +1,2 @@
+cd /home/fchirigati/digitRecognition
+/usr/bin/python performRecognition.py
\ No newline at end of file

From 268ab094820f4a446c5d9a56af74e4f136aa260d Mon Sep 17 00:00:00 2001
From: Khushnaseeb Ali <khushnaseebali@172-16-19-152.dynapool.nyu.edu>
Date: Mon, 30 Jul 2018 14:09:44 -0400
Subject: [PATCH 5/8] removed runscript from sample app

---
 reprounzip-singularity/sample_app/scif/runscript | 2 --
 1 file changed, 2 deletions(-)
 delete mode 100755 reprounzip-singularity/sample_app/scif/runscript

diff --git a/reprounzip-singularity/sample_app/scif/runscript b/reprounzip-singularity/sample_app/scif/runscript
deleted file mode 100755
index b7a021892..000000000
--- a/reprounzip-singularity/sample_app/scif/runscript
+++ /dev/null
@@ -1,2 +0,0 @@
-cd /home/fchirigati/digitRecognition
-/usr/bin/python performRecognition.py
\ No newline at end of file

From f0dea0a55457e6f0e5346daaf9597ad74ffaed4d Mon Sep 17 00:00:00 2001
From: Khushnaseeb Ali <khushnaseebali@172-16-19-152.dynapool.nyu.edu>
Date: Mon, 30 Jul 2018 15:11:49 -0400
Subject: [PATCH 6/8] fixed upload/download issue

---
 reprounzip-singularity/Singularity_unpacker.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/reprounzip-singularity/Singularity_unpacker.py b/reprounzip-singularity/Singularity_unpacker.py
index a2ceb2f8b..b4c686589 100644
--- a/reprounzip-singularity/Singularity_unpacker.py
+++ b/reprounzip-singularity/Singularity_unpacker.py
@@ -220,19 +220,19 @@ def run(IMAGE_TAR_FILE, app):
         except subprocess.CalledProcessError:
             print("Error running image '{0}'".format(IMAGE_TAR_FILE))
 
-def download(IMAGE_TAR_FILE, filename):
+def download(IMAGE_DIR,filename):
     home = os.environ['HOME']
     current_dir = os.getcwd()
-    bashCommand = "singularity run  -B {0}:{1} --overlay {2} -C -H {3}:/temp_home {4} download {5}".format(current_dir,MOUNT_DIR,OVERLAY_IMAGE, home, IMAGE_TAR_FILE, filename)
+    bashCommand = "singularity run  -B {0}:{1} --overlay {2} -C -H {3}:/temp_home {4} download {5}".format(current_dir,MOUNT_DIR,IMAGE_DIR+"/"+OVERLAY_IMAGE, home, IMAGE_DIR+"/"+IMAGE_TAR_FILE, filename)
     try:
         subprocess.check_call([bashCommand],shell=True)
     except subprocess.CalledProcessError:
         print("Error downloading '{0}'".format(filename))
 
-def upload(IMAGE_TAR_FILE, filename):
+def upload(IMAGE_DIR, filename):
     home = os.environ['HOME']
     current_dir = os.getcwd()
-    bashCommand = "singularity run  -B {0}:{1} --overlay {2} -C -H {3}:/temp_home {4} upload {5}".format(current_dir,MOUNT_DIR,OVERLAY_IMAGE, home, IMAGE_TAR_FILE, filename)
+    bashCommand = "singularity run  -B {0}:{1} --overlay {2} -C -H {3}:/temp_home {4} upload {5}".format(current_dir,MOUNT_DIR,IMAGE_DIR+"/"+OVERLAY_IMAGE, home, IMAGE_DIR+"/"+IMAGE_TAR_FILE, filename)
     try:
         subprocess.check_call([bashCommand],shell=True)
     except subprocess.CalledProcessError:
@@ -268,10 +268,9 @@ def destroy(IMAGE_DIR):
     run(IMAGE_TAR_FILE,app)
 elif cmd in ["upload","download"]:
     IMAGE_DIR,filename = args[1:]
-    os.chdir(IMAGE_DIR)
     if not filename:
         print("output file missing!")
-    globals()[cmd](IMAGE_TAR_FILE,filename)
+    globals()[cmd](IMAGE_DIR,filename)
 elif cmd=="destroy":
      IMAGE_DIR = args[1]
      destroy(IMAGE_DIR)

From 012e5653c51d88fbdc9fc6f22d9da99ea887dfb1 Mon Sep 17 00:00:00 2001
From: Khushnaseeb Ali <khushnaseebali@MacBook-Pro-5.local>
Date: Mon, 6 Aug 2018 13:30:05 -0400
Subject: [PATCH 7/8] fixed upload/download issue

---
 .../Singularity_unpacker.py                   | 45 +++++++++++--------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/reprounzip-singularity/Singularity_unpacker.py b/reprounzip-singularity/Singularity_unpacker.py
index b4c686589..fe83f8ea2 100644
--- a/reprounzip-singularity/Singularity_unpacker.py
+++ b/reprounzip-singularity/Singularity_unpacker.py
@@ -65,14 +65,14 @@ def make_app_runscript(tar, cmd, app_name=None):
     tar.addfile(t, io.BytesIO(cmd))
 
    
-def make_runscript(tar, cmd, workingdir):
+def make_runscript(tar, cmd):
     run_cmd = "run()\n"
     run_cmd += "{"+ cmd + "\n}\n\n"
     upload_download_cmd = "upload()\n{\n"
-    upload_download_cmd += "\tcp " + MOUNT_DIR + "\"$1\" " + workingdir + "\n}\n\n"
+    upload_download_cmd += "\tcp " + MOUNT_DIR + "\"$1\" \"$2\"\n}\n\n"
     upload_download_cmd += "download()\n{\n"
-    upload_download_cmd += "\tcp {0}/".format(workingdir) + "\"$1\" " + MOUNT_DIR + "\n}\n\n"
-    upload_download_cmd += "if [ \"$#\" -gt 0 ]\nthen \n \tcase \"$1\" in \n\t\t\"upload\"|\"download\")\n\t\t\t\"$1\" \"$2\" \n\t\t;;\n\t\t*)\n\t\techo \"Invalid arguments!!\" >&2\n\t\texit 1\n\t\t;;\n\tesac\nelse\n\trun\nfi"
+    upload_download_cmd += "\tcp \"$1\" " + MOUNT_DIR + "\"$2\"\n}\n\n"
+    upload_download_cmd += "if [ \"$#\" -gt 0 ]\nthen \n \tcase \"$1\" in \n\t\t\"download\"|\"upload\")\n\t\t\t\"$1\" \"$2\" \"$3\" \n\t\t;;\n\t\t*)\n\t\techo \"Invalid arguments!!\" >&2\n\t\texit 1\n\t\t;;\n\tesac\nelse\n\trun\nfi"
     file = ".singularity.d/"+ RUNSCRIPT
     run_cmd = run_cmd + upload_download_cmd
     run_cmd=run_cmd.encode("utf-8")
@@ -160,7 +160,7 @@ def add_singularity_folder(tar):
         main_run_cmd += "\n\tsource /scif/apps/"+ app_name + "/scif/env/" + RUN_ENV_FILE 
         main_run_cmd += "\n\tsource /scif/apps/"+ app_name + "/scif/" + RUNSCRIPT 
     if main_run_cmd:
-         make_runscript(tar,main_run_cmd,runs[0]['workingdir'])
+         make_runscript(tar,main_run_cmd)
     if main_app_base_cmd:
         make_main_app_base_script(tar,main_app_base_cmd)
 
@@ -220,23 +220,32 @@ def run(IMAGE_TAR_FILE, app):
         except subprocess.CalledProcessError:
             print("Error running image '{0}'".format(IMAGE_TAR_FILE))
 
-def download(IMAGE_DIR,filename):
+def download(IMAGE_DIR, src, dest):
     home = os.environ['HOME']
-    current_dir = os.getcwd()
-    bashCommand = "singularity run  -B {0}:{1} --overlay {2} -C -H {3}:/temp_home {4} download {5}".format(current_dir,MOUNT_DIR,IMAGE_DIR+"/"+OVERLAY_IMAGE, home, IMAGE_DIR+"/"+IMAGE_TAR_FILE, filename)
+    dest_dir = os.path.dirname(dest)
+    if os.path.isdir(dest):
+        filename = ""
+    else:
+        filename = os.path.basename(dest)
+    bashCommand = "singularity run  -B {0}:{1} --overlay {2} -C -H {3}:/temp_home {4} download {5} {6}".format(dest_dir,MOUNT_DIR,IMAGE_DIR+"/"+OVERLAY_IMAGE, home, IMAGE_DIR+"/"+IMAGE_TAR_FILE, src, filename)
+    print(bashCommand)
     try:
         subprocess.check_call([bashCommand],shell=True)
     except subprocess.CalledProcessError:
         print("Error downloading '{0}'".format(filename))
 
-def upload(IMAGE_DIR, filename):
+def upload(IMAGE_DIR, src, dest):
+    src_dir = os.path.dirname(src)
     home = os.environ['HOME']
-    current_dir = os.getcwd()
-    bashCommand = "singularity run  -B {0}:{1} --overlay {2} -C -H {3}:/temp_home {4} upload {5}".format(current_dir,MOUNT_DIR,IMAGE_DIR+"/"+OVERLAY_IMAGE, home, IMAGE_DIR+"/"+IMAGE_TAR_FILE, filename)
+    if os.path.isdir(src):
+        filename = ""
+    else:
+        filename = os.path.basename(src)
+    bashCommand = "singularity run  -B {0}:{1} --overlay {2} -C -H {3}:/temp_home {4} upload {5} {6}".format(src_dir,MOUNT_DIR,IMAGE_DIR+"/"+OVERLAY_IMAGE, home, IMAGE_DIR+"/"+IMAGE_TAR_FILE, filename, dest)
     try:
         subprocess.check_call([bashCommand],shell=True)
     except subprocess.CalledProcessError:
-        error_out("Error uploading '{0}' to '{1}".format(filename,IMAGE_DIR))
+        print("Error uploading '{0}' to '{1}".format(filename,IMAGE_DIR))
 
 def destroy(IMAGE_DIR):
     bashCommand = "rm -rf {}".format(IMAGE_DIR)
@@ -248,8 +257,8 @@ def destroy(IMAGE_DIR):
 args = sys.argv[1:]
 cmd = args[0]
 
-if cmd not in ["setup","run","upload","download"]:
-    print("Invalid Commands - only setup/run/download/upload are allowed")
+if cmd not in ["setup","run","upload","download", "destroy"]:
+    print("Invalid Commands - only setup/run/download/upload/destroy are allowed")
     exit()
 
 if cmd=="setup":
@@ -267,10 +276,10 @@ def destroy(IMAGE_DIR):
     os.chdir(IMAGE_DIR)
     run(IMAGE_TAR_FILE,app)
 elif cmd in ["upload","download"]:
-    IMAGE_DIR,filename = args[1:]
-    if not filename:
-        print("output file missing!")
-    globals()[cmd](IMAGE_DIR,filename)
+    IMAGE_DIR, src, dest = args[1:]
+    if not dest:
+        print("file missing!")
+    globals()[cmd](IMAGE_DIR, src, dest)
 elif cmd=="destroy":
      IMAGE_DIR = args[1]
      destroy(IMAGE_DIR)

From ad75c23d005bddf6a9672daff82e39a8b59ece30 Mon Sep 17 00:00:00 2001
From: Khushnaseeb Ali <khushnaseebali@MacBook-Pro-5.home>
Date: Sun, 12 Aug 2018 23:30:47 -0400
Subject: [PATCH 8/8] addded download of busybox and fixed upload/download bug

---
 .../Singularity_unpacker.py                   | 29 +++++++++++++++----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/reprounzip-singularity/Singularity_unpacker.py b/reprounzip-singularity/Singularity_unpacker.py
index fe83f8ea2..9e8a8b437 100644
--- a/reprounzip-singularity/Singularity_unpacker.py
+++ b/reprounzip-singularity/Singularity_unpacker.py
@@ -6,10 +6,11 @@
 import subprocess
 import io
 import string
+import requests 
 
 SINGULARITY_DIR = "../.singularity.d"
 APP_DIR = "../sample_app"
-MOUNT_DIR = "/mnt/"
+MOUNT_DIR = "/tmp/"
 RUN_ENV_FILE = "90-environment.sh"
 APP_BASE_FILE = "01-base.sh"
 MAIN_APP_BASE_FILE = ".singularity.d/env/94-appsbase.sh"
@@ -23,6 +24,14 @@
                        "0123456789"
                        "-+=/:.,%_")
 
+busybox_urls = {
+"x86_64": "https://s3.amazonaws.com/reprozip-files/busybox-x86_64",
+"i686": "https://s3.amazonaws.com/reprozip-files/busybox-i686"
+}
+
+
+
+
 def sanitize_appname(name):
     name = name.strip()
     valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
@@ -120,8 +129,15 @@ def make_app_specific_base_script(tar, file, app_name=None):
     tar.addfile(t, io.BytesIO(cmd))
 
 # Check if bin is present in the tar if not add bin and sh
-def copy_busybox(tar):
-  tar.add("../bin",arcname="bin")
+def copy_busybox(tar,busybox_url):
+    r = requests.get(busybox_url)
+    for cmd in ["sh","cp","ls"]:
+        file = "bin/" + cmd
+        t = tarfile.TarInfo(file)
+        t.size = len(r.content)
+        t.mode=0o755
+        tar.addfile(t, io.BytesIO(r.content))
+
 
 def add_singularity_folder(tar):
     # check if the runs are multiple or single
@@ -130,8 +146,11 @@ def add_singularity_folder(tar):
     runs = my_dict['runs']
     main_app_base_cmd = ''
     main_run_cmd = ''
-    copy_busybox(tar)
     # Add scif folders - apps and data
+    arch = runs[0]['architecture']
+    busybox_url = busybox_urls[arch]
+    copy_busybox(tar,busybox_url)
+
     for folder in ["apps","data"]:
         new_info = tarfile.TarInfo("scif/"+folder)
         new_info.type = tarfile.DIRTYPE
@@ -222,7 +241,7 @@ def run(IMAGE_TAR_FILE, app):
 
 def download(IMAGE_DIR, src, dest):
     home = os.environ['HOME']
-    dest_dir = os.path.dirname(dest)
+    dest_dir = os.path.dirname(dest) or os.getcwd()
     if os.path.isdir(dest):
         filename = ""
     else: