diff --git a/a-delete b/a-delete
index 4e6b703..78abd87 100755
--- a/a-delete
+++ b/a-delete
@@ -73,6 +73,10 @@ do
mode="s3cmd"
shift
;;
+ '--lumi' )
+ mode="lumi"
+ shift
+ ;;
'-h' | '--help' )
print_help=1
@@ -142,6 +146,9 @@ if [[ $mode == "s3cmd" ]]; then
storage_server="s3allas"
fi
+if [[ $mode == "lumi" ]]; then
+ storage_server="lumi-o"
+fi
#The method to remove a bucket
if [[ $remove_bucket -eq 1 ]]; then
diff --git a/a-encrypt b/a-encrypt
index 854e239..d2de303 100755
--- a/a-encrypt
+++ b/a-encrypt
@@ -1,3 +1,4 @@
+#!/bin/bash
#default user
@@ -51,6 +52,11 @@ do
mode=("s3cmd")
shift
;;
+
+ '--lumi' )
+ mode=("lumi")
+ shift
+ ;;
'-b' | '--bucket' )
bucket="$2"
shift
@@ -190,6 +196,12 @@ if [[ $mode == "s3cmd" ]];then
storage_server="s3allas"
fi
+#Rclone through s3
+if [[ $mode == "lumi" ]];then
+ storage_server="lumi-o"
+fi
+
+
#source /appl/opt/allas_conf
#input=("$1")
diff --git a/a-find b/a-find
index 65f2ec3..a4ee3d9 100755
--- a/a-find
+++ b/a-find
@@ -15,17 +15,17 @@ source $inst_root/a_env_conf
source $inst_root/allas-lib
#local variables
-user=($USER)
-bucket_name=("not_defined")
-silent=(0)
-print_help=(0)
-os_project_name=("$OS_PROJECT_NAME")
-input_def=("")
-mode=("swift")
+user="$USER"
+bucket_name="not_defined"
+silent=0
+print_help=0
+os_project_name="$OS_PROJECT_NAME"
+input_def=""
+mode="swift"
#tmp_dir=("${tmp_root}/a_put_$$_tmp")
-show_filelist=(0)
-query=("")
-all_buckets=(0)
+show_filelist=0
+query=""
+all_buckets=0
# read customer defaults
@@ -41,7 +41,7 @@ while [[ $# -ge 1 ]]
do
case "$1" in
'--user' | '-u' )
- user=("$2")
+ user="$2"
shift
shift
;;
@@ -57,19 +57,23 @@ do
shift
;;
'-silent' | '-s')
- silent=(1)
+ silent=1
+ shift
+ ;;
+ '--lumi')
+ mode="lumi"
shift
;;
'--files' | '-f' )
- show_filelist=(1)
+ show_filelist=1
shift
;;
'--all_bukets' | '-a' | '--all_buckets' )
- bucket_name=("$(swift list| tr '\n' ' ')")
+ all_buckets=1
shift
;;
'-h' | '--help' )
- print_help=(1)
+ print_help=1
shift
;;
*)
@@ -78,7 +82,7 @@ do
echo "Query term is: $query"
exit 1
fi
- query=("$1")
+ query="$1"
shift # No more switches
;;
esac
@@ -170,6 +174,7 @@ project_label=$(echo ${os_project_name} | sed -e s/"project_"/""/g)
#Check if connection works
if [[ $mode == "swift" ]]
then
+ storage_server="allas"
test=$(rclone about ${storage_server}: 2> /dev/null | wc -l)
#test=$(swift stat 2> /dev/null | grep -c "Account:")
if [[ $test -lt 1 ]]
@@ -184,6 +189,21 @@ then
fi
+if [[ $mode == "s3cmf" ]]
+then
+ storage_server="s3allas"
+fi
+
+if [[ $mode == "lumi" ]]
+then
+ storage_server="lumi-o"
+fi
+
+if [[ $all_buckets -eq 1 ]]; then
+ #bucket_name=("$(swift list| tr '\n' ' ')")
+ bucket_name=("$(rclone lsd ${storage_server}: | awk '{print $NF}'| tr '\n' ' ')")
+fi
+
#define standard buckets
std_buckets=("${user}-${project_label}-MISC ${user}-${project_label}-HOME ${project_label}-puhti-SCRATCH ${project_label}-puhti-PROJAPPL ${user}-${project_label}-pub ${user}-${project_label}-flip")
@@ -240,7 +260,7 @@ do
echo "----------------------------------------------"
echo "Checking bucket: $bn"
fi
- rclone ls ${storage_server}:$bn 2> /dev/null | awk '{print $2}' | grep -v "_ameta$" | grep "$query" > ./$$_find_tmp
+ rclone ls ${storage_server}:$bn 2> /dev/null | awk '{print "x " $0}' | tr -s ' '| cut -d " " -f 3- | grep -v "_ameta$" | grep "$query" > ./$$_find_tmp
object_hits=$(cat ./$$_find_tmp | wc -l)
if [[ $object_hits -gt 0 ]];then
if [ $silent -eq 0 ]; then
diff --git a/a-flip b/a-flip
index 1eaab13..6ebdc85 100755
--- a/a-flip
+++ b/a-flip
@@ -25,6 +25,15 @@ input_def=("")
mode=("swift")
#tmp_dir=("${tmp_root}/a_put_$$_tmp")
+
+# read customer defaults
+if [[ -e $HOME/.a_tools_conf ]]; then
+ customized=1
+ source $HOME/.a_tools_conf
+else
+ customized=0
+fi
+
#Process command line
while [[ $# -ge 1 ]]
do
@@ -84,8 +93,17 @@ then
echo "Please try setting up the connection again."
exit 1
fi
-else
- echo "a-flip is available only in swift mode"
+fi
+
+if [[ $mode == "s3cmd" ]]
+then
+ echo "a-flip is not available s3allas."
+ exit
+fi
+
+if [[ $mode == "lumi" ]]
+then
+ storage_server="lumi-pub"
fi
@@ -104,6 +122,12 @@ echo "Files to be uploaded: $input_def"
project_label=$(echo ${os_project_name} | sed -e s/"project_"/""/g)
bucket_name=("${user}-${project_label}-flip")
+if [[ $mode == "lumi" ]]; then
+ echo "rclone mkdir ${storage_server}:${bucket_name}"
+ rclone mkdir ${storage_server}:${bucket_name}
+fi
+
+
make_temp_dir
for input in $input_def
@@ -196,15 +220,14 @@ do
#upload
- if [ $mode == "swift" ]
- then
- # For less than 5GB files rclone is used for uploading
+ #if [[ $mode == "swift" ]]; then
+ # For less than 5GB files rclone is used for uploading
- echo "Uploading data to allas."
- # echo "rclone copy --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}/${partial_path}"
- rclone copy -L --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}
- exitcode=$?
- if [ $exitcode -ne 0 ]; then
+ echo "Uploading data to allas."
+ echo "rclone copy --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}"
+ rclone copy -L --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}
+ exitcode=$?
+ if [ $exitcode -ne 0 ]; then
echo ""
echo "File upload for $infile failed"
@@ -213,29 +236,28 @@ do
rm -f ${tmp_dir}/${tmp_file}_ameta
clean_temp_dir
exit 1
- fi
+ fi
- # rclone md5sums can be calculated only for files that are smaller than 5GB
- if [[ $tot_size -lt 5000000 ]]
- then
- echo "Confirming upload..."
- #checksums for local and allas files
- sum1=($(md5sum ${tmp_dir}/$tmp_file))
- sum2=($(rclone md5sum ${storage_server}:${bucket_name}/$tmp_file))
-
- #check is checksums match
- if [[ ${sum1[0]} != ${sum2[0]} ]]
- then
- echo "Upload of $input was not successful!"
- echo "Cleaning the failed upload"
- rclone deletefile ${storage_server}:${bucket_name}/$tmp_file
- rm -f ${tmp_dir}/$tmp_file
- rm -f ${tmp_dir}/${tmp_file}_ameta
- clean_temp_dir
- exit 1
- fi
- echo "$input OK"
- fi
+ # rclone md5sums can be calculated only for files that are smaller than 5GB
+ if [[ $tot_size -lt 5000000 ]]
+ then
+ echo "Confirming upload..."
+ #checksums for local and allas files
+ sum1=($(md5sum ${tmp_dir}/$tmp_file))
+ sum2=($(rclone md5sum ${storage_server}:${bucket_name}/$tmp_file))
+
+ #check is checksums match
+ if [[ ${sum1[0]} != ${sum2[0]} ]]
+ then
+ echo "Upload of $input was not successful!"
+ echo "Cleaning the failed upload"
+ rclone deletefile ${storage_server}:${bucket_name}/$tmp_file
+ rm -f ${tmp_dir}/$tmp_file
+ rm -f ${tmp_dir}/${tmp_file}_ameta
+ clean_temp_dir
+ exit 1
+ fi
+ echo "$input OK"
fi
@@ -251,7 +273,12 @@ do
rm -f ${tmp_dir}/${tmp_file}_ameta
echo "$input uploaded to ${bucket_name}"
- echo "Public link: https://a3s.fi/${bucket_name}/$tmp_file"
+ if [[ $mode == "swift" ]]; then
+ echo "Public link: https://a3s.fi/${bucket_name}/$tmp_file"
+ fi
+ if [[ $mode == "lumi" ]]; then
+ echo "Public link: https://${os_project_name}.lumidata.eu/${bucket_name}/$tmp_file"
+ fi
tmp_file=("not_defined")
@@ -271,18 +298,21 @@ done
#Publish and rebuild index file
echo '' > ${tmp_dir}/index.html
-for i in $(swift list $bucket_name | grep -v '_ameta$' )
+#for i in $(swift list $bucket_name | grep -v '_ameta$' )
+for i in $(rclone ls ${storage_server}:${bucket_name} | grep -v '_ameta$' | awk '{print "X "$0}' | tr -s " " | cut -d " " -f 3- )
do
-echo '
'$i'' >> ${tmp_dir}/index.html
+ echo ''$i'' >> ${tmp_dir}/index.html
done
echo '' >> ${tmp_dir}/index.html
-rclone copy ${tmp_dir}/index.html ${storage_server}:${bucket_name}/
+rclone copy ${tmp_dir}/index.html ${storage_server}:${bucket_name}/
rm -f ${tmp_dir}/index.html
-#set access control
-swift post ${bucket_name} --read-acl ".r:*,.rlistings"
-swift post ${bucket_name}_segments --read-acl ".r:*,.rlistings"
+#set access control in case of Allas/swift
+if [[ $mode == "swift" ]]; then
+ swift post ${bucket_name} --read-acl ".r:*,.rlistings"
+ swift post ${bucket_name}_segments --read-acl ".r:*,.rlistings"
+fi
clean_temp_dir
echo ""
diff --git a/a-get b/a-get
index 875aad9..c509ab9 100755
--- a/a-get
+++ b/a-get
@@ -175,7 +175,7 @@ then
#test=$(swift stat 2> /dev/null | grep -c "Account:")
if [[ $test -lt 1 ]]
then
- echo "No connection to Allas!"
+ echo "No connection to $storage_server!"
echo "Please try setting up the connection again"
echo "by running command:"
echo ""
@@ -189,6 +189,14 @@ if [[ $mode == "s3cmd" ]];then
storage_server="s3allas"
fi
+#lumi
+if [[ $mode == "lumi" ]];then
+ storage_server="lumi-o"
+fi
+
+
+
+
#source /appl/opt/allas_conf
#input=("$1")
@@ -491,7 +499,6 @@ else
rclone cat "${storage_server}:$object_name" | crypt4gh decrypt --sk "$secret_key" > $target_name
exitcode=$?
else
- echo "eka ${target_name:0:1}"
if [[ ${target_name:0:1} != "/" ]]; then
#this is needed to cope with file names containing :
rclone -P copyto "${storage_server}:$object_name" "./$target_name"
diff --git a/a-info b/a-info
index 4653830..a360975 100755
--- a/a-info
+++ b/a-info
@@ -104,6 +104,12 @@ if [[ $mode == "s3cmd" ]]; then
storage_server="s3allas"
fi
+if [[ $mode == "lumi" ]]; then
+ storage_server="lumi-o"
+fi
+
+
+
if [[ $object_name == "" ]] ; then
for buc in $(rclone lsd ${storage_server}: | awk '{print $NF}')
diff --git a/a-list b/a-list
index 8ef8450..adcf439 100755
--- a/a-list
+++ b/a-list
@@ -34,7 +34,7 @@ while [[ $# -ge 1 ]]
do
case "$1" in
'--user' | '-u' )
- user=("$2")
+ user="$2"
shift
shift
;;
@@ -43,6 +43,10 @@ do
options=("$options --lh ")
shift
;;
+ '--lumi' | '-L' )
+ mode="lumi"
+ shift
+ ;;
'--dir' | '-d' )
dir_like=(1)
shift
@@ -91,6 +95,10 @@ fi
if [[ $mode == "s3cmd" ]];then
storage_server="s3allas"
fi
+if [[ $mode == "lumi" ]];then
+ storage_server="lumi-o"
+fi
+
#check that we have a bucket only
if [ $(echo $bucket_name | grep -o "/" | wc -l) -gt 0 ]; then
@@ -148,14 +156,10 @@ shared_list="buckets_shared_to"
if [[ $mode == "swift" ]] && [[ $bucket_name != "" ]]; then
- #my_auth=$( echo $OS_STORAGE_URL | awk -F "/" '{print $NF}' )
bucket_readable=$(rclone ls ${storage_server}:$bucket_name 2> /dev/null | wc -l)
own_bucket=$(rclone lsd allas: | awk '{ print $NF}' | grep -c -w $bucket_name )
- #bucket_auth=$(swift stat $bucket_name 2>/dev/null | grep "Account:" | awk '{print $2}')
- #echo my: $my_auth bucket $bucket_auth
# case: bucket reable but not owned
if [[ $bucket_readable -gt 0 ]] && [[ $own_bucket -eq 0 ]]; then
- #if [[ $my_auth != $bucket_auth ]] && [[ $bucket_auth != "" ]]; then
# do we already have a list of shared buckets
check_os=$(rclone ls ${storage_server}:${a_tools_bucket}/$shared_list 2> /dev/null | wc -l)
if [[ $check_os -eq 1 ]]; then
@@ -168,7 +172,6 @@ if [[ $mode == "swift" ]] && [[ $bucket_name != "" ]]; then
#rm .$$_a_shared_tmp
fi
else
- #echo " rclone mkdir ${storage_server}:${a_tools_bucket} "
rclone mkdir ${storage_server}:${a_tools_bucket}
echo "$bucket_name" | rclone rcat ${storage_server}:${a_tools_bucket}/$shared_list
echo "Adding bucket $bucket_name to the list of buckets shared to this project"
@@ -190,7 +193,7 @@ if [[ $mode == "swift" ]] && [[ $bucket_name != "" ]]; then
rclone delete ${storage_server}:${a_tools_bucket}/$shared_list
fi
#rm .$$_a_shared_tmp
- echo "Bucket $bucket_name is no longe accessible. Removing it form the list of shared buckets"
+ echo "Bucket $bucket_name is no longer accessible. Removing it form the list of shared buckets."
fi
fi
fi
@@ -202,7 +205,6 @@ fi
if [[ $dir_like -eq 1 ]]; then
- echo rclone lsf ${storage_server}:${full_name}
rclone lsf ${storage_server}:${full_name} | grep -v "_ameta$"
exit 0
fi
@@ -223,11 +225,13 @@ if [[ $bucket_name == "" ]]; then
# swift list $options
else
- if [[ $mode == "s3cmd" ]]; then
+ if [[ $mode == "s3cmd" ]] || [[ $mode == "lumi" ]] ; then
if [[ $long_listing -eq 0 ]]; then
- rclone ls ${storage_server}:$full_name | grep -v "_ameta$" | awk '{print "'$full_name'/" $NF}'
+ rclone ls ${storage_server}:$bucket_name | grep -v "_ameta$" | awk '{print "x " $0}' | tr -s ' ' | cut -d " " -f 3- | awk '{ print "'$bucket_name'/" $0}' | grep $full_name
+ #rclone ls ${storage_server}:$full_name | grep -v "_ameta$" | awk '{print "'$full_name'/" $NF}'
else
- rclone lsl ${storage_server}:$full_name | awk '{print $1"\t"$2"\t"$3"\t'$full_name'/" $4}'
+ #rclone lsl ${storage_server}:$bucket_name | grep -v "_ameta$" | awk '{print "x " $0}' | tr -s ' ' | cut -d " " -f 3- | awk '{ print "'$full_name'/ " $0}'
+ rclone lsl ${storage_server}:$full_name | grep -v "_ameta$"
fi
else
# listing in swift mode
diff --git a/a-publish b/a-publish
index f21819e..8cf2d6e 100755
--- a/a-publish
+++ b/a-publish
@@ -28,6 +28,16 @@ mode="swift"
#tmp_dir="${tmp_root}/a_put_$$_tmp"
make_temp_dir
+# read customer defaults
+if [[ -e $HOME/.a_tools_conf ]]; then
+ customized=1
+ source $HOME/.a_tools_conf
+else
+ customized=0
+fi
+
+
+
#Process command line
while [[ $# -ge 1 ]]
do
@@ -53,7 +63,14 @@ do
mode=("s3cmd")
shift
;;
-
+ '--lumi' )
+ mode=("lumi")
+ shift
+ ;;
+ '--allas' )
+ mode=("swift")
+ shift
+ ;;
'--index' | '-i' )
index_mode=$2
if [[ $index_mode == "static" || $index_mode == "dynamic" ]]; then
@@ -134,43 +151,6 @@ exit
fi
-#Configure rclone
-if [ ! -e $HOME/.config/rclone ]
-then
- mkdir -p $HOME/.config/rclone
-fi
-if [ -e $HOME/.config/rclone/rclone.conf ]
-then
- rc_check=$(grep -c $storage_server $HOME/.config/rclone/rclone.conf)
- if [ $rc_check -lt 1 ]
- then
- echo '[pouta]' >> $HOME/.config/rclone/rclone.conf
- echo 'type = swift' >> $HOME/.config/rclone/rclone.conf
- echo 'env_auth = true' >> $HOME/.config/rclone/rclone.conf
- fi
-else
- echo '[pouta]' >> $HOME/.config/rclone/rclone.conf
- echo 'type = swift' >> $HOME/.config/rclone/rclone.conf
- echo 'env_auth = true' >> $HOME/.config/rclone/rclone.conf
-fi
-
-#Assign project to be used if not defined
-if [ $os_project_name == "" ]
-then
- if [ -e $HOME/.allas_default ]
- then
- source $HOME/.allas_default
- else
- echo "Default project is not defined"
- source $allas_conf_path -user $user
- echo "os_project_name=$OS_PROJECT_NAME" > $HOME/.allas_default
- echo "Default allas project is stored to \$HOME/.allas_default"
- echo ""
- fi
- source $HOME/.allas_default
-fi
-
-
#Check if connection works
if [[ $mode == "swift" ]]
then
@@ -184,12 +164,6 @@ then
fi
fi
-#source /appl/opt/allas_conf
-#input=("$1")
-
-#check free space in $WRKDIR
-#quota_s=($(lfs quota -q -u $USER $WRKDIR))
-#free_space=$(expr ${quota_s[2]} - ${quota_s[1]})
echo "Files to be uploaded: $input_def"
@@ -199,9 +173,20 @@ if [[ "$bucket_name" == "not_defined" ]]
then
bucket_name=("${user}-${project_label}-pub")
fi
+
+#In case of lumi the bucket must be cereated
+if [[ $mode == "lumi" ]]; then
+ storage_server="lumi-pub"
+ echo "rclone mkdir ${storage_server}:${bucket_name}"
+ rclone mkdir ${storage_server}:${bucket_name}
+fi
+
+
echo "Bucket: $bucket_name"
+
+
for input in $input_def
do
echo "Processing: $input"
@@ -269,82 +254,67 @@ do
then
echo "This file or directory is too big for this tool"
echo "Total size: ${tot_size}K"
- echo "Please use swift or rclone command to upload the data to allas"
+ echo "Please use swift or rclone command to upload the data to $storage_server"
rm -f ${tmp_dir}/${tmp_file}_ameta
clean_temp_dir
exit 1
fi
-# if [[ $tot_size -gt $free_space ]]
-# then
-# echo "There is not enough space for the temporary files."
-# echo "$input contains $num_files files or directories that take ${tot_size}K of disk space"
-# echo "Available free space is ${free_space}K"
-# echo ""
-# rm -f ${tmp_dir}/${tmp_file}_ameta
-# rmdir ${tmp_dir}
-# exit 1
-# fi
-
- #this is not a good approach.
- #test if a symbolic link would do
- #cp $input ${tmp_dir}/$tmp_file
ln -s ${file_path} ${tmp_dir}/$tmp_file
#upload
- if [ $mode == "swift" ]
- then
- # For less than 5GB files rclone is used for uploading
+ #if [[ $mode == "swift" ]]
+ #then
+ # For less than 5GB files rclone is used for uploading
- echo "Uploading data to allas."
- # echo "rclone copy --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}/${partial_path}"
- rclone copy -L --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}/${partial_path}
- exitcode=$?
- if [ $exitcode -ne 0 ]; then
- echo ""
- echo "File upload for $infile failed"
+ echo "Uploading data to allas."
+ # echo "rclone copy --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}/${partial_path}"
+ rclone copy -L --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}/${partial_path}
+ exitcode=$?
+ if [[ $exitcode -ne 0 ]]; then
+ echo ""
+ echo "File upload for $infile failed"
+ if [[ $partial_path == "" ]]; then
+ rclone deletefile ${storage_server}:${bucket_name}/$tmp_file
+ else
+ rclone deletefile ${storage_server}:${bucket_name}/${partial_path}/$tmp_file
+ fi
+ rm -f ${tmp_dir}/$tmp_file
+ if [[ $include_ameta -eq 1 ]]; then
+ rm -f ${tmp_dir}/${tmp_file}_ameta
+ fi
+ clean_temp_dir
+ exit 1
+ fi
+
+ # rclone md5sums can be calculated only for files that are smaller than 5GB
+ if [[ $tot_size -lt 5000000 ]]
+ then
+ echo "Confirming upload..."
+ #checksums for local and allas files
+ sum1=($(md5sum ${tmp_dir}/$tmp_file))
+
+ if [[ $partial_path == "" ]]; then
+ sum2=($(rclone md5sum ${storage_server}:${bucket_name}/$tmp_file))
+ else
+ sum2=($(rclone md5sum ${storage_server}:${bucket_name}/${partial_path}/$tmp_file))
+ fi
+ #check if checksums match
+ if [[ ${sum1[0]} != ${sum2[0]} ]]
+ then
+ echo "Upload of $input was not successful!"
+ echo "Cleaning the failed upload"
if [[ $partial_path == "" ]]; then
- rclone deletefile ${storage_server}:${bucket_name}/$tmp_file
+ rclone deletefile ${storage_server}:${bucket_name}/$tmp_file
else
- rclone deletefile ${storage_server}:${bucket_name}/${partial_path}/$tmp_file
+ rclone deletefile ${storage_server}:${bucket_name}/${partial_path}/$tmp_file
fi
rm -f ${tmp_dir}/$tmp_file
- if [[ $include_ameta -eq 1 ]]; then
- rm -f ${tmp_dir}/${tmp_file}_ameta
- fi
+ rm -f ${tmp_dir}/${tmp_file}_ameta
clean_temp_dir
exit 1
- fi
-
- # rclone md5sums can be calculated only for files that are smaller than 5GB
- if [[ $tot_size -lt 5000000 ]]
- then
- echo "Confirming upload..."
- #checksums for local and allas files
- sum1=($(md5sum ${tmp_dir}/$tmp_file))
-
- if [[ $partial_path == "" ]]; then
- sum2=($(rclone md5sum ${storage_server}:${bucket_name}/$tmp_file))
- else
- sum2=($(rclone md5sum ${storage_server}:${bucket_name}/${partial_path}/$tmp_file))
- fi
- #check if checksums match
- if [[ ${sum1[0]} != ${sum2[0]} ]]
- then
- echo "Upload of $input was not successful!"
- echo "Cleaning the failed upload"
- if [[ $partial_path == "" ]]; then
- rclone deletefile ${storage_server}:${bucket_name}/$tmp_file
- else
- rclone deletefile ${storage_server}:${bucket_name}/${partial_path}/$tmp_file
- fi
- rm -f ${tmp_dir}/$tmp_file
- rm -f ${tmp_dir}/${tmp_file}_ameta
- clean_temp_dir
- exit 1
- fi
- echo "$input OK"
- fi
+ fi
+ echo "$input OK"
fi
@@ -362,10 +332,18 @@ do
fi
echo "$input uploaded to ${bucket_name}"
+
+ if [[ $mode == "swift" ]]; then
+ https_root="https://a3s.fi"
+ fi
+ if [[ $mode == "lumi" ]]; then
+ https_root="https://${os_project_name}.lumidata.eu"
+ fi
+
if [[ $partial_path == "" ]]; then
- echo "Public link: https://a3s.fi/${bucket_name}/$tmp_file"
+ echo "Public link: ${https_root}/${bucket_name}/$tmp_file"
else
- echo "Public link: https://a3s.fi/${bucket_name}/${partial_path}/$tmp_file"
+ echo "Public link: ${https_root}/${bucket_name}/${partial_path}/$tmp_file"
fi
tmp_file=("not_defined")
@@ -377,7 +355,7 @@ if [[ $index_mode == "static" ]];then
echo '' > ${tmp_dir}/index.html
echo '' >> ${tmp_dir}/index.html
echo "Contents of bucket $bucket_name" >> ${tmp_dir}/index.html
- rclone lsl allas:$bucket_name | grep -v "_ameta" | awk '{print ""$1" | "$2" | "$3" | "$4" | info |
"}' >> ${tmp_dir}/index.html
+ rclone lsl ${storage_server}:$bucket_name | grep -v "_ameta" | awk '{print ""$1" | "$2" | "$3" | "$4" | info |
"}' >> ${tmp_dir}/index.html
echo '
' >> ${tmp_dir}/index.html
echo '' >> ${tmp_dir}/index.html
else
@@ -390,8 +368,10 @@ rclone copy ${tmp_dir}/index.html ${storage_server}:${bucket_name}/
rm -f ${tmp_dir}/index.html
#set access control
-swift post ${bucket_name} --read-acl ".r:*,.rlistings"
-swift post "${bucket_name}_segments" --read-acl ".r:*,.rlistings"
+if [[ $mode == "swift" ]]; then
+ swift post ${bucket_name} --read-acl ".r:*,.rlistings"
+ swift post "${bucket_name}_segments" --read-acl ".r:*,.rlistings"
+fi
clean_temp_dir
echo ""
diff --git a/a-put b/a-put
index d93173f..550ea64 100755
--- a/a-put
+++ b/a-put
@@ -69,7 +69,7 @@ do
shift
;;
'--project' | '-p' )
- os_project_name=($2)
+ os_project_name="$2"
shift
shift
;;
@@ -89,8 +89,12 @@ do
shift
shift
;;
+ '--lumi' | '-L' )
+ mode="lumi"
+ shift
+ ;;
'--s3cmd' | '--s3' | '-S' )
- mode=("s3cmd")
+ mode="s3cmd"
shift
;;
'--compress' | '-c')
@@ -103,7 +107,7 @@ do
shift
;;
'-s' | '--silent' )
- silent=(1)
+ silent=1
shift
;;
'-h' | '--help' )
@@ -111,12 +115,12 @@ do
shift
;;
'--user' | '-u' )
- user=("$2")
+ user="$2"
shift
shift
;;
'--tmpdir' | '-t' )
- tmp_root=("$2")
+ tmp_root="$2"
shift
shift
;;
@@ -428,6 +432,9 @@ fi
if [[ $mode == "s3cmd" ]]; then
storage_server="s3allas"
fi
+if [[ $mode == "lumi" ]]; then
+ storage_server="lumi-o"
+fi
#check free space in tmpdir
@@ -675,7 +682,29 @@ dmku3fKA/wrOpWntUTkkoQvknjZDisdmSwU4oFk/on0=
fi
partial_path=$(dirname $file_path | sed -e s/"\/projappl\/$puhti_project"/""/g)
fi
-
+
+ #Lumi project
+ if [ $(echo $file_path | cut -c1-8) == "/project" ]
+ then
+ puhti_project=$(echo $file_path/ |awk -F "/" '{print $3}')
+ bucket_name=("${project_label}-${local_host}-project")
+ if [[ $os_project_name != $puhti_project ]] && [[ $user_answer != "y" ]] ; then
+ echo ""
+ echo "NOTE: data locates in Project area of project: $puhti_project"
+ echo "But it will be stored to ${storage_server} under project: $os_project_name"
+ echo "Bucket to be used is: $bucket_name"
+ echo "Is this OK (y/n)?"
+ read user_answer
+ if [[ $user_answer != "y" ]]; then
+ echo "Exiting, data not uploaded."
+ exit 0
+ fi
+ fi
+ partial_path=$(dirname $file_path | sed -e s/"\/projappl\/$puhti_project"/""/g)
+ fi
+
+
+
#Puhti FMI-projappl
if [ $(echo $file_path | cut -c1-13) == "/fmi/projappl" ]
then
@@ -763,7 +792,11 @@ dmku3fKA/wrOpWntUTkkoQvknjZDisdmSwU4oFk/on0=
echo "host_name: $local_host" >> ${tmp_dir}/${tmp_file}_ameta
echo "input: $input" >> ${tmp_dir}/${tmp_file}_ameta
echo "original_location: $file_path " >> ${tmp_dir}/${tmp_file}_ameta
- echo "protocol: $mode " >> ${tmp_dir}/${tmp_file}_ameta
+ if [[ $mode == "s3cmd" ]] || [[ $mode == "lumi" ]]; then
+ echo "protocol: S3 " >> ${tmp_dir}/${tmp_file}_ameta
+ else
+ echo "protocol: swift" >> ${tmp_dir}/${tmp_file}_ameta
+ fi
if [[ $compression -eq 1 ]]; then
echo "compression: zstd" >> ${tmp_dir}/${tmp_file}_ameta
else
@@ -963,11 +996,17 @@ dmku3fKA/wrOpWntUTkkoQvknjZDisdmSwU4oFk/on0=
if [[ $silent -eq 0 ]] ; then
- echo "Uploading data to allas."
+ echo "Uploading data to ${storage_server}."
fi
-
+
+ # Lumi-O dont allow upper case letters in bucket names
+ if [[ $mode == "lumi" ]]; then
+ bucket_name=$(echo $bucket_name | tr [:upper:] [:lower:] )
+ fi
+
#S3 protocol requires that directory exist before copying
- if [[ $mode == "s3cmd" ]]; then
+ if [[ $mode == "s3cmd" ]] || [[ $mode == "lumi" ]] ; then
+ echo "rclone mkdir ${storage_server}:${bucket_name}"
rclone mkdir ${storage_server}:${bucket_name}
fi
@@ -998,7 +1037,7 @@ dmku3fKA/wrOpWntUTkkoQvknjZDisdmSwU4oFk/on0=
echo "File upload for $infile failed!"
echo "Upload summary:"
cat ${tmp_dir}/upload.log
- rclone deletefile ${storage_server}:${bucket_name}/${partial_path}/$tmp_file
+ rclone deletefile ${storage_server}:${bucket_name}${partial_path}/$tmp_file
rm -f ${tmp_dir}/$tmp_file
rm -f ${tmp_dir}/${tmp_file}_ameta
rm -f ${tmp_dir}/upload.log
@@ -1083,7 +1122,7 @@ dmku3fKA/wrOpWntUTkkoQvknjZDisdmSwU4oFk/on0=
fi
fi
#echo "rclone copy ./${tmp_file}_ameta ${storage_server}:${bucket_name}/${partial_path}"
- if [[ $mode == "swift" ]]||[[ $mode == "s3cmd" ]]; then
+ if [[ $mode == "swift" ]]||[[ $mode == "s3cmd" ]]||[[ $mode == "lumi" ]] ; then
if [[ $silent -eq 0 ]] ; then
#echo "rclone copy -L --progress ${tmp_dir}/${tmp_file}_ameta ${storage_server}:${bucket_name}/${partial_path}"
rclone copy -L --progress ${tmp_dir}/${tmp_file}_ameta ${storage_server}:${bucket_name}/${partial_path}
diff --git a/a-stream b/a-stream
index 695d6e6..eecbd6e 100755
--- a/a-stream
+++ b/a-stream
@@ -52,10 +52,19 @@ do
asis_mode=1
shift
;;
- '--s3cmd' )
+ '--s3cmd' | '--s3allas' )
mode=("s3cmd")
shift
;;
+ '--lumi' )
+ mode=("lumi")
+ shift
+ ;;
+ '--allas' )
+ mode=("swift")
+ shift
+ ;;
+
'--sk' )
secret_key="$2"
if [[ ! -e $secret_key ]]; then
@@ -148,6 +157,10 @@ if [[ $mode == "s3cmd" ]];then
storage_server="s3allas"
fi
+#Rclone through s3
+if [[ $mode == "lumi" ]];then
+ storage_server="lumi-o"
+fi
#source /appl/opt/allas_conf
#input=("$1")