diff --git a/a-delete b/a-delete index 4e6b703..78abd87 100755 --- a/a-delete +++ b/a-delete @@ -73,6 +73,10 @@ do mode="s3cmd" shift ;; + '--lumi' ) + mode="lumi" + shift + ;; '-h' | '--help' ) print_help=1 @@ -142,6 +146,9 @@ if [[ $mode == "s3cmd" ]]; then storage_server="s3allas" fi +if [[ $mode == "lumi" ]]; then + storage_server="lumi-o" +fi #The method to remove a bucket if [[ $remove_bucket -eq 1 ]]; then diff --git a/a-encrypt b/a-encrypt index 854e239..d2de303 100755 --- a/a-encrypt +++ b/a-encrypt @@ -1,3 +1,4 @@ +#!/bin/bash #default user @@ -51,6 +52,11 @@ do mode=("s3cmd") shift ;; + + '--lumi' ) + mode=("lumi") + shift + ;; '-b' | '--bucket' ) bucket="$2" shift @@ -190,6 +196,12 @@ if [[ $mode == "s3cmd" ]];then storage_server="s3allas" fi +#Rclone through s3 +if [[ $mode == "lumi" ]];then + storage_server="lumi-o" +fi + + #source /appl/opt/allas_conf #input=("$1") diff --git a/a-find b/a-find index 65f2ec3..a4ee3d9 100755 --- a/a-find +++ b/a-find @@ -15,17 +15,17 @@ source $inst_root/a_env_conf source $inst_root/allas-lib #local variables -user=($USER) -bucket_name=("not_defined") -silent=(0) -print_help=(0) -os_project_name=("$OS_PROJECT_NAME") -input_def=("") -mode=("swift") +user="$USER" +bucket_name="not_defined" +silent=0 +print_help=0 +os_project_name="$OS_PROJECT_NAME" +input_def="" +mode="swift" #tmp_dir=("${tmp_root}/a_put_$$_tmp") -show_filelist=(0) -query=("") -all_buckets=(0) +show_filelist=0 +query="" +all_buckets=0 # read customer defaults @@ -41,7 +41,7 @@ while [[ $# -ge 1 ]] do case "$1" in '--user' | '-u' ) - user=("$2") + user="$2" shift shift ;; @@ -57,19 +57,23 @@ do shift ;; '-silent' | '-s') - silent=(1) + silent=1 + shift + ;; + '--lumi') + mode="lumi" shift ;; '--files' | '-f' ) - show_filelist=(1) + show_filelist=1 shift ;; '--all_bukets' | '-a' | '--all_buckets' ) - bucket_name=("$(swift list| tr '\n' ' ')") + all_buckets=1 shift ;; '-h' | '--help' ) - print_help=(1) + print_help=1 shift ;; *) @@ -78,7 +82,7 @@ do echo "Query term is: $query" exit 1 fi - query=("$1") + query="$1" shift # No more switches ;; esac @@ -170,6 +174,7 @@ project_label=$(echo ${os_project_name} | sed -e s/"project_"/""/g) #Check if connection works if [[ $mode == "swift" ]] then + storage_server="allas" test=$(rclone about ${storage_server}: 2> /dev/null | wc -l) #test=$(swift stat 2> /dev/null | grep -c "Account:") if [[ $test -lt 1 ]] @@ -184,6 +189,21 @@ then fi +if [[ $mode == "s3cmf" ]] +then + storage_server="s3allas" +fi + +if [[ $mode == "lumi" ]] +then + storage_server="lumi-o" +fi + +if [[ $all_buckets -eq 1 ]]; then + #bucket_name=("$(swift list| tr '\n' ' ')") + bucket_name=("$(rclone lsd ${storage_server}: | awk '{print $NF}'| tr '\n' ' ')") +fi + #define standard buckets std_buckets=("${user}-${project_label}-MISC ${user}-${project_label}-HOME ${project_label}-puhti-SCRATCH ${project_label}-puhti-PROJAPPL ${user}-${project_label}-pub ${user}-${project_label}-flip") @@ -240,7 +260,7 @@ do echo "----------------------------------------------" echo "Checking bucket: $bn" fi - rclone ls ${storage_server}:$bn 2> /dev/null | awk '{print $2}' | grep -v "_ameta$" | grep "$query" > ./$$_find_tmp + rclone ls ${storage_server}:$bn 2> /dev/null | awk '{print "x " $0}' | tr -s ' '| cut -d " " -f 3- | grep -v "_ameta$" | grep "$query" > ./$$_find_tmp object_hits=$(cat ./$$_find_tmp | wc -l) if [[ $object_hits -gt 0 ]];then if [ $silent -eq 0 ]; then diff --git a/a-flip b/a-flip index 1eaab13..6ebdc85 100755 --- a/a-flip +++ b/a-flip @@ -25,6 +25,15 @@ input_def=("") mode=("swift") #tmp_dir=("${tmp_root}/a_put_$$_tmp") + +# read customer defaults +if [[ -e $HOME/.a_tools_conf ]]; then + customized=1 + source $HOME/.a_tools_conf +else + customized=0 +fi + #Process command line while [[ $# -ge 1 ]] do @@ -84,8 +93,17 @@ then echo "Please try setting up the connection again." exit 1 fi -else - echo "a-flip is available only in swift mode" +fi + +if [[ $mode == "s3cmd" ]] +then + echo "a-flip is not available s3allas." + exit +fi + +if [[ $mode == "lumi" ]] +then + storage_server="lumi-pub" fi @@ -104,6 +122,12 @@ echo "Files to be uploaded: $input_def" project_label=$(echo ${os_project_name} | sed -e s/"project_"/""/g) bucket_name=("${user}-${project_label}-flip") +if [[ $mode == "lumi" ]]; then + echo "rclone mkdir ${storage_server}:${bucket_name}" + rclone mkdir ${storage_server}:${bucket_name} +fi + + make_temp_dir for input in $input_def @@ -196,15 +220,14 @@ do #upload - if [ $mode == "swift" ] - then - # For less than 5GB files rclone is used for uploading + #if [[ $mode == "swift" ]]; then + # For less than 5GB files rclone is used for uploading - echo "Uploading data to allas." - # echo "rclone copy --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}/${partial_path}" - rclone copy -L --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name} - exitcode=$? - if [ $exitcode -ne 0 ]; then + echo "Uploading data to allas." + echo "rclone copy --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}" + rclone copy -L --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name} + exitcode=$? + if [ $exitcode -ne 0 ]; then echo "" echo "File upload for $infile failed" @@ -213,29 +236,28 @@ do rm -f ${tmp_dir}/${tmp_file}_ameta clean_temp_dir exit 1 - fi + fi - # rclone md5sums can be calculated only for files that are smaller than 5GB - if [[ $tot_size -lt 5000000 ]] - then - echo "Confirming upload..." - #checksums for local and allas files - sum1=($(md5sum ${tmp_dir}/$tmp_file)) - sum2=($(rclone md5sum ${storage_server}:${bucket_name}/$tmp_file)) - - #check is checksums match - if [[ ${sum1[0]} != ${sum2[0]} ]] - then - echo "Upload of $input was not successful!" - echo "Cleaning the failed upload" - rclone deletefile ${storage_server}:${bucket_name}/$tmp_file - rm -f ${tmp_dir}/$tmp_file - rm -f ${tmp_dir}/${tmp_file}_ameta - clean_temp_dir - exit 1 - fi - echo "$input OK" - fi + # rclone md5sums can be calculated only for files that are smaller than 5GB + if [[ $tot_size -lt 5000000 ]] + then + echo "Confirming upload..." + #checksums for local and allas files + sum1=($(md5sum ${tmp_dir}/$tmp_file)) + sum2=($(rclone md5sum ${storage_server}:${bucket_name}/$tmp_file)) + + #check is checksums match + if [[ ${sum1[0]} != ${sum2[0]} ]] + then + echo "Upload of $input was not successful!" + echo "Cleaning the failed upload" + rclone deletefile ${storage_server}:${bucket_name}/$tmp_file + rm -f ${tmp_dir}/$tmp_file + rm -f ${tmp_dir}/${tmp_file}_ameta + clean_temp_dir + exit 1 + fi + echo "$input OK" fi @@ -251,7 +273,12 @@ do rm -f ${tmp_dir}/${tmp_file}_ameta echo "$input uploaded to ${bucket_name}" - echo "Public link: https://a3s.fi/${bucket_name}/$tmp_file" + if [[ $mode == "swift" ]]; then + echo "Public link: https://a3s.fi/${bucket_name}/$tmp_file" + fi + if [[ $mode == "lumi" ]]; then + echo "Public link: https://${os_project_name}.lumidata.eu/${bucket_name}/$tmp_file" + fi tmp_file=("not_defined") @@ -271,18 +298,21 @@ done #Publish and rebuild index file echo '' > ${tmp_dir}/index.html -for i in $(swift list $bucket_name | grep -v '_ameta$' ) +#for i in $(swift list $bucket_name | grep -v '_ameta$' ) +for i in $(rclone ls ${storage_server}:${bucket_name} | grep -v '_ameta$' | awk '{print "X "$0}' | tr -s " " | cut -d " " -f 3- ) do -echo '
  • '$i'
  • ' >> ${tmp_dir}/index.html + echo '
  • '$i'
  • ' >> ${tmp_dir}/index.html done echo '' >> ${tmp_dir}/index.html -rclone copy ${tmp_dir}/index.html ${storage_server}:${bucket_name}/ +rclone copy ${tmp_dir}/index.html ${storage_server}:${bucket_name}/ rm -f ${tmp_dir}/index.html -#set access control -swift post ${bucket_name} --read-acl ".r:*,.rlistings" -swift post ${bucket_name}_segments --read-acl ".r:*,.rlistings" +#set access control in case of Allas/swift +if [[ $mode == "swift" ]]; then + swift post ${bucket_name} --read-acl ".r:*,.rlistings" + swift post ${bucket_name}_segments --read-acl ".r:*,.rlistings" +fi clean_temp_dir echo "" diff --git a/a-get b/a-get index 875aad9..c509ab9 100755 --- a/a-get +++ b/a-get @@ -175,7 +175,7 @@ then #test=$(swift stat 2> /dev/null | grep -c "Account:") if [[ $test -lt 1 ]] then - echo "No connection to Allas!" + echo "No connection to $storage_server!" echo "Please try setting up the connection again" echo "by running command:" echo "" @@ -189,6 +189,14 @@ if [[ $mode == "s3cmd" ]];then storage_server="s3allas" fi +#lumi +if [[ $mode == "lumi" ]];then + storage_server="lumi-o" +fi + + + + #source /appl/opt/allas_conf #input=("$1") @@ -491,7 +499,6 @@ else rclone cat "${storage_server}:$object_name" | crypt4gh decrypt --sk "$secret_key" > $target_name exitcode=$? else - echo "eka ${target_name:0:1}" if [[ ${target_name:0:1} != "/" ]]; then #this is needed to cope with file names containing : rclone -P copyto "${storage_server}:$object_name" "./$target_name" diff --git a/a-info b/a-info index 4653830..a360975 100755 --- a/a-info +++ b/a-info @@ -104,6 +104,12 @@ if [[ $mode == "s3cmd" ]]; then storage_server="s3allas" fi +if [[ $mode == "lumi" ]]; then + storage_server="lumi-o" +fi + + + if [[ $object_name == "" ]] ; then for buc in $(rclone lsd ${storage_server}: | awk '{print $NF}') diff --git a/a-list b/a-list index 8ef8450..adcf439 100755 --- a/a-list +++ b/a-list @@ -34,7 +34,7 @@ while [[ $# -ge 1 ]] do case "$1" in '--user' | '-u' ) - user=("$2") + user="$2" shift shift ;; @@ -43,6 +43,10 @@ do options=("$options --lh ") shift ;; + '--lumi' | '-L' ) + mode="lumi" + shift + ;; '--dir' | '-d' ) dir_like=(1) shift @@ -91,6 +95,10 @@ fi if [[ $mode == "s3cmd" ]];then storage_server="s3allas" fi +if [[ $mode == "lumi" ]];then + storage_server="lumi-o" +fi + #check that we have a bucket only if [ $(echo $bucket_name | grep -o "/" | wc -l) -gt 0 ]; then @@ -148,14 +156,10 @@ shared_list="buckets_shared_to" if [[ $mode == "swift" ]] && [[ $bucket_name != "" ]]; then - #my_auth=$( echo $OS_STORAGE_URL | awk -F "/" '{print $NF}' ) bucket_readable=$(rclone ls ${storage_server}:$bucket_name 2> /dev/null | wc -l) own_bucket=$(rclone lsd allas: | awk '{ print $NF}' | grep -c -w $bucket_name ) - #bucket_auth=$(swift stat $bucket_name 2>/dev/null | grep "Account:" | awk '{print $2}') - #echo my: $my_auth bucket $bucket_auth # case: bucket reable but not owned if [[ $bucket_readable -gt 0 ]] && [[ $own_bucket -eq 0 ]]; then - #if [[ $my_auth != $bucket_auth ]] && [[ $bucket_auth != "" ]]; then # do we already have a list of shared buckets check_os=$(rclone ls ${storage_server}:${a_tools_bucket}/$shared_list 2> /dev/null | wc -l) if [[ $check_os -eq 1 ]]; then @@ -168,7 +172,6 @@ if [[ $mode == "swift" ]] && [[ $bucket_name != "" ]]; then #rm .$$_a_shared_tmp fi else - #echo " rclone mkdir ${storage_server}:${a_tools_bucket} " rclone mkdir ${storage_server}:${a_tools_bucket} echo "$bucket_name" | rclone rcat ${storage_server}:${a_tools_bucket}/$shared_list echo "Adding bucket $bucket_name to the list of buckets shared to this project" @@ -190,7 +193,7 @@ if [[ $mode == "swift" ]] && [[ $bucket_name != "" ]]; then rclone delete ${storage_server}:${a_tools_bucket}/$shared_list fi #rm .$$_a_shared_tmp - echo "Bucket $bucket_name is no longe accessible. Removing it form the list of shared buckets" + echo "Bucket $bucket_name is no longer accessible. Removing it form the list of shared buckets." fi fi fi @@ -202,7 +205,6 @@ fi if [[ $dir_like -eq 1 ]]; then - echo rclone lsf ${storage_server}:${full_name} rclone lsf ${storage_server}:${full_name} | grep -v "_ameta$" exit 0 fi @@ -223,11 +225,13 @@ if [[ $bucket_name == "" ]]; then # swift list $options else - if [[ $mode == "s3cmd" ]]; then + if [[ $mode == "s3cmd" ]] || [[ $mode == "lumi" ]] ; then if [[ $long_listing -eq 0 ]]; then - rclone ls ${storage_server}:$full_name | grep -v "_ameta$" | awk '{print "'$full_name'/" $NF}' + rclone ls ${storage_server}:$bucket_name | grep -v "_ameta$" | awk '{print "x " $0}' | tr -s ' ' | cut -d " " -f 3- | awk '{ print "'$bucket_name'/" $0}' | grep $full_name + #rclone ls ${storage_server}:$full_name | grep -v "_ameta$" | awk '{print "'$full_name'/" $NF}' else - rclone lsl ${storage_server}:$full_name | awk '{print $1"\t"$2"\t"$3"\t'$full_name'/" $4}' + #rclone lsl ${storage_server}:$bucket_name | grep -v "_ameta$" | awk '{print "x " $0}' | tr -s ' ' | cut -d " " -f 3- | awk '{ print "'$full_name'/ " $0}' + rclone lsl ${storage_server}:$full_name | grep -v "_ameta$" fi else # listing in swift mode diff --git a/a-publish b/a-publish index f21819e..8cf2d6e 100755 --- a/a-publish +++ b/a-publish @@ -28,6 +28,16 @@ mode="swift" #tmp_dir="${tmp_root}/a_put_$$_tmp" make_temp_dir +# read customer defaults +if [[ -e $HOME/.a_tools_conf ]]; then + customized=1 + source $HOME/.a_tools_conf +else + customized=0 +fi + + + #Process command line while [[ $# -ge 1 ]] do @@ -53,7 +63,14 @@ do mode=("s3cmd") shift ;; - + '--lumi' ) + mode=("lumi") + shift + ;; + '--allas' ) + mode=("swift") + shift + ;; '--index' | '-i' ) index_mode=$2 if [[ $index_mode == "static" || $index_mode == "dynamic" ]]; then @@ -134,43 +151,6 @@ exit fi -#Configure rclone -if [ ! -e $HOME/.config/rclone ] -then - mkdir -p $HOME/.config/rclone -fi -if [ -e $HOME/.config/rclone/rclone.conf ] -then - rc_check=$(grep -c $storage_server $HOME/.config/rclone/rclone.conf) - if [ $rc_check -lt 1 ] - then - echo '[pouta]' >> $HOME/.config/rclone/rclone.conf - echo 'type = swift' >> $HOME/.config/rclone/rclone.conf - echo 'env_auth = true' >> $HOME/.config/rclone/rclone.conf - fi -else - echo '[pouta]' >> $HOME/.config/rclone/rclone.conf - echo 'type = swift' >> $HOME/.config/rclone/rclone.conf - echo 'env_auth = true' >> $HOME/.config/rclone/rclone.conf -fi - -#Assign project to be used if not defined -if [ $os_project_name == "" ] -then - if [ -e $HOME/.allas_default ] - then - source $HOME/.allas_default - else - echo "Default project is not defined" - source $allas_conf_path -user $user - echo "os_project_name=$OS_PROJECT_NAME" > $HOME/.allas_default - echo "Default allas project is stored to \$HOME/.allas_default" - echo "" - fi - source $HOME/.allas_default -fi - - #Check if connection works if [[ $mode == "swift" ]] then @@ -184,12 +164,6 @@ then fi fi -#source /appl/opt/allas_conf -#input=("$1") - -#check free space in $WRKDIR -#quota_s=($(lfs quota -q -u $USER $WRKDIR)) -#free_space=$(expr ${quota_s[2]} - ${quota_s[1]}) echo "Files to be uploaded: $input_def" @@ -199,9 +173,20 @@ if [[ "$bucket_name" == "not_defined" ]] then bucket_name=("${user}-${project_label}-pub") fi + +#In case of lumi the bucket must be cereated +if [[ $mode == "lumi" ]]; then + storage_server="lumi-pub" + echo "rclone mkdir ${storage_server}:${bucket_name}" + rclone mkdir ${storage_server}:${bucket_name} +fi + + echo "Bucket: $bucket_name" + + for input in $input_def do echo "Processing: $input" @@ -269,82 +254,67 @@ do then echo "This file or directory is too big for this tool" echo "Total size: ${tot_size}K" - echo "Please use swift or rclone command to upload the data to allas" + echo "Please use swift or rclone command to upload the data to $storage_server" rm -f ${tmp_dir}/${tmp_file}_ameta clean_temp_dir exit 1 fi -# if [[ $tot_size -gt $free_space ]] -# then -# echo "There is not enough space for the temporary files." -# echo "$input contains $num_files files or directories that take ${tot_size}K of disk space" -# echo "Available free space is ${free_space}K" -# echo "" -# rm -f ${tmp_dir}/${tmp_file}_ameta -# rmdir ${tmp_dir} -# exit 1 -# fi - - #this is not a good approach. - #test if a symbolic link would do - #cp $input ${tmp_dir}/$tmp_file ln -s ${file_path} ${tmp_dir}/$tmp_file #upload - if [ $mode == "swift" ] - then - # For less than 5GB files rclone is used for uploading + #if [[ $mode == "swift" ]] + #then + # For less than 5GB files rclone is used for uploading - echo "Uploading data to allas." - # echo "rclone copy --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}/${partial_path}" - rclone copy -L --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}/${partial_path} - exitcode=$? - if [ $exitcode -ne 0 ]; then - echo "" - echo "File upload for $infile failed" + echo "Uploading data to allas." + # echo "rclone copy --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}/${partial_path}" + rclone copy -L --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}/${partial_path} + exitcode=$? + if [[ $exitcode -ne 0 ]]; then + echo "" + echo "File upload for $infile failed" + if [[ $partial_path == "" ]]; then + rclone deletefile ${storage_server}:${bucket_name}/$tmp_file + else + rclone deletefile ${storage_server}:${bucket_name}/${partial_path}/$tmp_file + fi + rm -f ${tmp_dir}/$tmp_file + if [[ $include_ameta -eq 1 ]]; then + rm -f ${tmp_dir}/${tmp_file}_ameta + fi + clean_temp_dir + exit 1 + fi + + # rclone md5sums can be calculated only for files that are smaller than 5GB + if [[ $tot_size -lt 5000000 ]] + then + echo "Confirming upload..." + #checksums for local and allas files + sum1=($(md5sum ${tmp_dir}/$tmp_file)) + + if [[ $partial_path == "" ]]; then + sum2=($(rclone md5sum ${storage_server}:${bucket_name}/$tmp_file)) + else + sum2=($(rclone md5sum ${storage_server}:${bucket_name}/${partial_path}/$tmp_file)) + fi + #check if checksums match + if [[ ${sum1[0]} != ${sum2[0]} ]] + then + echo "Upload of $input was not successful!" + echo "Cleaning the failed upload" if [[ $partial_path == "" ]]; then - rclone deletefile ${storage_server}:${bucket_name}/$tmp_file + rclone deletefile ${storage_server}:${bucket_name}/$tmp_file else - rclone deletefile ${storage_server}:${bucket_name}/${partial_path}/$tmp_file + rclone deletefile ${storage_server}:${bucket_name}/${partial_path}/$tmp_file fi rm -f ${tmp_dir}/$tmp_file - if [[ $include_ameta -eq 1 ]]; then - rm -f ${tmp_dir}/${tmp_file}_ameta - fi + rm -f ${tmp_dir}/${tmp_file}_ameta clean_temp_dir exit 1 - fi - - # rclone md5sums can be calculated only for files that are smaller than 5GB - if [[ $tot_size -lt 5000000 ]] - then - echo "Confirming upload..." - #checksums for local and allas files - sum1=($(md5sum ${tmp_dir}/$tmp_file)) - - if [[ $partial_path == "" ]]; then - sum2=($(rclone md5sum ${storage_server}:${bucket_name}/$tmp_file)) - else - sum2=($(rclone md5sum ${storage_server}:${bucket_name}/${partial_path}/$tmp_file)) - fi - #check if checksums match - if [[ ${sum1[0]} != ${sum2[0]} ]] - then - echo "Upload of $input was not successful!" - echo "Cleaning the failed upload" - if [[ $partial_path == "" ]]; then - rclone deletefile ${storage_server}:${bucket_name}/$tmp_file - else - rclone deletefile ${storage_server}:${bucket_name}/${partial_path}/$tmp_file - fi - rm -f ${tmp_dir}/$tmp_file - rm -f ${tmp_dir}/${tmp_file}_ameta - clean_temp_dir - exit 1 - fi - echo "$input OK" - fi + fi + echo "$input OK" fi @@ -362,10 +332,18 @@ do fi echo "$input uploaded to ${bucket_name}" + + if [[ $mode == "swift" ]]; then + https_root="https://a3s.fi" + fi + if [[ $mode == "lumi" ]]; then + https_root="https://${os_project_name}.lumidata.eu" + fi + if [[ $partial_path == "" ]]; then - echo "Public link: https://a3s.fi/${bucket_name}/$tmp_file" + echo "Public link: ${https_root}/${bucket_name}/$tmp_file" else - echo "Public link: https://a3s.fi/${bucket_name}/${partial_path}/$tmp_file" + echo "Public link: ${https_root}/${bucket_name}/${partial_path}/$tmp_file" fi tmp_file=("not_defined") @@ -377,7 +355,7 @@ if [[ $index_mode == "static" ]];then echo '' > ${tmp_dir}/index.html echo '' >> ${tmp_dir}/index.html echo "

    Contents of bucket $bucket_name

    " >> ${tmp_dir}/index.html - rclone lsl allas:$bucket_name | grep -v "_ameta" | awk '{print "

    "}' >> ${tmp_dir}/index.html + rclone lsl ${storage_server}:$bucket_name | grep -v "_ameta" | awk '{print ""}' >> ${tmp_dir}/index.html echo '
    "$1""$2""$3""$4"info
    "$1""$2""$3""$4"info
    ' >> ${tmp_dir}/index.html echo '' >> ${tmp_dir}/index.html else @@ -390,8 +368,10 @@ rclone copy ${tmp_dir}/index.html ${storage_server}:${bucket_name}/ rm -f ${tmp_dir}/index.html #set access control -swift post ${bucket_name} --read-acl ".r:*,.rlistings" -swift post "${bucket_name}_segments" --read-acl ".r:*,.rlistings" +if [[ $mode == "swift" ]]; then + swift post ${bucket_name} --read-acl ".r:*,.rlistings" + swift post "${bucket_name}_segments" --read-acl ".r:*,.rlistings" +fi clean_temp_dir echo "" diff --git a/a-put b/a-put index d93173f..550ea64 100755 --- a/a-put +++ b/a-put @@ -69,7 +69,7 @@ do shift ;; '--project' | '-p' ) - os_project_name=($2) + os_project_name="$2" shift shift ;; @@ -89,8 +89,12 @@ do shift shift ;; + '--lumi' | '-L' ) + mode="lumi" + shift + ;; '--s3cmd' | '--s3' | '-S' ) - mode=("s3cmd") + mode="s3cmd" shift ;; '--compress' | '-c') @@ -103,7 +107,7 @@ do shift ;; '-s' | '--silent' ) - silent=(1) + silent=1 shift ;; '-h' | '--help' ) @@ -111,12 +115,12 @@ do shift ;; '--user' | '-u' ) - user=("$2") + user="$2" shift shift ;; '--tmpdir' | '-t' ) - tmp_root=("$2") + tmp_root="$2" shift shift ;; @@ -428,6 +432,9 @@ fi if [[ $mode == "s3cmd" ]]; then storage_server="s3allas" fi +if [[ $mode == "lumi" ]]; then + storage_server="lumi-o" +fi #check free space in tmpdir @@ -675,7 +682,29 @@ dmku3fKA/wrOpWntUTkkoQvknjZDisdmSwU4oFk/on0= fi partial_path=$(dirname $file_path | sed -e s/"\/projappl\/$puhti_project"/""/g) fi - + + #Lumi project + if [ $(echo $file_path | cut -c1-8) == "/project" ] + then + puhti_project=$(echo $file_path/ |awk -F "/" '{print $3}') + bucket_name=("${project_label}-${local_host}-project") + if [[ $os_project_name != $puhti_project ]] && [[ $user_answer != "y" ]] ; then + echo "" + echo "NOTE: data locates in Project area of project: $puhti_project" + echo "But it will be stored to ${storage_server} under project: $os_project_name" + echo "Bucket to be used is: $bucket_name" + echo "Is this OK (y/n)?" + read user_answer + if [[ $user_answer != "y" ]]; then + echo "Exiting, data not uploaded." + exit 0 + fi + fi + partial_path=$(dirname $file_path | sed -e s/"\/projappl\/$puhti_project"/""/g) + fi + + + #Puhti FMI-projappl if [ $(echo $file_path | cut -c1-13) == "/fmi/projappl" ] then @@ -763,7 +792,11 @@ dmku3fKA/wrOpWntUTkkoQvknjZDisdmSwU4oFk/on0= echo "host_name: $local_host" >> ${tmp_dir}/${tmp_file}_ameta echo "input: $input" >> ${tmp_dir}/${tmp_file}_ameta echo "original_location: $file_path " >> ${tmp_dir}/${tmp_file}_ameta - echo "protocol: $mode " >> ${tmp_dir}/${tmp_file}_ameta + if [[ $mode == "s3cmd" ]] || [[ $mode == "lumi" ]]; then + echo "protocol: S3 " >> ${tmp_dir}/${tmp_file}_ameta + else + echo "protocol: swift" >> ${tmp_dir}/${tmp_file}_ameta + fi if [[ $compression -eq 1 ]]; then echo "compression: zstd" >> ${tmp_dir}/${tmp_file}_ameta else @@ -963,11 +996,17 @@ dmku3fKA/wrOpWntUTkkoQvknjZDisdmSwU4oFk/on0= if [[ $silent -eq 0 ]] ; then - echo "Uploading data to allas." + echo "Uploading data to ${storage_server}." fi - + + # Lumi-O dont allow upper case letters in bucket names + if [[ $mode == "lumi" ]]; then + bucket_name=$(echo $bucket_name | tr [:upper:] [:lower:] ) + fi + #S3 protocol requires that directory exist before copying - if [[ $mode == "s3cmd" ]]; then + if [[ $mode == "s3cmd" ]] || [[ $mode == "lumi" ]] ; then + echo "rclone mkdir ${storage_server}:${bucket_name}" rclone mkdir ${storage_server}:${bucket_name} fi @@ -998,7 +1037,7 @@ dmku3fKA/wrOpWntUTkkoQvknjZDisdmSwU4oFk/on0= echo "File upload for $infile failed!" echo "Upload summary:" cat ${tmp_dir}/upload.log - rclone deletefile ${storage_server}:${bucket_name}/${partial_path}/$tmp_file + rclone deletefile ${storage_server}:${bucket_name}${partial_path}/$tmp_file rm -f ${tmp_dir}/$tmp_file rm -f ${tmp_dir}/${tmp_file}_ameta rm -f ${tmp_dir}/upload.log @@ -1083,7 +1122,7 @@ dmku3fKA/wrOpWntUTkkoQvknjZDisdmSwU4oFk/on0= fi fi #echo "rclone copy ./${tmp_file}_ameta ${storage_server}:${bucket_name}/${partial_path}" - if [[ $mode == "swift" ]]||[[ $mode == "s3cmd" ]]; then + if [[ $mode == "swift" ]]||[[ $mode == "s3cmd" ]]||[[ $mode == "lumi" ]] ; then if [[ $silent -eq 0 ]] ; then #echo "rclone copy -L --progress ${tmp_dir}/${tmp_file}_ameta ${storage_server}:${bucket_name}/${partial_path}" rclone copy -L --progress ${tmp_dir}/${tmp_file}_ameta ${storage_server}:${bucket_name}/${partial_path} diff --git a/a-stream b/a-stream index 695d6e6..eecbd6e 100755 --- a/a-stream +++ b/a-stream @@ -52,10 +52,19 @@ do asis_mode=1 shift ;; - '--s3cmd' ) + '--s3cmd' | '--s3allas' ) mode=("s3cmd") shift ;; + '--lumi' ) + mode=("lumi") + shift + ;; + '--allas' ) + mode=("swift") + shift + ;; + '--sk' ) secret_key="$2" if [[ ! -e $secret_key ]]; then @@ -148,6 +157,10 @@ if [[ $mode == "s3cmd" ]];then storage_server="s3allas" fi +#Rclone through s3 +if [[ $mode == "lumi" ]];then + storage_server="lumi-o" +fi #source /appl/opt/allas_conf #input=("$1")