Lumi-O compatibility added

CSCfi · Dec 16, 2022 · fa9de4c · fa9de4c
1 parent f053123
commit fa9de4c
Show file tree

Hide file tree

Showing 10 changed files with 310 additions and 192 deletions.
diff --git a/a-delete b/a-delete
@@ -73,6 +73,10 @@ do
                   mode="s3cmd"
                   shift
                 ;;
+             '--lumi' )
+                 mode="lumi"
+                 shift
+              ;;
 
              '-h' | '--help' )
                   print_help=1
@@ -142,6 +146,9 @@ if [[ $mode == "s3cmd" ]]; then
  storage_server="s3allas"
 fi
 
+if [[ $mode == "lumi" ]]; then
+ storage_server="lumi-o"
+fi
 
 #The method to remove a bucket
 if [[ $remove_bucket -eq 1 ]]; then

diff --git a/a-encrypt b/a-encrypt
@@ -1,3 +1,4 @@
+#!/bin/bash
 
 
 #default user
@@ -51,6 +52,11 @@ do
                   mode=("s3cmd")
                   shift
                 ;;
+
+              '--lumi' )
+                  mode=("lumi")
+                  shift
+                ;;
              '-b' | '--bucket' )
                   bucket="$2"
                   shift
@@ -190,6 +196,12 @@ if [[ $mode == "s3cmd" ]];then
    storage_server="s3allas"
 fi
 
+#Rclone through s3
+if [[ $mode == "lumi" ]];then
+   storage_server="lumi-o"
+fi
+
+
 #source /appl/opt/allas_conf
 #input=("$1")
 

diff --git a/a-find b/a-find
@@ -15,17 +15,17 @@ source $inst_root/a_env_conf
 source $inst_root/allas-lib
 
 #local variables
-user=($USER)
-bucket_name=("not_defined")
-silent=(0)
-print_help=(0)
-os_project_name=("$OS_PROJECT_NAME")
-input_def=("")
-mode=("swift")
+user="$USER"
+bucket_name="not_defined"
+silent=0
+print_help=0
+os_project_name="$OS_PROJECT_NAME"
+input_def=""
+mode="swift"
 #tmp_dir=("${tmp_root}/a_put_$$_tmp")
-show_filelist=(0)
-query=("")
-all_buckets=(0)
+show_filelist=0
+query=""
+all_buckets=0
 
 
 # read customer defaults
@@ -41,7 +41,7 @@ while [[ $# -ge 1 ]]
 do
   case "$1" in
              '--user' | '-u' )
-                  user=("$2")
+                  user="$2"
                   shift
                   shift
                 ;;
@@ -57,19 +57,23 @@ do
                   shift
                 ;;
              '-silent' | '-s')
-                  silent=(1)
+                  silent=1
+                  shift
+                ;;
+            '--lumi')
+                  mode="lumi"
                   shift
                 ;;
             '--files' | '-f' )
-                  show_filelist=(1)
+                  show_filelist=1
                   shift
                 ;;
             '--all_bukets' | '-a' | '--all_buckets' )
-                  bucket_name=("$(swift list| tr  '\n' ' ')")
+                  all_buckets=1
                   shift
                 ;;
              '-h' | '--help' )
-                  print_help=(1)
+                  print_help=1
                   shift
                 ;;
              *)
@@ -78,7 +82,7 @@ do
                       echo "Query term is: $query"
                       exit 1
                    fi
-                   query=("$1")
+                   query="$1"
                    shift                       # No more switches
                 ;;
     esac
@@ -170,6 +174,7 @@ project_label=$(echo ${os_project_name} |  sed -e s/"project_"/""/g)
 #Check if connection works
 if [[ $mode == "swift" ]]
 then
+  storage_server="allas"
   test=$(rclone about ${storage_server}: 2> /dev/null | wc -l)
   #test=$(swift stat 2> /dev/null | grep -c "Account:")
   if [[ $test -lt 1 ]]
@@ -184,6 +189,21 @@ then
 fi
 
 
+if [[ $mode == "s3cmf" ]]
+then
+  storage_server="s3allas"
+fi
+
+if [[ $mode == "lumi" ]]
+then
+  storage_server="lumi-o"
+fi
+
+if [[ $all_buckets -eq 1 ]]; then
+ #bucket_name=("$(swift list| tr  '\n' ' ')")
+ bucket_name=("$(rclone lsd  ${storage_server}: | awk '{print $NF}'| tr  '\n' ' ')")
+fi
+
 #define standard buckets
 std_buckets=("${user}-${project_label}-MISC ${user}-${project_label}-HOME ${project_label}-puhti-SCRATCH ${project_label}-puhti-PROJAPPL ${user}-${project_label}-pub ${user}-${project_label}-flip")
 
@@ -240,7 +260,7 @@ do
     echo "----------------------------------------------"
     echo "Checking bucket: $bn"
   fi
-  rclone ls ${storage_server}:$bn 2> /dev/null | awk '{print $2}' | grep -v "_ameta$" | grep "$query" > ./$$_find_tmp
+  rclone ls ${storage_server}:$bn 2> /dev/null | awk '{print "x " $0}' | tr -s ' '|  cut -d " " -f 3- | grep -v "_ameta$" | grep "$query" > ./$$_find_tmp
   object_hits=$(cat ./$$_find_tmp | wc -l)
   if [[ $object_hits -gt 0 ]];then
     if [ $silent -eq 0 ]; then

diff --git a/a-flip b/a-flip
@@ -25,6 +25,15 @@ input_def=("")
 mode=("swift")
 #tmp_dir=("${tmp_root}/a_put_$$_tmp")
 
+
+# read customer defaults
+if [[ -e $HOME/.a_tools_conf ]]; then
+   customized=1
+   source $HOME/.a_tools_conf
+else
+   customized=0
+fi
+
 #Process command line
 while [[ $# -ge 1 ]]
 do
@@ -84,8 +93,17 @@ then
     echo "Please try setting up the connection again."
     exit 1
   fi
-else
-  echo "a-flip is available only in swift mode"
+fi
+
+if [[ $mode == "s3cmd" ]]
+then
+  echo "a-flip is not available s3allas."
+  exit
+fi
+
+if [[ $mode == "lumi" ]]
+then
+  storage_server="lumi-pub"
 fi
 
 
@@ -104,6 +122,12 @@ echo "Files to be uploaded: $input_def"
 project_label=$(echo ${os_project_name} |  sed -e s/"project_"/""/g)
 bucket_name=("${user}-${project_label}-flip")
 
+if [[ $mode == "lumi" ]]; then
+   echo "rclone mkdir ${storage_server}:${bucket_name}"
+   rclone mkdir ${storage_server}:${bucket_name}
+fi
+
+
 make_temp_dir
 
 for input in $input_def
@@ -196,15 +220,14 @@ do
 
   #upload
 
-  if [ $mode == "swift" ]
-  then
-    # For less than 5GB files rclone is used for uploading
+  #if [[ $mode == "swift" ]]; then
+  # For less than 5GB files rclone is used for uploading
 
-    echo "Uploading data to allas."
-    # echo "rclone copy --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}/${partial_path}"
-    rclone copy -L --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}
-    exitcode=$?
-    if [ $exitcode -ne 0 ]; then
+  echo "Uploading data to allas."
+  echo "rclone copy --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}"
+  rclone copy -L --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}
+  exitcode=$?
+  if [ $exitcode -ne 0 ]; then
        echo ""
        echo "File upload for $infile failed"
 
@@ -213,29 +236,28 @@ do
        rm -f ${tmp_dir}/${tmp_file}_ameta
        clean_temp_dir
        exit 1
-    fi
+  fi
 
-    # rclone md5sums can be calculated only for files that are smaller than 5GB
-    if [[ $tot_size -lt 5000000 ]]
-    then
-       echo "Confirming upload..."
-       #checksums for local and allas files
-       sum1=($(md5sum ${tmp_dir}/$tmp_file))
-       sum2=($(rclone md5sum  ${storage_server}:${bucket_name}/$tmp_file))
-
-       #check is checksums match
-       if [[ ${sum1[0]} !=  ${sum2[0]} ]]
-       then
-         echo "Upload of $input was not successful!"
-         echo "Cleaning the failed upload"
-         rclone deletefile ${storage_server}:${bucket_name}/$tmp_file
-         rm -f ${tmp_dir}/$tmp_file
-         rm -f ${tmp_dir}/${tmp_file}_ameta
-	 clean_temp_dir
-         exit 1
-       fi
-       echo "$input OK"
-    fi
+  # rclone md5sums can be calculated only for files that are smaller than 5GB
+  if [[ $tot_size -lt 5000000 ]]
+  then
+     echo "Confirming upload..."
+     #checksums for local and allas files
+     sum1=($(md5sum ${tmp_dir}/$tmp_file))
+     sum2=($(rclone md5sum  ${storage_server}:${bucket_name}/$tmp_file))
+
+     #check is checksums match
+     if [[ ${sum1[0]} !=  ${sum2[0]} ]]
+     then
+       echo "Upload of $input was not successful!"
+       echo "Cleaning the failed upload"
+       rclone deletefile ${storage_server}:${bucket_name}/$tmp_file
+       rm -f ${tmp_dir}/$tmp_file
+       rm -f ${tmp_dir}/${tmp_file}_ameta
+       clean_temp_dir
+       exit 1
+     fi
+     echo "$input OK"
   fi
 
 
@@ -251,7 +273,12 @@ do
   rm -f ${tmp_dir}/${tmp_file}_ameta
 
   echo "$input uploaded to ${bucket_name}"
-  echo "Public link: https://a3s.fi/${bucket_name}/$tmp_file"
+  if [[ $mode == "swift" ]]; then
+     echo "Public link: https://a3s.fi/${bucket_name}/$tmp_file"
+  fi
+  if [[ $mode == "lumi" ]]; then
+     echo "Public link:  https://${os_project_name}.lumidata.eu/${bucket_name}/$tmp_file"
+  fi
 
   tmp_file=("not_defined")
 
@@ -271,18 +298,21 @@ done
 #Publish and rebuild index file
 
 echo '<html>' >  ${tmp_dir}/index.html
-for i in $(swift list $bucket_name | grep -v '_ameta$' )
+#for i in $(swift list $bucket_name | grep -v '_ameta$' )
+for i in $(rclone ls  ${storage_server}:${bucket_name} | grep -v '_ameta$' | awk '{print "X "$0}' | tr -s " " | cut -d " " -f 3- )
 do
-echo '<li><a href="'$i'">'$i'</a></li>' >> ${tmp_dir}/index.html
+  echo '<li><a href="'$i'">'$i'</a></li>' >> ${tmp_dir}/index.html
 done
 echo '</html>' >>  ${tmp_dir}/index.html
-rclone copy  ${tmp_dir}/index.html ${storage_server}:${bucket_name}/
 
+rclone copy  ${tmp_dir}/index.html ${storage_server}:${bucket_name}/
 rm -f   ${tmp_dir}/index.html
 
-#set access control
-swift post ${bucket_name} --read-acl ".r:*,.rlistings"
-swift post ${bucket_name}_segments --read-acl ".r:*,.rlistings"
+#set access control in case of Allas/swift
+if [[ $mode == "swift" ]]; then
+  swift post ${bucket_name} --read-acl ".r:*,.rlistings"
+  swift post ${bucket_name}_segments --read-acl ".r:*,.rlistings"
+fi
 
 clean_temp_dir
 echo ""

diff --git a/a-get b/a-get
@@ -175,7 +175,7 @@ then
   #test=$(swift stat 2> /dev/null | grep -c "Account:")
   if [[ $test -lt 1 ]]
   then 
-    echo "No connection to Allas!"
+    echo "No connection to $storage_server!"
     echo "Please try setting up the connection again"
     echo "by running command:"
     echo ""
@@ -189,6 +189,14 @@ if [[ $mode == "s3cmd" ]];then
    storage_server="s3allas"
 fi
 
+#lumi
+if [[ $mode == "lumi" ]];then
+   storage_server="lumi-o"
+fi
+
+
+
+
 #source /appl/opt/allas_conf
 #input=("$1")
 
@@ -491,7 +499,6 @@ else
              rclone cat "${storage_server}:$object_name" | crypt4gh decrypt --sk "$secret_key" >  $target_name
              exitcode=$?
            else
-             echo "eka ${target_name:0:1}"
              if [[ ${target_name:0:1} != "/" ]]; then
                 #this is needed to cope with file names containing :
                 rclone -P copyto "${storage_server}:$object_name"  "./$target_name"

diff --git a/a-info b/a-info
@@ -104,6 +104,12 @@ if [[ $mode == "s3cmd" ]]; then
   storage_server="s3allas"
 fi 
 
+if [[ $mode == "lumi" ]]; then
+  storage_server="lumi-o"
+fi 
+
+
+
 
 if [[ $object_name == "" ]] ; then 
   for buc in $(rclone lsd ${storage_server}: | awk '{print $NF}')