s3cmd mode switched to use rclone with s3allas

CSCfi · Apr 23, 2021 · 226d646 · 226d646
1 parent aaef20e
commit 226d646
Show file tree

Hide file tree

Showing 8 changed files with 249 additions and 192 deletions.
diff --git a/a-delete b/a-delete
@@ -7,26 +7,21 @@ check_swift_connection () {
 
     if [[ $test -lt 1 ]]
     then 
-      #if [ -n "$ACTIVE_TOKEN" ]; then
-      #   unset OS_AUTH_TOKEN
-      #   export OS_AUTH_TOKEN=$(check_atoken)
-      #   #echo "New  OS_AUTH_TOKEN = $OS_AUTH_TOKEN"
-      #fi
-
-    if [[ -n "$OS_PASSWORD" ]]; then 
-      if  [[ $silent -eq 0 ]] ; then     
+      if [[ -n "$OS_PASSWORD" ]]; then 
+        if  [[ $silent -eq 0 ]] ; then     
           echo "Updating token"
+        fi
+        source $allas_conf_path --user $user -k $OS_PROJECT_NAME  --mode $mode -f 
       fi
-      source $allas_conf_path --user $user -k $OS_PROJECT_NAME -f 
-    fi
-    test=$(swift stat 2> /dev/null | grep -c "Account:")
-    if [[ $test -lt 1 ]]
-    then 
-       echo "No connection to Allas!"
-       echo "Please try setting the the connection again."
-       exit 1
-    else
-      echo "swift connection updated"
+      #test=$(swift stat 2> /dev/null | grep -c "Account:")
+      test=$(rclone about ${storage_server}: 2> /dev/null | wc -l) 
+      if [[ $test -lt 1 ]]
+      then 
+         echo "No connection to Allas!"
+         echo "Please try setting the the connection again."
+         exit 1
+      else
+         echo "swift connection updated"
     fi
   else 
      echo "swift connection OK" 
@@ -58,6 +53,14 @@ remove_bucket=0
 force=0
 defined_user="x"
 
+
+# read customer defaults
+if [[ -e $HOME/.a_tools_conf ]]; then
+   echo "Reading customer settings"
+   source $HOME/.a_tools_conf
+fi
+
+
 #Process command line
 while [[ $# -ge 1 ]]
 do
@@ -92,6 +95,11 @@ do
                   remove_bucket=(1)
                   shift
                 ;;
+             '--s3cmd' )
+                  mode=("s3cmd")
+                  shift
+                ;;
+
              '-h' | '--help' )
                   print_help=(1)
                   shift
@@ -150,11 +158,14 @@ project_label=$(echo ${os_project_name} |  sed -e s/"project_"/""/g)
 
 
 #Check if connection works
-if [[ $mode == "swift" ]]
-then
+if [[ $mode == "swift" ]]; then
  check_swift_connection
 fi
 
+if [[ $mode == "s3cmd" ]]; then
+ storage_server="s3allas"
+fi
+
 
 #The method to remove a bucket
 if [[ $remove_bucket -eq 1 ]]; then

diff --git a/a-find b/a-find
@@ -27,6 +27,15 @@ show_filelist=(0)
 query=("")
 all_buckets=(0)
 
+
+# read customer defaults
+if [[ -e $HOME/.a_tools_conf ]]; then
+   customized=1
+   source $HOME/.a_tools_conf
+else 
+   customized=0
+fi
+
 #Process command line
 while [[ $# -ge 1 ]]
 do
@@ -75,6 +84,15 @@ do
     esac
 done
 
+# note about customization 
+if [[ $silent -eq 0 ]]; then
+   if [[ $customized -eq 1 ]]; then
+     echo "Customer settings red from $HOME/.a_tools_conf"
+   fi
+fi 
+
+
+
 #echo "buk $bucket_name"
 
 #exit

diff --git a/a-flip b/a-flip
@@ -76,43 +76,6 @@ exit
 
 fi 
 
-#Configure rclone
-if [ ! -e $HOME/.config/rclone ]
-then 
-  mkdir -p  $HOME/.config/rclone
-fi
-if [ -e $HOME/.config/rclone/rclone.conf ]
-then 
-   rc_check=$(grep -c $storage_server $HOME/.config/rclone/rclone.conf)
-   if [ $rc_check -lt 1 ]
-   then
-      echo '[pouta]' >>  $HOME/.config/rclone/rclone.conf
-      echo 'type = swift'  >>  $HOME/.config/rclone/rclone.conf
-      echo 'env_auth = true'   >>  $HOME/.config/rclone/rclone.conf
-   fi
-else
-   echo '[pouta]' >>  $HOME/.config/rclone/rclone.conf
-   echo 'type = swift'  >>  $HOME/.config/rclone/rclone.conf
-   echo 'env_auth = true'   >>  $HOME/.config/rclone/rclone.conf 
-fi
-
-#Assign project to be used if not defined 
-if [ $os_project_name == "" ]
-then
-  if [ -e $HOME/.allas_default ]
-  then
-     source $HOME/.allas_default
-  else
-     echo "Default project is not defined"
-     source $allas_conf_path -user $user
-     echo "os_project_name=$OS_PROJECT_NAME" > $HOME/.allas_default
-     echo "Default allas project is stored to  \$HOME/.allas_default"
-     echo ""
-  fi
-  source $HOME/.allas_default
-fi
-
-
 #Check if connection works
 if [[ $mode == "swift" ]]
 then
@@ -123,9 +86,14 @@ then
     echo "No connection to Allas!"
     echo "Please try setting the the connection again."
     exit 1
-  fi 
+  fi
+else 
+  echo "a-flip is available only in swift mode"
 fi
 
+
+
+
 #source /appl/opt/allas_conf
 #input=("$1")
 

diff --git a/a-get b/a-get
@@ -29,6 +29,14 @@ exe_dir=$(pwd)
 start_time=$(date +%s)
 encrypt=""
 
+
+# read customer defaults
+if [[ -e $HOME/.a_tools_conf ]]; then
+   echo "Reading customer settings"
+   source $HOME/.a_tools_conf
+fi
+
+
 #Process command line
 while [[ $# -ge 1 ]]
 do
@@ -186,6 +194,10 @@ then
   fi 
 fi
 
+#Rclone through s3
+if [[ $mode == "s3cmd" ]];then
+   storage_server="s3allas"
+fi
 
 #source /appl/opt/allas_conf
 #input=("$1")
@@ -394,26 +406,26 @@ if [[ $is_tar_file -eq 1 ]]; then
      #rclone cat "${storage_server}:/$object_name" | zstdmt -d | tar -x -o $file_name
      echo "Dowloading compressed object from Allas"
 
-     if [[ $mode == "swift" ]]; then
+     #if [[ $mode == "swift" ]]; then
         rclone copyto -P "${storage_server}:/$object_name" a-get_download_$$_tmp.zst
         exitcode=$?
-     else
-       s3cmd get "s3://${object_name}" a-get_download_$$_tmp.zst 
-     fi
+     #else
+     #  s3cmd get "s3://${object_name}" a-get_download_$$_tmp.zst 
+     #fi
 
      if [[ $exitcode -ne 0 ]]; then
         echo "Donload failed!"
         rm -f  a-get_download_$$_tmp.zst
         exit 1
      fi
      if [[ $encrypt == "gpg" ]]; then
-        echo "Based on the object name, thae object has been encrypted with gpg."
+        echo "Based on the object name, the object has been encrypted with gpg."
         gpg -d a-get_download_$$_tmp.zst >   a-get_download_$$_tmp.zst.tmp          
         rm -f a-get_download_$$_tmp.zst
         mv a-get_download_$$_tmp.zst.tmp a-get_download_$$_tmp.zst
      fi
      if [[ $encrypt == "crypt4gh" ]]; then
-        echo "Based on the object name, thae object has been encrypted with crypt4gh."
+        echo "Based on the object name, the object has been encrypted with crypt4gh."
         crypt4gh decrypt --sk "$secret_key" < a-get_download_$$_tmp.zst > a-get_download_$$_tmp.zst.tmp
         rm -f a-get_download_$$_tmp.zst
         mv a-get_download_$$_tmp.zst.tmp a-get_download_$$_tmp.zst
@@ -454,60 +466,60 @@ else
     fi
     # uncompress if needed
     if [[ $is_compressed -eq 1 ]]; then 
-       if [[ $mode == "swift" ]]; then 
+       #if [[ $mode == "swift" ]]; then 
            if [[ $encrypt == "gpg" ]]; then
              rclone cat "${storage_server}:/$object_name" | gpg -d | zstdmt -d >  $target_name
            elif [[  $encrypt == "crypt4gh" ]]; then
              rclone cat "${storage_server}:/$object_name" | crypt4gh decrypt --sk "$secret_key" | zstdmt -d >  $target_name
            else
              rclone cat "${storage_server}:/$object_name" | zstdmt -d >  $target_name
            fi
-       else
-          s3cmd get "s3://${object_name}" a-get_download_$$_tmp.zst
-	  if [[ $encrypt == "gpg" ]]; then
-             echo "Based on the object name, thae object has been encrypted with gpg."
-             gpg -d a-get_download_$$_tmp.zst >   a-get_download_$$_tmp.zst.tmp          
-             rm -f a-get_download_$$_tmp.zst
-             mv a-get_download_$$_tmp.zst.tmp a-get_download_$$_tmp.zst
-          fi
-          if [[ $encrypt == "crypt4gh" ]]; then
-             echo "Based on the object name, thae object has been encrypted with crypt4gh."
-             crypt4gh decrypt --sk "$secret_key" < a-get_download_$$_tmp.zst > a-get_download_$$_tmp.zst.tmp
-             rm -f a-get_download_$$_tmp.zst
-             mv a-get_download_$$_tmp.zst.tmp a-get_download_$$_tmp.zst
-          fi
-
-          zstdmt -d a-get_download_$$_tmp.zst 
-          mv a-get_download_$$_tmp > $target_name
-       fi
+       #else
+       #   s3cmd get "s3://${object_name}" a-get_download_$$_tmp.zst
+       #  if [[ $encrypt == "gpg" ]]; then
+       #      echo "Based on the object name, thae object has been encrypted with gpg."
+       #      gpg -d a-get_download_$$_tmp.zst >   a-get_download_$$_tmp.zst.tmp          
+       #      rm -f a-get_download_$$_tmp.zst
+       #      mv a-get_download_$$_tmp.zst.tmp a-get_download_$$_tmp.zst
+       #   fi
+       #   if [[ $encrypt == "crypt4gh" ]]; then
+       #      echo "Based on the object name, thae object has been encrypted with crypt4gh."
+       #      crypt4gh decrypt --sk "$secret_key" < a-get_download_$$_tmp.zst > a-get_download_$$_tmp.zst.tmp
+       #      rm -f a-get_download_$$_tmp.zst
+       #      mv a-get_download_$$_tmp.zst.tmp a-get_download_$$_tmp.zst
+       #   fi
+       #
+       #   zstdmt -d a-get_download_$$_tmp.zst 
+       #   mv a-get_download_$$_tmp > $target_name
+       #fi
     else 
        if [[ $target_dir == "" ]]; then 
             target_dir="."
        fi
        # Just download
-       if [[ $mode == "swift" ]]; then 
+       #if [[ $mode == "swift" ]]; then 
 	   if [[ $encrypt == "gpg" ]]; then
              rclone cat "${storage_server}:/$object_name" | gpg -d >  $target_name
            elif [[  $encrypt == "crypt4gh" ]]; then
              rclone cat "${storage_server}:/$object_name" | crypt4gh decrypt --sk "$secret_key" >  $target_name
            else
              rclone -P copy "${storage_server}:/$object_name"  $target_dir
            fi
-       else
-           s3cmd get "s3://${object_name}"  $target_dir/$target_name
-           if [[ $enrypted == "gpg" ]]; then
-              echo "Based on the object name, thae object has been encrypted with gpg."
-              gpg -d $target_dir/$target_name >  $target_dir/${target_name}.tmp       
-              rm $target_dir/$target_name
-              mv $target_dir/${target_name}.tmp $target_dir/$target_name
-           fi 
-           if [[ $enrypted == "crypt4gh" ]]; then
-              echo "Based on the object name, thae object has been encrypted with crypt4gh."
-              crypt4gh decrypt --sk "$secret_key" < $target_dir/$target_name >  $target_dir/${target_name}.tmp       
-              rm $target_dir/$target_name
-              mv $target_dir/${target_name}.tmp $target_dir/$target_name
-           fi 
-       fi
+       #else
+       #    s3cmd get "s3://${object_name}"  $target_dir/$target_name
+       #    if [[ $enrypted == "gpg" ]]; then
+       #       echo "Based on the object name, thae object has been encrypted with gpg."
+       #       gpg -d $target_dir/$target_name >  $target_dir/${target_name}.tmp       
+       #       rm $target_dir/$target_name
+       #       mv $target_dir/${target_name}.tmp $target_dir/$target_name
+       #    fi 
+       #    if [[ $enrypted == "crypt4gh" ]]; then
+       #       echo "Based on the object name, thae object has been encrypted with crypt4gh."
+       #       crypt4gh decrypt --sk "$secret_key" < $target_dir/$target_name >  $target_dir/${target_name}.tmp       
+       #       rm $target_dir/$target_name
+       #       mv $target_dir/${target_name}.tmp $target_dir/$target_name
+       #    fi 
+       #fi
     fi
 
 

diff --git a/a-info b/a-info
@@ -71,6 +71,16 @@ show_filelist=(0)
 query=("")
 object_with_bucket=(0)
 
+
+# read customer defaults
+if [[ -e $HOME/.a_tools_conf ]]; then
+   customized=1
+   source $HOME/.a_tools_conf
+else 
+   customized=0
+fi
+
+
 #Process command line
 while [[ $# -ge 1 ]]
 do
@@ -137,10 +147,13 @@ fi
 project_label=$(echo ${os_project_name} |  sed -e s/"project_"/""/g)
 
 #Check if connection works
-if [[ $mode == "swift" ]]
-then
+if [[ $mode == "swift" ]]; then
     check_swift_connection 
 fi
+
+if [[ $mode == "s3cmd" ]]; then
+  storage_server="s3allas"
+fi 
 
 
 if [[ $object_name == "" ]] ; then