a-find

#!/bin/bash 

#rclone is needed
if [[ $(which rclone 2> /dev/null | wc -l ) -ne 1 ]];then
      echo ""
      echo "rclone is not available!"
      echo "Please install rclone."
      exit 1
fi


#read static variables
inst_root=$(dirname $(readlink -f $0))
source $inst_root/a_env_conf

#local variables
user=($USER)
bucket_name=("not_defined")
silent=(0)
print_help=(0)
os_project_name=("none")
input_def=("")
mode=("swift")
tmp_dir=("${tmp_root}/a_put_$$_tmp")
show_filelist=(0)
query=("")
all_buckets=(0)

#Process command line
while [[ $# -ge 1 ]]
do
  case "$1" in
             '--user' | '-u' )
                  user=("$2")
                  shift
                  shift
                ;;               
             '--bucket' | '-b' )
             # query file
                  bucket_name=($2)
                  shift
                  shift
                ;;
             '--project' | '-p')
                  os_project_name=($2)
                  shift
                  shift
                ;;
             '-silent' | '-s')
                  silent=(1)
                  shift
                ;;
            '--files' | '-f' )
                  show_filelist=(1)
                  shift
                ;;
            '--all_bukets' | '-a' )
                  bucket_name=("$(swift list| tr  '\n' ' ')")
                  shift
                ;;               
             '-h' | '--help' )
                  print_help=(1)
                  shift
                ;;
             *)
                   if [[ $query != "" ]]; then
                      echo "unknown option: $1"
                      echo "Query term is: $query"
                      exit 1
                   fi
                   query=("$1")
                   shift                       # No more switches
                ;;
    esac
done

#echo "buk $bucket_name"

#exit

if [ $print_help -eq 1 ]; then
cat <<EOF
This tool is used to find files that have been uploaded to Allas service using the a-put command.
The basic syntax of the comand is:

   a-find query_term

The query term is compared to the names and original paths of the files that have been uploaded to
Allas and the matching obects are reported (but not downloaded). Note that data, that has been uploded 
to Allas using other tools than a-put is not included in this serach process.

The query term is procecced as a reqular repression where some characters, for example dot (.), have a special meaning.
The same regular expression syntax is used with e.g. grep, awk and sed commands.
The most commonly occurring special characters are listed below:

    the dot (.) is used to define any single character.
    ^ means the beginning of a line
    $ means the end of a line
    [ ] matches any of the characters inside the brackets. For example [abc] would match a,b or c.
    [^ ] matches any character, except the characters inside the brackets. 
    For example [^abc] would select all rows that contain also other characters
    than just a,b and c.
    * matchs zero or more of the preceding character or expression
    \{n,m\} matches n to m occurrences of the preceding character or expression

Options:

-f, --files                  Lists the names of the matching files inside the objects in addition to the object name.

-p, --project <project_ID>   Search matches form the buckets of the defined project in stead of the currently configured project. 

-b, --bucket <bucket_name>   By default all the standard buckets, used by a-put, are searched. Option --bucket allows you to specify a 
                        single bucket that will be used for the search.

-a, --all                    By default all the standard buckets, used by a-put, are searched. Option --all defines that all the bukets of
                             of the project will be included in the search.

-s, --silent                 Ouput just the object names and number of hits. If -file in cluded print object name and 
                        matching file name on one row.


Related commands: a-put, a-get, a-delete, a-info
EOF

exit

fi 

#Assign project to be used if not defined 
if [ $os_project_name == "none" ]
then
  if [ -e $HOME/.allas_default ]
  then
     source $HOME/.allas_default

     #echo      $os_project_name    $OS_PROJECT_NAME
     if [[ $os_project_name != $OS_PROJECT_NAME ]]
     then 
        echo "Switching allas configuration to use project $os_project_name"
        source $allas_conf_path -user $user $os_project_name -keeppasswd
        export OS_PROJECT_NAME=$os_project_name
     fi 
  else
     echo "Default project is not defined"
     source $allas_conf_path -user $user
     echo "os_project_name=$OS_PROJECT_NAME" > $HOME/.allas_default
     echo "Default allas project is stored to  \$HOME/.allas_default"
     echo ""
  fi
  else
     echo "Switching allas configuration to use project $os_project_name"
     source $allas_conf_path -user $user $os_project_name -keeppasswd
     export OS_PROJECT_NAME=$os_project_name
fi

#source $HOME/.allas_default
project_label=$(echo ${os_project_name} |  sed -e s/"project_"/""/g)

#Check if connection works
if [[ $mode == "swift" ]]
then
  test=$(swift stat 2> /dev/null | grep -c "Account:")
  if [[ $test -lt 1 ]]
  then 
    echo "No connection to Allas!"
    echo "Please try setting the the connection again."
    echo "by running command:"
    echo ""
    echo "   source $allas_conf_path"
    exit 1
  fi 
fi


#define standatrd buckests 
std_buckets=("${user}-${project_label}-MISC ${user}-${project_label}-HOME ${project_label}-puhti-SCRATCH ${project_label}-puhti-PROJAPPL ${user}-${project_label}-pub ${user}-${project_label}-flip")

#check all buckerts if a specific bucket is not defined
if [[ "$bucket_name" == "not_defined" ]]
then
  buckets=$std_buckets
  #echo $buckets
else
  buckets=$bucket_name 
fi



#list all objects if query is not defined
if [[ $query == "" ]]
then
  echo ""
  echo "No query term given!" 
  echo "Listing all objects but not files within them:"
  echo " "

  for bn in $buckets
  do 
    for metafile in $(rclone ls ${storage_server}:$bn 2> /dev/null | awk '{print $2}' | grep "_ameta$")
    do
        os_dirname=$(dirname "$metafile" )
        if [ $os_dirname == "." ]; then
           os_dirname=""
        else
           os_dirname="/$os_dirname"
        fi
        os_filename=$(basename "$metafile" _ameta)
        os_object=("${bn}${os_dirname}/${os_filename}")
        echo $os_object
    done
  done
  exit
fi




#source /appl/opt/allas_conf
#input=("$1")

total_hits=(0)
total_objects=(0)
for bn in $buckets
do 
  if [ $silent -eq 0 ]; then
    echo "----------------------------------------------"
    echo "Cheking bucket: $bn"; echo ""
  fi
  for metafile in $(rclone ls ${storage_server}:$bn 2> /dev/null | awk '{print $2}' | grep "_ameta$")
  do
     num_hits=$(rclone cat ${storage_server}:${bn}/$metafile | awk '{print $9}' | grep -c "$query")
     if [[ $num_hits -gt 0 ]]; then
        os_dirname=$(dirname "$metafile" )
        if [ $os_dirname == "." ]; then
           os_dirname=""
        else
           os_dirname="/$os_dirname"
        fi
        os_filename=$(basename "$metafile" _ameta)
        os_object=("${bn}${os_dirname}/${os_filename}")
        if [ $show_filelist -eq 1 ]; then
           if [ $silent -eq 0 ]; then
              echo "Matching files in object $os_object :" 
              rclone cat ${storage_server}:${bn}/$metafile | awk '{print $9}' | grep "$query"
              echo ""
           else
              #silent output
              rclone cat ${storage_server}:${bn}/$metafile | awk '{print $9}' | grep "$query" | awk '{print "'$os_object' "$0}'  
           fi  
        else 
           if [ $silent -eq 0 ]; then
             echo ""      
             #echo " Object:  ${bn}/$(dirname "$metafile")/$(basename $metafile _ameta)"
             echo " Object: $os_object"
             echo " includes $num_hits file names that that match query: $query"
             echo ""
           else
             echo $os_object $num_hits
           fi 
        fi   
        (( total_objects = total_objects + 1 ))
        (( total_hits = total_hits + num_hits ))
     fi
  done

done

if [ $silent -eq 0 ]; then
  echo "------------------------------------------------ "
  echo "Query: $query"
  echo "Total of $total_hits hits were found in $total_objects objects"
  echo "-------------------------------------------------"
fi
exit