#!/bin/bash
#Color setting variables
MENU=`echo "\033[36m"` #Blue
RED_TEXT=`echo "\033[31m"`
NORMAL=`echo "\033[m"`
RED=`echo "\033[01;31m"`
GREEN=`echo "\033[01;32m"`
YELLOW=`echo "\033[01;33m"`
PURPLE=`echo "\033[01;35m"`
CYAN=`echo "\033[01;36m"`
WHITE=`echo "\033[01;37m"`
BOLD=`echo "\033[1m"`
UNDERLINE=`echo "\033[4m"`
############################################################################################################
############################################### USAGE ######################################################
############################################################################################################
usage()
{
echo -e "
${BOLD}
Usage: pic-migration -i inputfile -t threads [-h mongohost] [-d dbname] [-p port]
${NORMAL}
${PURPLE} -i ${NORMAL}
: Input file with id list
${PURPLE} -t ${NORMAL} : Number of parallel threads (default 4)
${PURPLE} -h ${NORMAL}
: Mongo server hostname (default hostname)
${PURPLE} -d ${NORMAL}
: Mongo database name (default dbname)
${PURPLE} -p ${NORMAL}
: Mongo port (default 27108)
${GREEN} Example : ${NORMAL}
./pic-migration -i=inputfile -t=<threads count> -h=<host> -d=<db> -p=<port#>
or
./pic-migration -i=inputfile
"
}
##################################################################################################################
######################################### INPUT VALIDATION ######################################################
##################################################################################################################
if [ "$#" -eq 0 ]
then
usage
exit 0
fi
mongohost=<hostname>
dbname=<dbname>
port=27108
thread_count=4
mpath=/mongo/sw/mongo-3.0.1/bin
picpath=/opt/pic
usr=testuser
pwd=user1234
pause()
{
echo -e "Press ${BOLD} any key to exit ${NORMAL} "
read -p " "
}
#echo "$#"
#echo "$@"
if [[ "$#" > 0 ]]; then
for i in "$@"
do
# echo "$i"
case $i in
-i=*|--inputfile=*)
inputfile="${i#*=}"
shift # past argument=value
;;
-t=*|--threads=*)
thread_count="${i#*=}"
shift # past argument=value
;;
-h=*|--mongohost=*)
mongohost="${i#*=}"
shift # past argument=value
;;
-d=*|--dbname=*)
dbname="${i#*=}"
shift # past argument=value
;;
-p=*|--port=*)
port="${i#*=}"
shift # past argument=value
;;
*)
echo "input error.."
usage
exit
;;
esac
done
fi
TS=./log/pic_downloaded_$(date +"%m-%d-%y-%T")
#######################################################################################
###################### SSO_ID to PIC_ID CONVERSION ####################################
###################### VALIDATE GIVEN SSOIDS ARE VALID ################################
###################### VALIDATE GIVEN SSOIDS ARE HAVING PIC/IMAGE MAPPED OR NOT########
#######################################################################################
#echo "DBQuery.shellBatchSize = 90000000000000000;" > img_cnt.py
#rm -f picid_list.py
rm -f "$inputfile"_ssoid_notfound.lst
rm -f "$inputfile"_picid.lst
rm -f "$inputfile"_imgid_notfound.lst
rm -f "$inputfile"_prefdata.out
rm -f "$inputfile"_picdata.out
while read ssoid
do
echo $ssoid
echo "db.employee.find({'"employee.DigitalProfiles.ssoid"':'"$ssoid"'},{_id:0,'"Profiles.DigitalProfiles.profilePictureId"':1})" >imgqry.js
echo "db.employee.find({'"employee.DigitalProfiles.ssoid"':'"$ssoid"'})" >pref_data.js
#echo "db.ProfilePicture.files.find(
#cat imgqry.js
imgqry_output=$($mpath/mongo --host $mongohost $dbname --port=$port -u $usr -p $pwd --authenticationDatabase=admin --quiet < imgqry.js)
#echo "img query out $imgqry_output"
qryout_length=`printf "%s" "$imgqry_output" | wc -m`
if [ $qryout_length -ne 0 ]; then
$mpath/mongo --host $mongohost $dbname --port=$port -u $usr -p $pwd --authenticationDatabase=admin --quiet < pref_data.js >> ./data/"$inputfile"_prefdata.out
fi
if [ $qryout_length -eq 0 ]; then
echo -e "${RED}$ssoid <<ssoid not found>> ${NORMAL}"
echo -e "${RED}$ssoid ${NORMAL}" >> ./log/"$inputfile"_ssoid_notfound.lst
elif [[ ${imgqry_output} != *"ObjectId"* ]];then
echo -e "${RED}$ssoid ${NORMAL}" >> ./log/"$inputfile"_imgid_notfound.lst
echo -e "${RED}$ssoid <<pic not found>> ${NORMAL}"
else
#echo "awk output"
#picid=$(echo -e $imgqry_output|awk -F ':' '{print $4}'|awk -F '}' '{print $1}')
picid=$(echo -e $imgqry_output|awk -F ':' '{print $4}'|awk -F '}' '{print $1}')
picid=$(echo "db.ProfilePicture.files.find({_id : $picid })")
echo -e "$picid" > picdata_qry.js
#cat picdata_qry.js
$mpath/mongo --host $mongohost $dbname --port=$port -u $usr -p $pwd --authenticationDatabase=admin --quiet < picdata_qry.js >> ./data/"$inputfile"_picdata.out
echo -e $imgqry_output|awk -F ':' '{print $4}'|awk -F '}' '{print $1}'>>"$inputfile"_picid.lst
fi
done < "$inputfile"
#read -p "waiting for cancel "
###########################################################################################
############################# PIC DOWNLOAD ################################################
###########################################################################################
picids="$(cat <<EOF
`cat "$inputfile"_picid.lst`
EOF
)"
total_files=$(echo "$picids"|wc -l|awk {'print $1'})
lines_per_file="$(((($total_files + $thread_count) - 1) / $thread_count))"
#echo "total files : $total_files"
#echo "lines per file $lines_per_file"
#read -p "wait"
#echo "dollar picids ------------> $picids"
#TS=Pic_download_"$inputfile"_report_$(date +"%m-%d-%y-%T")
echo -e " ${BOLD} ----------------------------------------------------------------------------------------------------------------------------------${NORMAL}" >>"$TS".out
echo "Host :
$mongohost
">>"$TS".out
echo "Databse :
$dbname
">>"$TS".out
echo "Begin Date :
$(date)
">>"$TS".out
echo -e " ${BOLD}-----------------------------------------------------------------------------------------------------------------------------------${NORMAL}" >>"$TS".out
pic_download()
{
FILENAME=`echo $pic_id | awk -F '"' '{print $2}'`
echo "PIC FILENAME $FILENAME -------------> `date`"
$mpath/mongofiles --quiet -d $dbname get_id $pic_id --host $mongohost --port $port --prefix ProfilePicture --authenticationDatabase admin -u $usr -p $pwd --local $picpath/$FILENAME.jpg >> $TS.out 2>&1
#echo "Image downloaded list"
#cat pic_downloaded.out
}
##################################################################################################
############################## Thread Loop ######################################################
##################################################################################################
#echo "`date`"
# param: $1 worker identity number
# param: $2 starting line number in the file to process
process_lines () {
#cat x
while read -r line; do
#get filename
pic_id=$line
pic_id=$(echo -e "$line" | awk -F'\t' '{ print $1 }')
#if connected message then continue
[[ $file == 'connected to'* ]] && continue
# sync the file with the server
pic_download $pic_id
done < x$i
}
#######################################################################################################
####################### Multi Thread initiator #######################################################
#######################################################################################################
for ((i=1; i <= $thread_count; ++i)); do
#echo "starting worker $i"
#echo "lines per file : $lines_per_file, counter i : $i"
headpos=$(($lines_per_file * $i))
#echo "head pos : " $headpos
echo -e "$picids" | head -n $headpos | tail -n $lines_per_file > x$i
#cat x$i
#call process on this chunk of files
#process_lines $i $(($lines_per_file * i)) &
process_lines &
#record the pid for cleanup and waiting
#echo "starting line number $i"
_worker_pids+=($!)
echo -e "${BOLD}${PURPLE}worker process id : $_worker_pids[$!] ${NORMAL}"
done
#######################################################################################################
####################### Wait for Threads to complete ################################################
#######################################################################################################
#wait for each process to finish
for ((i=0; i <= ${#_worker_pids[@]}; ++i)); do
wait ${_worker_pids[i]} > /dev/null 2>&1
done
#if no errors say we are complete
if [ $? -eq 0 ]; then
cnt=$(grep -i "finished writing" $TS.out|wc -l|awk -F " " {'print $1'})
echo -e "${BOLD}-------------------------------------------------------------------------------------------------------------${NORMAL}">>$TS.out
echo "End Time : $(date) Number of images downloaded : $cnt ">>$TS.out
echo -e "${BOLD}-------------------------------------------------------------------------------------------------------------${NORMAL}">>$TS.out
echo -e "${BOLD}==================================================================================================================${NORMAL}"
echo -e "${BOLD}${GREEN} Execution complete!! Please validate execution output in the log and results in data & pic folders !! ${NORMAL}"
echo -e "${BOLD}==================================================================================================================${NORMAL}"
fi
