Extracing Mongo Collection data and GRIDFS Pictures using Multi Thread

#!/bin/bash

#Color setting variables

MENU=`echo "\033[36m"` #Blue
RED_TEXT=`echo "\033[31m"`
NORMAL=`echo "\033[m"`
RED=`echo "\033[01;31m"`
GREEN=`echo "\033[01;32m"`
YELLOW=`echo "\033[01;33m"`
PURPLE=`echo "\033[01;35m"`
CYAN=`echo "\033[01;36m"`
WHITE=`echo "\033[01;37m"`
BOLD=`echo "\033[1m"`
UNDERLINE=`echo "\033[4m"`

############################################################################################################
############################################### USAGE ######################################################
############################################################################################################
usage()
{
echo -e  "
${BOLD}
Usage: pic-migration -i inputfile -t threads [-h mongohost] [-d dbname] [-p port]
${NORMAL} 
${PURPLE}  -i ${NORMAL}    : Input file with id list
${PURPLE}  -t ${NORMAL}         : Number of parallel threads (default 4)
${PURPLE}  -h ${NORMAL} : Mongo server hostname (default hostname)
${PURPLE}  -d ${NORMAL} : Mongo database name  (default dbname)
${PURPLE}  -p ${NORMAL} : Mongo port  (default 27108)

 ${GREEN} Example : ${NORMAL}
    ./pic-migration -i=inputfile -t=<threads count>  -h=<host> -d=<db> -p=<port#>
or
            ./pic-migration -i=inputfile
  "
}
##################################################################################################################
######################################### INPUT VALIDATION  ######################################################
##################################################################################################################

if [ "$#" -eq 0 ]
then
   usage
   exit 0
fi

mongohost=<hostname>
dbname=<dbname>
port=27108
thread_count=4
mpath=/mongo/sw/mongo-3.0.1/bin
picpath=/opt/pic
usr=testuser
pwd=user1234

pause()
{
echo -e "Press ${BOLD} any key to exit ${NORMAL} "
read -p "                "
}


#echo "$#"

#echo "$@"

if [[ "$#" > 0 ]]; then
  for i in "$@"
  do
#        echo  "$i"
case $i in

-i=*|--inputfile=*)
                  inputfile="${i#*=}"
                        shift # past argument=value
                        ;;
-t=*|--threads=*)
                        thread_count="${i#*=}"
                        shift # past argument=value
                        ;;
    -h=*|--mongohost=*)
    mongohost="${i#*=}"
    shift # past argument=value
    ;;
    -d=*|--dbname=*)
    dbname="${i#*=}"
    shift # past argument=value
    ;;
    -p=*|--port=*)
    port="${i#*=}"
    shift # past argument=value
    ;;
    *)
          echo "input error.."
usage
    exit
    ;;
esac
   done
fi

TS=./log/pic_downloaded_$(date +"%m-%d-%y-%T")

#######################################################################################
###################### SSO_ID to PIC_ID CONVERSION ####################################
###################### VALIDATE GIVEN SSOIDS ARE VALID ################################
###################### VALIDATE GIVEN SSOIDS ARE HAVING PIC/IMAGE MAPPED OR NOT########
#######################################################################################

#echo "DBQuery.shellBatchSize = 90000000000000000;" > img_cnt.py
#rm -f picid_list.py
rm -f "$inputfile"_ssoid_notfound.lst
rm -f "$inputfile"_picid.lst
rm -f "$inputfile"_imgid_notfound.lst
rm -f "$inputfile"_prefdata.out
rm -f "$inputfile"_picdata.out

while read ssoid
do
echo $ssoid
echo "db.employee.find({'"employee.DigitalProfiles.ssoid"':'"$ssoid"'},{_id:0,'"Profiles.DigitalProfiles.profilePictureId"':1})" >imgqry.js
        echo "db.employee.find({'"employee.DigitalProfiles.ssoid"':'"$ssoid"'})" >pref_data.js
        #echo "db.ProfilePicture.files.find(
#cat imgqry.js
imgqry_output=$($mpath/mongo  --host $mongohost  $dbname  --port=$port -u $usr -p $pwd --authenticationDatabase=admin --quiet < imgqry.js)
#echo "img query out   $imgqry_output"
qryout_length=`printf "%s" "$imgqry_output" | wc -m`

if [ $qryout_length -ne 0 ]; then
$mpath/mongo  --host $mongohost  $dbname  --port=$port -u $usr -p $pwd --authenticationDatabase=admin --quiet < pref_data.js >> ./data/"$inputfile"_prefdata.out
        fi
if [ $qryout_length -eq 0 ]; then
echo -e "${RED}$ssoid <<ssoid not found>> ${NORMAL}"
echo -e "${RED}$ssoid ${NORMAL}" >> ./log/"$inputfile"_ssoid_notfound.lst
elif [[ ${imgqry_output} != *"ObjectId"* ]];then
echo -e "${RED}$ssoid ${NORMAL}" >> ./log/"$inputfile"_imgid_notfound.lst
echo -e "${RED}$ssoid <<pic not found>> ${NORMAL}"
else
        #echo "awk output"
        #picid=$(echo -e $imgqry_output|awk -F ':' '{print $4}'|awk -F '}' '{print $1}')
        picid=$(echo -e $imgqry_output|awk -F ':' '{print $4}'|awk -F '}' '{print $1}')
picid=$(echo "db.ProfilePicture.files.find({_id : $picid })")
echo -e "$picid" > picdata_qry.js
#cat picdata_qry.js
$mpath/mongo  --host $mongohost  $dbname  --port=$port -u $usr -p $pwd --authenticationDatabase=admin --quiet < picdata_qry.js  >> ./data/"$inputfile"_picdata.out
echo -e $imgqry_output|awk -F ':' '{print $4}'|awk -F '}' '{print $1}'>>"$inputfile"_picid.lst
fi
done < "$inputfile"

#read -p "waiting for cancel "
###########################################################################################
############################# PIC DOWNLOAD ################################################
###########################################################################################

picids="$(cat <<EOF
`cat "$inputfile"_picid.lst`
EOF
)"

total_files=$(echo "$picids"|wc -l|awk {'print $1'})
lines_per_file="$(((($total_files + $thread_count) - 1) / $thread_count))"

#echo "total files :  $total_files"
#echo "lines per file $lines_per_file"

#read -p "wait"

#echo "dollar picids ------------> $picids"

#TS=Pic_download_"$inputfile"_report_$(date +"%m-%d-%y-%T")
echo -e " ${BOLD} ----------------------------------------------------------------------------------------------------------------------------------${NORMAL}" >>"$TS".out
echo "Host         : $mongohost              ">>"$TS".out
echo "Databse      : $dbname              ">>"$TS".out
echo "Begin Date   : $(date)                                    ">>"$TS".out
echo -e " ${BOLD}-----------------------------------------------------------------------------------------------------------------------------------${NORMAL}" >>"$TS".out

pic_download()
{
FILENAME=`echo $pic_id | awk -F '"' '{print $2}'`
echo "PIC FILENAME $FILENAME -------------> `date`"
$mpath/mongofiles --quiet -d $dbname get_id $pic_id --host $mongohost --port $port  --prefix ProfilePicture --authenticationDatabase admin -u $usr -p $pwd --local $picpath/$FILENAME.jpg  >> $TS.out 2>&1
#echo "Image downloaded list"
#cat pic_downloaded.out
}

##################################################################################################
##############################  Thread Loop ######################################################
##################################################################################################
#echo "`date`"

# param: $1 worker identity number
# param: $2 starting line number in the file to process
process_lines () {
  #cat x
  while read -r line; do
    #get filename
    pic_id=$line
    pic_id=$(echo -e "$line" | awk -F'\t' '{ print $1 }')
    #if connected message then continue
    [[ $file == 'connected to'* ]] && continue
    # sync the file with the server
    pic_download $pic_id
  done < x$i
}

#######################################################################################################
#######################  Multi Thread initiator #######################################################
#######################################################################################################

for ((i=1; i <= $thread_count; ++i)); do
  #echo "starting worker $i"
  #echo "lines per file : $lines_per_file,   counter i :  $i"
  headpos=$(($lines_per_file * $i))
  #echo "head pos : " $headpos
  echo -e "$picids" | head -n $headpos | tail -n $lines_per_file > x$i 
  #cat x$i
  #call process on this chunk of files
  #process_lines $i $(($lines_per_file * i)) &
  process_lines  &
  #record the pid for cleanup and waiting
  #echo "starting line number $i"
  _worker_pids+=($!)
  echo -e "${BOLD}${PURPLE}worker process id : $_worker_pids[$!] ${NORMAL}"
done

#######################################################################################################
#######################   Wait for Threads to complete ################################################
#######################################################################################################

#wait for each process to finish
for ((i=0; i <= ${#_worker_pids[@]}; ++i)); do
  wait ${_worker_pids[i]} > /dev/null 2>&1
done

#if no errors say we are complete
if [ $? -eq 0 ]; then
        cnt=$(grep -i "finished writing" $TS.out|wc -l|awk -F " " {'print $1'})
echo -e "${BOLD}-------------------------------------------------------------------------------------------------------------${NORMAL}">>$TS.out
echo "End Time  :      $(date)                   Number of images downloaded : $cnt ">>$TS.out
echo -e "${BOLD}-------------------------------------------------------------------------------------------------------------${NORMAL}">>$TS.out
echo -e "${BOLD}==================================================================================================================${NORMAL}"
echo -e "${BOLD}${GREEN}   Execution complete!! Please validate execution output in the  log and results in data & pic folders !! ${NORMAL}"
echo -e "${BOLD}==================================================================================================================${NORMAL}"
fi

Login to Mongo prompt

pth=`ps -ef|grep -v grep|grep mongod|tail -1|awk -F " " '{print $8}'|cut -d '/'  -f 1,2,3,4,5`
port=`ps -ef|grep -v grep|grep mongod|tail -1|awk -F " " '{print $10}'|awk -F "/" '{print $4}'|cut -d "-" -f 2|cut -d "." -f 1`
pth=$pth"/mongo --port "$port
$pth --quiet admin  -u Admin -p pwd
rs.status()

Reading Mongo Log for a particular day and particular error

for file in `ls -lrt|grep -v grep|grep "Jun 27"|awk -F " " '{print $9}'`
do
if [ "$file" != "a.sh" ]; then
grep -i "Rehydration successfully completed for node :" $file
fi
done

Mongo_log_monitoring

Note:

3 files used in this script are as below

######################## MSTAT.sh ############################################

/imdb001/sw/mongo-3.2.1/bin/mongostat --port=27101 -u username -p pwd -n 5  --authenticationDatabase=admin --noheaders >> /home/pmv742/Mstat.txt

####################### fname.txt #############################################

#log file path

#Eg:

/imdb001/log/27101/mongod-27101.log

###################### MSTAT.sh  ###############################################

#!/bin/bash

MENU=`echo "\033[36m"` #Blue

RED_TEXT=`echo "\033[31m"`

NORMAL=`echo "\033[m"`

RED=`echo "\033[01;31m"`

GREEN=`echo "\033[01;32m"`

YELLOW=`echo "\033[01;33m"`

PURPLE=`echo "\033[01;35m"`

CYAN=`echo "\033[01;36m"`

WHITE=`echo "\033[01;37m"`

BOLD=`echo "\033[1m"`

UNDERLINE=`echo "\033[4m"`

FNAME=`cat fname.txt`

pause()

{

echo -e "${BOLD} Press [Enter] key to continue... ${NORMAL}"

read -p "                "

}

status_check()

{

pth=`ps -ef|grep mongod|head -1|awk -F " " '{print $8}'|cut -d '/'  -f 1,2,3,4,5`

port=`ps -ef|grep mongod|head -1|awk -F " " '{print $10}'|awk -F "/" '{print $4}'|cut -d "-" -f 2|cut -d "." -f 1`

pth=$pth"/mongo --port "$port

#echo $pth

$pth --quiet << EOF > 1.lst

db.isMaster().primary

EOF

$pth  --quiet << EOF > 2.lst

db.isMaster().setName

EOF

master=`cat 1.lst | awk -F ':' '{print $1}'`

rs=`cat 2.lst`

#echo `${#master}`

#if [${#master} > 0]; then 

      if [ "`hostname`" = "$master" ]; then

         status="PRIMARY"

         else

         status="SECONDARY"

      fi

#else

#    status="Unable to find the status"

#fi

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "    Mongo node is  ${MENU} " $status

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo ""

echo ""

echo "" 

pause

}

LQ_time()

{

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD} enter how many lines you want to trace in the log... ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read log_time

export LOGTIME=$log_time

#tail -${LOGTIME} ${FNAME}|grep -i "QUERY"|awk '{ print $NF }'|awk -F "ms" '{print $1}'

tail -${LOGTIME} ${FNAME}|grep -i "QUERY"|awk '{print $1,$NF}'

echo ""

echo ""

echo ""

pause

}

Stpdown()

{

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD} enter how many lines you want to trace in the log... ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read log_time

tail -${log_time} ${FNAME}|grep -i "step" 

echo ""

echo ""

echo ""

pause

}

Stpdown-detail()

{

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD} enter how many lines you want to trace in the log... ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read log_time

export LOGTIME=$log_time

tail -${LOGTIME} ${FNAME}|grep -C 10 "step" 

echo ""

echo ""

echo ""

pause

}

Log_PIT_detail()

{

FNAME=`cat fname.txt`

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD}enter the time stamp for checking log ${MENU} eg: 2017-06-07T23:11:16  ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read time_stamp

export TIMESTMP=$time_stamp

grep -i ${time_stamp} ${FNAME}|tail -50

echo ""

echo ""

echo ""

pause

}

LQ_Count()

{

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD} enter how many lines you want to trace in the log... ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read log_time

export LOGTIME=$log_time

echo -e "${MENU} Number of queries having latency > 100ms ${NORMAL}"

echo "-------------------------------------------------------------------------------------------------------------------------"

tail -${LOGTIME} ${FNAME}|grep -i "QUERY"|awk '{ print $NF }'|awk -F "ms" '{print $1}'|wc -l

echo "-------------------------------------------------------------------------------------------------------------------------"

tail -${log_time} ${FNAME}|grep -i "QUERY"|head  

echo ""

echo ""

echo ""

pause

}

collscan()

{

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD} enter how many lines you want to trace in the log... ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read log_time

export LOGTIME=$log_time

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD} Queries which are using FULL TABLE SCAN ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

tail -${log_time} ${FNAME}|grep -i "COLLSCAN"|head

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo ""

pause

}

tail_log()

{

tail -f ${FNAME}

show_menu

}

read_count()

{

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD}                       Past how many  hours query load info you want to see? ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read -p "            " past_hours

x=`expr $past_hours \* 20`

echo -e "${GREEN}====================================================================================================${NORMAL}"

tail -$x  /root/Mstat.txt |awk -F " " '{print $19,"Number of queries   ====>  ",$2}'

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo ""

echo ""

echo ""

pause

}

read_load()

{

echo -e "${GREEN}======================================================================================================================${NORMAL}"

echo -e "${BOLD}Past how many hours load on the db server you want to see? ${NORMAL}"

echo -e "${GREEN}======================================================================================================================${NORMAL}"

read -p "            "  past_hours

x=`expr $past_hours \* 20`

echo -e "${GREEN}======================================================================================================================${NORMAL}"

#tail -$x  /root/Mstat.txt |awk -F " " '{print $19,"====>",$2}'

echo -e "${GREEN} Time                               Queries    Inserts    Updates    Deletes   NW_IN   NW_Out  Connections${NORMAL}" 

tail -$x /root/Mstat.txt|awk -F " " '{print $19," =======> ",$2,"      ",$1,"      ",$3,"      ",$4,"      ",$14,"     ",$15,"        ",$16}'

echo -e "${GREEN}======================================================================================================================${NORMAL}"

echo ""

echo ""

echo ""

pause

}

login()

{

echo `ps -ef|grep -v grep|grep mongod|tail -1`

echo `df -k | grep -i imdb`

pth=`ps -ef|grep -v grep|grep mongod|tail -1|awk -F " " '{print $8}'|cut -d '/'  -f 1,2,3,4,5`

port=`ps -ef|grep -v grep|grep mongod|tail -1|awk -F " " '{print $10}'|awk -F "/" '{print $4}'|cut -d "-" -f 2|cut -d "." -f 1`

echo `service mongod-$port status`

pth=$pth"/mongo --port "$port

$pth --quiet admin  -u user -p pwd

echo ""

pause

}

mstat()

{

pth=`ps -ef|grep -v grep|grep mongod|tail -1|awk -F " " '{print $8}'|cut -d '/'  -f 1,2,3,4,5`

port=`ps -ef|grep -v grep|grep mongod|tail -1|awk -F " " '{print $10}'|awk -F "/" '{print $4}'|cut -d "-" -f 2|cut -d "." -f 1`

$pth/\mongostat  --port=$port -u username -p password -n 50  --authenticationDatabase=admin

echo ""

pause

}

show_menu()

{

echo -e "${MENU}=================================================================================${NORMAL}"

echo -e "${BOLD}${RED}//                     MONGO DATABASE HEALTH CHECKS                   //  ${NORMAL}"

echo -e "${MENU}=================================================================================${NORMAL}"

echo -e "${RED} (0) ${GREEN}  :   Node is PRIMARY/SECONDARY"

echo ""

echo -e "${RED} (1) ${GREEN}  :   Search for any latency entry"

echo ""

echo -e "${RED} (2) ${GREEN}  :   Primary stepdown Timestamp"

echo ""

echo -e "${RED} (3) ${GREEN}  :   Primary Stepdown BEFORE and AFTER details"

echo ""

echo -e "${RED} (4) ${GREEN}  :   Searching details for a particular timestamp"

echo ""

echo -e "${RED} (5) ${GREEN}  :   Latency queries count and display sample Latency queries"

echo ""

echo -e "${RED} (6) ${GREEN}  :   Display Past Hr(s) - Application query load count info"

echo ""

echo -e "${RED} (7) ${GREEN}  :   Tail the log indefinite"

echo ""

echo -e "${RED} (8) ${GREEN}  :   List queries which are using collscan"

echo ""

echo -e "${RED} (9) ${GREEN}  :   Display Past Hr(s) DB server Read, Write, NW In/Out and Connections load info"

echo ""

echo -e "${RED} (10) ${GREEN} :   Login to Mongo prompt"

echo ""

echo -e "${RED} (11) ${GREEN} :   Display current mongo load statistics"

echo ""

echo -e "${RED} (12) ${GREEN} :   Exit ${NORMAL}"

echo -e "${MENU}====================================================${NORMAL}"

}

read_menu()

{

local choice

echo -e "${BOLD} Enter choice [0-12] ${NORMAL}"

read -p "                    " choice

case $choice in

   0)clear;

     status_check;

   ;;

   1) clear;

      LQ_time;

   ;;

   2) clear;

      Stpdown;

   ;;

   3) clear;

     Stpdown-detail;

   ;;

   4)clear;

     Log_PIT_detail;

   ;;

   5)clear;

     LQ_Count;

   ;;

   6)clear;

     read_count;

   ;;

   7)clear;

     tail_log;

   ;;

   8)clear;

     collscan;

   ;;

   9)clear;

     read_load;

   ;;

   10)clear;

      login;

   ;;  

   11)clear;

      mstat;

   ;;

   12) break

   ;;

   *)clear;

   echo -e "${RED}Wrong choice............choose [0-12]  :  ${NORMAL}   "

esac

}

while true

do

show_menu

read_menu

done

#################################################################################################

Roles-Privs-create user

Built-in Roles:

read
readWrite
dbAdmin
userAdin
clusterAdmin

readAnyDatabase
readWriteAnyDatabase
dbAdminAnyDatabase
userAdminAnyDatabase

User prompt changing:

prompt = function() {
    user = db.runCommand({connectionStatus:1}).authInfo.authenticatedUsers[0]
    host = db.getMongo().toString().split(" ")[2]
    curDB = db.getName()
    if (user) {
       uname = user.user
    }
    else {
       uname = "local"
    }
    return uname + "@" + host + ":" + curDB + "> "
}


Creating roles:

db.createRole({ role: "appReadRole", privileges: [ { resource: { db: "test", collection: "" }, actions:

[ "find" ] } ], roles: [] })


Listing all the roles:

db.getRoles(
    {
      rolesInfo: 1,
      showPrivileges:false,
      showBuiltinRoles: false
    }
)

Creating user with roles:

var a={user:"mani", pwd:"mani", roles:[{role:"read",db:"test"}]}
db
db.createUser(a)


db.createUser({ user: "finance", pwd: "password", roles: [ { role: "appReadRole", db: "test" } ] })

mongo Mani-PC:27002/test -u finance -p password

Changing password:

db.changeUserPassword("finance", "welcome123")

Current user details:

db.runCommand({connectionStatus : 1})

use admin
db.system.users.find().pretty()
db.system.users.remove({user:"userA"})


db.grantRolesToUser(
  "report",
  [{ "role" : "readWriteAnyDatabase", "db" : "admin" }]
)


db.revokeRolesFromUser(
  "report",
  [{ "role" : "readWriteAnyDatabase", "db" : "admin" }]
)

db.getUser("report")


use products
db.grantPrivilegesToRole(
  "inventoryCntrl01",
  [
    {
      resource: { db: "products", collection: "" },
      actions: [ "insert" ]
    },
    {
      resource: { db: "products", collection: "system.js" },
      actions: [ "find" ]
    }
  ],
  { w: "majority" }
)


===============================================================
Edit .mongorc.js in your home directory for changing the prompt:

function prompt() {
    var username = "anon";
    var user = db.runCommand({connectionStatus : 1}).authInfo.authenticatedUsers[0];
    var host = db.getMongo().toString().split(" ")[2];
    var current_db = db.getName();

    if (!!user) {
        username = user.user;
    }

    return username + "@" + host + ":" + current_db + "> ";
}
==============================================================

Mongo Prompt with DB name and current user

Past the below code in the mongo shell


prompt = function() {
    user = db.runCommand({connectionStatus:1}).authInfo.authenticatedUsers[0]
    host = db.getMongo().toString().split(" ")[2]
    curDB = db.getName()
    if (user) {
       uname = user.user
    }
    else {
       uname = "local"
    }
    return uname + "@" + host + ":" + curDB + "> "
}

Roles and authentication

Common Roles:
============
read
readWrite
dbAdmin
userAdin
clusterAdmin

readAnyDatabase
readWriteAnyDatabase
dbAdminAnyDatabase
userAdminAnyDatabase



var a={user:"abcd", pwd:"efgh", roles:["readWrite]}

db.createUser(a)

mongo hostname/dbname -u abcd -p efgh