Reading Mongo Log for a particular day and particular error

for file in `ls -lrt|grep -v grep|grep "Jun 27"|awk -F " " '{print $9}'`
do
if [ "$file" != "a.sh" ]; then
grep -i "Rehydration successfully completed for node :" $file
fi
done

Mongo_log_monitoring

Note:

3 files used in this script are as below

######################## MSTAT.sh ############################################

/imdb001/sw/mongo-3.2.1/bin/mongostat --port=27101 -u username -p pwd -n 5  --authenticationDatabase=admin --noheaders >> /home/pmv742/Mstat.txt

####################### fname.txt #############################################

#log file path

#Eg:

/imdb001/log/27101/mongod-27101.log

###################### MSTAT.sh  ###############################################

#!/bin/bash

MENU=`echo "\033[36m"` #Blue

RED_TEXT=`echo "\033[31m"`

NORMAL=`echo "\033[m"`

RED=`echo "\033[01;31m"`

GREEN=`echo "\033[01;32m"`

YELLOW=`echo "\033[01;33m"`

PURPLE=`echo "\033[01;35m"`

CYAN=`echo "\033[01;36m"`

WHITE=`echo "\033[01;37m"`

BOLD=`echo "\033[1m"`

UNDERLINE=`echo "\033[4m"`

FNAME=`cat fname.txt`

pause()

{

echo -e "${BOLD} Press [Enter] key to continue... ${NORMAL}"

read -p "                "

}

status_check()

{

pth=`ps -ef|grep mongod|head -1|awk -F " " '{print $8}'|cut -d '/'  -f 1,2,3,4,5`

port=`ps -ef|grep mongod|head -1|awk -F " " '{print $10}'|awk -F "/" '{print $4}'|cut -d "-" -f 2|cut -d "." -f 1`

pth=$pth"/mongo --port "$port

#echo $pth

$pth --quiet << EOF > 1.lst

db.isMaster().primary

EOF

$pth  --quiet << EOF > 2.lst

db.isMaster().setName

EOF

master=`cat 1.lst | awk -F ':' '{print $1}'`

rs=`cat 2.lst`

#echo `${#master}`

#if [${#master} > 0]; then 

      if [ "`hostname`" = "$master" ]; then

         status="PRIMARY"

         else

         status="SECONDARY"

      fi

#else

#    status="Unable to find the status"

#fi

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "    Mongo node is  ${MENU} " $status

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo ""

echo ""

echo "" 

pause

}

LQ_time()

{

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD} enter how many lines you want to trace in the log... ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read log_time

export LOGTIME=$log_time

#tail -${LOGTIME} ${FNAME}|grep -i "QUERY"|awk '{ print $NF }'|awk -F "ms" '{print $1}'

tail -${LOGTIME} ${FNAME}|grep -i "QUERY"|awk '{print $1,$NF}'

echo ""

echo ""

echo ""

pause

}

Stpdown()

{

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD} enter how many lines you want to trace in the log... ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read log_time

tail -${log_time} ${FNAME}|grep -i "step" 

echo ""

echo ""

echo ""

pause

}

Stpdown-detail()

{

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD} enter how many lines you want to trace in the log... ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read log_time

export LOGTIME=$log_time

tail -${LOGTIME} ${FNAME}|grep -C 10 "step" 

echo ""

echo ""

echo ""

pause

}

Log_PIT_detail()

{

FNAME=`cat fname.txt`

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD}enter the time stamp for checking log ${MENU} eg: 2017-06-07T23:11:16  ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read time_stamp

export TIMESTMP=$time_stamp

grep -i ${time_stamp} ${FNAME}|tail -50

echo ""

echo ""

echo ""

pause

}

LQ_Count()

{

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD} enter how many lines you want to trace in the log... ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read log_time

export LOGTIME=$log_time

echo -e "${MENU} Number of queries having latency > 100ms ${NORMAL}"

echo "-------------------------------------------------------------------------------------------------------------------------"

tail -${LOGTIME} ${FNAME}|grep -i "QUERY"|awk '{ print $NF }'|awk -F "ms" '{print $1}'|wc -l

echo "-------------------------------------------------------------------------------------------------------------------------"

tail -${log_time} ${FNAME}|grep -i "QUERY"|head  

echo ""

echo ""

echo ""

pause

}

collscan()

{

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD} enter how many lines you want to trace in the log... ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read log_time

export LOGTIME=$log_time

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD} Queries which are using FULL TABLE SCAN ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

tail -${log_time} ${FNAME}|grep -i "COLLSCAN"|head

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo ""

pause

}

tail_log()

{

tail -f ${FNAME}

show_menu

}

read_count()

{

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo -e "${BOLD}                       Past how many  hours query load info you want to see? ${NORMAL}"

echo -e "${GREEN}====================================================================================================${NORMAL}"

read -p "            " past_hours

x=`expr $past_hours \* 20`

echo -e "${GREEN}====================================================================================================${NORMAL}"

tail -$x  /root/Mstat.txt |awk -F " " '{print $19,"Number of queries   ====>  ",$2}'

echo -e "${GREEN}====================================================================================================${NORMAL}"

echo ""

echo ""

echo ""

pause

}

read_load()

{

echo -e "${GREEN}======================================================================================================================${NORMAL}"

echo -e "${BOLD}Past how many hours load on the db server you want to see? ${NORMAL}"

echo -e "${GREEN}======================================================================================================================${NORMAL}"

read -p "            "  past_hours

x=`expr $past_hours \* 20`

echo -e "${GREEN}======================================================================================================================${NORMAL}"

#tail -$x  /root/Mstat.txt |awk -F " " '{print $19,"====>",$2}'

echo -e "${GREEN} Time                               Queries    Inserts    Updates    Deletes   NW_IN   NW_Out  Connections${NORMAL}" 

tail -$x /root/Mstat.txt|awk -F " " '{print $19," =======> ",$2,"      ",$1,"      ",$3,"      ",$4,"      ",$14,"     ",$15,"        ",$16}'

echo -e "${GREEN}======================================================================================================================${NORMAL}"

echo ""

echo ""

echo ""

pause

}

login()

{

echo `ps -ef|grep -v grep|grep mongod|tail -1`

echo `df -k | grep -i imdb`

pth=`ps -ef|grep -v grep|grep mongod|tail -1|awk -F " " '{print $8}'|cut -d '/'  -f 1,2,3,4,5`

port=`ps -ef|grep -v grep|grep mongod|tail -1|awk -F " " '{print $10}'|awk -F "/" '{print $4}'|cut -d "-" -f 2|cut -d "." -f 1`

echo `service mongod-$port status`

pth=$pth"/mongo --port "$port

$pth --quiet admin  -u user -p pwd

echo ""

pause

}

mstat()

{

pth=`ps -ef|grep -v grep|grep mongod|tail -1|awk -F " " '{print $8}'|cut -d '/'  -f 1,2,3,4,5`

port=`ps -ef|grep -v grep|grep mongod|tail -1|awk -F " " '{print $10}'|awk -F "/" '{print $4}'|cut -d "-" -f 2|cut -d "." -f 1`

$pth/\mongostat  --port=$port -u username -p password -n 50  --authenticationDatabase=admin

echo ""

pause

}

show_menu()

{

echo -e "${MENU}=================================================================================${NORMAL}"

echo -e "${BOLD}${RED}//                     MONGO DATABASE HEALTH CHECKS                   //  ${NORMAL}"

echo -e "${MENU}=================================================================================${NORMAL}"

echo -e "${RED} (0) ${GREEN}  :   Node is PRIMARY/SECONDARY"

echo ""

echo -e "${RED} (1) ${GREEN}  :   Search for any latency entry"

echo ""

echo -e "${RED} (2) ${GREEN}  :   Primary stepdown Timestamp"

echo ""

echo -e "${RED} (3) ${GREEN}  :   Primary Stepdown BEFORE and AFTER details"

echo ""

echo -e "${RED} (4) ${GREEN}  :   Searching details for a particular timestamp"

echo ""

echo -e "${RED} (5) ${GREEN}  :   Latency queries count and display sample Latency queries"

echo ""

echo -e "${RED} (6) ${GREEN}  :   Display Past Hr(s) - Application query load count info"

echo ""

echo -e "${RED} (7) ${GREEN}  :   Tail the log indefinite"

echo ""

echo -e "${RED} (8) ${GREEN}  :   List queries which are using collscan"

echo ""

echo -e "${RED} (9) ${GREEN}  :   Display Past Hr(s) DB server Read, Write, NW In/Out and Connections load info"

echo ""

echo -e "${RED} (10) ${GREEN} :   Login to Mongo prompt"

echo ""

echo -e "${RED} (11) ${GREEN} :   Display current mongo load statistics"

echo ""

echo -e "${RED} (12) ${GREEN} :   Exit ${NORMAL}"

echo -e "${MENU}====================================================${NORMAL}"

}

read_menu()

{

local choice

echo -e "${BOLD} Enter choice [0-12] ${NORMAL}"

read -p "                    " choice

case $choice in

   0)clear;

     status_check;

   ;;

   1) clear;

      LQ_time;

   ;;

   2) clear;

      Stpdown;

   ;;

   3) clear;

     Stpdown-detail;

   ;;

   4)clear;

     Log_PIT_detail;

   ;;

   5)clear;

     LQ_Count;

   ;;

   6)clear;

     read_count;

   ;;

   7)clear;

     tail_log;

   ;;

   8)clear;

     collscan;

   ;;

   9)clear;

     read_load;

   ;;

   10)clear;

      login;

   ;;  

   11)clear;

      mstat;

   ;;

   12) break

   ;;

   *)clear;

   echo -e "${RED}Wrong choice............choose [0-12]  :  ${NORMAL}   "

esac

}

while true

do

show_menu

read_menu

done

#################################################################################################

Roles-Privs-create user

Built-in Roles:

read
readWrite
dbAdmin
userAdin
clusterAdmin

readAnyDatabase
readWriteAnyDatabase
dbAdminAnyDatabase
userAdminAnyDatabase

User prompt changing:

prompt = function() {
    user = db.runCommand({connectionStatus:1}).authInfo.authenticatedUsers[0]
    host = db.getMongo().toString().split(" ")[2]
    curDB = db.getName()
    if (user) {
       uname = user.user
    }
    else {
       uname = "local"
    }
    return uname + "@" + host + ":" + curDB + "> "
}


Creating roles:

db.createRole({ role: "appReadRole", privileges: [ { resource: { db: "test", collection: "" }, actions:

[ "find" ] } ], roles: [] })


Listing all the roles:

db.getRoles(
    {
      rolesInfo: 1,
      showPrivileges:false,
      showBuiltinRoles: false
    }
)

Creating user with roles:

var a={user:"mani", pwd:"mani", roles:[{role:"read",db:"test"}]}
db
db.createUser(a)


db.createUser({ user: "finance", pwd: "password", roles: [ { role: "appReadRole", db: "test" } ] })

mongo Mani-PC:27002/test -u finance -p password

Changing password:

db.changeUserPassword("finance", "welcome123")

Current user details:

db.runCommand({connectionStatus : 1})

use admin
db.system.users.find().pretty()
db.system.users.remove({user:"userA"})


db.grantRolesToUser(
  "report",
  [{ "role" : "readWriteAnyDatabase", "db" : "admin" }]
)


db.revokeRolesFromUser(
  "report",
  [{ "role" : "readWriteAnyDatabase", "db" : "admin" }]
)

db.getUser("report")


use products
db.grantPrivilegesToRole(
  "inventoryCntrl01",
  [
    {
      resource: { db: "products", collection: "" },
      actions: [ "insert" ]
    },
    {
      resource: { db: "products", collection: "system.js" },
      actions: [ "find" ]
    }
  ],
  { w: "majority" }
)


===============================================================
Edit .mongorc.js in your home directory for changing the prompt:

function prompt() {
    var username = "anon";
    var user = db.runCommand({connectionStatus : 1}).authInfo.authenticatedUsers[0];
    var host = db.getMongo().toString().split(" ")[2];
    var current_db = db.getName();

    if (!!user) {
        username = user.user;
    }

    return username + "@" + host + ":" + current_db + "> ";
}
==============================================================

Mongo Prompt with DB name and current user

Past the below code in the mongo shell


prompt = function() {
    user = db.runCommand({connectionStatus:1}).authInfo.authenticatedUsers[0]
    host = db.getMongo().toString().split(" ")[2]
    curDB = db.getName()
    if (user) {
       uname = user.user
    }
    else {
       uname = "local"
    }
    return uname + "@" + host + ":" + curDB + "> "
}

Roles and authentication

Common Roles:
============
read
readWrite
dbAdmin
userAdin
clusterAdmin

readAnyDatabase
readWriteAnyDatabase
dbAdminAnyDatabase
userAdminAnyDatabase



var a={user:"abcd", pwd:"efgh", roles:["readWrite]}

db.createUser(a)

mongo hostname/dbname -u abcd -p efgh

Wconcern - Replication - Best practice

Best practice

Write concern -> wmajority setting is important
Wtimeout value should be appropriate to ensure taking Write ackg within timeout interval
Connection pool max size should be appropriate

Replication failover

FAILOVER
=========

ps  -Aef|grep mongod
Ps -Aef|grep mongod|grep <pid>    ---- belongs to primary instance port id
kill  -9   <pid>
rs.slaveOk() for making the secondary replica set to allow user queries...this has to be executed on secondary dbs only

After all the failover testings are done, bringing back the failed node to online:
mongod --replSet abc  --dbpath 1 --port 27001 --oplogSize 50 --logpath log.1 --logappend --fork

Read Preference :
rs.slaveOk() for making the secondary replica set to allow user queries...this has to be executed on secondary dbs only
primary  - Default
primary preferred - Try primary if not reachable then read from secondary
secondary - Read from scondary, offload read from primary
secondary preferred - Scondary first then primary
nearest - Nearest member first