Shell Script to compare row counts between two Greenplum Postgresql Database

This shell script takes 3 arguments: source db, target db and a list file of tables.

List file is in format:
$ cat listfile.lst
schema.table : where<> ::
schema2.table2 : where<> ::

Also ~/gpdbinfo.dat file contains database name, host and port details.

#!/bin/ksh
######################################################################
#USAGE : gpdb_row_compare.ksh <source> <target> <list of tables>
# format for list of tables --> schema.table:where clause::
######################################################################

module load gpdb/client/4.3.6.1

touch Report.file
touch greenReport.file
touch redReport.file
echo "" > Report.file
echo "" > greenReport.file
echo "" > redReport.file

SOURCE_GP=$1
TARGET_GP=$2
listfile=$3

header_write()
{
echo "<table border=1>" >> Report.file
echo "tr><th SIZE=5>QUERY</th><th SIZE=5>$SOURCE_GP Count</th><th SIZE=5>$TARGET_GP Count</th><th SIZE=5>MATCH</th></tr>" >> Report.file
} 

green_match()
{
#echo "<tr bgcolor=\"lime\">" >> greenReport.file
echo "<tr>" >> greenReport.file
echo "<td align=centre><FONT SIZE=2>$1</FONT></td>" >> greenReport.file
echo "<td align=centre><FONT SIZE=2>$2</FONT></td>" >> greenReport.file
echo "<td align=centre><FONT SIZE=2>$3/FONT></td>" >> greenReport.file
echo "<td align=centre><FONT SIZE=2>YES</FONT></td>" >> greenReport.file
echo "</tr>" >> greenReport.file
}

red_match()
{
#echo "<tr bgcolor=\"lightpink\">" >> redReport.file
echo "<tr>" >> redReport.file
echo "<td align=centre><FONT SIZE=2>$1</FONT></td>" >> redReport.file
echo "<td align=centre><FONT SIZE=2>$2</FONT></td>" >> redReport.file
echo "<td align=centre><FONT SIZE=2>$3/FONT></td>" >> redReport.file
echo "<td align=centre><FONT SIZE=2>NO</FONT></td>" >> redReport.file
echo "</tr>" >> redReport.file
}

#######################################

echo -e "\n Connecting to GP --- $SOURCE_GP "
echo -e "\nDETAILS --"
cat ~/gpdbinfo.dat | grep "$SOURCE_GP:" | awk -F':' '{print "-t -d "$3"-h "$2" -p "$5 }' | read GPDBINFO
cat ~/gpdbinfo.dat | grep "$SOURCE_GP:" | awk -F':' '{print $3}' | read GPSHORTNAME
cat ~/gpdbinfo.dat | grep "$SOURCE_GP:" | awk -F':' '{print $2}' | read HOST
cat ~/gpdbinfo.dat | grep "$SOURCE_GP:" | awk -F':' '{print $3}' | read PORT

echo "gp: $GPSHORTNAME"
echo "host: $HOST"
echo "port : $PORT"
USER="$(whoami)"
echo "user : $USER"
echo -e "/n"

set -A prod_count
i=0

while IFS=':' read -r table filter dummy dummy2 || [[ -n "$line" ]];do
    query="select count(*) from $table $filter ;"
    prod_count[$i]=$(psql $GPDBINFO -c "$query")
    echo "$query -->${prod_count[$i]}"
    ((i++))
done < "$listfile"

echo ${prod_count[@]}

######################################

echo -e "\n Connecting to GP --- $TARGET_GP "
echo -e "\nDETAILS --"
cat ~/gpdbinfo.dat | grep "$TARGET_GP:" | awk -F':' '{print "-t -d "$3"-h "$2" -p "$5 }' | read GPDBINFO
cat ~/gpdbinfo.dat | grep "$TARGET_GP:" | awk -F':' '{print $3}' | read GPSHORTNAME
cat ~/gpdbinfo.dat | grep "$TARGET_GP:" | awk -F':' '{print $2}' | read HOST
cat ~/gpdbinfo.dat | grep "$TARGET_GP:" | awk -F':' '{print $3}' | read PORT

echo "gp: $GPSHORTNAME"
echo "host: $HOST"
echo "port : $PORT"
USER="$(whoami)"
echo "user : $USER"
echo -e "/n"

set -A stg_count
i=0

while IFS=':' read -r table filter dummy dummy2 || [[ -n "$line" ]];do
    query="select count(*) from $table $filter ;"
    prod_count[$i]=$(psql $GPDBINFO -c "$query")
    echo "$query -->${stg_count[$i]}"
    ((i++))
done < "$listfile"

echo ${stg_count[@]}

################################

### Prepare Report File

i=0

while IFS=':' read -r table filter dummy dummy2 || [[ -n "$line" ]];
do
    query="select count(*) from $table $filter ;"
    echo "${stg_count[$i]}"
    echo "${prod_count[$i]}"
    
    if [ ${stg_count[$i] == ${prod_count[$i]} ]
    then
        echo "green"
        green_match "$query" ${prod_count[$i]} ${stg_count[$i]}
    else
        echo "red"
        red_match "$query" ${prod_count[$i]} ${stg_count[$i]}
    fi

    ((i++))
done < "$listfile"

header_write
cat redReport.file >> Report.file
cat greenReport.file >> Report.file

#####################################

### Send Report File as Email
./sendmail.ksh $@

#####################################

Below is the script to send the report file generated above as email :

#!/bin/ksh

{
echo "From: GP COUNT COMPARE REPORT"
echo "To: subhankd@gmail.com"
echo "Cc: "
echo "MIME-Version: 1.0"
echo "Subject: GP Compare: $1 vs $2"
echo "Content-Type: text/html"

cat ./Report.file

} | sendemail -t

echo " Email Sent."

Leave a comment

Design a site like this with WordPress.com
Get started