From fb04865e3ae23ccd10be5aa1a871bf5898349133 Mon Sep 17 00:00:00 2001 From: prudhvirajp Date: Tue, 3 Mar 2026 16:02:08 +0530 Subject: [PATCH 1/3] Renamed the script from reorgTablesIndexesInplace2.sh to reorgTablesIndexesInplace.sh to copy files to respected paths Refactor backup scripts to improve readability and functionality. Updated the new ICD links to the script Added parameter checks, updated variable naming conventions, and enhanced logging. Renamed the reorgTablesIndexesInplace2.sh to reorgTablesIndexesInplace.sh Refactor runstats_rebind.sh to run daily runstats and weekly rebinds --- .../files/CopyDBScripts.sh | 4 +- .../120-ibm-db2u-database/files/DB2_Backup.sh | 471 +++++++++--------- .../files/RUN_OnDemandFULL_BKP.sh | 145 +++--- .../120-ibm-db2u-database/files/Run_Backup.sh | 169 ++++--- ...place2.sh => reorgTablesIndexesInplace.sh} | 0 .../files/runstats_rebind.sh | 111 +++-- 6 files changed, 495 insertions(+), 405 deletions(-) rename instance-applications/120-ibm-db2u-database/files/{reorgTablesIndexesInplace2.sh => reorgTablesIndexesInplace.sh} (100%) diff --git a/instance-applications/120-ibm-db2u-database/files/CopyDBScripts.sh b/instance-applications/120-ibm-db2u-database/files/CopyDBScripts.sh index 0395ae254..c95d180a3 100644 --- a/instance-applications/120-ibm-db2u-database/files/CopyDBScripts.sh +++ b/instance-applications/120-ibm-db2u-database/files/CopyDBScripts.sh @@ -30,7 +30,7 @@ cp -rp RUN_OnDemandFULL_BKP.sh ${INSTHOME}/bin/ cp -rp runstats_rebind.sh ${INSTHOME}/bin/ cp -rp CreateRoles.sh ${INSTHOME}/bin/ cp -rp grant_check.sh ${INSTHOME}/bin/ -cp -rp reorgTablesIndexesInplace2.sh ${INSTHOME}/bin/ +cp -rp reorgTablesIndexesInplace.sh ${INSTHOME}/bin/ cp -rp extract_authorization.sh ${INSTHOME}/bin cp -rp HADRMON.sh ${INSTHOME}/bin @@ -48,7 +48,7 @@ cp PostBackFlow.sh ${INSTHOME}/Managed cp OwnerCheck.txt ${INSTHOME}/Managed echo -e "\nCopying files to maintenance directory under Instance Home . . . "; -cp -rp reorgTablesIndexesInplace2_maintenance.sh ${INSTHOME}/maintenance/reorgTablesIndexesInplace2.sh +cp -rp reorgTablesIndexesInplace2_maintenance.sh ${INSTHOME}/maintenance/reorgTablesIndexesInplace.sh if [ ! -d ${INSTHOME}/maintenance/logs ] ; then mkdir -p ${INSTHOME}/maintenance/logs echo "${DATETIME}:Creating directory ${INSTHOME}/maintenance/logs" diff --git a/instance-applications/120-ibm-db2u-database/files/DB2_Backup.sh b/instance-applications/120-ibm-db2u-database/files/DB2_Backup.sh index d8f21436c..d6f4c2413 100755 --- a/instance-applications/120-ibm-db2u-database/files/DB2_Backup.sh +++ b/instance-applications/120-ibm-db2u-database/files/DB2_Backup.sh @@ -1,12 +1,11 @@ #!/bin/ksh -set -x +#set -x ######################################################### -# DB2_Backup.sh -# -# Things to do: -# Recovery history retention (days) (REC_HIS_RETENTN) = 0 >>> Need to set to 15 days +# DB2_Backup.sh # +# Things to do: +# Recovery history retention (days) (REC_HIS_RETENTN) = 0 >>> Need to set to 15 days # # The cron job on the cluster will supply the needed parameters for this script # If an on demand backup (Full) is required, the DB2_Backup.sh script can be called with the following parameters @@ -18,242 +17,263 @@ set -x # ######################################################### - -. /mnt/backup/bin/.PROPS +# -- Script Usage +if [[ $# -eq 4 ]]; then + typeset -l instance=$1 dbname=$2 + typeset -u INSTANCE=$1 DBNAME=$2 + typeset -i NUM_BACKUPS_TO_KEEP=$3 + typeset -l BKUP_TYPE=$4 -#### COSBACKUPBUCKET=masms-pp-1-cos-backup-pseg-test-pr-wdc -#### For testing -TESTMSG="######## TESTING ###########" -echo ${COSBACKUPBUCKET} -COSBACKUPBUCKET=${CONTAINER} -#### TESTING URL +else + print `tput smso` "Usage! $0 instance database number_of_backups_to_keep" `tput rmso` + exit 1 +fi -Server=`hostname` -instance=`whoami` -FULLIMAGE= -DATETIME=`date +%Y-%m-%d_%H%M%S`; -BACKUP_BASE=/mnt/backup -BACKUP_LOGS=${BACKUP_BASE}/${DB2INSTANCE} -BACKUP_PATH=DB2REMOTE://AWSCOS/${COSBACKUPBUCKET}/backups-manage/${HOSTNAME} -ARCBKP_PATH=${BACKUP_PATH}/${DATETIME} -CLEAN_LOG=${BACKUP_PATH}/.cleanup.log -instance_home=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${instance}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` +# -- Standard Parameters +HOSTNAME=`hostname` +NAMESPACE=`hostname -A | awk -F '.' '{print $3}'` +DBINSTANCE=`whoami` +DATETIME=`date +'%F_%T'`; +INSTANCE_HOME=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${DBINSTANCE}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` IP=`/sbin/ifconfig | grep "inet" | grep broadcast | awk '{print $2}'` -BACK_LOG=$instance_home/bin/.$2_BackupLOG.out +CUSTNAME=`hostname | sed 's/c-db2wh-//; s/c-//; s/-db2u-0//; s/db2u/-/; s/-manage//;' | tr '[:lower:]' '[:upper:]'` +SCRIPT_DIR=${INSTANCE_HOME}/bin + +# -- Source DB2 Profile + +if [[ ! -f "${INSTANCE_HOME}/sqllib/db2profile" ]]; then + echo "ERROR - ${INSTANCE_HOME}/sqllib/db2profile not found" + EXIT_STATUS=1 +else + . ${INSTANCE_HOME}/sqllib/db2profile +fi + +set -x +# -- Source the PROPS file +. /mnt/backup/bin/.PROPS + +# -- Backup Parameters + +COSBACKUPBUCKET="${CONTAINER}" +BUCKET_ALIAS=`db2 list storage access | grep ${COSBACKUPBUCKET} -B4 | grep ALIAS | awk -F '=' '{print $2}'` +BACKUP_BASE="/mnt/backup" +BACKUP_LOGS=${SCRIPT_DIR}/${DBINSTANCE} +BACKUP_PATH=DB2REMOTE://${BUCKET_ALIAS}/${COSBACKUPBUCKET}/backups-${APPENV}/${HOSTNAME} +ARCBKP_PATH=${BACKUP_LOGS}/${DATETIME} +CLEAN_LOG=${BACKUP_LOGS}/.cleanup.LOG Maillog="/tmp/.backup_maillog" - +BACK_LOG=${SCRIPT_DIR}/.${DBNAME}_BackupLOG.out +ICD_LOG=${SCRIPT_DIR}/.Maillive.log +HSTYPE="Backup" -SLACK_NOTIFY() -{ - des="$instance - Backup - $Server ${database} -- DATABASE Backup issues" - echo "${CUSTNAME} - $instance - Backup - $Server $IP ${database} DATABASE Backup issues" > .Maillive.log - echo "############################" >> .Maillive.log - cat ${BACK_LOG} >> .Maillive.log - longdes=`cat .Maillive.log | sed 's/"//g' | sed "s/'//g"` - slackdes=" BACKUP FAILED for ${Server} - ${CONTAINER}} ...Please investigate " - -### Send Failure notification to a slack channel ## -cat << ! >.curl_$database.sh - curl -X POST -H 'Content-type: application/json' --data '{"text":"$slackdes"}' ${SLACKURL} -! -if [[ -n "${SLACKURL}" ]]; then - /bin/bash .curl_$database.sh > .curl_$database.out 2>&1 +# -- Valid only for MAS-CP4D Customers +if (( ${CUSTNAME} )) ; then + CUSTNAME=`echo ${CONTAINER} | awk -F '-backup-' '{print $2}' | awk -F '-pr-' '{print $1}' | tr '[:lower:]' '[:upper:]'` +fi + +# -- Database Environment +if [[ ${BUCKET_ALIAS} == "IBMCOS" ]]; then + DBENV="MASMS" +else + DBENV="MAS_SaaS" fi - ##### Create ICD Incident #### - ####### If Backup fails ### - des="${CUSTNAME} - ${instance} - Backup - ${HOSTNAME} ${database} - MASMS -- Backup Failed" - echo "############################" >> .Maillive.log - longdes=`cat .Maillive.log | sed 's/"//g' | sed "s/'//g"` - longdes=`echo "
 ${longdes} 
"` - ICD_URL="https://servicedesk.mro.com" - if ! curl -k -s --connect-timeout 3 ${ICD_URL} >/dev/null; then - ICD_URL="https://servicedesk.cds.mro.com" - fi -cat << ! >.curl_${database}_ICD.sh - curl --insecure --location --request POST "${ICD_URL}/maximo_mif/oslc/os/hsincident?lean=1" \ - --header "Authorization: Basic ${ICD_AUTH_KEY}" \ - --header 'Content-Type: application/json' \ - --data '{ - "description":"$des", - "reportedpriority":4, - "internalpriority":4, - "reportedby":"DB2", - "affectedperson":"CTGINST1", - "description_longdescription":"$longdes", - "siteid":"001", - "classstructureid":"1341", - "classificationid":"IN-DBPERF", - "hshost":"{servicedesk-pdb-sjc03-2.cds.mro.com:0:50}", - "hstype":"BACKUP" - }' +# -- Function to send Slack notification + +SLACK_NOTIFY() { + + SLACKDES="$1" + # -- Send Failure notification to a slack channel + cat << ! >.curl_${DBNAME}.sh + curl -X POST -H 'Content-type: application/json' --data '{"text":"${SLACKDES}"}' ${SLACKURL} ! -if [[ -n "${ICD_AUTH_KEY}" ]]; then - /bin/bash .curl_${database}_ICD.sh > .curl_${database}_ICD.out 2>&1 -fi + /bin/bash .curl_${DBNAME}.sh > .curl_${DBNAME}.out 2>&1 } +# -- Create ICD Incident , If Backup fails +CREATE_ICD() { + HTYPE=`echo ${HSTYPE} | tr '[:lower:]' '[:upper:]'` + DES="$1" + echo "############################" >> ${ICD_LOG} + LONGDES=`cat ${ICD_LOG} | sed 's/"//g' | sed "s/'//g"` + LONGDES=`echo "
 ${LONGDES} 
"` -if [ ! -f "$instance_home/sqllib/db2profile" ] -then - echo "ERROR - $instance_home/sqllib/db2profile not found" - EXIT_STATUS=1 -else - . $instance_home/sqllib/db2profile -fi + # -- Verify the ICD Status + if curl -k -s --connect-timeout 3 ${ICD_URL_SAAS} >/dev/null; then + CURL_REQ="--request POST --url ${ICD_URL_SAAS} " + AUTH_REQ="apikey: ${ICD_API_KEY}" + fi + + # -- Generate Curl Syntax to push to ICD + cat << ! >.curl_${DBNAME}_ICD.sh + curl ${CURL_REQ} \ + --header '${AUTH_REQ}' \ + --header 'Content-Type: application/json' \ + --data '{ + "description":"${DES}", + "reportedpriority":4, + "internalpriority":4, + "reportedby":"DB2", + "affectedperson":"${DBENV}", + "ownergroup":"HSDBA", + "description_longdescription":"${LONGDES}", + "siteid":"001", + "classstructureid":"1341", + "classificationid":"IN-DBPERF", + "hshost":"${HOSTNAME}", + "hstype":"${HTYPE}" + }' +! + /bin/bash .curl_${DBNAME}_ICD.sh > .curl_${DBNAME}_ICD.out 2>&1 +} -if [ -f $Maillog ] -then - rm $Maillog +# -- Delete old log file +if [[ -f $Maillog ]]; then + rm $Maillog fi -echo "COS bucket = ${COSBACKUPBUCKET} " > $BACK_LOG -echo "BACKUP Start time : ${DATETIME}" >> $BACK_LOG -echo " " >> $BACK_LOG -echo ${HOSTNAME} >> $BACK_LOG -echo " " >> $BACK_LOG -echo " " >> $BACK_LOG - -if [[ $# -eq 4 ]] -then - typeset -l instance=$1 database=$2 - typeset -u INSTANCE=$1 DATABASE=$2 - typeset -i num_backups_to_keep=$3 - typeset -l BKUP_TYPE=$4 - - ##### until the db is bounced to pickup the TRACKMOD parm..We have to hardcode a FULL backup - #####BKUP_TYPE=full - ### BKUP_TYPE = full or inc #### -else - print `tput smso` "Usage! $0 instance database number_of_backups_to_keep" `tput rmso` - exit 1 +# -- Create the backup log directory if it doesnt exists +if [[ ! -d ${BACKUP_LOGS} ]]; then + mkdir -m 755 ${BACKUP_LOGS} fi - -### Check for the existance of /home/ctginst1/sqllib/db2dump/libdb2compr.so...if it exists, delete it -COMPRESS_LOC=$instance_home/sqllib/db2dump/libdb2compr.so -if [[ -f ${COMPRESS_LOC} ]] -then + +# -- Setting backup type +if [[ ${BKUP_TYPE} == 'full' ]]; then + BKPTYPE="FULL" +else + BKPTYPE="DIFF" +fi + +# -- Script Execution starts from here + +echo -e "\n-------------------------------------" | tee ${BACK_LOG} +echo -e "Backup Start Time \t :: ${DATETIME}" | tee -a ${BACK_LOG} +echo -e "COS Bucket \t\t :: ${COSBACKUPBUCKET}" | tee -a ${BACK_LOG} +echo -e "\nHostname \t\t :: ${HOSTNAME}" | tee -a ${BACK_LOG} +echo -e "Namespace \t\t :: ${NAMESPACE}" | tee -a ${BACK_LOG} +echo -e "HostIP \t\t\t :: ${IP}" | tee -a ${BACK_LOG} +echo -e "-------------------------------------\n" | tee -a ${BACK_LOG} + +# -- Check for the existance of /home/ctginst1/sqllib/db2dump/libdb2compr.so...if it exists, delete it +COMPRESS_LOC=${INSTANCE_HOME}/sqllib/db2dump/libdb2compr.so +if [[ -f ${COMPRESS_LOC} ]]; then rm ${COMPRESS_LOC} fi -### Check to see if the database is Running -ps -ef | grep db2sys | grep -v grep > /dev/null 2>&1 -if [ $? -eq 1 ]; then - echo "Database is not active " - echo "$Server,Database Not Active,BACKUP Not Run" > $instance_home/bin/LASTbkupRUN - - ### Send error alert - SLACK_NOTIFY +# -- Check to see if the Instance is up and Running +ps -ef | grep db2sysc | grep -v grep > /dev/null 2>&1 +if [[ $? -eq 1 ]]; then + echo "Instance is not active " + echo "${HOSTNAME}, Instance is not Active, BACKUP cannot Run!" | tee ${INSTANCE_HOME}/bin/LASTbkupRUN ${ICD_LOG} >/dev/null + echo "############################" >> ${ICD_LOG} + cat ${BACK_LOG} >> ${ICD_LOG} + + SLACKDES="${CUSTNAME} - ${DBENV} - ${HOSTNAME}, Instance is not Active, Backup cannot Run! " + DES="${CUSTNAME} - ${DBENV} - ${DBNAME} - ${HOSTNAME} -- Instance is not Active, Backup cannot Run!! " + + # -- Send error notification to Slack + SLACK_NOTIFY "${SLACKDES}" + + # -- Create ICD ticket if fails + CREATE_ICD "${DES}" + + # -- End the script execution exit fi - -### Check to see if the database is HADR -db2pd -hadr -db ${database} | awk -F= '/HADR_ROLE/ {print $2}' | grep STANDBY > /dev/null 2>&1 -if [ $? -eq 0 ]; then - echo "This is a HADR database" - echo "Backup successful. The timestamp for this backup image is : HADR_DB" - echo "$Server,HADR,NO BACKUPS" > $instance_home/bin/LASTbkupRUN - exit 0 -fi - -### Create the backup directory if it doesnt already exist -if [[ -d $BACKUP_LOGS ]] -then : - else mkdir -m 755 ${BACKUP_LOGS} + +# -- Verify whether Database is Standby +db2pd -hadr -db ${DBNAME} | awk -F= '/HADR_ROLE/ {print $2}' | grep STANDBY > /dev/null 2>&1 +if [[ $? -eq 0 ]]; then + echo "This is a HADR Database" + echo "Backup successful. The timestamp for this backup image is : HADR_DB" + echo "${HOSTNAME}, HADR, NO BACKUPS" > ${INSTANCE_HOME}/bin/LASTbkupRUN + exit 0 fi -echo "BACKUP Start time : ${DATETIME}" -echo " " -echo $Server -echo " " -echo " " - -db2 -v archive log for db $database | tee -a $BACK_LOG -sleep 30 - -if [[ $num_backups_to_keep -gt 0 ]] -then - ### Backup database - if [ ${BKUP_TYPE} = 'full' ] ; then - db2 -v backup db $database online to $BACKUP_PATH compress UTIL_IMPACT_PRIORITY 50 include logs without prompting | tee -a $BACK_LOG +# -- Archive the logs for Database +db2 -v "ARCHIVE LOG FOR DB ${DBNAME}" | tee -a ${BACK_LOG} +sleep 20 + +# -- Starting backup for the database +if [[ ${NUM_BACKUPS_TO_KEEP} -gt 0 ]]; then + + if [[ ${BKUP_TYPE} = 'full' ]]; then + db2 -v "BACKUP DB ${DBNAME} ONLINE TO ${BACKUP_PATH} COMPRESS UTIL_IMPACT_PRIORITY 50 INCLUDE LOGS WITHOUT PROMPTING" | tee -a ${BACK_LOG} else - db2 -v backup db $database online INCREMENTAL DELTA to $BACKUP_PATH compress UTIL_IMPACT_PRIORITY 50 include logs without prompting | tee -a $BACK_LOG + db2 -v "BACKUP DB ${DBNAME} ONLINE INCREMENTAL DELTA TO ${BACKUP_PATH} COMPRESS UTIL_IMPACT_PRIORITY 50 INCLUDE LOGS WITHOUT PROMPTING" | tee -a ${BACK_LOG} fi - grep -Fq "Backup successful." $BACK_LOG - if [ $? = 0 ]; then - Backup_timestamp=`grep timestamp $BACK_LOG | cut -d: -f2` - - ### Need to find all the files associate with the backup ## - ### - ### - ### Need to change this to the db2RemStgManager command to get a list of all backup images just created - ### - ### - - # fi - else - SLACK_NOTIFY - #exit + + grep -Fq "Backup successful." ${BACK_LOG} + if [[ $? -ne 0 ]]; then + echo "${CUSTNAME} - ${DBENV} - ${HOSTNAME}, ${BKPTYPE} DB Backup Failed ! Database Backup issues !!!" > ${ICD_LOG} + echo "############################" >> ${ICD_LOG} + cat ${BACK_LOG} >> ${ICD_LOG} + + SLACKDES="${CUSTNAME} - ${DBENV} - ${HOSTNAME}, ${BKPTYPE} DB Backup Failed . . . Please investigate ! ! ! " + DES="${CUSTNAME} - ${DBENV} - ${DBNAME} - ${HOSTNAME} -- ${BKPTYPE} ${HSTYPE} Failed !" + + # -- Send error notification to Slack + SLACK_NOTIFY "${SLACKDES}" + # -- Create ICD ticket if fails + CREATE_ICD "${DES}" fi fi -######## Copy keystore to COS -set -x +# -- Copy keystore to COS SOURCE1=/mnt/blumeta0/db2/keystore/keystore.p12 SOURCE2=/mnt/blumeta0/db2/keystore/keystore.sth -TARGET1=backups-manage/${HOSTNAME}/KEYSTORE/keystore.p12 -TARGET2=backups-manage/${HOSTNAME}/KEYSTORE/keystore.sth +TARGET1=backups-${APPENV}/${HOSTNAME}/KEYSTORE/keystore.p12 +TARGET2=backups-${APPENV}/${HOSTNAME}/KEYSTORE/keystore.sth DB2V=`db2level | grep Inform | awk '{print $5}' | sed 's/",//'` -if [ ${DB2V} = "v11.5.7.0" ] -then - db2RemStgManager S3 put server=${SERVER} auth1=${PARM1} auth2=${PARM2} container=${CONTAINER} source=${SOURCE1} target=${TARGET1} - db2RemStgManager S3 put server=${SERVER} auth1=${PARM1} auth2=${PARM2} container=${CONTAINER} source=${SOURCE2} target=${TARGET2} +if [[ ${DB2V} == "v11.5.7.0" ]]; then + + db2RemStgManager S3 put server=${HOSTNAME} auth1=${PARM1} auth2=${PARM2} container=${CONTAINER} source=${SOURCE1} target=${TARGET1} + db2RemStgManager S3 put server=${HOSTNAME} auth1=${PARM1} auth2=${PARM2} container=${CONTAINER} source=${SOURCE2} target=${TARGET2} + else - db2RemStgManager ALIAS PUT source=${SOURCE1} target=DB2REMOTE://AWSCOS//${TARGET1} - db2RemStgManager ALIAS PUT source=${SOURCE2} target=DB2REMOTE://AWSCOS//${TARGET2} + db2RemStgManager ALIAS PUT source=${SOURCE1} target=DB2REMOTE://${BUCKET_ALIAS}//${TARGET1} + db2RemStgManager ALIAS PUT source=${SOURCE2} target=DB2REMOTE://${BUCKET_ALIAS}//${TARGET2} fi -# exclude files that arent backups, e.g. backhist listing. -typeset -i no_backups=`./CheckCOS.sh | grep -i ${database}| cut -d/ -f3| grep 001 |wc -l` -echo " number of backups $no_backups" -### Prune the history file, if and only if the last backup succeeded. -### Remove archive transaction logs for expired backups, if there are a requisite number of successful backups. -### Remove expired backups in step. - -if [[ $num_backups_to_keep -gt 0 && $no_backups -ge $num_backups_to_keep ]] -then - db2 -v connect to $database | tee -a $BACK_LOG +# -- Exclude files that arent backups, e.g. backhist listing. + +typeset -i NO_BACKUPS=`~/bin/CheckCOS.sh | grep -i ${DBNAME} | cut -d/ -f3 | grep 001 | wc -l` +echo "Number of Backups in the bucket :: ${NO_BACKUPS}" + +# -- Prune the history file, if and only if the last backup succeeded. - timestmp=$(db2 -x "select coalesce(max(start), 17890713235959) from \ - (select bigint(start_time) - 1 as start, \ - row_number() over(order by start_time desc) as backup \ - from sysibmadm.db_history \ - where operation = 'B' \ - and objecttype = 'D' \ - and devicetype = 'D' \ - and sqlcode is null \ - and sqlwarn is null \ - ) as zzz \ - where backup = $num_backups_to_keep" ) - - db2 -v prune history $timestmp WITH FORCE OPTION and delete | tee -a $BACK_LOG - - ### loop until the recovery history file is stable and then report it +if [[ ${NUM_BACKUPS_TO_KEEP} -gt 0 && ${NO_BACKUPS} -ge ${NUM_BACKUPS_TO_KEEP} ]]; then + db2 -v CONNECT TO ${DBNAME} | tee -a ${BACK_LOG} + + TIMESTMP=$(db2 -x "select coalesce(max(start), 17890713235959) from \ + (select bigint(start_time) - 1 as start, \ + row_number() over(order by start_time desc) as backup \ + from sysibmadm.db_history \ + where operation = 'B' \ + and objecttype = 'D' \ + and devicetype = 'D' \ + and sqlcode is null \ + and sqlwarn is null \ + ) as zzz \ + where backup = ${NUM_BACKUPS_TO_KEEP}" ) + + db2 -v "PRUNE HISTORY ${TIMESTMP} WITH FORCE OPTION AND DELETE" | tee -a ${BACK_LOG} + + # -- Loop until the recovery history file is stable and then report it RC=999 typeset -i no_loops=0 while [[ $RC -gt 0 ]] do - db2 -v list history backup since $timestmp for $database > ${BACKUP_LOGS}/backhist + db2 -v list history backup since ${TIMESTMP} for ${DBNAME} > ${BACKUP_LOGS}/backhist RC=$? print RC for list history was $RC - cat ${BACKUP_LOGS}/backhist >> $BACK_LOG - if [[ $no_loops -gt 720 ]] - then - ### then youve been waiting an hour + cat ${BACKUP_LOGS}/backhist >> ${BACK_LOG} + if [[ $no_loops -gt 720 ]]; then + # -- then youve been waiting an hour print $0 "Im tired of waiting for the recovery history file to stabilise. Im giving up" break else @@ -261,44 +281,39 @@ then let no_loops=no_loops+1 fi done - echo "Content of backhist file:" + + echo "Content of backhist file: " cat ${BACKUP_LOGS}/backhist - db2 -v commit | tee -a $BACK_LOG - - db2 -v connect reset | tee -a $BACK_LOG - db2 -v terminate | tee -a $BACK_LOG + db2 -v commit | tee -a ${BACK_LOG} + db2 -v connect reset | tee -a ${BACK_LOG} + db2 -v terminate | tee -a ${BACK_LOG} fi -sleep 30 - -if [[ $num_backups_to_keep -eq 0 ]] -then - db2 -v connect to $database | tee -a $BACK_LOG - db2 -x "select location \ - from sysibmadm.db_history \ - where operation = 'X' \ - and operationtype = '1' " > $BACKUP_LOGS/archivelog.zaplist - for log in `cat $BACKUP_LOGS/archivelog.zaplist` +sleep 20 + +# -- Prune the archive logs and archive log history file +if [[ ${NUM_BACKUPS_TO_KEEP} -eq 0 ]]; then + db2 -v connect to ${DBNAME} | tee -a ${BACK_LOG} + db2 -x "select location from sysibmadm.db_history where operation = 'X' and operationtype = '1' " > ${BACKUP_LOGS}/archivelog.zaplist + + for LOG in `cat ${BACKUP_LOGS}/archivelog.zaplist` do - printf "`date +'%F %T'`\t%-110s\t%12d k\n" "${log}" "`du -sk ${log} | awk '{print $1}'`" >> ${CLEAN_LOG} + printf "`date +'%F %T'`\t%-110s\t%12d k\n" "${LOG}" "`du -sk ${LOG} | awk '{print $1}'`" >> ${CLEAN_LOG} done - ### prune history in step - timestmp=$(db2 -x "select max(start_time) from sysibmadm.db_history where operation = 'X' and operationtype = '1'") - db2 -v prune history $timestmp WITH FORCE OPTION and delete | tee -a $BACK_LOG + + # -- prune history in step + TIMESTMP=$(db2 -x "select max(start_time) from sysibmadm.db_history where operation = 'X' and operationtype = '1'") + db2 -v PRUNE HISTORY ${TIMESTMP} WITH FORCE OPTION AND DELETE | tee -a ${BACK_LOG} wait - db2 -v commit | tee -a $BACK_LOG - db2 -v connect reset | tee -a $BACK_LOG - db2 -v terminate | tee -a $BACK_LOG + db2 -v commit | tee -a ${BACK_LOG} + db2 -v connect reset | tee -a ${BACK_LOG} + db2 -v terminate | tee -a ${BACK_LOG} fi - + DATETIME=`date +%Y-%m-%d_%H%M%S`; -echo "BACKUP End time : ${DATETIME}" >> $BACK_LOG +echo "BACKUP End time :: ${DATETIME}" >> ${BACK_LOG} -if [[ ${BKUP_STATUS} -gt 0 ]] -then - ### Send error alert - SLACK_NOTIFY -fi +# -- Copy the current backup LOG to the Backup LOG history file +cat ${BACK_LOG} >> ${BACKUP_LOGS}/.BackupLOG -### Copy the current backup log to the Backup log history file -cat $BACK_LOG >> ${BACKUP_LOGS}/.BackupLOG +# -- END OF SCRIPT \ No newline at end of file diff --git a/instance-applications/120-ibm-db2u-database/files/RUN_OnDemandFULL_BKP.sh b/instance-applications/120-ibm-db2u-database/files/RUN_OnDemandFULL_BKP.sh index f8b97fd73..97a5903fd 100755 --- a/instance-applications/120-ibm-db2u-database/files/RUN_OnDemandFULL_BKP.sh +++ b/instance-applications/120-ibm-db2u-database/files/RUN_OnDemandFULL_BKP.sh @@ -2,8 +2,8 @@ #set -x ######################################################### -# Run_Backup.sh -# Run_Backup.sh will be called from the Cron Jobs +# RUN_OnDemandFULL_BKP.sh +# RUN_OnDemandFULL_BKP.sh will be called from the Cron Jobs # This script will list all local databases running in the instance on a node. It will call the # DB2_Backup.sh script to run a backup for each running database. # Variables are set at the top of the DB2_Backup.sh script to determine if a full backup needs to be run @@ -12,7 +12,7 @@ # # Variables to be set # SLACKURL = The channel were notifications are send -# BACKUP_SCRIPT = The backup script that Run_Backup.sh calls +# BACKUP_SCRIPT = The backup script that RUN_OnDemandFULL_BKP.sh calls # DAYOFFULL = Defines the day of the week that the full backup will on on (must match the same format as the output from `date`) # NUMOFBKUPTOKEEP = This defines the number of days to keep a backup image on local disk # @@ -26,78 +26,107 @@ # ./DB2_Backup.sh ${DB2INSTANCE} ${DBNAME} ${NUMOFBKUPTOKEEP} ${BACKUPTYPE} 2>>.BackupLOG.stderr > .BackupLOG.out ######################################################### +# -- Source the PROPS File . /mnt/backup/bin/.PROPS +# -- Standard Parameters DBINSTANCE=`whoami` HOSTNAME=`hostname` -BACKUP_DIR=${HOME}/bin -BACKUP_SCRIPT=DB2_Backup.sh +INSTANCE_HOME=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${DBINSTANCE}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` +SCRIPT_DIR=${INSTANCE_HOME}/bin DATETIME=`date +%Y-%m-%d_%H%M%S`; -if [ ! -f "${HOME}/sqllib/db2profile" ] -then - echo "ERROR - ${HOME}/sqllib/db2profile not found" +# -- Verify and source db2profile + +if [[ ! -f "${INSTANCE_HOME}/sqllib/db2profile" ]]; then + echo "ERROR - ${INSTANCE_HOME}/sqllib/db2profile not found" EXIT_STATUS=1 else - . ${HOME}/sqllib/db2profile + . ${INSTANCE_HOME}/sqllib/db2profile +fi + +# -- Backup parameters +BACKUPTYPE=full +BACKUP_SCRIPT="${SCRIPT_DIR}/DB2_Backup.sh" +CUSTNAME=`hostname | sed 's/c-db2wh-//; s/c-//; s/-db2u-0//; s/db2u/-/; s/-manage//;' | tr '[:lower:]' '[:upper:]'` +BUCKET_ALIAS=`db2 list storage access | grep ${CONTAINER} -B4 | grep ALIAS | awk -F '=' '{print $2}'` +HSTYPE="Backup" +ICD_LOG=${SCRIPT_DIR}/.Maillive.log + + +# -- Valid only for MAS-CP4D customers +if (( ${CUSTNAME} )) ; then + CUSTNAME=`echo ${CONTAINER} | awk -F '-backup-' '{print $2}' | awk -F '-pr-' '{print $1}' | tr '[:lower:]' '[:upper:]'` +fi + +# -- Database Environment +if [[ ${BUCKET_ALIAS} == "IBMCOS" ]]; then + DBENV="MAS MS" +else + DBENV="MAS SaaS" fi -DOW=`date | awk '{print $1}'` +# -- Create ICD Incident , If Backup fails + +CREATE_ICD() { + HTYPE=`echo ${HSTYPE} | tr '[:lower:]' '[:upper:]'` + DES="$1" + echo "############################" >> ${ICD_LOG} + LONGDES=`cat ${ICD_LOG} | sed 's/"//g' | sed "s/'//g"` + LONGDES=`echo "
 ${LONGDES} 
"` -# if [ ${DOW} = ${DAYOFFULL} ] ; then - BACKUPTYPE=full -# else -# BACKUPTYPE=inc -# fi + # -- Verify the ICD URL Status + if curl -k -s --connect-timeout 3 ${ICD_URL_SAAS} >/dev/null; then + CURL_REQ="--request POST --url ${ICD_URL_SAAS} " + AUTH_REQ="apikey: ${ICD_API_KEY}" + fi + + # -- Generate Curl Syntax to push to ICD + cat << ! >.curl_${DBNAME}_ICD.sh + curl ${CURL_REQ} \ + --header '${AUTH_REQ}' \ + --header 'Content-Type: application/json' \ + --data '{ + "description":"${DES}", + "reportedpriority":4, + "internalpriority":4, + "reportedby":"DB2", + "affectedperson":"${DBENV}", + "ownergroup":"HSDBA", + "description_longdescription":"${LONGDES}", + "siteid":"001", + "classstructureid":"1341", + "classificationid":"IN-DBPERF", + "hshost":"${HOSTNAME}", + "hstype":"${HTYPE}" + }' +! + /bin/bash .curl_${DBNAME}_ICD.sh > .curl_${DBNAME}_ICD.out 2>&1 + +} + +# -- Loop through the available databases in the instance DBS=`db2 list db directory | grep -B5 "Indirect" | grep "Database name" | awk '{ print $4 }'` for DBNAME in ${DBS} do - cd ${BACKUP_DIR} - ./DB2_Backup.sh ${DB2INSTANCE} ${DBNAME} ${NUMOFBKUPTOKEEP} ${BACKUPTYPE} 2>.BackupLOG.stderr > .BackupLOG.out + cd ${SCRIPT_DIR} + ${BACKUP_SCRIPT} ${DB2INSTANCE} ${DBNAME} ${NUMOFBKUPTOKEEP} ${BACKUPTYPE} 2>.BackupLOG.stderr > .BackupLOG.out + RC=$? + if [[ ${RC} -ne 0 ]]; then - RC=$? - if [ ${RC} -ne 0 ]; then - - longdes="Failure to start the Backup job ${DATETIME} CUST=${CUSTNAME} ${RC}" - ## Send Failure notification to a slack channel ## - cat << ! >.curl_${DBNAME}_RUN.sh - curl -X POST -H 'Content-type: application/json' --data '{"text":"$longdes"}' $SLACKURL -! -/bin/bash .curl_${DBNAME}_RUN.sh > .curl_${DBNAME}_RUN.out 2>&1 - - ##### Create ICD Incident #### - ####### If Backup fails ### - des="${DBINSTANCE} - Backup - ${HOSTNAME} ${DBNAME} ${CUSTNAME} - MASMS -- Backup Failed" - echo "TESTING $instance - Backup - $ ${DBNAME} - Backup Failed" > .Maillive.log - echo "############################" >> .Maillive.log - #cat $BACK_LOG >> .Maillive.log - longdes=`cat .Maillive.log | sed 's/"//g' | sed "s/'//g"` - ICD_URL="https://servicedesk.mro.com" - if ! curl -k -s --connect-timeout 3 ${ICD_URL} >/dev/null; then - ICD_URL="https://servicedesk.cds.mro.com" - fi - -cat << ! >.curl_${DBNAME}_ICD.sh - curl --insecure --location --request POST "${ICD_URL}/maximo_mif/oslc/os/hsincident?lean=1" \ - --header "Authorization: Basic ${ICD_AUTH_KEY}" \ - --header 'Content-Type: application/json' \ - --data '{ - "description":"$des", - "reportedpriority":4, - "internalpriority":4, - "reportedby":"DB2", - "affectedperson":"CTGINST1", - "description_longdescription":"$longdes", - "siteid":"001", - "classstructureid":"1341", - "classificationid":"IN-DBPERF", - "hshost":"{servicedesk-pdb-sjc03-2.cds.mro.com:0:50}", - "hstype":"BACKUP" - }' + LONGDES="Failure to start the Backup job ${DATETIME} CUST=${CUSTNAME} - ${RC}" + # -- Send Failure notification to a slack channel + cat << ! >.curl_${DBNAME}_RUN.sh + curl -X POST -H 'Content-type: application/json' --data '{"text":"$LONGDES"}' ${SLACKURL} ! -#####/bin/bash .curl_${DBNAME}_ICD.sh > .curl_${DBNAME}_ICD.out 2>&1 + /bin/bash .curl_${DBNAME}_RUN.sh > .curl_${DBNAME}_RUN.out 2>&1 -fi + # -- Create ICD ticket if fails + DES="${CUSTNAME} - ${DBENV} - ${DBNAME} - ${HOSTNAME} -- Failed to Start Backup!! " + CREATE_ICD "${DES}" + fi done + +# -- END OF SCRIPT \ No newline at end of file diff --git a/instance-applications/120-ibm-db2u-database/files/Run_Backup.sh b/instance-applications/120-ibm-db2u-database/files/Run_Backup.sh index 72f29f0c8..e6d8e660a 100755 --- a/instance-applications/120-ibm-db2u-database/files/Run_Backup.sh +++ b/instance-applications/120-ibm-db2u-database/files/Run_Backup.sh @@ -1,8 +1,6 @@ #!/bin/bash -#set -x - ######################################################### -# Run_Backup.sh +# Run_Backup.sh # Run_Backup.sh will be called from the Cron Jobs # This script will list all local databases running in the instance on a node. It will call the # DB2_Backup.sh script to run a backup for each running database. @@ -17,93 +15,136 @@ # NUMOFBKUPTOKEEP = This defines the number of days to keep a backup image on local disk # # Variables determined by the environment -# BACKUPTYPE = Is determined from the `date` command and the DAYOFFULL value +# BKPTYPE = Is determined from the `date` command and the DAYOFFULL value # DB2INSTANCE = Pulled from the environment # HOSTNAME # DBNAME = Pulled from the `db2 list db directory` # # Backup command issued -# ./DB2_Backup.sh ${DB2INSTANCE} ${DBNAME} ${NUMOFBKUPTOKEEP} ${BACKUPTYPE} 2>>.BackupLOG.stderr > .BackupLOG.out +# ./DB2_Backup.sh ${DB2INSTANCE} ${DBNAME} ${NUMOFBKUPTOKEEP} ${BKPTYPE} 2>>.BackupLOG.stderr > .BackupLOG.out +# +# -- Revision of script to include new ICD URL ######################################################### +# -- Source the Props File . /mnt/backup/bin/.PROPS +# -- Standard Parameters DBINSTANCE=`whoami` HOSTNAME=`hostname` -BACKUP_DIR=${HOME}/bin -BACKUP_SCRIPT=DB2_Backup.sh DATETIME=`date +%Y-%m-%d_%H%M%S`; +DOW=`date | awk '{print $1}'` + +# -- Verify and source db2profile -if [ ! -f "${HOME}/sqllib/db2profile" ] -then +if [[ ! -f "${HOME}/sqllib/db2profile" ]]; then echo "ERROR - ${HOME}/sqllib/db2profile not found" EXIT_STATUS=1 else . ${HOME}/sqllib/db2profile fi +# -- Debug Mode +set -x -DOW=`date | awk '{print $1}'` +# -- Backup Parameters +INSTANCE_HOME=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${DBINSTANCE}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` +SCRIPT_DIR=${INSTANCE_HOME}/bin +BACKUP_SCRIPT="${SCRIPT_DIR}/DB2_Backup.sh" +CUSTNAME=`hostname | sed 's/c-db2wh-//; s/c-//; s/-db2u-0//; s/db2u/-/; s/-manage//;' | tr '[:lower:]' '[:upper:]'` +BUCKET_ALIAS=`db2 list storage access | grep ${CONTAINER} -B4 | grep ALIAS | awk -F '=' '{print $2}'` +HSTYPE="Backup" +ICD_LOG=${SCRIPT_DIR}/.Maillive.log - if [ ${DOW} = ${DAYOFFULL} ] ; then - BACKUPTYPE=full - else - BACKUPTYPE=inc - fi +# -- Valid only for MAS-CP4D customers +if (( ${CUSTNAME} )) ; then + CUSTNAME=`echo ${CONTAINER} | awk -F '-backup-' '{print $2}' | awk -F '-pr-' '{print $1}' | tr '[:lower:]' '[:upper:]'` +fi + +# -- Database Environment +if [[ ${BUCKET_ALIAS} == "IBMCOS" ]]; then + DBENV="MAS MS" +else + DBENV="MAS SaaS" +fi + +# -- Create ICD Incident , If Backup fails + +CREATE_ICD() { + HTYPE=`echo ${HSTYPE} | tr '[:lower:]' '[:upper:]'` + DES="$1" + echo "############################" >> ${ICD_LOG} + LONGDES=`cat ${ICD_LOG} | sed 's/"//g' | sed "s/'//g"` + LONGDES=`echo "
 ${LONGDES} 
"` + + # -- Verify the ICD URL Status + if curl -k -s --connect-timeout 3 ${ICD_URL_SAAS} >/dev/null; then + CURL_REQ="--request POST --url ${ICD_URL_SAAS} " + AUTH_REQ="apikey: ${ICD_API_KEY}" + fi + + + # -- Generate Curl Syntax to push to ICD + cat << ! >.curl_${DBNAME}_ICD.sh + curl ${CURL_REQ} \ + --header '${AUTH_REQ}' \ + --header 'Content-Type: application/json' \ + --data '{ + "description":"${DES}", + "reportedpriority":4, + "internalpriority":4, + "reportedby":"DB2", + "affectedperson":"${DBENV}", + "ownergroup":"HSDBA", + "description_longdescription":"${LONGDES}", + "siteid":"001", + "classstructureid":"1341", + "classificationid":"IN-DBPERF", + "hshost":"${HOSTNAME}", + "hstype":"${HTYPE}" + }' +! + /bin/bash .curl_${DBNAME}_ICD.sh > .curl_${DBNAME}_ICD.out 2>&1 + +} + +# -- Verify the day of the week +if [[ ${DOW} = ${DAYOFFULL} ]] ; then + BKPTYPE="FULL" +else + BKPTYPE="DIFF" +fi + +# -- Loop through the available databases in the instance DBS=`db2 list db directory | grep -B5 "Indirect" | grep "Database name" | awk '{ print $4 }'` for DBNAME in ${DBS} do - cd ${BACKUP_DIR} - ./DB2_Backup.sh ${DB2INSTANCE} ${DBNAME} ${NUMOFBKUPTOKEEP} ${BACKUPTYPE} 2>.BackupLOG.stderr > .BackupLOG.out - - RC=$? - if [ ${RC} -ne 0 ]; then - - longdes="Failure to start the Backup job ${DATETIME} CUST=${CUSTNAME} ${RC}" - ## Send Failure notification to a slack channel ## - cat << ! >.curl_${DBNAME}_RUN.sh - if [[ -n "${SLACKURL}" ]]; then - curl -X POST -H 'Content-type: application/json' --data '{"text":"$longdes"}' $SLACKURL - fi + cd ${SCRIPT_DIR} + ${BACKUP_SCRIPT} ${DB2INSTANCE} ${DBNAME} ${NUMOFBKUPTOKEEP} ${BKPTYPE} 2>.BackupLOG.stderr > .BackupLOG.out + RC=$? + if [[ ${RC} -ne 0 ]]; then + LONGDES="Failure to start the Backup job ${DATETIME} CUST=${CUSTNAME} ${RC}" + # -- Send Failure notification to a slack channel + cat << ! >.curl_${DBNAME}_RUN.sh + curl -X POST -H 'Content-type: application/json' --data '{"text":"$LONGDES"}' ${SLACKURL} ! -/bin/bash .curl_${DBNAME}_RUN.sh > .curl_${DBNAME}_RUN.out 2>&1 - - ##### Create ICD Incident #### - ####### If Backup fails ### - des="${DBINSTANCE} - Backup - ${HOSTNAME} ${DBNAME} ${CUSTNAME} - MASMS -- Backup Failed" - echo "TESTING $instance - Backup - $ ${DBNAME} - Backup Failed" > .Maillive.log - echo "############################" >> .Maillive.log - #cat $BACK_LOG >> .Maillive.log - longdes=`cat .Maillive.log | sed 's/"//g' | sed "s/'//g"` - ICD_URL="https://servicedesk.mro.com" - if ! curl -k -s --connect-timeout 3 ${ICD_URL} >/dev/null; then - ICD_URL="https://servicedesk.cds.mro.com" - fi + /bin/bash .curl_${DBNAME}_RUN.sh > .curl_${DBNAME}_RUN.out 2>&1 -cat << ! >.curl_${DBNAME}_ICD.sh - curl --insecure --location --request POST "${ICD_URL}/maximo_mif/oslc/os/hsincident?lean=1" \ - --header "Authorization: Basic ${ICD_AUTH_KEY}" \ - --header 'Content-Type: application/json' \ - --data '{ - "description":"$des", - "reportedpriority":4, - "internalpriority":4, - "reportedby":"DB2", - "affectedperson":"CTGINST1", - "description_longdescription":"$longdes", - "siteid":"001", - "classstructureid":"1341", - "classificationid":"IN-DBPERF", - "hshost":"{servicedesk-pdb-sjc03-2.cds.mro.com:0:50}", - "hstype":"BACKUP" - }' -! -if [[ -n "${ICD_AUTH_KEY}" ]]; then - /bin/bash .curl_${DBNAME}_ICD.sh > .curl_${DBNAME}_ICD.out 2>&1 -fi + # -- Create ICD ticket if fails + DES="${CUSTNAME} - ${DBENV} - ${DBNAME} - ${HOSTNAME} -- Failed to Start Backup!! " + CREATE_ICD "${DES}" + fi + + # -- Execute Online Reorgs for qualified tables and indexes after every Full Backup + if [[ ${DOW} = ${DAYOFFULL} ]] ; then + /bin/bash ${SCRIPT_DIR}/reorgTablesIndexesInplace.sh -db ${DBNAME} -s MAXIMO -tb_stats -ix_stats -window 120 -tr > ${HOME}/maintenance/logs/reorgTablesIndexesInplace_${DATETIME}.log 2>&1 + fi -fi done -/bin/bash ${HOME}/bin/runstats_rebind.sh >${HOME}/bin/.runstats_rebind.out 2>&1 -/bin/bash ${HOME}/bin/grant_check.sh bludb >${HOME}/bin/.grant_check.out 2>&1 + +# -- Exeucte Runstats and Rebind for all tables daily +/bin/bash ${SCRIPT_DIR}/runstats_rebind.sh >${SCRIPT_DIR}/.runstats_rebind.out 2>&1 +#/bin/bash ${SCRIPT_DIR}/grant_check.sh bludb >${SCRIPT_DIR}/.grant_check.out 2>&1 + +# -- END OF SCRIPT \ No newline at end of file diff --git a/instance-applications/120-ibm-db2u-database/files/reorgTablesIndexesInplace2.sh b/instance-applications/120-ibm-db2u-database/files/reorgTablesIndexesInplace.sh similarity index 100% rename from instance-applications/120-ibm-db2u-database/files/reorgTablesIndexesInplace2.sh rename to instance-applications/120-ibm-db2u-database/files/reorgTablesIndexesInplace.sh diff --git a/instance-applications/120-ibm-db2u-database/files/runstats_rebind.sh b/instance-applications/120-ibm-db2u-database/files/runstats_rebind.sh index 7415d6b4a..a45e7e9f0 100755 --- a/instance-applications/120-ibm-db2u-database/files/runstats_rebind.sh +++ b/instance-applications/120-ibm-db2u-database/files/runstats_rebind.sh @@ -1,41 +1,43 @@ #!/bin/bash # *************************************************************************** -# Author: Fu Le Qing (Roking) -# Email: leqingfu@cn.ibm.com -# Date: 10-31-2018 +# Author: Fu Le Qing (Roking) +# Email: leqingfu@cn.ibm.com +# Date: 10-31-2018 # -# Description: This script updates statistics of tables, -# associated indexes in the database, and sends an email -# to a specified email list. +# Description: This script updates statistics of tables, +# associated indexes in the database, and sends an email +# to a specified email list. # # ******** THIS NEEDS TO BE RUN AS INSTANCE OWNER. ************** # -# Revision history: -# 10-31-2018 Fu Le Qing (Roking) -# Original version -# 11-16-2018 Fu Le Qing (Roking) -# Skip the tables which are ongoing with reorg -# 09-08-2023 Fu Le Qing (Roking) -# Update for MAS +# Revision history: +# 10-31-2018 Fu Le Qing (Roking) +# Original version +# 11-16-2018 Fu Le Qing (Roking) +# Skip the tables which are ongoing with reorg +# 09-08-2023 Fu Le Qing (Roking) +# Update for MAS # # *************************************************************************** # # *************************************************************************** + +# -- Source the Props file if [ -f /mnt/backup/bin/.PROPS ] then . /mnt/backup/bin/.PROPS - DOW=`date | awk '{print $1}'` - if [ ${DOW} != ${DAYOFFULL} ] - then - exit 0 - fi + DOW=`date | awk '{print $1}'` fi -instance=`whoami` -instance_home=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${instance}" | awk -F ',' '{print $5}'| sed 's/\/sqllib//'` +# -- Standard Parameters +INSTANCE=`whoami` +INSTANCE_HOME=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${INSTANCE}" | awk -F ',' '{print $5}'| sed 's/\/sqllib//'` +DOW=`date | awk '{print $1}'` +mkdir -p ${INSTANCE_HOME}/maintenance/logs +DATESTAMP=`date "+%Y-%m-%d-%H.%M.%S"` -pidfile="$instance_home/.`basename ${0}`.pid" -if [ -e ${pidfile} ] && $kill -0 `cat ${pidfile}` 2>/dev/null +pidfile="${INSTANCE_HOME}/.`basename ${0}`.pid" +if [ -e ${pidfile} ] && $kill -0 `cat ${pidfile}` 2>/dev/null then exit 0 fi @@ -43,48 +45,51 @@ fi echo $$ > ${pidfile} trap "rm -f ${pidfile}; exit" SIGHUP SIGINT SIGQUIT SIGABRT SIGTERM EXIT -if [ ! -f "$instance_home/sqllib/db2profile" ] +if [ ! -f "${INSTANCE_HOME}/sqllib/db2profile" ] then - echo "ERROR - $instance_home/sqllib/db2profile not found" + echo "ERROR - ${INSTANCE_HOME}/sqllib/db2profile not found" exit 1 else - . $instance_home/sqllib/db2profile + . ${INSTANCE_HOME}/sqllib/db2profile fi -RUNSTATS_TMP_FILE="$instance_home/.runstats.sql" -REBIND_TMP_FILE="$instance_home/.rebind.sql" - -mkdir -p $instance_home/maintenance/logs -DATESTAMP=`date "+%Y-%m-%d-%H.%M.%S"` +RUNSTATS_TMP_FILE="${INSTANCE_HOME}/bin/.runstats.sql" +REBIND_TMP_FILE="${INSTANCE_HOME}/bin/.rebind.sql" -for db in `db2 list db directory | grep -B 5 Indirect | grep "Database name" | cut -d= -f2` +for DB in `db2 list db directory | grep -B 5 Indirect | grep "Database name" | cut -d= -f2` do - role=`db2 get db cfg for ${db} | grep "HADR database role" | cut -d= -f2 |sed 's/ //g'` + RUNSTATS_REBIND_LOG="${INSTANCE_HOME}/maintenance/logs/runstats_rebind_${DB}_${DATESTAMP}.log" + + role=`db2 get db cfg for ${DB} | grep "HADR database role" | cut -d= -f2 |sed 's/ //g'` if [ "$role" != "STANDBY" ]; then - if [ -f $RUNSTATS_TMP_FILE ] - then - rm $RUNSTATS_TMP_FILE + if [[ -f ${RUNSTATS_TMP_FILE} ]]; then + rm ${RUNSTATS_TMP_FILE} fi - if [ -f $REBIND_TMP_FILE ] - then - rm $REBIND_TMP_FILE + if [[ -f ${REBIND_TMP_FILE} ]]; then + rm ${REBIND_TMP_FILE} fi - db2 connect to ${db} | tee $instance_home/maintenance/logs/runstats_${db}_${DATESTAMP} - if [ $? -eq 0 ]; then - db2 -x "select 'RUNSTATS ON TABLE \"' ||rtrim(tab.tabschema)||'\".\"'|| tab.tabname ||'\" WITH DISTRIBUTION ON KEY COLUMNS AND DETAILED INDEXES ALL ALLOW WRITE ACCESS;' - from syscat.tables tab left join sysibmadm.SNAPTAB_REORG reg on tab.tabschema=reg.TABSCHEMA and tab.tabname=reg.TABNAME and reg.REORG_STATUS not in ('COMPLETED','STOPPED') - where tab.type='T' and reg.tabname is null" > $RUNSTATS_TMP_FILE - #db2 -x "select 'rebind package \"' ||rtrim(PKGSCHEMA)||'\".\"'|| PKGNAME ||'\";' from syscat.packages where PKGSCHEMA not in ('NULLID','NULLIDR1','NULLIDRA','SYSIBMADM','SYSIBMINTERNAL') and PKGSCHEMA not like 'NULL%' " > $REBIND_TMP_FILE - echo "Begin processing of runstats @ $DATESTAMP ..." | tee -a $instance_home/maintenance/logs/runstats_${db}_${DATESTAMP} - db2 -txvf $RUNSTATS_TMP_FILE | tee -a $instance_home/maintenance/logs/runstats_${db}_${DATESTAMP} - echo "End processing of runstats @ $DATESTAMP" | tee -a $instance_home/maintenance/logs/runstats_${db}_${DATESTAMP} - #echo "Begin processing of rebind @ $DATESTAMP ..." | tee -a $instance_home/maintenance/logs/runstats_${db}_${DATESTAMP} - #db2 -txvf $REBIND_TMP_FILE | tee -a $instance_home/maintenance/logs/runstats_${db}_${DATESTAMP} - #echo "End processing of rebind @ $DATESTAMP" | tee -a $instance_home/maintenance/logs/runstats_${db}_${DATESTAMP} - rm $RUNSTATS_TMP_FILE - #rm $REBIND_TMP_FILE + db2 connect to ${DB} | tee ${RUNSTATS_REBIND_LOG} + if [[ $? -eq 0 ]]; then + db2 -x "select 'RUNSTATS ON TABLE \"' ||rtrim(tab.tabschema)||'\".\"'|| tab.tabname ||'\" WITH DISTRIBUTION ON KEY COLUMNS AND DETAILED INDEXES ALL ALLOW WRITE ACCESS;' from syscat.tables tab left join sysibmadm.SNAPTAB_REORG reg on tab.tabschema=reg.TABSCHEMA and tab.tabname=reg.TABNAME and reg.REORG_STATUS not in ('COMPLETED','STOPPED') where tab.type='T' and reg.tabname is null" > ${RUNSTATS_TMP_FILE} + + echo -e "Begin processing of runstats @ ${DATESTAMP} ...\n" | tee -a ${RUNSTATS_REBIND_LOG} + db2 -txvf ${RUNSTATS_TMP_FILE} | tee -a ${RUNSTATS_REBIND_LOG} + echo -e "\nEnd processing of runstats @ ${DATESTAMP}" | tee -a ${RUNSTATS_REBIND_LOG} + rm ${RUNSTATS_TMP_FILE} + + if [[ ${DOW} == ${DAYOFFULL} ]]; then + + db2 -x "select 'rebind package \"' ||rtrim(PKGSCHEMA)||'\".\"'|| PKGNAME ||'\";' from syscat.packages where PKGSCHEMA not like 'SYSIBM%' and PKGSCHEMA not like 'NULL%' " > ${REBIND_TMP_FILE} + + echo -e "\n ----------------------------------------------- " | tee -a ${RUNSTATS_REBIND_LOG} + echo -e "Begin processing of rebind @ ${DATESTAMP} ...\n" | tee -a ${RUNSTATS_REBIND_LOG} + db2 -txvf ${REBIND_TMP_FILE} | tee -a ${RUNSTATS_REBIND_LOG} + echo -e "\nEnd processing of rebind @ ${DATESTAMP}" | tee -a ${RUNSTATS_REBIND_LOG} + + rm ${REBIND_TMP_FILE} + fi db2 terminate - fi + fi fi done From 2397eae8fb12bab691f803cab8f98456d2ee248f Mon Sep 17 00:00:00 2001 From: prudhvirajp Date: Tue, 17 Mar 2026 17:30:02 +0530 Subject: [PATCH 2/3] Rectified the logging errros on the script and Remodified the runstats_rebind script with the actual script --- .../120-ibm-db2u-database/files/DB2_Backup.sh | 29 ++++++++++--------- .../files/RUN_OnDemandFULL_BKP.sh | 20 ++++++------- .../120-ibm-db2u-database/files/Run_Backup.sh | 18 ++++++------ .../files/runstats_rebind.sh | 29 +++++++++++-------- 4 files changed, 51 insertions(+), 45 deletions(-) diff --git a/instance-applications/120-ibm-db2u-database/files/DB2_Backup.sh b/instance-applications/120-ibm-db2u-database/files/DB2_Backup.sh index d6f4c2413..b5112aeb9 100755 --- a/instance-applications/120-ibm-db2u-database/files/DB2_Backup.sh +++ b/instance-applications/120-ibm-db2u-database/files/DB2_Backup.sh @@ -1,6 +1,4 @@ #!/bin/ksh -#set -x - ######################################################### # DB2_Backup.sh # @@ -48,7 +46,10 @@ else . ${INSTANCE_HOME}/sqllib/db2profile fi -set -x +# -- Debug Mode +# set -x; # Uncomment to debug this shell script +# set -n; # Uncomment to check your syntax, without execution. + # -- Source the PROPS file . /mnt/backup/bin/.PROPS @@ -73,9 +74,9 @@ fi # -- Database Environment if [[ ${BUCKET_ALIAS} == "IBMCOS" ]]; then - DBENV="MASMS" + DBENV="MAS MS" else - DBENV="MAS_SaaS" + DBENV="MAS SaaS" fi # -- Function to send Slack notification @@ -96,7 +97,7 @@ SLACK_NOTIFY() { CREATE_ICD() { HTYPE=`echo ${HSTYPE} | tr '[:lower:]' '[:upper:]'` DES="$1" - echo "############################" >> ${ICD_LOG} +# echo "############################" >> ${ICD_LOG} LONGDES=`cat ${ICD_LOG} | sed 's/"//g' | sed "s/'//g"` LONGDES=`echo "
 ${LONGDES} 
"` @@ -166,13 +167,13 @@ fi # -- Check to see if the Instance is up and Running ps -ef | grep db2sysc | grep -v grep > /dev/null 2>&1 if [[ $? -eq 1 ]]; then - echo "Instance is not active " - echo "${HOSTNAME}, Instance is not Active, BACKUP cannot Run!" | tee ${INSTANCE_HOME}/bin/LASTbkupRUN ${ICD_LOG} >/dev/null - echo "############################" >> ${ICD_LOG} + #echo "Instance is not active " + echo "Instance is not Active, Backup cannot Initiate !!!" | tee ${INSTANCE_HOME}/bin/LASTbkupRUN ${ICD_LOG} >/dev/null +# echo "############################" >> ${ICD_LOG} cat ${BACK_LOG} >> ${ICD_LOG} - SLACKDES="${CUSTNAME} - ${DBENV} - ${HOSTNAME}, Instance is not Active, Backup cannot Run! " - DES="${CUSTNAME} - ${DBENV} - ${DBNAME} - ${HOSTNAME} -- Instance is not Active, Backup cannot Run!! " + SLACKDES="${CUSTNAME} - ${DBENV} - ${HOSTNAME}, Instance is not Active, Backup cannot Initiate !! " + DES="${CUSTNAME} - ${DBENV} - ${DBNAME} - ${HOSTNAME} -- Instance is not Active, Backup cannot Initiate !! " # -- Send error notification to Slack SLACK_NOTIFY "${SLACKDES}" @@ -208,11 +209,11 @@ if [[ ${NUM_BACKUPS_TO_KEEP} -gt 0 ]]; then grep -Fq "Backup successful." ${BACK_LOG} if [[ $? -ne 0 ]]; then - echo "${CUSTNAME} - ${DBENV} - ${HOSTNAME}, ${BKPTYPE} DB Backup Failed ! Database Backup issues !!!" > ${ICD_LOG} - echo "############################" >> ${ICD_LOG} + echo "${CUSTNAME} - ${DBENV} - ${HOSTNAME} - ${BKPTYPE} ${HSTYPE} Failed !!!" > ${ICD_LOG} + # echo "############################" >> ${ICD_LOG} cat ${BACK_LOG} >> ${ICD_LOG} - SLACKDES="${CUSTNAME} - ${DBENV} - ${HOSTNAME}, ${BKPTYPE} DB Backup Failed . . . Please investigate ! ! ! " + SLACKDES="${CUSTNAME} - ${DBENV} - ${HOSTNAME} -- ${BKPTYPE} ${HSTYPE} Failed . . . Please investigate ! ! ! " DES="${CUSTNAME} - ${DBENV} - ${DBNAME} - ${HOSTNAME} -- ${BKPTYPE} ${HSTYPE} Failed !" # -- Send error notification to Slack diff --git a/instance-applications/120-ibm-db2u-database/files/RUN_OnDemandFULL_BKP.sh b/instance-applications/120-ibm-db2u-database/files/RUN_OnDemandFULL_BKP.sh index 97a5903fd..7124e9f80 100755 --- a/instance-applications/120-ibm-db2u-database/files/RUN_OnDemandFULL_BKP.sh +++ b/instance-applications/120-ibm-db2u-database/files/RUN_OnDemandFULL_BKP.sh @@ -1,28 +1,26 @@ #!/bin/bash -#set -x - ######################################################### -# RUN_OnDemandFULL_BKP.sh -# RUN_OnDemandFULL_BKP.sh will be called from the Cron Jobs +# RUN_OnDemandFULL_BKP.sh +# RUN_OnDemandFULL_BKP.sh will be called by OnDemand Jobs # This script will list all local databases running in the instance on a node. It will call the # DB2_Backup.sh script to run a backup for each running database. # Variables are set at the top of the DB2_Backup.sh script to determine if a full backup needs to be run # based on the day of the week. Currently, Saturday is when the full backup runs, incremental backups run # every all other days. # -# Variables to be set +# Variables to be set # SLACKURL = The channel were notifications are send # BACKUP_SCRIPT = The backup script that RUN_OnDemandFULL_BKP.sh calls # DAYOFFULL = Defines the day of the week that the full backup will on on (must match the same format as the output from `date`) # NUMOFBKUPTOKEEP = This defines the number of days to keep a backup image on local disk # -# Variables determined by the environment +# Variables determined by the environment # BACKUPTYPE = Is determined from the `date` command and the DAYOFFULL value # DB2INSTANCE = Pulled from the environment # HOSTNAME # DBNAME = Pulled from the `db2 list db directory` # -# Backup command issued +# Backup command issued # ./DB2_Backup.sh ${DB2INSTANCE} ${DBNAME} ${NUMOFBKUPTOKEEP} ${BACKUPTYPE} 2>>.BackupLOG.stderr > .BackupLOG.out ######################################################### @@ -45,6 +43,10 @@ else . ${INSTANCE_HOME}/sqllib/db2profile fi +# -- Debug Mode +# set -x; # Uncomment to debug this shell script +# set -n; # Uncomment to check your syntax, without execution. + # -- Backup parameters BACKUPTYPE=full BACKUP_SCRIPT="${SCRIPT_DIR}/DB2_Backup.sh" @@ -53,7 +55,6 @@ BUCKET_ALIAS=`db2 list storage access | grep ${CONTAINER} -B4 | grep ALIAS | awk HSTYPE="Backup" ICD_LOG=${SCRIPT_DIR}/.Maillive.log - # -- Valid only for MAS-CP4D customers if (( ${CUSTNAME} )) ; then CUSTNAME=`echo ${CONTAINER} | awk -F '-backup-' '{print $2}' | awk -F '-pr-' '{print $1}' | tr '[:lower:]' '[:upper:]'` @@ -66,13 +67,12 @@ else DBENV="MAS SaaS" fi - # -- Create ICD Incident , If Backup fails CREATE_ICD() { HTYPE=`echo ${HSTYPE} | tr '[:lower:]' '[:upper:]'` DES="$1" - echo "############################" >> ${ICD_LOG} +# echo "############################" >> ${ICD_LOG} LONGDES=`cat ${ICD_LOG} | sed 's/"//g' | sed "s/'//g"` LONGDES=`echo "
 ${LONGDES} 
"` diff --git a/instance-applications/120-ibm-db2u-database/files/Run_Backup.sh b/instance-applications/120-ibm-db2u-database/files/Run_Backup.sh index e6d8e660a..21bfe794d 100755 --- a/instance-applications/120-ibm-db2u-database/files/Run_Backup.sh +++ b/instance-applications/120-ibm-db2u-database/files/Run_Backup.sh @@ -1,6 +1,6 @@ #!/bin/bash ######################################################### -# Run_Backup.sh +# Run_Backup.sh # Run_Backup.sh will be called from the Cron Jobs # This script will list all local databases running in the instance on a node. It will call the # DB2_Backup.sh script to run a backup for each running database. @@ -8,19 +8,19 @@ # based on the day of the week. Currently, Saturday is when the full backup runs, incremental backups run # every all other days. # -# Variables to be set +# Variables to be set # SLACKURL = The channel were notifications are send # BACKUP_SCRIPT = The backup script that Run_Backup.sh calls # DAYOFFULL = Defines the day of the week that the full backup will on on (must match the same format as the output from `date`) # NUMOFBKUPTOKEEP = This defines the number of days to keep a backup image on local disk # -# Variables determined by the environment +# Variables determined by the environment # BKPTYPE = Is determined from the `date` command and the DAYOFFULL value # DB2INSTANCE = Pulled from the environment # HOSTNAME # DBNAME = Pulled from the `db2 list db directory` # -# Backup command issued +# Backup command issued # ./DB2_Backup.sh ${DB2INSTANCE} ${DBNAME} ${NUMOFBKUPTOKEEP} ${BKPTYPE} 2>>.BackupLOG.stderr > .BackupLOG.out # # -- Revision of script to include new ICD URL @@ -45,7 +45,8 @@ else fi # -- Debug Mode -set -x +# set -x; # Uncomment to debug this shell script +# set -n; # Uncomment to check your syntax, without execution. # -- Backup Parameters INSTANCE_HOME=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${DBINSTANCE}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` @@ -73,7 +74,7 @@ fi CREATE_ICD() { HTYPE=`echo ${HSTYPE} | tr '[:lower:]' '[:upper:]'` DES="$1" - echo "############################" >> ${ICD_LOG} +# echo "############################" >> ${ICD_LOG} LONGDES=`cat ${ICD_LOG} | sed 's/"//g' | sed "s/'//g"` LONGDES=`echo "
 ${LONGDES} 
"` @@ -83,7 +84,6 @@ CREATE_ICD() { AUTH_REQ="apikey: ${ICD_API_KEY}" fi - # -- Generate Curl Syntax to push to ICD cat << ! >.curl_${DBNAME}_ICD.sh curl ${CURL_REQ} \ @@ -132,7 +132,7 @@ do /bin/bash .curl_${DBNAME}_RUN.sh > .curl_${DBNAME}_RUN.out 2>&1 # -- Create ICD ticket if fails - DES="${CUSTNAME} - ${DBENV} - ${DBNAME} - ${HOSTNAME} -- Failed to Start Backup!! " + DES="${CUSTNAME} - ${DBENV} - ${DBNAME} - ${HOSTNAME} -- Failed to Initiate Backup!! " CREATE_ICD "${DES}" fi @@ -143,7 +143,7 @@ do done -# -- Exeucte Runstats and Rebind for all tables daily +# -- Exeucte Runstats and Rebind for all tables on weekly after full backup /bin/bash ${SCRIPT_DIR}/runstats_rebind.sh >${SCRIPT_DIR}/.runstats_rebind.out 2>&1 #/bin/bash ${SCRIPT_DIR}/grant_check.sh bludb >${SCRIPT_DIR}/.grant_check.out 2>&1 diff --git a/instance-applications/120-ibm-db2u-database/files/runstats_rebind.sh b/instance-applications/120-ibm-db2u-database/files/runstats_rebind.sh index a45e7e9f0..a2159405d 100755 --- a/instance-applications/120-ibm-db2u-database/files/runstats_rebind.sh +++ b/instance-applications/120-ibm-db2u-database/files/runstats_rebind.sh @@ -27,12 +27,16 @@ if [ -f /mnt/backup/bin/.PROPS ] then . /mnt/backup/bin/.PROPS DOW=`date | awk '{print $1}'` + if [ ${DOW} != ${DAYOFFULL} ] + then + echo "Runstats runs only on Day of Full backup. . . !!! Exiting !" + exit 0 + fi fi # -- Standard Parameters INSTANCE=`whoami` INSTANCE_HOME=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${INSTANCE}" | awk -F ',' '{print $5}'| sed 's/\/sqllib//'` -DOW=`date | awk '{print $1}'` mkdir -p ${INSTANCE_HOME}/maintenance/logs DATESTAMP=`date "+%Y-%m-%d-%H.%M.%S"` @@ -53,6 +57,10 @@ else . ${INSTANCE_HOME}/sqllib/db2profile fi +# -- Debug Mode +# set -x; # Uncomment to debug this shell script +# set -n; # Uncomment to check your syntax, without execution. + RUNSTATS_TMP_FILE="${INSTANCE_HOME}/bin/.runstats.sql" REBIND_TMP_FILE="${INSTANCE_HOME}/bin/.rebind.sql" @@ -71,24 +79,21 @@ do db2 connect to ${DB} | tee ${RUNSTATS_REBIND_LOG} if [[ $? -eq 0 ]]; then - db2 -x "select 'RUNSTATS ON TABLE \"' ||rtrim(tab.tabschema)||'\".\"'|| tab.tabname ||'\" WITH DISTRIBUTION ON KEY COLUMNS AND DETAILED INDEXES ALL ALLOW WRITE ACCESS;' from syscat.tables tab left join sysibmadm.SNAPTAB_REORG reg on tab.tabschema=reg.TABSCHEMA and tab.tabname=reg.TABNAME and reg.REORG_STATUS not in ('COMPLETED','STOPPED') where tab.type='T' and reg.tabname is null" > ${RUNSTATS_TMP_FILE} + db2 -x "select 'RUNSTATS ON TABLE \"' ||rtrim(tab.tabschema)||'\".\"'|| tab.tabname ||'\" ON ALL COLUMNS WITH DISTRIBUTION ON ALL COLUMNS AND DETAILED INDEXES ALL ALLOW WRITE ACCESS;' from syscat.tables tab left join sysibmadm.SNAPTAB_REORG reg on tab.tabschema=reg.TABSCHEMA and tab.tabname=reg.TABNAME and reg.REORG_STATUS not in ('COMPLETED','STOPPED') where tab.type='T' and reg.tabname is null" > ${RUNSTATS_TMP_FILE} echo -e "Begin processing of runstats @ ${DATESTAMP} ...\n" | tee -a ${RUNSTATS_REBIND_LOG} db2 -txvf ${RUNSTATS_TMP_FILE} | tee -a ${RUNSTATS_REBIND_LOG} echo -e "\nEnd processing of runstats @ ${DATESTAMP}" | tee -a ${RUNSTATS_REBIND_LOG} rm ${RUNSTATS_TMP_FILE} - - if [[ ${DOW} == ${DAYOFFULL} ]]; then + + db2 -x "select 'rebind package \"' ||rtrim(PKGSCHEMA)||'\".\"'|| PKGNAME ||'\";' from syscat.packages where PKGSCHEMA not like 'SYSIBM%' and PKGSCHEMA not like 'NULL%' " > ${REBIND_TMP_FILE} - db2 -x "select 'rebind package \"' ||rtrim(PKGSCHEMA)||'\".\"'|| PKGNAME ||'\";' from syscat.packages where PKGSCHEMA not like 'SYSIBM%' and PKGSCHEMA not like 'NULL%' " > ${REBIND_TMP_FILE} + echo -e "\n ----------------------------------------------- " | tee -a ${RUNSTATS_REBIND_LOG} + echo -e "Begin processing of rebind @ ${DATESTAMP} ...\n" | tee -a ${RUNSTATS_REBIND_LOG} + db2 -txvf ${REBIND_TMP_FILE} | tee -a ${RUNSTATS_REBIND_LOG} + echo -e "\nEnd processing of rebind @ ${DATESTAMP}" | tee -a ${RUNSTATS_REBIND_LOG} - echo -e "\n ----------------------------------------------- " | tee -a ${RUNSTATS_REBIND_LOG} - echo -e "Begin processing of rebind @ ${DATESTAMP} ...\n" | tee -a ${RUNSTATS_REBIND_LOG} - db2 -txvf ${REBIND_TMP_FILE} | tee -a ${RUNSTATS_REBIND_LOG} - echo -e "\nEnd processing of rebind @ ${DATESTAMP}" | tee -a ${RUNSTATS_REBIND_LOG} - - rm ${REBIND_TMP_FILE} - fi + rm ${REBIND_TMP_FILE} db2 terminate fi fi From be34ddd4cae3838cdb835de079f4f414ec04e546 Mon Sep 17 00:00:00 2001 From: prudhvirajp Date: Thu, 14 May 2026 16:42:41 +0530 Subject: [PATCH 3/3] Updated the logging of backups auditExtractUpload to upload the audit archives to COS bucket --- .../files/CopyDBScripts.sh | 2 + .../120-ibm-db2u-database/files/DB2_Backup.sh | 309 +-- .../files/RUN_OnDemandFULL_BKP.sh | 59 +- .../120-ibm-db2u-database/files/Run_Backup.sh | 78 +- .../files/auditExtractUpload.sh | 196 ++ .../files/reorgTablesIndexesInplace.sh | 2204 +++++++++-------- .../reorgTablesIndexesInplace2_maintenance.sh | 2202 ++++++++-------- .../files/runstats_rebind.sh | 107 +- 8 files changed, 2716 insertions(+), 2441 deletions(-) create mode 100644 instance-applications/120-ibm-db2u-database/files/auditExtractUpload.sh diff --git a/instance-applications/120-ibm-db2u-database/files/CopyDBScripts.sh b/instance-applications/120-ibm-db2u-database/files/CopyDBScripts.sh index c95d180a3..1a3edccbf 100644 --- a/instance-applications/120-ibm-db2u-database/files/CopyDBScripts.sh +++ b/instance-applications/120-ibm-db2u-database/files/CopyDBScripts.sh @@ -33,6 +33,8 @@ cp -rp grant_check.sh ${INSTHOME}/bin/ cp -rp reorgTablesIndexesInplace.sh ${INSTHOME}/bin/ cp -rp extract_authorization.sh ${INSTHOME}/bin cp -rp HADRMON.sh ${INSTHOME}/bin +cp -rp auditExtractUpload.sh ${INSTHOME}/bin/ + echo -e "\nCopying the file to bin/ITCS104 directory under Instance Home . . ." cp -rp FixInvalidObjects.sh ${INSTHOME}/bin/ITCS104/ diff --git a/instance-applications/120-ibm-db2u-database/files/DB2_Backup.sh b/instance-applications/120-ibm-db2u-database/files/DB2_Backup.sh index b5112aeb9..984c68e3d 100755 --- a/instance-applications/120-ibm-db2u-database/files/DB2_Backup.sh +++ b/instance-applications/120-ibm-db2u-database/files/DB2_Backup.sh @@ -1,9 +1,9 @@ #!/bin/ksh ######################################################### -# DB2_Backup.sh +# DB2_Backup.sh # -# Things to do: -# Recovery history retention (days) (REC_HIS_RETENTN) = 0 >>> Need to set to 15 days +# Things to do: +# Recovery history retention (days) (REC_HIS_RETENTN) = 0 >>> Need to set to 15 days # # The cron job on the cluster will supply the needed parameters for this script # If an on demand backup (Full) is required, the DB2_Backup.sh script can be called with the following parameters @@ -23,19 +23,21 @@ if [[ $# -eq 4 ]]; then typeset -l BKUP_TYPE=$4 else - print `tput smso` "Usage! $0 instance database number_of_backups_to_keep" `tput rmso` + print $(tput smso) "Usage! $0 instance database number_of_backups_to_keep" $(tput rmso) exit 1 fi # -- Standard Parameters -HOSTNAME=`hostname` -NAMESPACE=`hostname -A | awk -F '.' '{print $3}'` -DBINSTANCE=`whoami` -DATETIME=`date +'%F_%T'`; -INSTANCE_HOME=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${DBINSTANCE}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` -IP=`/sbin/ifconfig | grep "inet" | grep broadcast | awk '{print $2}'` -CUSTNAME=`hostname | sed 's/c-db2wh-//; s/c-//; s/-db2u-0//; s/db2u/-/; s/-manage//;' | tr '[:lower:]' '[:upper:]'` +HOSTNAME=$(hostname) +HOSTIP=$(/sbin/ifconfig | grep "inet" | grep broadcast | awk '{print $2}') +DATETIME=$(date +'%Y%m%d_%H%M%S'); +DBINSTANCE=$(whoami); +NAMESPACE=$(hostname -A | awk -F '.' '{print $3}') +INSTANCE_HOME=$(/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${DBINSTANCE}" | awk -F ',' '{print $5}'| sed 's/\/sqllib//' ) +CUSTNAME=$(hostname | sed 's/c-db2wh-//; s/c-db2u-//; s/c-//; s/-db2u-0//; s/db2u/-/; ' | tr '[:lower:]' '[:upper:]' ) SCRIPT_DIR=${INSTANCE_HOME}/bin +ICD_LOG=${SCRIPT_DIR}/.Maillive.log +Maillog="/tmp/.backup_maillog" # -- Source DB2 Profile @@ -47,59 +49,69 @@ else fi # -- Debug Mode -# set -x; # Uncomment to debug this shell script -# set -n; # Uncomment to check your syntax, without execution. + set -x; # Uncomment to debug this shell script +# set -n; # Uncomment to check your syntax, without execution. -# -- Source the PROPS file +# -- Source the PROPS file . /mnt/backup/bin/.PROPS # -- Backup Parameters COSBACKUPBUCKET="${CONTAINER}" -BUCKET_ALIAS=`db2 list storage access | grep ${COSBACKUPBUCKET} -B4 | grep ALIAS | awk -F '=' '{print $2}'` +BUCKET_ALIAS=$(db2 list storage access | grep ${COSBACKUPBUCKET} -B4 | grep ALIAS | awk -F '=' '{print $2}') BACKUP_BASE="/mnt/backup" BACKUP_LOGS=${SCRIPT_DIR}/${DBINSTANCE} BACKUP_PATH=DB2REMOTE://${BUCKET_ALIAS}/${COSBACKUPBUCKET}/backups-${APPENV}/${HOSTNAME} ARCBKP_PATH=${BACKUP_LOGS}/${DATETIME} CLEAN_LOG=${BACKUP_LOGS}/.cleanup.LOG -Maillog="/tmp/.backup_maillog" BACK_LOG=${SCRIPT_DIR}/.${DBNAME}_BackupLOG.out -ICD_LOG=${SCRIPT_DIR}/.Maillive.log HSTYPE="Backup" # -- Valid only for MAS-CP4D Customers if (( ${CUSTNAME} )) ; then - CUSTNAME=`echo ${CONTAINER} | awk -F '-backup-' '{print $2}' | awk -F '-pr-' '{print $1}' | tr '[:lower:]' '[:upper:]'` + CUSTNAME=$(echo ${CONTAINER} | awk -F '-backup-' '{print $2}' | awk -F '-pr-' '{print $1}' | tr '[:lower:]' '[:upper:]') fi # -- Database Environment if [[ ${BUCKET_ALIAS} == "IBMCOS" ]]; then - DBENV="MAS MS" + DBENV="MASMS" else - DBENV="MAS SaaS" + DBENV="MASSaaS" +fi + +# -- For mapping Hostname with Servicedesk + +NS=$( echo ${NAMESPACE} | sed 's/mas-//; s/-core//; s/-manage//; s/-facilities//; s/-db2u//;' ); +if [[ "$NAMESPACE" =~ "manage" ]] ; then + HSHOSTS="main.manage.${NS}.suite" +elif [[ "$NAMESPACE" =~ "core" ]]; then + HSHOSTS="main.home.${NS}.suite" +elif [[ "$NAMESPACE" =~ "monitor" ]]; then + HSHOSTS="main.monitor.${NS}.suite" +elif [[ "$NAMESPACE" =~ "facilities" ]]; then + HSHOSTS="main.facilities.${NS}.suite" fi # -- Function to send Slack notification SLACK_NOTIFY() { - SLACKDES="$1" - # -- Send Failure notification to a slack channel - cat << ! >.curl_${DBNAME}.sh - curl -X POST -H 'Content-type: application/json' --data '{"text":"${SLACKDES}"}' ${SLACKURL} + SLACKDES="$1" + # -- Send Failure notification to a slack channel + cat << ! >.curl_${DBNAME}.sh + curl -X POST -H 'Content-type: application/json' --data '{"text":"${SLACKDES}"}' ${SLACKURL} ! - /bin/bash .curl_${DBNAME}.sh > .curl_${DBNAME}.out 2>&1 + /bin/bash .curl_${DBNAME}.sh > .curl_${DBNAME}.out 2>&1 } # -- Create ICD Incident , If Backup fails CREATE_ICD() { - HTYPE=`echo ${HSTYPE} | tr '[:lower:]' '[:upper:]'` - DES="$1" -# echo "############################" >> ${ICD_LOG} - LONGDES=`cat ${ICD_LOG} | sed 's/"//g' | sed "s/'//g"` - LONGDES=`echo "
 ${LONGDES} 
"` + HTYPE=$(echo ${HSTYPE} | tr '[:lower:]' '[:upper:]') + DES="$1" + LONGDES=$(cat ${ICD_LOG} | sed 's/"//g' | sed "s/'//g") + LONGDES=$(echo "
${LONGDES}
") # -- Verify the ICD Status if curl -k -s --connect-timeout 3 ${ICD_URL_SAAS} >/dev/null; then @@ -123,7 +135,7 @@ CREATE_ICD() { "siteid":"001", "classstructureid":"1341", "classificationid":"IN-DBPERF", - "hshost":"${HOSTNAME}", + "hshost":"${HSHOSTS}", "hstype":"${HTYPE}" }' ! @@ -133,94 +145,92 @@ CREATE_ICD() { # -- Delete old log file if [[ -f $Maillog ]]; then - rm $Maillog + rm $Maillog fi # -- Create the backup log directory if it doesnt exists if [[ ! -d ${BACKUP_LOGS} ]]; then - mkdir -m 755 ${BACKUP_LOGS} + mkdir -m 755 ${BACKUP_LOGS} fi # -- Setting backup type if [[ ${BKUP_TYPE} == 'full' ]]; then - BKPTYPE="FULL" + BKPTYPE="FULL" else - BKPTYPE="DIFF" + BKPTYPE="DIFF" fi # -- Script Execution starts from here -echo -e "\n-------------------------------------" | tee ${BACK_LOG} -echo -e "Backup Start Time \t :: ${DATETIME}" | tee -a ${BACK_LOG} -echo -e "COS Bucket \t\t :: ${COSBACKUPBUCKET}" | tee -a ${BACK_LOG} -echo -e "\nHostname \t\t :: ${HOSTNAME}" | tee -a ${BACK_LOG} -echo -e "Namespace \t\t :: ${NAMESPACE}" | tee -a ${BACK_LOG} -echo -e "HostIP \t\t\t :: ${IP}" | tee -a ${BACK_LOG} -echo -e "-------------------------------------\n" | tee -a ${BACK_LOG} +ln=80 +printf "%${ln}s\n" | tr ' ' '-' | tee ${BACK_LOG}; +echo -e "\nBackup Start Time \t :: ${DATETIME}" | tee -a ${BACK_LOG}; +echo -e "COS Bucket \t\t :: ${COSBACKUPBUCKET}" | tee -a ${BACK_LOG}; +echo -e "\nHostname \t\t :: ${HOSTNAME}" | tee -a ${BACK_LOG}; +echo -e "Namespace \t\t :: ${NAMESPACE}" | tee -a ${BACK_LOG}; +echo -e "HostIP \t\t\t :: ${HOSTIP}" | tee -a ${BACK_LOG}; +printf "\n%${ln}s\n" | tr ' ' '-' | tee -a ${BACK_LOG}; # -- Check for the existance of /home/ctginst1/sqllib/db2dump/libdb2compr.so...if it exists, delete it COMPRESS_LOC=${INSTANCE_HOME}/sqllib/db2dump/libdb2compr.so if [[ -f ${COMPRESS_LOC} ]]; then - rm ${COMPRESS_LOC} + rm ${COMPRESS_LOC} fi - + # -- Check to see if the Instance is up and Running ps -ef | grep db2sysc | grep -v grep > /dev/null 2>&1 if [[ $? -eq 1 ]]; then - #echo "Instance is not active " - echo "Instance is not Active, Backup cannot Initiate !!!" | tee ${INSTANCE_HOME}/bin/LASTbkupRUN ${ICD_LOG} >/dev/null -# echo "############################" >> ${ICD_LOG} - cat ${BACK_LOG} >> ${ICD_LOG} + cat ${BACK_LOG} > ${ICD_LOG} + echo "Instance is not Active, Backup cannot Initiate !!!" | tee -a ${INSTANCE_HOME}/bin/LASTbkupRUN ${ICD_LOG} >/dev/null - SLACKDES="${CUSTNAME} - ${DBENV} - ${HOSTNAME}, Instance is not Active, Backup cannot Initiate !! " - DES="${CUSTNAME} - ${DBENV} - ${DBNAME} - ${HOSTNAME} -- Instance is not Active, Backup cannot Initiate !! " + SLACKDES="${CUSTNAME} - ${DBENV} - ${HOSTNAME}, Instance is not Active, Backup cannot Initiate !! " + DES="${CUSTNAME} - ${DBENV} - ${DBNAME} - ${HOSTNAME} -- Instance is not Active, Backup cannot Initiate !! " - # -- Send error notification to Slack - SLACK_NOTIFY "${SLACKDES}" + # -- Send error notification to Slack + SLACK_NOTIFY "${SLACKDES}" - # -- Create ICD ticket if fails - CREATE_ICD "${DES}" + # -- Create ICD ticket if fails + CREATE_ICD "${DES}" - # -- End the script execution - exit + # -- End the script execution + exit fi # -- Verify whether Database is Standby db2pd -hadr -db ${DBNAME} | awk -F= '/HADR_ROLE/ {print $2}' | grep STANDBY > /dev/null 2>&1 if [[ $? -eq 0 ]]; then - echo "This is a HADR Database" - echo "Backup successful. The timestamp for this backup image is : HADR_DB" - echo "${HOSTNAME}, HADR, NO BACKUPS" > ${INSTANCE_HOME}/bin/LASTbkupRUN + echo "This is a HADR Database" | tee -a ${ICD_LOG} + echo "Backup successful. The timestamp for this backup image is : HADR_DB" | tee -a ${ICD_LOG} + echo "${HOSTNAME}, HADR, NO BACKUPS" > ${INSTANCE_HOME}/bin/LASTbkupRUN exit 0 fi - -# -- Archive the logs for Database + +# -- Archive the logs for Database db2 -v "ARCHIVE LOG FOR DB ${DBNAME}" | tee -a ${BACK_LOG} sleep 20 # -- Starting backup for the database if [[ ${NUM_BACKUPS_TO_KEEP} -gt 0 ]]; then - if [[ ${BKUP_TYPE} = 'full' ]]; then - db2 -v "BACKUP DB ${DBNAME} ONLINE TO ${BACKUP_PATH} COMPRESS UTIL_IMPACT_PRIORITY 50 INCLUDE LOGS WITHOUT PROMPTING" | tee -a ${BACK_LOG} - else - db2 -v "BACKUP DB ${DBNAME} ONLINE INCREMENTAL DELTA TO ${BACKUP_PATH} COMPRESS UTIL_IMPACT_PRIORITY 50 INCLUDE LOGS WITHOUT PROMPTING" | tee -a ${BACK_LOG} - fi - - grep -Fq "Backup successful." ${BACK_LOG} - if [[ $? -ne 0 ]]; then - echo "${CUSTNAME} - ${DBENV} - ${HOSTNAME} - ${BKPTYPE} ${HSTYPE} Failed !!!" > ${ICD_LOG} - # echo "############################" >> ${ICD_LOG} - cat ${BACK_LOG} >> ${ICD_LOG} - - SLACKDES="${CUSTNAME} - ${DBENV} - ${HOSTNAME} -- ${BKPTYPE} ${HSTYPE} Failed . . . Please investigate ! ! ! " - DES="${CUSTNAME} - ${DBENV} - ${DBNAME} - ${HOSTNAME} -- ${BKPTYPE} ${HSTYPE} Failed !" - - # -- Send error notification to Slack - SLACK_NOTIFY "${SLACKDES}" - # -- Create ICD ticket if fails - CREATE_ICD "${DES}" - fi + if [[ ${BKUP_TYPE} = 'full' ]]; then + db2 -v "BACKUP DB ${DBNAME} ONLINE TO ${BACKUP_PATH} COMPRESS UTIL_IMPACT_PRIORITY 50 INCLUDE LOGS WITHOUT PROMPTING" | tee -a ${BACK_LOG} + else + db2 -v "BACKUP DB ${DBNAME} ONLINE INCREMENTAL DELTA TO ${BACKUP_PATH} COMPRESS UTIL_IMPACT_PRIORITY 50 INCLUDE LOGS WITHOUT PROMPTING" | tee -a ${BACK_LOG} + fi + + grep -Fq "Backup successful." ${BACK_LOG} + if [[ $? -ne 0 ]]; then + #echo "${CUSTNAME} - ${DBENV} - ${HOSTNAME} - ${BKPTYPE} ${HSTYPE} Failed !!!" > ${ICD_LOG} + cat ${BACK_LOG} > ${ICD_LOG} + + SLACKDES="${CUSTNAME} - ${DBENV} - ${HOSTNAME} -- ${BKPTYPE} ${HSTYPE} Failed . . . Please investigate ! ! ! " + DES="${CUSTNAME} - ${DBENV} - ${DBNAME} - ${HOSTNAME} -- ${BKPTYPE} ${HSTYPE} Failed !" + + # -- Send error notification to Slack + SLACK_NOTIFY "${SLACKDES}" + # -- Create ICD ticket if fails + CREATE_ICD "${DES}" + fi fi # -- Copy keystore to COS @@ -229,92 +239,95 @@ SOURCE2=/mnt/blumeta0/db2/keystore/keystore.sth TARGET1=backups-${APPENV}/${HOSTNAME}/KEYSTORE/keystore.p12 TARGET2=backups-${APPENV}/${HOSTNAME}/KEYSTORE/keystore.sth -DB2V=`db2level | grep Inform | awk '{print $5}' | sed 's/",//'` +DB2V=$(db2level | grep Inform | awk '{print $5}' | sed 's/",//') if [[ ${DB2V} == "v11.5.7.0" ]]; then - db2RemStgManager S3 put server=${HOSTNAME} auth1=${PARM1} auth2=${PARM2} container=${CONTAINER} source=${SOURCE1} target=${TARGET1} - db2RemStgManager S3 put server=${HOSTNAME} auth1=${PARM1} auth2=${PARM2} container=${CONTAINER} source=${SOURCE2} target=${TARGET2} + db2RemStgManager S3 put server=${HOSTNAME} auth1=${PARM1} auth2=${PARM2} container=${CONTAINER} source=${SOURCE1} target=${TARGET1} + db2RemStgManager S3 put server=${HOSTNAME} auth1=${PARM1} auth2=${PARM2} container=${CONTAINER} source=${SOURCE2} target=${TARGET2} else - db2RemStgManager ALIAS PUT source=${SOURCE1} target=DB2REMOTE://${BUCKET_ALIAS}//${TARGET1} - db2RemStgManager ALIAS PUT source=${SOURCE2} target=DB2REMOTE://${BUCKET_ALIAS}//${TARGET2} + db2RemStgManager ALIAS PUT source=${SOURCE1} target=DB2REMOTE://${BUCKET_ALIAS}//${TARGET1} + db2RemStgManager ALIAS PUT source=${SOURCE2} target=DB2REMOTE://${BUCKET_ALIAS}//${TARGET2} fi -# -- Exclude files that arent backups, e.g. backhist listing. +# -- Exclude files that arent backups, e.g. backhist listing. -typeset -i NO_BACKUPS=`~/bin/CheckCOS.sh | grep -i ${DBNAME} | cut -d/ -f3 | grep 001 | wc -l` -echo "Number of Backups in the bucket :: ${NO_BACKUPS}" +typeset -i NO_BACKUPS=$(~/bin/CheckCOS.sh | grep -i ${DBNAME} | cut -d/ -f3 | grep 001 | wc -l) +echo "Number of Backups in the bucket :: ${NO_BACKUPS}" | tee -a ${BACK_LOG} # -- Prune the history file, if and only if the last backup succeeded. - + if [[ ${NUM_BACKUPS_TO_KEEP} -gt 0 && ${NO_BACKUPS} -ge ${NUM_BACKUPS_TO_KEEP} ]]; then - db2 -v CONNECT TO ${DBNAME} | tee -a ${BACK_LOG} - - TIMESTMP=$(db2 -x "select coalesce(max(start), 17890713235959) from \ - (select bigint(start_time) - 1 as start, \ - row_number() over(order by start_time desc) as backup \ - from sysibmadm.db_history \ - where operation = 'B' \ - and objecttype = 'D' \ - and devicetype = 'D' \ - and sqlcode is null \ - and sqlwarn is null \ - ) as zzz \ - where backup = ${NUM_BACKUPS_TO_KEEP}" ) - - db2 -v "PRUNE HISTORY ${TIMESTMP} WITH FORCE OPTION AND DELETE" | tee -a ${BACK_LOG} - - # -- Loop until the recovery history file is stable and then report it - RC=999 - typeset -i no_loops=0 - while [[ $RC -gt 0 ]] - do - db2 -v list history backup since ${TIMESTMP} for ${DBNAME} > ${BACKUP_LOGS}/backhist - RC=$? - print RC for list history was $RC - cat ${BACKUP_LOGS}/backhist >> ${BACK_LOG} - if [[ $no_loops -gt 720 ]]; then - # -- then youve been waiting an hour - print $0 "Im tired of waiting for the recovery history file to stabilise. Im giving up" - break - else - sleep 5 - let no_loops=no_loops+1 - fi - done - - echo "Content of backhist file: " - cat ${BACKUP_LOGS}/backhist - db2 -v commit | tee -a ${BACK_LOG} - db2 -v connect reset | tee -a ${BACK_LOG} - db2 -v terminate | tee -a ${BACK_LOG} + db2 -v CONNECT TO ${DBNAME} > /dev/null 2>&1 + + TIMESTMP=$(db2 -x "select coalesce(max(start), 17890713235959) from \ + (select bigint(start_time) - 1 as start, \ + row_number() over(order by start_time desc) as backup \ + from sysibmadm.db_history \ + where operation = 'B' \ + and objecttype = 'D' \ + and devicetype = 'D' \ + and sqlcode is null \ + and sqlwarn is null \ + ) as zzz \ + where backup = ${NUM_BACKUPS_TO_KEEP}" | tr -d ' ' ) + + db2 -v "PRUNE HISTORY ${TIMESTMP} WITH FORCE OPTION AND DELETE" | tee -a ${BACK_LOG} + + # -- Loop until the recovery history file is stable and then report it + RC=999 + typeset -i no_loops=0 + while [[ $RC -gt 0 ]] + do + db2 -v list history backup since ${TIMESTMP} for ${DBNAME} > ${BACKUP_LOGS}/backhist + RC=$? + #print RC for list history was $RC + #cat ${BACKUP_LOGS}/backhist >> ${BACK_LOG} + if [[ $no_loops -gt 720 ]]; then + # -- then youve been waiting an hour + print $0 "Im tired of waiting for the recovery history file to stabilise. Im giving up" | tee -a ${BACK_LOG} + break + else + sleep 5 + let no_loops=no_loops+1 + fi + done + + #echo "Content of backhist file: " + #cat ${BACKUP_LOGS}/backhist + db2 -v commit > /dev/null 2>&1 + db2 -v connect reset > /dev/null 2>&1 + db2 -v terminate > /dev/null 2>&1 + fi sleep 20 # -- Prune the archive logs and archive log history file if [[ ${NUM_BACKUPS_TO_KEEP} -eq 0 ]]; then - db2 -v connect to ${DBNAME} | tee -a ${BACK_LOG} - db2 -x "select location from sysibmadm.db_history where operation = 'X' and operationtype = '1' " > ${BACKUP_LOGS}/archivelog.zaplist - - for LOG in `cat ${BACKUP_LOGS}/archivelog.zaplist` - do - printf "`date +'%F %T'`\t%-110s\t%12d k\n" "${LOG}" "`du -sk ${LOG} | awk '{print $1}'`" >> ${CLEAN_LOG} - done - - # -- prune history in step - TIMESTMP=$(db2 -x "select max(start_time) from sysibmadm.db_history where operation = 'X' and operationtype = '1'") - db2 -v PRUNE HISTORY ${TIMESTMP} WITH FORCE OPTION AND DELETE | tee -a ${BACK_LOG} - wait - db2 -v commit | tee -a ${BACK_LOG} - db2 -v connect reset | tee -a ${BACK_LOG} - db2 -v terminate | tee -a ${BACK_LOG} + db2 -v connect to ${DBNAME} > /dev/null 2>&1 + db2 -x "select location from sysibmadm.db_history where operation = 'X' and operationtype = '1' " > ${BACKUP_LOGS}/archivelog.zaplist + + for LOG in $(cat ${BACKUP_LOGS}/archivelog.zaplist) + do + printf "$(date +'%F %T')\t%-110s\t%12d k\n" "${LOG}" "$(du -sk ${LOG} | awk '{print $1}')" >> ${CLEAN_LOG} + done + + # -- prune history in step + TIMESTMP=$(db2 -x "select max(start_time) from sysibmadm.db_history where operation = 'X' and operationtype = '1'" | tr -d ' ') + db2 -v PRUNE HISTORY ${TIMESTMP} WITH FORCE OPTION AND DELETE | tee -a ${BACK_LOG} + wait + db2 -v commit > /dev/null 2>&1 + db2 -v connect reset > /dev/null 2>&1 + db2 -v terminate > /dev/null 2>&1 + fi - -DATETIME=`date +%Y-%m-%d_%H%M%S`; + + +DATETIME=$(date +%Y-%m-%d_%H%M%S); echo "BACKUP End time :: ${DATETIME}" >> ${BACK_LOG} -# -- Copy the current backup LOG to the Backup LOG history file +# -- Copy the current backup LOG to the Backup LOG history file cat ${BACK_LOG} >> ${BACKUP_LOGS}/.BackupLOG # -- END OF SCRIPT \ No newline at end of file diff --git a/instance-applications/120-ibm-db2u-database/files/RUN_OnDemandFULL_BKP.sh b/instance-applications/120-ibm-db2u-database/files/RUN_OnDemandFULL_BKP.sh index 7124e9f80..c1ca7e223 100755 --- a/instance-applications/120-ibm-db2u-database/files/RUN_OnDemandFULL_BKP.sh +++ b/instance-applications/120-ibm-db2u-database/files/RUN_OnDemandFULL_BKP.sh @@ -11,28 +11,29 @@ # Variables to be set # SLACKURL = The channel were notifications are send # BACKUP_SCRIPT = The backup script that RUN_OnDemandFULL_BKP.sh calls -# DAYOFFULL = Defines the day of the week that the full backup will on on (must match the same format as the output from `date`) +# DAYOFFULL = Defines the day of the week that the full backup will on on (must match the same format as the output from $(date)) # NUMOFBKUPTOKEEP = This defines the number of days to keep a backup image on local disk # # Variables determined by the environment -# BACKUPTYPE = Is determined from the `date` command and the DAYOFFULL value +# BACKUPTYPE = Is determined from the $(date) command and the DAYOFFULL value # DB2INSTANCE = Pulled from the environment # HOSTNAME -# DBNAME = Pulled from the `db2 list db directory` +# DBNAME = Pulled from the $(db2 list db directory) # # Backup command issued # ./DB2_Backup.sh ${DB2INSTANCE} ${DBNAME} ${NUMOFBKUPTOKEEP} ${BACKUPTYPE} 2>>.BackupLOG.stderr > .BackupLOG.out ######################################################### -# -- Source the PROPS File -. /mnt/backup/bin/.PROPS - # -- Standard Parameters -DBINSTANCE=`whoami` -HOSTNAME=`hostname` -INSTANCE_HOME=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${DBINSTANCE}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` +HOSTNAME=$(hostname) +NAMESPACE=$(hostname -A | awk -F '.' '{print $3}') +DBINSTANCE=$(whoami); +DATETIME=$(date +'%F_%T'); +INSTANCE_HOME=$(/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${DBINSTANCE}" | awk -F ',' '{print $5}'| sed 's/\/sqllib//' ) +HOSTIP=$(/sbin/ifconfig | grep "inet" | grep broadcast | awk '{print $2}') +CUSTNAME=$(hostname | sed 's/c-db2wh-//; s/c-db2u-//; s/c-//; s/-db2u-0//; s/db2u/-/; ' | tr '[:lower:]' '[:upper:]' ) SCRIPT_DIR=${INSTANCE_HOME}/bin -DATETIME=`date +%Y-%m-%d_%H%M%S`; +ICD_LOG=${SCRIPT_DIR}/.Maillive.log # -- Verify and source db2profile @@ -42,6 +43,8 @@ if [[ ! -f "${INSTANCE_HOME}/sqllib/db2profile" ]]; then else . ${INSTANCE_HOME}/sqllib/db2profile fi +# -- Source the PROPS File +. /mnt/backup/bin/.PROPS # -- Debug Mode # set -x; # Uncomment to debug this shell script @@ -50,33 +53,43 @@ fi # -- Backup parameters BACKUPTYPE=full BACKUP_SCRIPT="${SCRIPT_DIR}/DB2_Backup.sh" -CUSTNAME=`hostname | sed 's/c-db2wh-//; s/c-//; s/-db2u-0//; s/db2u/-/; s/-manage//;' | tr '[:lower:]' '[:upper:]'` -BUCKET_ALIAS=`db2 list storage access | grep ${CONTAINER} -B4 | grep ALIAS | awk -F '=' '{print $2}'` +BUCKET_ALIAS=$(db2 list storage access | grep ${CONTAINER} -B4 | grep ALIAS | awk -F '=' '{print $2}') HSTYPE="Backup" -ICD_LOG=${SCRIPT_DIR}/.Maillive.log # -- Valid only for MAS-CP4D customers if (( ${CUSTNAME} )) ; then - CUSTNAME=`echo ${CONTAINER} | awk -F '-backup-' '{print $2}' | awk -F '-pr-' '{print $1}' | tr '[:lower:]' '[:upper:]'` + CUSTNAME=$(echo ${CONTAINER} | awk -F '-backup-' '{print $2}' | awk -F '-pr-' '{print $1}' | tr '[:lower:]' '[:upper:]') fi # -- Database Environment if [[ ${BUCKET_ALIAS} == "IBMCOS" ]]; then - DBENV="MAS MS" + DBENV="MASMS" else - DBENV="MAS SaaS" + DBENV="MASSaaS" +fi + +# -- For mapping Hostname with Servicedesk + +NS=$( echo ${NAMESPACE} | sed 's/mas-//; s/-core//; s/-manage//; s/-facilities//; s/-db2u//;' ); +if [[ "$NAMESPACE" =~ "manage" ]] ; then + HSHOSTS="main.manage.${NS}.suite" +elif [[ "$NAMESPACE" =~ "core" ]]; then + HSHOSTS="main.home.${NS}.suite" +elif [[ "$NAMESPACE" =~ "monitor" ]]; then + HSHOSTS="main.monitor.${NS}.suite" +elif [[ "$NAMESPACE" =~ "facilities" ]]; then + HSHOSTS="main.facilities.${NS}.suite" fi # -- Create ICD Incident , If Backup fails CREATE_ICD() { - HTYPE=`echo ${HSTYPE} | tr '[:lower:]' '[:upper:]'` + HTYPE=$(echo ${HSTYPE} | tr '[:lower:]' '[:upper:]') DES="$1" -# echo "############################" >> ${ICD_LOG} - LONGDES=`cat ${ICD_LOG} | sed 's/"//g' | sed "s/'//g"` - LONGDES=`echo "
 ${LONGDES} 
"` + LONGDES=$(cat ${ICD_LOG} | sed 's/"//g' | sed "s/'//g") + LONGDES=$(echo "
${LONGDES} 
") - # -- Verify the ICD URL Status + # -- Verify the ICD Status if curl -k -s --connect-timeout 3 ${ICD_URL_SAAS} >/dev/null; then CURL_REQ="--request POST --url ${ICD_URL_SAAS} " AUTH_REQ="apikey: ${ICD_API_KEY}" @@ -98,7 +111,7 @@ CREATE_ICD() { "siteid":"001", "classstructureid":"1341", "classificationid":"IN-DBPERF", - "hshost":"${HOSTNAME}", + "hshost":"${HSHOSTS}", "hstype":"${HTYPE}" }' ! @@ -108,7 +121,7 @@ CREATE_ICD() { # -- Loop through the available databases in the instance -DBS=`db2 list db directory | grep -B5 "Indirect" | grep "Database name" | awk '{ print $4 }'` +DBS=$(db2 list db directory | grep -B5 "Indirect" | grep "Database name" | awk '{ print $4 }') for DBNAME in ${DBS} do cd ${SCRIPT_DIR} diff --git a/instance-applications/120-ibm-db2u-database/files/Run_Backup.sh b/instance-applications/120-ibm-db2u-database/files/Run_Backup.sh index 21bfe794d..eaf65f317 100755 --- a/instance-applications/120-ibm-db2u-database/files/Run_Backup.sh +++ b/instance-applications/120-ibm-db2u-database/files/Run_Backup.sh @@ -11,14 +11,14 @@ # Variables to be set # SLACKURL = The channel were notifications are send # BACKUP_SCRIPT = The backup script that Run_Backup.sh calls -# DAYOFFULL = Defines the day of the week that the full backup will on on (must match the same format as the output from `date`) +# DAYOFFULL = Defines the day of the week that the full backup will on on (must match the same format as the output from $(date)) # NUMOFBKUPTOKEEP = This defines the number of days to keep a backup image on local disk # # Variables determined by the environment -# BKPTYPE = Is determined from the `date` command and the DAYOFFULL value +# BKPTYPE = Is determined from the $(date) command and the DAYOFFULL value # DB2INSTANCE = Pulled from the environment # HOSTNAME -# DBNAME = Pulled from the `db2 list db directory` +# DBNAME = Pulled from the $(db2 list db directory) # # Backup command issued # ./DB2_Backup.sh ${DB2INSTANCE} ${DBNAME} ${NUMOFBKUPTOKEEP} ${BKPTYPE} 2>>.BackupLOG.stderr > .BackupLOG.out @@ -26,14 +26,16 @@ # -- Revision of script to include new ICD URL ######################################################### -# -- Source the Props File -. /mnt/backup/bin/.PROPS - # -- Standard Parameters -DBINSTANCE=`whoami` -HOSTNAME=`hostname` -DATETIME=`date +%Y-%m-%d_%H%M%S`; -DOW=`date | awk '{print $1}'` +HOSTNAME=$(hostname) +IP=$(/sbin/ifconfig | grep "inet" | grep broadcast | awk '{print $2}') +DATETIME=$(date +'%Y%m%d_%H%M%S'); +DBINSTANCE=$(whoami); +NAMESPACE=$(hostname -A | awk -F '.' '{print $3}') +INSTANCE_HOME=$(/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${DBINSTANCE}" | awk -F ',' '{print $5}'| sed 's/\/sqllib//' ) +CUSTNAME=$(hostname | sed 's/c-db2wh-//; s/c-db2u-//; s/c-//; s/-db2u-0//; s/db2u/-/; ' | tr '[:lower:]' '[:upper:]' ) +SCRIPT_DIR=${INSTANCE_HOME}/bin +ICD_LOG=${SCRIPT_DIR}/.Maillive.log # -- Verify and source db2profile @@ -44,41 +46,54 @@ else . ${HOME}/sqllib/db2profile fi +# -- Source the Props File +. /mnt/backup/bin/.PROPS + # -- Debug Mode # set -x; # Uncomment to debug this shell script # set -n; # Uncomment to check your syntax, without execution. # -- Backup Parameters -INSTANCE_HOME=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${DBINSTANCE}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` -SCRIPT_DIR=${INSTANCE_HOME}/bin BACKUP_SCRIPT="${SCRIPT_DIR}/DB2_Backup.sh" -CUSTNAME=`hostname | sed 's/c-db2wh-//; s/c-//; s/-db2u-0//; s/db2u/-/; s/-manage//;' | tr '[:lower:]' '[:upper:]'` -BUCKET_ALIAS=`db2 list storage access | grep ${CONTAINER} -B4 | grep ALIAS | awk -F '=' '{print $2}'` +BUCKET_ALIAS=$(db2 list storage access | grep ${CONTAINER} -B4 | grep ALIAS | awk -F '=' '{print $2}' ) +DOW=$(date | awk '{print $1}') HSTYPE="Backup" -ICD_LOG=${SCRIPT_DIR}/.Maillive.log # -- Valid only for MAS-CP4D customers if (( ${CUSTNAME} )) ; then - CUSTNAME=`echo ${CONTAINER} | awk -F '-backup-' '{print $2}' | awk -F '-pr-' '{print $1}' | tr '[:lower:]' '[:upper:]'` + CUSTNAME=$( echo ${CONTAINER} | awk -F '-backup-' '{print $2}' | awk -F '-pr-' '{print $1}' | tr '[:lower:]' '[:upper:]' ) fi # -- Database Environment if [[ ${BUCKET_ALIAS} == "IBMCOS" ]]; then - DBENV="MAS MS" + DBENV="MASMS" else - DBENV="MAS SaaS" + DBENV="MASSaaS" fi +# -- For mapping Hostname with Servicedesk + +NS=$( echo ${NAMESPACE} | sed 's/mas-//; s/-core//; s/-manage//; s/-facilities// ; s/-db2u//;' ); +if [[ "$NAMESPACE" =~ "manage" ]] ; then + HSHOSTS="main.manage.${NS}.suite" +elif [[ "$NAMESPACE" =~ "core" ]]; then + HSHOSTS="main.home.${NS}.suite" +elif [[ "$NAMESPACE" =~ "monitor" ]]; then + HSHOSTS="main.monitor.${NS}.suite" +elif [[ "$NAMESPACE" =~ "facilities" ]]; then + HSHOSTS="main.facilities.${NS}.suite" +fi + + # -- Create ICD Incident , If Backup fails CREATE_ICD() { - HTYPE=`echo ${HSTYPE} | tr '[:lower:]' '[:upper:]'` - DES="$1" -# echo "############################" >> ${ICD_LOG} - LONGDES=`cat ${ICD_LOG} | sed 's/"//g' | sed "s/'//g"` - LONGDES=`echo "
 ${LONGDES} 
"` + HTYPE=$(echo ${HSTYPE} | tr '[:lower:]' '[:upper:]') + DES="$1" + LONGDES=$(cat ${ICD_LOG} | sed 's/"//g' | sed "s/'//g") + LONGDES=$(echo "
${LONGDES}
") - # -- Verify the ICD URL Status + # -- Verify the ICD Status if curl -k -s --connect-timeout 3 ${ICD_URL_SAAS} >/dev/null; then CURL_REQ="--request POST --url ${ICD_URL_SAAS} " AUTH_REQ="apikey: ${ICD_API_KEY}" @@ -100,7 +115,7 @@ CREATE_ICD() { "siteid":"001", "classstructureid":"1341", "classificationid":"IN-DBPERF", - "hshost":"${HOSTNAME}", + "hshost":"${HSHOSTS}", "hstype":"${HTYPE}" }' ! @@ -117,7 +132,7 @@ fi # -- Loop through the available databases in the instance -DBS=`db2 list db directory | grep -B5 "Indirect" | grep "Database name" | awk '{ print $4 }'` +DBS=$(db2 list db directory | grep -B5 "Indirect" | grep "Database name" | awk '{ print $4 }') for DBNAME in ${DBS} do cd ${SCRIPT_DIR} @@ -136,10 +151,10 @@ do CREATE_ICD "${DES}" fi - # -- Execute Online Reorgs for qualified tables and indexes after every Full Backup - if [[ ${DOW} = ${DAYOFFULL} ]] ; then - /bin/bash ${SCRIPT_DIR}/reorgTablesIndexesInplace.sh -db ${DBNAME} -s MAXIMO -tb_stats -ix_stats -window 120 -tr > ${HOME}/maintenance/logs/reorgTablesIndexesInplace_${DATETIME}.log 2>&1 - fi + # -- Execute Online Reorgs for qualified tables and indexes after every Full Backup + if [[ ${DOW} = ${DAYOFFULL} ]] ; then + /bin/bash ${SCRIPT_DIR}/reorgTablesIndexesInplace.sh -db ${DBNAME} -s MAXIMO -tb_stats -ix_stats -tr -window 180 >${INSTANCE_HOME}/maintenance/logs/reorgTablesIndexesInplace_${DATETIME}.log 2>&1 + fi done @@ -147,4 +162,7 @@ done /bin/bash ${SCRIPT_DIR}/runstats_rebind.sh >${SCRIPT_DIR}/.runstats_rebind.out 2>&1 #/bin/bash ${SCRIPT_DIR}/grant_check.sh bludb >${SCRIPT_DIR}/.grant_check.out 2>&1 +# -- Extract the audit logs and transfer to Bucket +/bin/bash ${SCRIPT_DIR}/auditExtractUpload.sh >${INSTANCE_HOME}/maintenance/logs/auditExtractUpload_${DATETIME}.log & disown + # -- END OF SCRIPT \ No newline at end of file diff --git a/instance-applications/120-ibm-db2u-database/files/auditExtractUpload.sh b/instance-applications/120-ibm-db2u-database/files/auditExtractUpload.sh new file mode 100644 index 000000000..af0e40d4d --- /dev/null +++ b/instance-applications/120-ibm-db2u-database/files/auditExtractUpload.sh @@ -0,0 +1,196 @@ +#!/bin/sh + +# ---------------------------------------------------------------------------- +#% Script Name : auditExtractUpload.sh +#% Description : Script to Archive, Extract the audit logs and upload to COS +#% Created On : 10th February 2026 +#% +#% Author : Mujibur Rahman +#% Email : mujibur.rahman1@ibm.com +# ---------------------------------------------------------------------------- +# Version Date Changed By Description +# ---------------------------------------------------------------------------- +# 0.1 10-02-2026 Mujib Initial Version +# 0.2 04-05-2026 Prudhviraj P Updated the loggging, clean up of old archives, induced compression to archives +# +# ---------------------------------------------------------------------------- +# ************** THIS NEEDS TO BE RUN AS INSTANCE OWNER. ***************** +# USAGE: +# auditExtractUpload.sh +# +# *************************************************************************** + +#set -euo pipefail + +# ============================================================================ +# Parameters/Inputs +# ============================================================================ + +HOSTNAME=$(hostname) +HOSTIP=$(hostname -i) +WHOAMI=$(whoami) +INST=$(/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | awk -F, '{print $4}') +INSTHOME=$(/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${INST}" | awk -F ',' '{print $5}'| sed 's/\/sqllib//') +BASE_DIR=${INSTHOME}/bin +DT=$(date +"%Y%m%d_%H%M%S") +DBNAME="BLUDB" + +# =========================================================================== +# Invoking DB2 Profile +# =========================================================================== + +. ${INSTHOME}/sqllib/db2profile + +# ------------------------------- +# Load COS parameters +# ------------------------------- +. /mnt/backup/bin/.PROPS + +# =========================================================================== +# Debug Options +# =========================================================================== + +# set -x; # Uncomment to debug this shell script +# set -n; # Uncomment to check your syntax, without execution. + + +# -- Extract the info needed from db2audit + +DATA_DIR=$( db2audit describe | grep "Audit Data Path: " | awk -F ': ' '{gsub(/"/, ""); print $2}' | sed 's/\/$//' ); +ARCHIVE_DIR=$( db2audit describe | grep "Audit Archive Path:" | awk -F ': ' '{gsub(/"/, ""); print $2}' | sed 's/\/$//' ); + +# ------------------------------- +# Functions +# ------------------------------- +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" +} + +cleanup() { + log "INFO :: Ensuring db2audit is running " + db2audit start >/dev/null 2>&1 || true +} + +trap cleanup EXIT + +# ------------------------------- +# Start processing +# ------------------------------- +log "INFO :: Starting DB2 audit extraction for database $DBNAME" + +: ' +# Stop audit +db2audit stop +RC=$? +if [[ $RC -eq 0 ]]; then + log "INFO :: db2audit stopped" +else + log "ERROR :: db2audit cannot be stopped" +fi +' + +# Flush records +db2audit flush > /dev/null 2>&1 +RC=$? +if [[ $RC -eq 0 ]]; then + log "INFO :: db2audit flushed" +else + log "ERROR :: db2audit cannot be flushed" +fi + +# Archive audit logs +db2audit archive database "$DBNAME" > /dev/null 2>&1 +RC=$? +if [[ $RC -eq 0 ]]; then + log "INFO :: db2audit archived" +else + log "ERROR :: db2audit cannot be archived" + exit +fi +#db2audit.db.BLUDB.log.0.20260504093246 +#db2audit.db."$DBNAME".log.0. +DTN=$( date +'%Y%m%d%H' ) + +# Identify latest archived audit file +AUDIT_LOG=$(ls -t "$ARCHIVE_DIR"/db2audit.db."$DBNAME".log.0.$DTN* 2>/dev/null | head -1) +AUDIT_FILE="${ARCHIVE_DIR}/db2audit_report.${DT}" +AUDIT_ZIP="${AUDIT_FILE}.zip" + +log "INFO :: Verifying whether audit log is available or not" + +if [[ -z "${AUDIT_LOG:-}" ]]; then + log "ERROR :: No archived audit file found" + exit 1 +fi + +log "INFO :: Extracting audit data from $AUDIT_LOG" + +# Extract audit data +db2audit extract file "$AUDIT_FILE" from files "$AUDIT_LOG" > /dev/null 2>&1 +RC=$? +if [[ $RC -eq 0 ]]; then + log "INFO :: Audit log extract and created as $AUDIT_FILE" +else + log "ERROR :: Audit Failed to extract. " + exit +fi + +log "INFO :: Compressing the Audit file : $AUDIT_FILE" +zip -r ${AUDIT_ZIP} ${AUDIT_FILE} > /dev/null 2>&1 +RC=$? +if [[ $RC -eq 0 ]]; then + log "INFO :: Audit log Compressed to $AUDIT_ZIP" + log "INFO :: Removing the extracted and archived files " + log "INFO :: ${AUDIT_FILE} " + log "INFO :: ${AUDIT_LOG} " + rm ${AUDIT_FILE} ${AUDIT_LOG} + RC2=$? + if [[ $RC2 -eq 0 ]]; then + log "INFO :: Removed both extracted/Archive audit files " + else + log "ERROR :: Failed to delete the extracted/Archive audit files " + exit + fi +else + log "ERROR :: Audit Failed to Compress. " + exit +fi + +# Restart audit +#db2audit start +#log "db2audit started" + +# Upload to COS +log "INFO :: Uploading audit extract to COS" + +AUDZIP=$( echo $AUDIT_ZIP | awk -F '/' '{print $NF}' ) +COS_TARGET="DB2REMOTE://AWSCOS//AUDIT_LOGS/${AUDZIP}" + +db2RemStgManager alias put \ + source="$AUDIT_ZIP" \ + target="$COS_TARGET" > /dev/null 2>&1 +RC=$? +if [[ $RC -eq 0 ]]; then + log "INFO :: Upload completed to $COS_TARGET" +else + log "ERROR :: Failed to Upload. " + exit +fi + + +# -- Clean the old archives more than 30 days +log "INFO :: Deleting the old archive more than 30 Days from Pod " +find ${ARCHIVE_DIR} -name db2audit_report.*.zip -type f -mtime +30 -exec rm {} \; +RC=$? +if [[ $RC -eq 0 ]]; then + log "INFO :: Cleaned archives more than 30 Days" +else + log "ERROR :: Failed to clean the old archives " + exit +fi + +log "INFO :: Execution of audit extract and upload has completed. " +exit 0 + + +# -- End of the Script \ No newline at end of file diff --git a/instance-applications/120-ibm-db2u-database/files/reorgTablesIndexesInplace.sh b/instance-applications/120-ibm-db2u-database/files/reorgTablesIndexesInplace.sh index 6f594ba73..28a993c4e 100644 --- a/instance-applications/120-ibm-db2u-database/files/reorgTablesIndexesInplace.sh +++ b/instance-applications/120-ibm-db2u-database/files/reorgTablesIndexesInplace.sh @@ -5,66 +5,71 @@ ## http://www.ibm.com/developerworks/data/library/techarticle/dm-1307optimizerunstats/ ## see Identifying fragmented indexes from statistics ## http://www.ibm.com/developerworks/data/library/techarticle/dm-1307optimizerunstats/#Listing%207 -##0 23 * * 5 (. ~/sqllib/db2profile; ~/dba/bin/misc/reorgTablesIndexesInplace2.sh -s OMDB -tb_stats -if_stats -window 120 -tr >> ~/dba/logs/reorgTablesIndexesInplace2.sh.log 2>&1 ) +##0 23 * * 5 (. ~/sqllib/db2profile; ~/maintenance/bin/misc/reorgTablesIndexesInplace.sh -db dbname -s MAXIMO -tb_stats -if_stats -window 120 -tr >> ~/maintenance/logs/reorgTablesIndexesInplace.sh.log 2>&1 ) ## script to reorg tables online and indexes offline based on different criteria ## we need to be able to perform an online table reorg and an offline indexes all reorg in the same run of the script ## +# +# -- Modifications/Updates on 01/May/26 by Prudhviraj Patrata +# -- Increased the default tablesize from 20GB to 100GB +# -- Updated to use single database at a time +# -- Improved the readability of output of logs +# UsageHelp() { - - echo "Script to perform reorg tables, indexes online (inplace) " - echo " also to REORG INDEXES ALL FOR TABLE offline" - echo " db2 performs the online reorgs asynchronously" - echo "" - echo "Usage: ${0} [options]" - echo " where [options] is one of the following:" - echo " -h: displays this usage screen" - echo " -db: dbname, default is all cataloged databases" - echo "" - echo " -s: table schemaname" - echo " -t: table(s) to reorg" - echo "-tb_stats: reorg tables reported by REORGCHK_TB_STATS" - echo " -ti: reorg table index(s), format must be TABSCHEMA.TABNAME.INDSCHEMA.INDNAME" - echo "-ix_stats: reorg table index(s) reported by REORGCHK_IX_STATS" - echo "-if_stats: reorg indexes all for table(s) offline as reported by index fragmentation NLEAF/SEQUENTIAL_PAGES columns" - echo "" - echo " -ls: list valid table sizes for a particular schema" - echo " -lf: list all fragmented index details for a particular schema, based on valid table sizes" - echo " -lt: list all tables to reorg based on REORGCHK_TB_STATS reorg column, based on valid table sizes" - echo " -li: list all indexes to reorg based on REORGCHK_IX_STATS reorg column, based on valid table sizes" - echo " -l: list tables/indexes that would be reorged" - echo "" - echo " -ittx: ignore tables over a specific threshold size in MBs, default is 20000 MB ie 20 GB" - echo " -ittn: ignore tables under a specific threshold size in MBs, default is 10 MB" - echo " -mar: maximum asynchronous reorgs allowed, default is 3" - echo " -log: don't kick off a reorg if transaction log usage is over a certain percentage, default is 90%" - echo " -window: stop reorg tables/indexes/runstats after a set maintenance timeout window, default is 240 minutes" - echo " -twa: timeout window action: default=2 for online, 1 for offline" - echo " 1=allow current reorg(s) to continue" - echo " 2=stop current reorg(s)" -# echo " 3=stop current reorg(s) if < 80% complete and continue script" - echo " -ignore: ignore specific tables from SYSIBMADM.ADMINTABINFO t0, SYSCAT.TABLES t1 " - echo " eg \"$IGNORE_TABLES_EX\"" - echo " -reorg: table F1 F2 F3 filter reorg, default is *" - echo " -sleep: SLEEP_INTERVAL_TIME, default is 60 seconds" - echo "" - echo " -tr: execute inplace table/index reorg" - echo "" - echo " -trsi: Retrieve table reorganization snapshot information from snap_get_tab_reorg and db2pd -reorgs index" - echo "" - echo "Examples:" - echo " 1. ${0} -h" - echo " 2. ${0} -db dbname -s omdb -ls" - echo " 3. ${0} -s OMDB -t \"YFS_ITEM YFS_TASK_Q YFS_SHIPMENT\" -tb_stats -tr " - echo " 4. ${0} -s OMDB -ti \"OMDB.YFS_SNAPSHOT.OMDB.YFS_SNAPSHOT_I1 OMDB.YFS_ITEM.OMDB.YFS_ITEM_PK\" -ix_stats -tr" - echo " 5. ${0} -s OMDB -t \"YFS_ITEM YFS_SNAPSHOT YFS_IMPORT YFS_EXPORT\" -if_stats -tr" - echo " 6. ${0} -s OMDB -tb_stats -mar 5 -window 10 -log 95 -ittx 30000 -tr" - echo " 7. ${0} -s OMDB -tb_stats -mar 5 -window 10 -log 95 -ignore \"$IGNORE_TABLES_EX\" -reorg \"***\" -tr" - echo " 8. ${0} -s OMDB -tb_stats -if_stats -ittx 100 -ittn 20 -tr" - echo " 9. ${0} -trsi" - - echo "" + echo "Script to perform reorg tables, indexes online (inplace) " + echo " also to REORG INDEXES ALL FOR TABLE offline" + echo " db2 performs the online reorgs asynchronously" + echo "" + echo "Usage: ${0} [options]" + echo " where [options] is one of the following:" + echo " -h: displays this usage screen" + echo " -db: dbname, default is all cataloged databases" + echo "" + echo " -s: table schemaname" + echo " -t: table(s) to reorg" + echo "-tb_stats: reorg tables reported by REORGCHK_TB_STATS" + echo " -ti: reorg table index(s), format must be TABSCHEMA.TABNAME.INDSCHEMA.INDNAME" + echo "-ix_stats: reorg table index(s) reported by REORGCHK_IX_STATS" + echo "-if_stats: reorg indexes all for table(s) offline as reported by index fragmentation NLEAF/SEQUENTIAL_PAGES columns" + echo "" + echo " -ls: list valid table sizes for a particular schema" + echo " -lf: list all fragmented index details for a particular schema, based on valid table sizes" + echo " -lt: list all tables to reorg based on REORGCHK_TB_STATS reorg column, based on valid table sizes" + echo " -li: list all indexes to reorg based on REORGCHK_IX_STATS reorg column, based on valid table sizes" + echo " -l: list tables/indexes that would be reorged" + echo "" + echo " -ittx: ignore tables over a specific threshold size in MBs, default is 100000 MB ie 100 GB" + echo " -ittn: ignore tables under a specific threshold size in MBs, default is 10 MB" + echo " -mar: maximum asynchronous reorgs allowed, default is 3" + echo " -log: don't kick off a reorg if transaction log usage is over a certain percentage, default is 90%" + echo " -window: stop reorg tables/indexes/runstats after a set maintenance timeout window, default is 240 minutes" + echo " -twa: timeout window action: default=2 for online, 1 for offline" + echo " 1=allow current reorg(s) to continue" + echo " 2=stop current reorg(s)" +# echo " 3=stop current reorg(s) if < 80% complete and continue script" + echo " -ignore: ignore specific tables from SYSIBMADM.ADMINTABINFO t0, SYSCAT.TABLES t1 " + echo " eg \"$IGNORE_TABLES_EX\"" + echo " -reorg: table F1 F2 F3 filter reorg, default is *" + echo " -sleep: SLEEP_INTERVAL_TIME, default is 60 seconds" + echo "" + echo " -tr: execute inplace table/index reorg" + echo "" + echo " -trsi: Retrieve table reorganization snapshot information from snap_get_tab_reorg and db2pd -reorgs index" + echo "" + echo "Examples :" + echo " 1. ${0} -h" + echo " 2. ${0} -db dbname -s MAXIMO -ls" + echo " 3. ${0} -db dbname -s MAXIMO -t \"YFS_ITEM YFS_TASK_Q YFS_SHIPMENT\" -tb_stats -tr " + echo " 4. ${0} -db dbname -s MAXIMO -ti \"MAXIMO.YFS_SNAPSHOT.MAXIMO.YFS_SNAPSHOT_I1 MAXIMO.YFS_ITEM.MAXIMO.YFS_ITEM_PK\" -ix_stats -tr" + echo " 5. ${0} -db dbname -s MAXIMO -t \"YFS_ITEM YFS_SNAPSHOT YFS_IMPORT YFS_EXPORT\" -if_stats -tr" + echo " 6. ${0} -db dbname -s MAXIMO -tb_stats -mar 5 -window 10 -log 95 -ittx 30000 -tr" + echo " 7. ${0} -db dbname -s MAXIMO -tb_stats -mar 5 -window 10 -log 95 -ignore \"$IGNORE_TABLES_EX\" -reorg \"***\" -tr" + echo " 8. ${0} -db dbname -s MAXIMO -tb_stats -if_stats -ittx 100 -ittn 20 -tr" + echo " 9. ${0} -trsi" + + echo "" } @@ -73,133 +78,131 @@ UsageHelp() ## isNumeric() { - echo $1 | grep -E '^[0-9]+$' > /dev/null - - return $? + echo $1 | grep -E '^[0-9]+$' > /dev/null + return $? } TRSI() { - db2 -v "select varchar(tabschema,9) as tabschema, varchar(tabname,32) as tabname, - REORG_STATUS, REORG_COMPLETION, REORG_PHASE, REORG_CURRENT_COUNTER, REORG_MAX_COUNTER, --- varchar( varchar_format(REORG_START, 'YYYY-MM-DD HH24:MI:SS'),19) as REORG_START, --- varchar( varchar_format(REORG_END, 'YYYY-MM-DD HH24:MI:SS'),19) as REORG_END, - REORG_START, REORG_END, - REORG_INDEX_ID, REORG_TBSPC_ID - from table(snap_get_tab_reorg('')) - order by REORG_START asc - with ur" - - db2pd -db $DBNAME -reorgs index | sed -n "/Index Reorg Stats:/,//p" - + db2 -v "select varchar(tabschema,9) as tabschema, varchar(tabname,32) as tabname, + REORG_STATUS, REORG_COMPLETION, REORG_PHASE, REORG_CURRENT_COUNTER, REORG_MAX_COUNTER, + VARCHAR_FORMAT(REORG_START, 'YYYY-MM-DD-HH24:MI:SS') as REORG_START, + VARCHAR_FORMAT(REORG_END, 'YYYY-MM-DD-HH24:MI:SS') as REORG_END, +-- REORG_START, REORG_END, + REORG_INDEX_ID, REORG_TBSPC_ID + from table(snap_get_tab_reorg('')) + order by REORG_START asc + with ur" + + db2pd -db $DBNAME -reorgs index | sed -n "/Index Reorg Stats:/,//p" > /dev/null 2>&1 } log() { - TYPE=$1 - MSG="$2" - - DATE=$( date '+%d-%m-%Y %H:%M:%S' ); - - # TYPE: - # 0 = Critical - # 1 = Warn - # 3 = Info - # 5 = Debug' - if [ ${TYPE} -eq 0 ]; then - TYPEMSG="Error" - elif [ ${TYPE} -eq 1 ]; then - TYPEMSG="Warning" - elif [ ${TYPE} -eq 3 ]; then - TYPEMSG="Info" - elif [ ${TYPE} -eq 5 ]; then - TYPEMSG="Debug" - else - TYPEMSG="Other" - fi - - echo -e "${DATE} ${TYPEMSG}: ${MSG}" | tee -a $REORG_TABLE_INDEX_LOG - - return 0 + TYPE=$1 + MSG="$2" + + DATE=$( date '+%d-%m-%Y %H:%M:%S' ); + + # TYPE: + # 0 = Critical + # 1 = Warn + # 3 = Info + # 5 = Debug' + if [ ${TYPE} -eq 0 ]; then + TYPEMSG="Error" + elif [ ${TYPE} -eq 1 ]; then + TYPEMSG="Warning" + elif [ ${TYPE} -eq 3 ]; then + TYPEMSG="Info" + elif [ ${TYPE} -eq 5 ]; then + TYPEMSG="Debug" + else + TYPEMSG="Other" + fi + + echo -e "${DATE} ${TYPEMSG} : ${MSG}" | tee -a $REORG_TABLE_INDEX_LOG + + return 0 } initTABLE_IN_USE_ARRAY() { - local NUM_ITEMS=$1 - local jj; + local NUM_ITEMS=$1 + local jj; - ## - ## initialise the db2 TABLE_IN_USE_ARRAY - ## - for((jj=0; jj<$NUM_ITEMS; jj++)) - do - TABLE_IN_USE_ARRAY[$jj]="" - done + ## + ## initialise the db2 TABLE_IN_USE_ARRAY + ## + for((jj=0; jj<$NUM_ITEMS; jj++)) + do + TABLE_IN_USE_ARRAY[$jj]="" + done - return 0 + return 0 } existTABLE_TABLE_IN_USE_ARRAY() { - local TABLE=$1 - local jj; - ## - ## check if table is in use - ## - for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) - do - if [ "${TABLE_IN_USE_ARRAY[$jj]}" == "$TABLE" ]; then - return 0; - fi - done + local TABLE=$1 + local jj; + ## + ## check if table is in use + ## + for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) + do + if [ "${TABLE_IN_USE_ARRAY[$jj]}" == "$TABLE" ]; then + return 0; + fi + done - return 1; + return 1; } addTABLE_TABLE_IN_USE_ARRAY() { - local TABLE=$1 - local jj; - - ## - ## add table in empty slot - ## - for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) - do - if [ "${TABLE_IN_USE_ARRAY[$jj]}" == "" ]; then - TABLE_IN_USE_ARRAY[$jj]=$TABLE; - return 0; - fi - done - - return 1; + local TABLE=$1 + local jj; + + ## + ## add table in empty slot + ## + for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) + do + if [ "${TABLE_IN_USE_ARRAY[$jj]}" == "" ]; then + TABLE_IN_USE_ARRAY[$jj]=$TABLE; + return 0; + fi + done + + return 1; } removeTABLE_TABLE_IN_USE_ARRAY() { - local TABLE=$1 - local jj; - ## - ## remove entry - ## - for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) - do - if [ "${TABLE_IN_USE_ARRAY[$jj]}" == "$TABLE" ]; then - TABLE_IN_USE_ARRAY[$jj]=""; - return 0; - fi - done - - return 1; + local TABLE=$1 + local jj; + ## + ## remove entry + ## + for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) + do + if [ "${TABLE_IN_USE_ARRAY[$jj]}" == "$TABLE" ]; then + TABLE_IN_USE_ARRAY[$jj]=""; + return 0; + fi + done + + return 1; } @@ -207,195 +210,193 @@ removeTABLE_TABLE_IN_USE_ARRAY() listTABLE_IN_USE_ARRAY() { - local jj; - ## - ## list table entries - ## - for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) - do - log 5 "TABLE_IN_USE_ARRAY $jj ${TABLE_IN_USE_ARRAY[$jj]}"; - done + local jj; + ## + ## list table entries + ## + for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) + do + log 5 "TABLE_IN_USE_ARRAY $jj ${TABLE_IN_USE_ARRAY[$jj]}"; + done - return 0; + return 0; } getValidTablesToReorg() { - getValidTableSizes - - VALID_TABLES_TO_REORG="" - VALID_TABLES_TO_REORG_RAW="" - NUM_VALID_TABLES_TO_REORG=0 - for TABNAME in $VALID_TABLES - do - - RAW=$( db2 -x "call REORGCHK_TB_STATS('T','$SCHEMANAME_IN.$TABNAME')" ); - RAW=$( echo "$RAW" | grep $SCHEMANAME_IN | grep $TABNAME | awk '{ if (NF == 12) print $0 }' | sed 's/ \+/ /g' | grep $REORG ) - rc=$? - if [ $rc -eq 0 ]; then - TABNAME=$( echo "$RAW" | awk '{print $2}' ); - [ "$VALID_TABLES_TO_REORG_RAW_DATA" == "" ] && VALID_TABLES_TO_REORG_RAW_DATA="$RAW" || VALID_TABLES_TO_REORG_RAW_DATA="$VALID_TABLES_TO_REORG_RAW_DATA\n$RAW" - fi - - done - - ## sort the tables based on REORG column - VALID_TABLES_TO_REORG_RAW_DATA=$( echo -e "$VALID_TABLES_TO_REORG_RAW_DATA" | sort -k12 -r); - VALID_TABLES_TO_REORG=$( echo -e "$VALID_TABLES_TO_REORG_RAW_DATA" | awk '{print $2}' ); - NUM_VALID_TABLES_TO_REORG=$( echo -e "$VALID_TABLES_TO_REORG" | wc -l ); - - return 0 - + getValidTableSizes + + VALID_TABLES_TO_REORG="" + VALID_TABLES_TO_REORG_RAW="" + NUM_VALID_TABLES_TO_REORG=0 + for TABNAME in $VALID_TABLES + do + + RAW=$( db2 -x "call REORGCHK_TB_STATS('T','$SCHEMANAME_IN.$TABNAME')" ); + RAW=$( echo "$RAW" | grep $SCHEMANAME_IN | grep $TABNAME | awk '{ if (NF == 12) print $0 }' | sed 's/ \+/ /g' | grep $REORG ) + rc=$? + if [ $rc -eq 0 ]; then + TABNAME=$( echo "$RAW" | awk '{print $2}' ); + [ "$VALID_TABLES_TO_REORG_RAW_DATA" == "" ] && VALID_TABLES_TO_REORG_RAW_DATA="$RAW" || VALID_TABLES_TO_REORG_RAW_DATA="$VALID_TABLES_TO_REORG_RAW_DATA\n$RAW" + fi + + done + + ## sort the tables based on REORG column + VALID_TABLES_TO_REORG_RAW_DATA=$( echo -e "$VALID_TABLES_TO_REORG_RAW_DATA" | sort -k12 -r); + VALID_TABLES_TO_REORG=$( echo -e "$VALID_TABLES_TO_REORG_RAW_DATA" | awk '{print $2}' ); + NUM_VALID_TABLES_TO_REORG=$( echo -e "$VALID_TABLES_TO_REORG" | wc -l ); + + return 0 + } getValidIndexesToReorg() { - getValidTableSizes - - VALID_INDEXES_TO_REORG="" - VALID_INDEXES_TO_REORG_RAW="" - NUM_VALID_INDEXES_TO_REORG=0 - for TABNAME in $VALID_TABLES - do - - RAW=$( db2 -x "call REORGCHK_IX_STATS('T','$SCHEMANAME_IN.$TABNAME')" ); - ## this can return multiple indexes for same TABNAME - RAW=$( echo "$RAW" | grep $SCHEMANAME_IN | grep $TABNAME | awk '{ if (NF == 21) print $0 }' | sed 's/ \+/ /g' | grep $REORG ); - rc=$? - if [ $rc -eq 0 ]; then - INDNAME=$( echo "$RAW" | awk '{print $1"."$2"."$3"."$4}' ); - [ "$VALID_INDEXES_TO_REORG_RAW_DATA" == "" ] && VALID_INDEXES_TO_REORG_RAW_DATA=$RAW || VALID_INDEXES_TO_REORG_RAW_DATA="$VALID_INDEXES_TO_REORG_RAW_DATA\n$RAW" - fi - - done - - ## sort the indexes based on REORG column - VALID_INDEXES_TO_REORG_RAW_DATA=$( echo -e "$VALID_INDEXES_TO_REORG_RAW_DATA" | sort -k21 -r); - VALID_INDEXES_TO_REORG=$( echo -e "$VALID_INDEXES_TO_REORG_RAW_DATA" | awk '{print $1"."$2"."$3"."$4}' ); - NUM_VALID_INDEXES_TO_REORG=$( echo -e "$VALID_INDEXES_TO_REORG" | wc -l ); - - return 0 - + getValidTableSizes + + VALID_INDEXES_TO_REORG="" + VALID_INDEXES_TO_REORG_RAW="" + NUM_VALID_INDEXES_TO_REORG=0 + for TABNAME in $VALID_TABLES + do + + RAW=$( db2 -x "call REORGCHK_IX_STATS('T','$SCHEMANAME_IN.$TABNAME')" ); + ## this can return multiple indexes for same TABNAME + RAW=$( echo "$RAW" | grep $SCHEMANAME_IN | grep $TABNAME | awk '{ if (NF == 21) print $0 }' | sed 's/ \+/ /g' | grep $REORG ); + rc=$? + if [ $rc -eq 0 ]; then + INDNAME=$( echo "$RAW" | awk '{print $1"."$2"."$3"."$4}' ); + [ "$VALID_INDEXES_TO_REORG_RAW_DATA" == "" ] && VALID_INDEXES_TO_REORG_RAW_DATA=$RAW || VALID_INDEXES_TO_REORG_RAW_DATA="$VALID_INDEXES_TO_REORG_RAW_DATA\n$RAW" + fi + + done + + ## sort the indexes based on REORG column + VALID_INDEXES_TO_REORG_RAW_DATA=$( echo -e "$VALID_INDEXES_TO_REORG_RAW_DATA" | sort -k21 -r); + VALID_INDEXES_TO_REORG=$( echo -e "$VALID_INDEXES_TO_REORG_RAW_DATA" | awk '{print $1"."$2"."$3"."$4}' ); + NUM_VALID_INDEXES_TO_REORG=$( echo -e "$VALID_INDEXES_TO_REORG" | wc -l ); + + return 0 + } getValidFragmentedIndexes() { - ## - ## http://www.ibm.com/developerworks/data/library/techarticle/dm-1307optimizerunstats/#Listing%207 - ## - - getValidTableSizes - - VALID_FRAGMENTED_INDEXES_RAW_DATA=$( db2 -x "select rtrim(tabschema)||' '||rtrim(tabname)||' '||rtrim(indschema)||' '||rtrim(indname) - ||' '||indcard||' '||stats_time||' '||lastused||' '||nleaf||' '||sequential_pages - from syscat.indexes where tabschema='$SCHEMANAME_IN' - and not (nleaf = 1 and sequential_pages = 0) - and not (nleaf = 0 and sequential_pages = 1) - and (nleaf - sequential_pages > 10) - and tabname in ( $VALID_TABLES_FORMATTED ) - order by tabname - with ur"; ); - - VALID_FRAGMENTED_INDEXES_RAW_DATA=$(echo "${VALID_FRAGMENTED_INDEXES_RAW_DATA}" | sed 's/ *$//g' ); - VALID_FRAGMENTED_INDEXES=$( echo "${VALID_FRAGMENTED_INDEXES_RAW_DATA}" | sed 's/ *$//g' | cut -d' ' -f2 | uniq ) - NUM_VALID_FRAGMENTED_INDEXES=$( echo "${VALID_FRAGMENTED_INDEXES}" | wc -l ) - - + ## + ## http://www.ibm.com/developerworks/data/library/techarticle/dm-1307optimizerunstats/#Listing%207 + ## + + getValidTableSizes + + VALID_FRAGMENTED_INDEXES_RAW_DATA=$( db2 -x "select rtrim(tabschema)||' '||rtrim(tabname)||' '||rtrim(indschema)||' '||rtrim(indname) + ||' '||indcard||' '||stats_time||' '||lastused||' '||nleaf||' '||sequential_pages + from syscat.indexes where tabschema='$SCHEMANAME_IN' + and not (nleaf = 1 and sequential_pages = 0) + and not (nleaf = 0 and sequential_pages = 1) + and (nleaf - sequential_pages > 10) + and tabname in ( $VALID_TABLES_FORMATTED ) + order by tabname + with ur"; ); + + VALID_FRAGMENTED_INDEXES_RAW_DATA=$(echo "${VALID_FRAGMENTED_INDEXES_RAW_DATA}" | sed 's/ *$//g' ); + VALID_FRAGMENTED_INDEXES=$( echo "${VALID_FRAGMENTED_INDEXES_RAW_DATA}" | sed 's/ *$//g' | cut -d' ' -f2 | uniq ) + NUM_VALID_FRAGMENTED_INDEXES=$( echo "${VALID_FRAGMENTED_INDEXES}" | wc -l ) } getValidTableSizes() { - VALID_TABLE_SIZES_RAW_DATA=$( db2 "select t0.tabname, - ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) as TOTAL_TABLE_MB, - cast ((INDEX_OBJECT_P_SIZE / 1024) as integer) as INDEX_SIZE_MB - from SYSIBMADM.ADMINTABINFO t0, SYSCAT.TABLES t1 - where t0.tabschema='$SCHEMANAME_IN' - and t0.tabschema=t1.tabschema - and t0.tabname=t1.tabname - $IGNORE_TABLES - and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) < $IGNORE_TABLE_SIZE_THRESHOLD_MAX - and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) > $IGNORE_TABLE_SIZE_THRESHOLD_MIN - order by 2 desc - with ur"; ); - rc=$? - if [ $rc -eq 0 ]; then - VALID_TABLE_SIZES=$( echo "$VALID_TABLE_SIZES_RAW_DATA" | sed '1,3d' | sed '$d' | sed '$d' ); - VALID_TABLE_SIZES=$( echo "$VALID_TABLE_SIZES" | awk 'BEGIN {ORS="\t"} { for(ii=1 ; ii<=NF ; ii++) print $ii; printf "\n"; }'); - VALID_TABLES=$( echo "$VALID_TABLE_SIZES" | awk '{print $1}' ); - NUM_VALID_TABLES=$( echo "$VALID_TABLE_SIZES" | wc -l ); - # log 3 "NUM_VALID_TABLES=$NUM_VALID_TABLES" - - VALID_TABLES_FORMATTED="" - for TABLE in $VALID_TABLES - do - VALID_TABLES_FORMATTED="$VALID_TABLES_FORMATTED'$TABLE'," - done - VALID_TABLES_FORMATTED=$( echo "$VALID_TABLES_FORMATTED" | sed 's/,$//g' ) - else - VALID_TABLES_FORMATTED="'UNKNOWN_TABNAME'" - fi + VALID_TABLE_SIZES_RAW_DATA=$( db2 "select t0.tabname, + ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) as TOTAL_TABLE_MB, + cast ((INDEX_OBJECT_P_SIZE / 1024 ) as integer) as INDEX_SIZE_MB + from SYSIBMADM.ADMINTABINFO t0, SYSCAT.TABLES t1 + where t0.tabschema='$SCHEMANAME_IN' + and t0.tabschema=t1.tabschema + and t0.tabname=t1.tabname + $IGNORE_TABLES + and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) < $IGNORE_TABLE_SIZE_THRESHOLD_MAX + and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) > $IGNORE_TABLE_SIZE_THRESHOLD_MIN + order by 2 desc + with ur"; ); + rc=$? + if [ $rc -eq 0 ]; then + VALID_TABLE_SIZES=$( echo "$VALID_TABLE_SIZES_RAW_DATA" | sed '1,3d' | sed '$d' | sed '$d' ); + VALID_TABLE_SIZES=$( echo "$VALID_TABLE_SIZES" | awk 'BEGIN {ORS="\t"} { for(ii=1 ; ii<=NF ; ii++) print $ii; printf "\n"; }'); + VALID_TABLES=$( echo "$VALID_TABLE_SIZES" | awk '{print $1}' ); + NUM_VALID_TABLES=$( echo "$VALID_TABLE_SIZES" | wc -l ); + # log 3 "NUM_VALID_TABLES=$NUM_VALID_TABLES" + + VALID_TABLES_FORMATTED="" + for TABLE in $VALID_TABLES + do + VALID_TABLES_FORMATTED="$VALID_TABLES_FORMATTED'$TABLE'," + done + VALID_TABLES_FORMATTED=$( echo "$VALID_TABLES_FORMATTED" | sed 's/,$//g' ) + else + VALID_TABLES_FORMATTED="'UNKNOWN_TABNAME'" + fi } ## is TABLE within size limits < IGNORE_TABLE_SIZE_THRESHOLD_MAX and > IGNORE_TABLE_SIZE_THRESHOLD_MIN isTableWithinSizeLimit() { - local SCHEMANAME=$1 - local TABNAME=$2 - local RC="" - local rc=0 - - RC=$( db2 -x "select tabname, - ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) as TOTAL_TABLE_MB - from SYSIBMADM.ADMINTABINFO - where tabschema='$SCHEMANAME' - and tabname = '$TABNAME' - and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) < $IGNORE_TABLE_SIZE_THRESHOLD_MAX - and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) > $IGNORE_TABLE_SIZE_THRESHOLD_MIN - order by 2 desc - with ur" ); - - rc=$? - return $rc + local SCHEMANAME=$1 + local TABNAME=$2 + local RC="" + local rc=0 + + RC=$( db2 -x "select tabname, + ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) as TOTAL_TABLE_MB + from SYSIBMADM.ADMINTABINFO + where tabschema='$SCHEMANAME' + and tabname = '$TABNAME' + and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) < $IGNORE_TABLE_SIZE_THRESHOLD_MAX + and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) > $IGNORE_TABLE_SIZE_THRESHOLD_MIN + order by 2 desc + with ur" ); + + rc=$? + return $rc } ## create an list/array of table objects to reorg based on tabnames createTableOBJECT_ARRAY() { - local TABNAMES="$1" - local OBJECT_REORG_TABLE_TYPE=$2 - - ## - ## make the OBJECT_ARRAY for tables and indexes - ## - if [ -z "$OBJECT_ARRAY" ]; then - local let index=0; - else - local let index=${#OBJECT_ARRAY[@]}; - fi - local TID=0 ## Table ID - always 0 for online table reorg - local INDSCHEMA=NULL; - local INDNAME=NULL; - local LOCK_COUNT=0 - - for TABNAME in $TABNAMES - do - ## we need TABLEID, TBSPACEID for IF_STATS as the full TableName: may not be dispalyed in the db2pd output - local RC=$( db2 -x "select TABLEID, TBSPACEID from syscat.tables where tabname='$TABNAME' and tabschema='$SCHEMANAME_IN'" ) - local rc=$? - if [ $rc -eq 0 ]; then - local TABLEID=$( echo $RC | awk '{print $1}' ); - local TBSPACEID=$( echo $RC | awk '{print $2}' ); - OBJECT_ARRAY[$index]="$SCHEMANAME_IN#$TABNAME#$INDSCHEMA#$INDNAME#$TID#NOTSTARTED#2019-01-01-00.00.00#2019-01-01-00.00.00#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE" - let index+=1 - fi - done + local TABNAMES="$1" + local OBJECT_REORG_TABLE_TYPE=$2 + + ## + ## make the OBJECT_ARRAY for tables and indexes + ## + if [ -z "$OBJECT_ARRAY" ]; then + local let index=0; + else + local let index=${#OBJECT_ARRAY[@]}; + fi + local TID=0 ## Table ID - always 0 for online table reorg + local INDSCHEMA=NULL; + local INDNAME=NULL; + local LOCK_COUNT=0 + + for TABNAME in $TABNAMES + do + ## we need TABLEID, TBSPACEID for IF_STATS as the full TableName: may not be dispalyed in the db2pd output + local RC=$( db2 -x "select TABLEID, TBSPACEID from syscat.tables where tabname='$TABNAME' and tabschema='$SCHEMANAME_IN'" ) + local rc=$? + if [ $rc -eq 0 ]; then + local TABLEID=$( echo $RC | awk '{print $1}' ); + local TBSPACEID=$( echo $RC | awk '{print $2}' ); + OBJECT_ARRAY[$index]="$SCHEMANAME_IN#$TABNAME#$INDSCHEMA#$INDNAME#$TID#NOTSTARTED#-#-#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE" + let index+=1 + fi + done } @@ -403,60 +404,66 @@ createTableOBJECT_ARRAY() createIndexOBJECT_ARRAY() { - local INDNAMES="$1" - local OBJECT_REORG_TABLE_TYPE=$2 - local let index=0 - local TABLEID=9999; - local TBSPACEID=9999; - local LOCK_COUNT=0; - - for INDEX in $INDNAMES - do - local TABSCHEMA=$( echo $INDEX | cut -d. -f1); - local TABNAME=$( echo $INDEX | cut -d. -f2); - local INDSCHEMA=$( echo $INDEX | cut -d. -f3); - local INDNAME=$( echo $INDEX | cut -d. -f4); - local RC=$( db2 -x "select IID from syscat.indexes where tabschema = '$TABSCHEMA' and tabname = '$TABNAME' and indschema = '$INDSCHEMA' and indname = '$INDNAME'"); - local rc=$? - if [ $rc -eq 0 ]; then - local IID=$( echo $RC | cut -d' ' -f1); - OBJECT_ARRAY[$index]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#NOTSTARTED#2019-01-01-00.00.00#2019-01-01-00.00.00#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE" - let index+=1 - fi - done + local INDNAMES="$1" + local OBJECT_REORG_TABLE_TYPE=$2 + local let index=0 + local TABLEID=9999; + local TBSPACEID=9999; + local LOCK_COUNT=0; + + for INDEX in $INDNAMES + do + local TABSCHEMA=$( echo $INDEX | cut -d. -f1); + local TABNAME=$( echo $INDEX | cut -d. -f2); + local INDSCHEMA=$( echo $INDEX | cut -d. -f3); + local INDNAME=$( echo $INDEX | cut -d. -f4); + local RC=$( db2 -x "select IID from syscat.indexes where tabschema = '$TABSCHEMA' and tabname = '$TABNAME' and indschema = '$INDSCHEMA' and indname = '$INDNAME'"); + local rc=$? + if [ $rc -eq 0 ]; then + local IID=$( echo $RC | cut -d' ' -f1); + OBJECT_ARRAY[$index]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#NOTSTARTED#-#-#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE" + let index+=1 + fi + done } - ## ## list out the objects and state ## this can be used for debugging ## listOBJECT_ARRAY() { - - local ii; - log 3 "The following is for debug purposes, Num objects=${#OBJECT_ARRAY[@]}, $OBJECT_NUM_TB_STATS:$OBJECT_NUM_IX_STATS:$OBJECT_NUM_IF_STATS" - for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) - do - echo "${OBJECT_ARRAY[$ii]}" | tee -a $REORG_TABLE_INDEX_DEBUG - done - + + local ii; + log 3 "The following is for debug purposes, Num objects=${#OBJECT_ARRAY[@]}, $OBJECT_NUM_TB_STATS:$OBJECT_NUM_IX_STATS:$OBJECT_NUM_IF_STATS" + printf "%-15s %-40s %-15s %-40s %-14s %-20s %-20s %-8s \n" "TABSCHEMA" "TABNAME" "INDSCHEMA" "INDNAME" "REORG_STATUS" "REORG_START" "REORG_END" "LOCK_COUNT" + : ' for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) + do + echo "${OBJECT_ARRAY[$ii]}" | /usr/bin/column -t -s "#" | tee -a $REORG_TABLE_INDEX_DEBUG + done + ' + for ((ii=0; ii<${#OBJECT_ARRAY[@]}; ii++)) + do + IFS="#" read -r TABSCHEMA TABNAME INDSCHEMA INDNAME IID REORG_STATUS OBJECT_REORG_START OBJECT_REORG_END TABLEID TBSPACEID LOCK_COUNT <<< "${OBJECT_ARRAY[$ii]}" + + printf "%-15s %-40s %-15s %-40s %-14s %-20s %-20s %-8s \n" "$TABSCHEMA" "$TABNAME" "$INDSCHEMA" "$INDNAME" "$REORG_STATUS" "$OBJECT_REORG_START" "$OBJECT_REORG_END" "$LOCK_COUNT" + done } ## get the number tb_stats, ix_stats and if_stats objects getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY() { - local rc=0; - local ii; - for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) - do - if [ $( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f12 ) -eq $1 ]; then - let rc+=1; - fi - done + local rc=0; + local ii; + for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) + do + if [ $( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f12 ) -eq $1 ]; then + let rc+=1; + fi + done - return $rc; + return $rc; } @@ -466,379 +473,382 @@ getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY() reorgTables() { - ## variables that need to be reset on each run of the function - NUM_REORGS_IN_PROGRESS=0; - NUM_REORGS_KICKED_OFF=0; - NUM_REORGS_COMPLETED=0; - NUM_REORGS_STOPPED=0; - NUM_REORGS_ABORTED=0; - - while true - do - - ## check reorg window maintenance time - MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS=$( date '+%s' ); - DIFF=$(( MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS - REORG_TIMEOUT_WINDOW_START_TIME_SECONDS )); - - ## safety valve - if for some reason the logic can't stop the reorgs - if [ $DIFF -ge $(( REORG_TIMEOUT_WINDOW_SECONDS + REORG_TIMEOUT_OVERFLOW_VALVE )) ]; then - log 1 "REORG_TIMEOUT_OVERFLOW_VALVE detected"; - log 1 "Aborting reorgs" - break; - fi - - if [ $DIFF -ge $REORG_TIMEOUT_WINDOW_SECONDS ]; then - - REORG_TIMEOUT_WINDOW_COMPLETED=1; - - ## -twa: timeout window action: default=3 - ## 1=allow current reorg(s) to continue - ## 2=stop current reorg(s) - ## 3=stop current reorg(s) if < 80% complete - if [ $REORG_TIMEOUT_WINDOW_ACTION -eq 1 ]; then - - log 3 "Reorg window ending, reorg window time exceeded, twa=$REORG_TIMEOUT_WINDOW_ACTION" - break - - elif [ $REORG_TIMEOUT_WINDOW_ACTION -eq 2 ]; then - - ## use of the REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER - ## 0 = ABORT those not started and issue a STOP to those STARTED - ## 1 = loop again and see if script exits as all reorgs are COMPLETED and STOPPED and ABORTED - ## 2 = break out - if [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 0 ]; then - let REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER+=1 - log 3 "Reorg window ending, reorg window time exceeded, twa=$REORG_TIMEOUT_WINDOW_ACTION" - log 3 "Aborting reorgs NOTSTARTED and issuing a STOP to those that are STARTED" - elif [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 1 ]; then - let REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER+=1 - log 3 "REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER=$REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER" - elif [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 2 ]; then - log 3 "REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER=$REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER" - log 3 "breaking out of reorg loop" - break; - fi - - fi - - fi - - ## loop for all OBJECTS - extract relevant data from OBJECT array - ## if we have NOTSTARTED in the OBJECT_ARRAY[N] - then kick off a reorg - ## then check tables reorg status - for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) - do - ## get table related info - TABSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f1 ); - TABNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f2 ); - INDSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f3 ); - INDNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f4 ); - IID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f5 ); - OBJECT_REORG_STATUS=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f6 ); - OBJECT_REORG_START=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f7 ); - OBJECT_REORG_END=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f8 ); - TABLEID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f9 ); - TBSPACEID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f10 ); - LOCK_COUNT=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f11 ); - OBJECT_REORG_TABLE_TYPE=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f12 ); - isTable=0; - isIndex=0; - if [ "$INDSCHEMA" == "NULL" -a "$INDNAME" == "NULL" ]; then - isTable=1; - else - isIndex=1; - fi - - if [ $OBJECT_REORG_TABLE_TYPE -ne $REORG_TABLE_TYPE ]; then - continue; - fi - - # log 5 "${OBJECT_ARRAY[$ii]}" - - ## - ## has OBJECT COMPLETED - no need to continue here - ## - if [ "$OBJECT_REORG_STATUS" == "COMPLETED" -o "$OBJECT_REORG_STATUS" == "STOPPED" -o "$OBJECT_REORG_STATUS" == "ABORTED" ]; then - continue; - fi - - if [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then - - ## - ## query db2 for REORG_STATUS etc - there may not be an entry so carry on - ## - RC_SNAP=$( db2 -x "select REORG_STATUS, REORG_COMPLETION, REORG_PHASE, REORG_CURRENT_COUNTER, REORG_MAX_COUNTER, REORG_START, REORG_END, REORG_INDEX_ID, REORG_TBSPC_ID from table(snap_get_tab_reorg('')) where tabschema='$TABSCHEMA' and tabname='$TABNAME' and REORG_START > TIMESTAMP('$WINDOW_START_TIME_DB2') and REORG_INDEX_ID=$IID"); - rc=$? - if [ $rc -ge 2 ]; then - log 0 "Possible error running select query against db2\nrc=$rc\nRC=$RC" - continue; - fi - - if [ $rc -eq 0 ]; then - RC=$( echo "$RC_SNAP" | awk 'BEGIN {ORS="\t"} { for(ii=1 ; ii<=NF ; ii++) print $ii; }') - REORG_STATUS=$( echo "$RC_SNAP" | awk '{print $1}' ); - REORG_COMPLETION=$( echo "$RC_SNAP" | awk '{print $2}' ); - REORG_CURRENT_COUNTER=$( echo "$RC_SNAP" | awk '{print $4}' ); - REORG_MAX_COUNTER=$( echo "$RC_SNAP" | awk '{print $5}' ); - REORG_START=$( echo "$RC_SNAP" | awk '{print $6}' ); - REORG_END=$( echo "$RC_SNAP" | awk '{print $7}' ); - REORG_INDEX_ID=$( echo "$RC_SNAP" | awk '{print $8}' ); - - REORG_PERCENT_COMPLETE=0 - if [ ! -z "$REORG_CURRENT_COUNTER" -a $REORG_CURRENT_COUNTER -gt 0 ]; then - if [ ! -z "$REORG_MAX_COUNTER" -a $REORG_MAX_COUNTER -gt 0 ]; then - if [ $REORG_MAX_COUNTER -ge $REORG_CURRENT_COUNTER ]; then - REORG_PERCENT_COMPLETE=$( echo $REORG_CURRENT_COUNTER $REORG_MAX_COUNTER | awk '{ print int (($1/$2)*100) }' ); - fi - fi - fi - fi - - # log 5 "RC_SNAP=$RC_SNAP" - - elif [ $IF_STATS -eq 3 ]; then - - DB2PD_REORG_INDEX_RECORD=$( db2pd -db $DBNAME -reorgs index | grep -B1 -A11 -w "^TbspaceID: $TBSPACEID" | grep -B1 -A11 -w "TableID: $TABLEID" ); - rc=$? - if [ $rc -eq 0 ]; then - REORG_START=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep '^Start Time:' | awk '{ print $3,$4}' ); - REORG_START_SECONDS=$(date --d="$REORG_START" '+%s'); - - if [ $REORG_START_SECONDS -ge $IF_STATS_WINDOW_START_TIME_DB2 ]; then - REORG_STATUS=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep '^Status:' | awk '{ $1=""; print $0}' | sed 's/^[ \t]*//;s/[ \t]*$//' ); - ## some differences between snap_get_tab_reorg and db2pd -reogrs index output - if [ "$REORG_STATUS" == "In Progress" ]; then - REORG_STATUS="STARTED"; - elif [ "$REORG_STATUS" == "Completed" ]; then - REORG_STATUS="COMPLETED"; - elif [ "$REORG_STATUS" == "Stopped" ]; then - REORG_STATUS="STOPPED"; - fi - - REORG_END=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep '^Start Time:' | grep 'End Time:' | awk '{ print $7,$8}' ); - fi - - fi - - - fi - - - ## - ## has OBJECT been KICKED_OFF or STARTED - ## if it has check to see if is STARTED or COMPLETED - ## update OBJECT_ARRAY - ## update COMPLETION/STOPPED stats - ## - if [ "$OBJECT_REORG_STATUS" == "KICKED_OFF" ] || [ "$OBJECT_REORG_STATUS" == "STARTED" ]; then - if [ ! -z "$REORG_STATUS" ]; then - if [ "$REORG_STATUS" == "STARTED" -o "$REORG_STATUS" == "COMPLETED" -o "$REORG_STATUS" == "STOPPED" ]; then - OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$REORG_STATUS#$REORG_START#$REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE" - fi - - if [ "$REORG_STATUS" == "COMPLETED" -o "$REORG_STATUS" == "STOPPED" ]; then - - removeTABLE_TABLE_IN_USE_ARRAY "$TABSCHEMA.$TABNAME" - rc=$? - if [ $rc -eq 1 ]; then - log 1 "Failed to remove table $TABSCHEMA.$TABNAME from TABLE_IN_USE_ARRAY"; - fi - - if [ "$REORG_STATUS" == "COMPLETED" ]; then - let NUM_REORGS_COMPLETED+=1 - elif [ "$REORG_STATUS" == "STOPPED" ]; then - let NUM_REORGS_STOPPED+=1 - fi - - let NUM_REORGS_IN_PROGRESS-=1 - fi - - fi - - ## - ## OBJECT is NOSTARTED so KICK_OFF a reorg - ## - elif [ "$OBJECT_REORG_STATUS" == "NOTSTARTED" ]; then - - ## dont kick off any reorgs if window timeout passed and TWA=2 - ## OBJECTS become ABORTED - UPDATE OBJECT_ARRAY - if [ $REORG_TIMEOUT_WINDOW_COMPLETED -eq 1 ] && [ $REORG_TIMEOUT_WINDOW_ACTION -eq 2 ] && [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 1 ]; then - REORG_STATUS=ABORTED; - OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$REORG_STATUS#$OBJECT_REORG_START#$OBJECT_REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE"; - let NUM_REORGS_ABORTED+=1; - continue; - - fi - - ## is the TABLE already being used -if it is goto next OBJECT - existTABLE_TABLE_IN_USE_ARRAY "$TABSCHEMA.$TABNAME" - rc=$? - [ $rc -eq 0 ] && continue; - - ## we only want to kick off so many reorgs at any one time - if [ $NUM_REORGS_IN_PROGRESS -eq $MAX_ASYNC_REORGS_ALLOWED ]; then - continue; - fi - - ## - ## The table could already be locked - if it is then by-pass it - ## and ABORT if locked more than 10 times - ## - TABLE_LOCKED=$( db2 "select APPLICATION_HANDLE, LOCK_OBJECT_TYPE, LOCK_MODE, LOCK_CURRENT_MODE, LOCK_STATUS, LOCK_COUNT, LOCK_HOLD_COUNT, TBSP_ID, TAB_FILE_ID from TABLE (MON_GET_LOCKS(NULL, -2)) where TBSP_ID=$TBSPACEID and TAB_FILE_ID=$TABLEID and LOCK_OBJECT_TYPE='TABLE' and LOCK_MODE='IX' with ur"; ); - rc=$? - if [ $rc -eq 0 ]; then - let LOCK_COUNT+=1; - log 1 "Appears table $TABSCHEMA.$TABNAME is already locked by another application(s), LOCK_COUNT=$LOCK_COUNT"; - log 1 "$TABLE_LOCKED"; - if [ $LOCK_COUNT -gt 10 ]; then - OBJECT_REORG_STATUS=ABORTED; - let NUM_REORGS_ABORTED+=1; - log 1 "Aborting table $TABSCHEMA.$TABNAME , LOCK_COUNT=$LOCK_COUNT"; - fi -OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$OBJECT_REORG_STATUS#$OBJECT_REORG_START#$OBJECT_REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE"; - continue; - fi - - ## - ## do a check to see where we are on transaction log space - this could be improved!!!! - ## - LOG_USED=$( db2 "select cast(LOG_UTILIZATION_PERCENT as decimal(5,2)) as PCTUSED, cast((TOTAL_LOG_USED_KB/1024) as Integer) as TOTUSEDMB, cast((TOTAL_LOG_AVAILABLE_KB/1024) as Integer) as TOTAVAILMB, cast((TOTAL_LOG_USED_TOP_KB/1024) as Integer) as TOTUSEDTOPMB FROM SYSIBMADM.LOG_UTILIZATION "); - if [ ! -z "$LOG_USED" ]; then - PCTUSED=$( echo "$LOG_USED" | awk '{ if(NF==4 && $2 ~/^[0-9]+$/) print int($1)}' ); - if [ ! -z "$PCTUSED" ]; then - if [ $PCTUSED -gt $TRANSACTION_LOG_THRESHOLD_PCT ]; then - log 1 "Will not kick off another reorg due to logfile PCTUSED above threshold of $LOG_THRESHOLD\n$LOG_USED" - continue - else - log 3 "$LOG_USED" - fi - fi - fi - - ## - ## kick off another reorg - ## if rc=0 then ok, else we ABORT the OBJECT and don't try again - ## - log 3 "" - - if [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then - - if [ $isTable -eq 1 ]; then - - db2 -v "reorg table $TABSCHEMA.$TABNAME inplace allow write access" - rc=$? - - elif [ $isIndex -eq 1 ]; then - db2 -v "reorg table $TABSCHEMA.$TABNAME index $INDSCHEMA.$INDNAME inplace allow write access" - rc=$? - fi - - elif [ $IF_STATS -eq 3 ]; then - - ## for offline we throw a job at db2 and wait a few seconds and check the output - ## output could be "reorg indexes all for table ...", - ## or SQL error - ## or 'DB20000I The REORG command completed successfully.' - ## not sure if there is a better way to do this - TMPLOG="/tmp/$TABSCHEMA.$TABNAME.tmp"; - db2 -v "reorg indexes all for table $TABSCHEMA.$TABNAME allow write access" > $TMPLOG 2>&1 & - sleep 5; - cat $TMPLOG; - RC=$( grep '^SQL' $TMPLOG); - rc=$? - if [ $rc -eq 0 ]; then - log 1 "Failed to kick off reorg\n$RC"; - rc=1; - else - ## reorg could have finished then no need for the big sleep - RC=$( grep 'DB20000I The REORG command completed successfully.' $TMPLOG); - if [ $? -eq 0 ]; then - IF_STATS_BYPASS_SLEEP_INTERVAL_TIME=1; - fi - rc=0; - - fi - rm -f $TMPLOG; - - fi - - if [ $rc -eq 0 ]; then - REORG_STATUS=KICKED_OFF; - else - REORG_STATUS=ABORTED; - fi - OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$REORG_STATUS#$OBJECT_REORG_START#$OBJECT_REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE"; - if [ $rc -ne 0 ]; then - let NUM_REORGS_ABORTED+=1; - continue - fi - - ## add the table to the TABLE_IN_USE_ARRAY - addTABLE_TABLE_IN_USE_ARRAY "$TABSCHEMA.$TABNAME" - rc=$? - if [ $rc -eq 1 ]; then - log 1 "Failed to add table $TABSCHEMA.$TABNAME to TABLE_IN_USE_ARRAY"; - fi - let NUM_REORGS_KICKED_OFF+=1; - let NUM_REORGS_IN_PROGRESS+=1; - - continue; - - fi - - ## - ## ouput STATUS - ## - if [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then - log 3 "$NUM_REORGS_IN_PROGRESS:$NUM_REORGS_KICKED_OFF:$NUM_REORGS_ABORTED:$NUM_REORGS_STOPPED:$NUM_REORGS_COMPLETED:$NUM_REORG_OBJECTS $TABSCHEMA.$TABNAME : $RC : $REORG_PERCENT_COMPLETE %" | tee -a $REORG_TABLE_INDEX_DEBUG - elif [ $IF_STATS -eq 3 ]; then - log 3 "$NUM_REORGS_IN_PROGRESS:$NUM_REORGS_KICKED_OFF:$NUM_REORGS_ABORTED:$NUM_REORGS_STOPPED:$NUM_REORGS_COMPLETED:$NUM_REORG_OBJECTS $TABSCHEMA.$TABNAME \n$DB2PD_REORG_INDEX_RECORD "| tee -a $REORG_TABLE_INDEX_DEBUG - - fi - - ## - ## if reorg timeout then issue a stop to current reorgs that are STARTED - ## no error checking for stopping a reorg - ## no need to update OBJECT array as it will be updated on next loop - ## - - if [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then - - if [ "$REORG_STATUS" == "STARTED" ] && [ $REORG_TIMEOUT_WINDOW_COMPLETED -eq 1 ] && [ $REORG_TIMEOUT_WINDOW_ACTION -eq 2 ] && [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 1 ]; then - - - if [ $isTable -eq 1 ]; then - db2 -v "reorg table $TABSCHEMA.$TABNAME inplace stop" - rc=$? - elif [ $isIndex -eq 1 ]; then - db2 -v "reorg table $TABSCHEMA.$TABNAME index $INDSCHEMA.$INDNAME inplace stop" - rc=$? - fi - - fi - - fi - - done ## for OBJECT_ARRAY[@] - - ## check if we are done with all OBJECTS - if [ $((NUM_REORGS_COMPLETED + NUM_REORGS_STOPPED + NUM_REORGS_ABORTED)) -ge $NUM_REORG_OBJECTS ]; then - log 3 "All reorgs are completed, stopped or aborted, $NUM_REORGS_COMPLETED:$NUM_REORGS_STOPPED:$NUM_REORGS_ABORTED:$NUM_REORG_OBJECTS" - break - fi - - ## wait some time - if [ $IF_STATS -eq 3 ] && [ $IF_STATS_BYPASS_SLEEP_INTERVAL_TIME -eq 1 ]; then - sleep 1; - IF_STATS_BYPASS_SLEEP_INTERVAL_TIME=0; - else - sleep $SLEEP_INTERVAL_TIME - fi - - done ## while true + ## variables that need to be reset on each run of the function + NUM_REORGS_IN_PROGRESS=0; + NUM_REORGS_KICKED_OFF=0; + NUM_REORGS_COMPLETED=0; + NUM_REORGS_STOPPED=0; + NUM_REORGS_ABORTED=0; + + while true + do + + ## check reorg window maintenance time + MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS=$( date '+%s' ); + DIFF=$(( MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS - REORG_TIMEOUT_WINDOW_START_TIME_SECONDS )); + + ## safety valve - if for some reason the logic can't stop the reorgs + if [ $DIFF -ge $(( REORG_TIMEOUT_WINDOW_SECONDS + REORG_TIMEOUT_OVERFLOW_VALVE )) ]; then + log 1 "REORG_TIMEOUT_OVERFLOW_VALVE detected"; + log 1 "Aborting reorgs" + break; + fi + + if [ $DIFF -ge $REORG_TIMEOUT_WINDOW_SECONDS ]; then + + REORG_TIMEOUT_WINDOW_COMPLETED=1; + + ## -twa: timeout window action: default=3 + ## 1=allow current reorg(s) to continue + ## 2=stop current reorg(s) + ## 3=stop current reorg(s) if < 80% complete + if [ $REORG_TIMEOUT_WINDOW_ACTION -eq 1 ]; then + + log 3 "Reorg window ending, reorg window time exceeded, twa=$REORG_TIMEOUT_WINDOW_ACTION" + break + + elif [ $REORG_TIMEOUT_WINDOW_ACTION -eq 2 ]; then + + ## use of the REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER + ## 0 = ABORT those not started and issue a STOP to those STARTED + ## 1 = loop again and see if script exits as all reorgs are COMPLETED and STOPPED and ABORTED + ## 2 = break out + if [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 0 ]; then + let REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER+=1 + log 3 "Reorg window ending, reorg window time exceeded, twa=$REORG_TIMEOUT_WINDOW_ACTION" + log 3 "Aborting reorgs NOTSTARTED and issuing a STOP to those that are STARTED" + elif [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 1 ]; then + let REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER+=1 + log 3 "REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER=$REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER" + elif [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 2 ]; then + log 3 "REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER=$REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER" + log 3 "Breaking out of reorg loop" + break; + fi + fi + fi + + ## loop for all OBJECTS - extract relevant data from OBJECT array + ## if we have NOTSTARTED in the OBJECT_ARRAY[N] - then kick off a reorg + ## then check tables reorg status + for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) + do + ## get table related info + TABSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f1 ); + TABNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f2 ); + INDSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f3 ); + INDNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f4 ); + IID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f5 ); + OBJECT_REORG_STATUS=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f6 ); + OBJECT_REORG_START=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f7 ); + OBJECT_REORG_END=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f8 ); + TABLEID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f9 ); + TBSPACEID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f10 ); + LOCK_COUNT=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f11 ); + OBJECT_REORG_TABLE_TYPE=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f12 ); + isTable=0; + isIndex=0; + if [ "$INDSCHEMA" == "NULL" -a "$INDNAME" == "NULL" ]; then + isTable=1; + else + isIndex=1; + fi + + if [ $OBJECT_REORG_TABLE_TYPE -ne $REORG_TABLE_TYPE ]; then + continue; + fi + + # log 5 "${OBJECT_ARRAY[$ii]}" + + ## + ## has OBJECT COMPLETED - no need to continue here + ## + if [[ "$OBJECT_REORG_STATUS" == "COMPLETED" || "$OBJECT_REORG_STATUS" == "STOPPED" || "$OBJECT_REORG_STATUS" == "ABORTED" ]]; then + continue; + fi + + if [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then + + ## + ## query db2 for REORG_STATUS etc - there may not be an entry so carry on + ## + RC_SNAP=$( db2 -x "select REORG_STATUS, REORG_COMPLETION, REORG_PHASE, REORG_CURRENT_COUNTER, REORG_MAX_COUNTER, VARCHAR_FORMAT(REORG_START, 'YYYY-MM-DD-HH24:MI:SS') as REORG_START, VARCHAR_FORMAT(REORG_END, 'YYYY-MM-DD-HH24:MI:SS') as REORG_END, REORG_INDEX_ID, REORG_TBSPC_ID from table(snap_get_tab_reorg('')) where tabschema='$TABSCHEMA' and tabname='$TABNAME' and REORG_START > TIMESTAMP('$WINDOW_START_TIME_DB2') and REORG_INDEX_ID=$IID"); + rc=$? + if [ $rc -ge 2 ]; then + log 0 "Possible error running select query against db2\nrc=$rc\nRC=$RC" + continue; + fi + + if [ $rc -eq 0 ]; then + RC=$( echo "$RC_SNAP" | awk 'BEGIN {ORS="\t"} { for(ii=1 ; ii<=NF ; ii++) print $ii; }') + REORG_STATUS=$( echo "$RC_SNAP" | awk '{print $1}' ); + REORG_COMPLETION=$( echo "$RC_SNAP" | awk '{print $2}' ); + REORG_CURRENT_COUNTER=$( echo "$RC_SNAP" | awk '{print $4}' ); + REORG_MAX_COUNTER=$( echo "$RC_SNAP" | awk '{print $5}' ); + REORG_START=$( echo "$RC_SNAP" | awk '{print $6}' ); + REORG_END=$( echo "$RC_SNAP" | awk '{print $7}' ); + REORG_INDEX_ID=$( echo "$RC_SNAP" | awk '{print $8}' ); + + declare -i REORG_PERCENT_COMPLETE=0 + if [[ ! -z "$REORG_CURRENT_COUNTER" && $REORG_CURRENT_COUNTER -gt 0 ]]; then + if [[ ! -z "$REORG_MAX_COUNTER" && $REORG_MAX_COUNTER -gt 0 ]]; then + if [ $REORG_MAX_COUNTER -ge $REORG_CURRENT_COUNTER ]; then + REORG_PERCENT_COMPLETE=$(echo $REORG_CURRENT_COUNTER $REORG_MAX_COUNTER | awk '{ print int (($1/$2)*100) }' ); + fi + fi + fi + fi + + # log 5 "RC_SNAP=$RC_SNAP" + + elif [ $IF_STATS -eq 3 ]; then + + DB2PD_REORG_INDEX_RECORD=$( db2pd -db $DBNAME -reorgs index | grep -B1 -A11 -w "^TbspaceID: $TBSPACEID" | grep -B1 -A11 -w "TableID: $TABLEID" ); + rc=$? + if [ $rc -eq 0 ]; then + REORG_START=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep '^Start Time:' | awk '{ print $3,$4}' ); + REORG_START_SECONDS=$(date --date="$REORG_START" '+%s'); + + if [ $REORG_START_SECONDS -ge $IF_STATS_WINDOW_START_TIME_DB2 ]; then + REORG_STATUS=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep '^Status:' | awk '{ $1=""; print $0}' | sed 's/^[ \t]*//;s/[ \t]*$//' ); + ## some differences between snap_get_tab_reorg and db2pd -reogrs index output + if [ "$REORG_STATUS" == "In Progress" ]; then + REORG_STATUS="STARTED"; + elif [ "$REORG_STATUS" == "Completed" ]; then + REORG_STATUS="COMPLETED"; + elif [ "$REORG_STATUS" == "Stopped" ]; then + REORG_STATUS="STOPPED"; + fi + + REORG_END=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep '^Start Time:' | grep 'End Time:' | awk '{ print $7,$8}' ); + REORG_IND_CUR_STATUS=$( echo "$DB2PD_REORG_INDEX_RECORD" | grep "Status:" ); + REORG_IND_CUR=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep "Cur Index:" ); + REORG_INDEX_STATUS=$(printf "%-25s %-10s" "$REORG_IND_CUR_STATUS" "$REORG_IND_CUR") + fi + fi + fi + + ## + ## has OBJECT been KICKED_OFF or STARTED + ## if it has check to see if is STARTED or COMPLETED + ## update OBJECT_ARRAY + ## update COMPLETION/STOPPED stats + ## + if [ "$OBJECT_REORG_STATUS" == "KICKED_OFF" ] || [ "$OBJECT_REORG_STATUS" == "STARTED" ]; then + if [ ! -z "$REORG_STATUS" ]; then + if [ "$REORG_STATUS" == "STARTED" -o "$REORG_STATUS" == "COMPLETED" -o "$REORG_STATUS" == "STOPPED" ]; then + OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$REORG_STATUS#$REORG_START#$REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE" + fi + + if [ "$REORG_STATUS" == "COMPLETED" -o "$REORG_STATUS" == "STOPPED" ]; then + + removeTABLE_TABLE_IN_USE_ARRAY "$TABSCHEMA.$TABNAME" + rc=$? + if [ $rc -eq 1 ]; then + log 1 "Failed to remove table $TABSCHEMA.$TABNAME from TABLE_IN_USE_ARRAY"; + fi + + if [ "$REORG_STATUS" == "COMPLETED" ]; then + let NUM_REORGS_COMPLETED+=1 + elif [ "$REORG_STATUS" == "STOPPED" ]; then + let NUM_REORGS_STOPPED+=1 + fi + + let NUM_REORGS_IN_PROGRESS-=1 + fi + + fi + + ## + ## OBJECT is NOSTARTED so KICK_OFF a reorg + ## + elif [ "$OBJECT_REORG_STATUS" == "NOTSTARTED" ]; then + + ## dont kick off any reorgs if window timeout passed and TWA=2 + ## OBJECTS become ABORTED - UPDATE OBJECT_ARRAY + if [ $REORG_TIMEOUT_WINDOW_COMPLETED -eq 1 ] && [ $REORG_TIMEOUT_WINDOW_ACTION -eq 2 ] && [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 1 ]; then + REORG_STATUS=ABORTED; + OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$REORG_STATUS#$OBJECT_REORG_START#$OBJECT_REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE"; + let NUM_REORGS_ABORTED+=1; + continue; + + fi + + ## is the TABLE already being used -if it is goto next OBJECT + existTABLE_TABLE_IN_USE_ARRAY "$TABSCHEMA.$TABNAME" + rc=$? + [ $rc -eq 0 ] && continue; + + ## we only want to kick off so many reorgs at any one time + if [ $NUM_REORGS_IN_PROGRESS -eq $MAX_ASYNC_REORGS_ALLOWED ]; then + continue; + fi + + ## + ## The table could already be locked - if it is then by-pass it + ## and ABORT if locked more than 10 times + ## + TABLE_LOCKED=$( db2 "select APPLICATION_HANDLE, LOCK_OBJECT_TYPE, LOCK_MODE, LOCK_CURRENT_MODE, LOCK_STATUS, LOCK_COUNT, LOCK_HOLD_COUNT, TBSP_ID, TAB_FILE_ID from TABLE (MON_GET_LOCKS(NULL, -2)) where TBSP_ID=$TBSPACEID and TAB_FILE_ID=$TABLEID and LOCK_OBJECT_TYPE='TABLE' and LOCK_MODE='IX' with ur"; ); + rc=$? + if [ $rc -eq 0 ]; then + let LOCK_COUNT+=1; + log 1 "Appears table $TABSCHEMA.$TABNAME is already locked by another application(s), LOCK_COUNT=$LOCK_COUNT"; + log 1 "$TABLE_LOCKED"; + if [ $LOCK_COUNT -gt 10 ]; then + OBJECT_REORG_STATUS=ABORTED; + let NUM_REORGS_ABORTED+=1; + log 1 "Aborting table $TABSCHEMA.$TABNAME , LOCK_COUNT=$LOCK_COUNT"; + fi + OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$OBJECT_REORG_STATUS#$OBJECT_REORG_START#$OBJECT_REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE"; + continue; + fi + + ## + ## do a check to see where we are on transaction log space - this could be improved!!!! + ## + #LOG_USED=$( db2 "select cast(LOG_UTILIZATION_PERCENT as decimal(5,2)) as PCTUSED, cast((TOTAL_LOG_USED_KB/1024) as Integer) as TOTUSEDMB, cast((TOTAL_LOG_AVAILABLE_KB/1024) as Integer) as TOTAVAILMB, cast((TOTAL_LOG_USED_TOP_KB/1024) as Integer) as TOTUSEDTOPMB FROM SYSIBMADM.LOG_UTILIZATION "); + LOG_USED=$( db2 -x "select LOG_UTILIZATION_PERCENT as PCTUSED FROM SYSIBMADM.MON_TRANSACTION_LOG_UTILIZATION " | tr -d ' ' ) + if [ ! -z "$LOG_USED" ]; then + #PCTUSED=$( echo "$LOG_USED" | awk '{ if(NF==4 && $2 ~/^[0-9]+$/) print int($1)}' ); + PCTUSED=$( echo $LOG_USED | awk '{print int($1)}') + if [ ! -z "$PCTUSED" ]; then + if [ $PCTUSED -gt $TRANSACTION_LOG_THRESHOLD_PCT ]; then + log 1 "Will not kick off another reorg due to logfile PCTUSED above threshold of $LOG_THRESHOLD% : $LOG_USED%" + continue + else + log 3 "Present LOG Utilization Percentage : $LOG_USED%" + fi + fi + fi + + ## + ## kick off another reorg + ## if rc=0 then ok, else we ABORT the OBJECT and don't try again + ## + + if [[ $TB_STATS -eq 1 || $IX_STATS -eq 2 ]]; then + + if [ $isTable -eq 1 ]; then + log 3 "Processing reorg for table $TABSCHEMA.$TABNAME" + db2 "reorg table $TABSCHEMA.$TABNAME inplace allow write access" >> /dev/null 2>&1 + rc=$? + #RC=$( db2 -x "select REORG_STATUS, REORG_COMPLETION, REORG_PHASE, REORG_CURRENT_COUNTER, REORG_MAX_COUNTER from table(snap_get_tab_reorg('')) where tabname='$TABNAME' " ) + RC=$( db2 -x "select REORG_STATUS, REORG_COMPLETION, REORG_PHASE from table(snap_get_tab_reorg('')) where tabname='$TABNAME' " ) + + elif [ $isIndex -eq 1 ]; then + log 3 "Processing reorg for table $TABSCHEMA.$TABNAME" + db2 "reorg table $TABSCHEMA.$TABNAME index $INDSCHEMA.$INDNAME inplace allow write access" > /dev/null 2>&1 + rc=$? + #RC=$( db2 "select REORG_STATUS, REORG_COMPLETION, REORG_PHASE, REORG_CURRENT_COUNTER, REORG_MAX_COUNTER from table(snap_get_tab_reorg('')) where tabname='$TABNAME' " ) + RC=$( db2 -x "select REORG_STATUS, REORG_COMPLETION, REORG_PHASE from table(snap_get_tab_reorg('')) where tabname='$TABNAME' " ) + fi + + elif [ $IF_STATS -eq 3 ]; then + + ## for offline we throw a job at db2 and wait a few seconds and check the output + ## output could be "reorg indexes all for table ...", + ## or SQL error + ## or 'DB20000I The REORG command completed successfully.' + ## not sure if there is a better way to do this + TMPLOG="/tmp/$TABSCHEMA.$TABNAME.tmp"; + log 3 "Processing reorg on indexes for table $TABSCHEMA.$TABNAME" + db2 -v "reorg indexes all for table $TABSCHEMA.$TABNAME allow write access" > $TMPLOG > /dev/null 2>&1 & + sleep 5; + cat $TMPLOG; + RC=$( grep '^SQL' $TMPLOG); + rc=$? + if [ $rc -eq 0 ]; then + log 1 "Failed to kick off reorg\n$RC"; + rc=1; + else + ## reorg could have finished then no need for the big sleep + RC=$( grep 'DB20000I The REORG command completed successfully.' $TMPLOG); + if [ $? -eq 0 ]; then + IF_STATS_BYPASS_SLEEP_INTERVAL_TIME=1; + fi + rc=0; + + fi + rm -f $TMPLOG; + + fi + + if [ $rc -eq 0 ]; then + REORG_STATUS=KICKED_OFF; + else + REORG_STATUS=ABORTED; + fi + OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$REORG_STATUS#$OBJECT_REORG_START#$OBJECT_REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE"; + if [ $rc -ne 0 ]; then + let NUM_REORGS_ABORTED+=1; + continue + fi + + ## add the table to the TABLE_IN_USE_ARRAY + addTABLE_TABLE_IN_USE_ARRAY "$TABSCHEMA.$TABNAME" + rc=$? + if [ $rc -eq 1 ]; then + log 1 "Failed to add table $TABSCHEMA.$TABNAME to TABLE_IN_USE_ARRAY"; + fi + let NUM_REORGS_KICKED_OFF+=1; + let NUM_REORGS_IN_PROGRESS+=1; + + continue; + fi + + ## + ## ouput STATUS + ## + if [[ $TB_STATS -eq 1 || $IX_STATS -eq 2 ]]; then + #echo -e "$NUM_REORGS_IN_PROGRESS:$NUM_REORGS_KICKED_OFF:$NUM_REORGS_ABORTED:$NUM_REORGS_STOPPED:$NUM_REORGS_COMPLETED:$NUM_REORG_OBJECTS $TABSCHEMA.$TABNAME : $RC \t: $REORG_PERCENT_COMPLETE %" | tee -a $REORG_TABLE_INDEX_DEBUG + printf "%-15s %-50s %-50s %-10s\n" "$NUM_REORGS_IN_PROGRESS:$NUM_REORGS_KICKED_OFF:$NUM_REORGS_ABORTED:$NUM_REORGS_STOPPED:$NUM_REORGS_COMPLETED:$NUM_REORG_OBJECTS" "$TABSCHEMA.$TABNAME" "$RC" "COMPLETION: $REORG_PERCENT_COMPLETE %"| tee -a $REORG_TABLE_INDEX_DEBUG + + elif [ $IF_STATS -eq 3 ]; then + #echo -e "$NUM_REORGS_IN_PROGRESS:$NUM_REORGS_KICKED_OFF:$NUM_REORGS_ABORTED:$NUM_REORGS_STOPPED:$NUM_REORGS_COMPLETED:$NUM_REORG_OBJECTS $TABSCHEMA.$TABNAME : $REORG_INDEX_STATUS "| tee -a $REORG_TABLE_INDEX_DEBUG + printf "%-15s %-50s %-10s %15s\n" "$NUM_REORGS_IN_PROGRESS:$NUM_REORGS_KICKED_OFF:$NUM_REORGS_ABORTED:$NUM_REORGS_STOPPED:$NUM_REORGS_COMPLETED:$NUM_REORG_OBJECTS" "$TABSCHEMA.$TABNAME" "$REORG_INDEX_STATUS" | tee -a $REORG_TABLE_INDEX_DEBUG + fi + + ## + ## if reorg timeout then issue a stop to current reorgs that are STARTED + ## no error checking for stopping a reorg + ## no need to update OBJECT array as it will be updated on next loop + ## + + if [[ $TB_STATS -eq 1 || $IX_STATS -eq 2 ]]; then + + if [ "$REORG_STATUS" == "STARTED" ] && [ $REORG_TIMEOUT_WINDOW_COMPLETED -eq 1 ] && [ $REORG_TIMEOUT_WINDOW_ACTION -eq 2 ] && [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 1 ]; then + + if [ $isTable -eq 1 ]; then + db2 -v "reorg table $TABSCHEMA.$TABNAME inplace stop" > /dev/null 2>&1 + rc=$? + elif [ $isIndex -eq 1 ]; then + db2 -v "reorg table $TABSCHEMA.$TABNAME index $INDSCHEMA.$INDNAME inplace stop" > /dev/null 2>&1 + rc=$? + fi + fi + fi + + done ## for OBJECT_ARRAY[@] + + ## check if we are done with all OBJECTS + if [ $((NUM_REORGS_COMPLETED + NUM_REORGS_STOPPED + NUM_REORGS_ABORTED)) -ge $NUM_REORG_OBJECTS ]; then + log 3 "Reorgs Status :: Completed : $NUM_REORGS_COMPLETED , Stopped : $NUM_REORGS_STOPPED , Aborted : $NUM_REORGS_ABORTED , Total No of Objects : $NUM_REORG_OBJECTS" + break + fi + + ## wait some time + if [ $IF_STATS -eq 3 ] && [ $IF_STATS_BYPASS_SLEEP_INTERVAL_TIME -eq 1 ]; then + sleep 1; + IF_STATS_BYPASS_SLEEP_INTERVAL_TIME=0; + else + sleep $SLEEP_INTERVAL_TIME + fi + + done ## while true } +# -- Main function starts here ## init if [ -f ${HOME}/sqllib/db2profile ]; then @@ -847,9 +857,9 @@ fi ## script already running ? if [ $( ps -ef | grep $0 | grep -v grep | wc -l ) -gt 2 ]; then - echo "Warning: appears $0 already running" - echo "$( ps -ef | grep $0 | grep -v grep )"; - exit 1 + echo "Warning: appears $0 already running" + echo "$( ps -ef | grep $0 | grep -v grep )"; + exit 1 fi SCRIPT=$(basename $0) @@ -871,7 +881,7 @@ LIST_VALID_TABLE_SIZES=0 LIST_REORGCHK_TB_STATS_TABLES=0 LIST_REORGCHK_IX_STATS_TABLES=0 EXECUTE_TABLE_REORG=0 -IGNORE_TABLE_SIZE_THRESHOLD_MAX=20000; +IGNORE_TABLE_SIZE_THRESHOLD_MAX=100000; IGNORE_TABLE_SIZE_THRESHOLD_MIN=10; MAINTENANCE_TIMEOUT_WINDOW_MINUTES=240; REORG_TIMEOUT_WINDOW_ACTION=2; @@ -891,8 +901,8 @@ REORGCHK_IF_STATS=3; ## user check if [ $WHOAMI == "root" ]; then - log 0 " This script should be not run as '$WHOAMI', but as instance owner." - exit 1 + log 0 " This script should be not run as '$WHOAMI', but as instance owner." + exit 1 fi ## @@ -900,97 +910,99 @@ fi ## while [ $# -gt 0 ] do - case $1 in - -h|-H|-help|--help) UsageHelp; exit 1 ;; - - -db) shift; [ ! -z $1 ] && DB=$( echo $1 | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; - -s) shift; [ ! -z $1 ] && SCHEMANAME_IN=$( echo $1 | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; - -t) shift; [ ! -z "$1" ] && TABLE_IN=$( echo "$1" | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; - -tb_stats) REORGCHK_TB_IF_STATS_OPTION+=$REORGCHK_TB_STATS; TB_STATS=1 ;; - -ti) shift; [ ! -z "$1" ] && INDEX_IN=$( echo "$1" | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; - -ix_stats) IX_STATS=2 ;; - -if_stats) REORGCHK_TB_IF_STATS_OPTION+=$REORGCHK_IF_STATS; IF_STATS=3 ;; - -ittx) shift; isNumeric $1 && { IGNORE_TABLE_SIZE_THRESHOLD_MAX=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - -ittn) shift; isNumeric $1 && { IGNORE_TABLE_SIZE_THRESHOLD_MIN=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - - -l) LIST_ONLY=1 ;; - -lf) LIST_FRAGMENTED_INDEXES=1 ;; - -ls) LIST_VALID_TABLE_SIZES=1 ;; - -lt) LIST_REORGCHK_TB_STATS_TABLES=1 ;; - -li) LIST_REORGCHK_IX_STATS_TABLES=1 ;; - - -window) shift; isNumeric $1 && { MAINTENANCE_TIMEOUT_WINDOW_MINUTES=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - -twa) shift; isNumeric $1 && { REORG_TIMEOUT_WINDOW_ACTION=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - -mar) shift; isNumeric $1 && { MAX_ASYNC_REORGS_ALLOWED=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - -log) shift; isNumeric $1 && { TRANSACTION_LOG_THRESHOLD_PCT=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - -tr) EXECUTE_TABLE_REORG=1 ;; - - -trsi) TRSI=1 ;; - -reorg) shift; [ ! -z "$1" ] && REORG=$( echo "$1" ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; - - -ignore) shift; [ ! -z "$1" ] && { IGNORE_TABLES="$1"; } || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; - - -sleep) shift; isNumeric $1 && { SLEEP_INTERVAL_TIME=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - - (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;; - (*) break;; - esac - + case $1 in + -h|-H|-help|--help) UsageHelp; exit 1 ;; + + -db|-d) shift; [ ! -z $1 ] && DBNAME=$( echo $1 | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; + -s) shift; [ ! -z $1 ] && SCHEMANAME_IN=$( echo $1 | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; + -t) shift; [ ! -z "$1" ] && TABLE_IN=$( echo "$1" | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; + -tb_stats) REORGCHK_TB_IF_STATS_OPTION+=$REORGCHK_TB_STATS; TB_STATS=1 ;; + -ti) shift; [ ! -z "$1" ] && INDEX_IN=$( echo "$1" | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; + -ix_stats) IX_STATS=2 ;; + -if_stats) REORGCHK_TB_IF_STATS_OPTION+=$REORGCHK_IF_STATS; IF_STATS=3 ;; + -ittx) shift; isNumeric $1 && { IGNORE_TABLE_SIZE_THRESHOLD_MAX=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + -ittn) shift; isNumeric $1 && { IGNORE_TABLE_SIZE_THRESHOLD_MIN=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + + -l) LIST_ONLY=1 ;; + -lf) LIST_FRAGMENTED_INDEXES=1 ;; + -ls) LIST_VALID_TABLE_SIZES=1 ;; + -lt) LIST_REORGCHK_TB_STATS_TABLES=1 ;; + -li) LIST_REORGCHK_IX_STATS_TABLES=1 ;; + + -window) shift; isNumeric $1 && { MAINTENANCE_TIMEOUT_WINDOW_MINUTES=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + -twa) shift; isNumeric $1 && { REORG_TIMEOUT_WINDOW_ACTION=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + -mar) shift; isNumeric $1 && { MAX_ASYNC_REORGS_ALLOWED=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + -log) shift; isNumeric $1 && { TRANSACTION_LOG_THRESHOLD_PCT=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + -tr) EXECUTE_TABLE_REORG=1 ;; + + -trsi) TRSI=1 ;; + -reorg) shift; [ ! -z "$1" ] && REORG=$( echo "$1" ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; + -ignore) shift; [ ! -z "$1" ] && { IGNORE_TABLES="$1"; } || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; + -sleep) shift; isNumeric $1 && { SLEEP_INTERVAL_TIME=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + + (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;; + (*) break;; + esac shift - done ## ## some verification ## +if [[ -z "$DBNAME" && -z "$SCHEMANAME_IN" ]]; then + log 0 "Please provide Database Name and Schema Name " + exit 1 + +fi +: ' if [ -z "$SCHEMANAME_IN" ]; then - log 0 "must enter a schemaname" - exit 1 + log 0 "must enter a schemaname" + exit 1 fi - +' CHECK=1 if [ $CHECK -eq 0 ]; then - rc=0 - if [ $TB_STATS -eq 1 ] && [ $IX_STATS -eq 2 -o $IF_STATS -eq 3 ]; then - rc=1; - elif [ $IX_STATS -eq 2 ] && [ $TB_STATS -eq 1 -o $IF_STATS -eq 3 ]; then - rc=1; - elif [ $IF_STATS -eq 3 ] && [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then - rc=1; - fi - if [ $rc -eq 1 ]; then - log 0 "can't define more than one of -tb_stats, -ix_stats or -if_stats" - exit 1 - fi + rc=0 + if [ $TB_STATS -eq 1 ] && [ $IX_STATS -eq 2 -o $IF_STATS -eq 3 ]; then + rc=1; + elif [ $IX_STATS -eq 2 ] && [ $TB_STATS -eq 1 -o $IF_STATS -eq 3 ]; then + rc=1; + elif [ $IF_STATS -eq 3 ] && [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then + rc=1; + fi + if [ $rc -eq 1 ]; then + log 0 "can't define more than one of -tb_stats, -ix_stats or -if_stats" + exit 1 + fi fi ## CHECK if [ $TB_STATS -eq 1 ] && [ ! -z "$INDEX_IN" ]; then - log 0 "can't define -ti with -tb_stats" - exit 1 + log 0 "can't define -ti with -tb_stats" + exit 1 elif [ $IX_STATS -eq 2 ] && [ ! -z "$TABLE_IN" ]; then - log 0 "can't define -t with -ix_stats" - exit 1 + log 0 "can't define -t with -ix_stats" + exit 1 elif [ $IF_STATS -eq 3 ] && [ ! -z "$INDEX_IN" ]; then - log 0 "can't define -ti with -if_stats" - exit 1 + log 0 "can't define -ti with -if_stats" + exit 1 fi if [ $TRANSACTION_LOG_THRESHOLD_PCT -gt 99 ]; then - log 0 "-log option should be less than 100, TRANSACTION_LOG_THRESHOLD_PCT=$TRANSACTION_LOG_THRESHOLD_PCT" - exit 1 + log 0 "-log option should be less than 100, TRANSACTION_LOG_THRESHOLD_PCT=$TRANSACTION_LOG_THRESHOLD_PCT" + exit 1 fi if [ $IGNORE_TABLE_SIZE_THRESHOLD_MIN -ge $IGNORE_TABLE_SIZE_THRESHOLD_MAX ]; then - log 0 "option -ittx should be greater than option -ittn" - exit 1 + log 0 "option -ittx should be greater than option -ittn" + exit 1 fi ## override some defaults for offline reorgs #if [ $IF_STATS -eq 3 ]; then -# MAX_ASYNC_REORGS_ALLOWED=1; -# REORG_TIMEOUT_WINDOW_ACTION=1; -# ## make SLEEP_INTERVAL_TIME 1/3 for IF_STATS -# SLEEP_INTERVAL_TIME=$( echo $SLEEP_INTERVAL_TIME | awk '{ print $1/3 }'); +# MAX_ASYNC_REORGS_ALLOWED=1; +# REORG_TIMEOUT_WINDOW_ACTION=1; +# ## make SLEEP_INTERVAL_TIME 1/3 for IF_STATS +# SLEEP_INTERVAL_TIME=$( echo $SLEEP_INTERVAL_TIME | awk '{ print $1/3 }'); #fi ## fixup REORG filter string for grep @@ -1009,328 +1021,328 @@ RUNSTATS_TIMEOUT_WINDOW_SECONDS=$( echo $MAINTENANCE_TIMEOUT_WINDOW_SECONDS | aw ## log 3 "Starting $0 at $(date) on $HOSTNAME" -DBNAMES=$( db2 list db directory | grep -E "alias|Indirect" | grep -B 1 Indirect | grep alias | awk '{print $4}' | sort ) +#DBNAMES=$( db2 list db directory | grep -E "alias|Indirect" | grep -B 1 Indirect | grep alias | awk '{print $4}' | sort ) ## ## loops for all dbs ## -for DBNAME in $DBNAMES -do - - ## just process the one db - if [ ! -z "$DB" ] && [ "$DB" != "$DBNAME" ] ; then - continue - fi - - ## can't run script on a STANDBY db - ROLE=$(db2 "get db cfg for $DBNAME" | grep 'HADR database role' | cut -d '=' -f2 | sed 's/ *//g') - if [ -z "$ROLE" ] || [ "$ROLE" == "" ]; then - log 1 " Can't determine hadr database role from 'db2 get db cfg for $DBNAME'" - continue - elif [ "$ROLE" == "STANDBY" ]; then - log 1 " Can't run script '${0}' for $DBNAME with hadr database role '$ROLE'" - continue - fi - - log 3 "DB=$DBNAME ..." - - db2 connect to $DBNAME >> /dev/null 2>&1 - rc=$? - if [ $rc -ne 0 ]; then - log 0 " can't connect to $DBNAME" - continue - fi - - if [ $TRSI -eq 1 ]; then - TRSI - continue - - elif [ $LIST_VALID_TABLE_SIZES -eq 1 ]; then - getValidTableSizes - log 3 "The following $NUM_VALID_TABLES are valid table sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB\n$VALID_TABLE_SIZES_RAW_DATA" - continue - - elif [ $LIST_FRAGMENTED_INDEXES -eq 1 ]; then - getValidFragmentedIndexes - VALID_FRAMENTED_INDEXES_HEADER="TABSCHEMA TABNAME INDSCHEMA INDNAME INDCARD STATS_TIME LAST_USED NLEAF SEQUENTIAL_PAGES"; - log 3 "The following $NUM_VALID_FRAGMENTED_INDEXES are fragmenated indexes based on tables sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB\n$VALID_FRAMENTED_INDEXES_HEADER\n$VALID_FRAGMENTED_INDEXES_RAW_DATA" - continue; - elif [ $LIST_REORGCHK_TB_STATS_TABLES -eq 1 ]; then - getValidTablesToReorg - REORGCHK_TB_STATS_HEADER="TABLE_SCHEMA TABLE_NAME CARD OVERFLOW NPAGES FPAGES ACTIVE_BLOCKS TSIZE F1 F2 F3 REORG"; - log 3 "The following $NUM_VALID_TABLES_TO_REORG are results from REORGCHK_TB_STATS based on tables sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB\n$REORGCHK_TB_STATS_HEADER\n$VALID_TABLES_TO_REORG_RAW_DATA" - continue; - - elif [ $LIST_REORGCHK_IX_STATS_TABLES -eq 1 ]; then - getValidIndexesToReorg - REORGCHK_IX_STAT_HEADER="TABLE_SCHEMA TABLE_NAME INDEX_SCHEMA INDEX_NAME INDCARD NLEAF NUM_EMPTY_LEAFS NLEVELS NUMRIDS_DELETED FULLKEYCARD LEAF_RECSIZE NONLEAF_RECSIZE LEAF_PAGE_OVERHEAD NONLEAF_PAGE_OVERHEAD PCT_PAGES_SAVED F4 F5 F6 F7 F8 REORG"; - log 3 "The following $NUM_VALID_INDEXES_TO_REORG are results from REORGCHK_IX_STATS based on tables sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB\n$REORGCHK_IX_STAT_HEADER\n$VALID_INDEXES_TO_REORG_RAW_DATA" - continue; - - - fi - - - INPLACE=1 - if [ $INPLACE -eq 1 ]; then - - log 3 " SCHEMA: $SCHEMANAME_IN" - log 3 " TABLES: $TABLE_IN" - log 3 "INDEXES: $INDEX_IN" - - ## - ## input table(s) verification - ## verify input table exist - ## and table is within size limits - ## create the OBJECT_ARRAY that holds the relevant table information - ## - if [ ! -z "$TABLE_IN" ]; then - - TABLE_IN=$( echo "$TABLE_IN" | tr ' ' '\n' ); - for TABNAME in $TABLE_IN - do - - ## make sure table exists - RC=$( db2 -x "select tabname from syscat.tables where tabname = '$TABNAME' and tabschema = '$SCHEMANAME_IN' and type = 'T'"); - rc=$? - if [ $rc -ne 0 ]; then - log 0 "input command line table '$TABNAME' does not exist or is invalid" - exit 1 - fi - - isTableWithinSizeLimit $SCHEMANAME_IN $TABNAME - rc=$? - if [ $rc -ne 0 ]; then - log 0 "Table $TABNAME is not within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB" - exit 1 - fi - - done - - if [ $TB_STATS -eq 1 ]; then - createTableOBJECT_ARRAY "$TABLE_IN" $TB_STATS - elif [ $IF_STATS -eq 3 ]; then - createTableOBJECT_ARRAY "$TABLE_IN" $IF_STATS - fi - - elif [ ${#REORGCHK_TB_IF_STATS_OPTION} -eq 2 ]; then - for REORG_TYPE in 0 1 - do - if [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "1" ]; then - getValidTablesToReorg - TABLE_IN="$VALID_TABLES_TO_REORG"; - createTableOBJECT_ARRAY "$TABLE_IN" $TB_STATS - elif [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "3" ]; then - getValidFragmentedIndexes - TABLE_IN="$VALID_FRAGMENTED_INDEXES" - createTableOBJECT_ARRAY "$TABLE_IN" $IF_STATS - fi - done - - elif [ $TB_STATS -eq 1 ]; then - getValidTablesToReorg - TABLE_IN="$VALID_TABLES_TO_REORG"; - createTableOBJECT_ARRAY "$TABLE_IN" $TB_STATS - elif [ $IF_STATS -eq 3 ]; then - getValidFragmentedIndexes - TABLE_IN="$VALID_FRAGMENTED_INDEXES" - createTableOBJECT_ARRAY "$TABLE_IN" $IF_STATS - fi - - ## - ## input indexes verification - ## verify input indexes exist - ## - if [ ! -z "$INDEX_IN" ]; then - INDEX_IN=$( echo "$INDEX_IN" | tr ' ' '\n' ); - for INDEX in $INDEX_IN - do - ## make sure index exists, especially those input on command line - TABSCHEMA=$( echo $INDEX | cut -d. -f1); - TABNAME=$( echo $INDEX | cut -d. -f2); - INDSCHEMA=$( echo $INDEX | cut -d. -f3); - INDNAME=$( echo $INDEX | cut -d. -f4); - RC=$( db2 -x "select indname from syscat.indexes where tabschema = '$TABSCHEMA' and tabname = '$TABNAME' and indschema = '$INDSCHEMA' and indname = '$INDNAME'"); - rc=$? - if [ $rc -ne 0 ]; then - log 0 " input command line index '$INDEX' does not exist" - exit 1 - fi - - isTableWithinSizeLimit $TABSCHEMA $TABNAME - rc=$? - if [ $rc -ne 0 ]; then - log 0 "Table $TABNAME is not within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB" - exit 1 - fi - - done - - createIndexOBJECT_ARRAY "$INDEX_IN" $IX_STATS; - ## - ## - elif [ $IX_STATS -eq 2 ]; then - getValidIndexesToReorg - INDEX_IN="$VALID_INDEXES_TO_REORG" - createIndexOBJECT_ARRAY "$INDEX_IN" $IX_STATS; - fi - - ## get the NUMBER of tables per reorg table type - getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY $TB_STATS; OBJECT_NUM_TB_STATS=$?; - getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY $IX_STATS; OBJECT_NUM_IX_STATS=$?; - getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY $IF_STATS; OBJECT_NUM_IF_STATS=$?; - - ## just list out the OBJECT_ARRAY and exit - if [ $LIST_ONLY -eq 1 ]; then - - listOBJECT_ARRAY; - exit 1 - fi - - - if [ $EXECUTE_TABLE_REORG -eq 1 ]; then - - - ## - ## this is the main list of what we are going to reorg - ## - echo "" - - listOBJECT_ARRAY; - -# exit 1 - - ## - ## setup some control variables for the main loop - ## - ## REORG_STATUS COMPLETED PAUSED STARTED STOPPED TRUNCATE - ## - MAINTENANCE_TIMEOUT_WINDOW_START_TIME_SECONDS=$( date '+%s' ); - REORG_TIMEOUT_WINDOW_START_TIME_SECONDS=$( date '+%s' ); - WINDOW_START_TIME_DB2=$( date '+%Y-%m-%d-%H.%M.%S' ); - IF_STATS_WINDOW_START_TIME_DB2=$( date '+%s' ); - IF_STATS_BYPASS_SLEEP_INTERVAL_TIME=0; - NUM_REORG_OBJECTS="${#OBJECT_ARRAY[@]}"; -# NUM_REORGS_IN_PROGRESS=0; -# NUM_REORGS_KICKED_OFF=0; -# NUM_REORGS_COMPLETED=0; -# NUM_REORGS_STOPPED=0; -# NUM_REORGS_ABORTED=0; - REORG_TIMEOUT_OVERFLOW_VALVE=300; - REORG_TIMEOUT_WINDOW_COMPLETED=0; - REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER=0; - initTABLE_IN_USE_ARRAY $MAX_ASYNC_REORGS_ALLOWED - - ## multi table reorg option - ## online table reorg and offline index reorgs have different options - MAX_ASYNC_REORGS_ALLOWED_ORG=$MAX_ASYNC_REORGS_ALLOWED; - REORG_TIMEOUT_WINDOW_ACTION_ORG=$REORG_TIMEOUT_WINDOW_ACTION; - SLEEP_INTERVAL_TIME_ORG=$SLEEP_INTERVAL_TIME; - - ## override some defaults for offline reorgs - #if [ $IF_STATS -eq 3 ]; then - # MAX_ASYNC_REORGS_ALLOWED=1; - # REORG_TIMEOUT_WINDOW_ACTION=1; - # ## make SLEEP_INTERVAL_TIME 1/3 for IF_STATS - # SLEEP_INTERVAL_TIME=$( echo $SLEEP_INTERVAL_TIME | awk '{ print $1/3 }'); - #fi - - echo "" - log 3 "Starting reorg of ..." - - if [ ${#REORGCHK_TB_IF_STATS_OPTION} -eq 2 ]; then - for REORG_TYPE in 0 1 - do - - if [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "1" ]; then - TB_STATS=1; - IF_STATS=0; - NUM_REORG_OBJECTS=$OBJECT_NUM_TB_STATS; - REORG_TABLE_TYPE=$TB_STATS; - MAX_ASYNC_REORGS_ALLOWED=$MAX_ASYNC_REORGS_ALLOWED_ORG; - REORG_TIMEOUT_WINDOW_ACTION=$REORG_TIMEOUT_WINDOW_ACTION_ORG; - SLEEP_INTERVAL_TIME=$SLEEP_INTERVAL_TIME_ORG; - - elif [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "3" ]; then - TB_STATS=0; - IF_STATS=3; - NUM_REORG_OBJECTS=$OBJECT_NUM_IF_STATS; - REORG_TABLE_TYPE=$IF_STATS; - MAX_ASYNC_REORGS_ALLOWED=1; - REORG_TIMEOUT_WINDOW_ACTION=1; - SLEEP_INTERVAL_TIME=$( echo $SLEEP_INTERVAL_TIME_ORG | awk '{ print $1/3 }'); - fi - - reorgTables - done - else - if [ $TB_STATS -eq 1 ]; then - REORG_TABLE_TYPE=$TB_STATS; - elif [ $IX_STATS -eq 2 ]; then - REORG_TABLE_TYPE=$IX_STATS; - elif [ $IF_STATS -eq 3 ]; then - REORG_TABLE_TYPE=$IF_STATS; - fi - reorgTables - fi - - ## list current state of OBJECT_ARRAY - listOBJECT_ARRAY; - - ## - ## now do runstats - ## - log 3 "Starting runstats of ..." - RUNSTATS_TIMEOUT_WINDOW_START_TIME_SECONDS=$( date '+%s' ); - - for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) - do - - ## check runstats window maintenance time - MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS=$( date '+%s' ); - DIFF=$(( MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS - REORG_TIMEOUT_WINDOW_START_TIME_SECONDS )); - if [ $DIFF -ge $(( REORG_TIMEOUT_WINDOW_SECONDS + RUNSTATS_TIMEOUT_WINDOW_SECONDS)) ]; then - log 3 "$REORG_TIMEOUT_WINDOW_START_TIME_SECONDS -$MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS $REORG_TIMEOUT_WINDOW_SECONDS $RUNSTATS_TIMEOUT_WINDOW_SECONDS $DIFF" - log 3 "Runstats window ending, runstats window time exceeded" - break - fi - - ## get table info - TABSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f1 ); - TABNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f2 ); - INDSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f3 ); - INDNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f4 ); - IID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f5 ); - OBJECT_REORG_STATUS=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f6 ); - - ## only do a runstats if table/index has completed - ## verify stats time so we dont kick off another runstats on the same table - if [ "$OBJECT_REORG_STATUS" == "COMPLETED" ]; then - STATS_TIME=$( db2 -x "select stats_time from syscat.tables where tabschema='$TABSCHEMA' and tabname='$TABNAME' and stats_time < TIMESTAMP('$WINDOW_START_TIME_DB2') " ); - rc=$? - if [ $rc -eq 0 ]; then - log 3 "Starting runstats on $TABSCHEMA.$TABNAME" - # db2 -v "runstats on table $TABSCHEMA.$TABNAME WITH DISTRIBUTION ON ALL COLUMNS AND SAMPLED DETAILED INDEXES ALL ALLOW WRITE ACCESS"; - db2 -v "runstats on table $TABSCHEMA.$TABNAME WITH DISTRIBUTION ON KEY COLUMNS AND SAMPLED DETAILED INDEXES ALL ALLOW WRITE ACCESS UTIL_IMPACT_PRIORITY 50"; - log 3 "Finished runstats on $TABSCHEMA.$TABNAME" - fi - fi - - done - - fi ## EXECUTE_TABLE_REORG - - - fi ## INPLACE - - -done ## DBNAMES +#for DBNAME in $DBNAMES +#do + + ## just process the one db +# if [ ! -z "$DB" ] && [ "$DB" != "$DBNAME" ] ; then +# continue +# fi +# DBNAME=${DB} + ## can't run script on a STANDBY db + ROLE=$(db2 "get db cfg for $DBNAME" | grep 'HADR database role' | cut -d '=' -f2 | sed 's/ *//g') + if [ -z "$ROLE" ] || [ "$ROLE" == "" ]; then + log 1 " Can't determine hadr database role from 'db2 get db cfg for $DBNAME'" + continue + elif [ "$ROLE" == "STANDBY" ]; then + log 1 " Can't run script '${0}' for $DBNAME with hadr database role '$ROLE'" + continue + fi + + log 3 "DBNAME :: $DBNAME" + + db2 connect to $DBNAME >> /dev/null 2>&1 + rc=$? + if [ $rc -ne 0 ]; then + log 0 " can't connect to $DBNAME" + continue + fi + + if [ $TRSI -eq 1 ]; then + TRSI + continue + + elif [ $LIST_VALID_TABLE_SIZES -eq 1 ]; then + getValidTableSizes + log 3 "The following $NUM_VALID_TABLES are valid table sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX GB\n$VALID_TABLE_SIZES_RAW_DATA" + continue + + elif [ $LIST_FRAGMENTED_INDEXES -eq 1 ]; then + getValidFragmentedIndexes + VALID_FRAMENTED_INDEXES_HEADER="TABSCHEMA TABNAME INDSCHEMA INDNAME INDCARD STATS_TIME LAST_USED NLEAF SEQUENTIAL_PAGES"; + log 3 "The following $NUM_VALID_FRAGMENTED_INDEXES are fragmenated indexes based on tables sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX GB\n$VALID_FRAMENTED_INDEXES_HEADER\n$VALID_FRAGMENTED_INDEXES_RAW_DATA" + continue; + elif [ $LIST_REORGCHK_TB_STATS_TABLES -eq 1 ]; then + getValidTablesToReorg + REORGCHK_TB_STATS_HEADER="TABLE_SCHEMA TABLE_NAME CARD OVERFLOW NPAGES FPAGES ACTIVE_BLOCKS TSIZE F1 F2 F3 REORG"; + log 3 "The following $NUM_VALID_TABLES_TO_REORG are results from REORGCHK_TB_STATS based on tables sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX GB\n$REORGCHK_TB_STATS_HEADER\n$VALID_TABLES_TO_REORG_RAW_DATA" + continue; + + elif [ $LIST_REORGCHK_IX_STATS_TABLES -eq 1 ]; then + getValidIndexesToReorg + REORGCHK_IX_STAT_HEADER="TABLE_SCHEMA TABLE_NAME INDEX_SCHEMA INDEX_NAME INDCARD NLEAF NUM_EMPTY_LEAFS NLEVELS NUMRIDS_DELETED FULLKEYCARD LEAF_RECSIZE NONLEAF_RECSIZE LEAF_PAGE_OVERHEAD NONLEAF_PAGE_OVERHEAD PCT_PAGES_SAVED F4 F5 F6 F7 F8 REORG"; + log 3 "The following $NUM_VALID_INDEXES_TO_REORG are results from REORGCHK_IX_STATS based on tables sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX GB\n$REORGCHK_IX_STAT_HEADER\n$VALID_INDEXES_TO_REORG_RAW_DATA" + continue; + + fi + + INPLACE=1 + if [ $INPLACE -eq 1 ]; then + + log 3 "SCHEMA :: $SCHEMANAME_IN" + log 3 "TABLES :: $TABLE_IN" + log 3 "INDEXES :: $INDEX_IN" + log 3 "Looking for Qualified Tables or Indexes for $SCHEMANAME_IN in $DBNAME . . . " + echo "" + + ## + ## input table(s) verification + ## verify input table exist + ## and table is within size limits + ## create the OBJECT_ARRAY that holds the relevant table information + ## + if [ ! -z "$TABLE_IN" ]; then + + TABLE_IN=$( echo "$TABLE_IN" | tr ' ' '\n' ); + for TABNAME in $TABLE_IN + do + + ## make sure table exists + RC=$( db2 -x "select tabname from syscat.tables where tabname = '$TABNAME' and tabschema = '$SCHEMANAME_IN' and type = 'T'"); + rc=$? + if [ $rc -ne 0 ]; then + log 0 "input command line table '$TABNAME' does not exist or is invalid" + exit 1 + fi + + isTableWithinSizeLimit $SCHEMANAME_IN $TABNAME + rc=$? + if [ $rc -ne 0 ]; then + log 0 "Table $TABNAME is not within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB" + exit 1 + fi + + done + + if [ $TB_STATS -eq 1 ]; then + createTableOBJECT_ARRAY "$TABLE_IN" $TB_STATS + elif [ $IF_STATS -eq 3 ]; then + createTableOBJECT_ARRAY "$TABLE_IN" $IF_STATS + fi + + elif [ ${#REORGCHK_TB_IF_STATS_OPTION} -eq 2 ]; then + for REORG_TYPE in 0 1 + do + if [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "1" ]; then + getValidTablesToReorg + TABLE_IN="$VALID_TABLES_TO_REORG"; + createTableOBJECT_ARRAY "$TABLE_IN" $TB_STATS + elif [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "3" ]; then + getValidFragmentedIndexes + TABLE_IN="$VALID_FRAGMENTED_INDEXES" + createTableOBJECT_ARRAY "$TABLE_IN" $IF_STATS + fi + done + + elif [ $TB_STATS -eq 1 ]; then + getValidTablesToReorg + TABLE_IN="$VALID_TABLES_TO_REORG"; + createTableOBJECT_ARRAY "$TABLE_IN" $TB_STATS + elif [ $IF_STATS -eq 3 ]; then + getValidFragmentedIndexes + TABLE_IN="$VALID_FRAGMENTED_INDEXES" + createTableOBJECT_ARRAY "$TABLE_IN" $IF_STATS + fi + + ## + ## input indexes verification + ## verify input indexes exist + ## + if [ ! -z "$INDEX_IN" ]; then + INDEX_IN=$( echo "$INDEX_IN" | tr ' ' '\n' ); + for INDEX in $INDEX_IN + do + ## make sure index exists, especially those input on command line + TABSCHEMA=$( echo $INDEX | cut -d. -f1); + TABNAME=$( echo $INDEX | cut -d. -f2); + INDSCHEMA=$( echo $INDEX | cut -d. -f3); + INDNAME=$( echo $INDEX | cut -d. -f4); + RC=$( db2 -x "select indname from syscat.indexes where tabschema = '$TABSCHEMA' and tabname = '$TABNAME' and indschema = '$INDSCHEMA' and indname = '$INDNAME'"); + rc=$? + if [ $rc -ne 0 ]; then + log 0 " input command line index '$INDEX' does not exist" + exit 1 + fi + + isTableWithinSizeLimit $TABSCHEMA $TABNAME + rc=$? + if [ $rc -ne 0 ]; then + log 0 "Table $TABNAME is not within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB" + exit 1 + fi + + done + + createIndexOBJECT_ARRAY "$INDEX_IN" $IX_STATS; + ## + ## + elif [ $IX_STATS -eq 2 ]; then + getValidIndexesToReorg + INDEX_IN="$VALID_INDEXES_TO_REORG" + createIndexOBJECT_ARRAY "$INDEX_IN" $IX_STATS; + fi + + ## get the NUMBER of tables per reorg table type + getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY $TB_STATS; OBJECT_NUM_TB_STATS=$?; + getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY $IX_STATS; OBJECT_NUM_IX_STATS=$?; + getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY $IF_STATS; OBJECT_NUM_IF_STATS=$?; + + ## just list out the OBJECT_ARRAY and exit + if [ $LIST_ONLY -eq 1 ]; then + + #printf "%-15s %-30s %-15s %-30s %-5s %-10s %-25s %-25s %-10s %-10s %-10s %-15s\n" "TABSCHEMA" "TABNAME" "INDSCHEMA" "INDNAME" "IID" "REORG_STATUS" "REORG_START" "REORG_END" "TABLEID" "TBSPACEID" "LOCK_COUNT" "TABLE_TYPE" + listOBJECT_ARRAY; + exit 1 + fi + + + if [ $EXECUTE_TABLE_REORG -eq 1 ]; then + + + ## + ## this is the main list of what we are going to reorg + ## + #echo "" + #printf "%-15s %-30s %-15s %-30s %-5s %-10s %-25s %-25s %-10s %-10s %-10s %-15s\n" "TABSCHEMA" "TABNAME" "INDSCHEMA" "INDNAME" "IID" "REORG_STATUS" "REORG_START" "REORG_END" "TABLEID" "TBSPACEID" "LOCK_COUNT" "TABLE_TYPE" + + listOBJECT_ARRAY; + +# exit 1 + + ## + ## setup some control variables for the main loop + ## + ## REORG_STATUS COMPLETED PAUSED STARTED STOPPED TRUNCATE + ## + MAINTENANCE_TIMEOUT_WINDOW_START_TIME_SECONDS=$( date '+%s' ); + REORG_TIMEOUT_WINDOW_START_TIME_SECONDS=$( date '+%s' ); + WINDOW_START_TIME_DB2=$( date '+%Y-%m-%d-%H.%M.%S' ); + IF_STATS_WINDOW_START_TIME_DB2=$( date '+%s' ); + IF_STATS_BYPASS_SLEEP_INTERVAL_TIME=0; + NUM_REORG_OBJECTS="${#OBJECT_ARRAY[@]}"; +# NUM_REORGS_IN_PROGRESS=0; +# NUM_REORGS_KICKED_OFF=0; +# NUM_REORGS_COMPLETED=0; +# NUM_REORGS_STOPPED=0; +# NUM_REORGS_ABORTED=0; + REORG_TIMEOUT_OVERFLOW_VALVE=300; + REORG_TIMEOUT_WINDOW_COMPLETED=0; + REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER=0; + initTABLE_IN_USE_ARRAY $MAX_ASYNC_REORGS_ALLOWED + + ## multi table reorg option + ## online table reorg and offline index reorgs have different options + MAX_ASYNC_REORGS_ALLOWED_ORG=$MAX_ASYNC_REORGS_ALLOWED; + REORG_TIMEOUT_WINDOW_ACTION_ORG=$REORG_TIMEOUT_WINDOW_ACTION; + SLEEP_INTERVAL_TIME_ORG=$SLEEP_INTERVAL_TIME; + + ## override some defaults for offline reorgs + #if [ $IF_STATS -eq 3 ]; then + # MAX_ASYNC_REORGS_ALLOWED=1; + # REORG_TIMEOUT_WINDOW_ACTION=1; + # ## make SLEEP_INTERVAL_TIME 1/3 for IF_STATS + # SLEEP_INTERVAL_TIME=$( echo $SLEEP_INTERVAL_TIME | awk '{ print $1/3 }'); + #fi + + echo "" + log 3 "Starting reorg of ..." + + if [ ${#REORGCHK_TB_IF_STATS_OPTION} -eq 2 ]; then + for REORG_TYPE in 0 1 + do + + if [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "1" ]; then + TB_STATS=1; + IF_STATS=0; + NUM_REORG_OBJECTS=$OBJECT_NUM_TB_STATS; + REORG_TABLE_TYPE=$TB_STATS; + MAX_ASYNC_REORGS_ALLOWED=$MAX_ASYNC_REORGS_ALLOWED_ORG; + REORG_TIMEOUT_WINDOW_ACTION=$REORG_TIMEOUT_WINDOW_ACTION_ORG; + SLEEP_INTERVAL_TIME=$SLEEP_INTERVAL_TIME_ORG; + + elif [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "3" ]; then + TB_STATS=0; + IF_STATS=3; + NUM_REORG_OBJECTS=$OBJECT_NUM_IF_STATS; + REORG_TABLE_TYPE=$IF_STATS; + MAX_ASYNC_REORGS_ALLOWED=1; + REORG_TIMEOUT_WINDOW_ACTION=1; + SLEEP_INTERVAL_TIME=$( echo $SLEEP_INTERVAL_TIME_ORG | awk '{ print $1/3 }'); + fi + + reorgTables + done + else + if [ $TB_STATS -eq 1 ]; then + REORG_TABLE_TYPE=$TB_STATS; + elif [ $IX_STATS -eq 2 ]; then + REORG_TABLE_TYPE=$IX_STATS; + elif [ $IF_STATS -eq 3 ]; then + REORG_TABLE_TYPE=$IF_STATS; + fi + reorgTables + fi + + ## list current state of OBJECT_ARRAY + #printf "%-15s %-30s %-15s %-30s %-5s %-10s %-25s %-25s %-10s %-10s %-10s %-15s\n" "TABSCHEMA" "TABNAME" "INDSCHEMA" "INDNAME" "IID" "REORG_STATUS" "REORG_START" "REORG_END" "TABLEID" "TBSPACEID" "LOCK_COUNT" "TABLE_TYPE" + listOBJECT_ARRAY; + + ## + ## now do runstats + ## + log 3 "Starting runstats of ..." + RUNSTATS_TIMEOUT_WINDOW_START_TIME_SECONDS=$( date '+%s' ); + + for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) + do + + ## check runstats window maintenance time + MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS=$( date '+%s' ); + DIFF=$(( MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS - REORG_TIMEOUT_WINDOW_START_TIME_SECONDS )); + if [ $DIFF -ge $(( REORG_TIMEOUT_WINDOW_SECONDS + RUNSTATS_TIMEOUT_WINDOW_SECONDS)) ]; then + #log 3 "$REORG_TIMEOUT_WINDOW_START_TIME_SECONDS $MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS $REORG_TIMEOUT_WINDOW_SECONDS $RUNSTATS_TIMEOUT_WINDOW_SECONDS $DIFF" + log 3 "Runstats window ending, runstats window time exceeded" + break + fi + + ## get table info + TABSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f1 ); + TABNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f2 ); + INDSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f3 ); + INDNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f4 ); + IID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f5 ); + OBJECT_REORG_STATUS=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f6 ); + + ## only do a runstats if table/index has completed + ## verify stats time so we dont kick off another runstats on the same table + if [ "$OBJECT_REORG_STATUS" == "COMPLETED" ]; then + STATS_TIME=$( db2 -x "select stats_time from syscat.tables where tabschema='$TABSCHEMA' and tabname='$TABNAME' and stats_time < TIMESTAMP('$WINDOW_START_TIME_DB2') " ); + rc=$? + if [ $rc -eq 0 ]; then + log 3 "Starting runstats on $TABSCHEMA.$TABNAME" + # db2 -v "runstats on table $TABSCHEMA.$TABNAME WITH DISTRIBUTION ON ALL COLUMNS AND SAMPLED DETAILED INDEXES ALL ALLOW WRITE ACCESS"; + #db2 -v "runstats on table $TABSCHEMA.$TABNAME WITH DISTRIBUTION ON KEY COLUMNS AND SAMPLED DETAILED INDEXES ALL ALLOW WRITE ACCESS UTIL_IMPACT_PRIORITY 50" > /dev/null 2>&1 + db2 -v "runstats on table $TABSCHEMA.$TABNAME ON ALL COLUMNS WITH DISTRIBUTION ON ALL COLUMNS AND DETAILED INDEXES ALL ALLOW WRITE ACCESS " > /dev/null 2>&1 + log 3 "Finished runstats on $TABSCHEMA.$TABNAME" + fi + fi + + done + + fi ## EXECUTE_TABLE_REORG + + fi ## INPLACE + +#done ## DBNAMES ## ## cleanup ## log 3 "Completed $0 at $(date)" - -exit 0 \ No newline at end of file +exit 0 diff --git a/instance-applications/120-ibm-db2u-database/files/reorgTablesIndexesInplace2_maintenance.sh b/instance-applications/120-ibm-db2u-database/files/reorgTablesIndexesInplace2_maintenance.sh index 94e16231e..28a993c4e 100755 --- a/instance-applications/120-ibm-db2u-database/files/reorgTablesIndexesInplace2_maintenance.sh +++ b/instance-applications/120-ibm-db2u-database/files/reorgTablesIndexesInplace2_maintenance.sh @@ -5,66 +5,71 @@ ## http://www.ibm.com/developerworks/data/library/techarticle/dm-1307optimizerunstats/ ## see Identifying fragmented indexes from statistics ## http://www.ibm.com/developerworks/data/library/techarticle/dm-1307optimizerunstats/#Listing%207 -## +##0 23 * * 5 (. ~/sqllib/db2profile; ~/maintenance/bin/misc/reorgTablesIndexesInplace.sh -db dbname -s MAXIMO -tb_stats -if_stats -window 120 -tr >> ~/maintenance/logs/reorgTablesIndexesInplace.sh.log 2>&1 ) ## script to reorg tables online and indexes offline based on different criteria ## we need to be able to perform an online table reorg and an offline indexes all reorg in the same run of the script ## +# +# -- Modifications/Updates on 01/May/26 by Prudhviraj Patrata +# -- Increased the default tablesize from 20GB to 100GB +# -- Updated to use single database at a time +# -- Improved the readability of output of logs +# UsageHelp() { - - echo "Script to perform reorg tables, indexes online (inplace) " - echo " also to REORG INDEXES ALL FOR TABLE offline" - echo " db2 performs the online reorgs asynchronously" - echo "" - echo "Usage: ${0} [options]" - echo " where [options] is one of the following:" - echo " -h: displays this usage screen" - echo " -db: dbname, default is all cataloged databases" - echo "" - echo " -s: table schemaname" - echo " -t: table(s) to reorg" - echo "-tb_stats: reorg tables reported by REORGCHK_TB_STATS" - echo " -ti: reorg table index(s), format must be TABSCHEMA.TABNAME.INDSCHEMA.INDNAME" - echo "-ix_stats: reorg table index(s) reported by REORGCHK_IX_STATS" - echo "-if_stats: reorg indexes all for table(s) offline as reported by index fragmentation NLEAF/SEQUENTIAL_PAGES columns" - echo "" - echo " -ls: list valid table sizes for a particular schema" - echo " -lf: list all fragmented index details for a particular schema, based on valid table sizes" - echo " -lt: list all tables to reorg based on REORGCHK_TB_STATS reorg column, based on valid table sizes" - echo " -li: list all indexes to reorg based on REORGCHK_IX_STATS reorg column, based on valid table sizes" - echo " -l: list tables/indexes that would be reorged" - echo "" - echo " -ittx: ignore tables over a specific threshold size in MBs, default is 20000 MB ie 20 GB" - echo " -ittn: ignore tables under a specific threshold size in MBs, default is 10 MB" - echo " -mar: maximum asynchronous reorgs allowed, default is 3" - echo " -log: don't kick off a reorg if transaction log usage is over a certain percentage, default is 90%" - echo " -window: stop reorg tables/indexes/runstats after a set maintenance timeout window, default is 240 minutes" - echo " -twa: timeout window action: default=2 for online, 1 for offline" - echo " 1=allow current reorg(s) to continue" - echo " 2=stop current reorg(s)" -# echo " 3=stop current reorg(s) if < 80% complete and continue script" - echo " -ignore: ignore specific tables from SYSIBMADM.ADMINTABINFO t0, SYSCAT.TABLES t1 " - echo " eg \"$IGNORE_TABLES_EX\"" - echo " -reorg: table F1 F2 F3 filter reorg, default is *" - echo " -sleep: SLEEP_INTERVAL_TIME, default is 60 seconds" - echo "" - echo " -tr: execute inplace table/index reorg" - echo "" - echo " -trsi: Retrieve table reorganization snapshot information from snap_get_tab_reorg and db2pd -reorgs index" - echo "" - echo "Examples:" - echo " 1. ${0} -h" - echo " 2. ${0} -db dbname -s BLUDB -ls" - echo " 3. ${0} -s MAXIMO -t \"YFS_ITEM YFS_TASK_Q YFS_SHIPMENT\" -tb_stats -tr " - echo " 4. ${0} -s MAXIMO -ti \"MAXIMO.YFS_SNAPSHOT.MAXIMO.YFS_SNAPSHOT_I1 MAXIMO.YFS_ITEM.MAXIMO.YFS_ITEM_PK\" -ix_stats -tr" - echo " 5. ${0} -s MAXIMO -t \"YFS_ITEM YFS_SNAPSHOT YFS_IMPORT YFS_EXPORT\" -if_stats -tr" - echo " 6. ${0} -s MAXIMO -tb_stats -mar 5 -window 10 -log 95 -ittx 30000 -tr" - echo " 7. ${0} -s MAXIMO -tb_stats -mar 5 -window 10 -log 95 -ignore \"$IGNORE_TABLES_EX\" -reorg \"***\" -tr" - echo " 8. ${0} -s MAXIMO -tb_stats -if_stats -ittx 100 -ittn 20 -tr" - echo " 9. ${0} -trsi" - - echo "" + echo "Script to perform reorg tables, indexes online (inplace) " + echo " also to REORG INDEXES ALL FOR TABLE offline" + echo " db2 performs the online reorgs asynchronously" + echo "" + echo "Usage: ${0} [options]" + echo " where [options] is one of the following:" + echo " -h: displays this usage screen" + echo " -db: dbname, default is all cataloged databases" + echo "" + echo " -s: table schemaname" + echo " -t: table(s) to reorg" + echo "-tb_stats: reorg tables reported by REORGCHK_TB_STATS" + echo " -ti: reorg table index(s), format must be TABSCHEMA.TABNAME.INDSCHEMA.INDNAME" + echo "-ix_stats: reorg table index(s) reported by REORGCHK_IX_STATS" + echo "-if_stats: reorg indexes all for table(s) offline as reported by index fragmentation NLEAF/SEQUENTIAL_PAGES columns" + echo "" + echo " -ls: list valid table sizes for a particular schema" + echo " -lf: list all fragmented index details for a particular schema, based on valid table sizes" + echo " -lt: list all tables to reorg based on REORGCHK_TB_STATS reorg column, based on valid table sizes" + echo " -li: list all indexes to reorg based on REORGCHK_IX_STATS reorg column, based on valid table sizes" + echo " -l: list tables/indexes that would be reorged" + echo "" + echo " -ittx: ignore tables over a specific threshold size in MBs, default is 100000 MB ie 100 GB" + echo " -ittn: ignore tables under a specific threshold size in MBs, default is 10 MB" + echo " -mar: maximum asynchronous reorgs allowed, default is 3" + echo " -log: don't kick off a reorg if transaction log usage is over a certain percentage, default is 90%" + echo " -window: stop reorg tables/indexes/runstats after a set maintenance timeout window, default is 240 minutes" + echo " -twa: timeout window action: default=2 for online, 1 for offline" + echo " 1=allow current reorg(s) to continue" + echo " 2=stop current reorg(s)" +# echo " 3=stop current reorg(s) if < 80% complete and continue script" + echo " -ignore: ignore specific tables from SYSIBMADM.ADMINTABINFO t0, SYSCAT.TABLES t1 " + echo " eg \"$IGNORE_TABLES_EX\"" + echo " -reorg: table F1 F2 F3 filter reorg, default is *" + echo " -sleep: SLEEP_INTERVAL_TIME, default is 60 seconds" + echo "" + echo " -tr: execute inplace table/index reorg" + echo "" + echo " -trsi: Retrieve table reorganization snapshot information from snap_get_tab_reorg and db2pd -reorgs index" + echo "" + echo "Examples :" + echo " 1. ${0} -h" + echo " 2. ${0} -db dbname -s MAXIMO -ls" + echo " 3. ${0} -db dbname -s MAXIMO -t \"YFS_ITEM YFS_TASK_Q YFS_SHIPMENT\" -tb_stats -tr " + echo " 4. ${0} -db dbname -s MAXIMO -ti \"MAXIMO.YFS_SNAPSHOT.MAXIMO.YFS_SNAPSHOT_I1 MAXIMO.YFS_ITEM.MAXIMO.YFS_ITEM_PK\" -ix_stats -tr" + echo " 5. ${0} -db dbname -s MAXIMO -t \"YFS_ITEM YFS_SNAPSHOT YFS_IMPORT YFS_EXPORT\" -if_stats -tr" + echo " 6. ${0} -db dbname -s MAXIMO -tb_stats -mar 5 -window 10 -log 95 -ittx 30000 -tr" + echo " 7. ${0} -db dbname -s MAXIMO -tb_stats -mar 5 -window 10 -log 95 -ignore \"$IGNORE_TABLES_EX\" -reorg \"***\" -tr" + echo " 8. ${0} -db dbname -s MAXIMO -tb_stats -if_stats -ittx 100 -ittn 20 -tr" + echo " 9. ${0} -trsi" + + echo "" } @@ -73,133 +78,131 @@ UsageHelp() ## isNumeric() { - echo $1 | grep -E '^[0-9]+$' > /dev/null - - return $? + echo $1 | grep -E '^[0-9]+$' > /dev/null + return $? } TRSI() { - db2 -v "select varchar(tabschema,9) as tabschema, varchar(tabname,32) as tabname, - REORG_STATUS, REORG_COMPLETION, REORG_PHASE, REORG_CURRENT_COUNTER, REORG_MAX_COUNTER, --- varchar( varchar_format(REORG_START, 'YYYY-MM-DD HH24:MI:SS'),19) as REORG_START, --- varchar( varchar_format(REORG_END, 'YYYY-MM-DD HH24:MI:SS'),19) as REORG_END, - REORG_START, REORG_END, - REORG_INDEX_ID, REORG_TBSPC_ID - from table(snap_get_tab_reorg('')) - order by REORG_START asc - with ur" - - db2pd -db $DBNAME -reorgs index | sed -n "/Index Reorg Stats:/,//p" - + db2 -v "select varchar(tabschema,9) as tabschema, varchar(tabname,32) as tabname, + REORG_STATUS, REORG_COMPLETION, REORG_PHASE, REORG_CURRENT_COUNTER, REORG_MAX_COUNTER, + VARCHAR_FORMAT(REORG_START, 'YYYY-MM-DD-HH24:MI:SS') as REORG_START, + VARCHAR_FORMAT(REORG_END, 'YYYY-MM-DD-HH24:MI:SS') as REORG_END, +-- REORG_START, REORG_END, + REORG_INDEX_ID, REORG_TBSPC_ID + from table(snap_get_tab_reorg('')) + order by REORG_START asc + with ur" + + db2pd -db $DBNAME -reorgs index | sed -n "/Index Reorg Stats:/,//p" > /dev/null 2>&1 } log() { - TYPE=$1 - MSG="$2" - - DATE=$( date '+%d-%m-%Y %H:%M:%S' ); - - # TYPE: - # 0 = Critical - # 1 = Warn - # 3 = Info - # 5 = Debug' - if [ ${TYPE} -eq 0 ]; then - TYPEMSG="Error" - elif [ ${TYPE} -eq 1 ]; then - TYPEMSG="Warning" - elif [ ${TYPE} -eq 3 ]; then - TYPEMSG="Info" - elif [ ${TYPE} -eq 5 ]; then - TYPEMSG="Debug" - else - TYPEMSG="Other" - fi - - echo -e "${DATE} ${TYPEMSG}: ${MSG}" | tee -a $REORG_TABLE_INDEX_LOG - - return 0 + TYPE=$1 + MSG="$2" + + DATE=$( date '+%d-%m-%Y %H:%M:%S' ); + + # TYPE: + # 0 = Critical + # 1 = Warn + # 3 = Info + # 5 = Debug' + if [ ${TYPE} -eq 0 ]; then + TYPEMSG="Error" + elif [ ${TYPE} -eq 1 ]; then + TYPEMSG="Warning" + elif [ ${TYPE} -eq 3 ]; then + TYPEMSG="Info" + elif [ ${TYPE} -eq 5 ]; then + TYPEMSG="Debug" + else + TYPEMSG="Other" + fi + + echo -e "${DATE} ${TYPEMSG} : ${MSG}" | tee -a $REORG_TABLE_INDEX_LOG + + return 0 } initTABLE_IN_USE_ARRAY() { - local NUM_ITEMS=$1 - local jj; + local NUM_ITEMS=$1 + local jj; - ## - ## initialise the db2 TABLE_IN_USE_ARRAY - ## - for((jj=0; jj<$NUM_ITEMS; jj++)) - do - TABLE_IN_USE_ARRAY[$jj]="" - done + ## + ## initialise the db2 TABLE_IN_USE_ARRAY + ## + for((jj=0; jj<$NUM_ITEMS; jj++)) + do + TABLE_IN_USE_ARRAY[$jj]="" + done - return 0 + return 0 } existTABLE_TABLE_IN_USE_ARRAY() { - local TABLE=$1 - local jj; - ## - ## check if table is in use - ## - for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) - do - if [ "${TABLE_IN_USE_ARRAY[$jj]}" == "$TABLE" ]; then - return 0; - fi - done + local TABLE=$1 + local jj; + ## + ## check if table is in use + ## + for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) + do + if [ "${TABLE_IN_USE_ARRAY[$jj]}" == "$TABLE" ]; then + return 0; + fi + done - return 1; + return 1; } addTABLE_TABLE_IN_USE_ARRAY() { - local TABLE=$1 - local jj; - - ## - ## add table in empty slot - ## - for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) - do - if [ "${TABLE_IN_USE_ARRAY[$jj]}" == "" ]; then - TABLE_IN_USE_ARRAY[$jj]=$TABLE; - return 0; - fi - done - - return 1; + local TABLE=$1 + local jj; + + ## + ## add table in empty slot + ## + for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) + do + if [ "${TABLE_IN_USE_ARRAY[$jj]}" == "" ]; then + TABLE_IN_USE_ARRAY[$jj]=$TABLE; + return 0; + fi + done + + return 1; } removeTABLE_TABLE_IN_USE_ARRAY() { - local TABLE=$1 - local jj; - ## - ## remove entry - ## - for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) - do - if [ "${TABLE_IN_USE_ARRAY[$jj]}" == "$TABLE" ]; then - TABLE_IN_USE_ARRAY[$jj]=""; - return 0; - fi - done - - return 1; + local TABLE=$1 + local jj; + ## + ## remove entry + ## + for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) + do + if [ "${TABLE_IN_USE_ARRAY[$jj]}" == "$TABLE" ]; then + TABLE_IN_USE_ARRAY[$jj]=""; + return 0; + fi + done + + return 1; } @@ -207,195 +210,193 @@ removeTABLE_TABLE_IN_USE_ARRAY() listTABLE_IN_USE_ARRAY() { - local jj; - ## - ## list table entries - ## - for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) - do - log 5 "TABLE_IN_USE_ARRAY $jj ${TABLE_IN_USE_ARRAY[$jj]}"; - done + local jj; + ## + ## list table entries + ## + for((jj=0; jj<${#TABLE_IN_USE_ARRAY[@]}; jj++)) + do + log 5 "TABLE_IN_USE_ARRAY $jj ${TABLE_IN_USE_ARRAY[$jj]}"; + done - return 0; + return 0; } getValidTablesToReorg() { - getValidTableSizes - - VALID_TABLES_TO_REORG="" - VALID_TABLES_TO_REORG_RAW="" - NUM_VALID_TABLES_TO_REORG=0 - for TABNAME in $VALID_TABLES - do - - RAW=$( db2 -x "call REORGCHK_TB_STATS('T','$SCHEMANAME_IN.$TABNAME')" ); - RAW=$( echo "$RAW" | grep $SCHEMANAME_IN | grep $TABNAME | awk '{ if (NF == 12) print $0 }' | sed 's/ \+/ /g' | grep $REORG ) - rc=$? - if [ $rc -eq 0 ]; then - TABNAME=$( echo "$RAW" | awk '{print $2}' ); - [ "$VALID_TABLES_TO_REORG_RAW_DATA" == "" ] && VALID_TABLES_TO_REORG_RAW_DATA="$RAW" || VALID_TABLES_TO_REORG_RAW_DATA="$VALID_TABLES_TO_REORG_RAW_DATA\n$RAW" - fi - - done - - ## sort the tables based on REORG column - VALID_TABLES_TO_REORG_RAW_DATA=$( echo -e "$VALID_TABLES_TO_REORG_RAW_DATA" | sort -k12 -r); - VALID_TABLES_TO_REORG=$( echo -e "$VALID_TABLES_TO_REORG_RAW_DATA" | awk '{print $2}' ); - NUM_VALID_TABLES_TO_REORG=$( echo -e "$VALID_TABLES_TO_REORG" | wc -l ); - - return 0 - + getValidTableSizes + + VALID_TABLES_TO_REORG="" + VALID_TABLES_TO_REORG_RAW="" + NUM_VALID_TABLES_TO_REORG=0 + for TABNAME in $VALID_TABLES + do + + RAW=$( db2 -x "call REORGCHK_TB_STATS('T','$SCHEMANAME_IN.$TABNAME')" ); + RAW=$( echo "$RAW" | grep $SCHEMANAME_IN | grep $TABNAME | awk '{ if (NF == 12) print $0 }' | sed 's/ \+/ /g' | grep $REORG ) + rc=$? + if [ $rc -eq 0 ]; then + TABNAME=$( echo "$RAW" | awk '{print $2}' ); + [ "$VALID_TABLES_TO_REORG_RAW_DATA" == "" ] && VALID_TABLES_TO_REORG_RAW_DATA="$RAW" || VALID_TABLES_TO_REORG_RAW_DATA="$VALID_TABLES_TO_REORG_RAW_DATA\n$RAW" + fi + + done + + ## sort the tables based on REORG column + VALID_TABLES_TO_REORG_RAW_DATA=$( echo -e "$VALID_TABLES_TO_REORG_RAW_DATA" | sort -k12 -r); + VALID_TABLES_TO_REORG=$( echo -e "$VALID_TABLES_TO_REORG_RAW_DATA" | awk '{print $2}' ); + NUM_VALID_TABLES_TO_REORG=$( echo -e "$VALID_TABLES_TO_REORG" | wc -l ); + + return 0 + } getValidIndexesToReorg() { - getValidTableSizes - - VALID_INDEXES_TO_REORG="" - VALID_INDEXES_TO_REORG_RAW="" - NUM_VALID_INDEXES_TO_REORG=0 - for TABNAME in $VALID_TABLES - do - - RAW=$( db2 -x "call REORGCHK_IX_STATS('T','$SCHEMANAME_IN.$TABNAME')" ); - ## this can return multiple indexes for same TABNAME - RAW=$( echo "$RAW" | grep $SCHEMANAME_IN | grep $TABNAME | awk '{ if (NF == 21) print $0 }' | sed 's/ \+/ /g' | grep $REORG ); - rc=$? - if [ $rc -eq 0 ]; then - INDNAME=$( echo "$RAW" | awk '{print $1"."$2"."$3"."$4}' ); - [ "$VALID_INDEXES_TO_REORG_RAW_DATA" == "" ] && VALID_INDEXES_TO_REORG_RAW_DATA=$RAW || VALID_INDEXES_TO_REORG_RAW_DATA="$VALID_INDEXES_TO_REORG_RAW_DATA\n$RAW" - fi - - done - - ## sort the indexes based on REORG column - VALID_INDEXES_TO_REORG_RAW_DATA=$( echo -e "$VALID_INDEXES_TO_REORG_RAW_DATA" | sort -k21 -r); - VALID_INDEXES_TO_REORG=$( echo -e "$VALID_INDEXES_TO_REORG_RAW_DATA" | awk '{print $1"."$2"."$3"."$4}' ); - NUM_VALID_INDEXES_TO_REORG=$( echo -e "$VALID_INDEXES_TO_REORG" | wc -l ); - - return 0 - + getValidTableSizes + + VALID_INDEXES_TO_REORG="" + VALID_INDEXES_TO_REORG_RAW="" + NUM_VALID_INDEXES_TO_REORG=0 + for TABNAME in $VALID_TABLES + do + + RAW=$( db2 -x "call REORGCHK_IX_STATS('T','$SCHEMANAME_IN.$TABNAME')" ); + ## this can return multiple indexes for same TABNAME + RAW=$( echo "$RAW" | grep $SCHEMANAME_IN | grep $TABNAME | awk '{ if (NF == 21) print $0 }' | sed 's/ \+/ /g' | grep $REORG ); + rc=$? + if [ $rc -eq 0 ]; then + INDNAME=$( echo "$RAW" | awk '{print $1"."$2"."$3"."$4}' ); + [ "$VALID_INDEXES_TO_REORG_RAW_DATA" == "" ] && VALID_INDEXES_TO_REORG_RAW_DATA=$RAW || VALID_INDEXES_TO_REORG_RAW_DATA="$VALID_INDEXES_TO_REORG_RAW_DATA\n$RAW" + fi + + done + + ## sort the indexes based on REORG column + VALID_INDEXES_TO_REORG_RAW_DATA=$( echo -e "$VALID_INDEXES_TO_REORG_RAW_DATA" | sort -k21 -r); + VALID_INDEXES_TO_REORG=$( echo -e "$VALID_INDEXES_TO_REORG_RAW_DATA" | awk '{print $1"."$2"."$3"."$4}' ); + NUM_VALID_INDEXES_TO_REORG=$( echo -e "$VALID_INDEXES_TO_REORG" | wc -l ); + + return 0 + } getValidFragmentedIndexes() { - ## - ## http://www.ibm.com/developerworks/data/library/techarticle/dm-1307optimizerunstats/#Listing%207 - ## - - getValidTableSizes - - VALID_FRAGMENTED_INDEXES_RAW_DATA=$( db2 -x "select rtrim(tabschema)||' '||rtrim(tabname)||' '||rtrim(indschema)||' '||rtrim(indname) - ||' '||indcard||' '||stats_time||' '||lastused||' '||nleaf||' '||sequential_pages - from syscat.indexes where tabschema='$SCHEMANAME_IN' - and not (nleaf = 1 and sequential_pages = 0) - and not (nleaf = 0 and sequential_pages = 1) - and (nleaf - sequential_pages > 10) - and tabname in ( $VALID_TABLES_FORMATTED ) - order by tabname - with ur"; ); - - VALID_FRAGMENTED_INDEXES_RAW_DATA=$(echo "${VALID_FRAGMENTED_INDEXES_RAW_DATA}" | sed 's/ *$//g' ); - VALID_FRAGMENTED_INDEXES=$( echo "${VALID_FRAGMENTED_INDEXES_RAW_DATA}" | sed 's/ *$//g' | cut -d' ' -f2 | uniq ) - NUM_VALID_FRAGMENTED_INDEXES=$( echo "${VALID_FRAGMENTED_INDEXES}" | wc -l ) - - + ## + ## http://www.ibm.com/developerworks/data/library/techarticle/dm-1307optimizerunstats/#Listing%207 + ## + + getValidTableSizes + + VALID_FRAGMENTED_INDEXES_RAW_DATA=$( db2 -x "select rtrim(tabschema)||' '||rtrim(tabname)||' '||rtrim(indschema)||' '||rtrim(indname) + ||' '||indcard||' '||stats_time||' '||lastused||' '||nleaf||' '||sequential_pages + from syscat.indexes where tabschema='$SCHEMANAME_IN' + and not (nleaf = 1 and sequential_pages = 0) + and not (nleaf = 0 and sequential_pages = 1) + and (nleaf - sequential_pages > 10) + and tabname in ( $VALID_TABLES_FORMATTED ) + order by tabname + with ur"; ); + + VALID_FRAGMENTED_INDEXES_RAW_DATA=$(echo "${VALID_FRAGMENTED_INDEXES_RAW_DATA}" | sed 's/ *$//g' ); + VALID_FRAGMENTED_INDEXES=$( echo "${VALID_FRAGMENTED_INDEXES_RAW_DATA}" | sed 's/ *$//g' | cut -d' ' -f2 | uniq ) + NUM_VALID_FRAGMENTED_INDEXES=$( echo "${VALID_FRAGMENTED_INDEXES}" | wc -l ) } getValidTableSizes() { - VALID_TABLE_SIZES_RAW_DATA=$( db2 "select t0.tabname, - ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) as TOTAL_TABLE_MB, - cast ((INDEX_OBJECT_P_SIZE / 1024) as integer) as INDEX_SIZE_MB - from SYSIBMADM.ADMINTABINFO t0, SYSCAT.TABLES t1 - where t0.tabschema='$SCHEMANAME_IN' - and t0.tabschema=t1.tabschema - and t0.tabname=t1.tabname - $IGNORE_TABLES - and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) < $IGNORE_TABLE_SIZE_THRESHOLD_MAX - and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) > $IGNORE_TABLE_SIZE_THRESHOLD_MIN - order by 2 desc - with ur"; ); - rc=$? - if [ $rc -eq 0 ]; then - VALID_TABLE_SIZES=$( echo "$VALID_TABLE_SIZES_RAW_DATA" | sed '1,3d' | sed '$d' | sed '$d' ); - VALID_TABLE_SIZES=$( echo "$VALID_TABLE_SIZES" | awk 'BEGIN {ORS="\t"} { for(ii=1 ; ii<=NF ; ii++) print $ii; printf "\n"; }'); - VALID_TABLES=$( echo "$VALID_TABLE_SIZES" | awk '{print $1}' ); - NUM_VALID_TABLES=$( echo "$VALID_TABLE_SIZES" | wc -l ); - # log 3 "NUM_VALID_TABLES=$NUM_VALID_TABLES" - - VALID_TABLES_FORMATTED="" - for TABLE in $VALID_TABLES - do - VALID_TABLES_FORMATTED="$VALID_TABLES_FORMATTED'$TABLE'," - done - VALID_TABLES_FORMATTED=$( echo "$VALID_TABLES_FORMATTED" | sed 's/,$//g' ) - else - VALID_TABLES_FORMATTED="'UNKNOWN_TABNAME'" - fi + VALID_TABLE_SIZES_RAW_DATA=$( db2 "select t0.tabname, + ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) as TOTAL_TABLE_MB, + cast ((INDEX_OBJECT_P_SIZE / 1024 ) as integer) as INDEX_SIZE_MB + from SYSIBMADM.ADMINTABINFO t0, SYSCAT.TABLES t1 + where t0.tabschema='$SCHEMANAME_IN' + and t0.tabschema=t1.tabschema + and t0.tabname=t1.tabname + $IGNORE_TABLES + and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) < $IGNORE_TABLE_SIZE_THRESHOLD_MAX + and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) > $IGNORE_TABLE_SIZE_THRESHOLD_MIN + order by 2 desc + with ur"; ); + rc=$? + if [ $rc -eq 0 ]; then + VALID_TABLE_SIZES=$( echo "$VALID_TABLE_SIZES_RAW_DATA" | sed '1,3d' | sed '$d' | sed '$d' ); + VALID_TABLE_SIZES=$( echo "$VALID_TABLE_SIZES" | awk 'BEGIN {ORS="\t"} { for(ii=1 ; ii<=NF ; ii++) print $ii; printf "\n"; }'); + VALID_TABLES=$( echo "$VALID_TABLE_SIZES" | awk '{print $1}' ); + NUM_VALID_TABLES=$( echo "$VALID_TABLE_SIZES" | wc -l ); + # log 3 "NUM_VALID_TABLES=$NUM_VALID_TABLES" + + VALID_TABLES_FORMATTED="" + for TABLE in $VALID_TABLES + do + VALID_TABLES_FORMATTED="$VALID_TABLES_FORMATTED'$TABLE'," + done + VALID_TABLES_FORMATTED=$( echo "$VALID_TABLES_FORMATTED" | sed 's/,$//g' ) + else + VALID_TABLES_FORMATTED="'UNKNOWN_TABNAME'" + fi } ## is TABLE within size limits < IGNORE_TABLE_SIZE_THRESHOLD_MAX and > IGNORE_TABLE_SIZE_THRESHOLD_MIN isTableWithinSizeLimit() { - local SCHEMANAME=$1 - local TABNAME=$2 - local RC="" - local rc=0 - - RC=$( db2 -x "select tabname, - ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) as TOTAL_TABLE_MB - from SYSIBMADM.ADMINTABINFO - where tabschema='$SCHEMANAME' - and tabname = '$TABNAME' - and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) < $IGNORE_TABLE_SIZE_THRESHOLD_MAX - and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) > $IGNORE_TABLE_SIZE_THRESHOLD_MIN - order by 2 desc - with ur" ); - - rc=$? - return $rc + local SCHEMANAME=$1 + local TABNAME=$2 + local RC="" + local rc=0 + + RC=$( db2 -x "select tabname, + ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) as TOTAL_TABLE_MB + from SYSIBMADM.ADMINTABINFO + where tabschema='$SCHEMANAME' + and tabname = '$TABNAME' + and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) < $IGNORE_TABLE_SIZE_THRESHOLD_MAX + and ( ( DATA_OBJECT_P_SIZE + INDEX_OBJECT_P_SIZE + LONG_OBJECT_P_SIZE + LOB_OBJECT_P_SIZE + XML_OBJECT_P_SIZE ) / 1024 ) > $IGNORE_TABLE_SIZE_THRESHOLD_MIN + order by 2 desc + with ur" ); + + rc=$? + return $rc } ## create an list/array of table objects to reorg based on tabnames createTableOBJECT_ARRAY() { - local TABNAMES="$1" - local OBJECT_REORG_TABLE_TYPE=$2 - - ## - ## make the OBJECT_ARRAY for tables and indexes - ## - if [ -z "$OBJECT_ARRAY" ]; then - local let index=0; - else - local let index=${#OBJECT_ARRAY[@]}; - fi - local TID=0 ## Table ID - always 0 for online table reorg - local INDSCHEMA=NULL; - local INDNAME=NULL; - local LOCK_COUNT=0 - - for TABNAME in $TABNAMES - do - ## we need TABLEID, TBSPACEID for IF_STATS as the full TableName: may not be dispalyed in the db2pd output - local RC=$( db2 -x "select TABLEID, TBSPACEID from syscat.tables where tabname='$TABNAME' and tabschema='$SCHEMANAME_IN'" ) - local rc=$? - if [ $rc -eq 0 ]; then - local TABLEID=$( echo $RC | awk '{print $1}' ); - local TBSPACEID=$( echo $RC | awk '{print $2}' ); - OBJECT_ARRAY[$index]="$SCHEMANAME_IN#$TABNAME#$INDSCHEMA#$INDNAME#$TID#NOTSTARTED#2019-01-01-00.00.00#2019-01-01-00.00.00#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE" - let index+=1 - fi - done + local TABNAMES="$1" + local OBJECT_REORG_TABLE_TYPE=$2 + + ## + ## make the OBJECT_ARRAY for tables and indexes + ## + if [ -z "$OBJECT_ARRAY" ]; then + local let index=0; + else + local let index=${#OBJECT_ARRAY[@]}; + fi + local TID=0 ## Table ID - always 0 for online table reorg + local INDSCHEMA=NULL; + local INDNAME=NULL; + local LOCK_COUNT=0 + + for TABNAME in $TABNAMES + do + ## we need TABLEID, TBSPACEID for IF_STATS as the full TableName: may not be dispalyed in the db2pd output + local RC=$( db2 -x "select TABLEID, TBSPACEID from syscat.tables where tabname='$TABNAME' and tabschema='$SCHEMANAME_IN'" ) + local rc=$? + if [ $rc -eq 0 ]; then + local TABLEID=$( echo $RC | awk '{print $1}' ); + local TBSPACEID=$( echo $RC | awk '{print $2}' ); + OBJECT_ARRAY[$index]="$SCHEMANAME_IN#$TABNAME#$INDSCHEMA#$INDNAME#$TID#NOTSTARTED#-#-#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE" + let index+=1 + fi + done } @@ -403,60 +404,66 @@ createTableOBJECT_ARRAY() createIndexOBJECT_ARRAY() { - local INDNAMES="$1" - local OBJECT_REORG_TABLE_TYPE=$2 - local let index=0 - local TABLEID=9999; - local TBSPACEID=9999; - local LOCK_COUNT=0; - - for INDEX in $INDNAMES - do - local TABSCHEMA=$( echo $INDEX | cut -d. -f1); - local TABNAME=$( echo $INDEX | cut -d. -f2); - local INDSCHEMA=$( echo $INDEX | cut -d. -f3); - local INDNAME=$( echo $INDEX | cut -d. -f4); - local RC=$( db2 -x "select IID from syscat.indexes where tabschema = '$TABSCHEMA' and tabname = '$TABNAME' and indschema = '$INDSCHEMA' and indname = '$INDNAME'"); - local rc=$? - if [ $rc -eq 0 ]; then - local IID=$( echo $RC | cut -d' ' -f1); - OBJECT_ARRAY[$index]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#NOTSTARTED#2019-01-01-00.00.00#2019-01-01-00.00.00#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE" - let index+=1 - fi - done + local INDNAMES="$1" + local OBJECT_REORG_TABLE_TYPE=$2 + local let index=0 + local TABLEID=9999; + local TBSPACEID=9999; + local LOCK_COUNT=0; + + for INDEX in $INDNAMES + do + local TABSCHEMA=$( echo $INDEX | cut -d. -f1); + local TABNAME=$( echo $INDEX | cut -d. -f2); + local INDSCHEMA=$( echo $INDEX | cut -d. -f3); + local INDNAME=$( echo $INDEX | cut -d. -f4); + local RC=$( db2 -x "select IID from syscat.indexes where tabschema = '$TABSCHEMA' and tabname = '$TABNAME' and indschema = '$INDSCHEMA' and indname = '$INDNAME'"); + local rc=$? + if [ $rc -eq 0 ]; then + local IID=$( echo $RC | cut -d' ' -f1); + OBJECT_ARRAY[$index]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#NOTSTARTED#-#-#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE" + let index+=1 + fi + done } - ## ## list out the objects and state ## this can be used for debugging ## listOBJECT_ARRAY() { - - local ii; - log 3 "The following is for debug purposes, Num objects=${#OBJECT_ARRAY[@]}, $OBJECT_NUM_TB_STATS:$OBJECT_NUM_IX_STATS:$OBJECT_NUM_IF_STATS" - for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) - do - echo "${OBJECT_ARRAY[$ii]}" | tee -a $REORG_TABLE_INDEX_DEBUG - done - + + local ii; + log 3 "The following is for debug purposes, Num objects=${#OBJECT_ARRAY[@]}, $OBJECT_NUM_TB_STATS:$OBJECT_NUM_IX_STATS:$OBJECT_NUM_IF_STATS" + printf "%-15s %-40s %-15s %-40s %-14s %-20s %-20s %-8s \n" "TABSCHEMA" "TABNAME" "INDSCHEMA" "INDNAME" "REORG_STATUS" "REORG_START" "REORG_END" "LOCK_COUNT" + : ' for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) + do + echo "${OBJECT_ARRAY[$ii]}" | /usr/bin/column -t -s "#" | tee -a $REORG_TABLE_INDEX_DEBUG + done + ' + for ((ii=0; ii<${#OBJECT_ARRAY[@]}; ii++)) + do + IFS="#" read -r TABSCHEMA TABNAME INDSCHEMA INDNAME IID REORG_STATUS OBJECT_REORG_START OBJECT_REORG_END TABLEID TBSPACEID LOCK_COUNT <<< "${OBJECT_ARRAY[$ii]}" + + printf "%-15s %-40s %-15s %-40s %-14s %-20s %-20s %-8s \n" "$TABSCHEMA" "$TABNAME" "$INDSCHEMA" "$INDNAME" "$REORG_STATUS" "$OBJECT_REORG_START" "$OBJECT_REORG_END" "$LOCK_COUNT" + done } ## get the number tb_stats, ix_stats and if_stats objects getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY() { - local rc=0; - local ii; - for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) - do - if [ $( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f12 ) -eq $1 ]; then - let rc+=1; - fi - done + local rc=0; + local ii; + for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) + do + if [ $( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f12 ) -eq $1 ]; then + let rc+=1; + fi + done - return $rc; + return $rc; } @@ -466,379 +473,382 @@ getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY() reorgTables() { - ## variables that need to be reset on each run of the function - NUM_REORGS_IN_PROGRESS=0; - NUM_REORGS_KICKED_OFF=0; - NUM_REORGS_COMPLETED=0; - NUM_REORGS_STOPPED=0; - NUM_REORGS_ABORTED=0; - - while true - do - - ## check reorg window maintenance time - MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS=$( date '+%s' ); - DIFF=$(( MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS - REORG_TIMEOUT_WINDOW_START_TIME_SECONDS )); - - ## safety valve - if for some reason the logic can't stop the reorgs - if [ $DIFF -ge $(( REORG_TIMEOUT_WINDOW_SECONDS + REORG_TIMEOUT_OVERFLOW_VALVE )) ]; then - log 1 "REORG_TIMEOUT_OVERFLOW_VALVE detected"; - log 1 "Aborting reorgs" - break; - fi - - if [ $DIFF -ge $REORG_TIMEOUT_WINDOW_SECONDS ]; then - - REORG_TIMEOUT_WINDOW_COMPLETED=1; - - ## -twa: timeout window action: default=3 - ## 1=allow current reorg(s) to continue - ## 2=stop current reorg(s) - ## 3=stop current reorg(s) if < 80% complete - if [ $REORG_TIMEOUT_WINDOW_ACTION -eq 1 ]; then - - log 3 "Reorg window ending, reorg window time exceeded, twa=$REORG_TIMEOUT_WINDOW_ACTION" - break - - elif [ $REORG_TIMEOUT_WINDOW_ACTION -eq 2 ]; then - - ## use of the REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER - ## 0 = ABORT those not started and issue a STOP to those STARTED - ## 1 = loop again and see if script exits as all reorgs are COMPLETED and STOPPED and ABORTED - ## 2 = break out - if [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 0 ]; then - let REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER+=1 - log 3 "Reorg window ending, reorg window time exceeded, twa=$REORG_TIMEOUT_WINDOW_ACTION" - log 3 "Aborting reorgs NOTSTARTED and issuing a STOP to those that are STARTED" - elif [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 1 ]; then - let REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER+=1 - log 3 "REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER=$REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER" - elif [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 2 ]; then - log 3 "REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER=$REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER" - log 3 "breaking out of reorg loop" - break; - fi - - fi - - fi - - ## loop for all OBJECTS - extract relevant data from OBJECT array - ## if we have NOTSTARTED in the OBJECT_ARRAY[N] - then kick off a reorg - ## then check tables reorg status - for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) - do - ## get table related info - TABSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f1 ); - TABNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f2 ); - INDSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f3 ); - INDNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f4 ); - IID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f5 ); - OBJECT_REORG_STATUS=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f6 ); - OBJECT_REORG_START=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f7 ); - OBJECT_REORG_END=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f8 ); - TABLEID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f9 ); - TBSPACEID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f10 ); - LOCK_COUNT=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f11 ); - OBJECT_REORG_TABLE_TYPE=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f12 ); - isTable=0; - isIndex=0; - if [ "$INDSCHEMA" == "NULL" -a "$INDNAME" == "NULL" ]; then - isTable=1; - else - isIndex=1; - fi - - if [ $OBJECT_REORG_TABLE_TYPE -ne $REORG_TABLE_TYPE ]; then - continue; - fi - - # log 5 "${OBJECT_ARRAY[$ii]}" - - ## - ## has OBJECT COMPLETED - no need to continue here - ## - if [ "$OBJECT_REORG_STATUS" == "COMPLETED" -o "$OBJECT_REORG_STATUS" == "STOPPED" -o "$OBJECT_REORG_STATUS" == "ABORTED" ]; then - continue; - fi - - if [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then - - ## - ## query db2 for REORG_STATUS etc - there may not be an entry so carry on - ## - RC_SNAP=$( db2 -x "select REORG_STATUS, REORG_COMPLETION, REORG_PHASE, REORG_CURRENT_COUNTER, REORG_MAX_COUNTER, REORG_START, REORG_END, REORG_INDEX_ID, REORG_TBSPC_ID from table(snap_get_tab_reorg('')) where tabschema='$TABSCHEMA' and tabname='$TABNAME' and REORG_START > TIMESTAMP('$WINDOW_START_TIME_DB2') and REORG_INDEX_ID=$IID"); - rc=$? - if [ $rc -ge 2 ]; then - log 0 "Possible error running select query against db2\nrc=$rc\nRC=$RC" - continue; - fi - - if [ $rc -eq 0 ]; then - RC=$( echo "$RC_SNAP" | awk 'BEGIN {ORS="\t"} { for(ii=1 ; ii<=NF ; ii++) print $ii; }') - REORG_STATUS=$( echo "$RC_SNAP" | awk '{print $1}' ); - REORG_COMPLETION=$( echo "$RC_SNAP" | awk '{print $2}' ); - REORG_CURRENT_COUNTER=$( echo "$RC_SNAP" | awk '{print $4}' ); - REORG_MAX_COUNTER=$( echo "$RC_SNAP" | awk '{print $5}' ); - REORG_START=$( echo "$RC_SNAP" | awk '{print $6}' ); - REORG_END=$( echo "$RC_SNAP" | awk '{print $7}' ); - REORG_INDEX_ID=$( echo "$RC_SNAP" | awk '{print $8}' ); - - REORG_PERCENT_COMPLETE=0 - if [ ! -z "$REORG_CURRENT_COUNTER" -a $REORG_CURRENT_COUNTER -gt 0 ]; then - if [ ! -z "$REORG_MAX_COUNTER" -a $REORG_MAX_COUNTER -gt 0 ]; then - if [ $REORG_MAX_COUNTER -ge $REORG_CURRENT_COUNTER ]; then - REORG_PERCENT_COMPLETE=$( echo $REORG_CURRENT_COUNTER $REORG_MAX_COUNTER | awk '{ print int (($1/$2)*100) }' ); - fi - fi - fi - fi - - # log 5 "RC_SNAP=$RC_SNAP" - - elif [ $IF_STATS -eq 3 ]; then - - DB2PD_REORG_INDEX_RECORD=$( db2pd -db $DBNAME -reorgs index | grep -B1 -A11 -w "^TbspaceID: $TBSPACEID" | grep -B1 -A11 -w "TableID: $TABLEID" ); - rc=$? - if [ $rc -eq 0 ]; then - REORG_START=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep '^Start Time:' | awk '{ print $3,$4}' ); - REORG_START_SECONDS=$(date --d="$REORG_START" '+%s'); - - if [ $REORG_START_SECONDS -ge $IF_STATS_WINDOW_START_TIME_DB2 ]; then - REORG_STATUS=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep '^Status:' | awk '{ $1=""; print $0}' | sed 's/^[ \t]*//;s/[ \t]*$//' ); - ## some differences between snap_get_tab_reorg and db2pd -reogrs index output - if [ "$REORG_STATUS" == "In Progress" ]; then - REORG_STATUS="STARTED"; - elif [ "$REORG_STATUS" == "Completed" ]; then - REORG_STATUS="COMPLETED"; - elif [ "$REORG_STATUS" == "Stopped" ]; then - REORG_STATUS="STOPPED"; - fi - - REORG_END=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep '^Start Time:' | grep 'End Time:' | awk '{ print $7,$8}' ); - fi - - fi - - - fi - - - ## - ## has OBJECT been KICKED_OFF or STARTED - ## if it has check to see if is STARTED or COMPLETED - ## update OBJECT_ARRAY - ## update COMPLETION/STOPPED stats - ## - if [ "$OBJECT_REORG_STATUS" == "KICKED_OFF" ] || [ "$OBJECT_REORG_STATUS" == "STARTED" ]; then - if [ ! -z "$REORG_STATUS" ]; then - if [ "$REORG_STATUS" == "STARTED" -o "$REORG_STATUS" == "COMPLETED" -o "$REORG_STATUS" == "STOPPED" ]; then - OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$REORG_STATUS#$REORG_START#$REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE" - fi - - if [ "$REORG_STATUS" == "COMPLETED" -o "$REORG_STATUS" == "STOPPED" ]; then - - removeTABLE_TABLE_IN_USE_ARRAY "$TABSCHEMA.$TABNAME" - rc=$? - if [ $rc -eq 1 ]; then - log 1 "Failed to remove table $TABSCHEMA.$TABNAME from TABLE_IN_USE_ARRAY"; - fi - - if [ "$REORG_STATUS" == "COMPLETED" ]; then - let NUM_REORGS_COMPLETED+=1 - elif [ "$REORG_STATUS" == "STOPPED" ]; then - let NUM_REORGS_STOPPED+=1 - fi - - let NUM_REORGS_IN_PROGRESS-=1 - fi - - fi - - ## - ## OBJECT is NOSTARTED so KICK_OFF a reorg - ## - elif [ "$OBJECT_REORG_STATUS" == "NOTSTARTED" ]; then - - ## dont kick off any reorgs if window timeout passed and TWA=2 - ## OBJECTS become ABORTED - UPDATE OBJECT_ARRAY - if [ $REORG_TIMEOUT_WINDOW_COMPLETED -eq 1 ] && [ $REORG_TIMEOUT_WINDOW_ACTION -eq 2 ] && [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 1 ]; then - REORG_STATUS=ABORTED; - OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$REORG_STATUS#$OBJECT_REORG_START#$OBJECT_REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE"; - let NUM_REORGS_ABORTED+=1; - continue; - - fi - - ## is the TABLE already being used -if it is goto next OBJECT - existTABLE_TABLE_IN_USE_ARRAY "$TABSCHEMA.$TABNAME" - rc=$? - [ $rc -eq 0 ] && continue; - - ## we only want to kick off so many reorgs at any one time - if [ $NUM_REORGS_IN_PROGRESS -eq $MAX_ASYNC_REORGS_ALLOWED ]; then - continue; - fi - - ## - ## The table could already be locked - if it is then by-pass it - ## and ABORT if locked more than 10 times - ## - TABLE_LOCKED=$( db2 "select APPLICATION_HANDLE, LOCK_OBJECT_TYPE, LOCK_MODE, LOCK_CURRENT_MODE, LOCK_STATUS, LOCK_COUNT, LOCK_HOLD_COUNT, TBSP_ID, TAB_FILE_ID from TABLE (MON_GET_LOCKS(NULL, -2)) where TBSP_ID=$TBSPACEID and TAB_FILE_ID=$TABLEID and LOCK_OBJECT_TYPE='TABLE' and LOCK_MODE='IX' with ur"; ); - rc=$? - if [ $rc -eq 0 ]; then - let LOCK_COUNT+=1; - log 1 "Appears table $TABSCHEMA.$TABNAME is already locked by another application(s), LOCK_COUNT=$LOCK_COUNT"; - log 1 "$TABLE_LOCKED"; - if [ $LOCK_COUNT -gt 10 ]; then - OBJECT_REORG_STATUS=ABORTED; - let NUM_REORGS_ABORTED+=1; - log 1 "Aborting table $TABSCHEMA.$TABNAME , LOCK_COUNT=$LOCK_COUNT"; - fi -OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$OBJECT_REORG_STATUS#$OBJECT_REORG_START#$OBJECT_REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE"; - continue; - fi - - ## - ## do a check to see where we are on transaction log space - this could be improved!!!! - ## - LOG_USED=$( db2 "select cast(LOG_UTILIZATION_PERCENT as decimal(5,2)) as PCTUSED, cast((TOTAL_LOG_USED_KB/1024) as Integer) as TOTUSEDMB, cast((TOTAL_LOG_AVAILABLE_KB/1024) as Integer) as TOTAVAILMB, cast((TOTAL_LOG_USED_TOP_KB/1024) as Integer) as TOTUSEDTOPMB FROM SYSIBMADM.LOG_UTILIZATION "); - if [ ! -z "$LOG_USED" ]; then - PCTUSED=$( echo "$LOG_USED" | awk '{ if(NF==4 && $2 ~/^[0-9]+$/) print int($1)}' ); - if [ ! -z "$PCTUSED" ]; then - if [ $PCTUSED -gt $TRANSACTION_LOG_THRESHOLD_PCT ]; then - log 1 "Will not kick off another reorg due to logfile PCTUSED above threshold of $LOG_THRESHOLD\n$LOG_USED" - continue - else - log 3 "$LOG_USED" - fi - fi - fi - - ## - ## kick off another reorg - ## if rc=0 then ok, else we ABORT the OBJECT and don't try again - ## - log 3 "" - - if [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then - - if [ $isTable -eq 1 ]; then - - db2 -v "reorg table $TABSCHEMA.$TABNAME inplace allow write access" - rc=$? - - elif [ $isIndex -eq 1 ]; then - db2 -v "reorg table $TABSCHEMA.$TABNAME index $INDSCHEMA.$INDNAME inplace allow write access" - rc=$? - fi - - elif [ $IF_STATS -eq 3 ]; then - - ## for offline we throw a job at db2 and wait a few seconds and check the output - ## output could be "reorg indexes all for table ...", - ## or SQL error - ## or 'DB20000I The REORG command completed successfully.' - ## not sure if there is a better way to do this - TMPLOG="/tmp/$TABSCHEMA.$TABNAME.tmp"; - db2 -v "reorg indexes all for table $TABSCHEMA.$TABNAME allow write access" > $TMPLOG 2>&1 & - sleep 5; - cat $TMPLOG; - RC=$( grep '^SQL' $TMPLOG); - rc=$? - if [ $rc -eq 0 ]; then - log 1 "Failed to kick off reorg\n$RC"; - rc=1; - else - ## reorg could have finished then no need for the big sleep - RC=$( grep 'DB20000I The REORG command completed successfully.' $TMPLOG); - if [ $? -eq 0 ]; then - IF_STATS_BYPASS_SLEEP_INTERVAL_TIME=1; - fi - rc=0; - - fi - rm -f $TMPLOG; - - fi - - if [ $rc -eq 0 ]; then - REORG_STATUS=KICKED_OFF; - else - REORG_STATUS=ABORTED; - fi - OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$REORG_STATUS#$OBJECT_REORG_START#$OBJECT_REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE"; - if [ $rc -ne 0 ]; then - let NUM_REORGS_ABORTED+=1; - continue - fi - - ## add the table to the TABLE_IN_USE_ARRAY - addTABLE_TABLE_IN_USE_ARRAY "$TABSCHEMA.$TABNAME" - rc=$? - if [ $rc -eq 1 ]; then - log 1 "Failed to add table $TABSCHEMA.$TABNAME to TABLE_IN_USE_ARRAY"; - fi - let NUM_REORGS_KICKED_OFF+=1; - let NUM_REORGS_IN_PROGRESS+=1; - - continue; - - fi - - ## - ## ouput STATUS - ## - if [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then - log 3 "$NUM_REORGS_IN_PROGRESS:$NUM_REORGS_KICKED_OFF:$NUM_REORGS_ABORTED:$NUM_REORGS_STOPPED:$NUM_REORGS_COMPLETED:$NUM_REORG_OBJECTS $TABSCHEMA.$TABNAME : $RC : $REORG_PERCENT_COMPLETE %" | tee -a $REORG_TABLE_INDEX_DEBUG - elif [ $IF_STATS -eq 3 ]; then - log 3 "$NUM_REORGS_IN_PROGRESS:$NUM_REORGS_KICKED_OFF:$NUM_REORGS_ABORTED:$NUM_REORGS_STOPPED:$NUM_REORGS_COMPLETED:$NUM_REORG_OBJECTS $TABSCHEMA.$TABNAME \n$DB2PD_REORG_INDEX_RECORD "| tee -a $REORG_TABLE_INDEX_DEBUG - - fi - - ## - ## if reorg timeout then issue a stop to current reorgs that are STARTED - ## no error checking for stopping a reorg - ## no need to update OBJECT array as it will be updated on next loop - ## - - if [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then - - if [ "$REORG_STATUS" == "STARTED" ] && [ $REORG_TIMEOUT_WINDOW_COMPLETED -eq 1 ] && [ $REORG_TIMEOUT_WINDOW_ACTION -eq 2 ] && [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 1 ]; then - - - if [ $isTable -eq 1 ]; then - db2 -v "reorg table $TABSCHEMA.$TABNAME inplace stop" - rc=$? - elif [ $isIndex -eq 1 ]; then - db2 -v "reorg table $TABSCHEMA.$TABNAME index $INDSCHEMA.$INDNAME inplace stop" - rc=$? - fi - - fi - - fi - - done ## for OBJECT_ARRAY[@] - - ## check if we are done with all OBJECTS - if [ $((NUM_REORGS_COMPLETED + NUM_REORGS_STOPPED + NUM_REORGS_ABORTED)) -ge $NUM_REORG_OBJECTS ]; then - log 3 "All reorgs are completed, stopped or aborted, $NUM_REORGS_COMPLETED:$NUM_REORGS_STOPPED:$NUM_REORGS_ABORTED:$NUM_REORG_OBJECTS" - break - fi - - ## wait some time - if [ $IF_STATS -eq 3 ] && [ $IF_STATS_BYPASS_SLEEP_INTERVAL_TIME -eq 1 ]; then - sleep 1; - IF_STATS_BYPASS_SLEEP_INTERVAL_TIME=0; - else - sleep $SLEEP_INTERVAL_TIME - fi - - done ## while true + ## variables that need to be reset on each run of the function + NUM_REORGS_IN_PROGRESS=0; + NUM_REORGS_KICKED_OFF=0; + NUM_REORGS_COMPLETED=0; + NUM_REORGS_STOPPED=0; + NUM_REORGS_ABORTED=0; + + while true + do + + ## check reorg window maintenance time + MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS=$( date '+%s' ); + DIFF=$(( MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS - REORG_TIMEOUT_WINDOW_START_TIME_SECONDS )); + + ## safety valve - if for some reason the logic can't stop the reorgs + if [ $DIFF -ge $(( REORG_TIMEOUT_WINDOW_SECONDS + REORG_TIMEOUT_OVERFLOW_VALVE )) ]; then + log 1 "REORG_TIMEOUT_OVERFLOW_VALVE detected"; + log 1 "Aborting reorgs" + break; + fi + + if [ $DIFF -ge $REORG_TIMEOUT_WINDOW_SECONDS ]; then + + REORG_TIMEOUT_WINDOW_COMPLETED=1; + + ## -twa: timeout window action: default=3 + ## 1=allow current reorg(s) to continue + ## 2=stop current reorg(s) + ## 3=stop current reorg(s) if < 80% complete + if [ $REORG_TIMEOUT_WINDOW_ACTION -eq 1 ]; then + + log 3 "Reorg window ending, reorg window time exceeded, twa=$REORG_TIMEOUT_WINDOW_ACTION" + break + + elif [ $REORG_TIMEOUT_WINDOW_ACTION -eq 2 ]; then + + ## use of the REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER + ## 0 = ABORT those not started and issue a STOP to those STARTED + ## 1 = loop again and see if script exits as all reorgs are COMPLETED and STOPPED and ABORTED + ## 2 = break out + if [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 0 ]; then + let REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER+=1 + log 3 "Reorg window ending, reorg window time exceeded, twa=$REORG_TIMEOUT_WINDOW_ACTION" + log 3 "Aborting reorgs NOTSTARTED and issuing a STOP to those that are STARTED" + elif [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 1 ]; then + let REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER+=1 + log 3 "REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER=$REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER" + elif [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 2 ]; then + log 3 "REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER=$REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER" + log 3 "Breaking out of reorg loop" + break; + fi + fi + fi + + ## loop for all OBJECTS - extract relevant data from OBJECT array + ## if we have NOTSTARTED in the OBJECT_ARRAY[N] - then kick off a reorg + ## then check tables reorg status + for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) + do + ## get table related info + TABSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f1 ); + TABNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f2 ); + INDSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f3 ); + INDNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f4 ); + IID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f5 ); + OBJECT_REORG_STATUS=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f6 ); + OBJECT_REORG_START=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f7 ); + OBJECT_REORG_END=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f8 ); + TABLEID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f9 ); + TBSPACEID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f10 ); + LOCK_COUNT=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f11 ); + OBJECT_REORG_TABLE_TYPE=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f12 ); + isTable=0; + isIndex=0; + if [ "$INDSCHEMA" == "NULL" -a "$INDNAME" == "NULL" ]; then + isTable=1; + else + isIndex=1; + fi + + if [ $OBJECT_REORG_TABLE_TYPE -ne $REORG_TABLE_TYPE ]; then + continue; + fi + + # log 5 "${OBJECT_ARRAY[$ii]}" + + ## + ## has OBJECT COMPLETED - no need to continue here + ## + if [[ "$OBJECT_REORG_STATUS" == "COMPLETED" || "$OBJECT_REORG_STATUS" == "STOPPED" || "$OBJECT_REORG_STATUS" == "ABORTED" ]]; then + continue; + fi + + if [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then + + ## + ## query db2 for REORG_STATUS etc - there may not be an entry so carry on + ## + RC_SNAP=$( db2 -x "select REORG_STATUS, REORG_COMPLETION, REORG_PHASE, REORG_CURRENT_COUNTER, REORG_MAX_COUNTER, VARCHAR_FORMAT(REORG_START, 'YYYY-MM-DD-HH24:MI:SS') as REORG_START, VARCHAR_FORMAT(REORG_END, 'YYYY-MM-DD-HH24:MI:SS') as REORG_END, REORG_INDEX_ID, REORG_TBSPC_ID from table(snap_get_tab_reorg('')) where tabschema='$TABSCHEMA' and tabname='$TABNAME' and REORG_START > TIMESTAMP('$WINDOW_START_TIME_DB2') and REORG_INDEX_ID=$IID"); + rc=$? + if [ $rc -ge 2 ]; then + log 0 "Possible error running select query against db2\nrc=$rc\nRC=$RC" + continue; + fi + + if [ $rc -eq 0 ]; then + RC=$( echo "$RC_SNAP" | awk 'BEGIN {ORS="\t"} { for(ii=1 ; ii<=NF ; ii++) print $ii; }') + REORG_STATUS=$( echo "$RC_SNAP" | awk '{print $1}' ); + REORG_COMPLETION=$( echo "$RC_SNAP" | awk '{print $2}' ); + REORG_CURRENT_COUNTER=$( echo "$RC_SNAP" | awk '{print $4}' ); + REORG_MAX_COUNTER=$( echo "$RC_SNAP" | awk '{print $5}' ); + REORG_START=$( echo "$RC_SNAP" | awk '{print $6}' ); + REORG_END=$( echo "$RC_SNAP" | awk '{print $7}' ); + REORG_INDEX_ID=$( echo "$RC_SNAP" | awk '{print $8}' ); + + declare -i REORG_PERCENT_COMPLETE=0 + if [[ ! -z "$REORG_CURRENT_COUNTER" && $REORG_CURRENT_COUNTER -gt 0 ]]; then + if [[ ! -z "$REORG_MAX_COUNTER" && $REORG_MAX_COUNTER -gt 0 ]]; then + if [ $REORG_MAX_COUNTER -ge $REORG_CURRENT_COUNTER ]; then + REORG_PERCENT_COMPLETE=$(echo $REORG_CURRENT_COUNTER $REORG_MAX_COUNTER | awk '{ print int (($1/$2)*100) }' ); + fi + fi + fi + fi + + # log 5 "RC_SNAP=$RC_SNAP" + + elif [ $IF_STATS -eq 3 ]; then + + DB2PD_REORG_INDEX_RECORD=$( db2pd -db $DBNAME -reorgs index | grep -B1 -A11 -w "^TbspaceID: $TBSPACEID" | grep -B1 -A11 -w "TableID: $TABLEID" ); + rc=$? + if [ $rc -eq 0 ]; then + REORG_START=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep '^Start Time:' | awk '{ print $3,$4}' ); + REORG_START_SECONDS=$(date --date="$REORG_START" '+%s'); + + if [ $REORG_START_SECONDS -ge $IF_STATS_WINDOW_START_TIME_DB2 ]; then + REORG_STATUS=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep '^Status:' | awk '{ $1=""; print $0}' | sed 's/^[ \t]*//;s/[ \t]*$//' ); + ## some differences between snap_get_tab_reorg and db2pd -reogrs index output + if [ "$REORG_STATUS" == "In Progress" ]; then + REORG_STATUS="STARTED"; + elif [ "$REORG_STATUS" == "Completed" ]; then + REORG_STATUS="COMPLETED"; + elif [ "$REORG_STATUS" == "Stopped" ]; then + REORG_STATUS="STOPPED"; + fi + + REORG_END=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep '^Start Time:' | grep 'End Time:' | awk '{ print $7,$8}' ); + REORG_IND_CUR_STATUS=$( echo "$DB2PD_REORG_INDEX_RECORD" | grep "Status:" ); + REORG_IND_CUR=$(echo "$DB2PD_REORG_INDEX_RECORD" | grep "Cur Index:" ); + REORG_INDEX_STATUS=$(printf "%-25s %-10s" "$REORG_IND_CUR_STATUS" "$REORG_IND_CUR") + fi + fi + fi + + ## + ## has OBJECT been KICKED_OFF or STARTED + ## if it has check to see if is STARTED or COMPLETED + ## update OBJECT_ARRAY + ## update COMPLETION/STOPPED stats + ## + if [ "$OBJECT_REORG_STATUS" == "KICKED_OFF" ] || [ "$OBJECT_REORG_STATUS" == "STARTED" ]; then + if [ ! -z "$REORG_STATUS" ]; then + if [ "$REORG_STATUS" == "STARTED" -o "$REORG_STATUS" == "COMPLETED" -o "$REORG_STATUS" == "STOPPED" ]; then + OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$REORG_STATUS#$REORG_START#$REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE" + fi + + if [ "$REORG_STATUS" == "COMPLETED" -o "$REORG_STATUS" == "STOPPED" ]; then + + removeTABLE_TABLE_IN_USE_ARRAY "$TABSCHEMA.$TABNAME" + rc=$? + if [ $rc -eq 1 ]; then + log 1 "Failed to remove table $TABSCHEMA.$TABNAME from TABLE_IN_USE_ARRAY"; + fi + + if [ "$REORG_STATUS" == "COMPLETED" ]; then + let NUM_REORGS_COMPLETED+=1 + elif [ "$REORG_STATUS" == "STOPPED" ]; then + let NUM_REORGS_STOPPED+=1 + fi + + let NUM_REORGS_IN_PROGRESS-=1 + fi + + fi + + ## + ## OBJECT is NOSTARTED so KICK_OFF a reorg + ## + elif [ "$OBJECT_REORG_STATUS" == "NOTSTARTED" ]; then + + ## dont kick off any reorgs if window timeout passed and TWA=2 + ## OBJECTS become ABORTED - UPDATE OBJECT_ARRAY + if [ $REORG_TIMEOUT_WINDOW_COMPLETED -eq 1 ] && [ $REORG_TIMEOUT_WINDOW_ACTION -eq 2 ] && [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 1 ]; then + REORG_STATUS=ABORTED; + OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$REORG_STATUS#$OBJECT_REORG_START#$OBJECT_REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE"; + let NUM_REORGS_ABORTED+=1; + continue; + + fi + + ## is the TABLE already being used -if it is goto next OBJECT + existTABLE_TABLE_IN_USE_ARRAY "$TABSCHEMA.$TABNAME" + rc=$? + [ $rc -eq 0 ] && continue; + + ## we only want to kick off so many reorgs at any one time + if [ $NUM_REORGS_IN_PROGRESS -eq $MAX_ASYNC_REORGS_ALLOWED ]; then + continue; + fi + + ## + ## The table could already be locked - if it is then by-pass it + ## and ABORT if locked more than 10 times + ## + TABLE_LOCKED=$( db2 "select APPLICATION_HANDLE, LOCK_OBJECT_TYPE, LOCK_MODE, LOCK_CURRENT_MODE, LOCK_STATUS, LOCK_COUNT, LOCK_HOLD_COUNT, TBSP_ID, TAB_FILE_ID from TABLE (MON_GET_LOCKS(NULL, -2)) where TBSP_ID=$TBSPACEID and TAB_FILE_ID=$TABLEID and LOCK_OBJECT_TYPE='TABLE' and LOCK_MODE='IX' with ur"; ); + rc=$? + if [ $rc -eq 0 ]; then + let LOCK_COUNT+=1; + log 1 "Appears table $TABSCHEMA.$TABNAME is already locked by another application(s), LOCK_COUNT=$LOCK_COUNT"; + log 1 "$TABLE_LOCKED"; + if [ $LOCK_COUNT -gt 10 ]; then + OBJECT_REORG_STATUS=ABORTED; + let NUM_REORGS_ABORTED+=1; + log 1 "Aborting table $TABSCHEMA.$TABNAME , LOCK_COUNT=$LOCK_COUNT"; + fi + OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$OBJECT_REORG_STATUS#$OBJECT_REORG_START#$OBJECT_REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE"; + continue; + fi + + ## + ## do a check to see where we are on transaction log space - this could be improved!!!! + ## + #LOG_USED=$( db2 "select cast(LOG_UTILIZATION_PERCENT as decimal(5,2)) as PCTUSED, cast((TOTAL_LOG_USED_KB/1024) as Integer) as TOTUSEDMB, cast((TOTAL_LOG_AVAILABLE_KB/1024) as Integer) as TOTAVAILMB, cast((TOTAL_LOG_USED_TOP_KB/1024) as Integer) as TOTUSEDTOPMB FROM SYSIBMADM.LOG_UTILIZATION "); + LOG_USED=$( db2 -x "select LOG_UTILIZATION_PERCENT as PCTUSED FROM SYSIBMADM.MON_TRANSACTION_LOG_UTILIZATION " | tr -d ' ' ) + if [ ! -z "$LOG_USED" ]; then + #PCTUSED=$( echo "$LOG_USED" | awk '{ if(NF==4 && $2 ~/^[0-9]+$/) print int($1)}' ); + PCTUSED=$( echo $LOG_USED | awk '{print int($1)}') + if [ ! -z "$PCTUSED" ]; then + if [ $PCTUSED -gt $TRANSACTION_LOG_THRESHOLD_PCT ]; then + log 1 "Will not kick off another reorg due to logfile PCTUSED above threshold of $LOG_THRESHOLD% : $LOG_USED%" + continue + else + log 3 "Present LOG Utilization Percentage : $LOG_USED%" + fi + fi + fi + + ## + ## kick off another reorg + ## if rc=0 then ok, else we ABORT the OBJECT and don't try again + ## + + if [[ $TB_STATS -eq 1 || $IX_STATS -eq 2 ]]; then + + if [ $isTable -eq 1 ]; then + log 3 "Processing reorg for table $TABSCHEMA.$TABNAME" + db2 "reorg table $TABSCHEMA.$TABNAME inplace allow write access" >> /dev/null 2>&1 + rc=$? + #RC=$( db2 -x "select REORG_STATUS, REORG_COMPLETION, REORG_PHASE, REORG_CURRENT_COUNTER, REORG_MAX_COUNTER from table(snap_get_tab_reorg('')) where tabname='$TABNAME' " ) + RC=$( db2 -x "select REORG_STATUS, REORG_COMPLETION, REORG_PHASE from table(snap_get_tab_reorg('')) where tabname='$TABNAME' " ) + + elif [ $isIndex -eq 1 ]; then + log 3 "Processing reorg for table $TABSCHEMA.$TABNAME" + db2 "reorg table $TABSCHEMA.$TABNAME index $INDSCHEMA.$INDNAME inplace allow write access" > /dev/null 2>&1 + rc=$? + #RC=$( db2 "select REORG_STATUS, REORG_COMPLETION, REORG_PHASE, REORG_CURRENT_COUNTER, REORG_MAX_COUNTER from table(snap_get_tab_reorg('')) where tabname='$TABNAME' " ) + RC=$( db2 -x "select REORG_STATUS, REORG_COMPLETION, REORG_PHASE from table(snap_get_tab_reorg('')) where tabname='$TABNAME' " ) + fi + + elif [ $IF_STATS -eq 3 ]; then + + ## for offline we throw a job at db2 and wait a few seconds and check the output + ## output could be "reorg indexes all for table ...", + ## or SQL error + ## or 'DB20000I The REORG command completed successfully.' + ## not sure if there is a better way to do this + TMPLOG="/tmp/$TABSCHEMA.$TABNAME.tmp"; + log 3 "Processing reorg on indexes for table $TABSCHEMA.$TABNAME" + db2 -v "reorg indexes all for table $TABSCHEMA.$TABNAME allow write access" > $TMPLOG > /dev/null 2>&1 & + sleep 5; + cat $TMPLOG; + RC=$( grep '^SQL' $TMPLOG); + rc=$? + if [ $rc -eq 0 ]; then + log 1 "Failed to kick off reorg\n$RC"; + rc=1; + else + ## reorg could have finished then no need for the big sleep + RC=$( grep 'DB20000I The REORG command completed successfully.' $TMPLOG); + if [ $? -eq 0 ]; then + IF_STATS_BYPASS_SLEEP_INTERVAL_TIME=1; + fi + rc=0; + + fi + rm -f $TMPLOG; + + fi + + if [ $rc -eq 0 ]; then + REORG_STATUS=KICKED_OFF; + else + REORG_STATUS=ABORTED; + fi + OBJECT_ARRAY[$ii]="$TABSCHEMA#$TABNAME#$INDSCHEMA#$INDNAME#$IID#$REORG_STATUS#$OBJECT_REORG_START#$OBJECT_REORG_END#$TABLEID#$TBSPACEID#$LOCK_COUNT#$OBJECT_REORG_TABLE_TYPE"; + if [ $rc -ne 0 ]; then + let NUM_REORGS_ABORTED+=1; + continue + fi + + ## add the table to the TABLE_IN_USE_ARRAY + addTABLE_TABLE_IN_USE_ARRAY "$TABSCHEMA.$TABNAME" + rc=$? + if [ $rc -eq 1 ]; then + log 1 "Failed to add table $TABSCHEMA.$TABNAME to TABLE_IN_USE_ARRAY"; + fi + let NUM_REORGS_KICKED_OFF+=1; + let NUM_REORGS_IN_PROGRESS+=1; + + continue; + fi + + ## + ## ouput STATUS + ## + if [[ $TB_STATS -eq 1 || $IX_STATS -eq 2 ]]; then + #echo -e "$NUM_REORGS_IN_PROGRESS:$NUM_REORGS_KICKED_OFF:$NUM_REORGS_ABORTED:$NUM_REORGS_STOPPED:$NUM_REORGS_COMPLETED:$NUM_REORG_OBJECTS $TABSCHEMA.$TABNAME : $RC \t: $REORG_PERCENT_COMPLETE %" | tee -a $REORG_TABLE_INDEX_DEBUG + printf "%-15s %-50s %-50s %-10s\n" "$NUM_REORGS_IN_PROGRESS:$NUM_REORGS_KICKED_OFF:$NUM_REORGS_ABORTED:$NUM_REORGS_STOPPED:$NUM_REORGS_COMPLETED:$NUM_REORG_OBJECTS" "$TABSCHEMA.$TABNAME" "$RC" "COMPLETION: $REORG_PERCENT_COMPLETE %"| tee -a $REORG_TABLE_INDEX_DEBUG + + elif [ $IF_STATS -eq 3 ]; then + #echo -e "$NUM_REORGS_IN_PROGRESS:$NUM_REORGS_KICKED_OFF:$NUM_REORGS_ABORTED:$NUM_REORGS_STOPPED:$NUM_REORGS_COMPLETED:$NUM_REORG_OBJECTS $TABSCHEMA.$TABNAME : $REORG_INDEX_STATUS "| tee -a $REORG_TABLE_INDEX_DEBUG + printf "%-15s %-50s %-10s %15s\n" "$NUM_REORGS_IN_PROGRESS:$NUM_REORGS_KICKED_OFF:$NUM_REORGS_ABORTED:$NUM_REORGS_STOPPED:$NUM_REORGS_COMPLETED:$NUM_REORG_OBJECTS" "$TABSCHEMA.$TABNAME" "$REORG_INDEX_STATUS" | tee -a $REORG_TABLE_INDEX_DEBUG + fi + + ## + ## if reorg timeout then issue a stop to current reorgs that are STARTED + ## no error checking for stopping a reorg + ## no need to update OBJECT array as it will be updated on next loop + ## + + if [[ $TB_STATS -eq 1 || $IX_STATS -eq 2 ]]; then + + if [ "$REORG_STATUS" == "STARTED" ] && [ $REORG_TIMEOUT_WINDOW_COMPLETED -eq 1 ] && [ $REORG_TIMEOUT_WINDOW_ACTION -eq 2 ] && [ $REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER -eq 1 ]; then + + if [ $isTable -eq 1 ]; then + db2 -v "reorg table $TABSCHEMA.$TABNAME inplace stop" > /dev/null 2>&1 + rc=$? + elif [ $isIndex -eq 1 ]; then + db2 -v "reorg table $TABSCHEMA.$TABNAME index $INDSCHEMA.$INDNAME inplace stop" > /dev/null 2>&1 + rc=$? + fi + fi + fi + + done ## for OBJECT_ARRAY[@] + + ## check if we are done with all OBJECTS + if [ $((NUM_REORGS_COMPLETED + NUM_REORGS_STOPPED + NUM_REORGS_ABORTED)) -ge $NUM_REORG_OBJECTS ]; then + log 3 "Reorgs Status :: Completed : $NUM_REORGS_COMPLETED , Stopped : $NUM_REORGS_STOPPED , Aborted : $NUM_REORGS_ABORTED , Total No of Objects : $NUM_REORG_OBJECTS" + break + fi + + ## wait some time + if [ $IF_STATS -eq 3 ] && [ $IF_STATS_BYPASS_SLEEP_INTERVAL_TIME -eq 1 ]; then + sleep 1; + IF_STATS_BYPASS_SLEEP_INTERVAL_TIME=0; + else + sleep $SLEEP_INTERVAL_TIME + fi + + done ## while true } +# -- Main function starts here ## init if [ -f ${HOME}/sqllib/db2profile ]; then @@ -847,9 +857,9 @@ fi ## script already running ? if [ $( ps -ef | grep $0 | grep -v grep | wc -l ) -gt 2 ]; then - echo "Warning: appears $0 already running" - echo "$( ps -ef | grep $0 | grep -v grep )"; - exit 1 + echo "Warning: appears $0 already running" + echo "$( ps -ef | grep $0 | grep -v grep )"; + exit 1 fi SCRIPT=$(basename $0) @@ -871,7 +881,7 @@ LIST_VALID_TABLE_SIZES=0 LIST_REORGCHK_TB_STATS_TABLES=0 LIST_REORGCHK_IX_STATS_TABLES=0 EXECUTE_TABLE_REORG=0 -IGNORE_TABLE_SIZE_THRESHOLD_MAX=20000; +IGNORE_TABLE_SIZE_THRESHOLD_MAX=100000; IGNORE_TABLE_SIZE_THRESHOLD_MIN=10; MAINTENANCE_TIMEOUT_WINDOW_MINUTES=240; REORG_TIMEOUT_WINDOW_ACTION=2; @@ -891,8 +901,8 @@ REORGCHK_IF_STATS=3; ## user check if [ $WHOAMI == "root" ]; then - log 0 " This script should be not run as '$WHOAMI', but as instance owner." - exit 1 + log 0 " This script should be not run as '$WHOAMI', but as instance owner." + exit 1 fi ## @@ -900,97 +910,99 @@ fi ## while [ $# -gt 0 ] do - case $1 in - -h|-H|-help|--help) UsageHelp; exit 1 ;; - - -db) shift; [ ! -z $1 ] && DB=$( echo $1 | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; - -s) shift; [ ! -z $1 ] && SCHEMANAME_IN=$( echo $1 | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; - -t) shift; [ ! -z "$1" ] && TABLE_IN=$( echo "$1" | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; - -tb_stats) REORGCHK_TB_IF_STATS_OPTION+=$REORGCHK_TB_STATS; TB_STATS=1 ;; - -ti) shift; [ ! -z "$1" ] && INDEX_IN=$( echo "$1" | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; - -ix_stats) IX_STATS=2 ;; - -if_stats) REORGCHK_TB_IF_STATS_OPTION+=$REORGCHK_IF_STATS; IF_STATS=3 ;; - -ittx) shift; isNumeric $1 && { IGNORE_TABLE_SIZE_THRESHOLD_MAX=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - -ittn) shift; isNumeric $1 && { IGNORE_TABLE_SIZE_THRESHOLD_MIN=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - - -l) LIST_ONLY=1 ;; - -lf) LIST_FRAGMENTED_INDEXES=1 ;; - -ls) LIST_VALID_TABLE_SIZES=1 ;; - -lt) LIST_REORGCHK_TB_STATS_TABLES=1 ;; - -li) LIST_REORGCHK_IX_STATS_TABLES=1 ;; - - -window) shift; isNumeric $1 && { MAINTENANCE_TIMEOUT_WINDOW_MINUTES=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - -twa) shift; isNumeric $1 && { REORG_TIMEOUT_WINDOW_ACTION=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - -mar) shift; isNumeric $1 && { MAX_ASYNC_REORGS_ALLOWED=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - -log) shift; isNumeric $1 && { TRANSACTION_LOG_THRESHOLD_PCT=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - -tr) EXECUTE_TABLE_REORG=1 ;; - - -trsi) TRSI=1 ;; - -reorg) shift; [ ! -z "$1" ] && REORG=$( echo "$1" ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; - - -ignore) shift; [ ! -z "$1" ] && { IGNORE_TABLES="$1"; } || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; - - -sleep) shift; isNumeric $1 && { SLEEP_INTERVAL_TIME=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; - - (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;; - (*) break;; - esac - + case $1 in + -h|-H|-help|--help) UsageHelp; exit 1 ;; + + -db|-d) shift; [ ! -z $1 ] && DBNAME=$( echo $1 | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; + -s) shift; [ ! -z $1 ] && SCHEMANAME_IN=$( echo $1 | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; + -t) shift; [ ! -z "$1" ] && TABLE_IN=$( echo "$1" | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; + -tb_stats) REORGCHK_TB_IF_STATS_OPTION+=$REORGCHK_TB_STATS; TB_STATS=1 ;; + -ti) shift; [ ! -z "$1" ] && INDEX_IN=$( echo "$1" | tr '[a-z]' '[A-Z]' ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; + -ix_stats) IX_STATS=2 ;; + -if_stats) REORGCHK_TB_IF_STATS_OPTION+=$REORGCHK_IF_STATS; IF_STATS=3 ;; + -ittx) shift; isNumeric $1 && { IGNORE_TABLE_SIZE_THRESHOLD_MAX=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + -ittn) shift; isNumeric $1 && { IGNORE_TABLE_SIZE_THRESHOLD_MIN=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + + -l) LIST_ONLY=1 ;; + -lf) LIST_FRAGMENTED_INDEXES=1 ;; + -ls) LIST_VALID_TABLE_SIZES=1 ;; + -lt) LIST_REORGCHK_TB_STATS_TABLES=1 ;; + -li) LIST_REORGCHK_IX_STATS_TABLES=1 ;; + + -window) shift; isNumeric $1 && { MAINTENANCE_TIMEOUT_WINDOW_MINUTES=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + -twa) shift; isNumeric $1 && { REORG_TIMEOUT_WINDOW_ACTION=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + -mar) shift; isNumeric $1 && { MAX_ASYNC_REORGS_ALLOWED=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + -log) shift; isNumeric $1 && { TRANSACTION_LOG_THRESHOLD_PCT=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + -tr) EXECUTE_TABLE_REORG=1 ;; + + -trsi) TRSI=1 ;; + -reorg) shift; [ ! -z "$1" ] && REORG=$( echo "$1" ) || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; + -ignore) shift; [ ! -z "$1" ] && { IGNORE_TABLES="$1"; } || { echo "Error: Must enter an argument for this option"; UsageHelp; exit 1 ; } ;; + -sleep) shift; isNumeric $1 && { SLEEP_INTERVAL_TIME=$1; } || { echo "Error: Must enter an numeric argument for this option"; UsageHelp; exit 1 ; } ;; + + (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;; + (*) break;; + esac shift - done ## ## some verification ## +if [[ -z "$DBNAME" && -z "$SCHEMANAME_IN" ]]; then + log 0 "Please provide Database Name and Schema Name " + exit 1 + +fi +: ' if [ -z "$SCHEMANAME_IN" ]; then - log 0 "must enter a schemaname" - exit 1 + log 0 "must enter a schemaname" + exit 1 fi - +' CHECK=1 if [ $CHECK -eq 0 ]; then - rc=0 - if [ $TB_STATS -eq 1 ] && [ $IX_STATS -eq 2 -o $IF_STATS -eq 3 ]; then - rc=1; - elif [ $IX_STATS -eq 2 ] && [ $TB_STATS -eq 1 -o $IF_STATS -eq 3 ]; then - rc=1; - elif [ $IF_STATS -eq 3 ] && [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then - rc=1; - fi - if [ $rc -eq 1 ]; then - log 0 "can't define more than one of -tb_stats, -ix_stats or -if_stats" - exit 1 - fi + rc=0 + if [ $TB_STATS -eq 1 ] && [ $IX_STATS -eq 2 -o $IF_STATS -eq 3 ]; then + rc=1; + elif [ $IX_STATS -eq 2 ] && [ $TB_STATS -eq 1 -o $IF_STATS -eq 3 ]; then + rc=1; + elif [ $IF_STATS -eq 3 ] && [ $TB_STATS -eq 1 -o $IX_STATS -eq 2 ]; then + rc=1; + fi + if [ $rc -eq 1 ]; then + log 0 "can't define more than one of -tb_stats, -ix_stats or -if_stats" + exit 1 + fi fi ## CHECK if [ $TB_STATS -eq 1 ] && [ ! -z "$INDEX_IN" ]; then - log 0 "can't define -ti with -tb_stats" - exit 1 + log 0 "can't define -ti with -tb_stats" + exit 1 elif [ $IX_STATS -eq 2 ] && [ ! -z "$TABLE_IN" ]; then - log 0 "can't define -t with -ix_stats" - exit 1 + log 0 "can't define -t with -ix_stats" + exit 1 elif [ $IF_STATS -eq 3 ] && [ ! -z "$INDEX_IN" ]; then - log 0 "can't define -ti with -if_stats" - exit 1 + log 0 "can't define -ti with -if_stats" + exit 1 fi if [ $TRANSACTION_LOG_THRESHOLD_PCT -gt 99 ]; then - log 0 "-log option should be less than 100, TRANSACTION_LOG_THRESHOLD_PCT=$TRANSACTION_LOG_THRESHOLD_PCT" - exit 1 + log 0 "-log option should be less than 100, TRANSACTION_LOG_THRESHOLD_PCT=$TRANSACTION_LOG_THRESHOLD_PCT" + exit 1 fi if [ $IGNORE_TABLE_SIZE_THRESHOLD_MIN -ge $IGNORE_TABLE_SIZE_THRESHOLD_MAX ]; then - log 0 "option -ittx should be greater than option -ittn" - exit 1 + log 0 "option -ittx should be greater than option -ittn" + exit 1 fi ## override some defaults for offline reorgs #if [ $IF_STATS -eq 3 ]; then -# MAX_ASYNC_REORGS_ALLOWED=1; -# REORG_TIMEOUT_WINDOW_ACTION=1; -# ## make SLEEP_INTERVAL_TIME 1/3 for IF_STATS -# SLEEP_INTERVAL_TIME=$( echo $SLEEP_INTERVAL_TIME | awk '{ print $1/3 }'); +# MAX_ASYNC_REORGS_ALLOWED=1; +# REORG_TIMEOUT_WINDOW_ACTION=1; +# ## make SLEEP_INTERVAL_TIME 1/3 for IF_STATS +# SLEEP_INTERVAL_TIME=$( echo $SLEEP_INTERVAL_TIME | awk '{ print $1/3 }'); #fi ## fixup REORG filter string for grep @@ -1009,328 +1021,328 @@ RUNSTATS_TIMEOUT_WINDOW_SECONDS=$( echo $MAINTENANCE_TIMEOUT_WINDOW_SECONDS | aw ## log 3 "Starting $0 at $(date) on $HOSTNAME" -DBNAMES=$( db2 list db directory | grep -E "alias|Indirect" | grep -B 1 Indirect | grep alias | awk '{print $4}' | sort ) +#DBNAMES=$( db2 list db directory | grep -E "alias|Indirect" | grep -B 1 Indirect | grep alias | awk '{print $4}' | sort ) ## ## loops for all dbs ## -for DBNAME in $DBNAMES -do - - ## just process the one db - if [ ! -z "$DB" ] && [ "$DB" != "$DBNAME" ] ; then - continue - fi - - ## can't run script on a STANDBY db - ROLE=$(db2 "get db cfg for $DBNAME" | grep 'HADR database role' | cut -d '=' -f2 | sed 's/ *//g') - if [ -z "$ROLE" ] || [ "$ROLE" == "" ]; then - log 1 " Can't determine hadr database role from 'db2 get db cfg for $DBNAME'" - continue - elif [ "$ROLE" == "STANDBY" ]; then - log 1 " Can't run script '${0}' for $DBNAME with hadr database role '$ROLE'" - continue - fi - - log 3 "DB=$DBNAME ..." - - db2 connect to $DBNAME >> /dev/null 2>&1 - rc=$? - if [ $rc -ne 0 ]; then - log 0 " can't connect to $DBNAME" - continue - fi - - if [ $TRSI -eq 1 ]; then - TRSI - continue - - elif [ $LIST_VALID_TABLE_SIZES -eq 1 ]; then - getValidTableSizes - log 3 "The following $NUM_VALID_TABLES are valid table sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB\n$VALID_TABLE_SIZES_RAW_DATA" - continue - - elif [ $LIST_FRAGMENTED_INDEXES -eq 1 ]; then - getValidFragmentedIndexes - VALID_FRAMENTED_INDEXES_HEADER="TABSCHEMA TABNAME INDSCHEMA INDNAME INDCARD STATS_TIME LAST_USED NLEAF SEQUENTIAL_PAGES"; - log 3 "The following $NUM_VALID_FRAGMENTED_INDEXES are fragmenated indexes based on tables sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB\n$VALID_FRAMENTED_INDEXES_HEADER\n$VALID_FRAGMENTED_INDEXES_RAW_DATA" - continue; - elif [ $LIST_REORGCHK_TB_STATS_TABLES -eq 1 ]; then - getValidTablesToReorg - REORGCHK_TB_STATS_HEADER="TABLE_SCHEMA TABLE_NAME CARD OVERFLOW NPAGES FPAGES ACTIVE_BLOCKS TSIZE F1 F2 F3 REORG"; - log 3 "The following $NUM_VALID_TABLES_TO_REORG are results from REORGCHK_TB_STATS based on tables sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB\n$REORGCHK_TB_STATS_HEADER\n$VALID_TABLES_TO_REORG_RAW_DATA" - continue; - - elif [ $LIST_REORGCHK_IX_STATS_TABLES -eq 1 ]; then - getValidIndexesToReorg - REORGCHK_IX_STAT_HEADER="TABLE_SCHEMA TABLE_NAME INDEX_SCHEMA INDEX_NAME INDCARD NLEAF NUM_EMPTY_LEAFS NLEVELS NUMRIDS_DELETED FULLKEYCARD LEAF_RECSIZE NONLEAF_RECSIZE LEAF_PAGE_OVERHEAD NONLEAF_PAGE_OVERHEAD PCT_PAGES_SAVED F4 F5 F6 F7 F8 REORG"; - log 3 "The following $NUM_VALID_INDEXES_TO_REORG are results from REORGCHK_IX_STATS based on tables sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB\n$REORGCHK_IX_STAT_HEADER\n$VALID_INDEXES_TO_REORG_RAW_DATA" - continue; - - - fi - - - INPLACE=1 - if [ $INPLACE -eq 1 ]; then - - log 3 " SCHEMA: $SCHEMANAME_IN" - log 3 " TABLES: $TABLE_IN" - log 3 "INDEXES: $INDEX_IN" - - ## - ## input table(s) verification - ## verify input table exist - ## and table is within size limits - ## create the OBJECT_ARRAY that holds the relevant table information - ## - if [ ! -z "$TABLE_IN" ]; then - - TABLE_IN=$( echo "$TABLE_IN" | tr ' ' '\n' ); - for TABNAME in $TABLE_IN - do - - ## make sure table exists - RC=$( db2 -x "select tabname from syscat.tables where tabname = '$TABNAME' and tabschema = '$SCHEMANAME_IN' and type = 'T'"); - rc=$? - if [ $rc -ne 0 ]; then - log 0 "input command line table '$TABNAME' does not exist or is invalid" - exit 1 - fi - - isTableWithinSizeLimit $SCHEMANAME_IN $TABNAME - rc=$? - if [ $rc -ne 0 ]; then - log 0 "Table $TABNAME is not within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB" - exit 1 - fi - - done - - if [ $TB_STATS -eq 1 ]; then - createTableOBJECT_ARRAY "$TABLE_IN" $TB_STATS - elif [ $IF_STATS -eq 3 ]; then - createTableOBJECT_ARRAY "$TABLE_IN" $IF_STATS - fi - - elif [ ${#REORGCHK_TB_IF_STATS_OPTION} -eq 2 ]; then - for REORG_TYPE in 0 1 - do - if [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "1" ]; then - getValidTablesToReorg - TABLE_IN="$VALID_TABLES_TO_REORG"; - createTableOBJECT_ARRAY "$TABLE_IN" $TB_STATS - elif [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "3" ]; then - getValidFragmentedIndexes - TABLE_IN="$VALID_FRAGMENTED_INDEXES" - createTableOBJECT_ARRAY "$TABLE_IN" $IF_STATS - fi - done - - elif [ $TB_STATS -eq 1 ]; then - getValidTablesToReorg - TABLE_IN="$VALID_TABLES_TO_REORG"; - createTableOBJECT_ARRAY "$TABLE_IN" $TB_STATS - elif [ $IF_STATS -eq 3 ]; then - getValidFragmentedIndexes - TABLE_IN="$VALID_FRAGMENTED_INDEXES" - createTableOBJECT_ARRAY "$TABLE_IN" $IF_STATS - fi - - ## - ## input indexes verification - ## verify input indexes exist - ## - if [ ! -z "$INDEX_IN" ]; then - INDEX_IN=$( echo "$INDEX_IN" | tr ' ' '\n' ); - for INDEX in $INDEX_IN - do - ## make sure index exists, especially those input on command line - TABSCHEMA=$( echo $INDEX | cut -d. -f1); - TABNAME=$( echo $INDEX | cut -d. -f2); - INDSCHEMA=$( echo $INDEX | cut -d. -f3); - INDNAME=$( echo $INDEX | cut -d. -f4); - RC=$( db2 -x "select indname from syscat.indexes where tabschema = '$TABSCHEMA' and tabname = '$TABNAME' and indschema = '$INDSCHEMA' and indname = '$INDNAME'"); - rc=$? - if [ $rc -ne 0 ]; then - log 0 " input command line index '$INDEX' does not exist" - exit 1 - fi - - isTableWithinSizeLimit $TABSCHEMA $TABNAME - rc=$? - if [ $rc -ne 0 ]; then - log 0 "Table $TABNAME is not within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB" - exit 1 - fi - - done - - createIndexOBJECT_ARRAY "$INDEX_IN" $IX_STATS; - ## - ## - elif [ $IX_STATS -eq 2 ]; then - getValidIndexesToReorg - INDEX_IN="$VALID_INDEXES_TO_REORG" - createIndexOBJECT_ARRAY "$INDEX_IN" $IX_STATS; - fi - - ## get the NUMBER of tables per reorg table type - getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY $TB_STATS; OBJECT_NUM_TB_STATS=$?; - getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY $IX_STATS; OBJECT_NUM_IX_STATS=$?; - getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY $IF_STATS; OBJECT_NUM_IF_STATS=$?; - - ## just list out the OBJECT_ARRAY and exit - if [ $LIST_ONLY -eq 1 ]; then - - listOBJECT_ARRAY; - exit 1 - fi - - - if [ $EXECUTE_TABLE_REORG -eq 1 ]; then - - - ## - ## this is the main list of what we are going to reorg - ## - echo "" - - listOBJECT_ARRAY; - -# exit 1 - - ## - ## setup some control variables for the main loop - ## - ## REORG_STATUS COMPLETED PAUSED STARTED STOPPED TRUNCATE - ## - MAINTENANCE_TIMEOUT_WINDOW_START_TIME_SECONDS=$( date '+%s' ); - REORG_TIMEOUT_WINDOW_START_TIME_SECONDS=$( date '+%s' ); - WINDOW_START_TIME_DB2=$( date '+%Y-%m-%d-%H.%M.%S' ); - IF_STATS_WINDOW_START_TIME_DB2=$( date '+%s' ); - IF_STATS_BYPASS_SLEEP_INTERVAL_TIME=0; - NUM_REORG_OBJECTS="${#OBJECT_ARRAY[@]}"; -# NUM_REORGS_IN_PROGRESS=0; -# NUM_REORGS_KICKED_OFF=0; -# NUM_REORGS_COMPLETED=0; -# NUM_REORGS_STOPPED=0; -# NUM_REORGS_ABORTED=0; - REORG_TIMEOUT_OVERFLOW_VALVE=300; - REORG_TIMEOUT_WINDOW_COMPLETED=0; - REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER=0; - initTABLE_IN_USE_ARRAY $MAX_ASYNC_REORGS_ALLOWED - - ## multi table reorg option - ## online table reorg and offline index reorgs have different options - MAX_ASYNC_REORGS_ALLOWED_ORG=$MAX_ASYNC_REORGS_ALLOWED; - REORG_TIMEOUT_WINDOW_ACTION_ORG=$REORG_TIMEOUT_WINDOW_ACTION; - SLEEP_INTERVAL_TIME_ORG=$SLEEP_INTERVAL_TIME; - - ## override some defaults for offline reorgs - #if [ $IF_STATS -eq 3 ]; then - # MAX_ASYNC_REORGS_ALLOWED=1; - # REORG_TIMEOUT_WINDOW_ACTION=1; - # ## make SLEEP_INTERVAL_TIME 1/3 for IF_STATS - # SLEEP_INTERVAL_TIME=$( echo $SLEEP_INTERVAL_TIME | awk '{ print $1/3 }'); - #fi - - echo "" - log 3 "Starting reorg of ..." - - if [ ${#REORGCHK_TB_IF_STATS_OPTION} -eq 2 ]; then - for REORG_TYPE in 0 1 - do - - if [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "1" ]; then - TB_STATS=1; - IF_STATS=0; - NUM_REORG_OBJECTS=$OBJECT_NUM_TB_STATS; - REORG_TABLE_TYPE=$TB_STATS; - MAX_ASYNC_REORGS_ALLOWED=$MAX_ASYNC_REORGS_ALLOWED_ORG; - REORG_TIMEOUT_WINDOW_ACTION=$REORG_TIMEOUT_WINDOW_ACTION_ORG; - SLEEP_INTERVAL_TIME=$SLEEP_INTERVAL_TIME_ORG; - - elif [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "3" ]; then - TB_STATS=0; - IF_STATS=3; - NUM_REORG_OBJECTS=$OBJECT_NUM_IF_STATS; - REORG_TABLE_TYPE=$IF_STATS; - MAX_ASYNC_REORGS_ALLOWED=1; - REORG_TIMEOUT_WINDOW_ACTION=1; - SLEEP_INTERVAL_TIME=$( echo $SLEEP_INTERVAL_TIME_ORG | awk '{ print $1/3 }'); - fi - - reorgTables - done - else - if [ $TB_STATS -eq 1 ]; then - REORG_TABLE_TYPE=$TB_STATS; - elif [ $IX_STATS -eq 2 ]; then - REORG_TABLE_TYPE=$IX_STATS; - elif [ $IF_STATS -eq 3 ]; then - REORG_TABLE_TYPE=$IF_STATS; - fi - reorgTables - fi - - ## list current state of OBJECT_ARRAY - listOBJECT_ARRAY; - - ## - ## now do runstats - ## - log 3 "Starting runstats of ..." - RUNSTATS_TIMEOUT_WINDOW_START_TIME_SECONDS=$( date '+%s' ); - - for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) - do - - ## check runstats window maintenance time - MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS=$( date '+%s' ); - DIFF=$(( MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS - REORG_TIMEOUT_WINDOW_START_TIME_SECONDS )); - if [ $DIFF -ge $(( REORG_TIMEOUT_WINDOW_SECONDS + RUNSTATS_TIMEOUT_WINDOW_SECONDS)) ]; then - log 3 "$REORG_TIMEOUT_WINDOW_START_TIME_SECONDS -$MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS $REORG_TIMEOUT_WINDOW_SECONDS $RUNSTATS_TIMEOUT_WINDOW_SECONDS $DIFF" - log 3 "Runstats window ending, runstats window time exceeded" - break - fi - - ## get table info - TABSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f1 ); - TABNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f2 ); - INDSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f3 ); - INDNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f4 ); - IID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f5 ); - OBJECT_REORG_STATUS=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f6 ); - - ## only do a runstats if table/index has completed - ## verify stats time so we dont kick off another runstats on the same table - if [ "$OBJECT_REORG_STATUS" == "COMPLETED" ]; then - STATS_TIME=$( db2 -x "select stats_time from syscat.tables where tabschema='$TABSCHEMA' and tabname='$TABNAME' and stats_time < TIMESTAMP('$WINDOW_START_TIME_DB2') " ); - rc=$? - if [ $rc -eq 0 ]; then - log 3 "Starting runstats on $TABSCHEMA.$TABNAME" - # db2 -v "runstats on table $TABSCHEMA.$TABNAME WITH DISTRIBUTION ON ALL COLUMNS AND SAMPLED DETAILED INDEXES ALL ALLOW WRITE ACCESS"; - db2 -v "runstats on table $TABSCHEMA.$TABNAME WITH DISTRIBUTION ON KEY COLUMNS AND SAMPLED DETAILED INDEXES ALL ALLOW WRITE ACCESS UTIL_IMPACT_PRIORITY 50"; - log 3 "Finished runstats on $TABSCHEMA.$TABNAME" - fi - fi - - done - - fi ## EXECUTE_TABLE_REORG - - - fi ## INPLACE - - -done ## DBNAMES +#for DBNAME in $DBNAMES +#do + + ## just process the one db +# if [ ! -z "$DB" ] && [ "$DB" != "$DBNAME" ] ; then +# continue +# fi +# DBNAME=${DB} + ## can't run script on a STANDBY db + ROLE=$(db2 "get db cfg for $DBNAME" | grep 'HADR database role' | cut -d '=' -f2 | sed 's/ *//g') + if [ -z "$ROLE" ] || [ "$ROLE" == "" ]; then + log 1 " Can't determine hadr database role from 'db2 get db cfg for $DBNAME'" + continue + elif [ "$ROLE" == "STANDBY" ]; then + log 1 " Can't run script '${0}' for $DBNAME with hadr database role '$ROLE'" + continue + fi + + log 3 "DBNAME :: $DBNAME" + + db2 connect to $DBNAME >> /dev/null 2>&1 + rc=$? + if [ $rc -ne 0 ]; then + log 0 " can't connect to $DBNAME" + continue + fi + + if [ $TRSI -eq 1 ]; then + TRSI + continue + + elif [ $LIST_VALID_TABLE_SIZES -eq 1 ]; then + getValidTableSizes + log 3 "The following $NUM_VALID_TABLES are valid table sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX GB\n$VALID_TABLE_SIZES_RAW_DATA" + continue + + elif [ $LIST_FRAGMENTED_INDEXES -eq 1 ]; then + getValidFragmentedIndexes + VALID_FRAMENTED_INDEXES_HEADER="TABSCHEMA TABNAME INDSCHEMA INDNAME INDCARD STATS_TIME LAST_USED NLEAF SEQUENTIAL_PAGES"; + log 3 "The following $NUM_VALID_FRAGMENTED_INDEXES are fragmenated indexes based on tables sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX GB\n$VALID_FRAMENTED_INDEXES_HEADER\n$VALID_FRAGMENTED_INDEXES_RAW_DATA" + continue; + elif [ $LIST_REORGCHK_TB_STATS_TABLES -eq 1 ]; then + getValidTablesToReorg + REORGCHK_TB_STATS_HEADER="TABLE_SCHEMA TABLE_NAME CARD OVERFLOW NPAGES FPAGES ACTIVE_BLOCKS TSIZE F1 F2 F3 REORG"; + log 3 "The following $NUM_VALID_TABLES_TO_REORG are results from REORGCHK_TB_STATS based on tables sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX GB\n$REORGCHK_TB_STATS_HEADER\n$VALID_TABLES_TO_REORG_RAW_DATA" + continue; + + elif [ $LIST_REORGCHK_IX_STATS_TABLES -eq 1 ]; then + getValidIndexesToReorg + REORGCHK_IX_STAT_HEADER="TABLE_SCHEMA TABLE_NAME INDEX_SCHEMA INDEX_NAME INDCARD NLEAF NUM_EMPTY_LEAFS NLEVELS NUMRIDS_DELETED FULLKEYCARD LEAF_RECSIZE NONLEAF_RECSIZE LEAF_PAGE_OVERHEAD NONLEAF_PAGE_OVERHEAD PCT_PAGES_SAVED F4 F5 F6 F7 F8 REORG"; + log 3 "The following $NUM_VALID_INDEXES_TO_REORG are results from REORGCHK_IX_STATS based on tables sizes within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX GB\n$REORGCHK_IX_STAT_HEADER\n$VALID_INDEXES_TO_REORG_RAW_DATA" + continue; + + fi + + INPLACE=1 + if [ $INPLACE -eq 1 ]; then + + log 3 "SCHEMA :: $SCHEMANAME_IN" + log 3 "TABLES :: $TABLE_IN" + log 3 "INDEXES :: $INDEX_IN" + log 3 "Looking for Qualified Tables or Indexes for $SCHEMANAME_IN in $DBNAME . . . " + echo "" + + ## + ## input table(s) verification + ## verify input table exist + ## and table is within size limits + ## create the OBJECT_ARRAY that holds the relevant table information + ## + if [ ! -z "$TABLE_IN" ]; then + + TABLE_IN=$( echo "$TABLE_IN" | tr ' ' '\n' ); + for TABNAME in $TABLE_IN + do + + ## make sure table exists + RC=$( db2 -x "select tabname from syscat.tables where tabname = '$TABNAME' and tabschema = '$SCHEMANAME_IN' and type = 'T'"); + rc=$? + if [ $rc -ne 0 ]; then + log 0 "input command line table '$TABNAME' does not exist or is invalid" + exit 1 + fi + + isTableWithinSizeLimit $SCHEMANAME_IN $TABNAME + rc=$? + if [ $rc -ne 0 ]; then + log 0 "Table $TABNAME is not within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB" + exit 1 + fi + + done + + if [ $TB_STATS -eq 1 ]; then + createTableOBJECT_ARRAY "$TABLE_IN" $TB_STATS + elif [ $IF_STATS -eq 3 ]; then + createTableOBJECT_ARRAY "$TABLE_IN" $IF_STATS + fi + + elif [ ${#REORGCHK_TB_IF_STATS_OPTION} -eq 2 ]; then + for REORG_TYPE in 0 1 + do + if [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "1" ]; then + getValidTablesToReorg + TABLE_IN="$VALID_TABLES_TO_REORG"; + createTableOBJECT_ARRAY "$TABLE_IN" $TB_STATS + elif [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "3" ]; then + getValidFragmentedIndexes + TABLE_IN="$VALID_FRAGMENTED_INDEXES" + createTableOBJECT_ARRAY "$TABLE_IN" $IF_STATS + fi + done + + elif [ $TB_STATS -eq 1 ]; then + getValidTablesToReorg + TABLE_IN="$VALID_TABLES_TO_REORG"; + createTableOBJECT_ARRAY "$TABLE_IN" $TB_STATS + elif [ $IF_STATS -eq 3 ]; then + getValidFragmentedIndexes + TABLE_IN="$VALID_FRAGMENTED_INDEXES" + createTableOBJECT_ARRAY "$TABLE_IN" $IF_STATS + fi + + ## + ## input indexes verification + ## verify input indexes exist + ## + if [ ! -z "$INDEX_IN" ]; then + INDEX_IN=$( echo "$INDEX_IN" | tr ' ' '\n' ); + for INDEX in $INDEX_IN + do + ## make sure index exists, especially those input on command line + TABSCHEMA=$( echo $INDEX | cut -d. -f1); + TABNAME=$( echo $INDEX | cut -d. -f2); + INDSCHEMA=$( echo $INDEX | cut -d. -f3); + INDNAME=$( echo $INDEX | cut -d. -f4); + RC=$( db2 -x "select indname from syscat.indexes where tabschema = '$TABSCHEMA' and tabname = '$TABNAME' and indschema = '$INDSCHEMA' and indname = '$INDNAME'"); + rc=$? + if [ $rc -ne 0 ]; then + log 0 " input command line index '$INDEX' does not exist" + exit 1 + fi + + isTableWithinSizeLimit $TABSCHEMA $TABNAME + rc=$? + if [ $rc -ne 0 ]; then + log 0 "Table $TABNAME is not within the range $IGNORE_TABLE_SIZE_THRESHOLD_MIN MB and $IGNORE_TABLE_SIZE_THRESHOLD_MAX MB" + exit 1 + fi + + done + + createIndexOBJECT_ARRAY "$INDEX_IN" $IX_STATS; + ## + ## + elif [ $IX_STATS -eq 2 ]; then + getValidIndexesToReorg + INDEX_IN="$VALID_INDEXES_TO_REORG" + createIndexOBJECT_ARRAY "$INDEX_IN" $IX_STATS; + fi + + ## get the NUMBER of tables per reorg table type + getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY $TB_STATS; OBJECT_NUM_TB_STATS=$?; + getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY $IX_STATS; OBJECT_NUM_IX_STATS=$?; + getNUM_OBJECT_REORG_TABLE_TYPE_OBJECT_ARRAY $IF_STATS; OBJECT_NUM_IF_STATS=$?; + + ## just list out the OBJECT_ARRAY and exit + if [ $LIST_ONLY -eq 1 ]; then + + #printf "%-15s %-30s %-15s %-30s %-5s %-10s %-25s %-25s %-10s %-10s %-10s %-15s\n" "TABSCHEMA" "TABNAME" "INDSCHEMA" "INDNAME" "IID" "REORG_STATUS" "REORG_START" "REORG_END" "TABLEID" "TBSPACEID" "LOCK_COUNT" "TABLE_TYPE" + listOBJECT_ARRAY; + exit 1 + fi + + + if [ $EXECUTE_TABLE_REORG -eq 1 ]; then + + + ## + ## this is the main list of what we are going to reorg + ## + #echo "" + #printf "%-15s %-30s %-15s %-30s %-5s %-10s %-25s %-25s %-10s %-10s %-10s %-15s\n" "TABSCHEMA" "TABNAME" "INDSCHEMA" "INDNAME" "IID" "REORG_STATUS" "REORG_START" "REORG_END" "TABLEID" "TBSPACEID" "LOCK_COUNT" "TABLE_TYPE" + + listOBJECT_ARRAY; + +# exit 1 + + ## + ## setup some control variables for the main loop + ## + ## REORG_STATUS COMPLETED PAUSED STARTED STOPPED TRUNCATE + ## + MAINTENANCE_TIMEOUT_WINDOW_START_TIME_SECONDS=$( date '+%s' ); + REORG_TIMEOUT_WINDOW_START_TIME_SECONDS=$( date '+%s' ); + WINDOW_START_TIME_DB2=$( date '+%Y-%m-%d-%H.%M.%S' ); + IF_STATS_WINDOW_START_TIME_DB2=$( date '+%s' ); + IF_STATS_BYPASS_SLEEP_INTERVAL_TIME=0; + NUM_REORG_OBJECTS="${#OBJECT_ARRAY[@]}"; +# NUM_REORGS_IN_PROGRESS=0; +# NUM_REORGS_KICKED_OFF=0; +# NUM_REORGS_COMPLETED=0; +# NUM_REORGS_STOPPED=0; +# NUM_REORGS_ABORTED=0; + REORG_TIMEOUT_OVERFLOW_VALVE=300; + REORG_TIMEOUT_WINDOW_COMPLETED=0; + REORG_TIMEOUT_WINDOW_COMPLETED_COUNTER=0; + initTABLE_IN_USE_ARRAY $MAX_ASYNC_REORGS_ALLOWED + + ## multi table reorg option + ## online table reorg and offline index reorgs have different options + MAX_ASYNC_REORGS_ALLOWED_ORG=$MAX_ASYNC_REORGS_ALLOWED; + REORG_TIMEOUT_WINDOW_ACTION_ORG=$REORG_TIMEOUT_WINDOW_ACTION; + SLEEP_INTERVAL_TIME_ORG=$SLEEP_INTERVAL_TIME; + + ## override some defaults for offline reorgs + #if [ $IF_STATS -eq 3 ]; then + # MAX_ASYNC_REORGS_ALLOWED=1; + # REORG_TIMEOUT_WINDOW_ACTION=1; + # ## make SLEEP_INTERVAL_TIME 1/3 for IF_STATS + # SLEEP_INTERVAL_TIME=$( echo $SLEEP_INTERVAL_TIME | awk '{ print $1/3 }'); + #fi + + echo "" + log 3 "Starting reorg of ..." + + if [ ${#REORGCHK_TB_IF_STATS_OPTION} -eq 2 ]; then + for REORG_TYPE in 0 1 + do + + if [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "1" ]; then + TB_STATS=1; + IF_STATS=0; + NUM_REORG_OBJECTS=$OBJECT_NUM_TB_STATS; + REORG_TABLE_TYPE=$TB_STATS; + MAX_ASYNC_REORGS_ALLOWED=$MAX_ASYNC_REORGS_ALLOWED_ORG; + REORG_TIMEOUT_WINDOW_ACTION=$REORG_TIMEOUT_WINDOW_ACTION_ORG; + SLEEP_INTERVAL_TIME=$SLEEP_INTERVAL_TIME_ORG; + + elif [ "${REORGCHK_TB_IF_STATS_OPTION:$REORG_TYPE:1}" == "3" ]; then + TB_STATS=0; + IF_STATS=3; + NUM_REORG_OBJECTS=$OBJECT_NUM_IF_STATS; + REORG_TABLE_TYPE=$IF_STATS; + MAX_ASYNC_REORGS_ALLOWED=1; + REORG_TIMEOUT_WINDOW_ACTION=1; + SLEEP_INTERVAL_TIME=$( echo $SLEEP_INTERVAL_TIME_ORG | awk '{ print $1/3 }'); + fi + + reorgTables + done + else + if [ $TB_STATS -eq 1 ]; then + REORG_TABLE_TYPE=$TB_STATS; + elif [ $IX_STATS -eq 2 ]; then + REORG_TABLE_TYPE=$IX_STATS; + elif [ $IF_STATS -eq 3 ]; then + REORG_TABLE_TYPE=$IF_STATS; + fi + reorgTables + fi + + ## list current state of OBJECT_ARRAY + #printf "%-15s %-30s %-15s %-30s %-5s %-10s %-25s %-25s %-10s %-10s %-10s %-15s\n" "TABSCHEMA" "TABNAME" "INDSCHEMA" "INDNAME" "IID" "REORG_STATUS" "REORG_START" "REORG_END" "TABLEID" "TBSPACEID" "LOCK_COUNT" "TABLE_TYPE" + listOBJECT_ARRAY; + + ## + ## now do runstats + ## + log 3 "Starting runstats of ..." + RUNSTATS_TIMEOUT_WINDOW_START_TIME_SECONDS=$( date '+%s' ); + + for((ii=0; ii< ${#OBJECT_ARRAY[@]}; ii++)) + do + + ## check runstats window maintenance time + MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS=$( date '+%s' ); + DIFF=$(( MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS - REORG_TIMEOUT_WINDOW_START_TIME_SECONDS )); + if [ $DIFF -ge $(( REORG_TIMEOUT_WINDOW_SECONDS + RUNSTATS_TIMEOUT_WINDOW_SECONDS)) ]; then + #log 3 "$REORG_TIMEOUT_WINDOW_START_TIME_SECONDS $MAINTENANCE_TIMEOUT_WINDOW_TIME_NOW_SECONDS $REORG_TIMEOUT_WINDOW_SECONDS $RUNSTATS_TIMEOUT_WINDOW_SECONDS $DIFF" + log 3 "Runstats window ending, runstats window time exceeded" + break + fi + + ## get table info + TABSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f1 ); + TABNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f2 ); + INDSCHEMA=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f3 ); + INDNAME=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f4 ); + IID=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f5 ); + OBJECT_REORG_STATUS=$( echo ${OBJECT_ARRAY[$ii]} | cut -d# -f6 ); + + ## only do a runstats if table/index has completed + ## verify stats time so we dont kick off another runstats on the same table + if [ "$OBJECT_REORG_STATUS" == "COMPLETED" ]; then + STATS_TIME=$( db2 -x "select stats_time from syscat.tables where tabschema='$TABSCHEMA' and tabname='$TABNAME' and stats_time < TIMESTAMP('$WINDOW_START_TIME_DB2') " ); + rc=$? + if [ $rc -eq 0 ]; then + log 3 "Starting runstats on $TABSCHEMA.$TABNAME" + # db2 -v "runstats on table $TABSCHEMA.$TABNAME WITH DISTRIBUTION ON ALL COLUMNS AND SAMPLED DETAILED INDEXES ALL ALLOW WRITE ACCESS"; + #db2 -v "runstats on table $TABSCHEMA.$TABNAME WITH DISTRIBUTION ON KEY COLUMNS AND SAMPLED DETAILED INDEXES ALL ALLOW WRITE ACCESS UTIL_IMPACT_PRIORITY 50" > /dev/null 2>&1 + db2 -v "runstats on table $TABSCHEMA.$TABNAME ON ALL COLUMNS WITH DISTRIBUTION ON ALL COLUMNS AND DETAILED INDEXES ALL ALLOW WRITE ACCESS " > /dev/null 2>&1 + log 3 "Finished runstats on $TABSCHEMA.$TABNAME" + fi + fi + + done + + fi ## EXECUTE_TABLE_REORG + + fi ## INPLACE + +#done ## DBNAMES ## ## cleanup ## log 3 "Completed $0 at $(date)" - exit 0 diff --git a/instance-applications/120-ibm-db2u-database/files/runstats_rebind.sh b/instance-applications/120-ibm-db2u-database/files/runstats_rebind.sh index a2159405d..dc97ace71 100755 --- a/instance-applications/120-ibm-db2u-database/files/runstats_rebind.sh +++ b/instance-applications/120-ibm-db2u-database/files/runstats_rebind.sh @@ -1,22 +1,22 @@ #!/bin/bash # *************************************************************************** -# Author: Fu Le Qing (Roking) -# Email: leqingfu@cn.ibm.com -# Date: 10-31-2018 +# Author: Fu Le Qing (Roking) +# Email: leqingfu@cn.ibm.com +# Date: 10-31-2018 # -# Description: This script updates statistics of tables, -# associated indexes in the database, and sends an email +# Description: This script updates statistics of tables, +# associated indexes in the database, and sends an email # to a specified email list. # # ******** THIS NEEDS TO BE RUN AS INSTANCE OWNER. ************** # -# Revision history: -# 10-31-2018 Fu Le Qing (Roking) -# Original version -# 11-16-2018 Fu Le Qing (Roking) -# Skip the tables which are ongoing with reorg -# 09-08-2023 Fu Le Qing (Roking) -# Update for MAS +# Revision history: +# 10-31-2018 Fu Le Qing (Roking) +# Original version +# 11-16-2018 Fu Le Qing (Roking) +# Skip the tables which are ongoing with reorg +# 09-08-2023 Fu Le Qing (Roking) +# Update for MAS # # *************************************************************************** # @@ -26,22 +26,22 @@ if [ -f /mnt/backup/bin/.PROPS ] then . /mnt/backup/bin/.PROPS - DOW=`date | awk '{print $1}'` + DOW=$(date | awk '{print $1}') if [ ${DOW} != ${DAYOFFULL} ] then - echo "Runstats runs only on Day of Full backup. . . !!! Exiting !" + echo "Runstats runs only on Day of Full backup. . . !!! Exiting !" exit 0 fi fi # -- Standard Parameters -INSTANCE=`whoami` -INSTANCE_HOME=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${INSTANCE}" | awk -F ',' '{print $5}'| sed 's/\/sqllib//'` +INSTANCE=$(whoami) +INSTANCE_HOME=$(/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${INSTANCE}" | awk -F ',' '{print $5}'| sed 's/\/sqllib//') mkdir -p ${INSTANCE_HOME}/maintenance/logs -DATESTAMP=`date "+%Y-%m-%d-%H.%M.%S"` +DATESTAMP=$(date "+%Y-%m-%d-%H.%M.%S") -pidfile="${INSTANCE_HOME}/.`basename ${0}`.pid" -if [ -e ${pidfile} ] && $kill -0 `cat ${pidfile}` 2>/dev/null +pidfile="${INSTANCE_HOME}/.$(basename ${0}).pid" +if [ -e ${pidfile} ] && $kill -0 $(cat ${pidfile}) 2>/dev/null then exit 0 fi @@ -64,37 +64,46 @@ fi RUNSTATS_TMP_FILE="${INSTANCE_HOME}/bin/.runstats.sql" REBIND_TMP_FILE="${INSTANCE_HOME}/bin/.rebind.sql" -for DB in `db2 list db directory | grep -B 5 Indirect | grep "Database name" | cut -d= -f2` +for DB in $(db2 list db directory | grep -B 5 Indirect | grep "Database name" | cut -d= -f2) do - RUNSTATS_REBIND_LOG="${INSTANCE_HOME}/maintenance/logs/runstats_rebind_${DB}_${DATESTAMP}.log" + RUNSTATS_REBIND_LOG="${INSTANCE_HOME}/maintenance/logs/runstats_rebind_${DB}_${DATESTAMP}.log" - role=`db2 get db cfg for ${DB} | grep "HADR database role" | cut -d= -f2 |sed 's/ //g'` - if [ "$role" != "STANDBY" ]; then - if [[ -f ${RUNSTATS_TMP_FILE} ]]; then - rm ${RUNSTATS_TMP_FILE} - fi - if [[ -f ${REBIND_TMP_FILE} ]]; then - rm ${REBIND_TMP_FILE} - fi + role=$(db2 get db cfg for ${DB} | grep "HADR database role" | cut -d= -f2 |sed 's/ //g') + if [ "$role" != "STANDBY" ]; then + if [[ -f ${RUNSTATS_TMP_FILE} ]]; then + rm ${RUNSTATS_TMP_FILE} + fi + if [[ -f ${REBIND_TMP_FILE} ]]; then + rm ${REBIND_TMP_FILE} + fi - db2 connect to ${DB} | tee ${RUNSTATS_REBIND_LOG} - if [[ $? -eq 0 ]]; then - db2 -x "select 'RUNSTATS ON TABLE \"' ||rtrim(tab.tabschema)||'\".\"'|| tab.tabname ||'\" ON ALL COLUMNS WITH DISTRIBUTION ON ALL COLUMNS AND DETAILED INDEXES ALL ALLOW WRITE ACCESS;' from syscat.tables tab left join sysibmadm.SNAPTAB_REORG reg on tab.tabschema=reg.TABSCHEMA and tab.tabname=reg.TABNAME and reg.REORG_STATUS not in ('COMPLETED','STOPPED') where tab.type='T' and reg.tabname is null" > ${RUNSTATS_TMP_FILE} - - echo -e "Begin processing of runstats @ ${DATESTAMP} ...\n" | tee -a ${RUNSTATS_REBIND_LOG} - db2 -txvf ${RUNSTATS_TMP_FILE} | tee -a ${RUNSTATS_REBIND_LOG} - echo -e "\nEnd processing of runstats @ ${DATESTAMP}" | tee -a ${RUNSTATS_REBIND_LOG} - rm ${RUNSTATS_TMP_FILE} - - db2 -x "select 'rebind package \"' ||rtrim(PKGSCHEMA)||'\".\"'|| PKGNAME ||'\";' from syscat.packages where PKGSCHEMA not like 'SYSIBM%' and PKGSCHEMA not like 'NULL%' " > ${REBIND_TMP_FILE} - - echo -e "\n ----------------------------------------------- " | tee -a ${RUNSTATS_REBIND_LOG} - echo -e "Begin processing of rebind @ ${DATESTAMP} ...\n" | tee -a ${RUNSTATS_REBIND_LOG} - db2 -txvf ${REBIND_TMP_FILE} | tee -a ${RUNSTATS_REBIND_LOG} - echo -e "\nEnd processing of rebind @ ${DATESTAMP}" | tee -a ${RUNSTATS_REBIND_LOG} - - rm ${REBIND_TMP_FILE} - db2 terminate - fi - fi + db2 connect to ${DB} | tee ${RUNSTATS_REBIND_LOG} + if [[ $? -eq 0 ]]; then + # -- Runstats of the tables + db2 -x "select 'RUNSTATS ON TABLE \"' ||rtrim(tab.tabschema)||'\".\"'|| tab.tabname ||'\" ON ALL COLUMNS WITH DISTRIBUTION ON ALL COLUMNS AND DETAILED INDEXES ALL ALLOW WRITE ACCESS;' \ + from syscat.tables tab left join sysibmadm.SNAPTAB_REORG reg on tab.tabschema=reg.TABSCHEMA and tab.tabname=reg.TABNAME \ + and reg.REORG_STATUS not in ('COMPLETED','STOPPED') where tab.type='T' and reg.tabname is null" > ${RUNSTATS_TMP_FILE} + + # -- Execution of runstats + echo -e "Begin processing of runstats @ ${DATESTAMP} ...\n" | tee -a ${RUNSTATS_REBIND_LOG} + db2 -txvf ${RUNSTATS_TMP_FILE} | tee -a ${RUNSTATS_REBIND_LOG} + echo -e "\nEnd processing of runstats @ ${DATESTAMP}" | tee -a ${RUNSTATS_REBIND_LOG} + rm ${RUNSTATS_TMP_FILE} + + # -- Rebind of the packages + db2 -x "select 'rebind package \"' ||rtrim(PKGSCHEMA)||'\".\"'|| PKGNAME ||'\";' from syscat.packages + where PKGSCHEMA not like 'SYSIBM%' and PKGSCHEMA not like 'NULL%' " > ${REBIND_TMP_FILE} + + # -- Execution of Rebind + echo -e "\n ----------------------------------------------- " | tee -a ${RUNSTATS_REBIND_LOG} + echo -e "Begin processing of rebind @ ${DATESTAMP} ...\n" | tee -a ${RUNSTATS_REBIND_LOG} + db2 -txvf ${REBIND_TMP_FILE} | tee -a ${RUNSTATS_REBIND_LOG} + echo -e "\nEnd processing of rebind @ ${DATESTAMP}" | tee -a ${RUNSTATS_REBIND_LOG} + + rm ${REBIND_TMP_FILE} + db2 terminate + fi + fi done + +# -- End of the Script \ No newline at end of file