Pages

Showing posts with label bash. Show all posts
Showing posts with label bash. Show all posts

Thursday, February 18, 2021

GG Restart from crontab

=========================
General
=========================

 Restart GG processes from crontab

=========================
Code
=========================
crontab entry
1 06 1 * * /software/oracle/oracle/scripts/gg_restart/gg_restart_all_groups.sh

gg_restart_all_groups.sh
#!/bin/bash

WORK_HOME=/software/oracle/oracle/scripts/gg_restart
LOG_FILE=gg_restart.log

cd $WORK_HOME

./gg_restart_log.sh "${WORK_HOME}" "${LOG_FILE}" "START"

./gg_restart_group.sh EXT_S "ext ext_s*" "${WORK_HOME}" "${LOG_FILE}"
./gg_restart_group.sh EXT_P "ext ext_p*" "${WORK_HOME}" "${LOG_FILE}"
./gg_restart_group.sh EXT_I "ext ext_i*" "${WORK_HOME}" "${LOG_FILE}"

./gg_restart_group.sh DPM_S "ext dpm_s*" "${WORK_HOME}" "${LOG_FILE}"
./gg_restart_group.sh DPM_P "ext dpm_p*" "${WORK_HOME}" "${LOG_FILE}"
./gg_restart_group.sh DPM_I "ext dpm_i*" "${WORK_HOME}" "${LOG_FILE}"

./gg_restart_group.sh REP_S "rep rep_s*" "${WORK_HOME}" "${LOG_FILE}"
./gg_restart_group.sh REP_P "rep rep_p*" "${WORK_HOME}" "${LOG_FILE}"
./gg_restart_group.sh REP_I "rep rep_i*" "${WORK_HOME}" "${LOG_FILE}"

./gg_restart_log.sh "${WORK_HOME}" "${LOG_FILE}" "FINISH" 


gg_restart_log.sh

#!/bin/bash

WORK_HOME=$1
LOG_FILE_NAME=$2
VERB=$3

cd $WORK_HOME

LOG_FILE=${WORK_HOME}/${LOG_FILE_NAME}
RUN_DATE=`date "+%Y%m%d"_"%H%M%S"`

touch $LOG_FILE

if [[ $VERB == "START" ]]; then

 echo "" >>$LOG_FILE
 echo ========================================== >>$LOG_FILE
 echo Start at $RUN_DATE >>$LOG_FILE
 echo ========================================== >>$LOG_FILE
 echo "" >>$LOG_FILE
 echo "Before Restart: " >>$LOG_FILE
 echo "" >>$LOG_FILE 
 free -m >> $LOG_FILE
 echo "" >> $LOG_FILE
 sar -r >> $LOG_FILE
  echo "" >>$LOG_FILE 
fi

if [[ $VERB == "FINISH" ]]; then
  echo "" >>$LOG_FILE
  echo "---------------------------" >>$LOG_FILE
  echo "After Restart: " >>$LOG_FILE
  echo "---------------------------" >>$LOG_FILE  
  echo "" >>$LOG_FILE  
  free -m >> $LOG_FILE
  echo "" >> $LOG_FILE
  sar -r >> $LOG_FILE
  echo "" >>$LOG_FILE    
  echo ========================================== >>$LOG_FILE
  echo Finished at $RUN_DATE >>$LOG_FILE
  echo ========================================== >>$LOG_FILE
  echo "" >>$LOG_FILE
fi

exit 0 

gg_restart_group.sh
#!/bin/bash

GROUP_NAME=$1
RESTART_GROUP=$2
WORK_DIR=$3
LOG_FILE_NAME=$4

#GROUP_NAME=EXT_S
#RESTART_GROUP="ext ext_s*"

LOG_FILE=${WORK_DIR}/${LOG_FILE_NAME}


clear
echo "Restarting ${GROUP_NAME} ${RESTART_GROUP}" >> ${LOG_FILE}
echo "Log File is: ${LOG_FILE}"
exit 0

. /etc/profile > /dev/null 2> /dev/null
TEMP_LOG_FILE="/tmp/restart_temp.tmp"

RUN_DATE=`date "+%Y%m%d"_"%H%M%S"`

# Setting Oracle GoldenGate environments
if [ "$OGGSH" = "" ]; then
   OGGSH=$INFRA_ROOT/oggsh
fi

if [ "$ORACLE_SID" = "" ]; then
   INSTANCE=igt
else
   INSTANCE=$ORACLE_SID
fi
. $OGGSH/gg_login.sh $INSTANCE

touch $LOG_FILE
touch $TEMP_LOG_FILE

echo "" >>$LOG_FILE
echo ========================================== >>$LOG_FILE
echo Start Restart for ${GROUP_NAME} group ${RESTART_GROUP} at $RUN_DATE >>$LOG_FILE
echo ========================================== >>$LOG_FILE
echo "" >>$LOG_FILE
echo ========================================== >>$LOG_FILE
echo Initial State >>$LOG_FILE
echo ========================================== >>$LOG_FILE
echo "" >>$LOG_FILE

$GG_HOME/ggsci >$TEMP_LOG_FILE <<EOD
info all
exit
EOD
cat  $TEMP_LOG_FILE | grep ${GROUP_NAME} >>$LOG_FILE

echo about to do  "stop ext $RESTART_GROUP" >>$LOG_FILE

$GG_HOME/ggsci >$TEMP_LOG_FILE <<EOF
stop ${RESTART_GROUP}
exit
EOF
cat  $TEMP_LOG_FILE >>$LOG_FILE

echo "" >>$LOG_FILE
echo ========================================== >>$LOG_FILE
echo After Stop >>$LOG_FILE
echo ========================================== >>$LOG_FILE
echo "" >>$LOG_FILE

sleep 10

$GG_HOME/ggsci >$TEMP_LOG_FILE <<EOD
info all
exit
EOD
cat  $TEMP_LOG_FILE | grep ${GROUP_NAME}  >>$LOG_FILE


echo "Starting  ${RESTART_GROUP}" >> $LOG_FILE

$GG_HOME/ggsci >$TEMP_LOG_FILE <<EOF
start ${RESTART_GROUP}
exit
EOF
cat $TEMP_LOG_FILE >>$LOG_FILE

echo "" >>$LOG_FILE
echo ========================================== >>$LOG_FILE
echo After Start >>$LOG_FILE
echo ========================================== >>$LOG_FILE
echo "" >>$LOG_FILE

sleep 10

$GG_HOME/ggsci >$TEMP_LOG_FILE <<EOD
info all
exit
EOD
cat  $TEMP_LOG_FILE | grep ${GROUP_NAME} >>$LOG_FILE


NOT_RUNNING=`grep STOPPED $TEMP_LOG_FILE |grep ${GROUP_NAME} | wc -l`

if [ $NOT_RUNNING -gt 0 ]; then

echo "Some Processes were not started. Attempting restart" >>$LOG_FILE

$GG_HOME/ggsci >$TEMP_LOG_FILE <<EOF
start ${RESTART_GROUP}
exit
EOF
cat $TEMP_LOG_FILE >>$LOG_FILE

fi

#============================================================
# Second restart
#============================================================
sleep 10

$GG_HOME/ggsci >$TEMP_LOG_FILE <<EOD
info all
exit
EOD
cat  $TEMP_LOG_FILE | grep ${GROUP_NAME} >>$LOG_FILE

NOT_RUNNING=`grep STOPPED $TEMP_LOG_FILE | wc -l`
if [ $NOT_RUNNING -gt 0 ]; then

echo "Some Processes were not started. Attempting restart" >>$LOG_FILE

$GG_HOME/ggsci > $TEMP_LOG_FILE <<EOF
start ${RESTART_GROUP}
exit
EOF
cat $TEMP_LOG_FILE >>$LOG_FILE

sleep 10

$GG_HOME/ggsci >$TEMP_LOG_FILE <<EOD
info all
exit
EOD
cat  $TEMP_LOG_FILE | grep ${GROUP_NAME} >>$LOG_FILE

fi

echo "" >>$LOG_FILE
echo ========================================== >>$LOG_FILE
echo Finished Restart for $GROUP_NAME $RESTART_GROUP at $RUN_DATE >>$LOG_FILE
echo ========================================== >>$LOG_FILE
echo "" >>$LOG_FILE
 




Tuesday, May 5, 2020

sqloader example. Loading several files types via bash and sqloader.

====================
General
====================
Load via bash and sqloader to three tables.

====================
Files
====================
ora_env.ini
ipn_profile_load.ini
main_ipn_conv_load.sh
load_handler.sh
sql_loader_cmd.sh
sql_loader_vrs_configuration.ctl
sql_loader_black_list.ctl
sql_loader_mccmnc.ctl

====================
Code
====================

==================================
ora_env.ini
==================================
ORA_VER=1120
ORACLE_SID=igt
ORACLE_BASE=/software/oracle
ORACLE_HOME=/software/oracle/112

==================================
ipn_profile_load.ini
==================================
DB_USER=ipn_conv
DB_PASS=ipn_conv
DB_INST=igt
DELIMITER="==================================="

WORK_DIR=/starhome/iu/workarea/IPN_LOAD
SCRIPTS_DIR=/starhome/iu/workarea/IPN_LOAD/scripts

LOG_FILE=/starhome/iu/workarea/IPN_LOAD/logs/ipn_load.log

INPUT_DIR=/starhome/iu/workarea/IPN_LOAD/files/input
PROCESS_DIR=/starhome/iu/workarea/IPN_LOAD/files/in_process
HANDLED_FILES_DIR=/starhome/iu/workarea/IPN_LOAD/files/handled
LOG_DIR=/starhome/iu/workarea/IPN_LOAD/logs

INPUT_FILE_NAME_ZIP=moco_managed_IPN_LOAD*dat.gz
INPUT_FILE_NAME=moco_managed_IPN_LOAD*dat

PATTERN_EXAMPLE=IPN_LOAD_[0-9]{2}.[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]{2}.csv
INPUT_FILE_PATTERN_1=vrs_configuration.csv
INPUT_FILE_PATTERN_2=black_list.csv
INPUT_FILE_PATTERN_3=mccmnc.csv

FLOW_1=vrs_configuration
FLOW_2=black_list
FLOW_3=mccmnc

SQL_LOADER_DIR=/starhome/iu/workarea/IPN_LOAD/loader_code
SQL_LOADER_DIR_IN=/starhome/iu/workarea/IPN_LOAD/loader_code/in
SQL_LOADER_DIR_BAD=/starhome/iu/workarea/IPN_LOAD/loader_code/bad
SQL_LOADER_DIR_LOG=/starhome/iu/workarea/IPN_LOAD/loader_code/log
SQL_LOADER_DIR_OLD=/starhome/iu/workarea/IPN_LOAD/loader_code/old

SQL_LOADER_DAT_FILE=/starhome/iu/workarea/IPN_LOAD/loader_code/in/load_data.dat
SQL_LOADER_BAD_FILE=/starhome/iu/workarea/IPN_LOAD/loader_code/bad/bad_load_data.dat
SQL_LOADER_HIST_FILE=/starhome/iu/workarea/IPN_LOAD/loader_code/old/load_data.old
SQL_LOADER_LOG_FILE=/starhome/iu/workarea/IPN_LOAD/loader_code/log/load_load.log

CTL_FILE_1=/starhome/iu/workarea/IPN_LOAD/loader_code/sql_loader_vrs_configuration.ctl
CTL_FILE_2=/starhome/iu/workarea/IPN_LOAD/loader_code/sql_loader_black_list.ctl
CTL_FILE_3=/starhome/iu/workarea/IPN_LOAD/loader_code/sql_loader_mccmnc.ctl

==================================
main_ipn_conv_load.sh
==================================
!/bin/bash
INI_FILE=/starhome/iu/workarea/IPN_LOAD/scripts/ipn_profile_load.ini

#=====================
#functions()
#=====================
get_value(){
 FILE_NAME=$1
 VALUE_NAME=$2
 #echo "Looking for $VALUE_NAME in file $FILE_NAME"
 v_return_value=`grep -w $VALUE_NAME $FILE_NAME | sed s/\=/" "/ | awk '{print $2}'`
 echo $v_return_value
}

write_to_log(){
 echo $1
 echo $1 >> $LOG_FILE
}

create_folder(){
 FOLDER_NAME=$1
 if [[ ! -d ${FOLDER_NAME} ]]; then
   mkdir ${FOLDER_NAME}
 fi
}
#=====================
#set variables
#=====================
DELIMITER=$(get_value ${INI_FILE} DELIMITER)
LOG_FILE=$(get_value ${INI_FILE} LOG_FILE)
RUN_DATE=`date +"%Y%m%d"_"%H%M%S"`
this_script=`basename $0`

#---------------------
#main()
#---------------------

echo "LOG_FILE = ${LOG_FILE}"
write_to_log ""
write_to_log $DELIMITER
write_to_log "$this_script .........: Starting at ${RUN_DATE}"
write_to_log $DELIMITER
write_to_log ""

#write_to_log "INPUT_DIR"
INPUT_DIR=$(get_value ${INI_FILE} INPUT_DIR)
create_folder ${INPUT_DIR}

#write_to_log "PROCESS_DIR"
PROCESS_DIR=$(get_value ${INI_FILE} PROCESS_DIR)
create_folder ${PROCESS_DIR}

#write_to_log "HANDLED_FILES_DIR"
HANDLED_FILES_DIR=$(get_value ${INI_FILE} HANDLED_FILES_DIR)
create_folder ${HANDLED_FILES_DIR}

#write_to_log "LOG_DIR"
LOG_DIR=$(get_value ${INI_FILE} LOG_DIR)
create_folder ${LOG_DIR}

#write_to_log "SQL_LOADER_DIR"
SQL_LOADER_DIR=$(get_value ${INI_FILE} SQL_LOADER_DIR)
create_folder ${SQL_LOADER_DIR}

#write_to_log "SQL_LOADER_DIR_IN"
SQL_LOADER_DIR_IN=$(get_value ${INI_FILE} SQL_LOADER_DIR_IN)
create_folder ${SQL_LOADER_DIR_IN}

#write_to_log "SQL_LOADER_DIR_BAD"
SQL_LOADER_DIR_BAD=$(get_value ${INI_FILE} SQL_LOADER_DIR_BAD)
create_folder ${SQL_LOADER_DIR_BAD}

#write_to_log "SQL_LOADER_DIR_LOG"
SQL_LOADER_DIR_LOG=$(get_value ${INI_FILE} SQL_LOADER_DIR_LOG)
create_folder ${SQL_LOADER_DIR_LOG}

#write_to_log "SQL_LOADER_DIR_OLD"
SQL_LOADER_DIR_OLD=$(get_value ${INI_FILE} SQL_LOADER_DIR_OLD)
create_folder ${SQL_LOADER_DIR_OLD}

write_to_log ""
. /etc/sh/orash/oracle_login.sh igt
WORK_AREA=/starhome/iu/workarea/IPN_LOAD/scripts

cd ${WORK_AREA}
./load_handler.sh
exit_status=$?
RUN_DATE=`date +"%Y%m%d"_"%H%M%S"`
write_to_log ""
write_to_log $DELIMITER
write_to_log "Finished Running $this_script. Process status: $exit_status"
write_to_log $DELIMITER
write_to_log ""

mv ${LOG_FILE} ${LOG_FILE}_${RUN_DATE}
write_to_log "Log File: ${LOG_FILE}_${RUN_DATE}"

exit $exit_status

==================================
load_handler.sh
==================================
INI_FILE=/starhome/iu/workarea/IPN_LOAD/scripts/ipn_profile_load.ini

#=====================
#functions()
#=====================
get_value(){
 FILE_NAME=$1
 VALUE_NAME=$2
 #echo "Looking for $VALUE_NAME in file $FILE_NAME"
 v_return_value=`grep -w $VALUE_NAME $FILE_NAME | sed s/\=/" "/ | awk '{print $2}'`
 echo $v_return_value
}

write_to_log(){
 echo $1
 echo $1 >> $LOG_FILE
}


load_file(){
 INPUT_FILE_PATTERN=$1
 FLOW_TYPE=$2
 #write_to_log "Input Files in ${INPUT_DIR}:"
 find ${INPUT_DIR} -type f -printf "%f\n"
 #write_to_log "Looking for ${INPUT_FILE_PATTERN} inside ${INPUT_DIR}"
 find ${INPUT_DIR} -type f | egrep ${INPUT_FILE_PATTERN}

 files_num=`find ${INPUT_DIR} -type f | egrep ${INPUT_FILE_PATTERN} 2>/dev/null | wc -l`
 #write_to_log "$this_script ...: Number of Input Files: ${INPUT_DIR}/*${INPUT_FILE_PATTERN}:  $files_num"

 if [[ $files_num == 0 ]];then
  write_to_log "$this_script ...: No Files to Process"
  return 0
 fi

 if [[ $files_num > 1 ]];then
  write_to_log "$this_script ...: More than 1 input File. Exiting !!! "
  exit 1
 fi

 file_name=`find ${INPUT_DIR} -type f | egrep ${INPUT_FILE_PATTERN}`
 BASENAME=`basename $file_name`
 write_to_log "Loading File: $file_name"
 #write_to_log "cp $file_name ${PROCESS_DIR}/${BASENAME}"
 #Remove Empty Lines
 less $file_name | grep -v ^$ > ${PROCESS_DIR}/${BASENAME}

 #!!!!!!!!!!!!!!!!!!!!!!!
 #Remove the file
 rm $file_name
 #!!!!!!!!!!!!!!!!!!!!!!!

 #cp $file_name ${PROCESS_DIR}/${BASENAME}
 #write_to_log "./sql_loader_cmd.sh ${INI_FILE} ${file_name} ${FLOW_TYPE}"

 ./sql_loader_cmd.sh ${INI_FILE} ${PROCESS_DIR}/${BASENAME} ${FLOW_TYPE}
 #write_to_log "$this_script .............: Finished "
 mv ${PROCESS_DIR}/${BASENAME} ${HANDLED_FILES_DIR}/${BASENAME}
 #write_to_log $DELIMITER
}

#=====================
#set variables
#=====================
RUN_DATE=`date +"%Y%m%d"_"%H%M%S"`
SCRIPTS_DIR=$(get_value ${INI_FILE} SCRIPTS_DIR)
LOG_DIR=$(get_value ${INI_FILE} LOG_DIR)
INPUT_FILE_PATTERN_1=$(get_value ${INI_FILE} INPUT_FILE_PATTERN_1)
INPUT_FILE_PATTERN_2=$(get_value ${INI_FILE} INPUT_FILE_PATTERN_2)
INPUT_FILE_PATTERN_3=$(get_value ${INI_FILE} INPUT_FILE_PATTERN_3)

DELIMITER=$(get_value ${INI_FILE} DELIMITER)
LOG_FILE=$(get_value ${INI_FILE} LOG_FILE)
WORK_DIR=$(get_value ${INI_FILE} WORK_DIR)
INPUT_DIR=$(get_value ${INI_FILE} INPUT_DIR)

INPUT_DIR=$(get_value ${INI_FILE} INPUT_DIR)
PROCESS_DIR=$(get_value ${INI_FILE} PROCESS_DIR)
HANDLED_FILES_DIR=$(get_value ${INI_FILE} HANDLED_FILES_DIR)


FLOW_1=$(get_value ${INI_FILE} FLOW_1 )
FLOW_2=$(get_value ${INI_FILE} FLOW_2 )
FLOW_3=$(get_value ${INI_FILE} FLOW_3 )

#=====================
#main
#=====================
. /etc/sh/orash/oracle_login.sh igt
this_script=`basename $0`
echo "Log File is: $LOG_FILE"
touch ${LOG_FILE}
#write_to_log ""
#write_to_log $DELIMITER
#write_to_log "$this_script .............: Starting Run at $RUN_DATE"
#write_to_log $DELIMITER
cd ${SCRIPTS_DIR}

write_to_log ""
write_to_log "${DELIMITER}"
write_to_log "Start Loading file type ${INPUT_FILE_PATTERN_1} Flow: ${FLOW_1}"
load_file "${INPUT_FILE_PATTERN_1}" "${FLOW_1}"
ret_code=$?
if [[ $ret_code > 0 ]]; then
 exit $ret_code
fi
write_to_log ""

write_to_log ""
write_to_log "${DELIMITER}"
write_to_log "Start Loading file type ${INPUT_FILE_PATTERN_2} Flow: ${FLOW_2}"
sleep 2
load_file "${INPUT_FILE_PATTERN_2}" "${FLOW_2}"
ret_code=$?
if [[ $ret_code > 0 ]]; then
 exit $ret_code
fi
write_to_log ""


write_to_log ""
write_to_log "${DELIMITER}"
write_to_log "Start Loading file type ${INPUT_FILE_PATTERN_3} Flow: ${FLOW_3}"
sleep 2
load_file "${INPUT_FILE_PATTERN_3}" "${FLOW_3}"
ret_code=$?
if [[ $ret_code > 0 ]]; then
 exit $ret_code
fi
write_to_log ""

write_to_log "${DELIMITER}"
write_to_log ""

exit 0

==================================
sql_loader_cmd.sh
==================================
#!/bin/bash
INI_FILE=$1
FILE_NAME=$2
FLOW_TYPE=$3
#=====================
#functions()
#=====================
get_value(){
 FILE_NAME=$1
 VALUE_NAME=$2
 #echo "Looking for $VALUE_NAME in file $FILE_NAME"
 v_return_value=`grep -w $VALUE_NAME $FILE_NAME | sed s/\=/" "/ | awk '{print $2}'`
 echo $v_return_value
}
write_to_log(){
 echo $1
 echo $1 >> $LOG_FILE
}
#=====================
#set variables
#=====================
RUN_DATE=`date +"%Y%m%d"_"%H%M%S"`
INPUT_FILE_PATTERN=$(get_value ${INI_FILE} INPUT_FILE_PATTERN)
INPUT_FILE_PATTERN_1=$(get_value ${INI_FILE} INPUT_FILE_PATTERN_1)
INPUT_FILE_PATTERN_2=$(get_value ${INI_FILE} INPUT_FILE_PATTERN_2)
INPUT_FILE_PATTERN_3=$(get_value ${INI_FILE} INPUT_FILE_PATTERN_3
)
LOG_FILE=$(get_value ${INI_FILE} LOG_FILE)
INPUT_DIR=$(get_value ${INI_FILE} INPUT_DIR)
SOURCE_DATA=$(get_value ${INI_FILE} SOURCE_DATA)
OUTPUT_DIR=$(get_value ${INI_FILE} OUTPUT_DIR)
INPUT_FILE_NAME=$(get_value ${INI_FILE} INPUT_FILE_NAME)
HANDLED_FILES_DIR=$(get_value ${INI_FILE} HANDLED_FILES_DIR)
SQL_LOADER_LOG_FILE=$(get_value ${INI_FILE} SQL_LOADER_LOG_FILE)

#CTL_FILE=$(get_value ${INI_FILE} CTL_FILE)
CTL_FILE_1=$(get_value ${INI_FILE} CTL_FILE_1)
CTL_FILE_2=$(get_value ${INI_FILE} CTL_FILE_2)
CTL_FILE_3=$(get_value ${INI_FILE} CTL_FILE_3)

FLOW_1=$(get_value ${INI_FILE} FLOW_1 )
FLOW_2=$(get_value ${INI_FILE} FLOW_2 )
FLOW_3=$(get_value ${INI_FILE} FLOW_3 )

SQL_LOADER_DAT_FILE=$(get_value ${INI_FILE} SQL_LOADER_DAT_FILE)
SQL_LOADER_BAD_FILE=$(get_value ${INI_FILE} SQL_LOADER_BAD_FILE)
SQL_LOADER_HIST_FILE=$(get_value ${INI_FILE} SQL_LOADER_HIST_FILE)
DB_USER=$(get_value ${INI_FILE} DB_USER)
DB_PASS=$(get_value ${INI_FILE} DB_PASS)
DB_INST=$(get_value ${INI_FILE} DB_INST)
DELIMITER=$(get_value ${INI_FILE} DELIMITER)

this_script=`basename $0`
write_to_log "$this_script ...: ----------------------------------------------------------"
write_to_log "$this_script ...: Starting ${this_script} at ${RUN_DATE}"
#write_to_log "$this_script ...: sqlldr ${DB_USER}/${DB_PASS}@${DB_INST} CONTROL=${CTL_FILE} LOG=${SQL_LOADER_LOG_FILE}"

. /etc/sh/orash/oracle_login.sh igt
. /starhome/iu/workarea/IPN_LOAD/scripts/ora_env.ini


#write_to_log "cp ${FILE_NAME} ${SQL_LOADER_DAT_FILE}"
cp ${FILE_NAME} ${SQL_LOADER_DAT_FILE}

if [[ $FLOW_TYPE == ${FLOW_1} ]]; then
 CTL_FILE=${CTL_FILE_1}
fi
if [[ $FLOW_TYPE == ${FLOW_2} ]]; then
 CTL_FILE=${CTL_FILE_2}
fi
if [[ $FLOW_TYPE == ${FLOW_3} ]]; then
 CTL_FILE=${CTL_FILE_3}
fi


if [[ -f ${FILE_NAME} ]]; then
  #write_to_log "$this_script ...: sqlldr ${DB_USER}/${DB_PASS}@${DB_INST} CONTROL=${CTL_FILE} LOG=${SQL_LOADER_LOG_FILE}"
  #write_to_log "$this_script ...: ls -l /starhome/iu/workarea/IPN_LOAD/loader_code/in/IPN_LOAD_data.dat"
  #write_to_log "$this_script ...: `ls -l /starhome/iu/workarea/IPN_LOAD/loader_code/in/IPN_LOAD_data.dat`"
  #write_to_log "$this_script ...: `env | grep ORA`"
  SQLDR_HOME=${ORACLE_HOME}/bin
  echo ${SQLDR_HOME}/sqlldr ${DB_USER}/${DB_PASS}@${DB_INST} CONTROL=${CTL_FILE} LOG=${SQL_LOADER_LOG_FILE}
  ${SQLDR_HOME}/sqlldr ${DB_USER}/${DB_PASS}@${DB_INST} CONTROL=${CTL_FILE} LOG=${SQL_LOADER_LOG_FILE}
  v_status=$?
  if [[ $v_status -ne 0 ]];then
    write_to_log "$this_script ...: ERROR in step: sqlldr ${DB_USER}/${DB_PASS}@${DB_INST} CONTROL=${CTL_FILE} LOG=${SQL_LOADER_LOG_FILE}"
    write_to_log "$this_script ...: Status: $v_status "
  fi
else
  write_to_log "$this_script ...: Error!! Could not find input file : ${FILE_NAME}"
fi


if [[ -f ${SQL_LOADER_LOG_FILE} ]]; then
  #write_to_log "$this_script ...: moving ${SQL_LOADER_LOG_FILE} : `ls -l ${SQL_LOADER_LOG_FILE}`"
  #write_to_log "$this_script ...: mv ${SQL_LOADER_LOG_FILE} ${SQL_LOADER_LOG_FILE}_${RUN_DATE}"
  mv ${SQL_LOADER_LOG_FILE} ${SQL_LOADER_LOG_FILE}_${RUN_DATE}
  write_to_log "sqloder log file: ${SQL_LOADER_LOG_FILE}_${RUN_DATE}"
else
  write_to_log "$this_script ...: ERROR! Cannot find Log File ${SQL_LOADER_LOG_FILE}"
fi

if [[ -f ${SQL_LOADER_BAD_FILE} ]]; then
  #write_to_log "$this_script ...: mv ${SQL_LOADER_BAD_FILE} ${SQL_LOADER_BAD_FILE}_${RUN_DATE}"
  mv ${SQL_LOADER_BAD_FILE} ${SQL_LOADER_BAD_FILE}_${RUN_DATE}
fi
if [[ -f ${SQL_LOADER_DAT_FILE} ]]; then
  #write_to_log "$this_script ...: moving ${SQL_LOADER_DAT_FILE}  : `ls -l ${SQL_LOADER_DAT_FILE} `"
  #write_to_log "$this_script ...: mv ${SQL_LOADER_DAT_FILE} ${SQL_LOADER_HIST_FILE}_${RUN_DATE}"
  mv ${SQL_LOADER_DAT_FILE} ${SQL_LOADER_HIST_FILE}_${RUN_DATE}
fi

RUN_DATE=`date +"%Y%m%d"_"%H%M%S"`
write_to_log "$this_script ...: ----------------------------------------------------------"
write_to_log "$this_script ...: Finished ${this_script} in mode ${RUN_MODE} at ${RUN_DATE}"
write_to_log "$this_script ...: ----------------------------------------------------------"

exit $v_status

==================================
sql_loader_vrs_configuration.ctl
==================================
LOAD DATA
INFILE '/starhome/iu/workarea/IPN_LOAD/loader_code/in/load_data.dat'
BADFILE '/starhome/iu/workarea/IPN_LOAD/loader_code/bad/bad_load_data.dat'
DISCARDMAX 0
TRUNCATE
INTO TABLE IPN_CONV_CONFIGURATION
WHEN (COUNTRY != BLANKS)
FIELDS TERMINATED BY ','
(
country              "TRIM (:country)",
comunity             "TRIM (:comunity)",
network_name         "TRIM (:network_name)",
redirection_code     "TRIM (:redirection_code)",
network_type         "TRIM (:network_type)",
rdc_pct              "TRIM (:rdc_pct)",
mcc                  "TRIM (:mcc)",
mnc                  "TRIM (:mnc)",
mcc_mnc              "TRIM (:mcc_mnc)",
tomia_network_name   "TRIM (:tomia_network_name)",
tomia_mcc_mnc        "TRIM (:tomia_mcc_mnc)",
tomia_community      "TRIM (:tomia_community)",
aux                  "TRIM (:aux)",
tomia_configuration  "TRIM (:tomia_configuration)",
tomia_network_barring "TRIM (:tomia_network_barring)",
ts_last_modified     "SYSDATE"
)

==================================
sql_loader_black_list.ctl
==================================
LOAD DATA
INFILE '/starhome/iu/workarea/IPN_LOAD/loader_code/in/load_data.dat'
BADFILE '/starhome/iu/workarea/IPN_LOAD/loader_code/bad/bad_load_data.dat'
DISCARDMAX 0
TRUNCATE
INTO TABLE IPN_CONV_BLACK_LIST
FIELDS TERMINATED BY ','
(
imsi                 "TRIM (:imsi)",
ts_last_modified     SYSDATE
)

==================================
sql_loader_mccmnc.ctl
==================================
LOAD DATA
INFILE '/starhome/iu/workarea/IPN_LOAD/loader_code/in/load_data.dat'
BADFILE '/starhome/iu/workarea/IPN_LOAD/loader_code/bad/load_data.dat'
DISCARDMAX 0
TRUNCATE
INTO TABLE IPN_CONV_TOMIA_MCCMNC
FIELDS TERMINATED BY ','
(
mcc              "TRIM (:mcc)",
mnc              "TRIM (:mnc)",
mccmnc           "TRIM (:mccmnc)",
iso              "TRIM (:iso)",
country_name     "TRIM (:country_name)",
country_code     "TRIM (:country_code)",
network_name     "TRIM (:network_name)",
country_network  "TRIM (:country_network)",
ts_last_modified SYSDATE
)

Wednesday, August 22, 2018

bash and sqlplus by Example: Kill Long Running Jobs

===============================
General
===============================
Known jobs should be running for only few minutes.
However, since they are connected via DB_LINKS to several remote databases, due to networks issues connection can be stuck, and job is hanged.
The workaround - it to kill this job, and start execution again.

Following code is executed from crontab every 15 minutes, and checks for jobs which run longer than 30 minutes.

===============================
Code
===============================

less kill_long_running_jobs.sh

#!/bin/bash
. /etc/profile
. /etc/sh/orash/oracle_login.sh igt

handle_return_code() {
  ret_code=$1
  step_name=$2
  if [[ $ret_code -ne 0 ]];then
    echo $step_name Finished With Error!! Return Code $ret_code
    exit $ret_code
  fi
}

WORK_DIR=/software/oracle/oracle/scripts
LOG_DIR=${WORK_DIR}
RUN_TIME=`date +"%Y%m%d_%H%M%S"`
LOG_FILE=killed_sessions.log
TEMP_FILE=long_running_sessions.tmp

cd ${WORK_DIR}
touch ${LOG_DIR}/${LOG_FILE}
rm -f ${TEMP_FILE}

sqlplus -s /nolog <<EOF
whenever sqlerror exit SQL.SQLCODE
@./kill_long_running_jobs.sql
EOF
handle_return_code $? ./kill_long_running_jobs.sql

if [[ ! -f ${TEMP_FILE} || ! -s ${TEMP_FILE} ]]; then
  echo "$RUN_TIME   No Long Running Sessions Found" >> ${LOG_DIR}/${LOG_FILE}
  exit 0
fi

echo reading file ${TEMP_FILE}

while read line
do
  KILL_CMD=`echo $line | awk 'BEGIN {FS="~"}{print $1}'`
  WHAT=`echo $line | awk 'BEGIN {FS="~"}{print $2}'`

  echo "$RUN_TIME   ${KILL_CMD} ${WHAT}" >> ${LOG_DIR}/${LOG_FILE}
  ${KILL_CMD}

done < ${TEMP_FILE}

less kill_long_running_jobs.sql

@./set_user.sql
connect &&user/&&pass@&&conn_str
PROMPT CONNECTED TO &&user/&&pass@&&conn_str

SET HEADING OFF
SET PAGESIZE 0
SET LINESIZE 200
SET NEWPAGE NONE
SET FEEDBACK OFF
SET VERIFY OFF
SPOOL long_running_sessions.tmp

SELECT 'kill -9 '||V_PROCESS.spid||'~'||DBA_JOBS.what AS LINUX_KILL
FROM   V$SESSION V_SESSION,
       V$PROCESS V_PROCESS,
       DBA_JOBS_RUNNING STUCK_JOBS,
       DBA_JOBS
WHERE  V_PROCESS.addr = V_SESSION.paddr
  AND  V_SESSION.type != 'BACKGROUND'
  AND  V_SESSION.sid = STUCK_JOBS.sid
  AND  DBA_JOBS.job = STUCK_JOBS.job
  AND  logon_time < (SYSDATE - 30/1440)
  AND  UPPER(DBA_JOBS.what) IN ('EXT_MOCO.SCHEDULE;','EXT_OVMD.SCHEDULE;','EXT_IPN.SCHEDULE;','EXT_GLR.SCHEDULE;','EXT_SPARX.SCH
EDULE;')
UNION ALL
SELECT 'kill -9 4545~EXT_SPARX.SCHEDULE;' AS LINUX_KILL
FROM DUAL
WHERE 1=2;

SPOOL OFF

less set_user.sql

DEFINE user=user_name
DEFINE pass=user_pass
DEFINE conn_str=orainst

Tuesday, February 6, 2018

Basic bash script Example: input parameters validation, get run_date, replace sting in a file, backup original file.

===========================
General
===========================
This script does search and remove a string inside a file.

===========================
Code
===========================

my_user@my_server:~/scripts>% less remove_table_from_rep.sh 
#!/bin/bash
echo
echo $0 Starting...........................
echo


#----------------------------
#Validate Input Parameters
#----------------------------
PARAM_NUMBER=$#
echo Input Parameters Number: $PARAM_NUMBER
if [[ $PARAM_NUMBER -ne 3 ]]; then
  echo "Usage $0 FOLDER_NAME FILE_NAME TABLE_NAME"
  exit 0
fi

FOLDER_NAME=$1
FILE_NAME=$2
TABLE_NAME=$3

BACKUP_FOLDER=/some/path/scripts/backup_ogg_param
RUN_DATE=`date +"%Y%m%d_%H%M%S"`

#----------------------------
#Replace String
#----------------------------
grep -v ${TABLE_NAME} ${FOLDER_NAME}/${FILE_NAME} > ${FOLDER_NAME}/${FILE_NAME}_bak
cp ${FOLDER_NAME}/${FILE_NAME} ${BACKUP_FOLDER}/${FILE_NAME}_${RUN_DATE}
mv ${FOLDER_NAME}/${FILE_NAME}_bak ${FOLDER_NAME}/${FILE_NAME}

echo 
echo File ${FOLDER_NAME}/${FILE_NAME} was copied to  ${BACKUP_FOLDER}/${FILE_NAME}_${RUN_DATE}
echo $0 Finished...........................
echo


Thursday, November 24, 2016

Code Example. crontab task to TRUNCATE table on ongoing basis

====================================
General
====================================
Due to a bad application design, debug tables are constantly being written into, without a process that is cleaning these tables.

This example, is a ctrontab task, that activates sh script, that is calling sql script, that generates actual truncate statements, and then executes them.

====================================
Code 
====================================

oracle@my_server:~/scripts>% crontab -l
15 4 * * * /software/oracle/oracle/scripts/delete_old_trace_files.sh
20 4 * * * /software/oracle/oracle/scripts/truncate_sga_w_events.sh

less /software/oracle/oracle/scripts/truncate_sga_w_events.sh
#!/bin/bash
HOME_DIR=/software/oracle/oracle/scripts

cd $HOME_DIR
. $HOME_DIR/.set_profile

export LOG_FILE=${HOME_DIR}/truncate_sga_w_events.log
touch $LOG_FILE
export RUN_TIME=`date "+%Y%m%d"_"%H%M%S"`

echo "----------------------------------------------" >> $LOG_FILE
echo "Starting Truncate at $RUN_TIME" >> $LOG_FILE
echo "----------------------------------------------" >> $LOG_FILE

sqlplus system/xen86pga@igt @truncate_sga_w_events.sql

less TRUNCATE_SGA_W_EVENTS_SQL.sql >> $LOG_FILE
echo "Done " >> $LOG_FILE
echo "----------------------------------------------" >> $LOG_FILE

rm TRUNCATE_SGA_W_EVENTS_SQL.sql


less /software/oracle/oracle/scripts/truncate_sga_w_events.sql
SET HEADING OFF
SET VERIFY OFF 
SET TERMOUT OFF
SET PAGESIZE 0
SET FEEDBACK OFF
spool TRUNCATE_SGA_W_EVENTS_SQL.sql

SELECT 'TRUNCATE TABLE '||OWNER||'.'||SEGMENT_NAME||';' 
  FROM DBA_SEGMENTS 
 WHERE segment_name = 'SGA_W_EVENTS' 
  HAVING ROUND(SUM(bytes)/1024/1024) > 100
  GROUP BY OWNER,SEGMENT_NAME;

spool off

@TRUNCATE_SGA_W_EVENTS_SQL.sql

exit;

TRUNCATE_SGA_W_EVENTS_SQL.sql
TRUNCATE TABLE OWNER1.SGA_W_EVENTS;
TRUNCATE TABLE OWNER2.SGA_W_EVENTS;
TRUNCATE TABLE OWNER3.SGA_W_EVENTS;
TRUNCATE TABLE OWNER4.SGA_W_EVENTS;

truncate_sga_w_events.log
----------------------------------------------
Starting Truncate at 20161124_122549
----------------------------------------------
TRUNCATE TABLE ALB_VODAF_SPARX.SGA_W_EVENTS;                                    
TRUNCATE TABLE MLT_VODAF_SPARX.SGA_W_EVENTS;                                    
TRUNCATE TABLE ROM_VODAF_SPARX.SGA_W_EVENTS;                                    
TRUNCATE TABLE ZAF_VODAC_SPARX.SGA_W_EVENTS;                                    
TRUNCATE TABLE NZL_VODAF_SPARX.SGA_W_EVENTS;                                    
TRUNCATE TABLE GRC_VODAF_SPARX.SGA_W_EVENTS;                                    
TRUNCATE TABLE GHA_VODAF_SPARX.SGA_W_EVENTS;                                    
Done 
----------------------------------------------
Starting Truncate at 20161124_123603
----------------------------------------------
Done 
----------------------------------------------