Pages

Showing posts with label jobs. Show all posts
Showing posts with label jobs. Show all posts

Monday, July 30, 2018

Code Example. PL/SQL Kill and re-create Stuck Jobs

=====================================
General
=====================================
For a job connecting to a remote DB via a db_link, sometimes due to network issues, the job is in "stuck" mode
as a workaround, there is a job running every 2 hours, and killing jobs that run over 1 hour.


=====================================
PL/SQL CODE

=====================================
CREATE OR REPLACE PACKAGE BODY ADMIN_UTIL IS

-----------------------------------------------
-- Known Exceptions
-----------------------------------------------
  EXP_ORA_SESS_MARK_FOR_KILL            EXCEPTION;
  PRAGMA EXCEPTION_INIT(EXP_ORA_SESS_MARK_FOR_KILL,-31);
  
  -------------------------------------------------
  PROCEDURE write_sga_w_log(p_procedure_name IN VARCHAR2,
                            p_data           IN VARCHAR2) IS
  PRAGMA AUTONOMOUS_TRANSACTION;
  BEGIN
    INSERT INTO SGA_W_LOG (module_name, msg_text, msg_date)
    VALUES (p_procedure_name, p_data, SYSDATE);
    COMMIT;
  EXCEPTION
    WHEN OTHERS THEN
       NULL;
  END write_sga_w_log;
  
  
  -------------------------------------------------
  PROCEDURE KILL_LONG_RUNNING_JOB  IS
    v_module_name      SGA_W_LOG.module_name%TYPE;
    v_msg_text         SGA_W_LOG.msg_text%TYPE;
    v_sql_cmd          VARCHAR2(1000);
    
    CURSOR long_running_jobs_cur IS
    SELECT 'ALTER SYSTEM KILL SESSION ' || SUBSTR(''''||V$SESSION.sid||','||V$SESSION.serial#||'''', 1,15)||' IMMEDIATE' AS kill_cmd,
           V$SESSION.sid, 
           V$SESSION.serial#,
           DBA_JOBS.what
      FROM DBA_JOBS ,
           DBA_JOBS_RUNNING,
           V$SESSION
     WHERE DBA_JOBS.this_date < SYSDATE - 1/24
       AND DBA_JOBS_RUNNING.job = DBA_JOBS.job
       AND DBA_JOBS_RUNNING.sid = V$SESSION.sid
       AND V$SESSION.status <> 'KILLED';       
           
  BEGIN  
    v_module_name := 'KILL_LONG_RUNNING_JOB';
    
    FOR long_running_jobs_rec IN long_running_jobs_cur LOOP
      BEGIN
        v_sql_cmd := long_running_jobs_rec.kill_cmd;
        v_msg_text := 'Killing Stuck Job :'||long_running_jobs_rec.what;
        WRITE_SGA_W_LOG(v_module_name,v_msg_text);        
        
        v_msg_text := 'Runing: '||v_sql_cmd;
        WRITE_SGA_W_LOG(v_module_name,v_msg_text);
        
        EXECUTE IMMEDIATE v_sql_cmd;
    
        
      EXCEPTION
        WHEN EXP_ORA_SESS_MARK_FOR_KILL THEN
          v_msg_text := 'Done! Session Was Marked For Kill';
          WRITE_SGA_W_LOG(v_module_name,v_msg_text);          
        
        WHEN OTHERS THEN
          v_msg_text := 'Unexpected Error: '||SQLERRM;
          WRITE_SGA_W_LOG(v_module_name,v_msg_text);    
      END;
    END LOOP;    
  EXCEPTION
    WHEN OTHERS THEN
      v_msg_text := 'Unexpected Error: '||SQLERRM;
      WRITE_SGA_W_LOG(v_module_name,v_msg_text);    
  END KILL_LONG_RUNNING_JOB;    

  -------------------------------------------------
  PROCEDURE RECREATE_JOB  IS
    v_module_name      SGA_W_LOG.module_name%TYPE;
    v_msg_text         SGA_W_LOG.msg_text%TYPE;
    v_sparx_what       VARCHAR2(1000);
    CURSOR get_job_list_cur(cp_job_what IN VARCHAR2) IS
    SELECT *
      FROM DBA_JOBS 
     WHERE UPPER(WHAT) = cp_job_what;
           
  BEGIN  
    v_module_name := 'RECREATE_JOB';
    v_sparx_what := UPPER('EXT_SPARX.schedule;');
        
    
    FOR get_job_list_rec IN get_job_list_cur(v_sparx_what) LOOP

        v_msg_text := 'Deleting Job :'||get_job_list_rec.job;
        WRITE_SGA_W_LOG(v_module_name,v_msg_text);        
        
        BEGIN
          DBMS_JOB.remove(get_job_list_rec.job);
          COMMIT;
        END;
        
        v_msg_text := 'Creating Job :'||get_job_list_rec.job;
        WRITE_SGA_W_LOG(v_module_name,v_msg_text);        
        
        BEGIN
          DBMS_JOB.ISUBMIT
          ( job       => get_job_list_rec.job 
           ,what      => get_job_list_rec.what
           ,next_date => get_job_list_rec.next_date
           ,interval  => get_job_list_rec.interval
          );
          COMMIT;
        END;
    END LOOP;    
  EXCEPTION
    WHEN OTHERS THEN
      v_msg_text := 'Unexpected Error: '||SQLERRM;
      WRITE_SGA_W_LOG(v_module_name,v_msg_text);    
  END RECREATE_JOB;    
  -------------------------------------------------
END;

=====================================
Job

=====================================
DECLARE
   v_job_number NUMBER(10);
BEGIN
 DBMS_JOB.SUBMIT (JOB => v_job_number, 
                  WHAT => 'ADMIN_UTIL.KILL_LONG_RUNNING_JOB;', 
                  NEXT_DATE => TRUNC(SYSDATE,'HH24')+((FLOOR(TO_NUMBER(TO_CHAR(SYSDATE,'MI'))/1)+121)*1)/(1440), 
                  INTERVAL => 'TRUNC(SYSDATE,''HH24'')+((FLOOR(TO_NUMBER(TO_CHAR(SYSDATE,''MI''))/1)+121)*1)/(1440)'
 );
 COMMIT;
END;
/

Monday, February 5, 2018

Killing Zombie Jobs in DBA_JOBS RUNNING

================================
General
================================
For unknown reason, jobs stopped from being run on a scheduler from USER_JOBS.
When checking for broken - it is in 'N' value
When checking application log, there is no error, the job seems to have been stuck in the middle of execution, and since the a new job was not submitted.

================================
What to Check
================================
DBA_JOBS
DBA_JOBS_RUNNING
ora_j0xx_oraigt Processes on Linux

Step1.
Checking DBA_JOBS_RUNNING - no job is related to an entry in V$SESSION.

job value is allways NULL.
Seems that these are "Zombie" jobs.

SELECT 'kill -9 '||proc.SPID AS "Linux Kill", 
       jobs.JOB as job,
       sess.sid||'.'||sess.serial# as sid_pid,
       sess.username 
 FROM V$SESSION sess, 
      V$PROCESS proc, 
      DBA_JOBS_RUNNING jobs
WHERE proc.addr = sess.paddr
  AND sess.type <> 'BACKGRGROUND'
  AND sess.logon_time < SYSDATE -1
  AND sess.username = 'MYS_CELCO_MOCOQ'
  AND sess.sid = jobs.sid(+)

Linux Kill          job sid_pid         username
------------- --------- --------------- ---------------
kill -9 22776 979.61739 MYS_CELCO_MOCOQ
kill -9 22765 790.2795 MYS_CELCO_MOCOQ
kill -9 4104 54.20611 MYS_CELCO_MOCOQ
kill -9 4096 30.27007 MYS_CELCO_MOCOQ
kill -9 4110 228.21157 MYS_CELCO_MOCOQ
kill -9 20859 810.58589 MYS_CELCO_MOCOQ
kill -9 4106 991.32265 MYS_CELCO_MOCOQ
kill -9 4118 98.38363 MYS_CELCO_MOCOQ
kill -9 4108 843.2089 MYS_CELCO_MOCOQ
kill -9 4092 106.45951 MYS_CELCO_MOCOQ
kill -9 1161 850.55871 MYS_CELCO_MOCOQ
kill -9 1169 212.38989 MYS_CELCO_MOCOQ
kill -9 30451 828.6607 MYS_CELCO_MOCOQ
kill -9 4100 178.61219 MYS_CELCO_MOCOQ
kill -9 30790 215.36845 MYS_CELCO_MOCOQ
kill -9 4090 195.21231 MYS_CELCO_MOCOQ
kill -9 30801 838.15399 MYS_CELCO_MOCOQ
kill -9 4098 959.63365 MYS_CELCO_MOCOQ
kill -9 30796 899.31015 MYS_CELCO_MOCOQ
kill -9 4114 817.4443 MYS_CELCO_MOCOQ
kill -9 4088 771.53083 MYS_CELCO_MOCOQ
kill -9 1165 148.39835 MYS_CELCO_MOCOQ
kill -9 22771 849.23091 MYS_CELCO_MOCOQ
kill -9 6087 138.29993 MYS_CELCO_MOCOQ
kill -9 30793 230.13119 MYS_CELCO_MOCOQ
kill -9 30798 199.4211 MYS_CELCO_MOCOQ
kill -9 4094 804.13365 MYS_CELCO_MOCOQ
kill -9 4112 963.23979 MYS_CELCO_MOCOQ
kill -9 4085 126.8799 MYS_CELCO_MOCOQ
kill -9 1145 977.23831 MYS_CELCO_MOCOQ


Step 2.
Checking How long ago these jobs were run.
The check was done at 05/02/2018

SELECT DBA_JOBS_RUNNING.sid,
       DBA_JOBS_RUNNING.job,
       DBA_JOBS_RUNNING.last_date,
       ROUND(SYSDATE-DBA_JOBS_RUNNING.last_date) as days_since_last_run
  FROM DBA_JOBS_RUNNING, DBA_JOBS
WHERE dba_jobs.job = DBA_JOBS_RUNNING.JOB
ORDER BY DBA_JOBS.SCHEMA_USER;

sid     job     last_date           days_since_last_run
------- ------- ------------------- -------------------
890 281 28/10/2017 00:00:13             101
971 283 27/10/2017 23:50:13             101            
198 284 27/10/2017 21:00:48                        101
787 282 28/10/2017 00:00:38             101
186 1382
953 1261
224 41 28/10/2017 00:00:23             101
226 43 27/10/2017 23:50:06             101
100 42 28/10/2017 00:00:43             101
124 44 27/10/2017 21:01:22             101
188 1361
941 123 27/10/2017 23:00:18             101
888 121 28/10/2017 00:01:48             101
39 243 27/10/2017 00:00:04             102
135 2585 27/10/2017 23:01:12             101
944 461 28/10/2017 00:01:54             101
924 1044
806 1043
837 1042
882 1041
207 1221
803 1241
156 85 28/10/2017 00:00:05             101
909 86 28/10/2017 00:00:31             101
812 88 27/10/2017 21:03:15             101
153 64 27/10/2017 21:03:01             101
945 62 28/10/2017 00:04:48             101
...
...

SELECT COUNT(*) FROM DBA_JOBS_RUNNING;
18

Step 3.
On Linux,
There are many ora_j0xx_igt processes.
oracle@my_server:~/scripts>% ps -ef | grep j0 | wc -l
36


ps -ef | grep j0
oracle   12331     1  0 12:46 ?        00:00:00 ora_j030_igt
oracle   12511     1  0 12:46 ?        00:00:00 ora_j031_igt
oracle   12519     1  0 12:46 ?        00:00:00 ora_j032_igt
oracle   12540     1  0 12:46 ?        00:00:00 ora_j033_igt
oracle   12562     1  0 12:46 ?        00:00:00 ora_j034_igt
oracle   12590     1  0 12:46 ?        00:00:00 ora_j039_igt
oracle   12672     1  0 12:46 ?        00:00:00 ora_j040_igt
oracle   12676     1  0 12:46 ?        00:00:00 ora_j042_igt
oracle   16114     1  0 12:49 ?        00:00:00 ora_j005_igt
oracle   18766     1  0 12:25 ?        00:00:00 ora_j013_igt
oracle   19570     1  0 12:25 ?        00:00:00 ora_j011_igt
oracle   19572     1  0 12:25 ?        00:00:00 ora_j014_igt
oracle   20697     1  0 12:26 ?        00:00:00 ora_j016_igt
oracle   20700     1  0 12:26 ?        00:00:00 ora_j022_igt
oracle   20702     1  0 12:26 ?        00:00:00 ora_j023_igt
oracle   20705     1  0 12:26 ?        00:00:00 ora_j026_igt
oracle   23072     1  0 12:55 ?        00:00:00 ora_j007_igt
oracle   27217     1  0 12:59 ?        00:00:00 ora_j003_igt
oracle   27938     1  0  2017 ?        00:00:00 ora_j010_igt
oracle   29689     1  0 12:07 ?        00:00:00 ora_j024_igt
oracle   29691     1  0 12:07 ?        00:00:00 ora_j025_igt
oracle   29829     1  0  2017 ?        00:03:10 ora_j015_igt
oracle   32684     1  0 13:03 ?        00:00:00 ora_j021_igt
oracle   32699     1  0 13:03 ?        00:00:00 ora_j017_igt


ps -ef | grep j0 | awk '{print "kill -9 "$2}'
Execute the generated commands:

kill -9 5285
kill -9 5291
kill -9 5469
kill -9 9288
kill -9 9290
kill -9 9294
kill -9 12590
kill -9 14892
kill -9 16114
kill -9 18766
etc...

SELECT DBA_JOBS_RUNNING.sid,
       DBA_JOBS_RUNNING.job,
       DBA_JOBS_RUNNING.last_date,
       ROUND(SYSDATE-DBA_JOBS_RUNNING.last_date) as days_since_last_run
  FROM DBA_JOBS_RUNNING, DBA_JOBS
WHERE dba_jobs.job = DBA_JOBS_RUNNING.JOB
ORDER BY DBA_JOBS.SCHEMA_USER


Step 4.
Most of the jobs should have been cleared by kill -9 command.
This might take some time to reflect in the Database.

Step 5.
If there are still jobs remaining: via DBMS_JOBS remove and recreate it.

Sunday, April 19, 2015

ksh and perl by Example: crontab task to find and kill zombie Oracle Jobs.

General.
Once a day, at 12:15 execute a script that would identify and kill Oracle Job processes, that are running longer than 24 hours.
This script is launched on Oracle server, as oracle user.

steps
A. crontab task - that runs ksh script once a day, at 12:15.
B. ksh script that launches perl script
C. perl script that connects to DB to find zombie Linux processes that were launched by Oracle job, and then kills these tasks.

crontab -l
15 12 * * * /backup/ora_exp/kill_zombie_jobs.sh

kill_long_zombie_jobs.sh
#!/bin/bash
. /etc/profile
. /etc/sh/orash/oracle_login.sh igt

perl /backup/ora_exp/kill_zombie_jobs.pl


kill_long_zombie_jobs.pl
#! /usr/bin/perl

use DBI;

use Time::gmtime;
use File::Copy;

####################################################################

#####  Subroutine to get the current date  #########################
####################################################################

sub getDate
{
  use Time::gmtime;
  $tm=gmtime;
  ($second,$minute,$hour,$day,$month,$year) = (gmtime) [0,1,2,3,4,5];
  my $locDate=sprintf("%04d%02d%02d_%02d%02d%02d",$tm->year+1900,($tm->mon)+1,$tm->mday,$tm->hour,$tm->min,$tm->sec );
  return $locDate;

}

########################

# main start here ######
########################
my $RetCode;
my $myDate=getDate;
my $logLocation="/backup/ora_exp/Logs/kill_zombie_jobs/";
my $logFile=$logLocation."kill_zombie_log_".$myDate.".log";

my $sql;
my $db="cgw_new";
my $db_driver="dbi:Oracle:".$db;
my $dbh=DBI->connect($db_driver,'collector','cgw154igt',{RaiseError =>1,AutoCommit=>0})|| die "$DBI::errstr";

print $DBI::errstr;

print "Log file:".$logFile."\n";

my $sql="SELECT 'kill -9 '||PROCESSES.spid AS LINUX_KILL, ";
$sql=$sql."PROCESSES.spid, PROCESSES.program, ";
$sql=$sql."SESSIONS.username, SESSIONS.sql_id, SESSIONS.logon_time, SESSIONS.event ";
$sql=$sql."FROM v\$process PROCESSES, V\$SESSION SESSIONS ";
$sql=$sql."WHERE PROCESSES.program like '%J0%' ";
$sql=$sql."  AND PROCESSES.addr=SESSIONS.paddr ";
$sql=$sql."  AND SESSIONS.schemaname <> 'SYS' ";
$sql=$sql."  AND SESSIONS.type = 'USER' ";
$sql=$sql."  AND PROCESSES.background IS NULL ";
$sql=$sql."  AND SESSIONS.sid NOT IN (SELECT sid FROM DBA_JOBS_RUNNING) ";
$sql=$sql."  AND logon_time < SYSDATE-1/4";

$sql=$sql." AND logon_time < SYSDATE-1";

open (MyLog,">>".$logFile);
print MyLog "----I---- Startting kill zombie process on ".$myDate."\n";
print MyLog "Running SQL: "."\n".$sql."\n";
print MyLog "\n\n";
print MyLog "These are the details of the job that were killed : \n";

print MyLog ".........................."."\n";

my $FNsth=$dbh->prepare($sql);
$FNsth->execute();
my @row;
my $kill_cmd;
while (@row=$FNsth->fetchrow())
{
  open (MyLog,">>".$logFile);
  print MyLog "V$SEESION.sid: ".$row[1]." \n";
  print MyLog "PROCESSES.program: ".$row[2]." \n";
  print MyLog "SESSIONS.username: ".$row[3]." \n";
  print MyLog "SESSIONS.sql_id: ".$row[4]." \n";
  print MyLog "SESSIONS.logon_time: ".$row[5]." \n";
  print MyLog "SESSIONS.event: ".$row[6]." \n";
  print MyLog ".........................."."\n";
  $kill_cmd = $row[0];
  print MyLog "Running command ".$kill_cmd."..........";
  system($kill_cmd);
  print MyLog "Done"."\n";
}
$FNsth->finish();

$dbh->disconnect;

print MyLog "\n=========================\n";
print MyLog "Script Finished Successfuly";
print MyLog "\n=========================\n";


close MyLog;

=================================
SQL to identify zombie jobs
=================================
SELECT 'kill -9 '||PROCESSES.spid AS LINUX_KILL, 
       PROCESSES.spid, 
       PROCESSES.program, 
       SESSIONS.username, 
       SESSIONS.sql_id, 
       SESSIONS.logon_time, 
       SESSIONS.event 
 FROM  V$PROCESS PROCESSES, 
       V$SESSION SESSIONS, 
       DBA_JOBS_RUNNING RUNNING_JOBS 
 WHERE PROCESSES.program like '%J0%' 
   AND PROCESSES.addr=SESSIONS.paddr 
   AND SESSIONS.sid = RUNNING_JOBS.sid(+) 
   AND SESSIONS.schemaname <> 'SYS' 
   AND SESSIONS.type = 'USER' 
   AND RUNNING_JOBS.job IS NULL
-- AND SESSIONS.logon_time < SYSDATE-1;


=======================================
profile files, not really important for this example...
=======================================
. /etc/profile

# /etc/profile

# System wide environment and startup programs, for login setup
# Functions and aliases go in /etc/bashrc

pathmunge () {
        if ! echo $PATH | /bin/egrep -q "(^|:)$1($|:)" ; then
           if [ "$2" = "after" ] ; then
              PATH=$PATH:$1
           else
              PATH=$1:$PATH
           fi
        fi
}

# Path manipulation
pathmunge /sbin
pathmunge /usr/sbin
pathmunge /usr/local/sbin
pathmunge /usr/local/bin
pathmunge /usr/X11R6/bin after


# No core files by default
ulimit -S -c 0 > /dev/null 2>&1

USER="`id -un`"
LOGNAME=$USER
MAIL="/var/spool/mail/$USER"

HOSTNAME=`/bin/hostname`
HISTSIZE=1000

if [ -z "$INPUTRC" -a ! -f "$HOME/.inputrc" ]; then
    INPUTRC=/etc/inputrc
fi

export PATH USER LOGNAME MAIL HOSTNAME HISTSIZE INPUTRC

for i in /etc/profile.d/*.sh ; do
    if [ -r "$i" ]; then
        . $i
    fi
done

unset i

. /etc/sh/orash/oracle_login.sh orainst
#!/bin/bash
#
# NAME:
#      oracle_login.sh
# !!!
#
# ABSTRACT:
#           Defining oracle's enironment variables. - Unix version
#
#
# ARGUMENTS:
#           [p1] - oracle sid
#
#       Note - 1) the parameter can be ommitted
#
#
# SPECIAL FILE:
#
# ENVIRONMENT VAR:
#     PROJECT_SID -
#        should be defined in the following way:
#        globaly        - the node run only one oracle instance
#        group login    - there is more than one instance on the current node
#        as a parameter - there is more than one instance per project,
#                         or for system users.
#
# SPECIAL CCC MACROS:
#
# INPUT FORMAT:
#
# OUTPUT FORMAT:
#
# RETURN/EXIT STATUS:
#            1) PROJECT_SID is not defined
#            2) the database PROJECT_SID does not exist
#            3) PROJECT_SID missing from $ORASH/sid.hosts
#            4) ORA_VER missing from $ORASH/sid.hosts
#            5) charcter missing for current sid from $ORASH/charset.dat
#
# LIMITATIONS:
#
# MODIFICATION HISTORY:
# +-------------+-------------------+-----------------------------------------+
# | Date        |     Name          |  Description                            |
# +-------------+-------------------+-----------------------------------------+
# +             +                   +                                         +
# +-------------+-------------------+-----------------------------------------+
if [ "$ORASH" = "" ]; then
   ORASH=$SH_ETC/etc/orash
fi

FACILITY="oracle_login:"
UNAME=`uname | cut -d_ -f1`   #=> Linux

  if [ "$#" -ge 1 ]; then
      PROJECT_SID=$1
   else
      project_sid_def=`set | grep "^PROJECT_SID=" |wc -l`
      if [ "$project_sid_def" -eq 0 ]; then
         echo "<oracle_login> ERROR>> PROJECT_SID is not defined"
         return 1
      fi
   fi

   ORACLE_BASE=/software/oracle
   project_sid_def=`set | grep "^PROJECT_SID=" |wc -l`
   if [ "$project_sid_def" -eq 0 ]; then
         echo "<oracle_login> ERROR>> PROJECT_SID is not defined"
         return 1
   fi

   ORACLE_HOME=`grep ^"$PROJECT_SID": /var/opt/oracle/oratab | awk '{if (NR==1) print $0}' | cut -d: -f2`
   if [ "$ORACLE_HOME" = "" ]; then
      echo "<oracle_login> ERROR>> the database $PROJECT_SID does not exist in /var/opt/oracle/oratab or /etc/oratab"
      return 2
   fi

   ORA_VER=`grep "$PROJECT_SID" $ORASH/sid.hosts | awk '{if (NR==1) print $0}' |cut -d: -f3`
   if [ "$ORA_VER" = "" ]; then
      echo "$FACILITY unable to verify ORA_VER for $PROJECT_SID using $ORASH/sid.hosts"
      return 4
   fi
   DEFAULT_SID=$PROJECT_SID

   ORACLE_SID=$DEFAULT_SID
   TNS_ADMIN=${ORACLE_HOME}/network/admin

#----------------------------------------------------------------#
# the hostname of the DEFAULT_SID appears first in file sid.hosts
#----------------------------------------------------------------#
   HOST=`grep ^"$PROJECT_SID": $ORASH/sid.hosts | awk '{if (NR==1) print $0}' | cut -d: -f2`
   if [ "$HOST" = "" ]; then
      echo "<oracle_login> ERROR>> the database $PROJECT_SID does not exist in $ORASH/sid.hosts"
      return 3
   fi

#-------------------------------------#
#  TWO_TASK is used for a unix clients
#-------------------------------------#
# Disable TWO_TASK feature eliminating the need to change hostname, after disk replication
#   NODE=`hostname`
#   if [ "$NODE" != "$HOST" ]; then
#      TWO_TASK=${DEFAULT_SID}_remote
#   fi

#-------------------------------------#
# setting up NLS_LANG
#-------------------------------------#
   CHARSET=`grep ^"$PROJECT_SID": $ORASH/charset.dat | awk '{if (NR==1) print $0}' | cut -d: -f2`
   if [ "$CHARSET" = "" ]; then
      echo "<oracle_login> ERROR>> the database $PROJECT_SID does not exist in $
ORASH/charset.dat"
      return 5
   fi

   NLS_DATE_FORMAT="DD-MON-RR"
   ORACLE_LPPROG=lpr
   ORACLE_LPARGS="-P la424"
   NLS_LANG="AMERICAN_AMERICA."${CHARSET}
   ORA_NLS32=$ORACLE_HOME/ocommon/nls/admin/data # for oracle
   ORA_NLS33=$ORACLE_HOME/ocommon/nls/admin/data # for oracle8

   cleaned_path=$PATH
   for ora_path in `grep -v "^#" /var/opt/oracle/oratab | cut -d: -f2 | uniq` ; do
      cleaned_path=`echo -e "${cleaned_path}\c" | tr ':' '\n' | grep -v "$ora_path" | tr '\n' ':'`
   done
   if [ $UNAME = 'SunOS' ] ; then
      cleaned_path=`echo -e "${cleaned_path}\c" | tr ':' '\n' | grep -v "$ora_path" | tr '\n' ':'`
   done
   if [ $UNAME = 'SunOS' ] ; then
     PATH=$cleaned_path:$ORACLE_HOME/bin:/usr/ccs/bin:/usr/openwin/bin
   else
     PATH=$cleaned_path$ORACLE_HOME/bin
   fi

   shlib_def=`set | grep "^LD_LIBRARY_PATH" |wc -l`
   if [ "$shlib_def" -eq 0 ]; then
      LD_LIBRARY_PATH=$ORACLE_HOME/lib:/usr/lib
   else
      cleaned_shlib_path=$LD_LIBRARY_PATH
      for ora_path in `grep -v "^#" /var/opt/oracle/oratab | cut -d: -f2 | uniq` ; do
         cleaned_shlib_path=`echo -e "${cleaned_shlib_path}\c" | tr ':' '\n' | grep -v "$ora_path" | tr '\n' ':'`
      done

      if [ $UNAME = 'SunOS' ] ; then
         LD_LIBRARY_PATH=$cleaned_shlib_path:$ORACLE_HOME/lib:/usr/lib
      else
         LD_LIBRARY_PATH=$cleaned_shlib_path$ORACLE_HOME/lib:/usr/lib
      fi
   fi

   export PROJECT_SID
   export ORACLE_TERM
   export ORACLE_HOME
   export ORACLE_BASE
   export ORACLE_SID
   export TNS_ADMIN
   export ORACLE_SERVER
   export PATH
   export LD_LIBRARY_PATH
   export NLS_LANG
   export ORA_NLS33
   unset ORA_NLS32
   ORACLE_ENV_DEFINED="yes"
   export ORACLE_ENV_DEFINED



   export ORA_VER