Pages

Monday, August 21, 2017

Oracle Clean up. Delete old Oracle trace files. Delete old dmp files.

=========================
General
=========================
Oracle creates trc files that are not being cleaned up from server.
Here is an example of delete unwanted files, using crontab task.


=========================
Evidences
=========================

crontab

1 6 * * * /software/oracle/oracle/scripts/scripts/delete_trace_files.sh

/software/oracle/oracle/scripts/delete_trace_files.sh

#!/bin/bash

ORACLE_SID=igt

#===============
#Listener Logs
#===============
SERVER_NAME=`hostname`
LISTENER_NAME=lsnr_igt
LISTENER_HOME=/software/oracle/diag/tnslsnr
LISTENER_PATH=${LISTENER_HOME}/${SERVER_NAME}/${LISTENER_NAME}/trace

mv ${LISTENER_PATH}/${LISTENER_NAME}.log_9  ${LISTENER_PATH}/${LISTENER_NAME}.log_10
mv ${LISTENER_PATH}/${LISTENER_NAME}.log_8  ${LISTENER_PATH}/${LISTENER_NAME}.log_9
mv ${LISTENER_PATH}/${LISTENER_NAME}.log_7  ${LISTENER_PATH}/${LISTENER_NAME}.log_8
mv ${LISTENER_PATH}/${LISTENER_NAME}.log_6  ${LISTENER_PATH}/${LISTENER_NAME}.log_7
mv ${LISTENER_PATH}/${LISTENER_NAME}.log_5  ${LISTENER_PATH}/${LISTENER_NAME}.log_6
mv ${LISTENER_PATH}/${LISTENER_NAME}.log_4  ${LISTENER_PATH}/${LISTENER_NAME}.log_5
mv ${LISTENER_PATH}/${LISTENER_NAME}.log_3  ${LISTENER_PATH}/${LISTENER_NAME}.log_4
mv ${LISTENER_PATH}/${LISTENER_NAME}.log_2  ${LISTENER_PATH}/${LISTENER_NAME}.log_3
mv ${LISTENER_PATH}/${LISTENER_NAME}.log_1  ${LISTENER_PATH}/${LISTENER_NAME}.log_2
mv ${LISTENER_PATH}/${LISTENER_NAME}.log  ${LISTENER_PATH}/${LISTENER_NAME}.log_1

DAYS_TO_KEEP=14
LISTENER_PATH=${LISTENER_HOME}/${SERVER_NAME}/${LISTENER_NAME}/alert/
find ${LISTENER_PATH} -type f -name "log*.xml" -mtime +${DAYS_TO_KEEP} -exec rm {} \;

#=======================
#delete old trace files
#=======================
LOG_DIR=/software/oracle/oracle/scripts/logs
if [[ ! -d $LOG_DIR ]]; then
  mkdir $LOG_DIR
fi

LOG_FILE=${LOG_DIR}/delete_trace.log
touch $LOG_FILE

ORA_INST=$ORACLE_SID
DAYS_TO_KEEP=10
RUN_DATE=`date +"%Y%m%d_%H%M%S"`
ORA_TRACE_HOME=/software/oracle/diag/rdbms
TRACE_PATH=${ORA_TRACE_HOME}/${ORA_INST}/${ORA_INST}/trace



echo "========================================" >> $LOG_FILE
echo "Start Deleting Files at $RUN_DATE       " >> $LOG_FILE
echo "========================================" >> $LOG_FILE


echo Starting Delete trc files older than $DAYS_TO_KEEP days: >> $LOG_FILE
find ${TRACE_PATH} -type f -name "*.trc" -mtime +${DAYS_TO_KEEP} -exec rm {} \;
echo " " >> $LOG_FILE

echo Starting Delete trm files older than $DAYS_TO_KEEP days: >> $LOG_FILE
find ${TRACE_PATH} -type f -name "*.trm" -mtime +${DAYS_TO_KEEP} -exec rm {} \;
echo " " >> $LOG_FILE


echo Starting Delete files bigger than 1Gb: >> $LOG_FILE
find ${TRACE_PATH} -type f -name "*.trc" -size +1G -exec rm {} \;
echo " " >> $LOG_FILE

echo "Starting Delete files with text kcbzib: dump suspect buffer:"  >> $LOG_FILE
find ${TRACE_PATH} -type f -name "*.trc" | xargs grep -l "kcbzib: dump suspect buffer"  >> $LOG_FILE
find ${TRACE_PATH} -type f -name "*.trc" | xargs grep -l "kcbzib: dump suspect buffer" | xargs rm -f
echo " " >> $LOG_FILE

echo "Starting Delete files with text kkpamMapRow under incident folder:" >>  $LOG_FILE
find ${ORA_TRACE_HOME}/${ORA_INST}/${ORA_INST}/incident -type f -name "*.trc" | xargs grep -l "kkpamMapRow" >>  $LOG_FILE
find ${ORA_TRACE_HOME}/${ORA_INST}/${ORA_INST}/incident -type f -name "*.trc" | xargs grep -l "kkpamMapRow" | xargs rm -f
echo " " >> $LOG_FILE

echo "Starting Delete cdump folders under trace folder:" >>  $LOG_FILE
find ${ORA_TRACE_HOME}/${ORA_INST}/${ORA_INST}/trace -type d -name "cdmp*" >>  $LOG_FILE 
find ${ORA_TRACE_HOME}/${ORA_INST}/${ORA_INST}/trace -type d -name "cdmp*" | xargs rm -rf
echo " " >> $LOG_FILE

echo "========================================" >> $LOG_FILE
echo "Delete and Archive Files                " >> $LOG_FILE
echo "========================================" >> $LOG_FILE

SEARCH_STING="DUMP REDO"

TRACE_FILES_LIST="$(find ${TRACE_PATH} -type f -name "*.trc" | xargs grep -l "${SEARCH_STING}") " 

for f in $TRACE_FILES_LIST
do
  echo "handling trace file $f" >> $LOG_FILE
  NEW_TRACE_NAME=/software/oracle/oracle/scripts/`basename $f`
  head -1000 $f > ${NEW_TRACE_NAME}
  tail -1000 $f >> ${NEW_TRACE_NAME}
  rm -f $f
done



echo " " >> $LOG_FILE
RUN_DATE=`date +"%Y%m%d_%H%M%S"`

echo "========================================" >> $LOG_FILE
echo "Finished Deleting Files at $RUN_DATE    " >> $LOG_FILE
echo "========================================" >> $LOG_FILE

exit 0



as shdaemon

1 6 * * * /home/shdaemon/scripts/move_old_log_files.sh

/home/shdaemon/scripts/move_old_log_files.sh

#!/bin/bash

#=============================
#move logs to old_logs folder
#=============================
FOR_BACKUP_HOME=/backup/ora_exp/for_backup
LOG_HOME=/backup/ora_exp/for_backup/old_log

for f in ${FOR_BACKUP_HOME}/export_igt*log 
do
  mv $f ${LOG_HOME}/
done

#=====================
#delete old dmp files
#=====================
BACKUP_HOME=/backup/ora_exp
DAYS_TO_KEEP=4

find ${BACKUP_HOME} -type f -name "export*.dmp" -mtime +${DAYS_TO_KEEP} -exec rm {} \;

exit 0


=======================
delete_old_exp_files.sh
=======================
1 6 * * * /home/shdaemon/scripts/delete_old_exp_files.sh

/home/shdaemon/scripts/delete_old_exp_files.sh

#!/bin/bash

BACKUP_HOME=/backup/ora_exp
FOR_BACKUP_HOME=/backup/ora_exp/for_backup
LOG_HOME=/backup/ora_exp/for_backup/old_log
DAYS_TO_KEEP=4

#=============================
#move logs to old_logs folder
#=============================
for f in ${FOR_BACKUP_HOME}/export_igt*log do
  mv -f ${FOR_BACKUP_HOME}/export_igt*log ${LOG_HOME}/
done
exit 0

#=====================
#delete old dmp files
#=====================
find ${BACKUP_HOME} -type f -name "export*.dmp" -mtime +${DAYS_TO_KEEP} -exec rm {} \;
exit 0

=====================
move_old_log_files.sh
=====================
1 6 * * * /home/shdaemon/scripts/move_old_log_files.sh

/home/shdaemon/scripts/move_old_log_files.sh

#!/bin/bash

BACKUP_HOME=/backup/ora_exp
FOR_BACKUP_HOME=/backup/ora_exp/for_backup
LOG_HOME=/backup/ora_exp/for_backup/old_log
DAYS_TO_KEEP=4

====================
delete_old_exp_files.sh
====================
1 6 * * * /home/shdaemon/scripts/delete_old_exp_files.sh

/home/shdaemon/scripts/delete_old_exp_files.sh

#!/bin/bash

BACKUP_HOME=/backup/ora_exp
FOR_BACKUP_HOME=/backup/ora_exp/for_backup
LOG_HOME=/backup/ora_exp/for_backup/old_log
DAYS_TO_KEEP=4

#================================================
#delete old dmp files
#================================================
find ${BACKUP_HOME} -type f -name "export*.dmp" -mtime +${DAYS_TO_KEEP} -exec rm {} \;

exit 0



====================
SQL Cleanup
====================
0 23 * * * /software/oracle/oracle/scripts/sql_cleanup.sh

sql_cleanup.sh
#!/bin/bash
. /etc/sh/orash/oracle_login.sh igt

WORK_DIR=/software/oracle/oracle/scripts
LOG_FILE=${WORK_DIR}/sql_cleanup.log
RUN_DATE=`date +"%Y%m%d_%H%M%S"`
echo "Logfile : $LOG_FILE"

echo "" >> ${LOG_FILE}
echo "Starting Truncate at $RUN_DATE" >> ${LOG_FILE}


cd /software/oracle/oracle/scripts
sqlplus system/XXXXXX@igt @sql_cleanup.sql | tee $LOGFILE

echo "Finished Truncate at $RUN_DATE" >> ${LOG_FILE}

sql_cleanup.sql
TRUNCATE TABLE MY_USER.MY_TABLE;
EXIT;

Sunday, August 20, 2017

Many trace files with ORA-00600 and error message Action: 'kpuActionDefault'

===========================
General
===========================
There are many *.trc files and incident/incident_<pid> files generated.
When checking alert.log, there are many error messages with error ORA-00600:

===========================
Evidences
===========================
alert.log
Sun Aug 20 07:14:18 2017
ORA-00600: internal error code, arguments: [kkpamRGet], [PTI 8.1], [0], [], [], [], [], [], [], [], [], []
Incident details in: /software/oracle/diag/rdbms/igt/igt/incident/incdir_246165/igt_ora_2263_i246165.trc
Sun Aug 20 07:14:18 2017
Errors in file /software/oracle/diag/rdbms/igt/igt/trace/igt_ora_5575.trc  (incident=246525):
ORA-00600: internal error code, arguments: [kkpamDGSPam2], [0], [0], [], [], [], [], [], [], [], [], []
Incident details in: /software/oracle/diag/rdbms/igt/igt/incident/incdir_246525/igt_ora_5575_i246525.trc

trace files
/software/oracle/diag/rdbms/igt/igt/trace/igt_ora_31903.trc

ORACLE_HOME = /software/oracle/111
System name:    Linux
Node name:      deu-vod-2-dbu-1
Release:        2.6.18-371.3.1.el5
Version:        #1 SMP Mon Nov 11 03:23:58 EST 2013
Machine:        x86_64
Instance name: igt
Redo thread mounted by this instance: 1
Oracle process number: 455
Unix process pid: 31903, image: oracle@deu-vod-2-dbu-1


*** 2017-08-20 12:59:18.420
*** SESSION ID:(2544.29033) 2017-08-20 12:59:18.420
*** CLIENT ID:() 2017-08-20 12:59:18.420
*** SERVICE NAME:(SYS$USERS) 2017-08-20 12:59:18.420
*** MODULE NAME:(JDBC Thin Client) 2017-08-20 12:59:18.420
*** ACTION NAME:() 2017-08-20 12:59:18.420
 
DDE: Problem Key 'ORA 600 [kkpamRGet]' was flood controlled (0x4) (incident: 246112)
ORA-00600: internal error code, arguments: [kkpamRGet], [PTI 8.1], [0], [], [], [], [], [], [], [], [], []


incident file /software/oracle/diag/rdbms/igt/igt/incident/incdir_243949/igt_ora_13764_i243949.trc
Oracle Database 11g Enterprise Edition Release 11.1.0.7.0 - 64bit Production
With the Partitioning, Oracle Label Security, OLAP, Data Mining,
Oracle Database Vault and Real Application Testing options
ORACLE_HOME = /software/oracle/111
System name:    Linux
Node name:      deu-vod-2-dbu-1
Release:        2.6.18-371.3.1.el5
Version:        #1 SMP Mon Nov 11 03:23:58 EST 2013
Machine:        x86_64
Instance name: igt
Redo thread mounted by this instance: 1
Oracle process number: 185
Unix process pid: 13764, image: oracle@deu-vod-2-dbu-1


*** 2017-08-04 08:59:25.201
*** SESSION ID:(3259.3287) 2017-08-04 08:59:25.201
*** CLIENT ID:() 2017-08-04 08:59:25.201
*** SERVICE NAME:(SYS$USERS) 2017-08-04 08:59:25.201
*** MODULE NAME:(JDBC Thin Client) 2017-08-04 08:59:25.201
*** ACTION NAME:() 2017-08-04 08:59:25.201
 
Dump continued from file: /software/oracle/diag/rdbms/igt/igt/trace/igt_ora_13764.trc
ORA-00600: internal error code, arguments: [kdtgsp-inc], [], [], [], [], [], [], [], [], [], [], []

========= Dump for incident 243949 (ORA 600 [kdtgsp-inc]) ========

*** 2017-08-04 08:59:25.201
----- Current SQL Statement for this session (sql_id=ancznzs51sycm) -----
INSERT INTO GA_W_COUNTERS_HISTORY (STATIC_ID , DYN1 , DYN2 , DYN3 , DYN4 , DYN5, DYN_S1, COUNTER_SUM , COUNTER_DELTA, TS_LAST_MODIFIED, NODE_ID, DAY) values (:1 ,:2 ,:3 ,:4 ,:5 ,:6 ,:7 ,:8 ,:9 ,:10 ,:11 ,:12 )

Begin OCI Call Context Dump
End OCI Call Context Dump


===========================
Oracle Metalink
===========================
Per Oracle document:
Bug 9399991 - Assorted Internal Errors and Dumps (mostly under kkpa*/kcb*) from SQL against partitioned tables (Doc ID 9399991.8):

Versions confirmed as being affected:
11.1.0.7
11.2.0.1
  
Description

Various dumps and / or ORA-600 errors can occur if a partitioned table has lots of dependent references in the shared pool.
eg: lots of different literal SQLs reference the same partitioned table.

The problem is that an internal field can overflow when the number of dependencies exceeds 65536 which can then allow the MVOB of the partitioned table tohave data freed leaving stale references in the SGA.

Rediscovery notes:
 If the problem is present the a library cache dump will show one or more MVOB entries which have a KGHP flag but no KEP flag. 

 A SQL like the following may help show the MVOB entries in the library cache: 
  select KGLHDADR, KGLNAHSH, KGLNAOWN, KGLFNOBJ, KGLHDNSP, KGLOBTYP, 
         Decode(bitand(KGLHDFLG,4),4,'YES','NO') "KPR",
         case when kglhdamk < 0 then (65536 + kglhdamk)
              else kglhdamk end "KDP"
   from x$kglob 
   where KGLOBTYP=70;

 In this select:
  "KPR" is the flag we are interested in. YES=flag set, NO=not set
  "KDP" is the dependency count. 
  You should see this increase if new SQL statements are parsed that reference the partitioned object. 
  If you see KDP>0 and KPR=NO then there is a problem in the SGA and it may be worth flushing.

 If you see KDP increasing above about 50000 for a specific entry then it may also be worth flushing the shared pool as the problem  could occur if the dependency count increases much further.

 Solution

 Per above: "If you see KDP>0 and KPR=NO then there is a problem in the SGA and it may be worth flushing."


===========================
In the specific case
===========================
Failing SQL was:
INSERT INTO GA_W_COUNTERS_HISTORY (STATIC_ID , DYN1 , DYN2 , DYN3 , DYN4 , DYN5, DYN_S1, COUNTER_SUM , COUNTER_DELTA, TS_LAST_MODIFIED, NODE_ID, DAY) 
values (:1 ,:2 ,:3 ,:4 ,:5 ,:6 ,:7 ,:8 ,:9 ,:10 ,:11 ,:12 )

GA_W_COUNTERS_HISTORY table is indeed partitioned.

SELECT TABLE_OWNER, TABLE_NAME, PARTITION_NAME 
  FROM DBA_TAB_PARTITIONS 
 WHERE table_name = 'GA_W_COUNTERS_HISTORY' 
   AND TABLE_OWNER = 'MY_USER'

TABLE_OWNER        TABLE_NAME                     PARTITION_NAME
------------------ ------------------------------ --------------
MY_USER            GA_W_COUNTERS_HISTORY          P_1
MY_USER            GA_W_COUNTERS_HISTORY          P_2
MY_USER            GA_W_COUNTERS_HISTORY          P_3
MY_USER            GA_W_COUNTERS_HISTORY          P_4
MY_USER            GA_W_COUNTERS_HISTORY          P_5
MY_USER            GA_W_COUNTERS_HISTORY          P_6
MY_USER            GA_W_COUNTERS_HISTORY          P_7
MY_USER            GA_W_COUNTERS_HISTORY          P_8
MY_USER            GA_W_COUNTERS_HISTORY          P_9
MY_USER            GA_W_COUNTERS_HISTORY          P_10
MY_USER            GA_W_COUNTERS_HISTORY          P_11
MY_USER            GA_W_COUNTERS_HISTORY          P_12
MY_USER            GA_W_COUNTERS_HISTORY          P_13
MY_USER            GA_W_COUNTERS_HISTORY          P_14
MY_USER            GA_W_COUNTERS_HISTORY          P_15
MY_USER            GA_W_COUNTERS_HISTORY          P_16
MY_USER            GA_W_COUNTERS_HISTORY          P_17
MY_USER            GA_W_COUNTERS_HISTORY          P_18
MY_USER            GA_W_COUNTERS_HISTORY          P_19
MY_USER            GA_W_COUNTERS_HISTORY          P_20
MY_USER            GA_W_COUNTERS_HISTORY          P_21
MY_USER            GA_W_COUNTERS_HISTORY          P_22
MY_USER            GA_W_COUNTERS_HISTORY          P_23
MY_USER            GA_W_COUNTERS_HISTORY          P_24
MY_USER            GA_W_COUNTERS_HISTORY          P_25
MY_USER            GA_W_COUNTERS_HISTORY          P_26
MY_USER            GA_W_COUNTERS_HISTORY          P_27
MY_USER            GA_W_COUNTERS_HISTORY          P_28
MY_USER            GA_W_COUNTERS_HISTORY          P_29
MY_USER            GA_W_COUNTERS_HISTORY          P_30
MY_USER            GA_W_COUNTERS_HISTORY          P_31

When running the SQL, from the above technote:
 select KGLHDADR, KGLNAHSH, KGLNAOWN, KGLFNOBJ, KGLHDNSP, KGLOBTYP, 
         Decode(bitand(KGLHDFLG,4),4,'YES','NO') "KPR",
         case when kglhdamk < 0 then (65536 + kglhdamk)
              else kglhdamk end "KDP"
   from x$kglob 
   where KGLOBTYP=70;

KGLHDADR         KGLNAHSH   KGLNAOWN  KGLFNOBJ          KGLHDNSP KGLOBTYP KPR   KDP
---------------- ---------- --------- ----------------- -------- -------- ----- ---
000000011D531030 1057352192 MY_USER_C GA_W_COUNTERS_ORY 45       70       NO    2
0000000156411590 1057352192 MY_USER_C GA_W_COUNTERS_ORY 45       70       YES   0
000000011980EBC0 4229035152 MY_USER_F GA_W_COUNTERS_ORY 45       70       YES   2
000000015699DCD0 2014443530 MY_USER_J GA_W_COUNTERS_ORY 45       70       YES   1
...
...
260 rows selected.

"If you see KDP>0 and KPR=NO then there is a problem in the SGA and it may be worth flushing."

So indeed, there is a corruption in memory in shared pool.
After flushing shared pool, the issue was resolved

ALTER SYSTEM FLUSH SHARED_POOL;

expdp process generates trace file with error ORA-31673

=========================
General
=========================
Oracle expdp process completes successfully, but upon completion, <pid>_dw01*.trc trace file is generated.

=========================
Evidences
=========================
Trace file contains following text:

Trace file /software/oracle/diag/rdbms/igt/igt/trace/igt_dw01_32631.trc
Oracle Database 11g Enterprise Edition Release 11.1.0.7.0 - 64bit Production
With the Partitioning, Oracle Label Security, OLAP, Data Mining,
Oracle Database Vault and Real Application Testing options
ORACLE_HOME = /software/oracle/111
System name:    Linux
Node name:      aut-mob-5-aps-1
Release:        2.6.18-194.el5
Version:        #1 SMP Tue Mar 16 21:52:39 EDT 2010
Machine:        x86_64
Instance name: igt
Redo thread mounted by this instance: 1
Oracle process number: 458
Unix process pid: 32631, image: oracle@aut-mob-5-aps-1 (DW01)


*** 2017-08-20 02:01:09.514
*** SESSION ID:(268.30456) 2017-08-20 02:01:09.514
*** CLIENT ID:() 2017-08-20 02:01:09.514
*** SERVICE NAME:(SYS$USERS) 2017-08-20 02:01:09.514
*** MODULE NAME:(Data Pump Worker) 2017-08-20 02:01:09.514
*** ACTION NAME:(SYS_EXPORT_FULL_01) 2017-08-20 02:01:09.514

DM-DBMS_DM_MODEL_EXP:schema_callout: schema=SYSTEM, pre_schema=NOOP for 11g

*** 2017-08-20 02:01:09.514
DM-DBMS_DM_MODEL_EXP:schema_callout: schema=OUTLN, pre_schema=NOOP for 11g

*** 2017-08-20 02:01:09.514
DM-DBMS_DM_MODEL_EXP:schema_callout: schema=TSMSYS, pre_schema=NOOP for 11g

*** 2017-08-20 02:01:09.515
DM-DBMS_DM_MODEL_EXP:schema_callout: schema=SHDAEMON, pre_schema=NOOP for 11g



*** 2017-08-20 02:14:43.329
DM-DBMS_DM_MODEL_EXP:schema_callout: schema=HRV_VIPNE_OVMDQ, pre_schema=NOOP for
 11g

*** 2017-08-20 02:18:53.785
KUPP: Error 31673 detected in worker process DW01, worker id=1
OPIRIP: Uncaught error 447. Error stack:
ORA-00447: fatal error in background process
ORA-31673: worker process interrupt for normal exit by master process
ORA-06512: at "SYS.KUPF$FILE_INT", line 972
ORA-06512: at "SYS.KUPF$FILE", line 7126
ORA-06512: at "SYS.KUPW$WORKER", line 4661
ORA-06512: at "SYS.KUPW$WORKER", line 8624
ORA-06512: at line 1
ORA-06512: at "SYS.DBMS_SQL", line 1575
ORA-06512: at "SYS.KUPW$WORKER", line 8342
ORA-06512: at "SYS.KUPW$WORKER", line 1545
ORA-06512: at line 2


=========================
Solution
=========================


This is a known issue, documented in Oracle technote:
 Data Pump Export Completed Successfully With Errors ORA-447 & ORA-31673 (Doc ID 1323792.1)

CAUSE
This is discussed in 
Bug 9962464 - ORA-447 AND ORA-31673 OCCURS IN THE WORKER PROCESS WHEN EXPDP IS EXECUTED.
which has been closed with status 33 - Suspended, Req'd Info not Avail

SOLUTION
Ignore the error as the export has already completed and so this is just an unnecessary trace file being generated on exit of the Data Pump Worker process and so has no impact on the database or export. 

In case you would run into the same issue, you could provide a working test case to restart to bug investigation.

=========================
Workaround
=========================

To avoid trace file occupied space, delete *.trc files with below crontab task


delete_expdp_trace_files.sh
#!/bin/bash

HOME_DIR=/software/oracle/diag/rdbms/igt/igt/trace
WORK_DIR=`pwd`
cd ${HOME_DIR}
ls -lt ${HOME_DIR} | grep dw01 | grep trc | awk '{print $9}' | xargs grep -l 00447 | xargs rm -f
cd $WORK_DIR

crontab -l

0 6 * * * /software/oracle/oracle/scripts/delete_expdp_trace_files.sh


=========================
A Similar Issue:

=========================
1956899.1
Lots of Trace Files Reporting "Kcbzib: Dump Suspect Buffer, Err2=1410" 

As a temporary workaround, you can simply delete the files as these should not have been generated for the non-user-visible errors in the first place.

Oracle hidden parameters

=========================
General
=========================
Oracle has more than 2000 hidden parameters.
There parameters have a '_' prefix.
They cannot be viewed via V$PARAMETER
Instead one can use the SQL below, to query X$KSPPI table.

=========================
Short reference
=========================

oracle@my_server:~>%sqlplus / as sysdba
SQL*Plus: Release 11.1.0.7.0 - Production on Sun Aug 20 08:52:10 2017

Copyright (c) 1982, 2008, Oracle.  All rights reserved.


Connected to:
Oracle Database 11g Enterprise Edition Release 11.1.0.7.0 - 64bit Production
With the Partitioning, Oracle Label Security, OLAP, Data Mining,
Oracle Database Vault and Real Application Testing options

SQL> SELECT name, value from V$PARAMETER WHERE name = '_disable_streams_pool_auto_tuning';

no rows selected

SELECT KSPPI.ksppinm PARAM ,
       KSPPCV.ksppstvl session_val ,
       KSPPSV.ksppstvl instance_val,
       KSPPI.ksppdesc description
FROM   X$KSPPI KSPPI,
       X$KSPPCV KSPPCV ,
       X$KSPPSV KSPPSV
WHERE  KSPPI.indx = KSPPCV.indx 
  AND  KSPPI.indx = KSPPSV.indx 
  AND  KSPPI.ksppinm LIKE '/_%' escape '/' 
  AND  KSPPI.ksppinm = '_disable_streams_pool_auto_tuning';

PARAM                              SESSION_VAL INSTANCE_VAL DESCRIPTION
---------------------------------- ----------- ------------ --------------------------------

_disable_streams_pool_auto_tuning  FALSE       FALSE        disable streams pool auto tuning

There are more than 2000 hidden parameters.

SQL> SELECT COUNT(*) FROM x$ksppi;
  COUNT(*)
----------
      2049

  

Thursday, August 17, 2017

Many trc files with message "KGX cleanup..." filling up disk space to 100%

=========================
General
=========================
Oracle version is 11.1.0.7
In alert.log there are no error messages.
But the trace folder is filled up with many trc files, having same text at the end of trc file:

KGX cleanup...
KGX Atomic Operation Log 0x2570f7e70
 Mutex 0x25ba10558(207, 0) idn 2b2d96fb6306 oper EXCL
 Library Cache uid 207 efd 21 whr 102 slp 0
 oper=0 pt1=(nil) pt2=(nil) pt3=(nil)
 pt4=(nil) pt5=(nil) ub4=0


=========================
Evidences
=========================

ls /software/oracle/diag/rdbms/${ORA_INST}/${ORA_INST}/trace/*.trc | xargs grep -l "KGX cleanup" | wc -l
9207

These file are generated frequently, with no apparent pattern:
-rw-r----- 1 oracle dba 2564096 Aug 17 15:45 igt_ora_7505.trc
-rw-r----- 1 oracle dba 1286144 Aug 17 15:50 igt_ora_4501.trc
-rw-r----- 1 oracle dba 1282048 Aug 17 15:50 igt_ora_5670.trc
-rw-r----- 1 oracle dba 1282048 Aug 17 15:52 igt_ora_7665.trc
-rw-r----- 1 oracle dba 1282048 Aug 17 16:01 igt_ora_9603.trc
-rw-r----- 1 oracle dba 2564096 Aug 17 16:05 igt_ora_465.trc
-rw-r----- 1 oracle dba 1280197 Aug 17 16:15 igt_ora_9351.trc
-rw-r----- 1 oracle dba 1280226 Aug 17 16:16 igt_ora_5427.trc
-rw-r----- 1 oracle dba 1280196 Aug 17 16:16 igt_ora_5794.trc
-rw-r----- 1 oracle dba 1280196 Aug 17 16:17 igt_ora_9574.trc
-rw-r----- 1 oracle dba 2560452 Aug 17 16:17 igt_ora_9579.trc
-rw-r----- 1 oracle dba 2560424 Aug 17 16:18 igt_ora_9848.trc
-rw-r----- 1 oracle dba 1280226 Aug 17 16:23 igt_ora_6918.trc
-rw-r----- 1 oracle dba 2560392 Aug 17 16:25 igt_ora_9280.trc
-rw-r----- 1 oracle dba 2560423 Aug 17 16:26 igt_ora_9282.trc

=========================
Oracle Documentation
=========================
Per oracle technote:

 Bug 6638558 - KGX cleanup trace files produced (Doc ID 6638558.8)

bug 6638558  KGX cleanup trace files produced
Range of versions believed to be affected Versions BELOW 11.2
This issue is fixed in 11.2.0.1 (Base Release)

KGX cleanup trace files containing shared examine mutexes
may be generated when a parse-related error occurs.
Trace files may be generated containing text of the form:
  KGX cleanup...
  KGX Atomic Operation Log c00000010de2cf90
   Mutex c0000000c0c17290(1006, 0) idn 0 oper EXAM
   Cursor Parent uid 1006 efd 5 whr 26 slp 0
   oper=DEFAULT pt1=c00000009b720b00 pt2=c00000009b720ce8 pt3=c00000009b721208
   pt4=0000000000000000 u41=0 stt=0
  *** 2007-11-14 16:19:19.645
  KGX cleanup...
  KGX Atomic Operation Log c00000010de2cf90
   Mutex c0000000c0c17290(1006, 0) idn 0 oper EXAM
   Cursor Parent uid 1006 efd 5 whr 26 slp 0
   oper=DEFAULT pt1=c000000072c5fe30 pt2=c000000072c5fe98 pt3=0000000000000000
   pt4=0000000000000000 u41=0 stt=0
Workaround
  Correct the cause of any parse-related error.
 or
  Ignore the trace entries as these are due to the code unnecessarily 
  logging its cleanup operation.


=========================
Solution
=========================
Create a crontab task to execute script, that will delete *.trc files with text "KGX cleanup"

The script to delete trace files:

less delete_trace_files.sh 
#!/bin/bash

export ORA_INST=igt
BASENAME=`basename $0`
LOG_FILE=`echo $BASENAME | sed s/.sh/.log/`
HOME_DIR=/software/oracle/oracle/scripts
echo Log File: ${LOG_FILE}
cd $HOME_DIR

./write_log.sh $0
echo "About to Delete these Files:" >> $LOG_FILE

ls /software/oracle/diag/rdbms/${ORA_INST}/${ORA_INST}/trace/*.trc | xargs grep -l "KGX cleanup" | xargs ls -ltr >> ./${LOG_FILE}
ls /software/oracle/diag/rdbms/${ORA_INST}/${ORA_INST}/trace/*.trc | xargs grep -l "KGX cleanup" | xargs rm -f

less write_log.sh 
#!/bin/bash

PROGRAM_NAME=$1
WORKDIR=/software/oracle/oracle/scripts
cd ${WORKDIR}
RUN_DATE=`date +"%Y%m%d_%H%M%S"`
DELIMITER="============================================="
BASENAME=`basename $PROGRAM_NAME`
LOG_FILE=`echo $BASENAME | sed s/.sh/.log/`

touch $LOG_FILE
echo $DELIMITER >> $LOG_FILE
echo "Running $PROGRAM_NAME at $RUN_DATE" >>  $LOG_FILE
echo $DELIMITER >> $LOG_FILE


exit 0


The crontab entry
2 * * * * /software/oracle/oracle/scripts/delete_kgx_cleanup_trace_files.sh