Pages

Thursday, March 31, 2016

Oracle Trick: Login without knowing another user password.

========================================
General
========================================
Suppose you need to login into another user account, without knowing his password, and without altering his password?

Step 1 - Get current password mask.
SELECT password FROM DBA_USERS WHERE username = 'ABC';
--Suppose the password is 'aswdededed'

Step 2 - Temporary change user password.
ALTER USER ABC IDENTIFIED BY ABC123;


Now it is possible to login with ABC/ABC123

Step 3 - Revert back to original password.
ALTER USER ABC IDENTIFIED BY VALUES 'aswdededed
';


========================================
Run PL/SQL code for another user - DBMS_JOB
========================================
In this example, run DBMS_JOB for another user.

When attempting to run a job that belongs to another user, even as privileged user, one would get this error message:
ORA-23421: job number 123 is not a job in the job queue


To overcome this, need to use DBMS_SYS_SQL Package.
DBMS_SYS_SQL Package allows user A to run SQL statements as user B.

DBMS_SYS_SQL is owned by SYS, and first must be granted.
sqlplus / as sysdba
GRANT EXECUTE ON SYS.DBMS_SYS_SQL TO admin_user;

Now, log as admin_user, and use DBMS_SYS_SQL, in a PL/SQL block.

In this example, all jobs of a certain type are executed under all users.

DECLARE
 v_uid     NUMBER;
 v_result  INTEGER;

 v_sql_template VARCHAR2(1000) := 'BEGIN DBMS_JOB.BROKEN(my_job_id, FALSE, SYSDATE+1/1440); COMMIT; END;  ';

 v_sqltext      VARCHAR2(1000);
 v_myint   INTEGER;

 CURSOR broken_jobs_cur IS SELECT schema_user, job FROM DBA_JOBS WHERE what = 'SUPPORT_UTIL_PKG.SUPPORT_COUNTERS_PRC;' AND broken = 'Y';

 BEGIN

   FOR broken_jobs_rec IN broken_jobs_cur loop

     SELECT user_id INTO v_uid FROM ALL_USERS WHERE username = broken_jobs_rec.schema_user;     

     v_sqltext := REPLACE (v_sql_template,'my_job_id',broken_jobs_rec.job);
     
     v_myint := SYS.DBMS_SYS_SQL.OPEN_CURSOR();

     SYS.DBMS_SYS_SQL.PARSE_AS_USER(v_myint, v_sqltext, DBMS_SQL.NATIVE, v_uid);

     v_result := SYS.DBMS_SYS_SQL.EXECUTE(v_myint);

     SYS.DBMS_SYS_SQL.CLOSE_CURSOR(v_myint);     

     COMMIT;
  END LOOP;   
 END ;
/

========================================
Example compiling Invalid Objects from same user.
========================================

CREATE USER DBA_USER IDENTIFIED BY DBA_PASS DEFAULT TABLESPACE IGT_TABLE;
GRANT DBA TO DBA_USER;

--Because PL/SQL require explicit grants:
GRANT SELECT ON DBA_USERS TO DBA_USER;
GRANT SELECT ON  DBA_OBJECTS TO DBA_USER;
GRANT EXECUTE ON SYS.DBMS_SYS_SQL TO DBA_USER;


CREATE OR REPLACE PACKAGE BODY ADMIN_UTIL IS
   --=======================================  
  C_EXP_GEN_APP_ERR      CONSTANT NUMBER := -20100;
   --=======================================

   PROCEDURE write_sga_w_log(p_module_name IN VARCHAR, p_msg_text IN VARCHAR2) IS 
     PRAGMA AUTONOMOUS_TRANSACTION;
     v_msg_text    VARCHAR2(1000);
   BEGIN
     INSERT INTO SGA_W_LOG(module_name,  msg_text, msg_date)
       VALUES ( p_module_name, p_msg_text, SYSDATE);
     COMMIT;
   EXCEPTION
     WHEN OTHERS THEN 
       v_msg_text := SQLERRM;
      RAISE_APPLICATION_ERROR(C_EXP_GEN_APP_ERR, v_msg_text);       

   END;    

  ---------------------------------------------------

  PROCEDURE EXECUTE_SQL (p_owner       IN DBA_OBJECTS.owner%TYPE,
                         p_sql_text    IN VARCHAR2) IS

    v_module_name        SGA_W_LOG.module_name%TYPE;
    v_msg_text           SGA_W_LOG.msg_text%TYPE;    
    v_uid     NUMBER;
    v_result  INTEGER;
    v_myint   INTEGER;    
  BEGIN
     v_module_name := 'EXECUTE_SQL';
     SELECT user_id INTO v_uid FROM DBA_USERS WHERE username = p_owner;     

     v_myint := SYS.DBMS_SYS_SQL.OPEN_CURSOR();

     SYS.DBMS_SYS_SQL.PARSE_AS_USER(v_myint, p_sql_text, DBMS_SQL.NATIVE, v_uid);

     v_result := SYS.DBMS_SYS_SQL.EXECUTE(v_myint);

     SYS.DBMS_SYS_SQL.CLOSE_CURSOR(v_myint);     

     COMMIT;
  EXCEPTION
    WHEN OTHERS THEN
      v_msg_text := 'Error Running Command: '|| p_sql_text||' : '|| SUBSTR(SQLERRM,1,800);
      write_sga_w_log(v_module_name,v_msg_text);    

 END EXECUTE_SQL;
  ---------------------------------------------------

  PROCEDURE COMPILE_PACKAGE_BODIES IS
    CURSOR get_invalid_package_cur IS
    SELECT owner, 
           'ALTER PACKAGE  '||owner||'.'||object_name||' COMPILE BODY' AS sql_command
      FROM DBA_OBJECTS
     WHERE DBA_OBJECTS.status = 'INVALID'
       AND DBA_OBJECTS.object_type = 'PACKAGE BODY';
       
    v_module_name        SGA_W_LOG.module_name%TYPE;
    v_msg_text           SGA_W_LOG.msg_text%TYPE;    
  BEGIN
   v_module_name := 'COMPILE_PACKAGE_BODIES';
   
   FOR get_invalid_package_rec IN get_invalid_package_cur LOOP
     BEGIN
       EXECUTE_SQL(get_invalid_package_rec.owner, get_invalid_package_rec.sql_command);
--       EXECUTE IMMEDIATE get_invalid_package_rec.sql_command;
     EXCEPTION
       WHEN OTHERS THEN         
         v_msg_text := 'Error Running Command: '|| get_invalid_package_rec.sql_command||' : '|| SUBSTR(SQLERRM,1,800);
         write_sga_w_log(v_module_name,v_msg_text);    
       END;  
   END LOOP;    
  EXCEPTION
    WHEN OTHERS THEN  
         v_msg_text := 'Unexpected Error: '|| SUBSTR(SQLERRM,1,800);
         write_sga_w_log(v_module_name,v_msg_text);    

  END COMPILE_PACKAGE_BODIES;    

  ---------------------------------------------------
  PROCEDURE COMPILE_INVALID_OBJECTS IS
  BEGIN
    COMPILE_PACKAGE_BODIES;
  EXCEPTION
    WHEN OTHERS THEN  
      NULL;
  END COMPILE_INVALID_OBJECTS;    
  ---------------------------------------------------

END ADMIN_UTIL;






Wednesday, March 30, 2016

Code Example: ksh, sqlplus, scp, ftp, zip

==============================
General
==============================
Code example.
ksh script reading from ini file.
for each ini entry, do scp to IP destination, listed in ini file.
for specific IPs, do ftp.
call sqlplus, and generate .csv file.
zip the generated file.

==================================================================
/my_project/ftm/ftm_counter_transfer.sh
==================================================================

#!/bin/bash -x

(
run_date=`date +"%d%m%Y" --date="1 days ago"`
log=/my_project/ftm/log.log

### bring SPU counters ###

num_spu=10
ini=/my_project/ftm/server_list.ini
ip=`cat $ini |awk '{print$3}'|grep -v SERVICE`
run_date=`date +"%d%m%Y" --date="1 days ago"`
work=/my_project/ftm/tmp

for i in $ip
        do
                scp $i:/my_project/igate/*/DSI/logs/MSU_License*${run_date}*.csv $work
        done

FTPHOST='111.222.333.444'
USR='my_user'
PASS='my_pass'
ftp -n -v $FTPHOST <<EOF
user $USR $PASS
prompt
binary
lcd $work
cd /my_project/igate/my_customer/DSI/logs/
mget MSU_License*${run_date}*.csv
bye
EOF

FTPHOST='111.222.333.555'
USR='my_user'
PASS='my_pass'
ftp -n -v $FTPHOST <<EOF
user $USR $PASS
prompt
binary
lcd $work
cd /my_project/igate/my_customer/DSI/logs/
mget MSU_License*${run_date}*.csv
bye
EOF


. /etc/sh/orash/oracle_login.sh orainst
. /my_project/my_user/.set_profile

cd /my_project/ftm/
/software/oracle/112/bin/sqlplus DB_USER/DB_PASS@orainst @/my_project/ftm/statistics.sql
mv /my_project/ftm/tmp/app_counters.csv /my_project/ftm/tmp/app_counters_${run_date}.csv
cp /my_project/ftm/tmp/app_counters_${run_date}.csv /my_project/ftm/archive/

cd $work
zip -m ${run_date}.zip *${run_date}*.csv

) | tee -a /my_project/ftm/log_${run_date}.log

==================================================================
/my_project/ftm/server_list.ini
==================================================================
SERVICE_IP      GATE_NAME       MANAGMENT_IP    SPU_ID
11.33.222.4     xxx-mpu-1-man   11.33.222.12 spu_id#1
11.33.222.5     xxx-mpu-2-man   11.33.222.13 spu_id#2
11.33.222.28    yyy-mpu-1-man   11.33.222.36 spu_id#3
11.33.222.29    yyy-mpu-2-man   11.33.222.37 spu_id#4
11.33.222.52    zzz-mpu-1-man   11.33.222.60 spu_id#5
11.33.222.53    zzz-mpu-2-man   11.33.222.61 spu_id#6
11.33.222.76    qqq-mpu-1-man   11.33.222.84 spu_id#7
11.33.222.77    qqq-mpu-2-man   11.33.222.85 spu_id#8

==================================================================
/my_project/my_user/.set_profile
==================================================================
export ORASH=/etc/sh/orash
export BAS_ORACLE_LIST=orainst
export ORA_VER=1120
export ORACLE_SID=orainst
export ORACLE_BASE=/software/oracle
export ORACLE_ENV_DEFINED=yes
export ORA_NLS33=/software/oracle/112/ocommon/nls/admin/data
export ORACLE_HOME=/software/oracle/112
export ORA_EXP=/backup/ora_exp

==================================================================
/my_project/ftm/statistics.sql
==================================================================
spool /my_project/ftm/tmp/app_counters.csv
SET NEWPAGE 0
SET SPACE 0
SET LINESIZE 100
SET PAGESIZE 0
SET ECHO OFF
SET FEEDBACK OFF
SET HEADING OFF
select to_char(ts_last_modified,'dd-mm-yyyy hh24:mi:ss')|| ',' ||
case STATIC_ID
when 102050101000000 then 'MPIA_total_LU'
when 102050103000000 then 'MPIA_total_CANCEL'
when 1011500104190000 then 'app_NumberOf_SMS_Messages_Success'
when 102080104000000 then 'smm_adaptors_LDAP_Requests'
when 102080101000000 then 'smm_adaptors_LDAP_Succesful_Respond'
end || ',' || sum(counter_delta)
from GA_W_COUNTERS_HISTORY
where STATIC_ID in(102050101000000,102050103000000,1011500104190000,102080104000000,102080101000000)
and to_char(ts_last_modified,'dd-mm-yyyy') = to_char(sysdate-1,'dd-mm-yyyy')
group by to_char(ts_last_modified,'dd-mm-yyyy hh24:mi:ss'),STATIC_ID
order by to_char(ts_last_modified,'dd-mm-yyyy hh24:mi:ss'),STATIC_ID;
spool off
exit

Tuesday, March 29, 2016

ORA-600 When running SELECT SQL via DB_LINK on a local Instance

==========================
Scenario
==========================
Oracle version is 11.1.0.7
In oracle alert log, there are ORA-600 messages, and coredump is occurring:

From alert.log
*** 2016-03-14 01:00:11.316
*** SESSION ID:(873.36749) 2016-03-14 01:00:11.316
*** CLIENT ID:() 2016-03-14 01:00:11.316
*** SERVICE NAME:(igt) 2016-03-14 01:00:11.316
*** MODULE NAME:(oracle@my_server (TNS V1-V3)) 2016-03-14 01:00:11.316
*** ACTION NAME:() 2016-03-14 01:00:11.316

Dump of change vector:
TYP:0 CLS: 1 AFN:21 DBA:0x0542c56b OBJ:468668 SCN:0x0000.f027f373 SEQ:  5 OP:10.2 ENC:0

*** 2016-03-14 01:00:11.316
Exception [type: SIGSEGV, Address not mapped to object] [ADDR:0x7FFFCA3EF125] [PC:0x7B12B32, kdvlin()+52]
Incident 75851 created, dump file: /software/oracle/diag/rdbms/igt/igt/incident/incdir_75851/igt_ora_22056_i75851.trc
ORA-07445: exception encountered: core dump [kdvlin()+52] [SIGSEGV] [ADDR:0x7FFFCA3EF125] [PC:0x7B12B32] [Address not mapped to object] []

Cleaning up copy latch 0
Copy latch cleanup completed
Incident 75852 created, dump file: /software/oracle/diag/rdbms/igt/igt/incident/incdir_75852/igt_ora_22056_i75852.trc
ORA-00600: internal error code, arguments: [4153], [16], [], [], [], [], [], [], [], [], [], []
ORA-00607: Internal error occurred while making a change to a data block
ORA-00602: internal programming exception
ORA-07445: exception encountered: core dump [

*** 2016-03-14 01:00:14.912
----- Error Stack Dump -----
ORA-00600: internal error code, arguments: [4153], [16], [], [], [], [], [], [], [], [], [], []
ORA-00607: Internal error occurred while making a change to a data block
ORA-00602: internal programming exception
ORA-07445: exception encountered: core dump [kdvlin()+52] [SIGSEGV] [ADDR:0x7FFFCA3EF125] [PC:0x7B12B32] [Address not mapped to object] []
----- Current SQL Statement for this session (sql_id=fbbvh8j9v9fhx) -----


SELECT /*+ OPAQUE_TRANSFORM */ "SCENARIO_ID","IMSI","MSISDN","EVENT_TYPE_ID","COUNTRY_ID","NETWORK_ID","VLR_ID","CAMPAIGN_ID","MESSAGE_TYPE","MESSAGE_ID","TS_LAST_MODIFIED","DAY","MESSAGE_TEXT","VISIT_ID","CAMEL_SERVICE_KEY","CORRELATION_ID" FROM "ONLINE_EVENTS" WHERE "TS_LAST_MODIFIED">=:1 AND "DAY"=:2 AND "TS_LAST_MODIFIED"<=:3

Why the code is failing on a simple SELECT part?
==========================
Investigation

==========================
What is weird in this error, is the hint /*+ OPAQUE_TRANSFORM */ .
This hint does not exists in the code. Where did it come from ?


Per Oracle documentation:

What is OPAQUE_TRANSFORM Hint and how to Control it (Doc ID 780503.1)
APPLIES TO: 
Oracle Database - Enterprise Edition - Version 10.2.0.3 and later

What is OPAQUE_TRANSFORM usage :
The OPAQUE_TRANSFORM hint is to help with the transformation of datatype when certain type of operations are done within the database.  
For example object types .
It is also used for a insert-as-remote-select operation on a remote database.
 This hint should not interfere with the query optimizer plan.

-Note that if local site is 11g and remote is  11g server, this opens 2 sessions 

   on the remote database and OPAQUE_TRANSFORM hint gives DX LOCK deadlock. 

==========================
Resolution

==========================
The SELECT statement in question is querying "ONLINE_EVENTS" table, inserting results into a local table.

After checking in USER_OBJECTS and in USER_SYNONYMS, it appears that ONLINE_EVENTS is in fact a synonym to a table in a remote schema, connected by DB LINK.

Normally that would be OK, but in this specific installation, both schemas were installed on the same instance.

So USER_A is querying table USER_B.ONLINE_EVENTS

Since both schemas are on the same instance, the DB LINK usage is in fact redundant.

After replacing the code to work with direct select, i.e. SELECT * FROM USER_B.ONLINE_EVENTS rather than via DB_LINK, the ORA-600 no longer appeared.

It seems that this is a bug in Oracle 11.1.0.7.

ORA-04021 When doing select from DUAL@DB_LINK

==========================
Scenario
==========================
There is an application which is using several remote db_links
In addition, there is a monitor program, which running every 1 hour, to check that the db_links are active.
This monitor program does the check by simple select, and reports in case of an error.
The select :
SELECT SYSDATE FROM DUAL@<DB_LINK>


For one of the remote DB_LINKS the SQL hangs for 15 minutes, and then return an error:

SQL> SELECT SYSDATE FROM DUAL@ZAF_DBLINK;
SELECT SYSDATE FROM DUAL@ZAF_DBLINK
*
ERROR at line 1:
ORA-04021: timeout occurred while waiting to lock object

==========================
Investigation
==========================
Following tables can be checked

V$SESSION
V$SESSION_EVENT
V$SESSION_WAIT
V$LOCKED_OBJECT 
DBA_DDL_LOCKS
V$ACCESS
DBA_DB_LINKS


V$SESSION
SELECT * FROM V$SESSION 
 WHERE UPPER(event) like '%JOB%'
There are several entries with event = 'jobq slave wait'


V$SESSION_EVENT
SELECT * FROM V$SESSION_EVENT 
 WHERE event = 'jobq slave wait';

SID EVENT           TOTAL_WAITS TOTAL_TIMEOUTS TIME_WAITED AVERAGE_WAIT  MAX_WAIT 
--- ---------------- ----------- -------------- ----------- ------------ ---------- 
109 jobq slave wait           60             59       17582          293        294     
110 jobq slave wait           11             11        3242          295        298     125 jobq slave wait            4              3         969          242        293     
129 jobq slave wait           10             10        2941          294        295     
133 jobq slave wait           26             25        7441          286        301     
141 jobq slave wait           11             11        3240          295        296     
142 jobq slave wait           11             11        3247          295        300     
151 jobq slave wait           11             11        3238          294        299     
164 jobq slave wait           11             11        3245          295        298     
168 jobq slave wait           31             30        9105          294        299     
170 jobq slave wait           11             11        3239          294        297     
178 jobq slave wait          172            169       50138          291        299     
192 jobq slave wait           20             19        5858          293        293     
193 jobq slave wait           40             38       11718          293        293     
200 jobq slave wait           11             11        3241          295        298     
216 jobq slave wait           11             11        3243          295        298     

V$SESSION_WAIT
SELECT SID, EVENT, WAIT_CLASS_ID, WAIT_CLASS#, WAIT_CLASS 
  FROM V$SESSION_WAIT 
 WHERE event = 'jobq slave wait';

       SID EVENT                WAIT_CLASS_ID WAIT_CLASS# WAIT_CLASS
---------- -------------------- ------------- ----------- -----------
       105 jobq slave wait         2723168908           6 Idle
       109 jobq slave wait         2723168908           6 Idle
       110 jobq slave wait         2723168908           6 Idle
       121 jobq slave wait         2723168908           6 Idle
       125 jobq slave wait         2723168908           6 Idle
       129 jobq slave wait         2723168908           6 Idle
       133 jobq slave wait         2723168908           6 Idle
       140 jobq slave wait         2723168908           6 Idle
       141 jobq slave wait         2723168908           6 Idle
       142 jobq slave wait         2723168908           6 Idle
       145 jobq slave wait         2723168908           6 Idle
       146 jobq slave wait         2723168908           6 Idle
       148 jobq slave wait         2723168908           6 Idle
       150 jobq slave wait         2723168908           6 Idle
       155 jobq slave wait         2723168908           6 Idle
       164 jobq slave wait         2723168908           6 Idle
       168 jobq slave wait         2723168908           6 Idle
       170 jobq slave wait         2723168908           6 Idle
       174 jobq slave wait         2723168908           6 Idle
       177 jobq slave wait         2723168908           6 Idle
       178 jobq slave wait         2723168908           6 Idle
       183 jobq slave wait         2723168908           6 Idle
       191 jobq slave wait         2723168908           6 Idle
       192 jobq slave wait         2723168908           6 Idle
       200 jobq slave wait         2723168908           6 Idle
       202 jobq slave wait         2723168908           6 Idle
       203 jobq slave wait         2723168908           6 Idle
       204 jobq slave wait         2723168908           6 Idle

       216 jobq slave wait         2723168908           6 Idle



V$LOCKED_OBJECT 
SELECT * FROM V$LOCKED_OBJECT
--no rows are returned

DBA_DDL_LOCKS
SELECT * FROM DBA_DDL_LOCKS
--no rows are returned


V$ACCESS
SELECT * FROM V$ACCESS
       SID OWNER                          OBJECT     TYPE
---------- ------------------------------ ---------- ------------------------
       101 MY_USER                        DUAL       NON-EXISTENT
       101 PUBLIC                         DUAL       SYNONYM
       101 SYS                            DUAL       TABLE
       108 MY_USER                        DUAL       NON-EXISTENT
       108 PUBLIC                         DUAL       SYNONYM
       108 SYS                            DUAL       TABLE
       110 MY_USER                        DUAL       NON-EXISTENT
       110 PUBLIC                         DUAL       SYNONYM
       110 SYS                            DUAL       TABLE
       111 MY_USER                        DUAL       NON-EXISTENT
       111 PUBLIC                         DUAL       SYNONYM
       111 SYS                            DUAL       TABLE
       112 SYS                            DUAL       TABLE
       122 SYS                            DUAL       TABLE
       127 MY_USER                        DUAL       NON-EXISTENT
       127 PUBLIC                         DUAL       SYNONYM
       127 SYS                            DUAL       TABLE
       132 MY_USER                        DUAL       NON-EXISTENT
       132 PUBLIC                         DUAL       SYNONYM
       132 SYS                            DUAL       TABLE
       141 MY_USER                        DUAL       NON-EXISTENT
       141 PUBLIC                         DUAL       SYNONYM
       141 SYS                            DUAL       TABLE
       143 MY_USER                        DUAL       NON-EXISTENT
       143 PUBLIC                         DUAL       SYNONYM
       143 SYS                            DUAL       TABLE
       144 MY_USER                        DUAL       NON-EXISTENT
       144 PUBLIC                         DUAL       SYNONYM
       144 SYS                            DUAL       TABLE
       149 MY_USER                        DUAL       NON-EXISTENT
       149 PUBLIC                         DUAL       SYNONYM
       149 SYS                            DUAL       TABLE
       152 SYS                            DUAL       TABLE
       153 SYS                            DUAL       TABLE
       154 MY_USER                        DUAL       NON-EXISTENT
       154 PUBLIC                         DUAL       SYNONYM
       154 SYS                            DUAL       TABLE
       160 MY_USER                        DUAL       NON-EXISTENT
       160 PUBLIC                         DUAL       SYNONYM
       160 SYS                            DUAL       TABLE
       164 SYS                            DUAL       TABLE
       165 MY_USER                        DUAL       NON-EXISTENT
       165 PUBLIC                         DUAL       SYNONYM
       165 SYS                            DUAL       TABLE
       167 MY_USER                        DUAL       NON-EXISTENT
       167 PUBLIC                         DUAL       SYNONYM
       167 SYS                            DUAL       TABLE

       171 MY_USER                        DUAL       NON-EXISTENT

This seems very weird. How can DUAL object be non existent?!

DBA_DB_LINKS
SELECT * FROM DBA_DB_LINKS WHERE db_link = 'ZAF_DBLINK';
No rows are returned!!!

After additional investigation, it appears that the db_link was dropped, but the PL/SQL code was still trying to check the connection to that db_link.

==========================
Resolution
==========================
After recreating the db_link:

SQL>  SELECT 1 FROM SOME_TABLE@ZAF_DBLINK where rownum < 2;

         1
----------
         1

SQL>  SELECT SYSDATE FROM DUAL@ZAF_DBLINK;
Again this SQL is stuck for 15 minutes, and then return the same error: 

ERROR at line 1:
ORA-04021: timeout occurred while waiting to lock object

As a workaround, the SQL checking the db_link was replaced, as not to use the DUAL@ZAF_DBLINK.

Monday, March 28, 2016

Oracle is stuck during shutdown and ORA-10873 upon startup

=======================
Scenario.
=======================
Following several ORA-600 coredumps, scheduled backup process were in failed/stuck status, which resulted in DB being hand for any DML operation.
The reson for ORA-600 is out of scope for this post.

Simple SHUTDOWN IMMEDIATE command could not be completed, because Oracle could not either commit or rollback transactions.

The solution in this case is to issue SHUTDOWN ABORT command.

Upon STARTUP, there is an ORA-10873: file 1 needs to be either taken out of backup mode or media recovered
This issue is addressed by forcing backup completion by:
 ALTER DATABASE END BACKUP;

From trace file igt_ora_10009.trc
=============================================================
Dump continued from file: /software/oracle/diag/rdbms/igt/igt/trace/igt_ora_10009.trc
ORA-00603: ORACLE server session terminated by fatal error
ORA-00600: internal error code, arguments: [4153], [16], [], [], [], [], [], [], [], [], [], []
ORA-00607: Internal error occurred while making a change to a data block
ORA-00602: internal programm
========= Dump for incident 88824 (ORA 603) ========

*** 2016-03-27 01:00:17.944
----- Error Stack Dump -----
ORA-00603: ORACLE server session terminated by fatal error
ORA-00600: internal error code, arguments: [4153], [16], [], [], [], [], [], [], [], [], [], []
ORA-00607: Internal error occurred while making a change to a data block
ORA-00602: internal programming exception
ORA-07445: exception encountered: core dump [kcocdm()+37] [SIGSEGV] [ADDR:0x0] [PC:0x7B9B88F] [SI_KERNEL(general_protection)] []
----- SQL Statement (None) -----
Current SQL information unavailable - no SGA.

Dump continued from file: /software/oracle/diag/rdbms/igt/igt/trace/igt_ora_10009.trc
ORA-00603: ORACLE server session terminated by fatal error
ORA-00600: internal error code, arguments: [4153], [16], [], [], [], [], [], [], [], [], [], []
ORA-00607: Internal error occurred while making a change to a data block
ORA-00602: internal programm
========= Dump for incident 88824 (ORA 603) ========

*** 2016-03-27 01:00:17.944
----- Error Stack Dump -----
ORA-00603: ORACLE server session terminated by fatal error
ORA-00600: internal error code, arguments: [4153], [16], [], [], [], [], [], [], [], [], [], []
ORA-00607: Internal error occurred while making a change to a data block
ORA-00602: internal programming exception
ORA-07445: exception encountered: core dump [kcocdm()+37] [SIGSEGV] [ADDR:0x0] [PC:0x7B9B88F] [SI_KERNEL(general_protection)] []
----- SQL Statement (None) -----
Current SQL information unavailable - no SGA.

SQL> SHUTDOWN IMMEDIATE;
The database just hangs.

SQL> ALTER SYSTEM CHECKPOINT;
The database just hangs.

SQL> SHUTDOWN ABORT;
ORACLE instance shut down.

SQL> STARTUP RESTRICT;
ORACLE instance started.

Total System Global Area 8551575552 bytes
Fixed Size                  2161400 bytes
Variable Size            4093641992 bytes
Database Buffers         4429185024 bytes
Redo Buffers               26587136 bytes
Database mounted.
ORA-10873: file 1 needs to be either taken out of backup mode or media
recovered
ORA-01110: data file 1: '/oracle_db/db1/db_igt/ora_system_01.dbf'

SQL> STARTUP NOMOUNT;
ORACLE instance started.

SQL> ALTER DATABASE OPEN;
ALTER DATABASE OPEN
*
ERROR at line 1:
ORA-10873: file 1 needs to be either taken out of backup mode or media
recovered
ORA-01110: data file 1: '/oracle_db/db1/db_igt/ora_system_01.dbf'

SQL> SELECT * FROM V$BACKUP;

     FILE# STATUS                CHANGE# TIME
---------- ------------------ ---------- ---------
         1 ACTIVE             4119977880 27-MAR-16
         2 ACTIVE             4119977880 27-MAR-16
         3 ACTIVE             4119977880 27-MAR-16
         4 ACTIVE             4119977880 27-MAR-16
         5 ACTIVE             4119977880 27-MAR-16
         6 ACTIVE             4119977880 27-MAR-16
         7 ACTIVE             4119977880 27-MAR-16
         8 ACTIVE             4119977880 27-MAR-16
         9 ACTIVE             4119977880 27-MAR-16
        10 ACTIVE             4119977880 27-MAR-16
        11 ACTIVE             4119977880 27-MAR-16
        12 ACTIVE             4119977880 27-MAR-16
        13 ACTIVE             4119977880 27-MAR-16
        14 ACTIVE             4119977880 27-MAR-16
        15 ACTIVE             4119977880 27-MAR-16
        16 ACTIVE             4119977880 27-MAR-16
        17 ACTIVE             4119977880 27-MAR-16
        18 ACTIVE             4119977880 27-MAR-16
        19 ACTIVE             4119977880 27-MAR-16
        20 ACTIVE             4119977880 27-MAR-16
        21 ACTIVE             4119977880 27-MAR-16

21 rows selected.


SQL> ALTER DATABASE END BACKUP;

Database altered.

SQL> SELECT * FROM V$BACKUP;

     FILE# STATUS                CHANGE# TIME
---------- ------------------ ---------- ---------
         1 NOT ACTIVE         4119977880 27-MAR-16
         2 NOT ACTIVE         4119977880 27-MAR-16
         3 NOT ACTIVE         4119977880 27-MAR-16
         4 NOT ACTIVE         4119977880 27-MAR-16
         5 NOT ACTIVE         4119977880 27-MAR-16
         6 NOT ACTIVE         4119977880 27-MAR-16
         7 NOT ACTIVE         4119977880 27-MAR-16
         8 NOT ACTIVE         4119977880 27-MAR-16
         9 NOT ACTIVE         4119977880 27-MAR-16
        10 NOT ACTIVE         4119977880 27-MAR-16
        11 NOT ACTIVE         4119977880 27-MAR-16
        12 NOT ACTIVE         4119977880 27-MAR-16
        13 NOT ACTIVE         4119977880 27-MAR-16
        14 NOT ACTIVE         4119977880 27-MAR-16
        15 NOT ACTIVE         4119977880 27-MAR-16
        16 NOT ACTIVE         4119977880 27-MAR-16
        17 NOT ACTIVE         4119977880 27-MAR-16
        18 NOT ACTIVE         4119977880 27-MAR-16
        19 NOT ACTIVE         4119977880 27-MAR-16
        20 NOT ACTIVE         4119977880 27-MAR-16
        21 NOT ACTIVE         4119977880 27-MAR-16

21 rows selected.

SQL>

SQL> ALTER DATABASE OPEN;

Database altered.