Basic:
-----------------------------------------
OS: Linux
DB: 10.2.0.5.5 RAC (4 nodes)
DB Name: jzywk
-----------------------------------------
Backup Backgroud:
--------------------------------------------------
NFS Server: backup
Full Backup FS: 1 /nfsbackup
2 /backds5100nfs
DB Script. Backup Node: jzywk2
--------------------------------------------------
2 Problem
Rman backup file exists in full backup file system /nfsbackup but backup info disappear in controlfile after running backup scripts to another full backup file system /backds5100nfs
3 Toubleshooting
--1) Backup file in Backup file system /nfsbackup
[root@backup ~]# df -h|grep nfsbackup
24T 17T 6.5T 72% /nfsbackup
[root@backup ~]# ls -lth /nfsbackup
total 17T
-rw-r----- 1 110 110 37G Dec 2 02:06 jzywk_arch_20121202_800935065_sdnrqj4p_1_1
-rw-r----- 1 110 110 38G Dec 2 01:58 jzywk_arch_20121202_800934365_sbnrqiet_1_1
-rw-r----- 1 110 110 37G Dec 2 01:57 jzywk_arch_20121202_800934365_scnrqiet_1_1
-rw-r----- 1 110 110 195G Dec 2 01:45 jzywk_full_20121202_800911363_s8nrps03_4_1
-rw-r----- 1 110 110 96K Dec 2 01:28 jzywk_full_20121202_800933318_sanrqhe6_1_1
-rw-r----- 1 110 110 28M Dec 2 01:28 jzywk_full_20121202_800933316_s9nrqhe4_1_1
-rw-r----- 1 110 110 173G Dec 2 01:28 jzywk_full_20121202_800910496_s7nrpr50_4_1
-rw-r----- 1 110 110 500G Dec 2 01:00 jzywk_full_20121201_800911363_s8nrps03_3_1
-rw-r----- 1 110 110 500G Dec 2 00:44 jzywk_full_20121201_800910496_s7nrpr50_3_1
-rw-r----- 1 110 110 500G Dec 1 23:15 jzywk_full_20121201_800911363_s8nrps03_2_1
-rw-r----- 1 110 110 500G Dec 1 22:57 jzywk_full_20121201_800910496_s7nrpr50_2_1
-rw-r----- 1 110 110 500G Dec 1 21:36 jzywk_full_20121201_800911363_s8nrps03_1_1
-rw-r----- 1 110 110 500G Dec 1 21:19 jzywk_full_20121201_800910496_s7nrpr50_1_1
-rw-r----- 1 110 110 199G Dec 1 19:22 jzywk_full_20121201_800887703_s6nrp4sn_4_1
-rw-r----- 1 110 110 186G Dec 1 19:08 jzywk_full_20121201_800886856_s5nrp428_4_1
-rw-r----- 1 110 110 500G Dec 1 18:26 jzywk_full_20121201_800887703_s6nrp4sn_3_1
-rw-r----- 1 110 110 500G Dec 1 18:15 jzywk_full_20121201_800886856_s5nrp428_3_1
-rw-r----- 1 110 110 500G Dec 1 16:34 jzywk_full_20121201_800887703_s6nrp4sn_2_1
-rw-r----- 1 110 110 500G Dec 1 16:15 jzywk_full_20121201_800886856_s5nrp428_2_1
-rw-r----- 1 110 110 500G Dec 1 14:48 jzywk_full_20121201_800887703_s6nrp4sn_1_1
-rw-r----- 1 110 110 500G Dec 1 14:34 jzywk_full_20121201_800886856_s5nrp428_1_1
-rw-r----- 1 110 110 194G Dec 1 12:48 jzywk_full_20121201_800863723_s4nrodfb_4_1
-rw-r----- 1 110 110 166G Dec 1 12:34 jzywk_full_20121201_800863507_s3nrod8j_4_1
-rw-r----- 1 110 110 500G Dec 1 11:53 jzywk_full_20121201_800863723_s4nrodfb_3_1
-rw-r----- 1 110 110 500G Dec 1 11:49 jzywk_full_20121201_800863507_s3nrod8j_3_1
-rw-r----- 1 110 110 500G Dec 1 10:08 jzywk_full_20121201_800863723_s4nrodfb_2_1
-rw-r----- 1 110 110 500G Dec 1 10:01 jzywk_full_20121201_800863507_s3nrod8j_2_1
-rw-r----- 1 110 110 500G Dec 1 08:12 jzywk_full_20121201_800863723_s4nrodfb_1_1
-rw-r----- 1 110 110 500G Dec 1 08:08 jzywk_full_20121201_800863507_s3nrod8j_1_1
-rw-r----- 1 110 110 199G Dec 1 06:08 jzywk_full_20121201_800745611_s2nrkq4b_4_1
-rw-r----- 1 110 110 215G Dec 1 06:05 jzywk_full_20121201_800744915_s1nrkpej_4_1
-rw-r----- 1 110 110 500G Dec 1 05:10 jzywk_full_20121201_800745611_s2nrkq4b_3_1
-rw-r----- 1 110 110 500G Dec 1 05:03 jzywk_full_20121201_800744915_s1nrkpej_3_1
-rw-r----- 1 110 110 500G Dec 1 03:05 jzywk_full_20121201_800745611_s2nrkq4b_2_1
-rw-r----- 1 110 110 500G Dec 1 02:49 jzywk_full_20121201_800744915_s1nrkpej_2_1
-rw-r----- 1 110 110 500G Dec 1 00:51 jzywk_full_20121130_800745611_s2nrkq4b_1_1
-rw-r----- 1 110 110 500G Dec 1 00:18 jzywk_full_20121130_800744915_s1nrkpej_1_1
-rw-r----- 1 110 110 191G Nov 30 21:20 jzywk_full_20121130_800722863_s0nrk3tf_4_1
-rw-r----- 1 110 110 159G Nov 30 21:08 jzywk_full_20121130_800722862_rvnrk3te_4_1
-rw-r----- 1 110 110 500G Nov 30 20:35 jzywk_full_20121130_800722862_rvnrk3te_3_1
-rw-r----- 1 110 110 500G Nov 30 20:34 jzywk_full_20121130_800722863_s0nrk3tf_3_1
-rw-r----- 1 110 110 500G Nov 30 18:47 jzywk_full_20121130_800722862_rvnrk3te_2_1
-rw-r----- 1 110 110 500G Nov 30 18:46 jzywk_full_20121130_800722863_s0nrk3tf_2_1
-rw-r----- 1 110 110 500G Nov 30 16:55 jzywk_full_20121130_800722862_rvnrk3te_1_1
-rw-r----- 1 110 110 500G Nov 30 16:54 jzywk_full_20121130_800722863_s0nrk3tf_1_1
--2) Backup info not exist in DB control file
[oracle@jzyw-r2 ~]$ rman target /
Recovery Manager: Release 10.2.0.5.0 - Production on Fri Dec 7 13:46:31 2012
Copyright (c) 1982, 2007, Oracle. All rights reserved.
connected to target database: JZYWK (DBID=3843034140)
RMAN> list backup summary;
using target database control file instead of recovery catalog
List of Backups
===============
Key TY LV S Device Type Completion Time #Pieces #Copies Compressed Tag
------- -- -- - ----------- --------------- ------- ------- ---------- ---
709 B A A DISK 05-DEC-12 1 1 NO TAG20121205T210017
710 B A A DISK 05-DEC-12 1 1 NO TAG20121205T210017
711 B A A DISK 05-DEC-12 1 1 NO TAG20121205T210017
712 B A A DISK 05-DEC-12 1 1 NO TAG20121205T210017
713 B A A DISK 06-DEC-12 1 1 NO TAG20121206T210018
714 B A A DISK 06-DEC-12 1 1 NO TAG20121206T210018
715 B A A DISK 06-DEC-12 1 1 NO TAG20121206T210018
RMAN> exit
Recovery Manager complete.
--3) Full backup scripts
#!/bin/bash
if [ -f $HOME/.bash_profile ];
then
. $HOME/.bash_profile
fi
export ORACLE_USER=/u01/home/oracle
export ORACLE_BASE=/u01/oracle
export ORACLE_HOME=$ORACLE_BASE/database
export ORACLE_SID=jzywk2
export PATH=$ORACLE_HOME/bin:$PATH
DATE=`date +%Y%m%d`
DBNAME=jzywk
BAKDIR=$1
<
crosscheck backup;
delete noprompt expired backup;
run {
allocate channel t1 type disk maxpiecesize=500g;
allocate channel t2 type disk maxpiecesize=500g;
sql 'alter system archive log current';
backup incremental level 0 format "${BAKDIR}/${DBNAME}_full_%T_%t_%U"
database include current controlfile;
sql 'alter system archive log current';
backup format "${BAKDIR}/${DBNAME}_arch_%T_%t_%U" archivelog all;
report obsolete;
release channel t1;
release channel t2;
}
EOF
--4) Cause: Rman crosscheck miss backup info due to /nfsbackup FORGET to mount on DB server Before running backup to another file system /backds5100nfs
[root@jzyw-r2 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sda3 29G 5.9G 22G 22% /
/dev/sda5 66G 20G 43G 32% /u01
/dev/sda1 99M 12M 82M 13% /boot
tmpfs 63G 0 63G 0% /dev/shm
10.148.65.70:/backds5100nfs
22T 201G 22T 1% /backds5100nfs
10.148.65.70:/ds5100bakinc
4.0T 277G 3.7T 7% /ds5100bakinc
[root@jzyw-r2 ~]# df -h |grep /nfsbackup
No return
--5) DB Server remount backup file system /nfsbackup
[oracle@jzyw-r2 ~]$ su - root
Password:
[root@jzyw-r2 ~]# mount -t nfs -o wsize=32768,rsize=32768 10.148.65.xx:/nfsbackup /nfsbackup
[root@jzyw-r2 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sda3 29G 5.9G 22G 22% /
/dev/sda5 66G 20G 43G 32% /u01
/dev/sda1 99M 12M 82M 13% /boot
tmpfs 63G 0 63G 0% /dev/shm
10.148.65.xx:/backds5100nfs
22T 78G 22T 1% /backds5100nfs
10.148.65.xx:/ds5100bakinc
4.0T 277G 3.7T 7% /ds5100bakinc
10.148.65.xx:/nfsbackup
24T 17T 6.5T 72% /nfsbackup
--6) Use catalog rman command to restore backup info to control file
[oracle@jzyw-r2 ~]$ rman target /
Recovery Manager: Release 10.2.0.5.0 - Production on Fri Dec 7 13:48:40 2012
Copyright (c) 1982, 2007, Oracle. All rights reserved.
connected to target database: JZYWK (DBID=3843034140)
RMAN> crosscheck backup
2> ;
using target database control file instead of recovery catalog
allocated channel: ORA_DISK_1
channel ORA_DISK_1: sid=6041 instance=jzywk2 devtype=DISK
crosschecked backup piece: found to be 'AVAILABLE'
backup piece handle=/ds5100bakinc/jzywk_arhc_20121205_801262827_sfns4j7b_1_1 recid=1400 stamp=801262829
crosschecked backup piece: found to be 'AVAILABLE'
backup piece handle=/ds5100bakinc/jzywk_arhc_20121205_801262827_sens4j7b_1_1 recid=1401 stamp=801262828
crosschecked backup piece: found to be 'AVAILABLE'
backup piece handle=/ds5100bakinc/jzywk_arhc_20121205_801264277_shns4kkl_1_1 recid=1402 stamp=801264280
crosschecked backup piece: found to be 'AVAILABLE'
backup piece handle=/ds5100bakinc/jzywk_arhc_20121205_801264250_sgns4kjq_1_1 recid=1403 stamp=801264278
crosschecked backup piece: found to be 'AVAILABLE'
backup piece handle=/ds5100bakinc/jzywk_arhc_20121206_801349222_sjns77j6_1_1 recid=1404 stamp=801349224
crosschecked backup piece: found to be 'AVAILABLE'
backup piece handle=/ds5100bakinc/jzywk_arhc_20121206_801350177_skns78h1_1_1 recid=1405 stamp=801350178
crosschecked backup piece: found to be 'AVAILABLE'
backup piece handle=/ds5100bakinc/jzywk_arhc_20121206_801349222_sins77j6_1_1 recid=1406 stamp=801349223
Crosschecked 7 objects
RMAN> catalog start with '/nfsbackup';
searching for all files that match the pattern /nfsbackup
no files found to be unknown to the database
RMAN> catalog start with '/nfsbackup/jzywk';
searching for all files that match the pattern /nfsbackup/jzywk
List of Files Unknown to the Database
=====================================
File Name: /nfsbackup/jzywk_full_20121130_800722862_rvnrk3te_1_1
File Name: /nfsbackup/jzywk_full_20121130_800722863_s0nrk3tf_1_1
File Name: /nfsbackup/jzywk_full_20121130_800722863_s0nrk3tf_2_1
File Name: /nfsbackup/jzywk_full_20121130_800722862_rvnrk3te_2_1
File Name: /nfsbackup/jzywk_full_20121130_800722863_s0nrk3tf_3_1
File Name: /nfsbackup/jzywk_full_20121130_800722862_rvnrk3te_3_1
File Name: /nfsbackup/jzywk_full_20121130_800722863_s0nrk3tf_4_1
File Name: /nfsbackup/jzywk_full_20121130_800722862_rvnrk3te_4_1
File Name: /nfsbackup/jzywk_full_20121130_800744915_s1nrkpej_1_1
File Name: /nfsbackup/jzywk_full_20121130_800745611_s2nrkq4b_1_1
File Name: /nfsbackup/jzywk_full_20121201_800744915_s1nrkpej_2_1
File Name: /nfsbackup/jzywk_full_20121201_800745611_s2nrkq4b_2_1
File Name: /nfsbackup/jzywk_full_20121201_800744915_s1nrkpej_3_1
File Name: /nfsbackup/jzywk_full_20121201_800745611_s2nrkq4b_3_1
File Name: /nfsbackup/jzywk_full_20121201_800744915_s1nrkpej_4_1
File Name: /nfsbackup/jzywk_full_20121201_800745611_s2nrkq4b_4_1
File Name: /nfsbackup/jzywk_full_20121201_800863507_s3nrod8j_1_1
File Name: /nfsbackup/jzywk_full_20121201_800863723_s4nrodfb_1_1
File Name: /nfsbackup/jzywk_full_20121201_800863507_s3nrod8j_2_1
File Name: /nfsbackup/jzywk_full_20121201_800863723_s4nrodfb_2_1
File Name: /nfsbackup/jzywk_full_20121201_800863507_s3nrod8j_3_1
File Name: /nfsbackup/jzywk_full_20121201_800863723_s4nrodfb_3_1
File Name: /nfsbackup/jzywk_full_20121201_800863507_s3nrod8j_4_1
File Name: /nfsbackup/jzywk_full_20121201_800863723_s4nrodfb_4_1
File Name: /nfsbackup/jzywk_full_20121201_800886856_s5nrp428_1_1
File Name: /nfsbackup/jzywk_full_20121201_800887703_s6nrp4sn_1_1
File Name: /nfsbackup/jzywk_full_20121201_800886856_s5nrp428_2_1
File Name: /nfsbackup/jzywk_full_20121201_800887703_s6nrp4sn_2_1
File Name: /nfsbackup/jzywk_full_20121201_800886856_s5nrp428_3_1
File Name: /nfsbackup/jzywk_full_20121201_800887703_s6nrp4sn_3_1
File Name: /nfsbackup/jzywk_full_20121201_800886856_s5nrp428_4_1
File Name: /nfsbackup/jzywk_full_20121201_800887703_s6nrp4sn_4_1
File Name: /nfsbackup/jzywk_full_20121201_800910496_s7nrpr50_1_1
File Name: /nfsbackup/jzywk_full_20121201_800911363_s8nrps03_1_1
File Name: /nfsbackup/jzywk_full_20121201_800910496_s7nrpr50_2_1
File Name: /nfsbackup/jzywk_full_20121201_800911363_s8nrps03_2_1
File Name: /nfsbackup/jzywk_full_20121201_800910496_s7nrpr50_3_1
File Name: /nfsbackup/jzywk_full_20121201_800911363_s8nrps03_3_1
File Name: /nfsbackup/jzywk_full_20121202_800910496_s7nrpr50_4_1
File Name: /nfsbackup/jzywk_full_20121202_800911363_s8nrps03_4_1
File Name: /nfsbackup/jzywk_full_20121202_800933316_s9nrqhe4_1_1
File Name: /nfsbackup/jzywk_full_20121202_800933318_sanrqhe6_1_1
File Name: /nfsbackup/jzywk_arch_20121202_800934365_sbnrqiet_1_1
File Name: /nfsbackup/jzywk_arch_20121202_800934365_scnrqiet_1_1
File Name: /nfsbackup/jzywk_arch_20121202_800935065_sdnrqj4p_1_1
Do you really want to catalog the above files (enter YES or NO)? YES
cataloging files...
cataloging done
List of Cataloged Files
=======================
File Name: /nfsbackup/jzywk_full_20121130_800722862_rvnrk3te_1_1
File Name: /nfsbackup/jzywk_full_20121130_800722863_s0nrk3tf_1_1
File Name: /nfsbackup/jzywk_full_20121130_800722863_s0nrk3tf_2_1
File Name: /nfsbackup/jzywk_full_20121130_800722862_rvnrk3te_2_1
File Name: /nfsbackup/jzywk_full_20121130_800722863_s0nrk3tf_3_1
File Name: /nfsbackup/jzywk_full_20121130_800722862_rvnrk3te_3_1
File Name: /nfsbackup/jzywk_full_20121130_800722863_s0nrk3tf_4_1
File Name: /nfsbackup/jzywk_full_20121130_800722862_rvnrk3te_4_1
File Name: /nfsbackup/jzywk_full_20121130_800744915_s1nrkpej_1_1
File Name: /nfsbackup/jzywk_full_20121130_800745611_s2nrkq4b_1_1
File Name: /nfsbackup/jzywk_full_20121201_800744915_s1nrkpej_2_1
File Name: /nfsbackup/jzywk_full_20121201_800745611_s2nrkq4b_2_1
File Name: /nfsbackup/jzywk_full_20121201_800744915_s1nrkpej_3_1
File Name: /nfsbackup/jzywk_full_20121201_800745611_s2nrkq4b_3_1
File Name: /nfsbackup/jzywk_full_20121201_800744915_s1nrkpej_4_1
File Name: /nfsbackup/jzywk_full_20121201_800745611_s2nrkq4b_4_1
File Name: /nfsbackup/jzywk_full_20121201_800863507_s3nrod8j_1_1
File Name: /nfsbackup/jzywk_full_20121201_800863723_s4nrodfb_1_1
File Name: /nfsbackup/jzywk_full_20121201_800863507_s3nrod8j_2_1
File Name: /nfsbackup/jzywk_full_20121201_800863723_s4nrodfb_2_1
File Name: /nfsbackup/jzywk_full_20121201_800863507_s3nrod8j_3_1
File Name: /nfsbackup/jzywk_full_20121201_800863723_s4nrodfb_3_1
File Name: /nfsbackup/jzywk_full_20121201_800863507_s3nrod8j_4_1
File Name: /nfsbackup/jzywk_full_20121201_800863723_s4nrodfb_4_1
File Name: /nfsbackup/jzywk_full_20121201_800886856_s5nrp428_1_1
File Name: /nfsbackup/jzywk_full_20121201_800887703_s6nrp4sn_1_1
File Name: /nfsbackup/jzywk_full_20121201_800886856_s5nrp428_2_1
File Name: /nfsbackup/jzywk_full_20121201_800887703_s6nrp4sn_2_1
File Name: /nfsbackup/jzywk_full_20121201_800886856_s5nrp428_3_1
File Name: /nfsbackup/jzywk_full_20121201_800887703_s6nrp4sn_3_1
File Name: /nfsbackup/jzywk_full_20121201_800886856_s5nrp428_4_1
File Name: /nfsbackup/jzywk_full_20121201_800887703_s6nrp4sn_4_1
File Name: /nfsbackup/jzywk_full_20121201_800910496_s7nrpr50_1_1
File Name: /nfsbackup/jzywk_full_20121201_800911363_s8nrps03_1_1
File Name: /nfsbackup/jzywk_full_20121201_800910496_s7nrpr50_2_1
File Name: /nfsbackup/jzywk_full_20121201_800911363_s8nrps03_2_1
File Name: /nfsbackup/jzywk_full_20121201_800910496_s7nrpr50_3_1
File Name: /nfsbackup/jzywk_full_20121201_800911363_s8nrps03_3_1
File Name: /nfsbackup/jzywk_full_20121202_800910496_s7nrpr50_4_1
File Name: /nfsbackup/jzywk_full_20121202_800911363_s8nrps03_4_1
File Name: /nfsbackup/jzywk_full_20121202_800933316_s9nrqhe4_1_1
File Name: /nfsbackup/jzywk_full_20121202_800933318_sanrqhe6_1_1
File Name: /nfsbackup/jzywk_arch_20121202_800934365_sbnrqiet_1_1
File Name: /nfsbackup/jzywk_arch_20121202_800934365_scnrqiet_1_1
File Name: /nfsbackup/jzywk_arch_20121202_800935065_sdnrqj4p_1_1
4 Verify
RMAN> list backup summary;
List of Backups
===============
Key TY LV S Device Type Completion Time #Pieces #Copies Compressed Tag
------- -- -- - ----------- --------------- ------- ------- ---------- ---
694 B 0 A DISK 30-NOV-12 4 1 NO TAG20121130T150100
695 B 0 A DISK 30-NOV-12 4 1 NO TAG20121130T150100
696 B 0 A DISK 01-DEC-12 4 1 NO TAG20121130T150100
697 B 0 A DISK 01-DEC-12 4 1 NO TAG20121130T150100
698 B 0 A DISK 01-DEC-12 4 1 NO TAG20121130T150100
699 B 0 A DISK 01-DEC-12 4 1 NO TAG20121130T150100
700 B 0 A DISK 01-DEC-12 4 1 NO TAG20121130T150100
701 B 0 A DISK 01-DEC-12 4 1 NO TAG20121130T150100
702 B 0 A DISK 02-DEC-12 4 1 NO TAG20121130T150100
703 B 0 A DISK 02-DEC-12 1 1 NO TAG20121130T150100
704 B 0 A DISK 02-DEC-12 1 1 NO TAG20121130T150100
705 B 0 A DISK 02-DEC-12 4 1 NO TAG20121130T150100
706 B A A DISK 02-DEC-12 1 1 NO TAG20121202T014601
707 B A A DISK 02-DEC-12 1 1 NO TAG20121202T014601
708 B A A DISK 02-DEC-12 1 1 NO TAG20121202T014601
709 B A A DISK 05-DEC-12 1 1 NO TAG20121205T210017
710 B A A DISK 05-DEC-12 1 1 NO TAG20121205T210017
711 B A A DISK 05-DEC-12 1 1 NO TAG20121205T210017
712 B A A DISK 05-DEC-12 1 1 NO TAG20121205T210017
713 B A A DISK 06-DEC-12 1 1 NO TAG20121206T210018
714 B A A DISK 06-DEC-12 1 1 NO TAG20121206T210018
715 B A A DISK 06-DEC-12 1 1 NO TAG20121206T210018