Patch (PSU) Automation

Applying patch is one the main maintenance task for a DBA. DBA team needs to give lots of effort to apply patch in every quarter. To minimize the manual effort I tried to make patching automation which will support following environments:

  • It will support for 11g and 12c Databases Including RAC and Standalone
  • It will support for Linux and AIX
  • Patching will support for Remote Node means don’t need to execute script from multiple Node. So this script need to run from one node in RAC Environment
  • User has flexibility to choose option
  • User don’t need to concentrate of patching each and every steps. User will able to apply patch in multiple environments as manual innervation is very minimal
  • It will support rollback also

I have mentioned the script as function wise to make it readable. But all functions will be under one script.

function set_grid_env
{
	cd /home/oracle
	
#	if [ ! -z "$(ls | grep grid.env)" ] ; then
#		. ./grid.env
#	fi	
	
	export ORACLE_HOME=$1
	export ORACLE_SID=$2		
	export PATH=$ORACLE_HOME/bin:$ORACLE_HOME/OPatch:$PATH
}

function set_oracle_home_env
{
#	cd /home/oracle
	export ORACLE_HOME=$1
	export ORACLE_SID=$2		
	export PATH=$ORACLE_HOME/bin:$ORACLE_HOME/OPatch:$PATH	
}

function all_rac_shutdown
{
	#	Stop Clusterware
	set_grid_env $GGRID_HOME $GASM_INSTANCE		
	R_HOSTNAME=$1	
	
	if [ ${R_HOSTNAME} == ${HOSTNAME_LOCAL_NODE} ] ; then 			
			# Grid Home 
			if [ "${SHUTDOWN_MODE}" == "unlock" ] ; then	
				echo
				echo "Bring down CRS on Local Node [$R_HOSTNAME] as UNLOCK mode..."
				echo "==================================================================================="				
				sudo $GGRID_HOME/crs/install/rootcrs.pl -unlock
				iSHUTDOWN_UNLOCK=1
				iSTARTUP_PATCH=0
			elif [ "${SHUTDOWN_MODE}" == "normal" ] ; then	
				echo
				echo "Bring down CRS on Local Node [$R_HOSTNAME] as normal mode..."
				echo "==================================================================================="				
				
				sudo $GGRID_HOME/bin/crsctl stop crs
			fi
			echo 
			echo "CRS has been stopped on Node [$R_HOSTNAME]."
			
	else 
			sleep 1
			echo 
			if [ "${SHUTDOWN_MODE}" == "unlock" ] ; then	
				echo 
				echo "Bring down CRS on Remote Node [$R_HOSTNAME] as UNLOCK mode..."
				echo "==================================================================================="			
				ssh -t ${R_HOSTNAME} "export ORACLE_HOME=$GGRID_HOME && export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH && sudo $GGRID_HOME/crs/install/rootcrs.pl -unlock && exit"
				iSHUTDOWN_UNLOCK=1
				iSTARTUP_PATCH=0
			elif [ "${SHUTDOWN_MODE}" == "normal" ] ; then	
				echo 
				echo "Bring down CRS on Remote Node [$R_HOSTNAME] as NORMAL mode..."				
				echo "==================================================================================="
				ssh -t ${R_HOSTNAME} "export ORACLE_HOME=$GGRID_HOME && export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH && sudo $GGRID_HOME/bin/crsctl stop crs && exit"
			fi
			
			echo 
			echo "CRS has been stopped on Node [$R_HOSTNAME]."
	fi
}
function rac_shutdown 
{	
	echo
	echo "Shuting down clusterware...."
	echo "==================================================================================="
	
#	Shuting down one by one node	
	if [ "$DB_TYPE" == "rac" ] ; then
	
		i=0
		for NODENAME in $NODE_LIST
		do
			all_rac_shutdown ${HOSTNAME_REMOTE_N[$i]}
			i=$i+1
		done	
		
# 	Parallel shutdown for all nodes 		
: '		if [ ${NODE_NUMBER} == 2 ] ; then 			
			all_rac_shutdown ${HOSTNAME_REMOTE_N[0]} & all_rac_shutdown ${HOSTNAME_REMOTE_N[1]} 
		elif [ ${NODE_NUMBER} == 3 ] ; then 
			all_rac_shutdown ${HOSTNAME_REMOTE_N[0]} & all_rac_shutdown ${HOSTNAME_REMOTE_N[1]} & all_rac_shutdown ${HOSTNAME_REMOTE_N[2]}
		elif [ ${NODE_NUMBER} == 4 ] ; then 
			all_rac_shutdown ${HOSTNAME_REMOTE_N[0]} & all_rac_shutdown ${HOSTNAME_REMOTE_N[1]} & all_rac_shutdown ${HOSTNAME_REMOTE_N[2]} & all_rac_shutdown ${HOSTNAME_REMOTE_N[3]} 
		fi
'
	fi	
	
	sleep 60
	echo "Clusterware Status:"
	echo "==================================================================================="
	crsctl check cluster -all
	
	echo
	echo "All databases have been shutdown"
	echo
   
	iSHUTDOWN=1
	iSTARTUP=0
	SHUTDOWN_MODE=normal
}
# listener stop
function stop_listener
{
	if [ "$DB_TYPE" == "rac" ] ; then
		srvctl stop listener 
		return 1;   
	fi
   
	for LNAME in $LISTENER_NAME
	do
		lsnrctl stop $LNAME
    done
}
function clusterware_start
{
	#	Stop Clusterware	
	set_grid_env $GGRID_HOME $GASM_INSTANCE
	
	R_HOSTNAME=$1	
	export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH
	export ORACLE_SID=$GASM_INSTANCE
	
	if [ ${R_HOSTNAME} == ${HOSTNAME_LOCAL_NODE} ] ; then 		
		if [ "${START_MODE}" == "patch" ] ; then
			echo "Starting CRS on Local Node [$R_HOSTNAME] with PATCH Mode..."
			sudo $GGRID_HOME/crs/install/rootcrs.pl -patch			
			iSHUTDOWN_UNLOCK=0
			iSTARTUP_PATCH=1
		elif [ "${START_MODE}" == "normal" ] ; then	
			echo "Starting CRS on Local Node [$R_HOSTNAME]..."
			sudo $GGRID_HOME/bin/crsctl start crs
		elif [ "${START_MODE}" == "has" ] ; then	
			echo "Starting HAS on Local Node [$R_HOSTNAME]..."
			sudo $GGRID_HOME/bin/crsctl start has	
		fi		
	else 
		sleep 15
					
#		ssh -t ${R_HOSTNAME} "cd /home/oracle && . ./grid.env && sudo crsctl start crs && exit" 
		if [ "${START_MODE}" == "patch" ] ; then
			echo "Starting CRS on Remote Node [$R_HOSTNAME] as Patching mode..."
			echo "==================================================================================="
			ssh -t ${R_HOSTNAME} "export ORACLE_HOME=$GGRID_HOME && export ORACLE_SID=$GASM_INSTANCE && export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH && sudo $GGRID_HOME/crs/install/rootcrs.pl -patch && exit"	
			iSHUTDOWN_UNLOCK=0
			iSTARTUP_PATCH=1	
		elif [ "${START_MODE}" == "normal" ] ; then	
			echo "Starting CRS on Remote Node [$R_HOSTNAME]..."
			echo "==================================================================================="
			ssh -t $R_HOSTNAME "export ORACLE_HOME=$GGRID_HOME && export ORACLE_SID=$GASM_INSTANCE && export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH && sudo $GGRID_HOME/bin/crsctl start crs && exit"
		elif [ "${START_MODE}" == "has" ] ; then
			echo "Starting HAS on Remote Node [$R_HOSTNAME]..."
			echo "==================================================================================="
			ssh -t ${R_HOSTNAME} "export ORACLE_HOME=$GGRID_HOME && export ORACLE_SID=$GASM_INSTANCE && export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH && sudo $GGRID_HOME/bin/crsctl start has && exit"
		fi
	fi
	
	echo 
	echo "Please wait 4-5 minutes to start CRS/HAS on $R_HOSTNAME ..."
	echo "==================================================================================="
	sleep 240 	
	echo 
		
	if [ "${START_MODE}" == "normal" ] ; then
		echo "CRS has been started as normal mode...."	
	elif [ "${START_MODE}" == "has" ] ; then	
		echo "HAS has been started...."	
	elif [ "${START_MODE}" == "patch" ] ; then		
		echo "CRS has been started as PATCH Mode...."	
	fi	
}
# Starts up all databases
function rac_startup 
{
	set_home_env
   
	echo
	echo "Clusterware is starting...."
	echo "==================================================================================="   
		if [ ${NODE_NUMBER} == 2 ] ; then 			
			clusterware_start ${HOSTNAME_REMOTE_N[0]} & clusterware_start ${HOSTNAME_REMOTE_N[1]} 
		elif [ ${NODE_NUMBER} == 3 ] ; then 
			clusterware_start ${HOSTNAME_REMOTE_N[0]} & clusterware_start ${HOSTNAME_REMOTE_N[1]} & clusterware_start ${HOSTNAME_REMOTE_N[2]} 
		elif [ ${NODE_NUMBER} == 4 ] ; then 
			clusterware_start ${HOSTNAME_REMOTE_N[0]} & clusterware_start ${HOSTNAME_REMOTE_N[1]} & clusterware_start ${HOSTNAME_REMOTE_N[2]} & clusterware_start ${HOSTNAME_REMOTE_N[3]} 
		fi
		
		echo "Cluster Status: "
		echo "==================================================================================="
		sudo crsctl check cluster -all	   	
	echo
	echo "Clusterware (crs/has) service has been started on all nodes.... "
	echo
	iSHUTDOWN=0
	iSTARTUP=1
	START_MODE=normal	
}
# Starts up all databases in UPGRADE mode
function startup_db_upgrade {
	echo
	echo "Starting databases with upgrade mode for [$NAME_LIST]"
	echo "==================================================================================="
	
	if [ "$DB_TYPE" == "rac" ] ; then
		set_oracle_home_env $GORACLE_HOME $NAME_LIST
		for NAME in $NAME_LIST
		do
			
			echo 
			echo "Progressing for database $NAME..."
			echo "==================================================================================="
			echo 
			
			set_oracle_home_env $GORACLE_HOME $NAME
			export PATH=$GORACLE_HOME:/bin:$PATH
			export ORACLE_HOME=$GORACLE_HOME
#			$GORACLE_HOME/bin/srvctl start database -d $NAME 
			echo "Instance name: ${NAME}${NODE_POSITION}"
			export ORACLE_SID=${NAME}${NODE_POSITION}
			
			echo 
			echo "Database parameter [cluster_database] is setting false...."
			echo "==================================================================================="
	
			sqlplus '/ as sysdba' << EOD			
			startup
			alter system set cluster_database=false scope=spfile;				
			shutdown immediate	
			startup upgrade
EOD
			
		done
	elif [ "$DB_TYPE" == "nonrac" ] ; then
		for NAME in $NAME_LIST
			do
			export ORACLE_SID=$NAME
			sqlplus '/ as sysdba' << EOD
			startup upgrade
EOD
		done
	fi
	

	echo
	echo "All databases have been started in upgrade mode"
	echo "==================================================================================="
}
function shutdown_db_normal
{
	set_home_env
	
	if [ "$DB_TYPE" == "rac" ] ; then	

: '	
		if [ "$OS_TYPE" == 'Linux' ] ; then		
			read -p "Do you want to stop database instance in all nodes or local node only? (all) [all]: " INPUT					
		elif [ "$OS_TYPE" == 'AIX' ] ; then	
			echo "Do you want to stop database instance in all nodes or local node only? (all) [all]: "
			read INPUT? 
		fi
'	
		for NAME in $NAME_LIST
		do			
			export PATH=$GORACLE_HOME:/bin:$PATH
			export ORACLE_HOME=$GORACLE_HOME
			echo "Instance name: ${NAME}${NODE_POSITION}"
			export ORACLE_SID=${NAME}${NODE_POSITION}
			
			sqlplus '/ as sysdba' << EOD
			shutdown immediate	
EOD
			if [ "$OS_TYPE" == "Linux" ] ; then
				vINSTANCE_ID=`ps -ef | grep pmon | grep -v grep | grep ${NAME} | grep -v ASM | awk '{ print $8 }' | cut -d '_' -f3 | sed 's/.\{1\}$//'`
			elif [ "$OS_TYPE" == "AIX" ] ; then
				vINSTANCE_ID=`ps -ef | grep pmon | grep -v grep | grep ${NAME} | grep -v ASM | awk '{ print $8 }' | cut -d '_' -f3 | sed 's/.\{1\}$//'`
			fi    
	
#			if [ "${vINSTANCE_ID##*( )}" == "" ] || [ -z "$vINSTANCE_ID" ] ; then	  
#				echo "Database $NAME is already down."				
#				srvctl status database -d $NAME -o immediate
#			else
			echo "Database [$NAME] is shutting down...."
			echo "==================================================================================="
#				if [ "$INPUT" == "all" ] || [ "${INPUT##*( )}" == "" ] ; then
					srvctl stop database -d $NAME -o immediate
					srvctl status database -d $NAME
#				elif [ "$INPUT" == "local" ] ; then  
#					srvctl stop instance -d $NAME  -i $NAME$NODE_POSITION -o immediate
#					srvctl status database -d $NAME
#				fi
#			fi			
		done   	

		
		
	elif [ "$DB_TYPE" == "nonrac" ] ; then
		for NAME in $NAME_LIST
		do
			echo "Database [$NAME] is shutting down...."
			echo "==================================================================================="
			
			export ORACLE_SID=$NAME  
			export PATH=$GORACLE_HOME/bin:$GORACLE_HOME/OPatch:$PATH

			if [ "$OS_TYPE" == "Linux" ] ; then
				vINSTANCE_ID=`ps -ef | grep pmon | grep -v grep | grep ${NAME} | grep -v ASM | awk '{ print $8 }' | cut -d '_' -f3`
			elif [ "$OS_TYPE" == "AIX" ] ; then
				vINSTANCE_ID=`ps -ef | grep pmon | grep -v grep | grep ${NAME} | grep -v ASM | awk '{ print $8 }' | cut -d '_' -f3`
			fi    
	
			if [ "${vINSTANCE_ID##*( )}" == "" ] || [ -z "$vINSTANCE_ID" ]; then	  			
				echo "Database $NAME is already down."
			else
				echo "Database [$NAME] is shutting down...."
				echo "==================================================================================="					  
				
				sqlplus '/ as sysdba' << EOD
				shutdown immediate
EOD
			fi  
		done	
	fi
    
   echo
   echo "All databases has been stopped..."
   echo
}
function startup_db_normal
{	
	if [ "$DB_TYPE" == "rac" ] ; then
		echo 
		set_oracle_home_env $GORACLE_HOME $NAME_LIST
		for NAME in $NAME_LIST
			do
			export ORACLE_HOME=$GORACLE_HOME
			export ORACLE_SID=${NAME}${NODE_POSITION}
			echo "Oracle SID: $ORACLE_SID"
			
			if [ "$iSTARTUP_DB_INST" == "patch" ] ; then
				echo 
				echo "Database parameter [cluster_database] is setting true...."
				echo "==================================================================================="
		
				sqlplus '/ as sysdba' << EOD
				alter system set cluster_database=true scope=spfile;
				shutdown immediate
EOD
			fi
			
			echo 
			echo "Database $NAME is starting in normal mode...."
			echo "==================================================================================="
			
			set_oracle_home_env $GORACLE_HOME $NAME
			
			${GORACLE_HOME}/bin/srvctl start database -d $NAME		
			
			echo 
			echo "Database $NAME Status:"
			echo "==================================================================================="
			${GORACLE_HOME}/bin/srvctl status database -d $NAME		
			
		done
	elif [ "$DB_TYPE" == "nonrac" ] ; then	
   		for NAME in $NAME_LIST
		do
			echo 
			echo "Database $NAME is starting in normal mode...."
			echo "==================================================================================="
			
			export ORACLE_SID=$NAME
			if [ "$OS_TYPE" == "Linux" ] ; then
				vINSTANCE_ID=`ps -ef | grep pmon | grep -v grep | grep $NAME | grep -v ASM | awk '{ print $8 }' | cut -d '_' -f3`
			elif [ "$OS_TYPE" == "AIX" ] ; then
				vINSTANCE_ID=`ps -ef | grep pmon | grep -v grep | grep $NAME | grep -v ASM | awk '{ print $9 }' | cut -d '_' -f3`
			fi
			
			if [ "${vINSTANCE_ID##*( )}" != "" ] || [ ! -z "$vINSTANCE_ID" ]; then			
				echo 
				echo "Database $NAME is already up."				
				echo "==================================================================================="
			else
				sqlplus '/ as sysdba' << EOD
				startup
EOD
			fi
		done
	fi

   echo
   echo "All databases have been started"
   echo "==================================================================================="	
   iSTARTUP_DB_INST=normal
   
}
function backup_ocr
{
	export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH
	
	i=0
	for NODENAME in $NODE_LIST
	do
		export ORACLE_SID=$GASM_INSTANCE
		echo ${HOME_BACKUP_REMOTE_N[$i]}
		echo "OCR Backup Details on $NODENAME"		
		echo "==================================================================================="	
		
		if [ ${NODENAME} == ${HOSTNAME_LOCAL_NODE} ] ; then		
			sudo $GGRID_HOME/bin/ocrconfig -export ${HOME_BACKUP_REMOTE_N[$i]}/OCRFile_expBackup_${NODENAME}_$DATENTIME.dmp
			echo "OCR Export Backup: ${HOME_BACKUP_REMOTE_N[$i]}/OCRFile_expBackup_${NODENAME}_$DATENTIME.dmp"
			sudo $GGRID_HOME/bin/ocrconfig -local -manualbackup
			sudo $GGRID_HOME/bin/ocrconfig -local -showbackup
		else 
			ssh -t ${NODENAME} "export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH && sudo ocrconfig -export ${HOME_BACKUP_REMOTE_N[$i]}/OCRFile_expBackup_${NODENAME}_$DATENTIME.dmp && sudo $GGRID_HOME/bin/ocrconfig -local -manualbackup && $GGRID_HOME/bin/ocrconfig -local -showbackup && echo 'OCR Export Backup: ${HOME_BACKUP_REMOTE_N[$i]}/OCRFile_expBackup_${NODENAME}_$DATENTIME.dmp' && exit"
		fi	
		i=$(( i + 1 ))	
	done		
}
function home_backup_rac_remote
{
	R_HOSTNAME=$1
	HOME_BACKUP=$2
	
	export ORACLE_SID=$GASM_INSTANCE
	
	echo "Backup Location for ${R_HOSTNAME}: ${HOME_BACKUP}"	
	
	if [ ${R_HOSTNAME} == ${HOSTNAME_LOCAL_NODE} ] ; then 
		sleep 10	
		
		if [ "$OS_TYPE" == "Linux" ] ; then		
			# Grid Home 
			
			# all required system files 
			
			sudo tar -Pcf ${HOME_BACKUP}/etc_oracle.tar /etc/oracle/* 
			sudo tar -Pcf ${HOME_BACKUP}/etc_inittab.tar /etc/inittab
			sudo tar -Pcf ${HOME_BACKUP}/etc_oratab.tar /etc/oratab			
						
			echo 
			echo $iECHO "\nLocal Node [$R_HOSTNAME] of Grid Home Backup is progressing..."
			echo $iECHO "\n==================================================================================="
#			echo 
			sudo tar -Pcf ${HOME_BACKUP}/GI_${R_HOSTNAME}_${TDATE}.tar $GGRID_HOME
			echo 
			echo $iECHO "\nLocal Node [$R_HOSTNAME] of Grid Home backup is completed..."			
			echo	
		
			# Oracle Home 			
			echo $iECHO "\nLocal Node [$R_HOSTNAME] Oracle home backup is progressing..."
#			echo 
#			echo $iECHO "\n==================================================================================="
			sudo tar -Pcf $HOME_BACKUP/OH_${R_HOSTNAME}_${TDATE}.tar $GORACLE_HOME
			echo 
			echo $iECHO "\nLocal Node [$R_HOSTNAME] of Oracle Home backup is completed..."
			echo 
			
		elif [ "$OS_TYPE" == "AIX" ] ; then
			# Grid Home 
			# all required system files 
			
			sudo tar cfp ${HOME_BACKUP}/etc_oracle.tar /etc/oracle/* 
			sudo tar cfp ${HOME_BACKUP}/etc_inittab.tar /etc/inittab
			sudo tar cfp ${HOME_BACKUP}/etc_oratab.tar /etc/oratab
			
			sudo ocrconfig -export ${HOME_BACKUP}/OCRFile_expBackup_${R_HOSTNAME}_$DATENTIME.dmp
			sudo ocrconfig -manualbackup
			
			echo 
			echo $iECHO "\nLocal Node [$R_HOSTNAME] of Grid Home Backup is progressing..."
			echo $iECHO "\n==================================================================================="
			sudo tar cfp $HOME_BACKUP/GI_${R_HOSTNAME}_${TDATE}.tar $GGRID_HOME
			echo 
			echo $iECHO "\nLocal Node [$R_HOSTNAME] of Grid Home backup is completed..."			
			# Oracle Home 			
			echo 
			echo $iECHO "\nLocal Node [$R_HOSTNAME] Oracle home backup is progressing..."			
			echo 
			sudo tar cfp $HOME_BACKUP/OH_${R_HOSTNAME}_${TDATE}.tar $GORACLE_HOME
			echo $iECHO "\nLocal Node [$R_HOSTNAME] of Oracle Home backup is completed...\n"			
		fi	
	else
		if [ "$OS_TYPE" == "Linux" ] ; then				
			# Grid Home 
			# all required system files 			
			ssh -t ${R_HOSTNAME} "sudo tar -Pcf ${HOME_BACKUP}/etc_oracle.tar /etc/oracle/* && sudo tar -Pcf ${HOME_BACKUP}/etc_inittab.tar /etc/inittab && sudo tar -Pcf ${HOME_BACKUP}/etc_oratab.tar /etc/oratab && exit" 
			
#			echo 
			echo $iECHO "\nNode [$R_HOSTNAME] of Grid Home Backup is progressing..."
			echo $iECHO "\n==================================================================================="
			ssh -t ${R_HOSTNAME} "sudo tar -Pcf $HOME_BACKUP/GI_${R_HOSTNAME}_${TDATE}.tar $GGRID_HOME && exit"  
#			sleep 2
			echo 
			echo $iECHO "\nNode [$R_HOSTNAME] of Grid Home backup is completed..."			
			# Oracle Home 			
#			echo
			echo $iECHO "\nNode [$R_HOSTNAME] Oracle home backup is progressing..."
#			echo 
#			echo $iECHO "\n==================================================================================="
			ssh -t ${R_HOSTNAME} "sudo tar -Pcf $HOME_BACKUP/OH_${R_HOSTNAME}_${TDATE}.tar $GORACLE_HOME && exit"
#			sleep 2
			echo 
			echo $iECHO "\nNode [$R_HOSTNAME] of Oracle Home backup is completed..."
		elif [ "$OS_TYPE" == "AIX" ] ; then
			# Grid Home 
			
			# all required system files 			
			ssh -t ${R_HOSTNAME} "sudo tar cfp ${HOME_BACKUP}/etc_oracle.tar /etc/oracle/* && sudo tar cfp ${HOME_BACKUP}/etc_inittab.tar /etc/inittab && sudo tar cfp ${HOME_BACKUP}/etc_oratab.tar /etc/oratab && exit" 
			
			echo 
			echo $iECHO "\nNode [$R_HOSTNAME] of Grid Home Backup is progressing..."
			echo 
#			echo $iECHO "\n==================================================================================="
			ssh -t ${R_HOSTNAME} "sudo tar cfp $HOME_BACKUP/GI_${R_HOSTNAME}_${TDATE}.tar $GGRID_HOME && exit"
#			sleep 2
			echo 
			echo $iECHO "\nNode [$R_HOSTNAME] of Grid Home backup is completed..."			
			# Oracle Home 			
			echo 
			echo $iECHO "\nNode [$R_HOSTNAME] Oracle home backup is progressing..."
#			echo 
#			echo $iECHO "\n==================================================================================="
			ssh -t ${R_HOSTNAME} "sudo tar cfp $HOME_BACKUP/OH_${R_HOSTNAME}_${TDATE}.tar $GORACLE_HOME && exit"
#			sleep 2
			echo $iECHO "\nNode [$R_HOSTNAME] of Oracle Home backup is completed..."
		fi
	fi

	iCOUNT=$((iCOUNT + 1))
	
	# sleep until backup is done for all nodes 
	while [ ${NODE_NUMBER} == $iCOUNT ]; 
	do 
		echo "Both node backup is done..."
		sleep 10
	done	
}
# take oracle home/binary backup
function home_backup
{
	patch_loc_validation	
	
	if [ "$DB_TYPE" == "rac" ] ; then		
			
		
		
		if [ "$OS_TYPE" == 'Linux' ] ; then
			vINSTANCE_ID=`ps -ef | grep smon | grep +ASM | awk '{ print $8 }' | cut -d '_' -f3 | sed 's/:\{1\}$//'`
		elif [ "$OS_TYPE" == 'AIX' ] ; then	
			vINSTANCE_ID=`ps -ef | grep smon | grep +ASM | awk '{ print $9 }' | cut -d '_' -f3 | sed 's/:\{1\}$//'`			
		fi
		
		if [ "${vINSTANCE_ID##*( )}" == "" ] || [ -z "$vINSTANCE_ID" ] ;  then
			echo 
			echo "It seems all services are down but recommnaded to start clusterware with unlock mode before starting backup and patching..."
			echo $iECHO "1. crs will be rebooted \n2. crs will start with unlock mode"
			echo "==================================================================================="
			echo 
			START_MODE=normal
			rac_startup
		fi
			
		echo "Stopping all services i.e. DB Instance, Listener..."
		echo "==================================================================================="

		#	OCR Backup		
		backup_ocr
		
		shutdown_db_normal
		SHUTDOWN_MODE=unlock
		rac_shutdown
	fi
	
	if [ "$OS_TYPE" == 'Linux' ] ; then
		read  -p "Do you want to take Oracle Home/Binary Backup before procced to apply patch:(y|n) [y]: " INPUT 
	elif [ "$OS_TYPE" == 'AIX' ] ; then	
		echo "Do you want to take Oracle Home/Binary Backup before procced to apply patch:(y|n) [y]: \c" 
		read INPUT? 
	fi
	
	if [ "$INPUT" == "y" ] || [ "$INPUT" == "yes" ] || [ "${INPUT##*( )}" == "" ] ; then 
		echo "Thanks for confirmation...."
    elif [ "$INPUT" == "n" ] || [ "$INPUT" == "no" ] ; then 
		echo $iECHO "This is highly recommanded to take home backup. Hope you already taken backup...\nPatching process will start without taking home backup.. "
		return 1
	fi
	
	echo 
	echo "Starting Oracle Home/Binary Backup"
	echo "==================================================================================="
	
	if [ "$DB_TYPE" == "rac" ] ; then		
		if [ ${NODE_NUMBER} == 2 ] ; then			
			home_backup_rac_remote "${HOSTNAME_REMOTE_N[0]}" "${HOME_BACKUP_REMOTE_N[0]}" & home_backup_rac_remote "${HOSTNAME_REMOTE_N[1]}" "${HOME_BACKUP_REMOTE_N[1]}"
		elif [ ${NODE_NUMBER} == 3 ] ; then 
			home_backup_rac_remote "${HOSTNAME_REMOTE_N[0]}" "${HOME_BACKUP_REMOTE_N[0]}" & home_backup_rac_remote "${HOSTNAME_REMOTE_N[1]}" "${HOME_BACKUP_REMOTE_N[0]}" & home_backup_rac_remote "${HOSTNAME_REMOTE_N[2]}" "${HOME_BACKUP_REMOTE_N[0]}"
		elif [ ${NODE_NUMBER} == 4 ] ; then 
			home_backup_rac_remote "${HOSTNAME_REMOTE_N[0]}" "${HOME_BACKUP_REMOTE_N[0]}" & home_backup_rac_remote "${HOSTNAME_REMOTE_N[1]}" "${HOME_BACKUP_REMOTE_N[0]}" & home_backup_rac_remote "${HOSTNAME_REMOTE_N[2]}" "${HOME_BACKUP_REMOTE_N[0]}" & home_backup_rac_remote "${HOSTNAME_REMOTE_N[3]}" "${HOME_BACKUP_REMOTE_N[0]}"
		fi
		
	elif [ "$DB_TYPE" == "nonrac" ] ; then			
		shutdown_db_normal
		
		vINSTANCE_ID=`ps -ef | grep tns | grep $GORACLE_HOME | grep -v grep | awk '{ print $9 }'`
		
		if [ [ "${vINSTANCE_ID##*( )}" != "" ] || [ ! -z "$vINSTANCE_ID" ] ; ] ;  then
			stop_listener
		fi				
		
		echo "Oracle Home backup is progressing....."	
		echo "==================================================================================="		
		
		if [ "$OS_TYPE" == "Linux" ] ; then
			tar -Pcf $HOSTNAME_LOCAL_NODE/OH_${HOSTNAME_LOCAL_NODE}_${TDATE}.tar $GORACLE_HOME					
		elif [ "$OS_TYPE" == "AIX" ] ; then
			tar cfp $HOSTNAME_LOCAL_NODE/OH_${HOSTNAME_LOCAL_NODE}_${TDATE}.tar $GORACLE_HOME
		fi
				
		echo $iECHO "\033[32m";
		echo "Oracle and/or Grid Home binary backup is done. Backup Location: $HOSTNAME_LOCAL_NODE/OH_${HOSTNAME}_${TDATE}.tar"
		echo $iECHO "\033[00m";
		echo 				
	fi	
	
	iHOME_BK=1
}
function patch_check_show
{
	echo "Patch prerequisite check will perform following steps: "	
	echo "==================================================================================="
	echo "1. Set required environment variable if needed"
	echo "2. Create response file for GRID/DATABASE & JVM patch and it will help to avoid user input during patching"
	echo "5. Check Prerequisite for GRID/DATABASE patch for all nodes in RAC. For Standalone will perform for ORACLE HOME"
	echo "6. Check JVM patch "
	echo "==================================================================================="
	echo 		
}

function patch_apply_show
{
	echo "Patch apply process will perform following steps: "	
	echo "==================================================================================="
	echo "1. Set required environment variable if needed"
	echo "2. Stop all services like database, listener, clusterware and so on"
	echo "3. Take GRID/Oracle home/binary backup [for rac - required system file including OCR backup & perform backup in paralle for all nodes]"
	echo "4. Start HAS for rac"
	echo "5. Apply GRID/DATABASE patch sequentially in RAC. For Standalone will apply patch in ORACLE HOME"
	echo "6. Apply JVM patch sequentially in RAC"
	echo "7. For RAC, Reboot clusterware in parallel and open CRS with PATCH mode. For Standalone, Start database"	
	echo "8. Apply Post Patch - like for 12c, datapatch for 11g postinstall.sql and catbundle psu and last UTLRP package"	
	echo "But it will ask your confirmation on most of the steps... :)"
	echo "==================================================================================="
	echo 
}
# Checks opatch functionality
function patch_check 
{
	patch_check_show
	
	patch_loc_validation	
    
	if [ "$DB_TYPE" == "rac" ] ; then
		i=0
		for NODENAME in $NODE_LIST
		do
			if [ ${HOSTNAME_LOCAL_NODE} == ${NODENAME} ] ; then # local node	
				echo 
				echo "Checking prerequisite for GRID/DB patch on Local Node [${HOSTNAME_LOCAL_NODE}]: $DB_PATCH_ID"
				echo "============================================================================================"
				set_grid_env $GGRID_HOME $GASM_INSTANCE
				
				cd ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID
				which opatch
				sudo $ORACLE_HOME/OPatch/opatchauto apply ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID -analyze

				set_oracle_home_env $GORACLE_HOME $NAME_LIST
				which opatch
				echo 
				echo "Checking prerequisite for JVM patch on Local Node [${HOSTNAME_LOCAL_NODE}]: $JVM_PATCH_ID"
				echo "==================================================================================="
				cd ${PATCH_LOC_REMOTE_N[$i]}/$JVM_PATCH_ID    
				opatch prereq CheckConflictAgainstOHWithDetail -ph ./ -invPtrLoc $ORACLE_HOME/oraInst.loc || { echo "opatch check failed"; }				
				i=$i+1
			else # remote node
				echo 
				echo "Checking prerequisite for GRID/DB patch on Remote Node [${NODENAME}]: $DB_PATCH_ID"
				echo "==================================================================================="
				set_grid_env $GGRID_HOME $GASM_INSTANCE		
				
				ssh -t ${NODENAME} "export ORACLE_HOME=$GGRID_HOME && export PATH=$ORACLE_HOME/bin:$ORACLE_HOME/OPatch:$PATH && cd ${PATCH_LOC_REMOTE_N[$i]}/${DB_PATCH_ID} && which opatch && sudo $ORACLE_HOME/OPatch/opatchauto apply ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID -analyze && exit"
				
				set_oracle_home_env $GORACLE_HOME $NAME_LIST
				
				which opatch
				echo 
				echo "Checking prerequisite for JVM patch on Node [${HOSTNAME_LOCAL_NODE}]: $JVM_PATCH_ID"
				echo "==================================================================================="
				echo 
				
				ssh -t ${NODENAME} "export ORACLE_HOME=$GORACLE_HOME && export PATH=$ORACLE_HOME/bin:$ORACLE_HOME/OPatch:$PATH && cd ${PATCH_LOC_REMOTE_N[$i]}/${JVM_PATCH_ID} && opatch prereq CheckConflictAgainstOHWithDetail -ph ./ -invPtrLoc $ORACLE_HOME/oraInst.loc && exit"
				
				i=$i+1				
			fi
		done			
	elif [ "$DB_TYPE" == "nonrac" ] ; then		
		echo 	
		echo "Checking prerequisite for DB patch on [${HOSTNAME_LOCAL_NODE}]: $DB_PATCH_ID"
		echo "==================================================================================="
		export PATH=$ORACLE_HOME/OPatch:$PATH
		cd $PATCH_LOCATION/$DB_PATCH_ID
		opatch prereq CheckConflictAgainstOHWithDetail -ph ./ -invPtrLoc $ORACLE_HOME/oraInst.loc || { echo "opatch check failed"; return 1; }						
		#nonrac JVM Patch
		echo 
		echo "Checking prerequisite for JVM patch on [${HOSTNAME_LOCAL_NODE}]: $JVM_PATCH_ID"
		echo "==================================================================================="		
		set_oracle_home_env $GORACLE_HOME $NAME_LIST
		which opatch
		echo "Checking prerequisite for JVM patch: $JVM_PATCH_ID"
		cd $PATCH_LOCATION/$JVM_PATCH_ID    
		opatch prereq CheckConflictAgainstOHWithDetail -ph ./ -invPtrLoc $ORACLE_HOME/oraInst.loc || { echo "opatch check failed"; return 1; }

	fi	
}
function patch_apply 
{
	# show details step will be performed by this 
	patch_apply_show
	
	check_prerequisite
	iCHECK_PRE=1
   
	set_env_variable
	set_home_env
   
	iSETENV=1	
	
	if [ "$DB_TYPE" == "rac" ] ; then
#		if [ "$iHOME_BK" == 0 ] ; then 
			home_backup		
			START_MODE=has
#			rac_startup	
#		elif [ "$iHOME_BK" == 1 ] ; then 
			START_MODE=has
			rac_startup	
#		fi
	elif [ "$DB_TYPE" == "nonrac" ] ; then
		home_backup
	fi		
	
#	shutdown_db_normal
	
	patch_loc_validation	
	
	if [ "$DB_TYPE" == "rac" ] ; then	
		i=0
		j=0
		iCOUNT=0
		
		for NODENAME in $NODE_LIST
		do
			if [ "$OS_TYPE" == 'Linux' ] ; then
				echo 
				read -p "Do you want to continue applying patch for GRID/DB patch # ${DB_PATCH_ID} on Node ${HOSTNAME_REMOTE_N[$i]} (y|n) [y]: " INPUT
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo 
				echo "Do you want to continue applying patch for GRID/DB patch # ${DB_PATCH_ID} on Node ${HOSTNAME_REMOTE_N[$i]} (y|n) [y]:"
				read INPUT? 
			fi	
			
			if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
				echo 
				echo "Patching is progressing...."
				echo "==================================================================================="
				
			else
				echo 
				echo "Canceling patching..."
				i=$i+1
				j=$(( j + 1 ))
				continue
			fi
					
			
			if [ ${HOSTNAME_LOCAL_NODE} == ${NODENAME} ] ; then # local node	
				echo 
				echo "Patch Location: ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID"
				export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH				
				which opatch
				export ORACLE_SID=GASM_INSTANCE
				echo "Applying GRID/DB patch on Local Node [${HOSTNAME_LOCAL_NODE}]: $DB_PATCH_ID"
				echo "==================================================================================="
				cd ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID				
				sudo $GGRID_HOME/OPatch/opatchauto apply ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID
				echo 
				i=$i+1
			else # remote node
				echo 
				echo "Patch Location: ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID"
				export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH
				export ORACLE_SID=GASM_INSTANCE			
				echo "Applying GRID/DB patch on Remote Node [${NODENAME}]: $DB_PATCH_ID"
				echo "==================================================================================="				
				ssh -t ${NODENAME} "export ORACLE_HOME=$GGRID_HOME && export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH && cd ${PATCH_LOC_REMOTE_N[$i]}/${DB_PATCH_ID} && which opatch && sudo $GGRID_HOME/OPatch/opatchauto apply ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID && exit"
					
				i=$i+1	
				
			fi
			
			echo $iSOUND "\007"
			
		done	
		# JVM Patch for RAC
		set_oracle_home_env $GORACLE_HOME $NAME_LIST				
		i=0	
			
		for NODENAME in $NODE_LIST
		do
			if [ "$OS_TYPE" == 'Linux' ] ; then
				echo 
				read -p "Do you want to continue apply JVM patch # ${JVM_PATCH_ID} on Node ${HOSTNAME_REMOTE_N[$i]} (y|n) [y]: " INPUT
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo 
				echo "Do you want to continue apply JVM patch # ${JVM_PATCH_ID} on Node ${HOSTNAME_REMOTE_N[$i]} (y|n) [y]:"
				read INPUT? 
			fi	
			
			if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
				echo 
				echo "Patching is progressing...."
				echo "==================================================================================="
			else
				echo 
				echo "Canceling patching..."
				i=$i+1
				j=$(( j + 1 ))
				continue
			fi
			
			export PATH=$GORACLE_HOME/bin:$GORACLE_HOME/OPatch:$PATH
			
			if [ ${HOSTNAME_LOCAL_NODE} == ${NODENAME} ] ; then # local node	
				echo 
				echo "Applying JVM patch on [${NODENAME}]: $JVM_PATCH_ID"
				echo "==================================================================================="
				cd ${PATCH_LOC_REMOTE_N[$i]}/$JVM_PATCH_ID
				echo "Patch Path: ${PATCH_LOC_REMOTE_N[$i]}/$JVM_PATCH_ID"
				
				$GORACLE_HOME/OPatch/opatch apply -local
				
				i=$i+1
			else # remote node
				echo
				echo "Applying JVM patch on [${NODENAME}]: $JVM_PATCH_ID"
				echo "==================================================================================="
				echo 
				echo "${PATCH_LOC_REMOTE_N[$i]}/$JVM_PATCH_ID"
				ssh -t ${NODENAME} "export ORACLE_HOME=$GORACLE_HOME && export PATH=$GORACLE_HOME/bin:$GORACLE_HOME/OPatch:$PATH && cd ${PATCH_LOC_REMOTE_N[$i]}/${JVM_PATCH_ID} && $GORACLE_HOME/OPatch/opatch apply -local && exit"				
				i=$i+1
			fi
			
			echo $iSOUND "\007"
			
		done	
		 
		if [ $j != 0 ] ; then 
			if [ "$OS_TYPE" == 'Linux' ] ; then
				echo 
				read -p "You canceled patching $j times. Do you want to continue for next step [ post patching] (y|n) [n] : " INPUT
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo 
				echo "You canceled patching $j times. Do you want to continue for next step [ post patching] (y|n) [n] : \c"
				read INPUT? 
			fi	
			
			if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "n" ] ; then			
				echo 
				echo "Post Patching is canceled by user.."
				echo "==================================================================================="
				return 1
			else
				echo 
				echo "Post Patching will continue...."
				echo "==================================================================================="			
		fi	
		fi
		
		
		
		SHUTDOWN_MODE=normal
		rac_shutdown 
		
		START_MODE=patch
		rac_startup

			
	elif [ "$DB_TYPE" == "nonrac" ] ; then		
		stop_listener
		
		if [ "$OS_TYPE" == 'Linux' ] ; then
			echo 
			read -p "Do you want to continue DB patch # ${DB_PATCH_ID} on $HOSTNAME_LOCAL_NODE (y|n) [y]: " INPUT
		elif [ "$OS_TYPE" == 'AIX' ] ; then	
			echo 
			echo "Do you want to continue DB patch # ${DB_PATCH_ID} on $HOSTNAME_LOCAL_NODE (y|n) [y]:\c"
			read INPUT? 
		fi	
			
		if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
			
			cd $PATCH_LOCATION/$DB_PATCH_ID
			echo 
			echo "Patch Location: $PATCH_LOCATION/$DB_PATCH_ID"
			echo "DB patch $DB_PATCH_ID is progressing...."
			echo "==================================================================================="
			
		else
			echo 
			echo "Patching is canceled by user..."
			break
		fi			
		
		echo $iSOUND "\007"   
		
		if [ "$DB_VERSION" == "11g" ] ; then
			#-silent -ocmrf /home/oracle/ocm.rsp 
			#/u001/app/oracle/product/11.2.0.4/OPatch/ocm/bin/emocmrsp
			$OPATCH apply -invPtrLoc $ORACLE_HOME/oraInst.loc || { echo "opatch apply $DB_PATCH_ID failed"; } || { echo "User Responded with: N"; return 1; }
		elif [ "$DB_VERSION" == "12c" ] ; then 
			$OPATCH apply -invPtrLoc $ORACLE_HOME/oraInst.loc || { echo "opatch apply $DB_PATCH_ID failed"; } || { echo "User Responded with: N"; return 1; }
		fi

		echo $iSOUND "\007"   
		if [ "$OS_TYPE" == 'Linux' ] ; then
			read  -p "Do you want to continue for JVM patch(y|n) [y]: " INPUT 
		elif [ "$OS_TYPE" == 'AIX' ] ; then	
			echo "Do you want to continue for JVM patch(y|n) [y]: \c"
			read INPUT? 
		fi
		
		if [ "$INPUT" == "n" ] || [ "$INPUT" == "no" ] ; then
			echo "User confirmed not to procced. Returing to main manu..."
			manu 
			return 1
		fi
   
		echo "Applying JVM Patch $JVM_PATCH_ID"
		echo "==================================================================================="
		cd $PATCH_LOCATION/$JVM_PATCH_ID

		if [ "$DB_VERSION" == "11g" ] ; then
		#	$OPATCH apply -silent -ocmrf /home/oracle/ocm.rsp -invPtrLoc $ORACLE_HOME/oraInst.loc || { echo "opatch apply $JVM_PATCH_ID failed"; return 1; }
			$OPATCH apply -invPtrLoc $ORACLE_HOME/oraInst.loc || { echo "opatch apply $JVM_PATCH_ID failed"; return 1; }
		elif [ "$DB_VERSION" == "12c" ] ; then 
			echo "patch command"
			$OPATCH apply -invPtrLoc $ORACLE_HOME/oraInst.loc || { echo "opatch apply $JVM_PATCH_ID failed"; return 1; }
		fi
	fi	   
	
	if [ "$OS_TYPE" == 'Linux' ] ; then
		read  -p "Do you want to continue for post patch installation like datapatch | postinstall.sql? (y|n) [y]: " INPUT 
	elif [ "$OS_TYPE" == 'AIX' ] ; then	
		echo "Do you want to continue for post patch installation like datapatch | postinstall.sql? (y|n) [y]: \c"
		read INPUT? 
	fi
		
	if [ "$INPUT" == "n" ] || [ "$INPUT" == "no" ] ; then
		echo
		echo "User confirmed not to procced. Returing to main manu..."
		echo "But it is mendatory to run post patch if you applied DB and JVM patch successfully... You can run it from main manu.."
		echo "==================================================================================="
		
		manu 
		return 1
	else 
		psu_apply
	fi
	
}
function psu_apply 
{
	patch_loc_validation
	
	shutdown_db_normal
	
	startup_db_upgrade
	
	mkdir $HOME_BACKUP_LOCAL/"patch_log_"$TDATE
	 
#	if [ "$DB_TYPE" == "rac" ] ; then
	for NAME in $NAME_LIST
		do	
			set_oracle_home_env $GORACLE_HOME ${NAME}${NODE_POSITION}
			export ORACLE_SID=${NAME}${NODE_POSITION}			
			
			if [ "$DB_VERSION" == "12c" ] ; then
				echo 
				echo "datapatch is progressing on $NAME database"
				echo "==================================================================================="
				echo 
				cd $GORACLE_HOME/OPatch 
				./datapatch -verbose >> $HOME_BACKUP_LOCAL/"patch_log_"$TDATE/${NAME}"_datapatch.log"	
				
				echo 
				echo "datapatch is completed on $NAME database. Please check following log file - for details"
				echo "Log File Path: $HOME_BACKUP_LOCAL/"patch_log_"$TDATE/${NAME}"_datapatch.log""
				echo "==================================================================================="
				echo 
				
			elif [ "$DB_VERSION" == "11g" ] ; then
				echo 
				echo "Postinstall is progressing on $NAME database" 
				echo "==================================================================================="
				echo 
				
				cd $GORACLE_HOME/sqlpatch/$JVM_PATCH_ID
				$ORACLE_HOME/bin/sqlplus "/ as sysdba" @postinstall.sql >> $HOME_BACKUP_LOCAL/"patch_log_"$TDATE/${NAME}"_postinstall.log" << EOD
EOD
				echo 
				echo "Postinstall is completed on $NAME database. Please check following log file - for details"  
				echo "Log File Path: $HOME_BACKUP_LOCAL/"patch_log_"$TDATE/${NAME}"_postinstall.log"" 
				echo "==================================================================================="
				echo 
			fi
		done	
		
		echo "Starting database in normal mode..."
		echo "==================================================================================="
		
		iSTARTUP_DB_INST=patch
		startup_db_normal			
		
		if [ "$DB_VERSION" == "11g" ] ; then
			catbundle_apply
		fi	
		
   echo $iSOUND "\007"   
   if [ "$OS_TYPE" == 'Linux' ] ; then
		read  -p "Do you want to run UTLRP package for validated invalide objects? (y|n) [y]: " INPUT
	elif [ "$OS_TYPE" == 'AIX' ] ; then	
		echo "Do you want to run UTLRP package for validated invalide objects? (y|n) [y]: \c"
		read INPUT? 
	fi

   if [ "$INPUT" == "y" ] || [ "$INPUT" == "yes" ] ; then
      run_utlrp
   else 
      echo $iECHO "This is recommanded to run this UTLRP package after patched or rollbacked...\nPlease run in from main manu.."
	  echo "==================================================================================="
   fi
}
function catbundle_apply
{	
	cd $GORACLE_HOME/rdbms/admin
	for NAME in $NAME_LIST
	do
		echo "Progressing catbundle PSU for $ORACLE_SID..."
		echo "==================================================================================="
		set_oracle_home_env $GORACLE_HOME ${NAME}${NODE_POSITION}
		export ORACLE_SID=${NAME}${NODE_POSITION}
	
		$ORACLE_HOME/bin/sqlplus "/ as sysdba" @catbundle.sql psu apply >> $HOME_BACKUP_LOCAL/"patch_log_"$TDATE/${NAME}"_psu_apply.log" << EOD
EOD
		echo "===========================================================================================" >> $HOME_BACKUP_LOCAL/"patch_log_"$TDATE/${NAME}"_psu_apply.log"
		echo 
		echo "Catbundle PSU for $ORACLE_SID is completed. Please check this logfile $HOME_BACKUP_LOCAL/"patch_log_"$TDATE/${NAME}"_psu_apply.log" for details"
		echo "========================================================================================================================"
		echo 
	done	
}	

# Run utlrp
function run_utlrp 
{
	if [ "$DB_TYPE" == "nonrac" ] ; then
		startup_db_normal 
	fi	
	
	mkdir $HOME_BACKUP_LOCAL/"patch_log_"$TDATE
	
	echo
	echo "Running UTLRP Package..."
	echo "==================================================================================="			
   
	cd $GORACLE_HOME/rdbms/admin
	for NAME in $NAME_LIST
	do
		set_oracle_home_env $GORACLE_HOME ${NAME}${NODE_POSITION}
		export ORACLE_SID=${NAME}${NODE_POSITION}
		UNAME=`echo $NAME | tr '[:lower:]' '[:upper:]'`
		
		echo 
		echo "Progressing UTLRP package for $ORACLE_SID..."
		echo "==================================================================================="	
		echo 		
		
		$ORACLE_HOME/bin/sqlplus "/ as sysdba" @?/rdbms/admin/utlrp.sql >> $HOME_BACKUP_LOCAL/"patch_log_"$TDATE/${NAME}"_utlrp.log" << EOD
EOD
		echo "===========================================================================================" >> $HOME_BACKUP_LOCAL/"patch_log_"$TDATE/${NAME}"_utlrp.log" 
		echo 
		echo "UTLRP package applied successfully on $ORACLE_SID. Please check this logfile - $HOME_BACKUP_LOCAL/"patch_log_"$TDATE/${NAME}"_utlrp.log" for details"
		echo "========================================================================================================================"
		echo 
	done
	echo
	echo "UTLRP has been applied on all databases"
	echo
}
function check_db_registry 
{
	echo
	echo "Checking database registry..."
	echo "==========================================================================================="
	
	if [ "$OS_TYPE" == 'Linux' ] ; then
		read -p "Services [clusterware|database|listener] are up? do you want to start all of these services [database, clusterware] now? (y|n) [n]: " INPUT
	elif [ "$OS_TYPE" == 'AIX' ] ; then	
		echo "Services [clusterware|database|listener] are up? do you want to start all of these services [database, clusterware] now? (y|n) [n]: \c"
		read INPUT? 
	fi
	
	if [ "$DB_TYPE" == "rac" ] ; then
		if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "n" ] ; then
			echo 
			echo "Assuming all services are up..."
			echo "==========================================================================================="
		else
			if [ "$OS_TYPE" == 'Linux' ] ; then
				vINSTANCE_ID=`ps -ef | grep smon | grep +ASM | awk '{ print $8 }' | cut -d '_' -f3 | sed 's/:\{1\}$//'`
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				vINSTANCE_ID=`ps -ef | grep smon | grep +ASM | awk '{ print $9 }' | cut -d '_' -f3 | sed 's/:\{1\}$//'`			
			fi
		
			if [ "${vINSTANCE_ID##*( )}" == "" ] || [ -z "$vINSTANCE_ID" ] ;  then
				echo 
				echo "It seems all services are down so it will start first before procced..."
				echo "==================================================================================="
				echo 
				START_MODE=normal
				rac_startup				
			fi
			startup_db_normal	
		fi
	elif [ "$DB_TYPE" == "nonrac" ] ; then
		if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "n" ] ; then
			echo
			echo "Assuming all services are up..."
			echo "==========================================================================================="
		else
			echo "Thanks for confirmation..."
			startup_db_normal
		fi		
	fi
	
	if [ "$DB_TYPE" == "rac" ] ; then
	
		for NODENAME in $NODE_LIST
		do			
			if [ ${HOSTNAME_LOCAL_NODE} == ${NODENAME} ] ; then # local node	
				echo	
				echo "Patch Summary for GRID_HOME on $NODENAME: (for details please run in seperate window $ opatch lsinv)"
				export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH
				echo "Which OPATCH:" 			
				which opatch 								
				echo "==========================================================================================="
		
				echo 
				$GGRID_HOME/OPatch/opatch lsinventory -detail -oh $GGRID_HOME | grep "Database Patch Set"
				echo 			

				export PATH=$GORACLE_HOME/OPatch:$GORACLE_HOME/bin:$PATH				
				
				echo 
				echo "Patch Summary for ORACLE_HOME on $NODENAME: (for details please run in seperate window $ opatch lsinv)"
				echo "Which OPATCH:" 			
				which opatch
				echo "==========================================================================================="
				echo 				
				
				$GORACLE_HOME/OPatch/opatch lsinventory | grep Patch
	
			else # remote node
				echo	
				echo "Patch Summary for GRID_HOME on $NODENAME: (for details please run in seperate window $ opatch lsinv)"
				echo "==========================================================================================="
		
				echo 
				
				ssh -t ${NODENAME} "export ORACLE_HOME=GGRID_HOME && export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH && echo 'Which OPatch: ' && which opatch && $GGRID_HOME/OPatch/opatch lsinventory -detail -oh $ORACLE_HOME | grep 'Database Patch Set' && exit"
				
				echo 
				echo "Patch Summary for ORACLE_HOME on $NODENAME: (for details please run in seperate window $ opatch lsinv)"
				echo "==========================================================================================="
				echo 												
				ssh -t ${NODENAME} "export PATH=$GORACLE_HOME/OPatch:$GORACLE_HOME/bin:$PATH && which opatch && $GORACLE_HOME/OPatch/opatch lsinventory -detail | grep Patch && exit"
			fi			
		done	
		
	elif [ "$DB_TYPE" == "nonrac" ] ; then   
		echo 
		echo "Patch Summary for ORACLE_HOME: (for details please run in seperate window $ opatch lsinv)"
		echo "==========================================================================================="
	
		export PATH=$GORACLE_HOME/OPatch:$GORACLE_HOME/bin:$PATH
		opatch lsinventory | grep Patch
	fi
	
	for NAME in $NAME_LIST
	do
		set_oracle_home_env $GORACLE_HOME $NAME
		
		export ORACLE_SID=${NAME}${NODE_POSITION}
		echo 
		echo "Oracle SID: $ORACLE_SID"
		echo "Which OPATCH:" 			
		which opatch				
		echo "==========================================================================================="
		echo 
		
		sqlplus '/ as sysdba' << EOD
      
		Select Name, DATABASE_ROLE, OPEN_MODE from V\$DATABASE;

		col owner for a20
		col object_name for a30
		col object_type for a25
      
		Select owner, object_name, object_type from dba_objects where status != 'VALID' order by 1,2,3;

		Select status, count(1) from dba_objects group by status;

		set linesize 200
		col COMP_NAME for A40
		SELECT COMP_NAME, VERSION, STATUS FROM SYS.DBA_REGISTRY;

        set linesize 200
        col ACTION_TIME for A30
        col ACTION for A10
        col VERSION for A25
        col COMMENTS for A30

        Select ACTION_TIME,ACTION,VERSION,COMMENTS from registry\$history order by 1;
EOD

		if [ "$DB_VERSION" == "12c" ] ; then
			export ORACLE_SID=${NAME}${NODE_POSITION}
			echo "Oracle SID: $ORACLE_SID"
			sqlplus '/ as sysdba' << EOD

			set linesize 120
			col DESCRIPTION for a50
			col ACTION for a8
			col STATUS for a8
			col ACTION_TIME for a28
			col VERSION for a8

			Select PATCH_ID,PATCH_UID,VERSION,ACTION,STATUS,ACTION_TIME,substr(logfile,1,55) from DBA_REGISTRY_SQLPATCH order by 6,2,1;
EOD
		fi
   done
}
# Rollback patch
function patch_rollback 
{	
	
	patch_loc_validation
	
	if [ "$OS_TYPE" == 'Linux' ] ; then
		vINSTANCE_ID=`ps -ef | grep smon | grep +ASM | awk '{ print $8 }' | cut -d '_' -f3 | sed 's/:\{1\}$//'`
	elif [ "$OS_TYPE" == 'AIX' ] ; then		
		vINSTANCE_ID=`ps -ef | grep smon | grep +ASM | awk '{ print $9 }' | cut -d '_' -f3 | sed 's/:\{1\}$//'`
	fi 
   
	shutdown_db_normal   	
	
	if [ "$DB_TYPE" == "rac" ] ; then	
		if [ "${vINSTANCE_ID##*( )}" == "" ] || [ -z "$vINSTANCE_ID" ] ;  then
			START_MODE=normal
			rac_startup
		fi
		
		# JVM Patch rollback for RAC
		set_oracle_home_env $GORACLE_HOME $NAME_LIST				
		export PATH=$GORACLE_HOME/OPatch:$GORACLE_HOME/bin:$PATH
		
		i=0
		for NODENAME in $NODE_LIST
		do		
		
				if [ "$OS_TYPE" == 'Linux' ] ; then
					echo 
					read -p "Do you want to continue JVM patch # ${JVM_PATCH_ID} rollback on Node ${HOSTNAME_REMOTE_N[$i]} (y|n) [y]: " INPUT
					echo 
				elif [ "$OS_TYPE" == 'AIX' ] ; then	
					echo 
					echo "Do you want to continue JVM patch # ${JVM_PATCH_ID} rollback on Node ${HOSTNAME_REMOTE_N[$i]} (y|n) [y]: \c"
					read INPUT? 
				fi	
	

: '		
			while true; 
			do
				if [ "$OS_TYPE" == 'Linux' ] ; then
					echo 
					read -p "Do you want to continue JVM patch # ${JVM_PATCH_ID} rollback on Node ${HOSTNAME_REMOTE_N[$i]} (y|n) [y]: " INPUT
					echo 
				elif [ "$OS_TYPE" == 'AIX' ] ; then	
					echo 
					echo "Do you want to continue JVM patch # ${JVM_PATCH_ID} rollback on Node ${HOSTNAME_REMOTE_N[$i]} (y|n) [y]: \c"
					read INPUT? 
				fi	
			
				#read -p "Do you wish to install this program?" yn
				case "$INPUT" in
					[Yy]* ) 
						echo 
						echo "Patching rollback is progressing...."
						echo "==================================================================================="
						
				;;
				[Nn]* ) 
						echo 
						echo "Canceling patching rollback..."
						i=$(( i + 1 ))
						j=$(( j + 1 ))
						echo $i
						echo $j						
						break
				;;
				* ) echo "Please answer y|n|yes|no"
				;;
				esac
			done
'		
			
			if [ "$INPUT" == "y" ] ; then
				echo 
				echo "Patching rollback is progressing...."
				echo "==================================================================================="				
			elif [ "$INPUT" == "n" ] ; then
				echo 
				echo "Canceling patching rollback..."
				echo 
				i=$i+1
				j=$(( j + 1 ))
				continue	
			fi
			
			export PATH=$GORACLE_HOME/bin:$GORACLE_HOME/OPatch:$PATH
				
				
			if [ ${HOSTNAME_LOCAL_NODE} == ${NODENAME} ] ; then # local node	
				echo 
				echo "Rolling back JVM patch on [${NODENAME}]: $JVM_PATCH_ID"
				echo "==================================================================================="
				
				cd ${PATCH_LOC_REMOTE_N[$i]}/$JVM_PATCH_ID				
				
				opatch rollback -id $JVM_PATCH_ID || { echo "Rollback JVM : $JVM_PATCH_ID failed";}
				i=$i+1
			else # remote node
				cd ${PATCH_LOC_REMOTE_N[$i]}/$JVM_PATCH_ID
				echo 
				echo "JVM Patch Location: ${PATCH_LOC_REMOTE_N[$i]}/$JVM_PATCH_ID"
				echo 				
				echo "Rolling back JVM patch on [${NODENAME}]: $JVM_PATCH_ID"
				echo "==================================================================================="
				echo 
				ssh -t ${NODENAME} "export ORACLE_HOME=$GORACLE_HOME && export PATH=$ORACLE_HOME/bin:$ORACLE_HOME/OPatch:$PATH && cd ${PATCH_LOC_REMOTE_N[$i]}/${JVM_PATCH_ID} && opatch rollback -id $JVM_PATCH_ID && exit" 
				
				i=$i+1	
				
			fi
		done	
		# GI and DB Patch
		
		i=0
		for NODENAME in $NODE_LIST
		do
			if [ "$OS_TYPE" == 'Linux' ] ; then
				echo 
				read -p "Do you want to continue for rolling back for GRID/DB patch # ${DB_PATCH_ID} on Node ${HOSTNAME_REMOTE_N[$i]} (y|n) [y]: " INPUT
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo 
				echo "Do you want to continue for rolling back for GRID/DB patch # ${DB_PATCH_ID} on Node ${HOSTNAME_REMOTE_N[$i]} (y|n) [y]: \c"
				read INPUT? 
			fi	
			
			if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
				echo 
				echo "Patching rollback is progressing...."
				echo "==================================================================================="
				
			else
				echo 
				echo "Canceling patching rollback..."
				j=$(( j + 1 ))
				i=$i+1
				continue
				
			fi
			
			export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH
			
			if [ ${HOSTNAME_LOCAL_NODE} == ${NODENAME} ] ; then # local node	
				echo 
				echo "Patch Location : ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID"
				echo "Rolling back GRID/DB patch on Local Node [${HOSTNAME_LOCAL_NODE}]: $DB_PATCH_ID"
				echo "==================================================================================="
				cd ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID
				export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH
				
				echo "Patch Location: ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID"
				which opatch
#				sudo $GGRID_HOME/OPatch/opatchauto rollback ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID -nonrolling
				sudo $GGRID_HOME/OPatch/opatchauto rollback ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID
				echo 
				i=$i+1
			else # remote node
				echo 
				echo "Patch Location : ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID"
				echo "Rolling back GRID/DB patch on Remote Node [${NODENAME}]: $DB_PATCH_ID"
				echo "==================================================================================="
				export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH				
				ssh -t ${NODENAME} "export ORACLE_HOME=$GGRID_HOME && export PATH=$GGRID_HOME/bin:$GGRID_HOME/OPatch:$PATH && cd ${PATCH_LOC_REMOTE_N[$i]}/${DB_PATCH_ID} && which opatch && sudo $GGRID_HOME/OPatch/opatchauto rollback ${PATCH_LOC_REMOTE_N[$i]}/$DB_PATCH_ID && exit"					
				i=$i+1	
				
			fi
		done	
		
		if [ $j != 0 ] ; then 
			if [ "$OS_TYPE" == 'Linux' ] ; then
				echo 
				read -p "You canceled patch rollback $j times. Do you want to continue for next step [ post patch deinstall] (y|n) [n] : " INPUT
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo 
				echo "You canceled patch rollback $j times. Do you want to continue for next step [ post patch deinstall] (y|n) [n] : \c"
				read INPUT? 
			fi	
		fi
		
		if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "n" ] ; then			
			echo 
			echo "Post Patch deinstall is canceled by user.."
			echo "==================================================================================="
			return 1
		else
			echo 
			echo "Post Patch deinstall will continue...."
			echo "==================================================================================="			
		fi	
		
		SHUTDOWN_MODE=normal
		rac_shutdown 
		
		START_MODE=normal
		rac_startup
		
#		iSTARTUP_DB_INST=patch
#		startup_db_normal
#		iSTARTUP_DB_INST=normal
		
	elif [ "$DB_TYPE" == "nonrac" ] ; then 	
	
		if [ "$OS_TYPE" == 'Linux' ] ; then
			echo 
			read -p "Do you want to continue for rolling back for JVM patch # ${JVM_PATCH_ID} on Node ${HOSTNAME_LOCAL_NODE} (y|n) [y]: " INPUT
			echo 
		elif [ "$OS_TYPE" == 'AIX' ] ; then	
			echo 
			echo "Do you want to continue for rolling back for JVM patch # ${JVM_PATCH_ID} on Node ${HOSTNAME_LOCAL_NODE} (y|n) [y]: \c"
			read INPUT? 
		fi	
			
		if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
			echo 
			echo "Patching rollback is progressing...."
			echo "==================================================================================="				
		else
			echo "Canceling patching rollback..."
			return 1
		fi
			
		echo "Rolling back JVM patch # $JVM_PATCH_ID"
		echo "==================================================================================="
		echo 
		
		PATH=$ORACLE_HOME/bin:$ORACLE_HOME/OPatch:$PATH		
		opatch rollback -id $JVM_PATCH_ID || { echo "Rollback JVM : $JVM_PATCH_ID failed"; return 1; }
   
		if [ "$OS_TYPE" == 'Linux' ] ; then
			echo 
			read -p "Do you want to continue for rolling back for DB patch # ${DB_PATCH_ID} on Node ${HOSTNAME_LOCAL_NODE} (y|n) [y]: " INPUT
			echo 
		elif [ "$OS_TYPE" == 'AIX' ] ; then	
			echo 
			echo "Do you want to continue for rolling back for DB patch # ${DB_PATCH_ID} on Node ${HOSTNAME_LOCAL_NODE} (y|n) [y]: \c"
			read INPUT? 
		fi	
			
		if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
			echo 
			echo "Patching rollback is progressing...."
			echo "==================================================================================="				
		else
			echo "Canceling patching rollback..."
			return 1
		fi
			
		echo "Rolling back DB patch # $DB_PATCH_ID"
		echo "==================================================================================="
		echo 
		PATH=$ORACLE_HOME/OPatch:$PATH
		opatch rollback -id $DB_PATCH_ID || { echo "Rollback DB : $DB_PATCH_ID failed"; return 1; }
		
		echo 
		echo "DB and JVM Patchs rolledback successfully!!!"   
		echo "==================================================================================="
		echo 	
	fi
   #echo -en "\007"
	
	if [ "$OS_TYPE" == 'Linux' ] ; then
		read -p "Do you want to continue post patch rollback?(y|n) [y]: " INPUT
	elif [ "$OS_TYPE" == 'AIX' ] ; then	
		echo "Do you want to continue post patch rollback?(y|n) [y]: \c"
		read INPUT? 
	fi
		   
	if [ "$INPUT" == "y" ] || [ "$INPUT" == "yes" ] || [ "${INPUT##*( )}" == "" ] ; then
		psu_rollback 
    else
		echo 
		echo "This is mendatory to run post_patch after applying or rolling back patch....You may run it from main manu.."		
		return 1;
	fi
   
}
function psu_rollback 
{   
	patch_loc_validation
	
	shutdown_db_normal		
	
	startup_db_upgrade
 

	for NAME in $NAME_LIST
	do
		
		export ORACLE_SID=${NAME}${NODE_POSITION}
		echo "Oracle SID: $ORACLE_SID"
		if [ "$DB_VERSION" == "12c" ] ; then
			echo 
			echo "datapatch is progressing for $NAME database..."   
			echo "==================================================================================="
			echo 
		
			cd $ORACLE_HOME/OPatch			
			./datapatch -verbose
		elif [ "$DB_VERSION" == "11g" ] ; then
			echo 
			echo "postdeinstall is progressing for $NAME database..."   
			echo "==================================================================================="
			echo 
			
			cd $ORACLE_HOME/sqlpatch/$JVM_PATCH_ID
			sqlplus '/ as sysdba' << EOD
			@postdeinstall.sql
EOD
			shutdown_db_normal
			startup_db_normal
			
			UNAME=`echo $NAME | tr '[:lower:]' '[:upper:]'`			
			cd $ORACLE_HOME/rdbms/admin
			
			echo 
			echo "@catbundle_PSU_${UNAME}_ROLLBACK.sql is progressing for $NAME database..."   
			echo "==================================================================================="
			echo 
		
			sqlplus '/ as sysdba' << EOD
			@catbundle_PSU_${UNAME}_ROLLBACK.sql
EOD

		fi
	done
	
	if [ "$DB_TYPE" == "rac" ] ; then
		iSTARTUP_DB_INST=patch			
		startup_db_normal			
	fi	
		
    
	echo 
	
	if [ "$OS_TYPE" == 'Linux' ] ; then
		read -p "Do you want to continue to run UTLRP package to validate for invalide objects?(y|n) [y]: " INPUT
	elif [ "$OS_TYPE" == 'AIX' ] ; then	
		echo "Do you want to continue to run UTLRP package to validate for invalide objects?(y|n) [y]: \c"
		read INPUT? 
	fi
		   
	if [ "$INPUT" == "y" ] || [ "$INPUT" == "yes" ] || [ "${INPUT##*( )}" == "" ] ; then
		run_utlrp 
    else
		echo 
		echo "Highly recommanded to run this package. You may run it from main manu.."
		echo 
	fi

}
function home_backup_location
{
	
	if [ "$DB_TYPE" == "rac" ] ; then
		echo 
		echo "Setting GRID/DATABASE Home backup location for all Nodes"
		echo "==================================================================================="
		
		i=0
		for NODENAME in $NODE_LIST
		do
			if [ "$OS_TYPE" == 'Linux' ] ; then
				read -p "GRID/DATABASE home backup location for ${HOSTNAME_REMOTE_N[$i]}: [${PATCH_LOC_REMOTE_N[$i]}]: " INPUT
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo "GRID/DATABASE home backup location for ${HOSTNAME_REMOTE_N[$i]}: [${PATCH_LOC_REMOTE_N[$i]}]: \c"
				read INPUT? 
			fi	
			
			if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then				
				HOME_BACKUP_REMOTE_N[$i]=${PATCH_LOC_REMOTE_N[$i]}
				
				if [ ${HOME_BACKUP_LOCAL} == ${HOSTNAME_REMOTE_N[$i]} ] ; then 
					HOME_BACKUP_LOCAL=${PATCH_LOC_REMOTE_N[$i]}
				fi				
				echo "Confirmed HOME backup location for ${HOSTNAME_REMOTE_N[$i]} : ${PATCH_LOC_REMOTE_N[$i]}"							
			else
				HOME_BACKUP_REMOTE_N[$i]=$INPUT
				
				echo "Confirmed HOME backup location for ${HOSTNAME_REMOTE_N[$i]}: ${HOME_BACKUP_REMOTE_N[$i]}"
				
				if [ ${HOME_BACKUP_LOCAL} == ${HOSTNAME_REMOTE_N[$i]} ] ; then 
					HOME_BACKUP_LOCAL=$INPUT
				fi
			fi
			
			
				
			i=$i+1
		done
	elif [ "$DB_TYPE" == "nonrac" ] ; then	
			if [ "$OS_TYPE" == 'Linux' ] ; then
				read -p "Location for binary/home backup: [$HOME_BACKUP]: " INPUT
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo "Location for binary/home backup: [$HOME_BACKUP]: \c"				
				read INPUT? 
			fi	   
		if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
			echo "Oracle home backup location: $HOME_BACKUP"
			HOME_BACKUP_LOCAL=${HOME_BACKUP}
		else
			HOME_BACKUP=$INPUT
			HOME_BACKUP_LOCAL=${HOME_BACKUP}
			echo "HOME backup location: $HOME_BACKUP" 
		fi
			
	fi	
}
function patch_loc_validation
{

if [ "$iPATCH_LOCATION" == 0 ] ; then
		
		if [ "$OS_TYPE" == 'Linux' ] ; then
				read -p "GRID/DB Patch Number:$DB_PATCH_ID [y or number]: " INPUT
		elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo "GRID/DB Patch Number:$DB_PATCH_ID [y or number]: \c"
				read INPUT? 
		fi
		if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
			echo "Confirmed default db patch number: $DB_PATCH_ID"
		else
			DB_PATCH_ID=$INPUT
			echo "DB Patch number: $DB_PATCH_ID"
		fi
		
		if [ "$OS_TYPE" == 'Linux' ] ; then
				read -p "JVM Patch number: $JVM_PATCH_ID [y or number]: " INPUT
		elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo "JVM Patch number: $JVM_PATCH_ID [y or number]: \c"
				read INPUT? 
		fi
		
		if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
			echo "Confirmed default jvm patch number: $JVM_PATCH_ID"
		else
			JVM_PATCH_ID=$INPUT
			echo "JVM Patch number: $DB_PATCH_ID"
		fi
#		echo "\007"

		if [ "$DB_TYPE" == "rac" ] ; then
			echo 
			echo "Setting GRID/DATABASE Patch binary location for all Nodes"
			echo "==================================================================================="
		
		i=0
		for NODENAME in $NODE_LIST
		do
			if [ "$OS_TYPE" == 'Linux' ] ; then
					read -p "Patch location for Node [${HOSTNAME_REMOTE_N[$i]}]: [${PATCH_LOC_REMOTE_N[$i]}]: " INPUT
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
					echo "Patch location for Node [${HOSTNAME_REMOTE_N[$i]}]: [${PATCH_LOC_REMOTE_N[$i]}]: \c"
					read INPUT? 
			fi	
			
			if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
				echo "Patch location for Node [${HOSTNAME_REMOTE_N[$i]}]: ${PATCH_LOC_REMOTE_N[$i]}"
			else
				PATCH_LOC_REMOTE_N[$i]=$INPUT
				echo "Patch location for Node [${HOSTNAME_REMOTE_N[$i]}]: ${PATCH_LOC_REMOTE_N[$i]}"
			fi
			i=$i+1
		done

		elif [ "$DB_TYPE" == "nonrac" ] ; then	
			echo 
			echo "Setting Oracle patch location"
			echo "==================================================================================="
					
			if [ "$OS_TYPE" == 'Linux' ] ; then
				read -p "Patch location: [$PATCH_LOCATION]: " INPUT
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo "Patch location: [$PATCH_LOCATION]: \c" 
				read INPUT? 
			fi
			if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
				echo "Confirmed default patch location: $PATCH_LOCATION"
			else
				PATCH_LOCATION=$INPUT
				echo "Patch location: $PATCH_LOCATION"
			fi
			
			cd $PATCH_LOCATION
			if [ -z "$( ls -lrt | grep $DB_PATCH_ID )" ] ; then
				echo "DB patch location is not valid."
				patch_loc_validation
			fi

			if [ -z "$( ls -lrt | grep $JVM_PATCH_ID )" ] ; then
				echo "JVM patch location is not valid."
				patch_loc_validation
			fi	
		fi
		
	home_backup_location
	
		iPATCH_LOCATION=1
fi
}
# Check Prerequisite
function check_prerequisite
{

   if [ "$iCHECK_PRE" == 1 ] ; then 
      return 1
   fi

   echo "Prerequsite Check:"
   echo "==================================================================================="
	
	if [ "$OS_TYPE" == 'Linux' ] ; then
		read -p "Has application been stopped?(y|n)[y]: " INPUT
	elif [ "$OS_TYPE" == 'AIX' ] ; then	
		echo "Has application been stopped?[y]: \c"
		read INPUT? 
	fi   
	if [ "$INPUT" == "n" ] || [ "$INPUT" == "no" ] ; then
		echo "Please contact with applicaton team to stop application before procced...[y]"
		quit     
	else 
		echo "Thanks for confirmation..."
	fi
   
	if [ "$OS_TYPE" == 'Linux' ] ; then
		read -p "Have you created blackout and stopped OEM agent and ITM?(y|n) [y]: " INPUT
	elif [ "$OS_TYPE" == 'AIX' ] ; then	
		echo "Have you created blackout and stopped OEM agent and ITM? (y|n) [y]: \c"
		read INPUT? 
	fi  
	
	if [ "$INPUT" == "n" ] || [ "$INPUT" == "no" ] ; then
		echo "Please create blackout and stop OEM agent before procced..."
		quit     
	else 
		echo "Thanks for confirmation..."
	fi
#   echo -en "\007"
	
	echo 	
	if [ "$OS_TYPE" == 'Linux' ] ; then
		read -p "Do you have enough free space on GRID/ORACLE Home filesystem [/u001 or /u01]? (y|n) [y]: " INPUT
	elif [ "$OS_TYPE" == 'AIX' ] ; then	
		echo "Do you have enough free space on GRID/ORACLE Home filesystem [/u001 or /u01]? (y|n) [y]: \c"
		read INPUT? 
	fi  
	
	if [ "$INPUT" == "n" ] || [ "$INPUT" == "no" ] ; then
		echo "Patching task will be failed...Try to make some room on /u001 or /u01"
		return 1
	else
		echo "Thanks for confirmation..."
	fi
#   echo -en "\007"  

	echo 
	if [ "$OS_TYPE" == 'Linux' ] ; then
		read -p "Database Version:11g|12c: [$DB_VERSION] :" INPUT
	elif [ "$OS_TYPE" == 'AIX' ] ; then	
		echo "Database Version:11g|12c: [$DB_VERSION] : \c"
		read INPUT? 
	fi  
	   
	if [ "$INPUT" == "11g" ] || [ "$INPUT" == "12c" ] || [ "$INPUT" == "y" ] ; then
		DB_VERSION=$INPUT
		echo "DB Version: $DB_VERSION"
	elif [ "${INPUT##*( )}" == "" ] ; then
		echo "Confirmed Default DB Version: $DB_VERSION "
	else 
		echo "Wrong Input...bye.." 
		quit
	fi
   
}
function set_home_env
{

 if [ "$iSET_HOME_ENV" == 0 ] ; then
# 	   echo -en "\007"
	   
		if [ "$INPUT" == "nonrac" ] ; then	
			
			if [ "$OS_TYPE" == 'Linux' ] ; then
				read -p "ORACLE HOME PATH: [$ORACLE_HOME]?: " INPUT
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo "ORACLE HOME PATH: [$ORACLE_HOME]?: \c"
				read INPUT? 
			fi
			
			if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
				echo "Confirmed default ORACLE_HOME: $ORACLE_HOME"
			else
				ORACLE_HOME=$INPUT
				echo "ORACLE_HOME: $ORACLE_HOME"
			fi
		elif [ "$INPUT" == "rac" ] ; then
			if [ "$OS_TYPE" == 'Linux' ] ; then
				read -p "ORACLE HOME PATH: [$ORACLE_HOME]?: " INPUT
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo "ORACLE HOME PATH: [$ORACLE_HOME]?: \c"
				read INPUT? 
			fi
			
			if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
				echo "Confirmed default ORACLE_HOME: $GORACLE_HOME"
			else
				GORACLE_HOME=$INPUT
				echo "ORACLE_HOME: $ORACLE_HOME"
			fi
			
			if [ "$OS_TYPE" == 'Linux' ] ; then
				read -p "GRID HOME PATH: [$GGRID_HOME]?: " INPUT
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo "GRID HOME PATH: [$GGRID_HOME]?: \c"
				read INPUT? 
			fi		
			
			if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
				echo "Confirmed default GRID_HOME: $GGRID_HOME"
			else
				GGRID_HOME=$INPUT
				echo "GRID_HOME: $GGRID_HOME"
			fi
	   fi

	   if [ "$OS_TYPE" == 'Linux' ] ; then
			read -p "Database List on this Host [enter DB name with space]: [$NAME_LIST] :" INPUT
		elif [ "$OS_TYPE" == 'AIX' ] ; then	
			echo "Database List on this Host [enter DB name with space]: [$NAME_LIST] : \c"
			read INPUT? 
		fi
	   
	   if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
		  echo "Confirmed DB List: $NAME_LIST"
	   else 
		  NAME_LIST=$INPUT 
		  echo "Confirmed DB List: $NAME_LIST"
	   fi
	   
	   iSET_HOME_ENV=1
   fi 
}
# Set Environment Variable
function set_env_variable
{
	if [ "$iSETENV" == 1 ] ; then
		return 1
	fi

	echo 
	echo "Please set/confirm environment variable"
	echo "============================================================================================" 
	echo 
	
	OPATCH=$ORACLE_HOME/OPatch/opatch
	echo "OPatch Details:"
	echo "============================================================================================" 
	echo 
	
	export PATH=$ORACLE_HOME/OPatch:$PATH
	which opatch   
	$OPATCH version
   
	if [ "$OS_TYPE" == 'Linux' ] ; then
		read -p "Please verify required opatch version from readme file otherwise you will encounter error during patching. want to continue? (y|n) [y]: " INPUT
	elif [ "$OS_TYPE" == 'AIX' ] ; then	
		echo "Please verify required opatch version from readme file otherwise you will encounter error during patching. want to continue? (y|n) [y]: \c"
		read INPUT? 
	fi		
   
	if [ "${INPUT##*( )}" == "" ] || [ "$INPUT" == "y" ] ; then
		echo "Thanks for confirmation..."
	else
		echo "Update the OPatch version then start again..."
		quit
	fi
  
	patch_loc_validation
}

function stop_all_services
{
	shutdown_db_normal
	rac_shutdown		
	
}

function start_all_services
{
	rac_startup
	startup_db_normal	
}

function rac_home_backup
{
	if [ "$DB_TYPE" == "rac" ] ; then
		home_backup
	elif [ "$DB_TYPE" == "nonrac" ] ; then
		home_backup
	fi
}
function manu
{
################
#	clear
	
		if [ "$DB_TYPE" == "nonrac" ] ; then			
			
			if [ "$OS_TYPE" == 'Linux' ] ; then
				echo -e "\033[32m";
				read -p "Choose Option: [db_startup|db_shutdown|home_backup|patch_prerequisite|patch_apply|post_patch|run_utlrp|patch_rollback|postpatch_rollback|validation|quit]: " option					
				echo -e "\033[00m";
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo "\033[32m";
				echo "Choose Option: [db_startup|db_shutdown|home_backup|patch_prerequisite|patch_apply|post_patch|run_utlrp|patch_rollback|postpatch_rollback|validation|quit]: \c" 
				read option? 
				echo "\033[00m";
			fi
		
		elif [ "$DB_TYPE" == "rac" ] || [ "${DB_TYPE##*( )}" == "" ] || [ "$DB_TYPE" == "y" ] ; then			
			if [ "$OS_TYPE" == 'Linux' ] ; then
				echo -e "\033[32m";
				read -p "Choose Option: [service_start|service_stop|home_backup|patch_prerequisite|patch_apply|post_patch|run_utlrp|patch_rollback|postpatch_rollback|validation|quit]: " option	
				echo -e "\033[00m";	
			elif [ "$OS_TYPE" == 'AIX' ] ; then	
				echo "\033[32m";
				echo "Choose Option: [service_start|service_stop|home_backup|patch_prerequisite|patch_apply|post_patch|run_utlrp|patch_rollback|postpatch_rollback|validation|quit]: \c"
				read option? 
				echo "\033[00m";
			fi
#			echo -e "\033[00m";			
		else
			echo "Wrong Input....Pleae try again..."
			#set_home_env
			return 1
		fi

################

  if [ "${option##*( )}" == "" ] ; then
    echo "Wrong option selected. Please try again..."
    manu
  fi

  #case "$1" in
  case "$option" in
    'db_startup')
      startup_db_normal
      manu
;;
	'service_stop')
      stop_all_services
      manu
;;
	'service_start')
      start_all_services
      manu
;;
    'run_utlrp')
      run_utlrp
      manu
;;
    'validation')
      check_db_registry
      manu
;;
    'db_shutdown')
      shutdown_db_normal 
      manu
;;
    'home_backup')
      rac_home_backup
      manu
;;
    'patch_prerequisite')
      patch_check
      manu
;;
    'patch_apply')
      patch_apply
      manu
;;
    'post_patch')
      psu_apply
      manu
;;
    'patch_rollback')
      patch_rollback
      manu
;;
    'postpatch_rollback')
      psu_rollback
      manu
;; 
    'quit')
     quit 
;;
*)
    echo "Wrong option selected. Please try again..."
    manu

esac

}

function quit
{
   echo "Thank you so much for using Oracle DB Patching automation script..."
   exit 1
}
##############################################################################################################
# 														main							                     #
##############################################################################################################

# Must NOT be run as root
if [ "$(id -n -u)" != "oracle" ]
then
   echo "This script must be run as oracle" 1>&2
   exit 1
fi

clear

OS_PRINT=0

echo "Loading initial variable..."
echo "============================================================================================" 

HOSTNAME=`hostname | awk -F . '{ print $1 }'`
echo "Hostname: $HOSTNAME"

OS_TYPE=`uname -a | awk ' { print $1} '`
echo "OS: $OS_TYPE"

ORACLE_HOME=`cat /etc/oratab | grep -v ASM | grep -v agent | grep "^[^#;]" | grep "^[^-;]" | awk '{ print $1 }' | cut -d ':' -f2 | sed 's/:\{1\}$//' | sort -u`
GORACLE_HOME=$ORACLE_HOME


ORACLE_HOME_COUNT=`cat /etc/oratab | grep -v ASM | grep -v agent | grep "^[^#;]" | grep "^[^-;]" | awk '{ print $1 }' | cut -d ':' -f2 | sed 's/:\{1\}$//' | sort -u | wc -l`

ORACLE_HOME_COUNT1=1

if [ "$ORACLE_HOME_COUNT" -gt "$ORACLE_HOME_COUNT1" ] ; then
	GORACLE_HOME=${ORACLE_HOME}
	echo "ORACLE_HOME:"
	echo "$GORACLE_HOME"

	if [ "$OS_TYPE" == 'Linux' ] ; then
		read -p "Found more than one Oracle Home. Please select [patching will support one home]: " INPUT					
	elif [ "$OS_TYPE" == 'AIX' ] ; then	
		echo "Found more than one Oracle Home. Please select [patching will support one home]: \c" 
		read INPUT? 				
	fi
	ORACLE_HOME=$INPUT
	GORACLE_HOME=$INPUT
fi 

echo "ORACLE_HOME: $GORACLE_HOME"

HOME_BACKUP=/u01/software/patches/jan19
HOME_BACKUP_LOCAL=/u01/software/patches/jan19/28980120
echo "Oralce home backup: $HOME_BACKUP"

OPATCH=$ORACLE_HOME/OPatch/opatch
echo "OPatch Path: $OPATCH"

#JRE=`ls -l $ORACLE_HOME|grep jre1|awk -F" " '{ print $9 }'`
PATCH_LOCATION=/u01/software/patches/jan19/28980120
echo "Patch Location: $PATCH_LOCATION"

DB_PATCH_ID=28813884
echo "DB Patch #: $DB_PATCH_ID"

JVM_PATCH_ID=28790654
echo "JVM Patch #: $JVM_PATCH_ID"

DATENTIME=`date '+%d_%m_%Y_%H_%M_%S'`
TDATE=`date '+%m%d%y'`

DB_VERSION=`$GORACLE_HOME/OPatch/opatch lsinventory | awk '/^Oracle Database/ {print $NF}' | cut -d '.' -f1`
DB_VERSION1=g
DB_VERSION2=c

if [ "$DB_VERSION" == '12' ] ; then
	DB_VERSION=$DB_VERSION"c"
elif [ "$DB_VERSION" == '11' ] ; then	
	DB_VERSION=$DB_VERSION"g"
fi 
echo "DB Version: $DB_VERSION"

DB_ROLE=primary
echo "DB Role: $DB_ROLE"

vINSTANCE_ID=''

# All Flag Variable 
iHOME_BK=0 
iPATCH_LOCATION=0
iCHECK_PRE=0
iSETENV=0
iSETRACHOME=0
iSET_HOME_ENV=0
iSTARTUP_DB_INST='normal'
iSHUTDOWN_DB_INST='normal'
iSHUTDOWN=0
iSTARTUP=0
iSHUTDOWN_UNLOCK=0
iSTARTUP_PATCH=0

iCOUNT=0

if [ "$OS_TYPE" == 'Linux' ] ; then
	export iECHO=-e
elif [ "$OS_TYPE" == 'AIX' ] ; then	
	export iECHO=""
fi
	
if [ "$OS_TYPE" == 'Linux' ] ; then
	export iSOUND=-en
elif [ "$OS_TYPE" == 'AIX' ] ; then	
	export iSOUND=""
fi
	
# Host Name for RAC

HOSTNAME_LOCAL_NODE=`hostname | awk -F . '{print $1}'`

GASM_INSTANCE=`cat /etc/oratab | grep +ASM | awk '{ print $1 }' | cut -d ':' -f1 | sed 's/:\{1\}$//' | sort -u`

if [ "${GASM_INSTANCE##*( )}" != "" ] || [ ! -z "$GASM_INSTANCE" ]; then	  
#	set_grid_env $GGRID_HOME $GASM_INSTANCE	

# cat inventory file to fine node list
	INVENTORY_LOC=`find / -name inventory.xml 2>/dev/null | grep oraInventory/ContentsXML/inventory.xml`
	
#	NODE_NUMBER=`olsnodes -n | wc -l`
	NODE_NUMBER=''
	DB_TYPE=rac

	#LOCAL_NODE	
	GGRID_HOME=`cat /etc/oratab | grep +ASM | awk '{ print $1 }' | cut -d ':' -f2 | sed 's/:\{1\}$//' | sort -u`
	GASM_INSTANCE=`cat /etc/oratab | grep +ASM | awk '{ print $1 }' | cut -d ':' -f1 | sed 's/:\{1\}$//'  | sort -u`
	NODE_POSITION=`cat /etc/oratab | grep +ASM | awk '{ print $1 }' | cut -d ':' -f1 | sed 's/:\{1\}$//' | tr -cd '[[:digit:]]'`
	GORACLE_HOME=`cat /etc/oratab | grep -v ASM | grep -v agent | grep "^[^#;]" | grep "^[^-;]" | awk '{ print $1 }' | cut -d ':' -f2 | sed 's/:\{1\}$//' | sort -u`
	
#	NODE_NUMBER=`olsnodes -n | wc -l`
	NODE_NUMBER=`cat "$INVENTORY_LOC" | grep "NODE NAME" | awk '!seen[$0]++' | cut -d '"' -f2 | sed 's/"\{1\}$//' | wc -l`
	NODE_LIST=`cat $INVENTORY_LOC | grep "NODE NAME" | awk '!seen[$0]++' | cut -d '"' -f2 | sed 's/"\{1\}$//' | awk '{print}' ORS=' '`

	PATCH_LOC_LOCAL_NODE=/u01/software/patches/jan19/28980120
	
	
	SHUTDOWN_MODE=normal
	START_MODE=normal

	#EMOTE_NODE
	HOSTNAME_REMOTE_N1=''
	HOSTNAME_REMOTE_N2=''
	HOSTNAME_REMOTE_N3=''
	
	HOSTNAME_REMOTE_N[0]=''
	HOSTNAME_REMOTE_N[1]=''
	HOSTNAME_REMOTE_N[2]='';
	HOSTNAME_REMOTE_N[3]=''
	
	PATCH_LOC_REMOTE_N[0]=/u01/software/patches/jan19/28980120
	PATCH_LOC_REMOTE_N[1]=''
	PATCH_LOC_REMOTE_N[2]=''
	PATCH_LOC_REMOTE_N[3]=''
	
	HOME_BACKUP_REMOTE_N[0]=''
	HOME_BACKUP_REMOTE_N[1]=''
	HOME_BACKUP_REMOTE_N[2]=''
	HOME_BACKUP_REMOTE_N[3]=''
	
	
	i=0
	for NODENAME in $NODE_LIST
	do
		HOSTNAME_REMOTE_N[$i]=$NODENAME
		echo "NODE$(($i+1)) Hostname: ${HOSTNAME_REMOTE_N[$i]}"
		i=$i+1
	done
   	
	echo "LOCAL NODE : $HOSTNAME_LOCAL_NODE"
	echo "GRID HOME: $GGRID_HOME"
	echo "ORACLE_HOME: $GORACLE_HOME"
else
	DB_TYPE=nonrac
fi

echo "DB TYPE: $DB_TYPE"

if [ "$OS_TYPE" == "Linux" ] ; then 
	LISTENER_NAME=`ps -ef | grep tns | grep $ORACLE_HOME | grep -v grep | awk '{ print $9 }'`
elif [ "$OS_TYPE" == "AIX" ] ; then
	LISTENER_NAME=`ps -ef | grep tns | grep $ORACLE_HOME | grep -v grep | awk '{ print $10 }'`
fi

if [ "$DB_TYPE" == "rac" ] ; then
	NAME_LIST=`cat /etc/oratab | grep -v ASM | grep "^[^#;]" | grep -v agent | grep $GORACLE_HOME | grep "^[^-;]" | awk '{ print $1 }' | cut -d ':' -f1 | sed 's/[0-9]*//g' | sort -u | awk '{print}' ORS=' '`
	echo "Database List on this server: $NAME_LIST"
else 
	NAME_LIST=`cat /etc/oratab | grep -v ASM | grep "^[^#;]" | grep -v agent | grep $GORACLE_HOME | grep "^[^-;]" | awk '{ print $1 }' | cut -d ':' -f1 | sed 's/:\{1\}$//' | awk '{print}' ORS=' '`
	echo "Database List on this server: $NAME_LIST"
	echo "Listener List: ${LISTENER_NAME}"
fi
manu

If you need this script please make a request on below message with your email address.

Leave a Reply