Serviceguard stand up Notes:<\/p>\n
Important parts of the .ascii file.<\/p>\n
CLUSTER_NAME prod_sgcluster<\/p>\n
HOSTNAME_ADDRESS_FAMILY IPV4<\/p>\n
FIRST_CLUSTER_LOCK_VG \/dev\/vgsgpackagename<\/p>\n
## Is it safer to maybe use a dedicated SG Lock disk.<\/p>\n
NODE_NAME sgclusternode01<\/p>\n
NETWORK_INTERFACE lan900<\/p>\n
HEARTBEAT_IP 192.168.208.209<\/p>\n
NETWORK_INTERFACE lan901<\/p>\n
HEARTBEAT_IP 10.43.128.209<\/p>\n
# CLUSTER_LOCK_LUN<\/p>\n
FIRST_CLUSTER_LOCK_PV \/dev\/disk\/disk20<\/p>\n
# Link Aggregate lan900 contains the following port(s): lan0,lan4<\/p>\n
# Warning: There are no standby network interfaces for lan900.<\/p>\n
# Link Aggregate lan901 contains the following port(s): lan8,lan12<\/p>\n
# Warning: There are no standby network interfaces for lan901.<\/p>\n
NODE_NAME sgclusternode02<\/p>\n
NETWORK_INTERFACE lan900<\/p>\n
HEARTBEAT_IP 192.168.209.98<\/p>\n
NETWORK_INTERFACE lan901<\/p>\n
HEARTBEAT_IP 10.43.128.210<\/p>\n
# CLUSTER_LOCK_LUN<\/p>\n
FIRST_CLUSTER_LOCK_PV \/dev\/disk\/disk59<\/p>\n
MEMBER_TIMEOUT 14000000<\/p>\n
# Configuration\/Reconfiguration Timing Parameters (microseconds).<\/p>\n
AUTO_START_TIMEOUT 600000000<\/p>\n
NETWORK_POLLING_INTERVAL 2000000<\/p>\n
NETWORK_FAILURE_DETECTION INOUT<\/p>\n
# NETWORK_AUTO_FAILBACK<\/p>\n
# When set to YES a recovery of the primary LAN interface will cause failback<\/p>\n
# from the standby LAN interface to the primary.<\/p>\n
# When set to NO a recovery of the primary LAN interface will do nothing and<\/p>\n
# the standby LAN interface will continue to be used until cmmodnet -e lanX<\/p>\n
# is issued for the primary LAN interface.<\/p>\n
NETWORK_AUTO_FAILBACK YES<\/p>\n
SUBNET 192.168.208.0<\/p>\n
IP_MONITOR ON<\/p>\n
POLLING_TARGET 192.168.208.1<\/p>\n
SUBNET 192.168.0.0<\/p>\n
IP_MONITOR OFF<\/p>\n
# Package Configuration Parameters.<\/p>\n
# Enter the maximum number of packages which will be configured in the cluster.<\/p>\n
# You can not add packages beyond this limit.<\/p>\n
# This parameter is required.<\/p>\n
MAX_CONFIGURED_PACKAGES 300<\/p>\n
You will want to make sure both nodes of the cluster are running and the volume group that the lock disks belongs to is cluster aware.<\/p>\n
NOTE: When standing up a one node cluster cmquerycl will remove lock disk configuration and a lock disk is not needed for such a configuration.<\/p>\n
cmrunnode #on each node<\/p>\n
vgchange \u2013a n vgsgpackagename<\/p>\n
vgchange \u2013c y vgsgpackagename<\/p>\n
cmquerycl -v -C prod_sgcluster.ascii -n sgclusternode01 -n sgclusternode02<\/p>\n
cmcheckconf -v -C prod_sgcluster.ascii<\/p>\n
cmapplyconf -v -C prod_sgcluster.ascii<\/p>\n
Cluster should look roughly like this:<\/p>\n
root@sgclusternode02:\/root> cmviewcl<\/p>\n
CLUSTER STATUS<\/p>\n
prod_sgcluster up<\/p>\n
NODE STATUS STATE<\/p>\n
sgclusternode01 up running<\/p>\n
sgclusternode02 up running<\/p>\n
PACKAGE STATUS STATE AUTO_RUN NODE<\/p>\n
root@sgclusternode02:\/root><\/p>\n
Standing up your first package.<\/p>\n
Scenario assumes you are standing up a package named sgpackagename.<\/p>\n
mkdir \u2013p \/etc\/cmcluster\/packages\/sgpackagename<\/p>\n
cd \/etc\/cmcluster\/packages\/sgpackagename<\/p>\n
Using your method of choice, copy in a package conference file and a task0 (external script to start Oracle. Choose your clone source carefully. Make sure you choose a working package to template. If you are standing up an SAP instance choose an SAP package to clone, sgpackagename is a standard database package and a better template candidate.<\/p>\n
Permissions may matter:<\/p>\n
oot@sgclusternode02:\/etc\/cmcluster\/packages\/sgpackagename> ll<\/p>\n
total 176<\/p>\n
-rw——- 1 root sys 71668 Dec 3 16:12 sgpackagename.conf<\/p>\n
-r-x—— 1 root sys 11005 Dec 9 11:35 task0<\/p>\n
root@sgclusternode02:\/etc\/cmcluster\/packages\/sgpackagename><\/p>\n
Use the method of choice to modify your configuration files:<\/p>\n
Here is what needs to be changed ( In bold ):<\/p>\n
package_name sgpackagename<\/p>\n
package_description “sgpackagename Instance”<\/p>\n
package_type failover<\/p>\n
node_name sgclusternode01<\/p>\n
node_name sgclusternode02<\/p>\n
ip_subnet 192.168.208.0<\/p>\n
ip_address 192.168.209.48<\/p>\n
Change the bold values below for each file system.<\/p>\n
fs_name \/dev\/vgsgpackagename\/oraarch<\/p>\n
fs_server “”<\/p>\n
fs_directory \/data\/oracle\/sgpackagename\/arch<\/p>\n
fs_type “vxfs”<\/p>\n
fs_mount_opt “-o ioerror=mwdisable,largefiles,delaylog,cio”<\/p>\n
fs_umount_opt “”<\/p>\n
fs_fsck_opt “”<\/p>\n
The actual data may vary. The fs_mount_opt are important and may change. The parameter cio is necessary for Oracle to have asynchronous I\/O.<\/p>\n
external_script \/etc\/cmcluster\/packages\/sgpackagename\/task0<\/p>\n
Create the mount points and change permission to oracle:dba. Again there are many ways to do this, your mileage may vary. You can check return codes if you are particularly precise, echo $?<\/p>\n
awk ‘\/^fs_dir\/{print $NF}’ sgpackagename.conf | while read \u2013r mp<\/p>\n
do<\/p>\n
mkdir \u2013p $mp<\/p>\n
ll \u2013d $mp<\/p>\n
chown oracle:dba $mp<\/p>\n
ll \u2013d $mp<\/p>\n
done<\/p>\n
Your final output from the ll \u2013d command should look this<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/arch<\/p>\n
drwxr-xr-x 4 oracle dba 96 Dec 10 11:40 \/data\/oracle\/sgpackagename\/export<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/undo1<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/redo1<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/redo2<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/ctrl1<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/ctrl2<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/data1<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/index1<\/p>\n
drwxr-xr-x 4 oracle dba 96 Dec 10 13:59 \/apps\/oracle\/sgpackagename<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/system<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/temp1<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/users<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/tools<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/fra<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/diag<\/p>\n
drwxr-xr-x 3 oracle dba 96 Dec 9 11:04 \/data\/oracle\/sgpackagename\/admin<\/p>\n
edit the task0 file<\/p>\n
function start_commmand<\/p>\n
su oracle -c “\/apps\/oracle\/general\/sh\/change_backup_mode.ksh sgpackagename cold end”<\/p>\n
Note this section is different for SAP. Change the instance name.<\/p>\n
function stop_command<\/p>\n
#### Database startup script, normally created by dba#####<\/p>\n
su oracle -c “\/apps\/oracle\/general\/sh\/change_backup_mode.ksh <\/b>sgpackagename cold begin”<\/p>\n
Comment out (in bold) the start\/stop code so first package run only stands up the filesystems:<\/p>\n
case ${1} in<\/p>\n
start)<\/p>\n
# start_command $*<\/p>\n
# exit_val=$?<\/p>\n
;;<\/p>\n
stop)<\/p>\n
# stop_command $*<\/p>\n
# exit_val=$?<\/p>\n
;;<\/p>\n
Detect your storage from storage system.<\/p>\n
ioscan \u2013fnNC disk<\/p>\n
# detects the storage<\/p>\n
For this example we detected the following agile devices:<\/p>\n
disk59<\/p>\n
disk60<\/p>\n
disk61<\/p>\n
disk62<\/p>\n
disk63<\/p>\n
disk64<\/p>\n
disk65<\/p>\n
disk66<\/p>\n
get the EMC serial number of each of the disks and match it against the serial number list given to you by storage in the dashboard request.<\/p>\n
This is a bit tedious so I created a simple script for the job:<\/p>\n
dislev.ksh<\/p>\n
#!\/usr\/bin\/ksh<\/p>\n
args=$#<\/p>\n
if [ $args -ne 1 ];then echo “1 argument last part of disk ex disk60”;exit 1;fi<\/p>\n
dv=$1<\/p>\n
diskinfo \/dev\/rdisk\/${dv}<\/p>\n
ldev=$(\/usr\/bin\/inq -nodots -sym_wwn | grep “$dv ” | grep -v rdsk |awk ‘{print $3}’ |awk ‘{print substr($0,length($0) – 3,length($0))}’);<\/p>\n
echo “LDEV is: ${ldev}”<\/p>\n
root@sgclusternode02:\/root\/shuffle> .\/disldev.ksh disk59<\/p>\n
SCSI describe of \/dev\/rdisk\/disk59:<\/p>\n
vendor: EMC<\/p>\n
product id: SYMMETRIX<\/p>\n
type: direct access<\/p>\n
size: 154176000 Kbytes<\/p>\n
bytes per sector: 512<\/p>\n
LDEV is: 4F2F<\/p>\n
The size is a reality check. You can also use inq utility to get the size. By using a second utility (diskinfo) this creates a natural double check. There are probably a dozen different right ways to complete this task.<\/p>\n
My disk list is kept for verification purposes in a text file root@sgclusternode02:\/root\/shuffle> cat request.610.dlist.txt<\/p>\n
610 is the request from the stand up rquest.<\/p>\n
awk \u2018{print $NF}\u2019 request.610.dlist.txt | while read \u2013r dvn<\/p>\n
do<\/p>\n
pvcreate $dvn<\/p>\n
rc=$?<\/p>\n
echo “RC from pvcreate is ${rc}”<\/p>\n
done<\/p>\n
Create the volume group using LVM 2.2<\/p>\n
vgcreate \u2013V <version number> -s <pe size> -S <Max size in TB> <volume group name> <disk list><\/p>\n
vgcreate -V 2.2 -s 32 -S 2t vgsgpackagename \/dev\/disk\/disk59 \u2026. \/dev\/disk\/disk66<\/p>\n
The next step is best scripted:<\/p>\n
<command> -n name \u2013L <size in MB> <volume group name><\/p>\n
lvcreate -n oraadmin -L 10000 vgsgpackagename<\/p>\n
echo $?<\/p>\n
newfs -F vxfs -o largefiles \/dev\/sgpackagename\/roraadmin<\/p>\n
echo $?<\/p>\n
.. repeat for every file system defined in the storage dashboard request .. I have scripted this entire process based on dashboard input and will provide the script as an appendix.<\/p>\n
Volume group must be exported in preview mode:<\/p>\n
vgexport -v -s -p -m vgsgpackagename.map vgsgpackagename<\/p>\n
Use scp to copy the map file to the failover node<\/p>\n
On failover node:<\/p>\n
vgimport -v -s -N -m vgsgpackagename.map vgsgpackagename<\/p>\n
vgchange \u2013a y vgsgpackagename<\/p>\n
vgdisplay \u2013v vgsgpackagename (Look for logical volumes disks being correct)<\/p>\n
vgchange \u2013a n vgsgpackagename<\/p>\n
vgchange \u2013c y vgsgpackagename (Must be successful check return code $? )<\/p>\n
vgchange \u2013c n vgsgpackagename<\/p>\n
On primary node (either will work but being consistent is helpful).<\/p>\n
vgchange \u2013c y vgsgpackagename<\/p>\n
cd \/etc\/cmcluster\/packages\/sgpackagename<\/p>\n
cmcheckconf \u2013P sgpackagename.conf<\/p>\n
(When all cmcheckconf issues are resolved)<\/p>\n
cmapplyconf \u2013P sgpackagename.conf<\/p>\n
cmrunpckg \u2013v \u2013n <node_hostname> sgpackagename<\/p>\n
cmmodpkg \u2013e sgpackagename<\/p>\n
cmviewcl (Look for your package to be running)<\/p>\n
Error logs live: \/var\/adm\/cmcluster\/log\/<packagename>.log<\/p>\n
Setting up asynchronous I\/O<\/p>\n
root@sat1hcmdb090:\/root> getprivgrp<\/p>\n
global privileges: CHOWN<\/p>\n
dba: MLOCK<\/p>\n
getprivgroup is a display command use command:<\/p>\n
setprivgrp dba MLOCK<\/p>\n
Standing up your first package:<\/p>\n
Make sure you are in the directory: \/etc\/cmcluster\/packages\/sgpackagename<\/p>\n
root@sgclusternode02:\/etc\/cmcluster\/packages\/sgpackagename> cmcheckconf -P sgpackagename.conf<\/p>\n
cmcheckconf: Verification completed. No errors found.<\/p>\n
Use the cmapplyconf command to apply the configuration.<\/p>\n
root@sgclusternode02:\/etc\/cmcluster\/packages\/sgpackagename> cmapplyconf -P sgpackagename.conf<\/p>\n
One or more of the specified packages are running. Any error in the<\/p>\n
proposed configuration change could cause these packages to fail.<\/p>\n
Ensure configuration changes have been tested before applying them.<\/p>\n
Modify the package configuration ([y]\/n)? y<\/p>\n
Completed the cluster update<\/p>\n
root@sgclusternode02:\/etc\/cmcluster\/packages\/sgpackagename> cmmodpkg -e sgpackagename<\/p>\n
cmmodpkg: Completed successfully on all packages specified<\/p>\n
root@sgclusternode02:\/etc\/cmcluster\/packages\/sgpackagename> cmviewcl -v<\/p>\n
CLUSTER STATUS<\/p>\n
prod_sgcluster up<\/p>\n
NODE STATUS STATE<\/p>\n
sgclusternode01 up running<\/p>\n
Cluster_Lock_LVM:<\/p>\n
VOLUME_GROUP PHYSICAL_VOLUME STATUS<\/p>\n
\/dev\/vgsgpackagename \/dev\/disk\/disk20 up<\/p>\n
Network_Parameters:<\/p>\n
INTERFACE STATUS PATH NAME<\/p>\n
PRIMARY up LinkAgg0 lan900<\/p>\n
PRIMARY up LinkAgg1 lan901<\/p>\n
NODE STATUS STATE<\/p>\n
sgclusternode02 up running<\/p>\n
Cluster_Lock_LVM:<\/p>\n
VOLUME_GROUP PHYSICAL_VOLUME STATUS<\/p>\n
\/dev\/vgsgpackagename \/dev\/disk\/disk59 up<\/p>\n
Network_Parameters:<\/p>\n
INTERFACE STATUS PATH NAME<\/p>\n
PRIMARY up LinkAgg0 lan900<\/p>\n
PRIMARY up LinkAgg1 lan901<\/p>\n
PACKAGE STATUS STATE AUTO_RUN NODE<\/p>\n
sgpackagename up running enabled sgclusternode02<\/p>\n
Policy_Parameters:<\/p>\n
POLICY_NAME CONFIGURED_VALUE<\/p>\n
Failover configured_node<\/p>\n
Failback manual<\/p>\n
Node_Switching_Parameters:<\/p>\n
NODE_TYPE STATUS SWITCHING NAME<\/p>\n
Primary up enabled sgclusternode01<\/p>\n
Alternate up enabled sgclusternode02 (current)<\/p>\n
Other_Attributes:<\/p>\n
ATTRIBUTE_NAME ATTRIBUTE_VALUE<\/p>\n
Style modular<\/p>\n
Priority no_priority<\/p>\n
Check that your file systems are mounted (About a dozen different ways to do this):<\/p>\n
root@sgclusternode02:\/etc\/cmcluster\/packages\/sgpackagename> bdf | grep sgpackagename<\/p>\n
102400000 91742 95913999 0% \/data\/oracle\/sgpackagename\/arch<\/p>\n
102400000 13178819 83644864 14% \/data\/oracle\/sgpackagename\/export<\/p>\n
102400000 91742 95913999 0% \/data\/oracle\/sgpackagename\/undo1<\/p>\n
10256384 20006 9596612 0% \/data\/oracle\/sgpackagename\/redo1<\/p>\n
10256384 20006 9596612 0% \/data\/oracle\/sgpackagename\/redo2<\/p>\n
10256384 20006 9596612 0% \/data\/oracle\/sgpackagename\/ctrl1<\/p>\n
10256384 20006 9596612 0% \/data\/oracle\/sgpackagename\/ctrl2<\/p>\n
307200000 141939 287866940 0% \/data\/oracle\/sgpackagename\/data1<\/p>\n
204800000 116841 191890469 0% \/data\/oracle\/sgpackagename\/index1<\/p>\n
20480000 3972498 15475861 20% \/apps\/oracle\/sgpackagename<\/p>\n
10256384 20006 9596612 0% \/data\/oracle\/sgpackagename\/system<\/p>\n
102400000 91742 95913999 0% \/data\/oracle\/sgpackagename\/temp1<\/p>\n
10256384 20006 9596612 0% \/data\/oracle\/sgpackagename\/users<\/p>\n
10256384 20006 9596612 0% \/data\/oracle\/sgpackagename\/tools<\/p>\n
102400000 91742 95913999 0% \/data\/oracle\/sgpackagename\/fra<\/p>\n
10256384 20006 9596612 0% \/data\/oracle\/sgpackagename\/diag<\/p>\n
10256384 20006 9596612 0% \/data\/oracle\/sgpackagename\/admin<\/p>\n
root@sgclusternode02:\/etc\/cmcluster\/packages\/sgpackagename> mount | grep sgpackagename<\/p>\n
\/data\/oracle\/sgpackagename\/arch on \/dev\/vgsgpackagename\/oraarch ioerror=mwdisable,largefiles,delaylog,cio,dev=80000002 on Mon Dec 9 11:41:17 2013<\/p>\n
\/data\/oracle\/sgpackagename\/export on \/dev\/vgsgpackagename\/oraexport ioerror=mwdisable,largefiles,delaylog,cio,dev=80000009 on Mon Dec 9 11:41:17 2013<\/p>\n
\/data\/oracle\/sgpackagename\/undo1 on \/dev\/vgsgpackagename\/oraundo1 ioerror=mwdisable,largefiles,delaylog,cio,dev=8000000c on Mon Dec 9 11:41:17 2013<\/p>\n
\/data\/oracle\/sgpackagename\/redo1 on \/dev\/vgsgpackagename\/oraredo1 ioerror=mwdisable,largefiles,delaylog,cio,dev=80000005 on Mon Dec 9 11:41:17 2013<\/p>\n
\/data\/oracle\/sgpackagename\/redo2 on \/dev\/vgsgpackagename\/oraredo2 ioerror=mwdisable,largefiles,delaylog,cio,dev=80000004 on Mon Dec 9 11:41:17 2013<\/p>\n
\/data\/oracle\/sgpackagename\/ctrl1 on \/dev\/vgsgpackagename\/oractrl1 ioerror=mwdisable,largefiles,delaylog,cio,dev=80000006 on Mon Dec 9 11:41:18 2013<\/p>\n
\/data\/oracle\/sgpackagename\/ctrl2 on \/dev\/vgsgpackagename\/oractrl2 ioerror=mwdisable,largefiles,delaylog,cio,dev=80000007 on Mon Dec 9 11:41:18 2013<\/p>\n
\/data\/oracle\/sgpackagename\/data1 on \/dev\/vgsgpackagename\/oradata1 ioerror=mwdisable,largefiles,delaylog,cio,dev=80000003 on Mon Dec 9 11:41:18 2013<\/p>\n
\/data\/oracle\/sgpackagename\/index1 on \/dev\/vgsgpackagename\/oraindex1 ioerror=mwdisable,largefiles,delaylog,cio,dev=80000008 on Mon Dec 9 11:41:18 2013<\/p>\n
\/apps\/oracle\/sgpackagename on \/dev\/vgsgpackagename\/orabin ioerror=mwdisable,largefiles,delaylog,nodatainlog,dev=80000001 on Mon Dec 9 11:41:18 2013<\/p>\n
\/data\/oracle\/sgpackagename\/system on \/dev\/vgsgpackagename\/orasystem ioerror=mwdisable,largefiles,delaylog,cio,dev=8000000a on Mon Dec 9 11:41:18 2013<\/p>\n
\/data\/oracle\/sgpackagename\/temp1 on \/dev\/vgsgpackagename\/oratemp1 ioerror=mwdisable,largefiles,delaylog,cio,dev=8000000b on Mon Dec 9 11:41:18 2013<\/p>\n
\/data\/oracle\/sgpackagename\/users on \/dev\/vgsgpackagename\/orausers ioerror=mwdisable,largefiles,delaylog,cio,dev=8000000d on Mon Dec 9 11:41:18 2013<\/p>\n
\/data\/oracle\/sgpackagename\/tools on \/dev\/vgsgpackagename\/oratools ioerror=mwdisable,largefiles,delaylog,cio,dev=8000000e on Mon Dec 9 11:41:19 2013<\/p>\n
\/data\/oracle\/sgpackagename\/fra on \/dev\/vgsgpackagename\/orafra ioerror=mwdisable,largefiles,delaylog,cio,dev=8000000f on Mon Dec 9 11:41:19 2013<\/p>\n
\/data\/oracle\/sgpackagename\/diag on \/dev\/vgsgpackagename\/oradiag ioerror=mwdisable,largefiles,delaylog,cio,dev=80000010 on Mon Dec 9 11:41:19 2013<\/p>\n
\/data\/oracle\/sgpackagename\/admin on \/dev\/vgsgpackagename\/oraadmin ioerror=mwdisable,largefiles,delaylog,cio,dev=80000011 on Mon Dec 9 11:41:19 2013<\/p>\n
Conduct failover test (Pre-installation)<\/p>\n
root@sgclusternode02:\/etc\/cmcluster\/packages\/sgpackagename> cmhaltpkg -v sgpackagename<\/p>\n
Disabling automatic failover for failover packages to be halted.<\/p>\n
Halting package sgpackagename<\/p>\n
Successfully halted package sgpackagename<\/p>\n
One or more packages or package instances have been halted.<\/p>\n
The failover packages have AUTO_RUN disabled and no new instance can start automatically. To allow automatic start, enable AUTO_RUN via cmmodpkg -e <package_name><\/p>\n
cmhaltpkg: Completed successfully on all packages specified<\/p>\n
root@sgclusternode02:\/etc\/cmcluster\/packages\/sgpackagename> cmviewcl<\/p>\n
CLUSTER STATUS<\/p>\n
prod_sgcluster up<\/p>\n
NODE STATUS STATE<\/p>\n
sgclusternode01 up running<\/p>\n
sgclusternode02 up running<\/p>\n
UNOWNED_PACKAGES<\/p>\n
PACKAGE STATUS STATE AUTO_RUN NODE<\/p>\n
sgpackagename down halted disabled unowned<\/p>\n
root@sgclusternode02:\/etc\/cmcluster\/packages\/sgpackagename> cmrunpkg -v -n sgclusternode01 sgpackagename<\/p>\n
Running package sgpackagename on node sgclusternode01<\/p>\n
Successfully started package sgpackagename on node sgclusternode01<\/p>\n
cmrunpkg: All specified packages are running<\/p>\n
root@sgclusternode02:\/etc\/cmcluster\/packages\/sgpackagename> cmviewcl<\/p>\n
CLUSTER STATUS<\/p>\n
prod_sgcluster up<\/p>\n
NODE STATUS STATE<\/p>\n
sgclusternode01 up running<\/p>\n
PACKAGE STATUS STATE AUTO_RUN NODE<\/p>\n
sgpackagename up running disabled sgclusternode01<\/p>\n
NODE STATUS STATE<\/p>\n
sgclusternode02 up running<\/p>\n
Re run your mount test on the node the package is now running on (see above).<\/p>\n
You may wish to leave package failover set to disabled to avoid surprises.<\/p>\n
Email your DBA give them the file system(s) to do installation. Flip the database dashboard request to “Application installation & configuration”<\/p>\n
Notify your IA\/PM via email.<\/p>\n
Be prepared to run root.sh when your DBA needs it run.<\/p>\n
Steps after DBA completion<\/p>\n
Modify task 0 uncomment the 4 lines of code you commented.<\/p>\n
case ${1} in<\/p>\n
start)<\/p>\n
start_command $*<\/p>\n
exit_val=$?<\/p>\n
;;<\/p>\n
stop)<\/p>\n
stop_command $*<\/p>\n
exit_val=$?<\/p>\n
Check your package changes:<\/p>\n
cmcheckconf \u2013P sgpackagename.conf<\/p>\n
cmapplyconf \u2013P sgpackagename.conf<\/p>\n
Make sure backups are configured before the next step. Also have storage test DR backup of LUNS if applicable (Model and dev usually do not have DR).<\/p>\n
Re-run the failover tests. Check log files:<\/p>\n
\/var\/adm\/cmcluster\/log\/sgpackagename.log<\/p>\n
Check database run status.<\/p>\n
ps \u2013ef | grep sgpackagename<\/p>\n
Lessons learned:<\/p>\n
\u00b7 DBA\u2019s do not necessarily understand clustering. If they say the start\/stop script works but you are getting errors in Serviceguard testing, ask them to check the return code (echo $?). Explain t the DBA that Serviceguard will not function correctly if it receives non-zero return codes.<\/p>\n
\u00b7 Try not to modify task0 to default to zero return codes. This defeats the purpose of Serviceguard checking return codes.<\/p>\n
\u00b7 Never run pvcreate \u2013f (If you feel the need, deconstruct your volume group and use pvremve)<\/p>\n
\u00b7 Never run cmdisklock -f reset <disk device>, this destroys all data on the lock disk and generally there is data on the lock disk. Note: Maybe a better design would be a dedicated lock disk.<\/p>\n
Appendix 1 Additional automation.<\/p>\n
This is optional, but making processes repeatable is important so you might like this.<\/p>\n
While validating the request on the storage dashboard, I noted (and slightly massaged) the data to make it machine (awk) readable. Scripts are local, I use them in \/root\/shuffle.<\/p>\n
root@sgclusternode02:\/root\/shuffle> cat request.610.data.txt<\/p>\n
sgclusternode01|NEW|\/apps\/oracle\/sgpackagename | 20 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/arch | 100 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/data1 | 300 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/redo2 | 10 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/redo1 | 10 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/ctrl1 | 10 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/ctrl2 | 10 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/index1 | 200 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/export | 100 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/SYSTEM | 10 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/temp1 | 100 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/undo1 | 100 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/users | 10 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/tools | 10 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/fra | 100 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/diag | 10 GB<\/p>\n
sgclusternode01|NEW|\/data\/oracle\/sgpackagename\/admin | 10 GB<\/p>\n
The following script reads the earlier request.{request number}.dlist.txt file to get its device list, reads the data.txt file and completely automates the VG build including the map export.<\/p>\n
Script requires the instance name (not the vg name, it does that itself) and the request number from the dashboard. Use it or not, the choice is yours.<\/p>\n
#!\/usr\/bin\/ksh<\/p>\n
args=$#<\/p>\n
if [ $args -ne 2 ];then echo “2 arguments instance name and request ### required”;exit 1;fi<\/p>\n
inn=$1<\/p>\n
vgn=”vg${inn}”<\/p>\n
req=$2<\/p>\n
disklist=”request.${req}.dlist.txt”<\/p>\n
actonreturn()<\/p>\n
{<\/p>\n
rcc=$1<\/p>\n
if [ $rcc -ne 0 ];then echo “FAIL FAIL $0 FAILED”; exit 1;fi<\/p>\n
}<\/p>\n
devl=””<\/p>\n
# pvcreate<\/p>\n
awk ‘{ print $1 }’ ${disklist} | while read -r dv<\/p>\n
do<\/p>\n
devr=”\/dev\/rdisk\/${dv}”<\/p>\n
echo “pvcreate ${devr}”<\/p>\n
pvcreate ${devr}<\/p>\n
rc=$?<\/p>\n
echo “pvcreate rc:${rc}”<\/p>\n
actonreturn ${rc}<\/p>\n
devl=”$devl \/dev\/disk\/${dv}”<\/p>\n
done<\/p>\n
#vgcreate -V 2.2 -s 32 -S 1t <vg name> disks in \/dev\/disk\/disk### format<\/p>\n
echo “vgcreate -V 2.2 -s 32 -S 1t ${vgn} ${devl}”<\/p>\n
vgcreate -V 2.2 -s 32 -S 2t ${vgn} ${devl}<\/p>\n
rc=$?<\/p>\n
echo “vgcreate rc:${rc}”<\/p>\n
actonreturn ${rc}<\/p>\n
infile=”request.${req}.data.txt”<\/p>\n
if [ -f ${infile} ]<\/p>\n
then<\/p>\n
echo “- pass – file ${infile} found”<\/p>\n
else<\/p>\n
echo “- FAIL – file ${infile} MIA”<\/p>\n
exit 1<\/p>\n
fi<\/p>\n
fstabstub=fstab.${vgn}<\/p>\n
echo “##### New Oracle Instance vg ${vgn} ####” > ${fstabstub}<\/p>\n
innu=$(echo ${inn} | awk ‘{print toupper($0)}’);<\/p>\n
echo “Instance Name: ${inn} ${innu}”<\/p>\n
awk ‘{print}’ ${infile} | while read -r DL<\/p>\n
do<\/p>\n
# echo “Data Line ${DL}”<\/p>\n
mp=$(echo ${DL} | awk -F\\| ‘{ print $3}’);<\/p>\n
# echo “mount point is ${mp}”<\/p>\n
mpszg=$(echo ${DL} | awk ‘{ print $3}’);<\/p>\n
mpszm=”${mpszg}000″<\/p>\n
# echo “mount point size in GB ${mpszg} in fake MB ${mpszm}”<\/p>\n
mpname=$(echo ${mp} | awk -F\\\/ ‘{print $NF}’);<\/p>\n
if [ “${mpname}” = “${innu}” ]<\/p>\n
then<\/p>\n
mpname=”orabin”<\/p>\n
else<\/p>\n
mpname=$(echo ${mpname} | awk ‘{print tolower($0)}’);<\/p>\n
mpname=”ora${mpname}”<\/p>\n
fi<\/p>\n
mkdir -p ${mp}<\/p>\n
chown oracle:dba ${mp}<\/p>\n
# echo “mount point name is ${mpname}”<\/p>\n
echo “lvcreate command is : lvcreate -n ${mpname} -L ${mpszm} ${vgn}”<\/p>\n
#### uncomment for realrun ###<\/p>\n
lvcreate -n ${mpname} -L ${mpszm} ${vgn}<\/p>\n
rc=$?<\/p>\n
actonreturn ${rc}<\/p>\n
echo “newfs command is : newfs -F vxfs -o largefiles \/dev\/${vgn}\/r${mpname}”<\/p>\n
newfs -F vxfs -o largefiles \/dev\/${vgn}\/r${mpname}<\/p>\n
rc=$?<\/p>\n
actonreturn ${rc}<\/p>\n
done<\/p>\n
vgexport -v -s -p -m ${vgn}.map ${vgn}<\/p>\n
1.<\/p>\n","protected":false},"excerpt":{"rendered":"
Serviceguard stand up Notes: Important parts of the .ascii file. CLUSTER_NAME prod_sgcluster HOSTNAME_ADDRESS_FAMILY IPV4 FIRST_CLUSTER_LOCK_VG \/dev\/vgsgpackagename ## Is it safer to maybe use a dedicated SG Lock disk. NODE_NAME sgclusternode01 NETWORK_INTERFACE lan900 HEARTBEAT_IP 192.168.208.209 NETWORK_INTERFACE lan901 HEARTBEAT_IP 10.43.128.209 # CLUSTER_LOCK_LUN FIRST_CLUSTER_LOCK_PV \/dev\/disk\/disk20 # Link Aggregate lan900 contains the following port(s): lan0,lan4 # Warning: There are […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_bbp_topic_count":0,"_bbp_reply_count":0,"_bbp_total_topic_count":0,"_bbp_total_reply_count":0,"_bbp_voice_count":0,"_bbp_anonymous_reply_count":0,"_bbp_topic_count_hidden":0,"_bbp_reply_count_hidden":0,"_bbp_forum_subforum_count":0,"_kadence_starter_templates_imported_post":false,"footnotes":""},"categories":[],"tags":[],"class_list":["post-304","post","type-post","status-publish","format-standard","hentry"],"_links":{"self":[{"href":"https:\/\/www.hpux.ws\/index.php?rest_route=\/wp\/v2\/posts\/304"}],"collection":[{"href":"https:\/\/www.hpux.ws\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.hpux.ws\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.hpux.ws\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.hpux.ws\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=304"}],"version-history":[{"count":0,"href":"https:\/\/www.hpux.ws\/index.php?rest_route=\/wp\/v2\/posts\/304\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.hpux.ws\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=304"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.hpux.ws\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=304"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.hpux.ws\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=304"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}