crm系统cluster启停.docx

crm系统vcs启停操作1、状态检查CRM系统共4个节点sfrac集群，平时只有crmdb1、crmdb2两台机器vcs集群处于启动状态，crmdb3、crmdb4集群关闭。用hastatus sum命令检查crmdb1/var/adm/streams#hastatus -sum- SYSTEM STATE- System State Frozen A crmdb1 RUNNING 0 A crmdb2 RUNNING 0 A crmdb3 EXITED 0 A crmdb4 EXITED 0 - GROUP STATE- Group System Probed AutoDisabled State B cfs_Mount crmdb1 Y N ONLINE B cfs_Mount crmdb2 Y N ONLINE B cfs_Mount crmdb3 Y N OFFLINE B cfs_Mount crmdb4 Y N OFFLINE B crmdb1vip crmdb1 Y N ONLINE B crmdb1vip crmdb2 Y N OFFLINE B crmdb2vip crmdb1 Y N OFFLINE B crmdb2vip crmdb2 Y N ONLINE B crmdb3vip crmdb3 Y N OFFLINE B crmdb3vip crmdb4 Y N OFFLINE B crmdb4vip crmdb3 Y N OFFLINE B crmdb4vip crmdb4 Y N OFFLINE B cvm crmdb1 Y N ONLINE B cvm crmdb2 Y N ONLINE B cvm crmdb3 Y N OFFLINE B cvm crmdb4 Y N OFFLINE vcs集群配置了三种资源组，分别是cfs_mount,vip,cvm。其中，cvm、cfs_mount是共享资源资源组，只要启动cluster，就应该处于online状态，可以多台机器同时online。Vip是 ha资源组，一次只能在一个节点上online。Vip资源组共有crmdb1vip、crmdb2vip、crmdb3vip、crmdb4vip四个，crmdb1vip和crmdb2vip互备，crmdb3vip和crmdb4互备。1可以切换到2，2可以切换到1，1、2不能切换到db3、db4主机上。同理crmdb3vip和crmdb4vip一样。手动切换资源组的命令是：hagrp switch -to 例如切换crmdb1vip到crmdb2主机上：hagrp switch crmdb1vip to crmdb2手动online资源组命令hagrp online -sys crm主机上dg资源组，共两个crmdb1/#vxdg listNAME STATE IDcrmdg enabled,shared,cds 1246257969.332.crmdb1orasrvdg enabled,shared,cds 1246249548.328.crmdb1crm系统镜像检查，注意KSTATE、STATE两列，ENABLED、ACTIVE状态是正常的。crmdb1/#vxprint -p Disk group: crmdgTY NAME ASSOC KSTATE LENGTH PLOFFS STATE TUTIL0 PUTIL0pl crmssd01-01 crmssd01 ENABLED 1126400000 - ACTIVE - -pl crmssd01-02 crmssd01 ENABLED 1126400000 - ACTIVE - -pl crmssd01_dcl-01 crmssd01_dcl ENABLED 77840 - ACTIVE - -pl crmssd01_dcl-02 crmssd01_dcl ENABLED 77840 - ACTIVE - -pl crm01-01 crm01 ENABLED 2469609472 - ACTIVE - -pl crm01-02 crm01 ENABLED 2469609472 - ACTIVE - -pl crm01_dcl-01 crm01_dcl ENABLED 170128 - ACTIVE - -pl crm01_dcl-02 crm01_dcl ENABLED 170128 - ACTIVE - -pl crm02-01 crm02 ENABLED 2469609472 - ACTIVE - -pl crm02-02 crm02 ENABLED 2469609472 - ACTIVE - -pl crm02_dcl-01 crm02_dcl ENABLED 170128 - ACTIVE - -pl crm02_dcl-02 crm02_dcl ENABLED 170128 - ACTIVE - -pl crm03-01 crm03 ENABLED 2469609472 - ACTIVE - -pl crm03-02 crm03 ENABLED 2469609472 - ACTIVE - -pl crm03_dcl-01 crm03_dcl ENABLED 170128 - ACTIVE - -pl crm03_dcl-02 crm03_dcl ENABLED 170128 - ACTIVE - -pl crm04-01 crm04 ENABLED 2469609472 - ACTIVE - -pl crm04-02 crm04 ENABLED 2469609472 - ACTIVE - -pl crm04_dcl-01 crm04_dcl ENABLED 170128 - ACTIVE - -pl crm04_dcl-02 crm04_dcl ENABLED 170128 - ACTIVE - -pl crm05-01 crm05 ENABLED 2469609472 - ACTIVE - -pl crm05-02 crm05 ENABLED 2469609472 - ACTIVE - -pl crm05_dcl-01 crm05_dcl ENABLED 170128 - ACTIVE - -pl crm05_dcl-02 crm05_dcl ENABLED 170128 - ACTIVE - -pl crm06-01 crm06 ENABLED 2469609472 - ACTIVE - -pl crm06-02 crm06 ENABLED 2469609472 - ACTIVE - -pl crm06_dcl-01 crm06_dcl ENABLED 170128 - ACTIVE - -pl crm06_dcl-02 crm06_dcl ENABLED 170128 - ACTIVE - -pl crm07-01 crm07 ENABLED 2469609472 - ACTIVE - -pl crm07-02 crm07 ENABLED 2469609472 - ACTIVE - -pl crm07_dcl-01 crm07_dcl ENABLED 170128 - ACTIVE - -pl crm07_dcl-02 crm07_dcl ENABLED 170128 - ACTIVE - -pl crm08-01 crm08 ENABLED 2469609472 - ACTIVE - -pl crm08-02 crm08 ENABLED 2469609472 - ACTIVE - -pl crm08_dcl-01 crm08_dcl ENABLED 170128 - ACTIVE - -pl crm08_dcl-02 crm08_dcl ENABLED 170128 - ACTIVE - -pl crm09-01 crm09 ENABLED 2469609472 - ACTIVE - -pl crm09-02 crm09 ENABLED 2469609472 - ACTIVE - -pl crm09_dcl-01 crm09_dcl ENABLED 170128 - ACTIVE - -pl crm09_dcl-02 crm09_dcl ENABLED 170128 - ACTIVE - -pl crm10-01 crm10 ENABLED 2469609472 - ACTIVE - -pl crm10-02 crm10 ENABLED 2469609472 - ACTIVE - -pl crm10_dcl-01 crm10_dcl ENABLED 170128 - ACTIVE - -pl crm10_dcl-02 crm10_dcl ENABLED 170128 - ACTIVE - -pl crm11-01 crm11 ENABLED 2469609472 - ACTIVE - -pl crm11-02 crm11 ENABLED 2469609472 - ACTIVE - -pl crm11_dcl-01 crm11_dcl ENABLED 170128 - ACTIVE - -pl crm11_dcl-02 crm11_dcl ENABLED 170128 - ACTIVE - -pl crm12-01 crm12 ENABLED 2469609472 - ACTIVE - -pl crm12-02 crm12 ENABLED 2469609472 - ACTIVE - -pl crm12_dcl-01 crm12_dcl ENABLED 170128 - ACTIVE - -pl crm12_dcl-02 crm12_dcl ENABLED 170128 - ACTIVE - -pl crm13-01 crm13 ENABLED 2469609472 - ACTIVE - -pl crm13-02 crm13 ENABLED 2469609472 - ACTIVE - -pl crm13_dcl-01 crm13_dcl ENABLED 170128 - ACTIVE - -pl crm13_dcl-02 crm13_dcl ENABLED 170128 - ACTIVE - -pl crm14-01 crm14 ENABLED 2469609472 - ACTIVE - -pl crm14-02 crm14 ENABLED 2469609472 - ACTIVE - -pl crm14_dcl-01 crm14_dcl ENABLED 170128 - ACTIVE - -pl crm14_dcl-02 crm14_dcl ENABLED 170128 - ACTIVE - -pl crm15-01 crm15 ENABLED 2469609472 - ACTIVE - -pl crm15-02 crm15 ENABLED 2469609472 - ACTIVE - -pl crm15_dcl-01 crm15_dcl ENABLED 170128 - ACTIVE - -pl crm15_dcl-02 crm15_dcl ENABLED 170128 - ACTIVE - -pl crm16-01 crm16 ENABLED 2469609472 - ACTIVE - -pl crm16-02 crm16 ENABLED 2469609472 - ACTIVE - -pl crm16_dcl-01 crm16_dcl ENABLED 170128 - ACTIVE - -pl crm16_dcl-02 crm16_dcl ENABLED 170128 - ACTIVE - -pl crm17-01 crm17 ENABLED 2469609472 - ACTIVE - -pl crm17-02 crm17 ENABLED 2469609472 - ACTIVE - -pl crm17_dcl-01 crm17_dcl ENABLED 170128 - ACTIVE - -pl crm17_dcl-02 crm17_dcl ENABLED 170128 - ACTIVE - -pl crm18-01 crm18 ENABLED 2469609472 - ACTIVE - -pl crm18-02 crm18 ENABLED 2469609472 - ACTIVE - -pl crm18_dcl-01 crm18_dcl ENABLED 170128 - ACTIVE - -pl crm18_dcl-02 crm18_dcl ENABLED 170128 - ACTIVE - -pl crm19-01 crm19 ENABLED 2469609472 - ACTIVE - -pl crm19-02 crm19 ENABLED 2469609472 - ACTIVE - -pl crm19_dcl-01 crm19_dcl ENABLED 170128 - ACTIVE - -pl crm19_dcl-02 crm19_dcl ENABLED 170128 - ACTIVE - -pl crm20-01 crm20 ENABLED 2469609472 - ACTIVE - -pl crm20-02 crm20 ENABLED 2469609472 - ACTIVE - -pl crm20_dcl-01 crm20_dcl ENABLED 170128 - ACTIVE - -pl crm20_dcl-02 crm20_dcl ENABLED 170128 - ACTIVE - -pl crm21-01 crm21 ENABLED 2469609472 - ACTIVE - -pl crm21-02 crm21 ENABLED 2469609472 - ACTIVE - -pl crm21_dcl-01 crm21_dcl ENABLED 170128 - ACTIVE - -pl crm21_dcl-02 crm21_dcl ENABLED 170128 - ACTIVE - -Disk group: orasrvdgTY NAME ASSOC KSTATE LENGTH PLOFFS STATE TUTIL0 PUTIL0pl orasrvvol-01 orasrvvol ENABLED 614400 - ACTIVE - -pl orasrvvol-02 orasrvvol ENABLED 614400 - ACTIVE - -pl orasrvvol_dcl-01 orasrvvol_dcl ENABLED 544 - ACTIVE - -pl orasrvvol_dcl-02 orasrvvol_dcl ENABLED 544 - ACTIVE - -检查volume状态 ，注意KSTATE、STATE两列，ENABLED，ACTIVE是正常的，其他异常crmdb1/#vxprint -vDisk group: crmdgTY NAME ASSOC KSTATE LENGTH PLOFFS STATE TUTIL0 PUTIL0v crmssd01 fsgen ENABLED 1126400000 - ACTIVE - -v crmssd01_dcl gen ENABLED 77840 - ACTIVE - -v crm01 fsgen ENABLED 2469606208 - ACTIVE - -v crm01_dcl gen ENABLED 170128 - ACTIVE - -v crm02 fsgen ENABLED 2469606208 - ACTIVE - -v crm02_dcl gen ENABLED 170128 - ACTIVE - -v crm03 fsgen ENABLED 2469606208 - ACTIVE - -v crm03_dcl gen ENABLED 170128 - ACTIVE - -v crm04 fsgen ENABLED 2469606208 - ACTIVE - -v crm04_dcl gen ENABLED 170128 - ACTIVE - -v crm05 fsgen ENABLED 2469606208 - ACTIVE - -v crm05_dcl gen ENABLED 170128 - ACTIVE - -v crm06 fsgen ENABLED 2469606208 - ACTIVE - -v crm06_dcl gen ENABLED 170128 - ACTIVE - -v crm07 fsgen ENABLED 2469606208 - ACTIVE - -v crm07_dcl gen ENABLED 170128 - ACTIVE - -v crm08 fsgen ENABLED 2469606208 - ACTIVE - -v crm08_dcl gen ENABLED 170128 - ACTIVE - -v crm09 fsgen ENABLED 2469606208 - ACTIVE - -v crm09_dcl gen ENABLED 170128 - ACTIVE - -v crm10 fsgen ENABLED 2469606208 - ACTIVE - -v crm10_dcl gen ENABLED 170128 - ACTIVE - -v crm11 fsgen ENABLED 2469606208 - ACTIVE - -v crm11_dcl gen ENABLED 170128 - ACTIVE - -v crm12 fsgen ENABLED 2469606208 - ACTIVE - -v crm12_dcl gen ENABLED 170128 - ACTIVE - -v crm13 fsgen ENABLED 2469606208 - ACTIVE - -v crm13_dcl gen ENABLED 170128 - ACTIVE - -v crm14 fsgen ENABLED 2469606208 - ACTIVE - -v crm14_dcl gen ENABLED 170128 - ACTIVE - -v crm15 fsgen ENABLED 2469606208 - ACTIVE - -v crm15_dcl gen ENABLED 170128 - ACTIVE - -v crm16 fsgen ENABLED 2469606208 - ACTIVE - -v crm16_dcl gen ENABLED 170128 - ACTIVE - -v crm17 fsgen ENABLED 2469606208 - ACTIVE - -v crm17_dcl gen ENABLED 170128 - ACTIVE - -v crm18 fsgen ENABLED 2469606208 - ACTIVE - -v crm18_dcl gen ENABLED 170128 - ACTIVE - -v crm19 fsgen ENABLED 2469606208 - ACTIVE - -v crm19_dcl gen ENABLED 170128 - ACTIVE - -v crm20 fsgen ENABLED 2469606208 - ACTIVE - -v crm20_dcl gen ENABLED 170128 - ACTIVE - -v crm21 fsgen ENABLED 2469606208 - ACTIVE - -v crm21_dcl gen ENABLED 170128 - ACTIVE - -Disk group: orasrvdgTY NAME ASSOC KSTATE LENGTH PLOFFS STATE TUTIL0 PUTIL0v orasrvvol fsgen ENABLED 614400 - ACTIVE - -v orasrvvol_dcl gen ENABLED 544 - ACTIVE - -查看主机链路，关注emc0、emc1，正常的状态是ENABLEDcrmdb1/#vxdmpadm listctlr allCTLR-NAME ENCLR-TYPE STATE ENCLR-NAME=scsi4 Disk ENABLED diskscsi5 Disk ENABLED diskscsi6 Disk ENABLED diskscsi2 Disk ENABLED diskscsi0 Disk ENABLED diskfscsi5 EMC ENABLED emc1fscsi15 EMC ENABLED emc1fscsi13 EMC ENABLED emc1fscsi7 EMC ENABLED emc1fscsi15 EMC ENABLED emc0fscsi13 EMC ENABLED emc0fscsi7 EMC ENABLED emc0fscsi5 EMC ENABLED emc0Crmdb1、crmdb2主机上有/crm01-21、/crmssd01这22个文件系统，用df g命令可以列出/dev/vx/dsk/crmdg/crm21 1177.60 714.74 40% 57 1% /crm21/dev/vx/dsk/crmdg/crm18 1177.60 755.43 36% 52 1% /crm18/dev/vx/dsk/crmdg/crm19 1177.60 748.44 37% 50 1% /crm19/dev/vx/dsk/crmdg/crm15 1177.60 716.95 40% 54 1% /crm15/dev/vx/dsk/crmdg/crm16 1177.60 746.08 37% 48 1% /crm16/dev/vx/dsk/crmdg/crm14 1177.60 470.81 61% 76 1% /crm14/dev/vx/dsk/crmdg/crm20 1177.60 775.40 35% 52 1% /crm20/dev/vx/dsk/crmdg/crm17 1177.60 738.88 38% 47 1% /crm17/dev/vx/dsk/crmdg/crmssd01 537.11 101.29 82% 30 1% /crmssd01/dev/vx/dsk/crmdg/crm01 1177.60 173.57 86% 155 1% /crm01/dev/vx/dsk/crmdg/crm13 1177.60 957.39 19% 11335 1% /crm13/dev/vx/dsk/crmdg/crm10 1177.60 190.89 84% 140 1% /crm10/dev/vx/dsk/crmdg/crm04 1177.60 166.80 86% 150 1% /crm04/dev/vx/dsk/crmdg/crm12 1177.60 194.12 84% 684 1% /crm12/dev/vx/dsk/crmdg/crm05 1177.60 176.76 85% 150 1% /crm05/dev/vx/dsk/crmdg/crm07 1177.60 210.01 83% 137 1% /crm07/dev/vx/dsk/crmdg/crm08 1177.60 166.76 86% 152 1% /crm08/dev/vx/dsk/crmdg/crm11 1177.60 204.31 83% 132 1% /crm11/dev/vx/dsk/crmdg/crm09 1177.60 146.81 88% 153 1% /crm09/dev/vx/dsk/crmdg/crm06 1177.60 183.68 85% 153 1% /crm06/dev/vx/dsk/crmdg/crm03 1177.60 155.63 87% 155 1% /crm03/dev/vx/dsk/crmdg/crm02 1177.60 159.44 87% 157 1% /crm022、应急情况下启动3、4节点下列情况下直接启动3、4节点clustera、 crmdb1、crmdb2主机均正常、vcs正常b、 crmdb1、crmdb2有一台正常，vcs正常 crmdb3、crmdb4主机上各有一个/veritas，该目录下有启动脚本启动命令：sh /veritas/start_vcs.sh注意：启动前线确认心跳网络正常、阵列链路正常。可以用cat /etc/llttab命令确定vcs用哪个心跳网口，然后用ping测试与其他节点通信情况。下列情况下不要启动3、4节点，下面请稍微复杂，需要联系symantec工程师协助a、 crmdb1、crmdb2同时宕机，且主机无法启动。b、 crmdb3、crmdb4心跳网络与crmdb1、crmdb2不通时c、 crmdb3、crmdb4连接阵列有问题时。3、启动3、4节点后检查登陆到3、4节点上用df g可以看到22个veritas文件系统。用gabconfig命令可以看到vcs的各port情况，后面有个membership 0、1、2、3分别表示crmdb1、2、3、4crmdb1/#gabconfig -aGAB Port Memberships=Port a gen 47600d membership 0123Port b gen 476010 membership 0123Port d gen 476013 membership 0123Por