今天因地铁机房停电原因导致一数据库机器系统不能启动，开机启动报错如下：
网上查询解决命令：
fsck -y /dev/mapper/vg_dbserver1-lv_root
重点来了.数据库节点不能启动。启动出错是如下：
CRS-0184: Cannot communicate with the CRS daemon
数据库集群crsd.log 无明显报错！节点1脱离集群。最终发现节点1重启后时间为2015年。更改系统时间并同步系统时钟。处理完成后数据库正常启动。检查处理如下！
2018/5/9 18:46:38
[root@dbserver1 bin]# crsctl check has
CRS-4638: Oracle High Availability Services is online
[root@dbserver2 ~]# crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
[root@dbserver2 ~]# crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora.ARCH.dg ora....up.type ONLINE ONLINE dbserver2
ora.DATA.dg ora....up.type ONLINE ONLINE dbserver2
ora.DG1.dg ora....up.type ONLINE ONLINE dbserver2
ora.DG2.dg ora....up.type ONLINE ONLINE dbserver2
ora.DG3.dg ora....up.type ONLINE ONLINE dbserver2
ora....ER.lsnr ora....er.type ONLINE ONLINE dbserver2
ora....N1.lsnr ora....er.type ONLINE ONLINE dbserver2
ora.RCY1.dg ora....up.type ONLINE ONLINE dbserver2
ora.asm ora.asm.type ONLINE ONLINE dbserver2
ora.cvu ora.cvu.type ONLINE ONLINE dbserver2
ora....er1.vip ora....t1.type ONLINE ONLINE dbserver2
ora....SM2.asm application ONLINE ONLINE dbserver2
ora....R2.lsnr application ONLINE ONLINE dbserver2
ora....er2.gsd application OFFLINE OFFLINE
ora....er2.ons application ONLINE ONLINE dbserver2
ora....er2.vip ora....t1.type ONLINE ONLINE dbserver2
ora.gsd ora.gsd.type OFFLINE OFFLINE
ora....network ora....rk.type ONLINE ONLINE dbserver2
ora.oc4j ora.oc4j.type ONLINE ONLINE dbserver2
ora.ons ora.ons.type ONLINE ONLINE dbserver2
ora.scan1.vip ora....ip.type ONLINE ONLINE dbserver2
ora....flow.db ora....se.type ONLINE ONLINE dbserver2
[root@dbserver1 bin]# ifconfig -a
eth0 Link encap:Ethernet HWaddr 00:25:90:5D:A8:46
inet addr:172.168.100.5 Bcast:172.168.100.255 Mask:255.255.255.0
inet6 addr: fe80::225:90ff:fe5d:a846/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:4103 errors:0 dropped:0 overruns:0 frame:0
TX packets:2937 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:503937 (492.1 KiB) TX bytes:441698 (431.3 KiB)
Memory:dfe20000-dfe3ffff
eth1 Link encap:Ethernet HWaddr 00:25:90:5D:A8:47
BROADCAST MULTICAST MTU:1500 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)
Memory:dfe60000-dfe7ffff
eth2 Link encap:Ethernet HWaddr 00:25:90:5D:A8:48
inet addr:10.10.10.1 Bcast:10.10.10.255 Mask:255.255.255.0
inet6 addr: fe80::225:90ff:fe5d:a848/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:64646 errors:0 dropped:0 overruns:0 frame:0
TX packets:25629 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:82156159 (78.3 MiB) TX bytes:4477407 (4.2 MiB)
Memory:dfea0000-dfebffff
eth2:1 Link encap:Ethernet HWaddr 00:25:90:5D:A8:48
inet addr:169.254.231.5 Bcast:169.254.255.255 Mask:255.255.0.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
Memory:dfea0000-dfebffff
eth3 Link encap:Ethernet HWaddr 00:25:90:5D:A8:49
BROADCAST MULTICAST MTU:1500 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)
Memory:dfee0000-dfefffff
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:65536 Metric:1
RX packets:2474 errors:0 dropped:0 overruns:0 frame:0
TX packets:2474 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:461462 (450.6 KiB) TX bytes:461462 (450.6 KiB)
[root@dbserver1 bin]#
[root@dbserver2 ~]# ifconfig -a
eth0 Link encap:Ethernet HWaddr 00:25:90:5D:D2:62
inet addr:172.168.100.6 Bcast:172.168.100.255 Mask:255.255.255.0
inet6 addr: fe80::225:90ff:fe5d:d262/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:1231494 errors:0 dropped:0 overruns:0 frame:0
TX packets:1162628 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:550758358 (525.2 MiB) TX bytes:766880440 (731.3 MiB)
Memory:dfe20000-dfe3ffff
eth0:1 Link encap:Ethernet HWaddr 00:25:90:5D:D2:62
inet addr:172.168.100.9 Bcast:172.168.100.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
Memory:dfe20000-dfe3ffff
eth0:2 Link encap:Ethernet HWaddr 00:25:90:5D:D2:62
inet addr:172.168.100.7 Bcast:172.168.100.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
Memory:dfe20000-dfe3ffff
eth0:3 Link encap:Ethernet HWaddr 00:25:90:5D:D2:62
inet addr:172.168.100.8 Bcast:172.168.100.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
Memory:dfe20000-dfe3ffff
eth1 Link encap:Ethernet HWaddr 00:25:90:5D:D2:63
BROADCAST MULTICAST MTU:1500 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)
Memory:dfe60000-dfe7ffff
eth2 Link encap:Ethernet HWaddr 00:25:90:5D:D2:64
inet addr:10.10.10.2 Bcast:10.10.10.255 Mask:255.255.255.0
inet6 addr: fe80::225:90ff:fe5d:d264/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:83825 errors:0 dropped:0 overruns:0 frame:0
TX packets:200633 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:16442256 (15.6 MiB) TX bytes:249300241 (237.7 MiB)
Memory:dfea0000-dfebffff
eth2:1 Link encap:Ethernet HWaddr 00:25:90:5D:D2:64
inet addr:169.254.59.187 Bcast:169.254.255.255 Mask:255.255.0.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
Memory:dfea0000-dfebffff
eth3 Link encap:Ethernet HWaddr 00:25:90:5D:D2:65
BROADCAST MULTICAST MTU:1500 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)
Memory:dfee0000-dfefffff
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:65536 Metric:1
RX packets:61389 errors:0 dropped:0 overruns:0 frame:0
TX packets:61389 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:62660462 (59.7 MiB) TX bytes:62660462 (59.7 MiB)
[root@dbserver2 ~]#
[root@dbserver1 bin]# date
Wed Jun 24 11:57:02 CST 2015
[root@dbserver1 bin]#
[root@dbserver2 ~]# date
Wed May 9 18:13:15 CST 2018
[root@dbserver1 bin]# date
Wed Jun 24 12:05:42 CST 2015
[root@dbserver1 bin]# date -s 05/09/2018
Wed May 9 00:00:00 CST 2018
[root@dbserver1 bin]# date
Wed May 9 00:00:09 CST 2018
[root@dbserver1 bin]# date -s 18:22:50
Wed May 9 18:22:50 CST 2018
[root@dbserver1 bin]# date
Wed May 9 18:22:54 CST 2018
[root@dbserver1 bin]# hwclock --systohc
[root@dbserver1 ~]# crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora.ARCH.dg ora....up.type ONLINE ONLINE dbserver1
ora.DATA.dg ora....up.type ONLINE ONLINE dbserver1
ora.DG1.dg ora....up.type ONLINE ONLINE dbserver1
ora.DG2.dg ora....up.type ONLINE ONLINE dbserver1
ora.DG3.dg ora....up.type ONLINE ONLINE dbserver1
ora....ER.lsnr ora....er.type ONLINE ONLINE dbserver1
ora....N1.lsnr ora....er.type ONLINE ONLINE dbserver2
ora.RCY1.dg ora....up.type ONLINE ONLINE dbserver1
ora.asm ora.asm.type ONLINE ONLINE dbserver1
ora.cvu ora.cvu.type ONLINE ONLINE dbserver2
ora....SM1.asm application ONLINE ONLINE dbserver1
ora....R1.lsnr application ONLINE ONLINE dbserver1
ora....er1.gsd application OFFLINE OFFLINE
ora....er1.ons application ONLINE ONLINE dbserver1
ora....er1.vip ora....t1.type ONLINE ONLINE dbserver1
ora....SM2.asm application ONLINE ONLINE dbserver2
ora....R2.lsnr application ONLINE ONLINE dbserver2
ora....er2.gsd application OFFLINE OFFLINE
ora....er2.ons application ONLINE ONLINE dbserver2
ora....er2.vip ora....t1.type ONLINE ONLINE dbserver2
ora.gsd ora.gsd.type OFFLINE OFFLINE
ora....network ora....rk.type ONLINE ONLINE dbserver1
ora.oc4j ora.oc4j.type ONLINE ONLINE dbserver2
ora.ons ora.ons.type ONLINE ONLINE dbserver1
ora.scan1.vip ora....ip.type ONLINE ONLINE dbserver2
ora....flow.db ora....se.type ONLINE ONLINE dbserver1
[root@dbserver1 ~]#
SQL> select instance_name,status from gv$instance;
INSTANCE_NAME STATUS
---------------- ------------
workflow1 OPEN
workflow2 OPEN
SQL>
