Oracle 11g RAC 本地环境部署

环境说明

1.1 基础环境说明

两台主机为VMworkstation虚拟机，配置规格为4C8G，分别配置两张虚拟网卡具体网段信息如下表所示，共享存储使用ISCSI+Multipath，存储放置在一台虚拟机上即可。本次安装Linux系统版本为CentOS Linux release 7.6.1810 (Core)，数据库版本为11.2.0.4。

（注意：oracleRAC11g适用系统中不包含centos7，适用性最好的为Centos6，但是由于Centos6目前已不再维护，这里选择使用Centos7，除此之外对于数据库版本11.2.0.1不建议安装在Centos7，可能出现安装oracle时无法识别集群的问题暂时没找到解决的办法。）

1.2 IP与磁盘规划

IP规划：

磁盘组规划：

安装环境初始化

该阶段除特殊说明外所有的步骤都是要在两个节点上依次执行

2.1 修改节点主机名

[root@orcl1 ~]# hostnamectl set-hostname orcl1

[root@orcl2 ~]# hostnamectl set-hostname orcl2

2.2 关闭防火墙禁用selinux

[root@orcl1 ~]# sed -i 's/^SELINUX=\(.*\)/SELINUX=disabled/g' /etc/selinux/config

[root@orcl1 ~]# systemctl stop firewalld && systemctl disable firewalld&&systemctl status firewalld

● firewalld.service - firewalld - dynamic firewall daemon

  Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)

  Active: inactive (dead)

    Docs: man:firewalld(1)

2.3 修改hosts文件

[root@orcl1 ~]# cat /etc/hosts

127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4

::1 localhost localhost.localdomain localhost6 localhost6.localdomain6

192.168.103.75 orcl1

192.168.103.76 orcl2

192.168.10.21 orcl1-priv

192.168.10.22 orcl2-priv

192.168.103.81 orcl1-vip

192.168.103.82 orcl2-vip

192.168.103.85 scan-ip

2.4 关闭transparent hugepages

[root@orcl1 ~]# cat /sys/kernel/mm/transparent_hugepage/enabled

[always] madvise never

2.5 修改系统内核参数

修改内核参数：

[root@orcl1 ~]# sysctl -p

net.ipv6.conf.all.disable_ipv6 = 1
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmall = 39546368
kernel.shmmax = 412316860416
kernel.shmmni = 4096

kernel.sem = 250 32000 100 128

net.ipv4.ip_local_port_range = 9000 65500

net.core.rmem_default = 262144

net.core.rmem_max = 4194304

net.core.wmem_default = 262144

net.core.wmem_max = 1048576

kernel.core_pattern = core.%e.%p

修改limits.conf：

[root@orcl1 ~]# tail -19 /etc/security/limits.conf

* soft nofile 655350
* hard nofile 655350
grid soft nproc 2047
grid hard nproc 65535
grid soft nofile 4096
grid hard nofile 65536
grid soft stack 10240
grid hard stack 32768
grid soft memlock unlimited
grid hard memlock unlimited
oracle soft nproc 2047
oracle hard nproc 65535
oracle soft nofile 4096
oracle hard nofile 65536
oracle soft stack 10240
oracle hard stack 32768
oracle soft memlock unlimited
oracle hard memlock unlimited
oracle hard nofile 65536
oracle soft stack 10240
oracle hard stack 32768
oracle soft memlock unlimited
oracle hard memlock unlimited

2.5 创建用户组修改密码

groupadd -g 54321 oinstall
groupadd -g 54322 dba
groupadd -g 54323 oper
groupadd -g 54324 backupdba
groupadd -g 54325 dgdba
groupadd -g 54326 kmdba
groupadd -g 54327 asmdba
groupadd -g 54328 asmoper
groupadd -g 54329 asmadmin
groupadd -g 54330 racdba
/usr/sbin/useradd -u 54321 -g oinstall -G oinstall,dba,oper,backupdba,dgdba,kmdba,asmdba,racdba oracle
/usr/sbin/useradd -u 54322 -g oinstall -G oinstall,dba,asmdba,asmoper,asmadmin,racdba grid

修改Oracle及grig密码

echo "oracle" |passwd --stdin grid
echo "oracle" |passwd --stdin oracle

2.6 创建安装目录并修改权限

mkdir -p /u01/app/grid_base
mkdir -p /u01/app/grid_home
mkdir -p /u01/app/oracle/product/11.2.0/db_1
mkdir -p /u01/app/oraInventory
chown -R grid:oinstall /u01/app/grid_base
chown -R grid:oinstall /u01/app/grid_home
chmod -R 775 /u01/app/grid_base
chmod -R 775 /u01/app/grid_home
chown -R oracle:oinstall /u01/app/oracle
chmod -R 775 /u01/app/oracle
chown -R grid:oinstall /u01/app/oraInventory

2.7 修改oracle及grid环境变量

Oracle #节点二修改 ORACLE_SID=orcl2 其他相同

[root@orcl1 /root]

#vim /home/oracle/.bash_profile

追加：

export PATH
export TMP=/tmp
export TMPDIR=$TMP
export ORACLE_SID=rac1
export ORACLE_BASE=/u01/app/oracle/
export ORACLE_HOME=/u01/app/oracle/product/11.2.0/db_1/
export TNS_ADMIN=$ORACLE_HOME/network/admin/
export PATH=/usr/sbin:$PATH
export PATH=$ORACLE_HOME/bin:$PATH
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib
export CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib
umask 022

Grid #节点二修改 ORACLE_SID=+ASM2 其他相同

[root@orcl2 /root]

#vim /home/grid/.bash_profile

export PATH
export TMP=/tmp
export TMPDIR=$TMP
export ORACLE_SID=+ASM1
export ORACLE_BASE=/u01/app/grid_base
export ORACLE_HOME=/u01/app/grid_home
export PATH=/usr/sbin:$PATH
export PATH=$ORACLE_HOME/bin:$PATH
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib
export CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib
umask 022

2.8 安装相关依赖包

yum -y install binutils compat-libstdc++-33* compat-libstdc++-33.i686 elfutils-libelf elfutils-libelf-devel elfutils-libelf-devel-static gcc gcc-c++ glibc* kernel-headers ksh libaio libaio.i686 libaio-devel libaio-devel.i686 libgcc libgcc.i686 libgomp libstdc++* libstdc++.i686 make sysstat unixODBC.i686 unixODBC.x86_64 unixODBC-devel.i686 unixODBC-devel.x86_64

2.9 配置asm磁盘组

根据相应的对应关系来制作 99-oracle-asmdevices.rules 配置文件内容如下：

[root@rac1 /etc/udev/rules.d]

#vim 12-dm-permissions.rules

ENV{DM_NAME}=="asm-ocr1", OWNER:="grid", GROUP:="oinstall", MODE:="660"

ENV{DM_NAME}=="asm-ocr2", OWNER:="grid", GROUP:="oinstall", MODE:="660"

ENV{DM_NAME}=="asm-ocr3", OWNER:="grid", GROUP:="oinstall", MODE:="660"

ENV{DM_NAME}=="asm-data1", OWNER:="grid", GROUP:="oinstall", MODE:="660"

ENV{DM_NAME}=="asm-mgmt", OWNER:="grid", GROUP:="oinstall", MODE:="660"

ENV{DM_NAME}=="asm-data2", OWNER:="grid", GROUP:="oinstall", MODE:="660"

ENV{DM_NAME}=="asm-redo1", OWNER:="grid", GROUP:="oinstall", MODE:="660"

ENV{DM_NAME}=="asm-redo2", OWNER:="grid", GROUP:="oinstall", MODE:="660"

Centos激活udev规则

udevadm control --reload-rules  
udevadm trigger

查看映射后的asm盘符

安装GRID集群

3.1 解压grid Oracle安装包并配置互信

#解压软件包，该操作单节点执行，解压位置无要求

--grid用户

[grid@orcl1 soft]$ unzip p13390677_112040_Linux-x86-64_3of7.zip

--oracle用户

[oracle@orcl1 soft]$ unzip p13390677_112040_Linux-x86-64_1of7.zip

[oracle@orcl1 soft]$ unzip p13390677_112040_Linux-x86-64_2of7.zip

#安装磁盘检测工具（该工具在grid安装包中）

[root@orcl1 rpm]# pwd

/home/soft/grid/rpm

[root@orcl1 rpm]# ls

cvuqdisk-1.0.9-1.rpm

[root@orcl1 rpm]# rpm -ivh cvuqdisk-1.0.9-1.rpm

Preparing... ################################# [100%]

Using default group oinstall to install package

Updating / installing...

  1:cvuqdisk-1.0.9-1 ################################# [100%]

#配置节点间互信

配置方式有很多种，这里手动配置的保证grid，oracle用户分别可以直接ssh到另一节点对应用户即可。

#/usr/bin/ssh -o FallBackToRsh=no -o PasswordAuthentication=no -o StrictHostKeyChecking=yes -o NumberOfPasswordPrompts=0 rac2 /usr/bin/true

[root@rac1 /root]

#echo $?

0 （返回0即为正常）

3.2 Grid 安装预检查

[grid@orcl1 grid]$ ./runcluvfy.sh stage -pre crsinst -n orcl1,orcl2 -fixup -verbose >> check.txt

Check: Package existence for "pdksh"

 Node Name Available Required Status

 ------------ ------------------------ ------------------------ ----------

 orcl2 missing pdksh-5.2.14 failed

 orcl1 missing pdksh-5.2.14 failed

缺少pdksh包忽略。

3.3 配置 grid 集群安装响应文件

[grid@rac1 /u01/app/grid_home/install/response]

$pwd

/u01/app/grid_home/install/response

[grid@rac1 /u01/app/grid_home/install/response]

$cat grid.rsp

oracle.install.responseFileVersion=/oracle/install/rspfmt_crsinstall_response_schema_v11_2_0

ORACLE_HOSTNAME=orcl1

INVENTORY_LOCATION=/u01/app/oraInventory

SELECTED_LANGUAGES=en

oracle.install.option=CRS_CONFIG

ORACLE_BASE=/u01/app/grid_base

ORACLE_HOME=/u01/app/grid_home

oracle.install.asm.OSDBA=asmdba

oracle.install.asm.OSOPER=asmoper

oracle.install.asm.OSASM=asmadmin

oracle.install.crs.config.gpnp.scanName=scan-ip

oracle.install.crs.config.gpnp.scanPort=1521

oracle.install.crs.config.clusterName=rac11g

oracle.install.crs.config.gpnp.configureGNS=false

oracle.install.crs.config.gpnp.gnsSubDomain=

oracle.install.crs.config.gpnp.gnsVIPAddress=

oracle.install.crs.config.autoConfigureClusterNodeVIP=false

oracle.install.crs.config.clusterNodes=orcl1:orcl1-vip,orcl2:orcl2-vip

oracle.install.crs.config.networkInterfaceList=eth0:172.16.8.0:1,eth1:172.16.10.0:2

oracle.install.crs.config.storageOption=ASM_STORAGE

oracle.install.crs.config.sharedFileSystemStorage.diskDriveMapping=

oracle.install.crs.config.sharedFileSystemStorage.votingDiskLocations=

oracle.install.crs.config.sharedFileSystemStorage.votingDiskRedundancy=NORMAL

oracle.install.crs.config.sharedFileSystemStorage.ocrLocations=

oracle.install.crs.config.sharedFileSystemStorage.ocrRedundancy=NORMAL

oracle.install.crs.config.useIPMI=false

oracle.install.crs.config.ipmi.bmcUsername=

oracle.install.crs.config.ipmi.bmcPassword=

oracle.install.asm.SYSASMPassword=oracle

oracle.install.asm.diskGroup.name=OCR

oracle.install.asm.diskGroup.redundancy=EXTERNAL

oracle.install.asm.diskGroup.AUSize=1

oracle.install.asm.diskGroup.disks=/dev/mapper/asm-ocr01

oracle.install.asm.diskGroup.diskDiscoveryString=/dev/mapper/asm*

oracle.install.asm.monitorPassword=oracle

oracle.install.crs.upgrade.clusterNodes=

oracle.install.asm.upgradeASM=false

oracle.installer.autoupdates.option=SKIP_UPDATES

oracle.installer.autoupdates.downloadUpdatesLoc=

AUTOUPDATES_MYORACLESUPPORT_USERNAME=

AUTOUPDATES_MYORACLESUPPORT_PASSWORD=

PROXY_HOST=

PROXY_PORT=

PROXY_USER=

PROXY_PWD=

PROXY_REALM=

3.4 执行grid集群安装

[grid@orcl1 grid]$ ./runInstaller -ignorePrereq -force -silent -responseFile /home/soft/grid/response/grid.rsp -showProgress

执行输出如下：

Starting Oracle Universal Installer...

Checking Temp space: must be greater than 120 MB. Actual 67729 MB Passed

Checking swap space: must be greater than 150 MB. Actual 2047 MB Passed

Preparing to launch Oracle Universal Installer from /tmp/OraInstall2021-02-04_03-02-34PM. Please wait ...[grid@orcl1 grid]$ [WARNING] [INS-30011] The SYS password entered does not conform to the Oracle recommended standards.

  CAUSE: Oracle recommends that the password entered should be at least 8 characters in length, contain at least 1 uppercase character, 1 lower case character and 1 digit [0-9].

  ACTION: Provide a password that conforms to the Oracle recommended standards.

[WARNING] [INS-30011] The ASMSNMP password entered does not conform to the Oracle recommended standards.

  CAUSE: Oracle recommends that the password entered should be at least 8 characters in length, contain at least 1 uppercase character, 1 lower case character and 1 digit [0-9].

  ACTION: Provide a password that conforms to the Oracle recommended standards.

You can find the log of this install session at:

/u01/app/oraInventory/logs/installActions2021-02-04_03-02-34PM.log

Prepare in progress.

.................................................. 9% Done.

Prepare successful.

Copy files in progress.

.................................................. 15% Done.

.................................................. 20% Done.

.................................................. 25% Done.

.................................................. 30% Done.

.................................................. 35% Done.

.................................................. 40% Done.

.................................................. 45% Done.

........................................

Copy files successful.

Link binaries in progress.

Link binaries successful.

.................................................. 62% Done.

Setup files in progress.

Setup files successful.

.................................................. 76% Done.

Perform remote operations in progress.

.................................................. 89% Done.

Perform remote operations successful.

The installation of Oracle Grid Infrastructure 11g was successful.

Please check '/u01/app/oraInventory/logs/silentInstall2021-02-04_03-02-34PM.log' for more details.

.................................................. 94% Done.

Execute Root Scripts in progress.

As a root user, execute the following script(s):

       1. /u01/app/oraInventory/orainstRoot.sh

       2. /u01/app/grid_home/root.sh

Execute /u01/app/oraInventory/orainstRoot.sh on the following nodes:

[orcl1, orcl2]

Execute /u01/app/grid_home/root.sh on the following nodes:

[orcl1, orcl2]

.................................................. 100% Done.

Execute Root Scripts successful.

As install user, execute the following script to complete the configuration.

       1. /u01/app/grid_home/cfgtoollogs/configToolAllCommands RESPONSE_FILE=<response_file>

       Note:

       1. This script must be run on the same host from where installer was run.

       2. This script needs a small password properties file for configuration assistants that require passwords (refer to install guide documentation).

Successfully Setup Software.

GI 安装完成

[root@orcl1 ~]# /u01/app/grid_home/bin/crsctl check crs

CRS-4638: Oracle High Availability Services is online

CRS-4537: Cluster Ready Services is online

CRS-4529: Cluster Synchronization Services is online

CRS-4533: Event Manager is online

创建磁盘组

登录到节点 1 asm（sqlplus / as sysasm）当中执行如下命令

create diskgroup DATA external redundancy disk '/dev/mapper/data' attribute 'compatible.asm' = '11.2';

create diskgroup FRI external redundancy disk '/dev/mapper/fri' attribute 'compatible.asm' = '11.2';

create diskgroup REDO1 external redundancy disk '/dev/mapper/redo1' attribute 'compatible.asm' = '11.2';

create diskgroup REDO2 external redundancy disk '/dev/mapper/redo2' attribute 'compatible.asm' = '11.2';

节点 2 上 mount 磁盘组：

alter diskgroup DATA mount;

alter diskgroup FRI mount;

alter diskGroup REDO1 mount;

alter diskGroup REDO2 mount;

安装数据库RDBMS

5.1 配置响应文件

预检查再 GI 安装时已经执行这里可以不再执行

[root@orcl1 response]# cat db.rsp

oracle.install.responseFileVersion=/oracle/install/rspfmt_dbinstall_response_schema_v11_2_0
oracle.install.option=INSTALL_DB_SWONLY
ORACLE_HOSTNAME=orcl1
UNIX_GROUP_NAME=oinstall
INVENTORY_LOCATION=/u01/app/oracle/oraInventory
SELECTED_LANGUAGES=en
ORACLE_HOME=/u01/app/oracle/product/11.2.0/db_1
ORACLE_BASE=/u01/app/oracle/
oracle.install.db.InstallEdition=EE
oracle.install.db.EEOptionsSelection=false
oracle.install.db.optionalComponents=oracle.rdbms.partitioning:11.2.0.4.0,oracle.oraolap:11.2.0.4.0,oracle.rdbms.dm:11.2.0.4.0,oracle.rdbms.dv:11.2.0.4.0,oracle.rdbms.lbac:11.2.0.4.0,oracle.rdbms.rat:11.2.0.4.0
oracle.install.db.DBA_GROUP=dba
oracle.install.db.OPER_GROUP=oper
oracle.install.db.CLUSTER_NODES=orcl1,orcl2
oracle.install.db.isRACOneInstall=false
oracle.install.db.racOneServiceName=
oracle.install.db.config.starterdb.type=GENERAL_PURPOSE
oracle.install.db.config.starterdb.globalDBName=
oracle.install.db.config.starterdb.SID=
oracle.install.db.config.starterdb.characterSet=AL32UTF8
oracle.install.db.config.starterdb.memoryOption=false
oracle.install.db.config.starterdb.memoryLimit=
oracle.install.db.config.starterdb.installExampleSchemas=false
oracle.install.db.config.starterdb.enableSecuritySettings=true
oracle.install.db.config.starterdb.password.ALL=
oracle.install.db.config.starterdb.password.SYS=
oracle.install.db.config.starterdb.password.SYSTEM=
oracle.install.db.config.starterdb.password.SYSMAN=
oracle.install.db.config.starterdb.password.DBSNMP=
oracle.install.db.config.starterdb.control=DB_CONTROL
oracle.install.db.config.starterdb.gridcontrol.gridControlServiceURL=
oracle.install.db.config.starterdb.automatedBackup.enable=false
oracle.install.db.config.starterdb.automatedBackup.osuid=
oracle.install.db.config.starterdb.automatedBackup.ospwd=
oracle.install.db.config.starterdb.storageType=
oracle.install.db.config.starterdb.fileSystemStorage.dataLocation=
oracle.install.db.config.starterdb.fileSystemStorage.recoveryLocation=
oracle.install.db.config.asm.diskGroup=
oracle.install.db.config.asm.ASMSNMPPassword=
MYORACLESUPPORT_USERNAME=
MYORACLESUPPORT_PASSWORD=
SECURITY_UPDATES_VIA_MYORACLESUPPORT=false
DECLINE_SECURITY_UPDATES=true
PROXY_HOST=
PROXY_PORT=
PROXY_USER=
PROXY_PWD=
PROXY_REALM=
COLLECTOR_SUPPORTHUB_URL=
oracle.installer.autoupdates.option=SKIP_UPDATES
oracle.installer.autoupdates.downloadUpdatesLoc=
AUTOUPDATES_MYORACLESUPPORT_USERNAME=
AUTOUPDATES_MYORACLESUPPORT_PASSWORD=

5.2 执行rdbms安装

在安装 rdbms 前，需要编辑一个脚本，因为 oracle11G 在 linux7 上面安装有bug

vi edit_emagent.sh

#/bin/bash

[ -f ~/.bash_profile ]&& . ~/.bash_profile

while true

do

   if [ -f $ORACLE_HOME/sysman/lib/ins_emagent.mk ];then

       sed -i 's/$(MK_EMAGENT_NMECTL)/$(MK_EMAGENT_NMECTL)-lnnz11/g' $ORACLE_HOME/sysman/lib/ins_emagent.mk

       break

   fi

done

bash edit_emagent.sh

执行安装

[oracle@orcl1 oracle]$ ./runInstaller -ignorePrereq -force -silent -responseFile /home/soft/database/response/db.rsp -showProgress

然后按照要求使用 root 用户在两个节点执行 root.sh 脚本

数据库安装完毕！

创建数据库实例

6.1 准备响应文件

[oracle@orcl1 response]$ cat ca.rsp

[GENERAL]
RESPONSEFILE_VERSION = "11.2.0"
OPERATION_TYPE = "createDatabase"
[CREATEDATABASE]
GDBNAME=orcl
SID=orcl
NODELIST=orcl1,orcl2
TEMPLATENAME = /u01/app/oracle/product/11.2.0/db_1/assistants/dbca/templates/General_Purpose.dbc
SYSPASSWORD = oracle
SYSTEMPASSWORD = oracle
DATAFILEDESTINATION = +DATA
STORAGETYPE=ASM
DISKGROUPNAME=DATA
CHARACTERSET = "AL32UTF8"
NATIONALCHARACTERSET= "UTF8"
TOTALMEMORY = "1600" #视实际内存而定

6.2 执行实例安装

[oracle@orcl1 database]$ dbca -silent -responsfile /u01/soft/database/response/ca.rsp

对于密码不符合建议的 warning 可自行忽略

至此整个 RAC 集群安装完毕！

安装后验证

7.1 状态验证

## [grid@orcl1 ~]$ crsctl stat res -t
## NAME TARGET STATE SERVER STATE_DETAILS
## Local Resources

ora.DATA.dg

              ONLINE ONLINE orcl1

              ONLINE ONLINE orcl2

ora.LISTENER.lsnr

              ONLINE ONLINE orcl1

              ONLINE ONLINE orcl2

ora.OCR.dg

              ONLINE ONLINE orcl1

              ONLINE ONLINE orcl2

ora.asm

              ONLINE ONLINE orcl1 Started

              ONLINE ONLINE orcl2 Started

ora.gsd

              OFFLINE OFFLINE orcl1

              OFFLINE OFFLINE orcl2

ora.net1.network

              ONLINE ONLINE orcl1

              ONLINE ONLINE orcl2

ora.ons

              ONLINE ONLINE orcl1
##               ONLINE ONLINE orcl2
## Cluster Resources

ora.LISTENER_SCAN1.lsnr

     1 ONLINE ONLINE orcl1

ora.cvu

     1 OFFLINE OFFLINE

ora.oc4j

     1 ONLINE OFFLINE

ora.orcl.db

     1 ONLINE ONLINE orcl1 Open

     2 ONLINE ONLINE orcl2 Open

ora.orcl1.vip

     1 ONLINE ONLINE orcl1

ora.orcl2.vip

     1 ONLINE ONLINE orcl2

ora.scan1.vip

     1 ONLINE ONLINE orcl1

7.2 SQLplus 验证

SQL> set linesize 300

SQL> select instance_number,instance_name,host_name,status from gv$instance;
## INSTANCE_NUMBER INSTANCE_NAME HOST_NAME STATUS ---------------- ---------------------------------------------------------------- ------------

             1 orcl1 orcl1 OPEN

2 orcl2 orcl2 OPEN