安装openGauss一主两备含3CM版

机器列表

修改主机名

hostnamectl set-hostname replicate1
hostnamectl set-hostname replicate2
hostnamectl set-hostname replicate3
复制

时区

\cp /usr/share/zoneinfo/Asia/Beijing /etc/localtime
复制

firewalld

systemctl disable firewalld.service
systemctl stop firewalld.service
systemctl status firewalld
复制

selinux

vim /etc/selinux/config
SELINUX=disabled
reboot
复制

详情

[root@replicate1 script]# uname -r
5.10.0-153.12.0.92.oe2203sp2.x86_64
[root@replicate1 script]#
[root@replicate1 ~]# cat /etc/os-release
NAME="openEuler"
VERSION="22.03 (LTS-SP2)"
ID="openEuler"
VERSION_ID="22.03"
PRETTY_NAME="openEuler 22.03 (LTS-SP2)"
ANSI_COLOR="0;31"
[root@replicate1 ~]#
复制

操作需要支持软件支持Python 3.7.X

- openEuler：支持Python 3.7.X
- CentOS：支持Python 3.6.X
- 麒麟：支持Python 3.7.X
- 说明 python需要通过--enable-shared方式编译
复制

安装python3

yum install openssl-devel
yum install libaio-devel readline-devel expect
yum install -y libaio-devel flex bison ncurses-devel glibc-devel patch redhat-lsb-core
yum install libffi-devel
https://www.python.org/ftp/python/3.7.9/Python-3.7.9.tgz
tar -zxvf Python-3.7.9.tgz
cd Python-3.7.9/
./configure --enable-shared --prefix=/usr/local/python3.7
make
make install
ln -s /usr/local/python3.7/bin/python3 /usr/bin/python3 
ln -s /usr/local/python3.7bin/pip3 /usr/bin/pip3
复制

加入环境变量

vim ~/.bashrc
export PATH=/usr/local/python3.7/bin:$PATH
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/python3.7/lib
export LD_LIBRARY_PATH=/soft/openGauss/script/gspylib/clib$LD_LIBRARY_PATH
复制

测试

[root@replicate1 script]#/usr/bin/env python3
Python3.7.9(default,Aug22024,16:01:56)
[GCC 10.3.1] on linux
Type"help","copyright","credits"or"license"for more information.
>>>
复制

建立用户并设置

groupadd dbgrp
useradd omm -g dbgrp
usermod -g dbgrp omm
passwd omm
omm
复制

安装必备软件(全安装)

yum install libaio-devel readline-devel expect bzip2 -y
复制

安装必备软件（root)

scp -rp /usr/local/python3.7 replicate2:/usr/local/python3.7
scp -rp /usr/local/python3.7 replicate1:/usr/local/python3.7
vim ~/.bashrc
export PATH=/usr/local/python3.7/bin:$PATH
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/python3.7/lib
复制

配置文件

<?xml version="1.0" encoding="UTF-8"?>
<ROOT>
<!-- openGauss整体信息 -->
<CLUSTER>
<!-- 数据库名称 -->
<PARAMname="clusterName"value="dbCluster"/>
<!-- 数据库节点名称(hostname) -->
<PARAMname="nodeNames"value="replicate1,replicate2,replicate3"/>
<!-- 数据库安装目录-->
<PARAMname="gaussdbAppPath"value="/gauss/huawei/install/app"/>
<!-- 日志目录-->
<PARAMname="gaussdbLogPath"value="/gauss/var/log/omm"/>
<!-- 临时文件目录-->
<PARAMname="tmpMppdbPath"value="/gauss/huawei/tmp"/>
<!-- 数据库工具目录-->
<PARAMname="gaussdbToolPath"value="/gauss/huawei/install/om"/>
<!-- 数据库core文件目录-->
<PARAMname="corePath"value="/gauss/huawei/corefile"/>
<!-- 节点IP，与数据库节点名称列表一一对应 -->
<PARAMname="backIp1s"value="10.0.2.8,10.0.2.9,10.0.2.10"/>
</CLUSTER>
<!-- 每台服务器上的节点部署信息 -->
<DEVICELIST>
<!-- 节点1上的部署信息 -->
<DEVICEsn="replicate1">
<!-- 节点1的主机名称 -->
<PARAMname="name"value="replicate1"/>
<!-- 节点1所在的AZ及AZ优先级 -->
<PARAMname="azName"value="AZ1"/>
<PARAMname="azPriority"value="1"/>
<!-- 节点1的IP，如果服务器只有一个网卡可用，将backIp1和sshIp1配置成同一个IP -->
<PARAMname="backIp1"value="10.0.2.8"/>
<PARAMname="sshIp1"value="10.0.2.8"/>
<!--dbnode-->
<PARAMname="dataNum"value="1"/>
<PARAMname="dataPortBase"value="8000"/>
<PARAMname="dataNode1"value="/opt/software/install/data/dn, replicate2,/opt/software/install/data/dn,replicate3,/opt/software/install/data/dn"/>
<PARAMname="dataNode1_syncNum"value="0"/>
<!--cm-->
<PARAMname="cmDir"value="/opt/software/data/cmserver"/>
<PARAMname="cmServerPortBase"value="15000"/>
<PARAMname="cmsNum"value="1"/>
<PARAMname="cmServerlevel"value="1"/>
<PARAMname="cmServerListenIp1"value="10.0.2.8,10.0.2.9,10.0.2.10"/>
<PARAMname="cmServerRelation"value="replicate1,replicate2,replicate3"/>
</DEVICE>
<!-- 节点2上的节点部署信息，其中“name”的值配置为主机名称 -->
<DEVICEsn="replicate2">
<!-- 节点2的主机名称 -->
<PARAMname="name"value="replicate2"/>
<!-- 节点2所在的AZ及AZ优先级 -->
<PARAMname="azName"value="AZ1"/>
<PARAMname="azPriority"value="1"/>
<!-- 节点2的IP，如果服务器只有一个网卡可用，将backIp1和sshIp1配置成同一个IP -->
<PARAMname="backIp1"value="10.0.2.9"/>
<PARAMname="sshIp1"value="10.0.2.9"/>
<!-- cm配置 -->
<PARAMname="cmDir"value="/opt/software/data/cmserver"/>
<PARAMname="cmServerPortBase"value="15000"/>
</DEVICE>
<!-- 节点3上的节点部署信息，其中“name”的值配置为主机名称 -->
<DEVICEsn="replicate3">
<!-- 节点3的主机名称 -->
<PARAMname="name"value="replicate3"/>
<!-- 节点3所在的AZ及AZ优先级 -->
<PARAMname="azName"value="AZ1"/>
<PARAMname="azPriority"value="1"/>
<!-- 节点3的IP，如果服务器只有一个网卡可用，将backIp1和sshIp1配置成同一个IP -->
<PARAMname="backIp1"value="10.0.2.10"/>
<PARAMname="sshIp1"value="10.0.2.10"/>
<!-- cm配置 -->
<PARAMname="cmDir"value="/opt/software/data/cmserver"/>
<PARAMname="cmServerPortBase"value="15000"/>
</DEVICE>
</DEVICELIST>
</ROOT>

复制

[root@replicate3 script]# /usr/local/python3.7/bin/python3 gs_preinstall -U omm -G dbgrp -X /home/omm/3cm.xml --unused-third-party
复制

预安装

Are you sure you want to create trust for root (yes/no)?yes
Are you sure you want to create the user[omm] and create trust for it (yes/no)? no
复制

开始

[root@replicate3 script]#/usr/local/python3.7/bin/python3 gs_preinstall -U omm -G dbgrp -X /home/omm/3cm.xml --unused-third-party
Parsing the configuration file.
Successfully parsed the configuration file.
Installing the tools on the local node.
Successfully installed the tools on the local node.
Are you sure you want to create trust for root (yes/no)?yes
Please enter password for root
Password:
Successfully created SSH trust for the root permission user.
Setting host ip env
Successfullyset host ip env.
Distributingpackage.
Begin to distribute package to tool path.
Successfully distribute package to tool path.
Begin to distribute package to package path.
Successfully distribute package to package path.
Successfully distributed package.
Are you sure you want to create the user[omm]and create trust for it (yes/no)?no
Preparing SSH service.
Successfully prepared SSH service.
Installing the tools in the cluster.
Successfully installed the tools in the cluster.
Checking hostname mapping.
Successfullychecked hostname mapping.
Checking OS software.
Successfully check os software.
Checking OS version.
Successfullychecked OS version.
Creating cluster's path.
Successfully created cluster's path.
Setand check OS parameter.
Setting OS parameters.
Successfullyset OS parameters.
Warning:Installation environment contains some warning messages.
Pleaseget more details by"/opt/opengauss3/script/gs_checkos -i A -h replicate1,replicate2,replicate3 --detail".
Setand check OS parameter completed.
Preparing CRON service.
Successfully prepared CRON service.
Setting user environmental variables.
Successfullyset user environmental variables.
Setting the dynamic link library.
Successfullyset the dynamic link library.
SettingCore file
Successfullyset core path.
Setting pssh path
Successfullyset pssh path.
SettingCgroup.
SuccessfullysetCgroup.
Set ARM Optimization.
No need to set ARM Optimization.
Fixing server package owner.
Setting finish flag.
Successfullyset finish flag.
Preinstallation succeeded.
[root@replicate3 script]# su - omm
[omm@replicate3 ~]$ python
Python3.9.9(main,Oct252023,15:16:57)
[GCC 10.3.1] on linux
Type"help","copyright","credits"or"license"for more information.
>>>
[omm@replicate3 ~]$ pwd
/home/omm
[omm@replicate3 ~]$ vim ~/.bashrc

vim ~/.bashrc
export PATH=/usr/local/python3.7/bin:$PATH
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/python3.7/lib

[omm@replicate3 ~]$ exit
logout
[root@replicate3 script]# su - omm
[omm@replicate3 ~]$ python3
Python3.7.9(default,Aug22024,16:01:56)
[GCC 10.3.1] on linux
Type"help","copyright","credits"or"license"for more information.
>>>
[omm@replicate3 ~]$ gs_install -X /home/omm/3cm.xml
Parsing the configuration file.
Check preinstall on every node.
Successfullychecked preinstall on every node.
Creating the backup directory.
Successfully created the backup directory.
begin deploy..
Installing the cluster.
begin prepare InstallCluster..
Checking the installation environment on all nodes.
begin install Cluster..
Installing applications on all nodes.
Successfully installed APP.
begin init Instance..
encrypt cipher and rand files for database.
Please enter password for database:
Please repeat for database:
begin to create CA cert files
The sslcert will be generated in/gauss/huawei/install/app/share/sslcert/om
Create CA files for cm beginning.
Create CA files on directory [/gauss/huawei/install/app_4db7019a/share/sslcert/cm]. file list:['server.key.rand','client.key.rand','server.key.cipher','cacert.pem','client.key.cipher','server.key','server.crt','client.crt','client.key']
Cluster installation is completed.
Configuring.
Deleting instances from all nodes.
Successfully deleted instances from all nodes.
Checking node configuration on all nodes.
Initializing instances on all nodes.
Updating instance configuration on all nodes.
Check consistence of memCheck and coresCheck on database nodes.
Successful check consistence of memCheck and coresCheck on all nodes.
Configuring pg_hba on all nodes.
Configurationis completed.
Starting cluster.
======================================================================
Successfully started primary instance.Waitfor standby instance.
======================================================================
.
Successfully started cluster.
======================================================================
cluster_state      :Normal
redistributing     :No
node_count         :3
DatanodeState
    primary           :1
    standby           :2
    secondary         :0
    cascade_standby   :0
    building          :0
    abnormal          :0
    down              :0

Successfully installed application.
end deploy..
复制

查看状态

[omm@replicate3 ~]$ gs_om -t status --detail
[CMServerState]

node          node_ip         instance                                   state
--------------------------------------------------------------------------------
1  replicate1 10.0.2.81/opt/software/data/cmserver/cm_server Primary
2  replicate2 10.0.2.92/opt/software/data/cmserver/cm_server Standby
3  replicate3 10.0.2.103/opt/software/data/cmserver/cm_server Standby

[ClusterState]

cluster_state   :Normal
redistributing  :No
balanced        :Yes
current_az      : AZ_ALL

[DatanodeState]

node          node_ip         instance                           state
----------------------------------------------------------------------------------
1  replicate1 10.0.2.86001/opt/software/install/data/dn P PrimaryNormal
2  replicate2 10.0.2.96002/opt/software/install/data/dn S StandbyNormal
3  replicate3 10.0.2.106003/opt/software/install/data/dn S StandbyNormal
[omm@replicate3 ~]$
复制

出错

[omm@replicate3 ~]$ id
uid=1000(omm) gid=1000(dbgrp) groups=1000(dbgrp) context=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023
[omm@replicate3 ~]$ gs_install -X /home/omm/3cm.xml
/usr/local/python3.7/lib/python3.7/site-packages/paramiko/pkey.py:100:CryptographyDeprecationWarning:TripleDES has been moved to cryptography.hazmat.decrepit.ciphers.algorithms.TripleDESand will be removed fromthismodulein48.0.0.
"cipher": algorithms.TripleDES,
/usr/local/python3.7/lib/python3.7/site-packages/paramiko/transport.py:259:CryptographyDeprecationWarning:TripleDES has been moved to cryptography.hazmat.decrepit.ciphers.algorithms.TripleDESand will be removed fromthismodulein48.0.0.
"class": algorithms.TripleDES,
Parsing the configuration file.
Check preinstall on every node.
[GAUSS-51400]:Failed to execute the command: python3 '/gauss/huawei/install/om/script/local/CheckPreInstall.py'-U omm -t preinstall.Result:{'replicate1':'Failure','replicate3':'Failure','replicate2':'Failure'}.
Error:
[FAILURE] replicate1:
[FAILURE] replicate2:
[FAILURE] replicate3:
复制

解决(没有批量认证)

[omm@replicate3 ~]$ ssh replicate1
The authenticity of host 'replicate1 (10.0.2.8)' can't be established.
ED25519 key fingerprint is SHA256:7CXNGBUoxKBjW8rCt/q8QXIDuS0rQq7rqAuaZzKP/FM.
This host key is known by the following other names/addresses:
    ~/.ssh/known_hosts:1: 10.0.2.10
    ~/.ssh/known_hosts:3: 10.0.2.9
    ~/.ssh/known_hosts:4: 10.0.2.8
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added 'replicate1' (ED25519) to the list of known hosts.

Authorized users only. All activities may be monitored and reported.

Authorized users only. All activities may be monitored and reported.
Last login: Sun Aug  4 14:58:10 2024
复制

查看所有状态

gs_om -t status --all
复制

端口查看

[omm@replicate3 ~]$ netstat -ntlp
(Not all processes could be identified, non-owned process info
 will not be shown, you would have to be root to see it all.)
ActiveInternet connections (only servers)
ProtoRecv-Q Send-Q LocalAddressForeignAddressState       PID/Program name
tcp        000.0.0.0:220.0.0.0:*               LISTEN      -
tcp        0010.0.2.10:150000.0.0.0:*               LISTEN      286754/cm_server
tcp        0010.0.2.10:150010.0.0.0:*               LISTEN      286754/cm_server
tcp        0010.0.2.10:150020.0.0.0:*               LISTEN      286715/cm_agent
tcp        0010.0.2.10:80000.0.0.0:*               LISTEN      286762/gaussdb
tcp        00127.0.0.1:80000.0.0.0:*               LISTEN      286762/gaussdb
tcp        0010.0.2.10:80010.0.0.0:*               LISTEN      286762/gaussdb
tcp        00127.0.0.1:80010.0.0.0:*               LISTEN      286762/gaussdb
tcp        0010.0.2.10:80050.0.0.0:*               LISTEN      286762/gaussdb
tcp6       00:::22:::*                    LISTEN      -
tcp6       00::1:8000:::*                    LISTEN      286762/gaussdb
tcp6       00::1:8001:::*                    LISTEN      286762/gaussdb
[omm@replicate3 ~]$
复制

连接测试

[omm@replicate3 ~]$ gsql -d postgres -p 8000
gsql ((openGauss 3.0.5 build 4db7019a) compiled at 2024-01-0420:20:21 commit 0last mr  )
Non-SSL connection (SSL connection is recommended when requiring high-security)
Type"help"for help.

openGauss=# create database dh;
ERROR:  cannot execute CREATE DATABASE in a read-only transaction
openGauss=# \q
[omm@replicate3 ~]$
[omm@replicate3 ~]$ gsql -d postgres -p 8000-h 10.0.2.10
gsql: FATAL:Forbid remote connection with trust method!
FATAL:Forbid remote connection with trust method!
[omm@replicate3 ~]$ gsql -d postgres -p 15000-h 10.0.2.10
failed to connect 10.0.2.10:15000.
[omm@replicate3 ~]$
复制

gs_guc [ set | reload ] [-N NODE-NAME] [-I INSTANCE-NAME | -D DATADIR] -h "HOSTTYPE DATABASE USERNAME IPADDR-WITH-IPMASK AUTHMEHOD"
复制

gs_guc set -N all -I all -h "host all jack 10.10.0.30/32 sha256"
复制

[omm@replicate3 ~]$ cm_ctl query -Cvi
[omm@replicate3 ~]$ cm_ctl query -Cvip
复制

在script/gspylib/etc/conf/cluster_config_template.xml获取XML文件模板。以下配置内容为示例，可自行替换。每行信息均有注释进行说明。
复制

相关命令

gs_om -t start
gs_om -t status --detail
gs_om -t stop
gs_ctl switchover -D /opt/software/install/data/dn
gs_ctl failover -D /opt/software/install/data/dn
cm_ctl start
cm_ctl query -Cv
cm_ctl stop
cm_ctl stop -n 1 -D /opt/software/install/data/dn
主备切换
cm_ctl switchover -n 1 -D /opt/software/install/data/dn
cm_ctl failover -n 1 -D /opt/software/install/data/dn

复制

执行命令

[omm@replicate3 cm_server]$ cm_ctl query -Cvp
[CMServerState]

node          instance state
------------------------------
1  replicate1 1Primary
2  replicate2 2Standby
3  replicate3 3Standby

[ClusterState]

cluster_state   :Normal
redistributing  :No
balanced        :Yes
current_az      : AZ_ALL

[DatanodeState]

node          instance        state            | node          instance        state            | node          instance state
---------------------------------------------------------------------------------------------------------------------------------------------------
1  replicate1 60018000       P PrimaryNormal|2  replicate2 60028000       S StandbyNormal|3  replicate3 60038000       S StandbyNormal
[omm@replicate3 cm_server]$

复制

出错

[omm@replicate3 ~]$  gsql -d postgres -p 15000 -h 10.0.2.10
failed to connect 10.0.2.10:15000.
[omm@replicate3 ~]$  gsql -d postgres -p 15000 -h 10.0.2.10
failed to connect 10.0.2.10:15000.
[omm@replicate3 ~]$
复制

日志

==> cm_server-2024-08-04_150336-current.log <==
2024-08-0510:10:50.589 tid=322708 MAIN ERROR:[ProcessStartupPacket] invalid frontend message type 0, nodeId=[(null):0], socket=61,inReadCommand
2024-08-0510:10:50.589 tid=322708 MAIN LOG: connection closed by client
2024-08-0510:10:50.589 tid=322708 MAIN LOG: close connection sock [fd=61], type is0, nodeid 0.
2024-08-0510:11:05.330 tid=322708 MAIN ERROR:[ProcessStartupPacket] invalid frontend message type 0, nodeId=[(null):0], socket=61,inReadCommand
2024-08-0510:11:05.330 tid=322708 MAIN LOG: connection closed by client
2024-08-0510:11:05.330 tid=322708 MAIN LOG: close connection sock [fd=61], type is0, nodeid 0.
复制

日志

[omm@replicate1 ~]$ gsql -d postgres -p 8000
gsql ((openGauss 3.0.5 build 4db7019a) compiled at 2024-01-0420:20:21 commit 0last mr  )
Non-SSL connection (SSL connection is recommended when requiring high-security)
Type"help"for help.

openGauss=# \l
List of databases
Name|Owner|Encoding|Collate|Ctype|Access privileges
-----------+-------+-----------+---------+-------+-------------------
 postgres  | omm   | SQL_ASCII | C       | C     |
 template0 | omm   | SQL_ASCII | C       | C     |=c/omm           +
||||| omm=CTc/omm
 template1 | omm   | SQL_ASCII | C       | C     |=c/omm           +
||||| omm=CTc/omm
(3 rows)

openGauss=# create database dh;
CREATE DATABASE
openGauss=#
复制