1.1服務(wù)器基本信息

[root@db02 software]# cat /etc/issue

Oracle Linux Server release 6.6

Kernel \r on an \m

[root@db02 software]# uname -a

Linux db02 3.8.13-55.1.2.el6uek.x86_64 #2 SMP Thu Dec 18 00:15:51 PST 2014 x86_64 x86_64 x86_64 GNU/Linux

1.2軟件信息

Db11.2.0.4：p13390677_112040_Linux-x86-64_1of7.zip

p13390677_112040_Linux-x86-64_2of7.zip

grid：      p13390677_112040_Linux-x86-64_3of7.zip 

安裝介質(zhì)存放在/u01/software中

p13390677_112040_Linux-x86-64_1of7.zip、p13390677_112040_Linux-x86-64_2of7.zip解壓生成datebase目錄；

p13390677_112040_Linux-x86-64_3of7.zip解壓生成grid目錄

1.3本地盤配置

將500GB的本地盤創(chuàng)建文件系統(tǒng)掛載到/u01目錄下面，并修改/etc/fstab，讓開(kāi)機(jī)自動(dòng)掛載/u01文件系統(tǒng)。

[root@db01 ~]# df -h

Filesystem     Size Used Avail Use% Mounted on

/dev/vda1       20G 3.0G  16G 16% /

tmpfs          8.0G 132M 7.9G  2% /dev/shm

/dev/vdc1      493G  23G 445G  5% /u01

二 基礎(chǔ)配置

基礎(chǔ)配置需要在兩個(gè)節(jié)點(diǎn)都要操作的。

2.1網(wǎng)絡(luò)配置

2.1.1 Host文件

#Public

192.168.0.2 db01

192.168.0.3 db02

#Virtual

192.168.0.102 db01-vip

192.168.0.103 db02-vip

#Private

192.168.101.2 db01-priv

192.168.101.3 db02-priv

#SCAN

192.168.0.100 db-scan

2.1.2網(wǎng)絡(luò)配置文件

節(jié)點(diǎn)1

[root@db01 ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth0

HWADDR=52:54:46:A6:94:40

TYPE=Ethernet

BOOTPROTO=none

IPADDR=192.168.101.2

PREFIX=24

DEFROUTE=yes

IPV4_FAILURE_FATAL=yes

IPV6INIT=no

NAME=eth0

UUID=d790a693-882a-4b5e-bc9f-44ac2706c5b7

ONBOOT=yes

[root@db01 ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth2

DEVICE=eth2

TYPE=Ethernet

IPADDR=192.168.0.2

NETMASK=255.255.255.0

GATEWAY=192.168.0.1

DEFROUTE=yes

NAME=eth2

ONBOOT=yes

HWADDR=52:54:a1:2a:34:3f

[root@db01 ~]#

節(jié)點(diǎn)2

[root@db02 ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth0

HWADDR=52:54:72:2C:50:63

TYPE=Ethernet

BOOTPROTO=none

IPADDR=192.168.101.3

PREFIX=24

DEFROUTE=yes

IPV4_FAILURE_FATAL=yes

IPV6INIT=no

NAME=eth0

UUID=5c37a16a-e9ac-4cf6-a244-c4d0faa592db

ONBOOT=yes

DEVICE=eth0

[root@db02 ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth2

DEVICE=eth2

HWADDR=52:54:b0:c6:97:26

TYPE=Ethernet

IPADDR=192.168.0.3

NETMASK=255.255.255.0

GATEWAY=192.168.0.1

DEFROUTE=yes

NAME=eth2

ONBOOT=yes

兩個(gè)節(jié)點(diǎn)都要操作：

關(guān)閉networkmanager

[root@i-zq8ytr3k network-scripts]# chkconfig NetworkManager --list

NetworkManager 0:off  1:off  2:on   3:on   4:on   5:on   6:off

[root@i-zq8ytr3k network-scripts]# chkconfig NetworkManager off

[root@i-zq8ytr3k network-scripts]# chkconfig NetworkManager --list

NetworkManager 0:off  1:off  2:off  3:off  4:off  5:off  6:off

重啟網(wǎng)絡(luò)：

service network restart

關(guān)閉防火墻

service iptables stop

chkconfig iptables off

關(guān)閉selinux

修改：/etc/selinux/config內(nèi)容改為：SELINUX=disabled

setenforce 0

2.2創(chuàng)建用戶

兩個(gè)節(jié)點(diǎn)都要操作：

groupadd -g 500 dba

 groupadd -g 501 oinstall

 groupadd -g 502 asmadmin

 useradd -g oinstall -u 500 oracle

 useradd -g oinstall -u 501 grid

 usermod -G dba oracle

 usermod -G dba grid

 usermod -aG asmadmin grid

[root@db02 software]# id oracle

uid=500(oracle) gid=501(oinstall)組=501(oinstall),500(dba)

[root@db02 software]# id grid

uid=501(grid) gid=501(oinstall)組=501(oinstall),500(dba),502(asmadmin)

[root@db01 ~]# id oracle

uid=500(oracle) gid=501(oinstall) groups=501(oinstall),500(dba)

[root@db01 ~]# id grid

uid=501(grid) gid=501(oinstall) groups=501(oinstall),500(dba),502(asmadmin)

[root@db01 ~]#

2.2.1設(shè)置profile

后續(xù)要將oracle的環(huán)境變量ORACLE_SID分別改為oadb1和oadb2

節(jié)點(diǎn)1：

Oracle用戶

[oracle@db01 ~]$ cat .bash_profile

# .bash_profile

# Get the aliases and functions

if [ -f ~/.bashrc ]; then

       . ~/.bashrc

fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH

TMP=/tmp; export TMP   

TMPDIR=$TMP; export TMPDIR

ORACLE_BASE=/u01/app/oracle; export ORACLE_BASE   

ORACLE_HOME=$ORACLE_BASE/product/11.2.0/db_1; export ORACLE_HOME   

ORACLE_SID=db1; export ORACLE_SID   

ORACLE_TERM=xterm; export ORACLE_TERM

PATH=/usr/sbin:$PATH; export PATH

PATH=$ORACLE_HOME/bin:$PATH; export PATH

LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib; export LD_LIBRARY_PATH

CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib;

grid用戶：

[grid@db01 ~]$ cat .bash_profile

# .bash_profile

# Get the aliases and functions

if [ -f ~/.bashrc ]; then

       . ~/.bashrc

fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH

TMP=/tmp; export TMP   

TMPDIR=$TMP; export TMPDIR

ORACLE_SID=+ASM1; export ORACLE_SID   

ORACLE_BASE=/u01/app/grid; export ORACLE_BASE

ORACLE_HOME=/u01/app/11.2.0/grid/product/db_1; export ORACLE_HOME

NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"; export NLS_DATE_FORMAT

THREADS_FLAG=native; export THREADS_FLAG

PATH=$ORACLE_HOME/bin:$PATH; export PATHTHREADS_FLAG=native; export THREADS_FLAG

PATH=$ORACLE_HOME/bin:$PATH; export PATH

節(jié)點(diǎn)2

Oracle用戶：

[oracle@db02 ~]$ cat .bash_profile

# .bash_profile

# Get the aliases and functions

if [ -f ~/.bashrc ]; then

       . ~/.bashrc

fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH

TMP=/tmp; export TMP   

TMPDIR=$TMP; export TMPDIR

ORACLE_BASE=/u01/app/oracle; export ORACLE_BASE   

ORACLE_HOME=$ORACLE_BASE/product/11.2.0/db_1; export ORACLE_HOME   

ORACLE_SID=db2; export ORACLE_SID   

ORACLE_TERM=xterm; export ORACLE_TERM

PATH=/usr/sbin:$PATH; export PATH

PATH=$ORACLE_HOME/bin:$PATH; export PATH

LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib; export LD_LIBRARY_PATH

CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib;

Grid用戶：

[grid@db02 ~]$ cat .bash_profile

# .bash_profile

# Get the aliases and functions

if [ -f ~/.bashrc ]; then

       . ~/.bashrc

fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH

TMP=/tmp; export TMP   

TMPDIR=$TMP; export TMPDIR

ORACLE_SID=+ASM2; export ORACLE_SID   

ORACLE_BASE=/u01/app/grid; export ORACLE_BASE

ORACLE_HOME=/u01/app/11.2.0/grid/product/db_1; export ORACLE_HOME

NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"; export NLS_DATE_FORMAT

THREADS_FLAG=native; export THREADS_FLAG

PATH=$ORACLE_HOME/bin:$PATH; export PATHTHREADS_FLAG=native; export THREADS_FLAG

PATH=$ORACLE_HOME/bin:$PATH; export PATH

2.3配置路徑

兩臺(tái)服務(wù)器都要配置：

mkdir -p /u01/app

 chown -R root:oinstall /u01/app

 mkdir -p /u01/app/11.2.0/ 

 mkdir -p /u01/app/grid/ 

 mkdir -p /u01/app/oracle/

 chown -R grid:oinstall /u01/app/11.2.0

 chown -R grid:oinstall /u01/app/grid

 chown -R oracle:oinstall /u01/app/oracle

2.4安裝rpm包

配置本地鏡像

將iso文件掛載

mount -t iso9660 -o loop /u01/software/V52218-01-OEL-6.6.iso /u01/software/mnt

重命名之前的文件

Mv /etc/yum.repos.d/public-yum-ol6.repo /etc/yum.repos.d/public-yum-ol6.repo_bak

[root@db02 ~]# cat /etc/yum.repos.d/public-yum-ol6.repo

[Server]

name=Oracle Linux $releasever Beta - $basearch - Server

baseurl=file:///u01/software/mnt

enabled=1

gpgcheck=1

gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle

安裝rpm

yum install gcc compat-libstdc++-33 elfutils-libelf-devel gcc-c++ libaio-devel libstdc++-devel ksh -y

后續(xù)根據(jù)實(shí)際情況也許需要安裝一些新的包，都可以在/u01/software/mnt/Packages中找到

2.5 sysctl.conf文件

修改/etc/sysctl.conf

新加

##add by oracle

kernel.shmmax = 4294967295

kernel.shmall = 2097152

kernel.shmmni = 4096

kernel.sem = 1100 32000 100 128

fs.file-max = 6815744

net.ipv4.ip_local_port_range = 9000 65000

net.core.rmem_default = 1048576

net.core.rmem_max = 4194304

net.core.wmem_default = 262144

net.core.wmem_max = 1048576

vm.min_free_kbytes= 1048576

fs.aio-max-nr= 1048576

設(shè)置生效：

/sbin/sysctl -p

2.6 limit文件

/etc/security/limits.conf添加：

#add by oracle

oracle hard nofile 65536

oracle  soft nofile 4096

oracle  hard nproc 16384

oracle  soft nproc 2047

grid hard nofile 65536

grid soft nofile 1024

grid hard nproc 16384

grid soft nproc 2047

2.7共享盤安裝

在云平臺(tái)上分配共享存儲(chǔ)之后，最好在界面上設(shè)置可以訪問(wèn)共享存儲(chǔ)的客戶端，本次設(shè)置的是：iqn.2014-12.com.qingcloud:e594a0ee9b82

和iqn.2014-12.com.qingcloud:5ba958139a1

1安裝工具包

yum install -y iscsi-initiator-utils

2根據(jù)前綴生成客戶端id

iscsi-iname -p iqn.2014-12.com.qingcloud

結(jié)果：

[root@db02 ~]# iscsi-iname -p iqn.2014-12.com.qingcloud

iqn.2014-12.com.qingcloud:e594a0ee9b82

[root@db01 ~]# iscsi-iname -p iqn.2014-12.com.qingcloud

iqn.2014-12.com.qingcloud:5ba958139a1

3修改配置文件

/etc/iscsi/initiatorname.iscsi 修改為下面的信息（名稱是界面上看到的）

節(jié)點(diǎn)1:

[root@db01 ~]# cat /etc/iscsi/initiatorname.iscsi

InitiatorName=iqn.2014-12.com.qingcloud:5ba958139a1

節(jié)點(diǎn)2：

[root@db02 ~]# cat /etc/iscsi/initiatorname.iscsi

InitiatorName=iqn.2014-12.com.qingcloud:e594a0ee9b82

4客戶端發(fā)現(xiàn)（兩個(gè)服務(wù)器都要配置）

[root@db01 iscsi]# iscsiadm -m discovery -t st -p 192.168.0.8

Starting iscsid:                                          [ OK ]

192.168.0.8:3260,1 iqn.2014-12.com.qingcloud.s2:sn.data

[root@i-zq8ytr3k iscsi]# iscsiadm -m node

192.168.0.8:3260,1 iqn.2014-12.com.qingcloud.s2:sn.data

5注冊(cè)（兩個(gè)服務(wù)器都要配置）

[root@db01 iscsi]# iscsiadm -m node -T iqn.2014-12.com.qingcloud.s2:sn.data -p 192.168.0.8 --login

Logging in to [iface: default, target: iqn.2014-12.com.qingcloud.s2:sn.data, portal: 192.168.0.8,3260] (multiple)

Login to [iface: default, target: iqn.2014-12.com.qingcloud.s2:sn.data, portal: 192.168.0.8,3260] successful.

6查看結(jié)果

[root@db02 ~]# lsblk

NAME  MAJ:MIN RM SIZE RO TYPE MOUNTPOINT

sda     8:0   0  10G 0 disk

sdb     8:16  0 500G 0 disk

sdc     8:32  0  10G 0 disk

sdd     8:48  0  10G 0 disk

vda   251:0   0  20G 0 disk

`-vda1 251:1   0  20G 0 part /

vdb   251:16  0  16G 0 disk [SWAP]

vdc   251:32  0 500G 0 disk

`-vdc1 251:33  0 500G 0 part /u01

[root@db01 ~]# lsblk

NAME  MAJ:MIN RM SIZE RO TYPE MOUNTPOINT

sda     8:0   0  10G 0 disk

sdb     8:16  0 500G 0 disk

sdc     8:32  0  10G 0 disk

sdd     8:48  0  10G 0 disk

vda   251:0   0  20G 0 disk

`-vda1 251:1   0  20G 0 part /

vdb   251:16  0  16G 0 disk [SWAP]

vdc   251:32  0 500G 0 disk

`-vdc1 251:33  0 500G 0 part /u01

7配置開(kāi)啟自動(dòng)啟動(dòng)識(shí)別

iscsiadm -m node -T iqn.2014-12.com.qingcloud.s2:sn.data -p 192.168.0.8 --op update -n node.startup -v automatic

8配置asm能識(shí)別的盤

echo "options=--whitelisted --replace-whitespace">/etc/scsi_id.config

rm -f /etc/udev/rules.d/99-oracle-asmdevices.rules

i=1

id=''

for x in a b c d

 do

   id=`scsi_id --whitelisted --replace-whitespace --device=/dev/sd$x`

   echo "KERNEL==\"sd*\", SUBSYSTEM==\"block\", PROGRAM==\"/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/\$name\", RESULT==\"$id\", NAME=\"asm-disk$i\", OWNER=\"grid\", GROUP=\"asmadmin\", MODE=\"0660\"">>/etc/udev/rules.d/99-oracle-asmdevices.rules

   let i++

 done

 /sbin/start_udev

最后文件生成的/etc/udev/rules.d/99-oracle-asmdevices.rules文件是：

[root@db01 ~]# cat /etc/udev/rules.d/99-oracle-asmdevices.rules

KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/$name", RESULT=="36001405745529a22afc4f8fb1a685e36", NAME="asm-disk1", OWNER="grid", GROUP="asmadmin", MODE="0660"

KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/$name", RESULT=="36001405e371bbbb031b4547b5fe217d1", NAME="asm-disk2", OWNER="grid", GROUP="asmadmin", MODE="0660"

KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/$name", RESULT=="36001405ff11eeac99ac40f69e34a3ec5", NAME="asm-disk3", OWNER="grid", GROUP="asmadmin", MODE="0660"

KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/$name", RESULT=="3600140517aafddef37c4305a3e6930ee", NAME="asm-disk4", OWNER="grid", GROUP="asmadmin", MODE="0660"

查看生成的設(shè)備符號(hào)

[root@db01 ~]# ll /dev/asm*

brw-rw---- 1 grid asmadmin 8, 0 Mar 23 19:12 /dev/asm-disk1

brw-rw---- 1 grid asmadmin 8, 16 Mar 23 19:12 /dev/asm-disk2

brw-rw---- 1 grid asmadmin 8, 32 Mar 23 19:12 /dev/asm-disk3

brw-rw---- 1 grid asmadmin 8, 48 Mar 23 19:12 /dev/asm-disk4

[root@db02 ~]# ll /dev/asm*

brw-rw---- 1 grid asmadmin 8, 0 Mar 23 19:12 /dev/asm-disk1

brw-rw---- 1 grid asmadmin 8, 16 Mar 23 19:12 /dev/asm-disk2

brw-rw---- 1 grid asmadmin 8, 32 Mar 23 19:12 /dev/asm-disk3

brw-rw---- 1 grid asmadmin 8, 48 Mar 23 19:12 /dev/asm-disk4

其中asm-disk2是500GB，用作數(shù)據(jù)。另外3塊10GB的盤用作集群vote disk。

2.8關(guān)閉ntp

Service ntpd stop

Chkconfig ntpd off

2.9配置操作系統(tǒng)用戶互信

修改ssh配置文件：/etc/ssh/ssh_config，注釋下面兩行

       #StrictHostKeyChecking no

       #UserKnownHostsFile /dev/null

Oracle和grid用戶都要操作

mkdir ~/.ssh

 chmod 700 ~/.ssh

 ssh-keygen -t rsa

 ssh-keygen -t dsa

在節(jié)點(diǎn)1：

$ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

$ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

$ ssh db02 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

$ ssh db02 cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

$ scp ~/.ssh/authorized_keys db02:~/.ssh/authorized_keys

之后分別使用grid、oracle用戶在兩個(gè)節(jié)點(diǎn)都運(yùn)行：

Ssh db01

ssh db02

ssh db01-priv

ssh db02-priv

按照提示輸入yes，不輸入密碼能正常登陸表示正常

三 預(yù)檢查

節(jié)點(diǎn)1，使用grid用戶，切換至目錄：

/u01/software/u01，運(yùn)行命令：

./runcluvfy.sh stage -pre crsinst -n db01,db02 -fixup -verbose

結(jié)果如下，按照需求再安裝rpm，讓failed成功：

[grid@db01 grid]$ ./runcluvfy.sh stage -pre crsinst -n db01,db02 -fixup -verbose

四 圖形化安裝

為避免網(wǎng)絡(luò)中途掉線，所有圖形化均在web控制臺(tái)上分別使用grid、oracle用戶登陸db01進(jìn)行操作。

4.1安裝grid軟件

使用grid用戶登陸db01，切換至目錄：

/u01/software/grid

執(zhí)行./runInstaller

選擇大小為10G的/dev/asm-disk1, /dev/asm-disk3,/dev/asm-disk4

這里需要點(diǎn)擊“change discovery path”，設(shè)置路徑為/dev/asm*

在進(jìn)行環(huán)境檢查時(shí)，會(huì)報(bào)錯(cuò)

這個(gè)警告在青云平臺(tái)也有說(shuō)明：

注解 grid安裝環(huán)境檢查腳本在Oracle Linux 6上面的一個(gè)bug導(dǎo)致錯(cuò)誤”PRVF-9802”無(wú)法解決，具體原因請(qǐng)見(jiàn)Debug Cluvfy error ERROR: PRVF-9802。

勾選忽略這個(gè)警告。

彈出腳本分別使用root在兩個(gè)服務(wù)器執(zhí)行：

/u01/app/oraInventory/orainstRoot.sh

/u01/app/11.2.0/grid/product/db_1/root.sh

第二個(gè)腳本執(zhí)行會(huì)比較好時(shí)間，大概20分鐘。

4.1.1執(zhí)行腳本

節(jié)點(diǎn)1

[root@db01 ~]# /u01/app/oraInventory/orainstRoot.sh

Changing permissions of /u01/app/oraInventory.

Adding read,write permissions for group.

Removing read,write,execute permissions for world.

Changing groupname of /u01/app/oraInventory to oinstall.

The execution of the script is complete.

[root@db01 ~]# /u01/app/11.2.0/grid/product/db_1/root.sh

[root@db01 ~]# su - grid

[grid@db01 ~]$ crs_stat -t

Name          Type          Target   State    Host       

------------------------------------------------------------

ora.CRS.dg    ora....up.type ONLINE   ONLINE   db01       

ora....N1.lsnr ora....er.type ONLINE   ONLINE   db01       

ora.asm       ora.asm.type  ONLINE   ONLINE   db01       

ora.cvu       ora.cvu.type  ONLINE   ONLINE   db01       

ora....SM1.asm application   ONLINE   ONLINE   db01       

ora.db01.gsd  application   OFFLINE  OFFLINE              

ora.db01.ons  application   ONLINE   ONLINE   db01       

ora.db01.vip  ora....t1.type ONLINE   ONLINE   db01       

ora.gsd       ora.gsd.type  OFFLINE  OFFLINE              

ora....network ora....rk.type ONLINE   ONLINE   db01       

ora.oc4j      ora.oc4j.type ONLINE   ONLINE   db01       

ora.ons       ora.ons.type  ONLINE   ONLINE   db01       

ora.scan1.vip ora....ip.type ONLINE   ONLINE   db01

節(jié)點(diǎn)2

[root@db02 ~]# /u01/app/oraInventory/orainstRoot.sh

Changing permissions of /u01/app/oraInventory.

Adding read,write permissions for group.

Removing read,write,execute permissions for world.

Changing groupname of /u01/app/oraInventory to oinstall.

The execution of the script is complete.

[root@db02 ~]# /u01/app/11.2.0/grid/product/db_1/root.sh

[root@db02 ~]# su - grid

[grid@db02 ~]$ crs_stat -t

4.2創(chuàng)建asm磁盤組

4.3安裝oracle軟件

Oracle用戶登陸，切換至目錄：

/u01/software/database