etcd实战

INSTALL

基础环境配置

关闭selinux

sed -i 's/^SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux

构建新的YUM

yum install -y wget
mkdir -p /etc/yum.repos.d/bak
mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/bak
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
yum install epel-release
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo

安装基础软件

yum install python-devel gcc zlib zlib-devel openssl-devel tcpdump net-tools lsof telnet ntp vim-* -y
yum install epel-release -y
yum install python-pip -y
pip install --upgrade pip

时间同步

yum install -y crontabs
systemctl restart crond
systemctl enable crond
systemctl reload crond
timedatectl set-timezone Asia/Shanghai
timedatectl set-local-rtc 1
hwclock --systohc --utc
cat >>/var/spool/cron/${USER}<<EOF
*/1 * * * * /usr/sbin/ntpdate ntp1.aliyun.com > /dev/null 2>&1
*/5 * * * * /usr/sbin/ntpdate ntp2.aliyun.com > /dev/null 2>&1
EOF

关闭防火墙服务和NetworkManager服务

systemctl stop firewalld.service
systemctl disable firewalld.service
iptables -F

systemctl stop NetworkManager.service
systemctl disable NetworkManager.service

关闭swap

swapoff -a
sed -i 's/.*swap.*/#&/' /etc/fstab

设置文件描述符

cat >>/etc/rc.local<<EOF
#open files
ulimit -SHn 65535
#stack size
ulimit -s 65535
EOF

优化sshd服务

cp /etc/ssh/sshd_config{,.bak}
cat >/etc/ssh/sshd_config<<EOF
HostKey /etc/ssh/ssh_host_rsa_key
HostKey /etc/ssh/ssh_host_ecdsa_key
HostKey /etc/ssh/ssh_host_ed25519_key
SyslogFacility AUTHPRIV
AuthorizedKeysFile	.ssh/authorized_keys
PasswordAuthentication yes
ChallengeResponseAuthentication no
GSSAPIAuthentication no
GSSAPICleanupCredentials no
UsePAM yes
X11Forwarding yes
AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES
AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT
AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE
AcceptEnv XMODIFIERS
Subsystem	sftp	/usr/libexec/openssh/sftp-server
UseDNS=no
UseDNS=no
IgnoreRhosts yes
EOF

内核优化

cat >/etc/sysctl.conf<<EOF
# 关闭 ipv6
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1

# 避免放大攻击
net.ipv4.icmp_echo_ignore_broadcasts = 1

# 开启恶意 icmp 错误消息保护
net.ipv4.icmp_ignore_bogus_error_responses = 1

# 关闭路由转发
net.ipv4.ip_forward = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0

# 开启反向路径过滤
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1

# 处理无源路由的包
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0

# 关闭 sysrq 功能
kernel.sysrq = 0

# core 文件名中添加 pid 作为扩展名
kernel.core_uses_pid = 1

# 开启 SYN 洪水攻击保护
net.ipv4.tcp_syncookies = 1

# 修改消息队列长度
kernel.msgmnb = 65536
kernel.msgmax = 65536

# 设置最大内存共享段大小 bytes
kernel.shmmax = 68719476736
kernel.shmall = 4294967296

# timewait 的数量，默认 180000
net.ipv4.tcp_max_tw_buckets = 6000
net.ipv4.tcp_sack = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_rmem = 4096 87380 4194304
net.ipv4.tcp_wmem = 4096 16384 4194304
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216

# 每个网络接口接收数据包的速率比内核处理这些包的速率快时，允许送到队列的数据包的最大数目
net.core.netdev_max_backlog = 262144

# 限制仅仅是为了防止简单的 DDoS 攻击
net.ipv4.tcp_max_orphans = 3276800

# 未收到客户端确认信息的连接请求的最大值
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_timestamps = 0

# 内核放弃建立连接之前发送 SYNACK 包的数量
net.ipv4.tcp_synack_retries = 1

# 内核放弃建立连接之前发送 SYN 包的数量
net.ipv4.tcp_syn_retries = 1

# 启用 timewait 快速回收
net.ipv4.tcp_tw_recycle = 1

# 开启重用，允许将 TIME-WAIT sockets 重新用于新的 TCP 连接
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_fin_timeout = 1

# 当 keepalive 起用的时候，TCP 发送 keepalive 消息的频度。缺省是 2 小时
net.ipv4.tcp_keepalive_time = 30

# 允许系统打开的端口范围
net.ipv4.ip_local_port_range = 1024    65000

# 修改防火墙表大小，默认 65536
#net.netfilter.nf_conntrack_max=655350
#net.netfilter.nf_conntrack_tcp_timeout_established=1200

# 确保无人能修改路由表
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.secure_redirects = 0
net.ipv4.conf.default.secure_redirects = 0
EOF

设置vim配置 主要是能编写格式

cat >>/etc/vimrc<<EOF
set expandtab
set tabstop=4
set softtabstop=4
set shiftwidth=4
autocmd FileType html setlocal shiftwidth=4 tabstop=4 softtabstop=4
autocmd FileType golang setlocal shiftwidth=4 tabstop=4 softtabstop=4
autocmd FileType go setlocal shiftwidth=4 tabstop=4 softtabstop=4
autocmd FileType yml setlocal shiftwidth=2 tabstop=2 softtabstop=2
autocmd FileType yaml setlocal shiftwidth=2 tabstop=2 softtabstop=2
autocmd FileType htmldjango setlocal shiftwidth=4 tabstop=4 softtabstop=4
autocmd FileType javascript setlocal shiftwidth=4 tabstop=4 softtabstop=4
set ls=2
set incsearch
set hlsearch
syntax on
set ruler
set autoindent
EOF

内核升级(重要)

rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
yum --disablerepo=\* --enablerepo=elrepo-kernel repolist
# 查看可用的rpm包
yum --disablerepo=\* --enablerepo=elrepo-kernel list kernel*
# 安装长期支持版本的kernel
yum --disablerepo=\* --enablerepo=elrepo-kernel install -y kernel-lt.x86_64
# 删除旧版本工具包
yum remove kernel-tools-libs.x86_64 kernel-tools.x86_64 -y
# 安装新版本工具包
yum --disablerepo=\* --enablerepo=elrepo-kernel install -y kernel-lt-tools.x86_64
#查看默认启动顺序
awk -F\' '$1=="menuentry " {print $2}' /etc/grub2.cfg  
#默认启动的顺序是从0开始，新内核是从头插入（目前位置在0，而4.4.4的是在1），所以需要选择0。
grub2-set-default 0  
#重启并检查
reboot

安装ETCD

配置ETCD证书(重要)

• 使用cfssl生成证书

curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo

• 生成根证书

mkdir /opt/pki
cfssl print-defaults config > ca-config.json
cfssl print-defaults csr > ca-csr.json
cat ca-config.json
{
    # 表示此CA证书可以用于签名其他证书
    "signing": {
        "default": {
            # 过期时间
            "expiry": "87600h" 
        },
        "profiles": {
            # 表示TLS Server Authentication
            "server": {
                "expiry": "87600h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth"
                ]
            },
            # 表示TLS Client Authentication
            "client": {
                "expiry": "87600h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth"
                ]
            },
            # 表示TLS 对等证书
            "peer": {
                "expiry": "87600h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}
cat ca-csr.json
{
    # Common Name
    "CN": "ETCD CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [{
        # 国家编码
        "C": "CN",
        # 省
        "ST": "Beijing",
        # 市
        "L": "Beijing",
        # 公司名称
        "O": "SMA",
        # 部门名称
        "OU": "DevOps"
    }]
}
# 生成自签证书
cfssl gencert -initca ca-csr.json | cfssljson -bare etcd-root-ca
# 证书名称为etcd-root-ca*.pem
# 根证书请求
# etcd-root-ca.csr
# 根证书私钥
# etcd-root-ca-key.pem
# 根证书
# etcd-root-ca.pem

• 生成server证书

cfssl print-defaults csr > server.json
{
    # CN和ca-csr.json中的profiles key尽量保证一致
    "CN": "server",
    # 能使用的机器
    "hosts": [
        "master.sma.org",
        "node01.sma.org",
        "node01.sma.org",
        "192.168.1.10",
        "192.168.1.11",
        "192.168.1.12"
    ],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "CN",
            "ST": "Beijing",
            "L": "Beijing",
            "O": "SMA",
            "OU": "DevOps"
        }
    ]
}
cfssl gencert -config=ca-config.json -ca=./etcd-root-ca.pem -ca-key=./etcd-root-ca-key.pem -profile=server server.json | cfssljson -bare etcd-server
# server证书请求
# etcd-server.csr
# server证书私钥
# etcd-server-key.pem
# server证书
# etcd-server.pem

• 生成peer证书

cfssl print-defaults csr > peer.json
cat peer.json
{
    "CN": "peer",
    "hosts": [
        "master.sma.org",
        "node01.sma.org",
        "node01.sma.org",
        "192.168.1.10",
        "192.168.1.11",
        "192.168.1.12"
    ],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "CN",
            "ST": "Beijing",
            "L": "Beijing",
            "O": "SMA",
            "OU": "DevOps"
        }
    ]
}
cfssl gencert -config=ca-config.json -ca=./etcd-root-ca.pem -ca-key=./etcd-root-ca-key.pem -profile=peer peer.json | cfssljson -bare etcd-peer
# peer证书请求
# etcd-peer.csr
# peer证书私钥
# etcd-peer-key.pem
# peer证书
# etcd-peer.pem

• 生成client证书

cfssl print-defaults csr > client.json
cat client.json
{
    "CN": "client",
    "hosts": [""],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "CN",
            "ST": "Beijing",
            "L": "Beijing",
            "O": "SMA",
            "OU": "DevOps"
        }
    ]
}
cfssl gencert -config=ca-config.json -ca=./etcd-root-ca.pem -ca-key=./etcd-root-ca-key.pem -profile=client client.json | cfssljson -bare etcd-client
# client证书请求
# etcd-client.csr
# client证书私钥
# etcd-client-key.pem
# client证书
# etcd-client.pem

• etcd配置(所有的节点,当前节点为10节点)

yum install -y etcd
mkdir -p /etc/etcd/pki
cd /opt/pki
cp etcd-client-key.pem  etcd-client.pem  etcd-peer-key.pem  etcd-peer.pem  etcd-root-ca.pem  etcd-server-key.pem  etcd-server.pem /etc/etcd/pki
chown -R etcd.etcd /etc/etcd/pki
scp -r /etc/etcd/pki root@192.168.1.11:/etc/etcd/
scp -r /etc/etcd/pki root@192.168.1.12:/etc/etcd/
cat etcd.conf
#[Member]
# 节点名称
ETCD_NAME="etcd00"
# 数据目录
ETCD_DATA_DIR="/var/lib/etcd/data"
# PEER 监听地址
ETCD_LISTEN_PEER_URLS="https://0.0.0.0:2380"
# 客户端 监听地址
ETCD_LISTEN_CLIENT_URLS="https://0.0.0.0:2379"
#[Clustering]
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.10:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.10:2380"
ETCD_INITIAL_CLUSTER="etcd00=https://192.168.1.10:2380,etcd01=https://192.168.1.11:2380,etcd02=https://192.168.1.12:2380,"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

#[Proxy]
# 关闭ETCD 代理
ETCD_PROXY="off"

#[Security]
# ETCD 服务端key和ca
ETCD_CERT_FILE="/etc/etcd/pki/etcd-server.pem"
ETCD_KEY_FILE="/etc/etcd/pki/etcd-server-key.pem"
ETCD_CLIENT_CERT_AUTH="true"
# ETCD root ca
ETCD_TRUSTED_CA_FILE="/etc/etcd/pki/etcd-root-ca.pem"
ETCD_AUTO_TLS="true"
# ETCD peer ca和key
ETCD_PEER_CERT_FILE="/etc/etcd/pki/etcd-peer.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/pki/etcd-peer-key.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"
# ETCD 生成peer ca和key的根证书
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/pki/etcd-root-ca.pem"
ETCD_PEER_AUTO_TLS="true"

systemctl restart etcd

• etcd验证

etcdctl   --endpoints=https://192.168.1.10:2379,https://192.168.1.11:2379,https://192.168.1.12:2379   --cert-file=/etc/etcd/pki/etcd-client.pem   --ca-file=/etc/etcd/pki/etcd-root-ca.pem   --key-file=/etc/etcd/pki/etcd-client-key.pem   cluster-health

• etcd操作

ETCDCTL_API=3
ENDPOINTS="https://192.168.1.10:2379,https://192.168.1.11:2379,https://192.168.1.12:2379"
ETCD_CA="/etc/etcd/pki/etcd-root-ca.pem"
ETCD_CLIENT_CA="/etc/etcd/pki/etcd-client.pem"
ETCD_CLIENT_KEY="/etc/etcd/pki/etcd-client-key.pem"
etcdctl --endpoints=${ENDPOINTS} --ca-file=${ETCD_CA} --cert-file=${ETCD_CLIENT_CA} --key-file=${ETCD_CLIENT_KEY} set test "hello world"
etcdctl --endpoints=${ENDPOINTS} --ca-file=${ETCD_CA} --cert-file=${ETCD_CLIENT_CA} --key-file=${ETCD_CLIENT_KEY} get test

注意点

生成client证书时会出现 ServerName error错误的字样
解决方式:

• https://github.com/etcd-io/etcd/issues/9398
• https://coreos.com/os/docs/latest/generate-self-signed-certificates.html