k8s完整搭建文档

2. 准备事项

机器环境：centos7.6 主节点：192.168.126.135 从节点：192.168.126.136， 192.168.126.137 2.1 机器hostname设置

hostnamectl set-hostname etcd1 # 192.168.126.135机器执行  
hostnamectl set-hostname etcd2 # 192.168.126.136机器执行  
hostnamectl set-hostname etcd3 # 192.168.126.137机器执行

2.2 机器hosts设置省略配置完成如下：

cat /etc/hosts:
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4  
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6  
192.168.126.135 etcd1  
192.168.126.136 etcd2  
192.168.126.137 etcd3

2.3 安装必要的软件

yum install kubernetes-master etcd flannel -y  # etcd1执行
yum install kubernetes-node etcd docker flannel *rhsm* -y  #etcd2执行
yum install kubernetes-node etcd docker flannel *rhsm* -y  #etcd3执行

2.4 关闭防火墙，配置软件源

systemctl stop firealld  
systemctl disable firewalld  

sudo tee /etc/docker/daemon.json <<-'EOF'  
{  
  "registry-mirrors": ["https://baz6f8j9.mirror.aliyuncs.com"]  
}  
EOF  
sudo systemctl daemon-reload  
sudo systemctl restart docker  

yum install *rhsm*  

wget http://mirror.centos.org/centos/7/os/x86_64/Packages/python-rhsm-certificates-1.19.10-1.el7_4.x86_64.rpm  

rpm2cpio python-rhsm-certificates-1.19.10-1.el7_4.x86_64.rpm | cpio -iv --to-stdout ./etc/rhsm/ca/redhat-uep.pem | tee /etc/rhsm/ca/redhat-uep.pem

3. etcd集群配置

3.1 etcd1配置配置/etc/etcd/etcd.conf:

[root@etcd1 ~]# cat /etc/etcd/etcd.conf  
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"  # 数据存放位置，如果出错，需要删除
ETCD_LISTEN_PEER_URLS="http://192.168.126.135:2380"  #当前客户端地址
ETCD_LISTEN_CLIENT_URLS="http://192.168.126.135:2379,http://127.0.0.1:2379"  #当前客户端地址
ETCD_MAX_SNAPSHOTS="5"  
ETCD_NAME="etcd1"  #重点，在集群中的名称
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.126.135:2380"  #广播地址
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.126.135:2379"  #广播地址
ETCD_INITIAL_CLUSTER="etcd1=http://192.168.126.135:2380,etcd2=http://192.168.126.136:2380,etcd3=http://192.168.126.137:2380"  #集群所有节点的地址
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"  #重点，etcd集群名称
ETCD_INITIAL_CLUSTER_STATE="new"

3.2 etcd2配置配置/etc/etcd/etcd.conf:

ETCD_DATA_DIR="/var/lib/etcd/default.etcd"  
ETCD_LISTEN_PEER_URLS="http://192.168.126.136:2380"  
ETCD_LISTEN_CLIENT_URLS="http://192.168.126.136:2379,http://127.0.0.1:2379" 
ETCD_MAX_SNAPSHOTS="5"  
ETCD_NAME="etcd2"  
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.126.136:2380"  
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.126.136:2379"  
ETCD_INITIAL_CLUSTER="etcd1=http://192.168.126.135:2380,etcd2=http://192.168.126.136:2380,etcd3=http://192.168.126.137:2380"  
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"  
ETCD_INITIAL_CLUSTER_STATE="new"

3.3 etcd3配置配置/etc/etcd/etcd.conf:

ETCD_DATA_DIR="/var/lib/etcd/default.etcd"  
ETCD_LISTEN_PEER_URLS="http://192.168.126.137:2380"  
ETCD_LISTEN_CLIENT_URLS="http://192.168.126.137:2379,http://127.0.0.1:2379" 
ETCD_MAX_SNAPSHOTS="5"  
ETCD_NAME="etcd3"  
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.126.137:2380"  
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.126.137:2379"  
ETCD_INITIAL_CLUSTER="etcd1=http://192.168.126.135:2380,etcd2=http://192.168.126.136:2380,etcd3=http://192.168.126.137:2380"  
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"  
ETCD_INITIAL_CLUSTER_STATE="new"

3.4 启动并验证etcd集群在每个机器上分别执行：systemctl start etcd，首次启动会卡顿，因为在等待所有节点加入

[root@etcd1 ~]# etcdctl member list  
168ae043a1f52da5: name=etcd2 peerURLs=http://192.168.126.136:2380 clientURLs=http://192.168.126.136:2379 isLeader=false  
33049130aa4e4a04: name=etcd3 peerURLs=http://192.168.126.137:2380 clientURLs=http://192.168.126.137:2379 isLeader=false  
e3bf8bcaa9f960b4: name=etcd1 peerURLs=http://192.168.126.135:2380 clientURLs=http://192.168.126.135:2379 isLeader=true

4. 配置kubernetes集群

4.1 etcd1配置 apiserver 配置：

[root@etcd1 ~]# cat /etc/kubernetes/apiserver  
KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"  ##不能绑定到本地
KUBE_API_PORT="--port=8080"  
KUBELET_PORT="--kubelet-port=10250"  
KUBE_ETCD_SERVERS="--etcd-servers=http://192.168.126.135:2379,http://192.168.126.136:2379,http://192.168.126.137:2379"  ##etcd集群节点地址
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"  
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,ResourceQuota"  ##删除了负责权限验证的配置
KUBE_API_ARGS=""

config配置：

[root@etcd1 ~]# cat /etc/kubernetes/config  
###  
KUBE_LOGTOSTDERR="--logtostderr=true"  
KUBE_LOG_LEVEL="--v=0"  
KUBE_ALLOW_PRIV="--allow-privileged=false"  
KUBE_MASTER="--master=http://192.168.126.135:8080"  #主节点

启动：

systemctl start kube-apiserver   
systemctl start kube-controller-manager  
systemctl start kube-scheduler

4.2 etcd2配置 kubelet配置：

[root@etcd2 ~]# cat /etc/kubernetes/kubelet  
KUBELET_ADDRESS="--address=0.0.0.0"  
KUBELET_HOSTNAME="--hostname-override=192.168.126.136"  #本机节点地址
KUBELET_API_SERVER="--api-servers=http://192.168.126.135:8080"  # 主节点地址
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"  
KUBELET_ARGS=""

config配置：

[root@etcd2 ~]# cat /etc/kubernetes/config  
KUBE_LOGTOSTDERR="--logtostderr=true"  
KUBE_LOG_LEVEL="--v=0"  
KUBE_ALLOW_PRIV="--allow-privileged=false"  
KUBE_MASTER="--master=http://192.168.126.135:8080"  #主节点位置

启动：

systemctl start kubelet  
systemctl start kube-proxy

4.3 etcd3配置 kubelet配置：

[root@etcd3 ~]# cat /etc/kubernetes/kubelet  
KUBELET_ADDRESS="--address=0.0.0.0"  
KUBELET_HOSTNAME="--hostname-override=192.168.126.137"  #本机节点地址
KUBELET_API_SERVER="--api-servers=http://192.168.126.135:8080"  # 主节点地址
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"  
KUBELET_ARGS=""

config配置：

[root@etcd3 ~]# cat /etc/kubernetes/config  
KUBE_LOGTOSTDERR="--logtostderr=true"  
KUBE_LOG_LEVEL="--v=0"  
KUBE_ALLOW_PRIV="--allow-privileged=false"  
KUBE_MASTER="--master=http://192.168.126.135:8080"  #主节点位置

启动：

systemctl start kubelet  
systemctl start kube-proxy

检查配置结果：

[root@etcd1 ~]# kubectl get nodes  
NAME              STATUS    AGE  
192.168.126.136   Ready     3h  
192.168.126.137   Ready     3h

5. 配置flanneld分布式网络

5.1 etcd1配置

[root@etcd1 ~]# cat /etc/sysconfig/flanneld  
FLANNEL_ETCD_ENDPOINTS="http://192.168.126.135:2379,http://192.168.126.136:2379,http://192.168.126.137:2379"  #构成flanneld的分布式
FLANNEL_ETCD_PREFIX="/atomic.io/network"  #此处值可以修改，对应etcd中存储的值
FLANNEL_OPTIONS="--logtostderr=false --log_dir=/var/log/flannel/ --iface=ens33"  #此处ens33为自己物理网卡名称，必须一致

5.2 etcd2配置

[root@etcd2 ~]# cat /etc/sysconfig/flanneld  
FLANNEL_ETCD_ENDPOINTS="http://192.168.126.135:2379,http://192.168.126.136:2379,http://192.168.126.137:2379"  #构成flanneld的分布式
FLANNEL_ETCD_PREFIX="/atomic.io/network"  #此处值可以修改，对应etcd中存储的值
FLANNEL_OPTIONS="--logtostderr=false --log_dir=/var/log/flannel/ --iface=ens33"  #此处ens33为自己物理网卡名称，必须一致

5.3 etcd3配置

[root@etcd3 ~]# cat /etc/sysconfig/flanneld  
FLANNEL_ETCD_ENDPOINTS="http://192.168.126.135:2379,http://192.168.126.136:2379,http://192.168.126.137:2379"  #构成flanneld的分布式
FLANNEL_ETCD_PREFIX="/atomic.io/network"  #此处值可以修改，对应etcd中存储的值
FLANNEL_OPTIONS="--logtostderr=false --log_dir=/var/log/flannel/ --iface=ens33"  #此处ens33为自己物理网卡名称，必须一致

5.4 为每个节点生成网络

etcdctl mk /atomic.io/network/config '{"Network":"172.17.0.0/16"}'# 存入etcd数据库中，在任意节点执行
etcdctl get /atomic.io/network/config #查看 
systemctl restart flanneld  #启动
etcdctl ls /atomic.io/network/subnets 查看分配的网络 
/atomic.io/network/subnets/172.17.49.0-24 #第一个网段
/atomic.io/network/subnets/172.17.88.0-24 #第二个网段
/atomic.io/network/subnets/172.17.61.0-24 #第三个网段

5.5 重点，检查防火墙规则，在所有节点执行

iptables -L -n  #查看防火墙规则  
iptables -P FORWARD ACCEPT ##转发

然后在etcd1 ping etcd2的flannel0网卡地址，和docker0网卡地址，互相测试，如果都可以连通，则证明flanneld分布式网络部署成功；检查docker0和flannel0是否在同一网段，如否，则有问题；启动顺序保证flanneld先启动，docker后启动，应为docker的网卡由flanneld分配

6. 部署Dashboard

坑：镜像registry.access.redhat.com/rhel7/pod-infrastructure无法下载，我们用docker提前直接下载，docker pull registry.access.redhat.com/rhel7/pod-infrastructure:latest 在etcd1进行创建，后续会由集群分发到不同的节点部署 dashboard-controller.yaml:

apiVersion: v1
kind: Service
metadata:
  name: kubernetes-dashboard
  namespace: kube-system
  labels:
    k8s-app: kubernetes-dashboard
    kubernetes.io/cluster-service: "true"
spec:
  selector:
    k8s-app: kubernetes-dashboard
  ports:
  - port: 80
    targetPort: 9090
[root@etcd1 ~]# clear
[root@etcd1 ~]# cat dashboard-controller
cat: dashboard-controller: No such file or directory
[root@etcd1 ~]# cat dashboard-controller.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: kubernetes-dashboard
  namespace: kube-system
  labels:
    k8s-app: kubernetes-dashboard
    kubernetes.io/cluster-service: "true"
spec:
  selector:
    matchLabels:
      k8s-app: kubernetes-dashboard
  template:
    metadata:
      labels:
        k8s-app: kubernetes-dashboard
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
    spec:
      containers:
      - name: kubernetes-dashboard
        image: bestwu/kubernetes-dashboard-amd64:v1.6.3
        resources:
          # keep request = limit to keep this container in guaranteed class
          limits:
            cpu: 100m
            memory: 50Mi
          requests:
            cpu: 100m
            memory: 50Mi
        ports:
        - containerPort: 9090
        args:
          - --apiserver-host=http://192.168.126.135:8080
        livenessProbe:
          httpGet:
            path: /
            port: 9090
          initialDelaySeconds: 30
          timeoutSeconds: 30

dashboard-service.yaml:

apiVersion: v1
kind: Service
metadata:
  name: kubernetes-dashboard
  namespace: kube-system
  labels:
    k8s-app: kubernetes-dashboard
    kubernetes.io/cluster-service: "true"
spec:
  selector:
    k8s-app: kubernetes-dashboard
  ports:
  - port: 80
    targetPort: 9090

7. 测试使用

访问浏览器https://192.168.126.135:6443/ui或者http://192.168.126.135:8080/ui

8. 总结

1）全是高可用，分布式，集群，master可以使用多个，node可以使用多个，flanneld使用多个，etcd组成集群

2）此文档未使用ca证书，之类，存在安全性风险，如果需要可以结合下

3）本文写的过程中写了不少笔记，放在博客上，https://linjinbao66.github.io/