k8s完整搭建文档
2. 准备事项
机器环境:centos7.6 主节点:192.168.126.135 从节点:192.168.126.136, 192.168.126.137 2.1 机器hostname设置
hostnamectl set-hostname etcd1 # 192.168.126.135机器执行
hostnamectl set-hostname etcd2 # 192.168.126.136机器执行
hostnamectl set-hostname etcd3 # 192.168.126.137机器执行
2.2 机器hosts设置 省略 配置完成如下:
cat /etc/hosts:
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.126.135 etcd1
192.168.126.136 etcd2
192.168.126.137 etcd3
2.3 安装必要的软件
yum install kubernetes-master etcd flannel -y # etcd1执行
yum install kubernetes-node etcd docker flannel *rhsm* -y #etcd2执行
yum install kubernetes-node etcd docker flannel *rhsm* -y #etcd3执行
2.4 关闭防火墙,配置软件源
systemctl stop firealld
systemctl disable firewalld
sudo tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": ["https://baz6f8j9.mirror.aliyuncs.com"]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker
yum install *rhsm*
wget http://mirror.centos.org/centos/7/os/x86_64/Packages/python-rhsm-certificates-1.19.10-1.el7_4.x86_64.rpm
rpm2cpio python-rhsm-certificates-1.19.10-1.el7_4.x86_64.rpm | cpio -iv --to-stdout ./etc/rhsm/ca/redhat-uep.pem | tee /etc/rhsm/ca/redhat-uep.pem
3. etcd集群配置
3.1 etcd1配置 配置/etc/etcd/etcd.conf:
[root@etcd1 ~]# cat /etc/etcd/etcd.conf
ETCD_DATA_DIR="/var/lib/etcd/default.etcd" # 数据存放位置,如果出错,需要删除
ETCD_LISTEN_PEER_URLS="http://192.168.126.135:2380" #当前客户端地址
ETCD_LISTEN_CLIENT_URLS="http://192.168.126.135:2379,http://127.0.0.1:2379" #当前客户端地址
ETCD_MAX_SNAPSHOTS="5"
ETCD_NAME="etcd1" #重点,在集群中的名称
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.126.135:2380" #广播地址
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.126.135:2379" #广播地址
ETCD_INITIAL_CLUSTER="etcd1=http://192.168.126.135:2380,etcd2=http://192.168.126.136:2380,etcd3=http://192.168.126.137:2380" #集群所有节点的地址
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" #重点,etcd集群名称
ETCD_INITIAL_CLUSTER_STATE="new"
3.2 etcd2配置 配置/etc/etcd/etcd.conf:
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="http://192.168.126.136:2380"
ETCD_LISTEN_CLIENT_URLS="http://192.168.126.136:2379,http://127.0.0.1:2379"
ETCD_MAX_SNAPSHOTS="5"
ETCD_NAME="etcd2"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.126.136:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.126.136:2379"
ETCD_INITIAL_CLUSTER="etcd1=http://192.168.126.135:2380,etcd2=http://192.168.126.136:2380,etcd3=http://192.168.126.137:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
3.3 etcd3配置 配置/etc/etcd/etcd.conf:
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="http://192.168.126.137:2380"
ETCD_LISTEN_CLIENT_URLS="http://192.168.126.137:2379,http://127.0.0.1:2379"
ETCD_MAX_SNAPSHOTS="5"
ETCD_NAME="etcd3"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.126.137:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.126.137:2379"
ETCD_INITIAL_CLUSTER="etcd1=http://192.168.126.135:2380,etcd2=http://192.168.126.136:2380,etcd3=http://192.168.126.137:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
3.4 启动并验证etcd集群 在每个机器上分别执行:systemctl start etcd,首次启动会卡顿,因为在等待所有节点加入
[root@etcd1 ~]# etcdctl member list
168ae043a1f52da5: name=etcd2 peerURLs=http://192.168.126.136:2380 clientURLs=http://192.168.126.136:2379 isLeader=false
33049130aa4e4a04: name=etcd3 peerURLs=http://192.168.126.137:2380 clientURLs=http://192.168.126.137:2379 isLeader=false
e3bf8bcaa9f960b4: name=etcd1 peerURLs=http://192.168.126.135:2380 clientURLs=http://192.168.126.135:2379 isLeader=true
4. 配置kubernetes集群
4.1 etcd1配置 apiserver 配置:
[root@etcd1 ~]# cat /etc/kubernetes/apiserver
KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0" ##不能绑定到本地
KUBE_API_PORT="--port=8080"
KUBELET_PORT="--kubelet-port=10250"
KUBE_ETCD_SERVERS="--etcd-servers=http://192.168.126.135:2379,http://192.168.126.136:2379,http://192.168.126.137:2379" ##etcd集群节点地址
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,ResourceQuota" ##删除了负责权限验证的配置
KUBE_API_ARGS=""
config配置:
[root@etcd1 ~]# cat /etc/kubernetes/config
###
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=false"
KUBE_MASTER="--master=http://192.168.126.135:8080" #主节点
启动:
systemctl start kube-apiserver
systemctl start kube-controller-manager
systemctl start kube-scheduler
4.2 etcd2配置 kubelet配置:
[root@etcd2 ~]# cat /etc/kubernetes/kubelet
KUBELET_ADDRESS="--address=0.0.0.0"
KUBELET_HOSTNAME="--hostname-override=192.168.126.136" #本机节点地址
KUBELET_API_SERVER="--api-servers=http://192.168.126.135:8080" # 主节点地址
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"
KUBELET_ARGS=""
config配置:
[root@etcd2 ~]# cat /etc/kubernetes/config
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=false"
KUBE_MASTER="--master=http://192.168.126.135:8080" #主节点位置
启动:
systemctl start kubelet
systemctl start kube-proxy
4.3 etcd3配置 kubelet配置:
[root@etcd3 ~]# cat /etc/kubernetes/kubelet
KUBELET_ADDRESS="--address=0.0.0.0"
KUBELET_HOSTNAME="--hostname-override=192.168.126.137" #本机节点地址
KUBELET_API_SERVER="--api-servers=http://192.168.126.135:8080" # 主节点地址
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"
KUBELET_ARGS=""
config配置:
[root@etcd3 ~]# cat /etc/kubernetes/config
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=false"
KUBE_MASTER="--master=http://192.168.126.135:8080" #主节点位置
启动:
systemctl start kubelet
systemctl start kube-proxy
检查配置结果:
[root@etcd1 ~]# kubectl get nodes
NAME STATUS AGE
192.168.126.136 Ready 3h
192.168.126.137 Ready 3h
5. 配置flanneld分布式网络
5.1 etcd1配置
[root@etcd1 ~]# cat /etc/sysconfig/flanneld
FLANNEL_ETCD_ENDPOINTS="http://192.168.126.135:2379,http://192.168.126.136:2379,http://192.168.126.137:2379" #构成flanneld的分布式
FLANNEL_ETCD_PREFIX="/atomic.io/network" #此处值可以修改,对应etcd中存储的值
FLANNEL_OPTIONS="--logtostderr=false --log_dir=/var/log/flannel/ --iface=ens33" #此处ens33为自己物理网卡名称,必须一致
5.2 etcd2配置
[root@etcd2 ~]# cat /etc/sysconfig/flanneld
FLANNEL_ETCD_ENDPOINTS="http://192.168.126.135:2379,http://192.168.126.136:2379,http://192.168.126.137:2379" #构成flanneld的分布式
FLANNEL_ETCD_PREFIX="/atomic.io/network" #此处值可以修改,对应etcd中存储的值
FLANNEL_OPTIONS="--logtostderr=false --log_dir=/var/log/flannel/ --iface=ens33" #此处ens33为自己物理网卡名称,必须一致
5.3 etcd3配置
[root@etcd3 ~]# cat /etc/sysconfig/flanneld
FLANNEL_ETCD_ENDPOINTS="http://192.168.126.135:2379,http://192.168.126.136:2379,http://192.168.126.137:2379" #构成flanneld的分布式
FLANNEL_ETCD_PREFIX="/atomic.io/network" #此处值可以修改,对应etcd中存储的值
FLANNEL_OPTIONS="--logtostderr=false --log_dir=/var/log/flannel/ --iface=ens33" #此处ens33为自己物理网卡名称,必须一致
5.4 为每个节点生成网络
etcdctl mk /atomic.io/network/config '{"Network":"172.17.0.0/16"}'# 存入etcd数据库中,在任意节点执行
etcdctl get /atomic.io/network/config #查看
systemctl restart flanneld #启动
etcdctl ls /atomic.io/network/subnets 查看分配的网络
/atomic.io/network/subnets/172.17.49.0-24 #第一个网段
/atomic.io/network/subnets/172.17.88.0-24 #第二个网段
/atomic.io/network/subnets/172.17.61.0-24 #第三个网段
5.5 重点,检查防火墙规则,在所有节点执行
iptables -L -n #查看防火墙规则
iptables -P FORWARD ACCEPT ##转发
然后在etcd1 ping etcd2的flannel0网卡地址,和docker0网卡地址,互相测试,如果都可以连通,则证明flanneld分布式网络部署成功; 检查docker0和flannel0是否在同一网段,如否,则有问题; 启动顺序保证flanneld先启动,docker后启动,应为docker的网卡由flanneld分配
6. 部署Dashboard
坑:镜像registry.access.redhat.com/rhel7/pod-infrastructure无法下载,我们用docker提前直接下载,docker pull registry.access.redhat.com/rhel7/pod-infrastructure:latest 在etcd1进行创建,后续会由集群分发到不同的节点部署 dashboard-controller.yaml:
apiVersion: v1
kind: Service
metadata:
name: kubernetes-dashboard
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
kubernetes.io/cluster-service: "true"
spec:
selector:
k8s-app: kubernetes-dashboard
ports:
- port: 80
targetPort: 9090
[root@etcd1 ~]# clear
[root@etcd1 ~]# cat dashboard-controller
cat: dashboard-controller: No such file or directory
[root@etcd1 ~]# cat dashboard-controller.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: kubernetes-dashboard
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
kubernetes.io/cluster-service: "true"
spec:
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
spec:
containers:
- name: kubernetes-dashboard
image: bestwu/kubernetes-dashboard-amd64:v1.6.3
resources:
# keep request = limit to keep this container in guaranteed class
limits:
cpu: 100m
memory: 50Mi
requests:
cpu: 100m
memory: 50Mi
ports:
- containerPort: 9090
args:
- --apiserver-host=http://192.168.126.135:8080
livenessProbe:
httpGet:
path: /
port: 9090
initialDelaySeconds: 30
timeoutSeconds: 30
dashboard-service.yaml:
apiVersion: v1
kind: Service
metadata:
name: kubernetes-dashboard
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
kubernetes.io/cluster-service: "true"
spec:
selector:
k8s-app: kubernetes-dashboard
ports:
- port: 80
targetPort: 9090
7. 测试使用
访问浏览器https://192.168.126.135:6443/ui或者http://192.168.126.135:8080/ui
8. 总结
1) 全是高可用,分布式,集群,master可以使用多个,node可以使用多个,flanneld使用多个,etcd组成集群
2) 此文档未使用ca证书,之类,存在安全性风险,如果需要可以结合下
3) 本文写的过程中写了不少笔记,放在博客上,https://linjinbao66.github.io/