源码构建
将客户端的证书由1年到10年并生成需要镜像
--构建环境
root@op:~# apt install curl make make-guile docker.io
root@op:~# docker version
Client:
Version:19.03.6
API version:1.40
Go version:go1.12.17
Git commit:369ce74a3c
Built:Fri Feb 28 23:45:43 2020
OS/Arch:linux/amd64
Experimental:false
Server:
Engine:
Version:19.03.6
API version:1.40 (minimum version 1.12)
Go version:go1.12.17
Git commit:369ce74a3c
Built:Wed Feb 19 01:06:16 2020
OS/Arch:linux/amd64
Experimental:false
containerd:
Version:1.3.3-0ubuntu1~18.04.2
GitCommit:
runc:
Version:spec: 1.0.1-dev
GitCommit:
docker-init:
Version:0.18.0
GitCommit:
--源码
root@op:/opt# wget https://github.com/rancher/k3s/archive/v1.18.8+k3s1.tar.gz
root@op:/opt# tar zxvf v1.18.8+k3s1.tar.gz
---修改证书的时间
root@op:/opt/k3s-1.18.8-k3s1/vendor/github.com/rancher/dynamiclistener/cert# vim cert.go
# vim cert.go
NotAfter:time.Now().Add(duration365d).UTC(),
改为
NotAfter:time.Now().Add(duration365d * 10).UTC(),
root@op:/opt/k3s-1.18.8-k3s1# git init
root@op:/opt/k3s-1.18.8-k3s1# git add .
root@op:/opt/k3s-1.18.8-k3s1# git config --global user.name root
root@op:/opt/k3s-1.18.8-k3s1# git config --global user.email root@yesnocom.com
root@op:/opt/k3s-1.18.8-k3s1# git commit -m "init"
---构建完整版本的二进制文件及需要的镜像
root@op:/opt/k3s-1.18.8-k3s1# SKIP_VALIDATE=true make
root@op:/opt/k3s-1.18.8-k3s1/dist/artifacts# ls -la
total 397292
drwxr-xr-x 2 root root4096 Sep7 04:42 .
drwxr-xr-x 3 root root4096 Sep7 04:41 ..
-rwxr-xr-x 1 root root53448704 Sep7 04:41 k3s
-rw------- 1 root root 352955392 Sep7 04:42 k3s-airgap-images-amd64.tar
-rw-r--r-- 1 root root272 Sep7 04:42 k3s-images.txt
root@op:/opt/k3s-1.18.8-k3s1/dist/artifacts# ./k3s -v
k3s version v1.18.8+k3s-c8d17880 (c8d17880)
在3个server(u1/u2/u3)节点上：
mkdir -p /data/rancher/logs_k3s/pods
mkdir -p /data/rancher/logs_k3s/containers
mkdir -p /data/rancher/kubelet_k3s/kubelet
mkdir -p /data/rancher/data_k3s/rancher/k3s/agent/images
ln -s /data/rancher/data_k3s/rancher /var/lib/
ln -s /data/rancher/kubelet_k3s/kubelet /var/lib/
ln -s /data/rancher/logs_k3s/pods /var/log/
ln -s /data/rancher/logs_k3s/containers /var/log/
将构建好的二进制软件（k3s)分别分发到3个server节点上（u1/u2/u3）的/usr/local/bin目录中
root@u1:~# ls -l /usr/local/bin/k3s
-rwxr-xr-x 1 root root 53448704 Sep7 08:41 /usr/local/bin/k3s
root@u2:~# ls -l /usr/local/bin/k3s
-rwxr-xr-x 1 root root 53448704 Sep7 08:41 /usr/local/bin/k3s
root@u3:~# ls -l /usr/local/bin/k3s
-rwxr-xr-x 1 root root 53448704 Sep7 08:41 /usr/local/bin/k3s
将构建好的需要的镜像包（k3s-airgap-images-amd64.tar）分别分发到3个server节点上（u1/u2/u3）的/var/lib/rancher/k3s/agent/images目录中
root@u1:~# ls -l /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar
-rw------- 1 root root 352955392 Sep7 08:42 /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar
root@u2:~# ls -l /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar
-rw------- 1 root root 352955392 Sep7 08:42 /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar
root@u3:~# ls -l /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar
-rw------- 1 root root 352955392 Sep7 08:42 /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar
部署server节点上初始集群（u1/u2/u3）
root@u1:~# curl -sfL https://get.k3s.io | \
> INSTALL_K3S_SKIP_DOWNLOAD=true \
> INSTALL_K3S_EXEC=" \
> server \
> --write-kubeconfig-mode 644 \
> --datastore-endpoint 'https://g1.yesnocom.com:2379,https://g2.yesnocom.com,https://g3.yesnocom.com' \
> --datastore-cafile '/srv/etcd/pki/ca.crt' \
> --datastore-certfile '/srv/etcd/pki/client.crt' \
> --datastore-keyfile '/srv/etcd/pki/client.key' \
> -t agent-secret \
> --tls-san vip.yesnocom.com" \
> sh -
[INFO]Skipping k3s download and verify
[INFO]Creating /usr/local/bin/kubectl symlink to k3s
[INFO]Creating /usr/local/bin/crictl symlink to k3s
[INFO]Creating /usr/local/bin/ctr symlink to k3s
[INFO]Creating killall script /usr/local/bin/k3s-killall.sh
[INFO]Creating uninstall script /usr/local/bin/k3s-uninstall.sh
[INFO]env: Creating environment file /etc/systemd/system/k3s.service.env
[INFO]systemd: Creating service file /etc/systemd/system/k3s.service
[INFO]systemd: Enabling k3s unit
Created symlink /etc/systemd/system/multi-user.target.wants/k3s.service → /etc/systemd/system/k3s.service.
[INFO]systemd: Starting k3s
root@u2:~# curl -sfL https://get.k3s.io | \
> INSTALL_K3S_SKIP_DOWNLOAD=true \
> INSTALL_K3S_EXEC=" \
> server \
> --write-kubeconfig-mode 644 \
> --datastore-endpoint 'https://g1.yesnocom.com:2379,https://g2.yesnocom.com,https://g3.yesnocom.com' \
> --datastore-cafile '/srv/etcd/pki/ca.crt' \
> --datastore-certfile '/srv/etcd/pki/client.crt' \
> --datastore-keyfile '/srv/etcd/pki/client.key' \
> -t agent-secret \
> --tls-san vip.yesnocom.com" \
> sh -
[INFO]Skipping k3s download and verify
[INFO]Creating /usr/local/bin/kubectl symlink to k3s
[INFO]Creating /usr/local/bin/crictl symlink to k3s
[INFO]Creating /usr/local/bin/ctr symlink to k3s
[INFO]Creating killall script /usr/local/bin/k3s-killall.sh
[INFO]Creating uninstall script /usr/local/bin/k3s-uninstall.sh
[INFO]env: Creating environment file /etc/systemd/system/k3s.service.env
[INFO]systemd: Creating service file /etc/systemd/system/k3s.service
[INFO]systemd: Enabling k3s unit
Created symlink /etc/systemd/system/multi-user.target.wants/k3s.service → /etc/systemd/system/k3s.service.
[INFO]systemd: Starting k3s
root@u3:~# curl -sfL https://get.k3s.io | \
> INSTALL_K3S_SKIP_DOWNLOAD=true \
> INSTALL_K3S_EXEC=" \
> server \
> --write-kubeconfig-mode 644 \
> --datastore-endpoint 'https://g1.yesnocom.com:2379,https://g2.yesnocom.com,https://g3.yesnocom.com' \
> --datastore-cafile '/srv/etcd/pki/ca.crt' \
> --datastore-certfile '/srv/etcd/pki/client.crt' \
> --datastore-keyfile '/srv/etcd/pki/client.key' \
> -t agent-secret \
> --tls-san vip.yesnocom.com" \
> sh -
[INFO]Skipping k3s download and verify
[INFO]Creating /usr/local/bin/kubectl symlink to k3s
[INFO]Creating /usr/local/bin/crictl symlink to k3s
[INFO]Creating /usr/local/bin/ctr symlink to k3s
[INFO]Creating killall script /usr/local/bin/k3s-killall.sh
[INFO]Creating uninstall script /usr/local/bin/k3s-uninstall.sh
[INFO]env: Creating environment file /etc/systemd/system/k3s.service.env
[INFO]systemd: Creating service file /etc/systemd/system/k3s.service
[INFO]systemd: Enabling k3s unit
Created symlink /etc/systemd/system/multi-user.target.wants/k3s.service → /etc/systemd/system/k3s.service.
[INFO]systemd: Starting k3s
root@u1:~# kubectl get node -o wide
NAMESTATUSROLESAGEVERSIONINTERNAL-IPEXTERNAL-IPOS-IMAGEKERNEL-VERSIONCONTAINER-RUNTIME
u1Readymaster2m31sv1.18.8+k3s-192.168.100.11<none>Ubuntu 18.04.5 LTS4.15.0-115-genericcontainerd://1.3.3-k3s2
u2Readymaster83sv1.18.8+k3s-192.168.100.12<none>Ubuntu 18.04.5 LTS4.15.0-115-genericcontainerd://1.3.3-k3s2
u3Readymaster52sv1.18.8+k3s-192.168.100.13<none>Ubuntu 18.04.5 LTS4.15.0-115-genericcontainerd://1.3.3-k3s2
root@u1:~# kubectl get pod -A
NAMESPACENAMEREADYSTATUSRESTARTSAGE
kube-systemcoredns-7944c66d8d-l7jtq1/1Running06m59s
kube-systemhelm-install-traefik-hctws0/1Completed06m59s
kube-systemlocal-path-provisioner-6d59f47c7-n9qjd1/1Running06m59s
kube-systemmetrics-server-7566d596c8-9wzsz1/1Running06m59s
kube-systemsvclb-traefik-47xws2/2Running05m43s
kube-systemsvclb-traefik-sz8b82/2Running06m40s
kube-systemsvclb-traefik-wcn7s2/2Running05m13s
kube-systemtraefik-758cd5fc85-f242c1/1Running06m40s
root@u1:~# kubectl get apiservices |grep 'metrics'
v1beta1.metrics.k8s.iokube-system/metrics-serverTrue6m44s
root@u1:~# kubectl top node
NAMECPU(cores)CPU%MEMORY(bytes)MEMORY%
u1148m7%968Mi49%
u294m4%726Mi36%
u393m4%728Mi36%
HA部署（u1/u2/u3）
# apt install haproxy -y
# apt install keepalived -y
haproxy 配置（3个节点上【u1/u2/u3】配置文件相同）
# cat /etc/haproxy/haproxy.cfg
global
log /dev/loglocal0
log /dev/loglocal1 notice
chroot /var/lib/haproxy
stats socket /var/run/haproxy-admin.sock mode 660 level admin
stats timeout 30s
userhaproxy
group haproxy
daemon
nbproc 1
defaults
logglobal
timeout connect 5000
timeout client10m
timeout server10m
listenadmin_stats
bind 0.0.0.0:10080
mode http
log 127.0.0.1 local0 err
stats refresh 30s
stats uri /status
stats realm welcome login\ Haproxy
stats auth admin:Jieshi11gR2.
stats hide-version
stats admin if TRUE
listen kube-master
bind 0.0.0.0:8443
mode tcp
option tcplog
balance source
server 192.168.100.91 192.168.100.11:6443 check inter 2000 fall 2 rise 2 weight 1
server 192.168.100.92 192.168.100.12:6443 check inter 2000 fall 2 rise 2 weight 1
server 192.168.100.93 192.168.100.13:6443 check inter 2000 fall 2 rise 2 weight 1
# systemctl restart haproxy.service
# ss -tunpla|cat |grep 8443
tcpLISTEN01280.0.0.0:84430.0.0.0:*users:(("haproxy",pid=9399,fd=9))
keepalived 配置文件（采用一主多备）
u1节点上：
root@u1:~# cat /etc/keepalived/keepalived.conf
global_defs {
router_id lb-master-105
}
vrrp_script check-haproxy {
script "killall -0 haproxy"
interval 5
weight -30
}
vrrp_instance VI-kube-master {
state MASTER
priority 120
dont_track_primary
interface ens33
virtual_router_id 68
advert_int 3
track_script {
check-haproxy
}
virtual_ipaddress {
192.168.100.88 dev ens33 label ens33:1
}
}
root@u1:~# systemctl restart keepalived.service
root@u1:~# ifconfig |grep -A 3 ens33:1
ens33:1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>mtu 1500
inet 192.168.100.88netmask 255.255.255.255broadcast 0.0.0.0
ether 00:0c:29:61:7d:9atxqueuelen 1000(Ethernet)
备节点上（u2/u3 配置文件一样）
# cat /etc/keepalived/keepalived.conf
global_defs {
router_id lb-backup-105
}
vrrp_script check-haproxy {
script "killall -0 haproxy"
interval 5
weight -30
}
vrrp_instance VI-kube-master {
state BACKUP
priority 110
dont_track_primary
interface ens33
virtual_router_id 68
advert_int 3
track_script {
check-haproxy
}
virtual_ipaddress {
192.168.100.88 dev ens33 label ens33:1
}
}
# systemctl restart keepalived.service
# ps -ef |grep keep
root788310 09:48 ?00:00:00 /usr/sbin/keepalived
root789378830 09:48 ?00:00:00 /usr/sbin/keepalived
root789578830 09:48 ?00:00:00 /usr/sbin/keepalived