From 5060a3f558e0647920c6905765141a9fd36c0337 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Wed, 24 Apr 2024 15:26:30 +0900 Subject: [PATCH 1/3] cicd: k3s-flannel-incluster scenario to support multi-master --- cicd/k3s-flannel-incluster-l2/EPconfig.txt | 34 +++++++++++++++ cicd/k3s-flannel-incluster-l2/Vagrantfile | 12 ++++++ cicd/k3s-flannel-incluster-l2/k3s.yaml | 19 --------- cicd/k3s-flannel-incluster-l2/kube-loxilb.yml | 13 ++++++ cicd/k3s-flannel-incluster-l2/lbconfig.txt | 41 +++++++++++++++++++ cicd/k3s-flannel-incluster-l2/loxilb.yml | 14 ++++++- cicd/k3s-flannel-incluster-l2/master1.sh | 9 ++-- cicd/k3s-flannel-incluster-l2/master2.sh | 9 ++-- cicd/k3s-flannel-incluster-l2/master3.sh | 13 ++++++ cicd/k3s-flannel-incluster-l2/node-token | 1 - cicd/k3s-flannel-incluster-l2/rmconfig.sh | 1 + cicd/k3s-flannel-incluster-l2/worker.sh | 2 +- 12 files changed, 139 insertions(+), 29 deletions(-) create mode 100644 cicd/k3s-flannel-incluster-l2/EPconfig.txt delete mode 100644 cicd/k3s-flannel-incluster-l2/k3s.yaml create mode 100644 cicd/k3s-flannel-incluster-l2/lbconfig.txt create mode 100755 cicd/k3s-flannel-incluster-l2/master3.sh delete mode 100644 cicd/k3s-flannel-incluster-l2/node-token diff --git a/cicd/k3s-flannel-incluster-l2/EPconfig.txt b/cicd/k3s-flannel-incluster-l2/EPconfig.txt new file mode 100644 index 000000000..5521b1d64 --- /dev/null +++ b/cicd/k3s-flannel-incluster-l2/EPconfig.txt @@ -0,0 +1,34 @@ +{ + "Attr":[ + { + "hostName":"192.168.80.10", + "name":"192.168.80.10_tcp_6443", + "inactiveReTries":2, + "probeType":"tcp", + "probeReq":"", + "probeResp":"", + "probeDuration":10, + "probePort":6443 + }, + { + "hostName":"192.168.80.11", + "name":"192.168.80.11_tcp_6443", + "inactiveReTries":2, + "probeType":"tcp", + "probeReq":"", + "probeResp":"", + "probeDuration":10, + "probePort":6443 + }, + { + "hostName":"192.168.80.12", + "name":"192.168.80.12_tcp_6443", + "inactiveReTries":2, + "probeType":"tcp", + "probeReq":"", + "probeResp":"", + "probeDuration":10, + "probePort":6443 + } + ] +} diff --git a/cicd/k3s-flannel-incluster-l2/Vagrantfile b/cicd/k3s-flannel-incluster-l2/Vagrantfile index 2132c4936..8d460f5e9 100644 --- a/cicd/k3s-flannel-incluster-l2/Vagrantfile +++ b/cicd/k3s-flannel-incluster-l2/Vagrantfile @@ -45,6 +45,18 @@ Vagrant.configure("2") do |config| end end + config.vm.define "master3" do |master| + master.vm.hostname = 'master3' + master.vm.network :private_network, ip: "192.168.90.12", :netmask => "255.255.255.0" + master.vm.network :private_network, ip: "192.168.80.12", :netmask => "255.255.255.0" + master.vm.provision :shell, :path => "master3.sh" + master.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 8192] + vbox.customize ["modifyvm", :id, "--cpus", 4] + end + end + + (1..workers).each do |node_number| config.vm.define "worker#{node_number}" do |worker| worker.vm.hostname = "worker#{node_number}" diff --git a/cicd/k3s-flannel-incluster-l2/k3s.yaml b/cicd/k3s-flannel-incluster-l2/k3s.yaml deleted file mode 100644 index f18efb521..000000000 --- a/cicd/k3s-flannel-incluster-l2/k3s.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUyT1RnM01UWTBPREV3SGhjTk1qTXhNRE14TURFME1USXhXaGNOTXpNeE1ESTRNREUwTVRJeApXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUyT1RnM01UWTBPREV3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFUcE0zVW11N0J3M1pVMzNwS3YrS0dwTHUwUXkvSllISUQrNVVqcWM4NGcKTnJudHZTcVdISmJEUExtWWhNVng5S3FiU0I3dU9HMmVvVGN2dzEwY1Z0RVRvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVWx4a2ZxcXp0dzhHMHcvb2VFb0EzCmZjejh0eDB3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnT2hUamlVYnU4TDl2YmNISlpTTWFPR3FsWlgwZ205dm0KWncxZ1hBV0VvTFlDSVFDbWpJQ1FSRzJUWnhpdldJUVhrVERhekJLbHZJTWdPWkk3bFlCTTJvVFd6dz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - server: https://192.168.80.10:6443 - name: default -contexts: -- context: - cluster: default - user: default - name: default -current-context: default -kind: Config -preferences: {} -users: -- name: default - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrRENDQVRlZ0F3SUJBZ0lJYUxFcUF6cEl5aGt3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOams0TnpFMk5EZ3hNQjRYRFRJek1UQXpNVEF4TkRFeU1Wb1hEVEkwTVRBegpNREF4TkRFeU1Wb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJBYnR3WEQvcWt2ajhrU0kKZlIzZXNWUGRDQnpyYU4zV1hrS3NvOTZhWnFBcUFiOHdkRlFPRnZIdTlJSEgyK2dEY0N0MXJOWC9TK1FNcFVlWgpmbEUremRtalNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCUWNNaW1IbFdCOTdjTE5YTW92OExpc2ZwemVvakFLQmdncWhrak9QUVFEQWdOSEFEQkUKQWlBSy9TVEJ0V0VJblpGNVF0Zkx1dVRQZ0pXZ3BvL2JCbThwNXhvTXRJN3JKd0lnRXZ3MkdOaVY5QmRtR1lLTwpmVk5lMlE2YVZwdW1hTTZ5eEFaZjdTRW1hV2c9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdFkyeHAKWlc1MExXTmhRREUyT1RnM01UWTBPREV3SGhjTk1qTXhNRE14TURFME1USXhXaGNOTXpNeE1ESTRNREUwTVRJeApXakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwWlc1MExXTmhRREUyT1RnM01UWTBPREV3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFRSzR3QzhyTGdXWmFMcWE0Yjh6NllLN0dkQTFxWUJjTHNhZ0R3TmNpcnQKaGN3SHVjUEJ2cTN2elN2STVsRGpua3VlenZUdmlydy9jR0doZGlIRGdwcVNvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVUhESXBoNVZnZmUzQ3pWektML0M0CnJINmMzcUl3Q2dZSUtvWkl6ajBFQXdJRFNRQXdSZ0loQU9EdW4vU09MT2w0MzhycmkyazdFWTV6bktXd2IzLzcKNVp3Z2pDUndaQkZsQWlFQXVsSjQwelFqT05SWXVVN3dNa29JQkEzNjRaR2FuaWdzaFdtd2JZTmZQVGs9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUFJTUloU1BCSFFaN3BHY2ZxNEVsZVVQY0wxR0g1TEVyRVV0akNnRTNxTDdvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQnUzQmNQK3FTK1B5UkloOUhkNnhVOTBJSE90bzNkWmVRcXlqM3BwbW9Db0J2ekIwVkE0Vwo4ZTcwZ2NmYjZBTndLM1dzMWY5TDVBeWxSNWwrVVQ3TjJRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= diff --git a/cicd/k3s-flannel-incluster-l2/kube-loxilb.yml b/cicd/k3s-flannel-incluster-l2/kube-loxilb.yml index b4c8dcbae..0f1ed7a5c 100644 --- a/cicd/k3s-flannel-incluster-l2/kube-loxilb.yml +++ b/cicd/k3s-flannel-incluster-l2/kube-loxilb.yml @@ -101,6 +101,19 @@ spec: operator: Exists - effect: NoExecute operator: Exists + - key: "node-role.kubernetes.io/master" + operator: Exists + - key: "node-role.kubernetes.io/control-plane" + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: Exists + - key: "node-role.kubernetes.io/control-plane" + operator: Exists priorityClassName: system-node-critical serviceAccountName: kube-loxilb terminationGracePeriodSeconds: 0 diff --git a/cicd/k3s-flannel-incluster-l2/lbconfig.txt b/cicd/k3s-flannel-incluster-l2/lbconfig.txt new file mode 100644 index 000000000..f96094054 --- /dev/null +++ b/cicd/k3s-flannel-incluster-l2/lbconfig.txt @@ -0,0 +1,41 @@ +{ + "lbAttr":[ + { + "serviceArguments":{ + "externalIP":"192.168.80.80", + "port":6443, + "protocol":"tcp", + "sel":0, + "mode":2, + "BGP":false, + "Monitor":true, + "inactiveTimeOut":240, + "block":0 + }, + "secondaryIPs":null, + "endpoints":[ + { + "endpointIP":"192.168.80.10", + "targetPort":6443, + "weight":1, + "state":"active", + "counter":"" + }, + { + "endpointIP":"192.168.80.11", + "targetPort":6443, + "weight":1, + "state":"active", + "counter":"" + }, + { + "endpointIP":"192.168.80.12", + "targetPort":6443, + "weight":1, + "state":"active", + "counter":"" + } + ] + } + ] +} diff --git a/cicd/k3s-flannel-incluster-l2/loxilb.yml b/cicd/k3s-flannel-incluster-l2/loxilb.yml index 6fee9b7bd..fc8e31e68 100644 --- a/cicd/k3s-flannel-incluster-l2/loxilb.yml +++ b/cicd/k3s-flannel-incluster-l2/loxilb.yml @@ -29,11 +29,23 @@ spec: operator: Exists - key: "node-role.kubernetes.io/control-plane" operator: Exists + volumes: + - name: hllb + hostPath: + path: /etc/loxilb + type: DirectoryOrCreate containers: - name: loxilb-app image: "ghcr.io/loxilb-io/loxilb:latest" imagePullPolicy: Always - command: [ "/root/loxilb-io/loxilb/loxilb", "--egr-hooks", "--blacklist=cni[0-9a-z]|veth.|flannel." ] + command: + - /root/loxilb-io/loxilb/loxilb + args: + - --egr-hooks + - --blacklist=cni[0-9a-z]|veth.|flannel. + volumeMounts: + - name: hllb + mountPath: /etc/loxilb ports: - containerPort: 11111 - containerPort: 179 diff --git a/cicd/k3s-flannel-incluster-l2/master1.sh b/cicd/k3s-flannel-incluster-l2/master1.sh index c6f0613d1..a70eb09de 100755 --- a/cicd/k3s-flannel-incluster-l2/master1.sh +++ b/cicd/k3s-flannel-incluster-l2/master1.sh @@ -1,12 +1,13 @@ sudo su export MASTER_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.90' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/') -curl -fL https://get.k3s.io | sh -s - server --node-ip=192.168.80.10 --disable servicelb --disable traefik --cluster-init external-hostname=192.168.80.10 --node-external-ip=192.168.80.10 --disable-cloud-controller +curl -fL https://get.k3s.io | sh -s - server --node-ip=192.168.80.10 --disable servicelb --disable traefik --cluster-init external-hostname=192.168.80.10 --node-external-ip=192.168.80.80 --disable-cloud-controller --flannel-iface=eth1 curl -sfL https://github.com/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh - sleep 60 echo $MASTER_IP > /vagrant/master-ip cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token -sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /etc/rancher/k3s/k3s.yaml cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml -sudo kubectl apply -f /vagrant/loxilb.yml -sudo kubectl apply -f /vagrant/kube-loxilb.yml +sed -i -e "s/127.0.0.1/192.168.80.80/g" /vagrant/k3s.yaml +sudo mkdir -p /etc/loxilb +sudo cp /vagrant/lbconfig.txt /etc/loxilb/ +sudo cp /vagrant/EPconfig.txt /etc/loxilb/ /vagrant/wait_ready.sh diff --git a/cicd/k3s-flannel-incluster-l2/master2.sh b/cicd/k3s-flannel-incluster-l2/master2.sh index 5ec72af6e..354d86a52 100755 --- a/cicd/k3s-flannel-incluster-l2/master2.sh +++ b/cicd/k3s-flannel-incluster-l2/master2.sh @@ -2,9 +2,12 @@ sudo su export WORKER_ADDR=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/') export MASTER_ADDR=$(cat /vagrant/master-ip) export NODE_TOKEN=$(cat /vagrant/node-token) - +sudo mkdir -p /etc/loxilb +sudo cp /vagrant/lbconfig.txt /etc/loxilb/ +sudo cp /vagrant/EPconfig.txt /etc/loxilb/ #curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.11 --disable-cloud-controller -t ${NODE_TOKEN} -curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.11 -t ${NODE_TOKEN} +curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.80 -t ${NODE_TOKEN} --flannel-iface=eth1 curl -sfL https://github.com/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh - - +#sudo kubectl apply -f /vagrant/loxilb.yml +#sudo kubectl apply -f /vagrant/kube-loxilb.yml /vagrant/wait_ready.sh diff --git a/cicd/k3s-flannel-incluster-l2/master3.sh b/cicd/k3s-flannel-incluster-l2/master3.sh new file mode 100755 index 000000000..d05e74d66 --- /dev/null +++ b/cicd/k3s-flannel-incluster-l2/master3.sh @@ -0,0 +1,13 @@ +sudo su +export WORKER_ADDR=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/') +export MASTER_ADDR=$(cat /vagrant/master-ip) +export NODE_TOKEN=$(cat /vagrant/node-token) +sudo mkdir -p /etc/loxilb +sudo cp /vagrant/lbconfig.txt /etc/loxilb/ +sudo cp /vagrant/EPconfig.txt /etc/loxilb/ +#curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.11 --disable-cloud-controller -t ${NODE_TOKEN} +curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.12 external-hostname=192.168.80.12 --node-external-ip=192.168.80.80 -t ${NODE_TOKEN} --flannel-iface=eth1 +curl -sfL https://github.com/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh - +sudo kubectl apply -f /vagrant/loxilb.yml +sudo kubectl apply -f /vagrant/kube-loxilb.yml +/vagrant/wait_ready.sh diff --git a/cicd/k3s-flannel-incluster-l2/node-token b/cicd/k3s-flannel-incluster-l2/node-token deleted file mode 100644 index 41447f218..000000000 --- a/cicd/k3s-flannel-incluster-l2/node-token +++ /dev/null @@ -1 +0,0 @@ -K104ba03b7d623244660768d0475dbaab00b38a44cf3dbd7f8cfb749899d6917dfe::server:53d79e122c4fb6b54f104932e26995dd diff --git a/cicd/k3s-flannel-incluster-l2/rmconfig.sh b/cicd/k3s-flannel-incluster-l2/rmconfig.sh index bd4b79e81..1f85eb636 100755 --- a/cicd/k3s-flannel-incluster-l2/rmconfig.sh +++ b/cicd/k3s-flannel-incluster-l2/rmconfig.sh @@ -4,4 +4,5 @@ vagrant destroy -f worker1 vagrant destroy -f worker2 vagrant destroy -f master1 vagrant destroy -f master2 +vagrant destroy -f master3 vagrant destroy -f host diff --git a/cicd/k3s-flannel-incluster-l2/worker.sh b/cicd/k3s-flannel-incluster-l2/worker.sh index b03d55fb5..016a47b40 100644 --- a/cicd/k3s-flannel-incluster-l2/worker.sh +++ b/cicd/k3s-flannel-incluster-l2/worker.sh @@ -4,7 +4,7 @@ export MASTER_ADDR=$(cat /vagrant/master-ip) export NODE_TOKEN=$(cat /vagrant/node-token) mkdir -p /etc/rancher/k3s cp -f /vagrant/k3s.yaml /etc/rancher/k3s/k3s.yaml -curl -sfL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - agent --server https://192.168.80.10:6443 --node-ip=${WORKER_ADDR} --node-external-ip=${WORKER_ADDR} -t ${NODE_TOKEN} +curl -sfL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - agent --server https://192.168.80.80:6443 --node-ip=${WORKER_ADDR} --node-external-ip=${WORKER_ADDR} -t ${NODE_TOKEN} --flannel-iface=eth1 #sudo kubectl apply -f /vagrant/loxilb-peer.yml #sudo kubectl apply -f /vagrant/nginx.yml #sudo kubectl apply -f /vagrant/udp.yml From 7b996fb5783442a62a113af7e7d9f9fe28203914 Mon Sep 17 00:00:00 2001 From: TrekkieCoder <111065900+TrekkieCoder@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:41:55 +0900 Subject: [PATCH 2/3] chore: Updated README to include new how-to guides --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c7ef9dacd..6072afaa3 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S - [How-To : Manual build/run](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/run.md) - [How-To : Standalone configuration](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/cmd.md) - [How-To : debug](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/debugging.md) +- [How-To : Access end-points outside K8s](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/ext-ep.md) +- [How-To : Deploy multi-server K3s HA with loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k3s-multi-master.md) ## Getting started with different K8s distributions/tools From b8ea862124e359ba4ecf32f3b9a35cf2ce0f29ad Mon Sep 17 00:00:00 2001 From: TrekkieCoder <111065900+TrekkieCoder@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:42:47 +0900 Subject: [PATCH 3/3] chore: Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6072afaa3..237ecfcab 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S - [How-To : Run loxilb in standalone mode](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/standalone.md) - [How-To : Manual build/run](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/run.md) - [How-To : Standalone configuration](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/cmd.md) -- [How-To : debug](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/debugging.md) +- [How-To : Debug loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/debugging.md) - [How-To : Access end-points outside K8s](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/ext-ep.md) - [How-To : Deploy multi-server K3s HA with loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k3s-multi-master.md)