cicd: k3s-flannel-incluster scenario to support multi-master

README.md

@@ -71,7 +71,9 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S
 - [How-To : Run loxilb in standalone mode](https:/loxilb-io/loxilbdocs/blob/main/docs/standalone.md)
 - [How-To : Manual build/run](https:/loxilb-io/loxilbdocs/blob/main/docs/run.md)
 - [How-To : Standalone configuration](https:/loxilb-io/loxilbdocs/blob/main/docs/cmd.md)
-- [How-To : debug](https:/loxilb-io/loxilbdocs/blob/main/docs/debugging.md)
+- [How-To : Debug loxilb](https:/loxilb-io/loxilbdocs/blob/main/docs/debugging.md)
+- [How-To : Access end-points outside K8s](https:/loxilb-io/loxilbdocs/blob/main/docs/ext-ep.md)
+- [How-To : Deploy multi-server K3s HA with loxilb](https:/loxilb-io/loxilbdocs/blob/main/docs/k3s-multi-master.md)
 
 ## Getting started with different K8s distributions/tools 
 

cicd/k3s-flannel-incluster-l2/EPconfig.txt

@@ -0,0 +1,34 @@
+{
+ "Attr":[
+ {
+ "hostName":"192.168.80.10",
+ "name":"192.168.80.10_tcp_6443",
+ "inactiveReTries":2,
+ "probeType":"tcp",
+ "probeReq":"",
+ "probeResp":"",
+ "probeDuration":10,
+ "probePort":6443
+ },
+ {
+ "hostName":"192.168.80.11",
+ "name":"192.168.80.11_tcp_6443",
+ "inactiveReTries":2,
+ "probeType":"tcp",
+ "probeReq":"",
+ "probeResp":"",
+ "probeDuration":10,
+ "probePort":6443
+ },
+ {
+ "hostName":"192.168.80.12",
+ "name":"192.168.80.12_tcp_6443",
+ "inactiveReTries":2,
+ "probeType":"tcp",
+ "probeReq":"",
+ "probeResp":"",
+ "probeDuration":10,
+ "probePort":6443
+ }
+ ]
+}

cicd/k3s-flannel-incluster-l2/Vagrantfile

@@ -45,6 +45,18 @@ Vagrant.configure("2") do |config|
  end
  end
 
+ config.vm.define "master3" do |master|
+ master.vm.hostname = 'master3'
+ master.vm.network :private_network, ip: "192.168.90.12", :netmask => "255.255.255.0"
+ master.vm.network :private_network, ip: "192.168.80.12", :netmask => "255.255.255.0"
+ master.vm.provision :shell, :path => "master3.sh"
+ master.vm.provider :virtualbox do |vbox|
+ vbox.customize ["modifyvm", :id, "--memory", 8192]
+ vbox.customize ["modifyvm", :id, "--cpus", 4]
+ end
+ end
+
+
  (1..workers).each do |node_number|
  config.vm.define "worker#{node_number}" do |worker|
  worker.vm.hostname = "worker#{node_number}"

cicd/k3s-flannel-incluster-l2/kube-loxilb.yml

@@ -101,6 +101,19 @@ spec:
  operator: Exists
  - effect: NoExecute
  operator: Exists
+ - key: "node-role.kubernetes.io/master"
+ operator: Exists
+ - key: "node-role.kubernetes.io/control-plane"
+ operator: Exists
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: "node-role.kubernetes.io/master"
+ operator: Exists
+ - key: "node-role.kubernetes.io/control-plane"
+ operator: Exists
  priorityClassName: system-node-critical
  serviceAccountName: kube-loxilb
  terminationGracePeriodSeconds: 0

cicd/k3s-flannel-incluster-l2/lbconfig.txt

@@ -0,0 +1,41 @@
+{
+ "lbAttr":[
+ {
+ "serviceArguments":{
+ "externalIP":"192.168.80.80",
+ "port":6443,
+ "protocol":"tcp",
+ "sel":0,
+ "mode":2,
+ "BGP":false,
+ "Monitor":true,
+ "inactiveTimeOut":240,
+ "block":0
+ },
+ "secondaryIPs":null,
+ "endpoints":[
+ {
+ "endpointIP":"192.168.80.10",
+ "targetPort":6443,
+ "weight":1,
+ "state":"active",
+ "counter":""
+ },
+ {
+ "endpointIP":"192.168.80.11",
+ "targetPort":6443,
+ "weight":1,
+ "state":"active",
+ "counter":""
+ },
+ {
+ "endpointIP":"192.168.80.12",
+ "targetPort":6443,
+ "weight":1,
+ "state":"active",
+ "counter":""
+ }
+ ]
+ }
+ ]
+}

cicd/k3s-flannel-incluster-l2/loxilb.yml

@@ -29,11 +29,23 @@ spec:
  operator: Exists
  - key: "node-role.kubernetes.io/control-plane"
  operator: Exists
+ volumes:
+ - name: hllb
+ hostPath:
+ path: /etc/loxilb
+ type: DirectoryOrCreate
  containers:
  - name: loxilb-app
  image: "ghcr.io/loxilb-io/loxilb:latest"
  imagePullPolicy: Always 
- command: [ "/root/loxilb-io/loxilb/loxilb", "--egr-hooks", "--blacklist=cni[0-9a-z]|veth.|flannel." ]
+ command:
+ - /root/loxilb-io/loxilb/loxilb
+ args:
+ - --egr-hooks
+ - --blacklist=cni[0-9a-z]|veth.|flannel.
+ volumeMounts:
+ - name: hllb
+ mountPath: /etc/loxilb
  ports:
  - containerPort: 11111
  - containerPort: 179

cicd/k3s-flannel-incluster-l2/master1.sh

@@ -1,12 +1,13 @@
 sudo su
 export MASTER_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.90' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
-curl -fL https://get.k3s.io | sh -s - server --node-ip=192.168.80.10 --disable servicelb --disable traefik --cluster-init external-hostname=192.168.80.10 --node-external-ip=192.168.80.10 --disable-cloud-controller
+curl -fL https://get.k3s.io | sh -s - server --node-ip=192.168.80.10 --disable servicelb --disable traefik --cluster-init external-hostname=192.168.80.10 --node-external-ip=192.168.80.80 --disable-cloud-controller --flannel-iface=eth1
 curl -sfL https:/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh -
 sleep 60
 echo $MASTER_IP > /vagrant/master-ip
 cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token
-sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /etc/rancher/k3s/k3s.yaml
 cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml
-sudo kubectl apply -f /vagrant/loxilb.yml
-sudo kubectl apply -f /vagrant/kube-loxilb.yml
+sed -i -e "s/127.0.0.1/192.168.80.80/g" /vagrant/k3s.yaml
+sudo mkdir -p /etc/loxilb
+sudo cp /vagrant/lbconfig.txt /etc/loxilb/
+sudo cp /vagrant/EPconfig.txt /etc/loxilb/
 /vagrant/wait_ready.sh

cicd/k3s-flannel-incluster-l2/master2.sh

@@ -2,9 +2,12 @@ sudo su
 export WORKER_ADDR=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
 export MASTER_ADDR=$(cat /vagrant/master-ip)
 export NODE_TOKEN=$(cat /vagrant/node-token)
-
+sudo mkdir -p /etc/loxilb
+sudo cp /vagrant/lbconfig.txt /etc/loxilb/
+sudo cp /vagrant/EPconfig.txt /etc/loxilb/
 #curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.11 --disable-cloud-controller -t ${NODE_TOKEN}
-curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.11 -t ${NODE_TOKEN}
+curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.80 -t ${NODE_TOKEN} --flannel-iface=eth1
 curl -sfL https:/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh -
-
+#sudo kubectl apply -f /vagrant/loxilb.yml
+#sudo kubectl apply -f /vagrant/kube-loxilb.yml
 /vagrant/wait_ready.sh

cicd/k3s-flannel-incluster-l2/master3.sh

@@ -0,0 +1,13 @@
+sudo su
+export WORKER_ADDR=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
+export MASTER_ADDR=$(cat /vagrant/master-ip)
+export NODE_TOKEN=$(cat /vagrant/node-token)
+sudo mkdir -p /etc/loxilb
+sudo cp /vagrant/lbconfig.txt /etc/loxilb/
+sudo cp /vagrant/EPconfig.txt /etc/loxilb/
+#curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.11 --disable-cloud-controller -t ${NODE_TOKEN}
+curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.12 external-hostname=192.168.80.12 --node-external-ip=192.168.80.80 -t ${NODE_TOKEN} --flannel-iface=eth1
+curl -sfL https:/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh -
+sudo kubectl apply -f /vagrant/loxilb.yml
+sudo kubectl apply -f /vagrant/kube-loxilb.yml
+/vagrant/wait_ready.sh

cicd/k3s-flannel-incluster-l2/rmconfig.sh

@@ -4,4 +4,5 @@ vagrant destroy -f worker1
 vagrant destroy -f worker2
 vagrant destroy -f master1
 vagrant destroy -f master2
+vagrant destroy -f master3
 vagrant destroy -f host

cicd/k3s-flannel-incluster-l2/worker.sh

@@ -4,7 +4,7 @@ export MASTER_ADDR=$(cat /vagrant/master-ip)
 export NODE_TOKEN=$(cat /vagrant/node-token)
 mkdir -p /etc/rancher/k3s
 cp -f /vagrant/k3s.yaml /etc/rancher/k3s/k3s.yaml
-curl -sfL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - agent --server https://192.168.80.10:6443 --node-ip=${WORKER_ADDR} --node-external-ip=${WORKER_ADDR} -t ${NODE_TOKEN}
+curl -sfL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - agent --server https://192.168.80.80:6443 --node-ip=${WORKER_ADDR} --node-external-ip=${WORKER_ADDR} -t ${NODE_TOKEN} --flannel-iface=eth1
 #sudo kubectl apply -f /vagrant/loxilb-peer.yml
 #sudo kubectl apply -f /vagrant/nginx.yml
 #sudo kubectl apply -f /vagrant/udp.yml