From 69c4dc288d6795ee5fa269b4620b9ceefa8ac7e3 Mon Sep 17 00:00:00 2001
From: Trekkie Coder <trekkie@netlox.io>
Date: Mon, 10 Jul 2023 22:58:39 +0900
Subject: [PATCH] Graceful handling on SIGTERM and SIGINT (#348)

---
 api/restapi/server.go   | 33 ++++++++++++++++++++++++++-------
 loxinet/dpebpf_linux.go | 28 ++++++++++++++++++++++++++++
 loxinet/loxinet.go      | 10 +++++++---
 3 files changed, 61 insertions(+), 10 deletions(-)

diff --git a/api/restapi/server.go b/api/restapi/server.go
index 5b9e1b162..0d94c505f 100644
--- a/api/restapi/server.go
+++ b/api/restapi/server.go
@@ -8,6 +8,11 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"github.com/go-openapi/runtime/flagext"
+	"github.com/go-openapi/swag"
+	flags "github.com/jessevdk/go-flags"
+	"github.com/loxilb-io/loxilb/api/restapi/operations"
+	"golang.org/x/net/netutil"
 	"log"
 	"net"
 	"net/http"
@@ -18,13 +23,6 @@ import (
 	"sync/atomic"
 	"syscall"
 	"time"
-
-	"github.com/go-openapi/runtime/flagext"
-	"github.com/go-openapi/swag"
-	flags "github.com/jessevdk/go-flags"
-	"golang.org/x/net/netutil"
-
-	"github.com/loxilb-io/loxilb/api/restapi/operations"
 )
 
 const (
@@ -104,6 +102,8 @@ type Server struct {
 	interrupt    chan os.Signal
 }
 
+var sigShutOk chan bool
+
 // Logf logs message either via defined user logger or via system one if no user logger is defined.
 func (s *Server) Logf(f string, args ...interface{}) {
 	if s.api != nil && s.api.Logger != nil {
@@ -167,6 +167,10 @@ func (s *Server) Serve() (err error) {
 		s.SetHandler(s.api.Serve(nil))
 	}
 
+	if sigShutOk == nil {
+		sigShutOk = make(chan bool)
+	}
+
 	wg := new(sync.WaitGroup)
 	once := new(sync.Once)
 	signalNotify(s.interrupt)
@@ -487,6 +491,20 @@ func (s *Server) TLSListener() (net.Listener, error) {
 	return s.httpsServerL, nil
 }
 
+func waitServerSigShutOk() {
+	for {
+		select {
+		case <-sigShutOk:
+			return
+		}
+	}
+}
+
+// ServerSigShutOk - Notifier for server to be shutdown on signals
+func ServerSigShutOk() {
+	sigShutOk <- true
+}
+
 func handleInterrupt(once *sync.Once, s *Server) {
 	once.Do(func() {
 		for range s.interrupt {
@@ -494,6 +512,7 @@ func handleInterrupt(once *sync.Once, s *Server) {
 				s.Logf("Server already shutting down")
 				continue
 			}
+			waitServerSigShutOk()
 			s.interrupted = true
 			s.Logf("Shutting down... ")
 			if err := s.Shutdown(); err != nil {
diff --git a/loxinet/dpebpf_linux.go b/loxinet/dpebpf_linux.go
index db08bdb7a..640a2a69c 100644
--- a/loxinet/dpebpf_linux.go
+++ b/loxinet/dpebpf_linux.go
@@ -286,6 +286,34 @@ func DpEbpfInit(clusterEn bool, nodeNum int, rssEn bool, logLevel tk.LogLevelT)
 	return ne
 }
 
+// DpEbpfUnInit - uninitialize the ebpf dp subsystem
+func (e *DpEbpfH) DpEbpfUnInit() {
+
+	e.tDone <- true
+	e.ToFinCh <- 1
+
+	// Make sure to unload eBPF programs
+	ifList, err := net.Interfaces()
+	if err != nil {
+		return
+	}
+
+	for _, intf := range ifList {
+
+		tk.LogIt(tk.LogInfo, "ebpf unload - %s\n", intf.Name)
+		ifStr := C.CString(intf.Name)
+		section := C.CString(string(C.TC_LL_SEC_DEFAULT))
+		if e.RssEn {
+			xSection := C.CString(string(C.XDP_LL_SEC_DEFAULT))
+			C.llb_dp_link_attach(ifStr, xSection, C.LL_BPF_MOUNT_XDP, 1)
+			C.free(unsafe.Pointer(xSection))
+		}
+		C.llb_dp_link_attach(ifStr, section, C.LL_BPF_MOUNT_TC, 1)
+		C.free(unsafe.Pointer(ifStr))
+		C.free(unsafe.Pointer(section))
+	}
+}
+
 func convNetIP2DPv6Addr(addr unsafe.Pointer, goIP net.IP) {
 	aPtr := (*C.uchar)(addr)
 	for bp := 0; bp < 16; bp++ {
diff --git a/loxinet/loxinet.go b/loxinet/loxinet.go
index 74c39b703..c9f8603db 100644
--- a/loxinet/loxinet.go
+++ b/loxinet/loxinet.go
@@ -22,6 +22,7 @@ import (
 	apiserver "github.com/loxilb-io/loxilb/api"
 	nlp "github.com/loxilb-io/loxilb/api/loxinlp"
 	prometheus "github.com/loxilb-io/loxilb/api/prometheus"
+	restapi "github.com/loxilb-io/loxilb/api/restapi"
 	cmn "github.com/loxilb-io/loxilb/common"
 	opts "github.com/loxilb-io/loxilb/options"
 	tk "github.com/loxilb-io/loxilib"
@@ -179,8 +180,12 @@ func loxiNetTicker() {
 					try++
 				}
 			} else if sig == syscall.SIGHUP {
-				fmt.Printf("SIGHUP called\n")
+				tk.LogIt(tk.LogCritical, "SIGHUP received\n")
 				pprof.StopCPUProfile()
+			} else if sig == syscall.SIGINT || sig == syscall.SIGTERM {
+				tk.LogIt(tk.LogCritical, "Shutdown on sig %v\n", sig)
+				mh.dpEbpf.DpEbpfUnInit()
+				restapi.ServerSigShutOk()
 			}
 		case t := <-mh.ticker.C:
 			tk.LogIt(-1, "Tick at %v\n", t)
@@ -225,8 +230,7 @@ func loxiNetInit() {
 	mh.sumDis = opts.Opts.CSumDisable
 	mh.pProbe = opts.Opts.PassiveEPProbe
 	mh.sigCh = make(chan os.Signal, 5)
-	signal.Notify(mh.sigCh, os.Interrupt, syscall.SIGCHLD)
-	signal.Notify(mh.sigCh, os.Interrupt, syscall.SIGHUP)
+	signal.Notify(mh.sigCh, os.Interrupt, syscall.SIGCHLD, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM)
 
 	// Check if profiling is enabled
 	if opts.Opts.CPUProfile != "none" {