Maint: rework openvpn restart on unhealthy

This commit is contained in:
Quentin McGaw (desktop)
2021-07-18 03:17:48 +00:00
parent 7e50c95823
commit c0d27b4bfc
5 changed files with 59 additions and 74 deletions

View File

@@ -46,7 +46,6 @@ type looper struct {
client *http.Client
openFile os.OpenFileFunc
tunnelReady chan<- struct{}
healthy <-chan bool
// Internal channels and values
stop <-chan struct{}
stopped chan<- struct{}
@@ -55,20 +54,18 @@ type looper struct {
portForwardSignals chan net.IP
userTrigger bool
// Internal constant values
backoffTime time.Duration
healthWaitTime time.Duration
backoffTime time.Duration
}
const (
defaultBackoffTime = 15 * time.Second
defaultHealthWaitTime = 6 * time.Second
defaultBackoffTime = 15 * time.Second
)
func NewLooper(settings configuration.OpenVPN,
username string, puid, pgid int, allServers models.AllServers,
conf Configurator, fw firewall.Configurator, routing routing.Routing,
logger logging.ParentLogger, client *http.Client, openFile os.OpenFileFunc,
tunnelReady chan<- struct{}, healthy <-chan bool) Looper {
tunnelReady chan<- struct{}) Looper {
start := make(chan struct{})
running := make(chan models.LoopStatus)
stop := make(chan struct{})
@@ -90,7 +87,6 @@ func NewLooper(settings configuration.OpenVPN,
client: client,
openFile: openFile,
tunnelReady: tunnelReady,
healthy: healthy,
start: start,
running: running,
stop: stop,
@@ -98,7 +94,6 @@ func NewLooper(settings configuration.OpenVPN,
portForwardSignals: make(chan net.IP),
userTrigger: true,
backoffTime: defaultBackoffTime,
healthWaitTime: defaultHealthWaitTime,
}
}
@@ -116,7 +111,7 @@ func (l *looper) signalOrSetStatus(status models.LoopStatus) {
}
}
func (l *looper) Run(ctx context.Context, done chan<- struct{}) { //nolint:gocognit
func (l *looper) Run(ctx context.Context, done chan<- struct{}) {
defer close(done)
select {
@@ -243,25 +238,6 @@ func (l *looper) Run(ctx context.Context, done chan<- struct{}) { //nolint:gocog
stayHere = false
l.state.Unlock()
case healthy := <-l.healthy:
if healthy {
continue
}
// ensure it stays unhealthy for some time before restarting it
healthy = l.waitForHealth(ctx)
if healthy || ctx.Err() != nil {
continue
}
l.logger.Warn("unhealthy program: restarting openvpn")
l.state.SetStatus(constants.Stopping)
openvpnCancel()
<-waitError
close(waitError)
closeStreams()
<-portForwardDone
l.state.SetStatus(constants.Stopped)
stayHere = false
}
}
openvpnCancel()
@@ -284,35 +260,6 @@ func (l *looper) logAndWait(ctx context.Context, err error) {
}
}
// waitForHealth waits for a true healthy signal
// after restarting openvpn in order to avoid restarting
// openvpn in a loop as it requires a few seconds to connect.
func (l *looper) waitForHealth(ctx context.Context) (healthy bool) {
l.logger.Info("unhealthy program: waiting %s for it to change to healthy", l.healthWaitTime)
timer := time.NewTimer(l.healthWaitTime)
l.healthWaitTime *= 2
for {
select {
case healthy = <-l.healthy:
if !healthy {
break
}
if !timer.Stop() {
<-timer.C
}
l.healthWaitTime = defaultHealthWaitTime
return true
case <-timer.C:
return false
case <-ctx.Done():
if !timer.Stop() {
<-timer.C
}
return false
}
}
}
// portForward is a blocking operation which may or may not be infinite.
// You should therefore always call it in a goroutine.
func (l *looper) portForward(ctx context.Context,