Maint: rework openvpn restart on unhealthy
This commit is contained in:
@@ -46,7 +46,6 @@ type looper struct {
|
||||
client *http.Client
|
||||
openFile os.OpenFileFunc
|
||||
tunnelReady chan<- struct{}
|
||||
healthy <-chan bool
|
||||
// Internal channels and values
|
||||
stop <-chan struct{}
|
||||
stopped chan<- struct{}
|
||||
@@ -55,20 +54,18 @@ type looper struct {
|
||||
portForwardSignals chan net.IP
|
||||
userTrigger bool
|
||||
// Internal constant values
|
||||
backoffTime time.Duration
|
||||
healthWaitTime time.Duration
|
||||
backoffTime time.Duration
|
||||
}
|
||||
|
||||
const (
|
||||
defaultBackoffTime = 15 * time.Second
|
||||
defaultHealthWaitTime = 6 * time.Second
|
||||
defaultBackoffTime = 15 * time.Second
|
||||
)
|
||||
|
||||
func NewLooper(settings configuration.OpenVPN,
|
||||
username string, puid, pgid int, allServers models.AllServers,
|
||||
conf Configurator, fw firewall.Configurator, routing routing.Routing,
|
||||
logger logging.ParentLogger, client *http.Client, openFile os.OpenFileFunc,
|
||||
tunnelReady chan<- struct{}, healthy <-chan bool) Looper {
|
||||
tunnelReady chan<- struct{}) Looper {
|
||||
start := make(chan struct{})
|
||||
running := make(chan models.LoopStatus)
|
||||
stop := make(chan struct{})
|
||||
@@ -90,7 +87,6 @@ func NewLooper(settings configuration.OpenVPN,
|
||||
client: client,
|
||||
openFile: openFile,
|
||||
tunnelReady: tunnelReady,
|
||||
healthy: healthy,
|
||||
start: start,
|
||||
running: running,
|
||||
stop: stop,
|
||||
@@ -98,7 +94,6 @@ func NewLooper(settings configuration.OpenVPN,
|
||||
portForwardSignals: make(chan net.IP),
|
||||
userTrigger: true,
|
||||
backoffTime: defaultBackoffTime,
|
||||
healthWaitTime: defaultHealthWaitTime,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,7 +111,7 @@ func (l *looper) signalOrSetStatus(status models.LoopStatus) {
|
||||
}
|
||||
}
|
||||
|
||||
func (l *looper) Run(ctx context.Context, done chan<- struct{}) { //nolint:gocognit
|
||||
func (l *looper) Run(ctx context.Context, done chan<- struct{}) {
|
||||
defer close(done)
|
||||
|
||||
select {
|
||||
@@ -243,25 +238,6 @@ func (l *looper) Run(ctx context.Context, done chan<- struct{}) { //nolint:gocog
|
||||
stayHere = false
|
||||
|
||||
l.state.Unlock()
|
||||
case healthy := <-l.healthy:
|
||||
if healthy {
|
||||
continue
|
||||
}
|
||||
// ensure it stays unhealthy for some time before restarting it
|
||||
healthy = l.waitForHealth(ctx)
|
||||
if healthy || ctx.Err() != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
l.logger.Warn("unhealthy program: restarting openvpn")
|
||||
l.state.SetStatus(constants.Stopping)
|
||||
openvpnCancel()
|
||||
<-waitError
|
||||
close(waitError)
|
||||
closeStreams()
|
||||
<-portForwardDone
|
||||
l.state.SetStatus(constants.Stopped)
|
||||
stayHere = false
|
||||
}
|
||||
}
|
||||
openvpnCancel()
|
||||
@@ -284,35 +260,6 @@ func (l *looper) logAndWait(ctx context.Context, err error) {
|
||||
}
|
||||
}
|
||||
|
||||
// waitForHealth waits for a true healthy signal
|
||||
// after restarting openvpn in order to avoid restarting
|
||||
// openvpn in a loop as it requires a few seconds to connect.
|
||||
func (l *looper) waitForHealth(ctx context.Context) (healthy bool) {
|
||||
l.logger.Info("unhealthy program: waiting %s for it to change to healthy", l.healthWaitTime)
|
||||
timer := time.NewTimer(l.healthWaitTime)
|
||||
l.healthWaitTime *= 2
|
||||
for {
|
||||
select {
|
||||
case healthy = <-l.healthy:
|
||||
if !healthy {
|
||||
break
|
||||
}
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
l.healthWaitTime = defaultHealthWaitTime
|
||||
return true
|
||||
case <-timer.C:
|
||||
return false
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// portForward is a blocking operation which may or may not be infinite.
|
||||
// You should therefore always call it in a goroutine.
|
||||
func (l *looper) portForward(ctx context.Context,
|
||||
|
||||
Reference in New Issue
Block a user