Maintenance: shutdown order

- Order of threads to shutdown (control then tickers then health etc.)
- Rely on closing channels instead of waitgroups
- Move exit logs from each package to the shutdown package

internal/shutdown/order.go

@@ -0,0 +1,50 @@
+package shutdown
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/qdm12/golibs/logging"
+)
+
+type Order interface {
+	Append(waves ...Wave)
+	Shutdown(timeout time.Duration, logger logging.Logger) (err error)
+}
+
+type order struct {
+	waves []Wave
+	total int // for logging only
+}
+
+func NewOrder() Order {
+	return &order{}
+}
+
+var ErrIncomplete = errors.New("one or more routines did not terminate gracefully")
+
+func (o *order) Append(waves ...Wave) {
+	o.waves = append(o.waves, waves...)
+}
+
+func (o *order) Shutdown(timeout time.Duration, logger logging.Logger) (err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	total := 0
+	incomplete := 0
+
+	for _, wave := range o.waves {
+		total += wave.size()
+		incomplete += wave.shutdown(ctx, logger)
+	}
+
+	if incomplete == 0 {
+		return nil
+	}
+
+	return fmt.Errorf("%w: %d not terminated on %d routines",
+		ErrIncomplete, incomplete, total)
+}

internal/shutdown/routine.go

@@ -0,0 +1,39 @@
+package shutdown
+
+import (
+	"context"
+	"fmt"
+	"time"
+)
+
+type routine struct {
+	name    string
+	cancel  context.CancelFunc
+	done    <-chan struct{}
+	timeout time.Duration
+}
+
+func newRoutine(name string) (r routine,
+	ctx context.Context, done chan struct{}) {
+	ctx, cancel := context.WithCancel(context.Background())
+	done = make(chan struct{})
+	return routine{
+		name:   name,
+		cancel: cancel,
+		done:   done,
+	}, ctx, done
+}
+
+func (r *routine) shutdown(ctx context.Context) (err error) {
+	ctx, cancel := context.WithTimeout(ctx, r.timeout)
+	defer cancel()
+
+	r.cancel()
+
+	select {
+	case <-r.done:
+		return nil
+	case <-ctx.Done():
+		return fmt.Errorf("for routine %q: %w", r.name, ctx.Err())
+	}
+}

internal/shutdown/wave.go

@@ -0,0 +1,66 @@
+package shutdown
+
+import (
+	"context"
+	"time"
+
+	"github.com/qdm12/golibs/logging"
+)
+
+type Wave interface {
+	Add(name string, timeout time.Duration) (
+		ctx context.Context, done chan struct{})
+	size() int
+	shutdown(ctx context.Context, logger logging.Logger) (incomplete int)
+}
+
+type wave struct {
+	name     string
+	routines []routine
+}
+
+func NewWave(name string) Wave {
+	return &wave{
+		name: name,
+	}
+}
+
+func (w *wave) Add(name string, timeout time.Duration) (ctx context.Context, done chan struct{}) {
+	ctx, cancel := context.WithCancel(context.Background())
+	done = make(chan struct{})
+	routine := routine{
+		name:    name,
+		cancel:  cancel,
+		done:    done,
+		timeout: timeout,
+	}
+	w.routines = append(w.routines, routine)
+	return ctx, done
+}
+
+func (w *wave) size() int { return len(w.routines) }
+
+func (w *wave) shutdown(ctx context.Context, logger logging.Logger) (incomplete int) {
+	completed := make(chan bool)
+
+	for _, r := range w.routines {
+		go func(r routine) {
+			if err := r.shutdown(ctx); err != nil {
+				logger.Warn(w.name + " routines: " + err.Error() + " ⚠️")
+				completed <- false
+			} else {
+				logger.Info(w.name + " routines: " + r.name + " terminated ✔️")
+				completed <- err == nil
+			}
+		}(r)
+	}
+
+	for range w.routines {
+		c := <-completed
+		if !c {
+			incomplete++
+		}
+	}
+
+	return incomplete
+}