Persistent server pools (#226)

* GetAllServers with version & timestamp tests
* Storage package to sync servers
* Use storage Sync to get and use servers

internal/storage/merge.go

@@ -0,0 +1,68 @@
+package storage
+
+import (
+	"time"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+func getUnixTimeDifference(unix1, unix2 int64) (difference time.Duration) {
+	difference = time.Unix(unix1, 0).Sub(time.Unix(unix2, 0))
+	if difference < 0 {
+		difference = -difference
+	}
+	return difference.Truncate(time.Second)
+}
+
+func (s *storage) mergeServers(hardcoded, persistent models.AllServers) (merged models.AllServers) {
+	merged.Version = hardcoded.Version
+	merged.Cyberghost = hardcoded.Cyberghost
+	if persistent.Cyberghost.Timestamp > hardcoded.Cyberghost.Timestamp {
+		s.logger.Info("Using Cyberghost servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Cyberghost.Timestamp, hardcoded.Cyberghost.Timestamp))
+		merged.Cyberghost = persistent.Cyberghost
+	}
+	merged.Mullvad = hardcoded.Mullvad
+	if persistent.Mullvad.Timestamp > hardcoded.Mullvad.Timestamp {
+		s.logger.Info("Using Mullvad servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Mullvad.Timestamp, hardcoded.Mullvad.Timestamp))
+		merged.Mullvad = persistent.Mullvad
+	}
+	merged.Nordvpn = hardcoded.Nordvpn
+	if persistent.Nordvpn.Timestamp > hardcoded.Nordvpn.Timestamp {
+		s.logger.Info("Using Nordvpn servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Nordvpn.Timestamp, hardcoded.Nordvpn.Timestamp))
+		merged.Nordvpn = persistent.Nordvpn
+	}
+	merged.Pia = hardcoded.Pia
+	if persistent.Pia.Timestamp > hardcoded.Pia.Timestamp {
+		s.logger.Info("Using Private Internet Access servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Pia.Timestamp, hardcoded.Pia.Timestamp))
+		merged.Pia = persistent.Pia
+	}
+	merged.Purevpn = hardcoded.Purevpn
+	if persistent.Purevpn.Timestamp > hardcoded.Purevpn.Timestamp {
+		s.logger.Info("Using Purevpn servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Purevpn.Timestamp, hardcoded.Purevpn.Timestamp))
+		merged.Purevpn = persistent.Purevpn
+	}
+	merged.Surfshark = hardcoded.Surfshark
+	if persistent.Surfshark.Timestamp > hardcoded.Surfshark.Timestamp {
+		s.logger.Info("Using Surfshark servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Surfshark.Timestamp, hardcoded.Surfshark.Timestamp))
+		merged.Surfshark = persistent.Surfshark
+	}
+	merged.Vyprvpn = hardcoded.Vyprvpn
+	if persistent.Vyprvpn.Timestamp > hardcoded.Vyprvpn.Timestamp {
+		s.logger.Info("Using Vyprvpn servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Vyprvpn.Timestamp, hardcoded.Vyprvpn.Timestamp))
+		merged.Vyprvpn = persistent.Vyprvpn
+	}
+	merged.Windscribe = hardcoded.Windscribe
+	if persistent.Windscribe.Timestamp > hardcoded.Windscribe.Timestamp {
+		s.logger.Info("Using Windscribe servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Windscribe.Timestamp, hardcoded.Windscribe.Timestamp))
+		merged.Windscribe = persistent.Windscribe
+	}
+	return merged
+}

internal/storage/storage.go

@@ -0,0 +1,29 @@
+package storage
+
+import (
+	"io/ioutil"
+	"os"
+
+	"github.com/qdm12/gluetun/internal/models"
+	"github.com/qdm12/golibs/logging"
+)
+
+type Storage interface {
+	SyncServers(hardcodedServers models.AllServers) (allServers models.AllServers, err error)
+}
+
+type storage struct {
+	osStat    func(name string) (os.FileInfo, error)
+	readFile  func(filename string) (data []byte, err error)
+	writeFile func(filename string, data []byte, perm os.FileMode) error
+	logger    logging.Logger
+}
+
+func New(logger logging.Logger) Storage {
+	return &storage{
+		osStat:    os.Stat,
+		readFile:  ioutil.ReadFile,
+		writeFile: ioutil.WriteFile,
+		logger:    logger.WithPrefix("storage: "),
+	}
+}

internal/storage/sync.go

@@ -0,0 +1,72 @@
+package storage
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+const (
+	jsonFilepath = "/gluetun/servers.json"
+)
+
+func countServers(allServers models.AllServers) int {
+	return len(allServers.Cyberghost.Servers) +
+		len(allServers.Mullvad.Servers) +
+		len(allServers.Nordvpn.Servers) +
+		len(allServers.Pia.Servers) +
+		len(allServers.Purevpn.Servers) +
+		len(allServers.Surfshark.Servers) +
+		len(allServers.Vyprvpn.Servers) +
+		len(allServers.Windscribe.Servers)
+}
+
+func (s *storage) SyncServers(hardcodedServers models.AllServers) (allServers models.AllServers, err error) {
+	// Eventually read file
+	var serversOnFile models.AllServers
+	_, err = s.osStat(jsonFilepath)
+	if err == nil {
+		serversOnFile, err = s.readFromFile()
+		if err != nil {
+			return allServers, err
+		}
+	} else if !os.IsNotExist(err) {
+		return allServers, err
+	}
+
+	// Merge data from file and hardcoded
+	s.logger.Info("Merging by most recent %d hardcoded servers and %d servers read from %s",
+		countServers(hardcodedServers), countServers(serversOnFile), jsonFilepath)
+	allServers = s.mergeServers(hardcodedServers, serversOnFile)
+
+	// Eventually write file
+	if reflect.DeepEqual(serversOnFile, allServers) {
+		return allServers, nil
+	}
+	return allServers, s.flushToFile(allServers)
+}
+
+func (s *storage) readFromFile() (servers models.AllServers, err error) {
+	bytes, err := s.readFile(jsonFilepath)
+	if err != nil {
+		return servers, err
+	}
+	if err := json.Unmarshal(bytes, &servers); err != nil {
+		return servers, err
+	}
+	return servers, nil
+}
+
+func (s *storage) flushToFile(servers models.AllServers) error {
+	bytes, err := json.MarshalIndent(servers, "", "  ")
+	if err != nil {
+		return fmt.Errorf("cannot write to file: %w", err)
+	}
+	if err := s.writeFile(jsonFilepath, bytes, 0644); err != nil {
+		return err
+	}
+	return nil
+}