Persistent server pools (#226)

* GetAllServers with version & timestamp tests
* Storage package to sync servers
* Use storage Sync to get and use servers

Dockerfile

@@ -104,5 +104,6 @@ RUN apk add -q --progress --no-cache --update openvpn ca-certificates iptables i
     rm -rf /var/cache/apk/* /etc/unbound/* /usr/sbin/unbound-* /etc/tinyproxy/tinyproxy.conf && \
     deluser openvpn && \
     deluser tinyproxy && \
-    deluser unbound
+    deluser unbound && \
+    mkdir /gluetun
 COPY --from=builder /tmp/gobuild/entrypoint /entrypoint

README.md

@@ -52,6 +52,7 @@ iptables, DNS over TLS, ShadowSocks and Tinyproxy*
     ```bash
     docker run -d --name gluetun --cap-add=NET_ADMIN \
     -e REGION="CA Montreal" -e USER=js89ds7 -e PASSWORD=8fd9s239G \
+    -v /yourpath:/gluetun \
     qmcgaw/private-internet-access
     ```
 

cmd/gluetun/main.go

@@ -23,6 +23,7 @@ import (
 	"github.com/qdm12/gluetun/internal/server"
 	"github.com/qdm12/gluetun/internal/settings"
 	"github.com/qdm12/gluetun/internal/shadowsocks"
+	"github.com/qdm12/gluetun/internal/storage"
 	"github.com/qdm12/gluetun/internal/tinyproxy"
 	"github.com/qdm12/golibs/command"
 	"github.com/qdm12/golibs/files"
@@ -88,6 +89,14 @@ func _main(background context.Context, args []string) int {
 	fatalOnError(err)
 	logger.Info(allSettings.String())
 
+	// TODO run this in a loop or in openvpn to reload from file without restarting
+	storage := storage.New(logger)
+	allServers, err := storage.SyncServers(constants.GetAllServers())
+	if err != nil {
+		logger.Error(err)
+		return 1
+	}
+
 	// Should never change
 	uid, gid := allSettings.System.UID, allSettings.System.GID
 
@@ -143,7 +152,7 @@ func _main(background context.Context, args []string) int {
 
 	wg := &sync.WaitGroup{}
 
-	openvpnLooper := openvpn.NewLooper(allSettings.VPNSP, allSettings.OpenVPN, uid, gid,
+	openvpnLooper := openvpn.NewLooper(allSettings.VPNSP, allSettings.OpenVPN, uid, gid, allServers,
 		ovpnConf, firewallConf, logger, client, fileManager, streamMerger, fatalOnError)
 	restartOpenvpn := openvpnLooper.Restart
 	portForward := openvpnLooper.PortForward

docker-compose.yml

@@ -12,6 +12,8 @@ services:
       - 8388:8388/udp # Shadowsocks
       - 8000:8000/tcp # Built-in HTTP control server
     # command:
+    volumes:
+      - /yourpath:/gluetun
     environment:
       # More variables are available, see the readme table
       - VPNSP=private internet access

internal/cli/cli.go

@@ -7,9 +7,11 @@ import (
 
 	"net"
 
+	"github.com/qdm12/gluetun/internal/constants"
 	"github.com/qdm12/gluetun/internal/params"
 	"github.com/qdm12/gluetun/internal/provider"
 	"github.com/qdm12/gluetun/internal/settings"
+	"github.com/qdm12/gluetun/internal/storage"
 	"github.com/qdm12/golibs/files"
 	"github.com/qdm12/golibs/logging"
 )
@@ -54,7 +56,11 @@ func OpenvpnConfig() error {
 	if err != nil {
 		return err
 	}
-	providerConf := provider.New(allSettings.OpenVPN.Provider.Name)
+	allServers, err := storage.New(logger).SyncServers(constants.GetAllServers())
+	if err != nil {
+		return err
+	}
+	providerConf := provider.New(allSettings.OpenVPN.Provider.Name, allServers)
 	connections, err := providerConf.GetOpenVPNConnections(allSettings.OpenVPN.Provider.ServerSelection)
 	if err != nil {
 		return err

internal/constants/servers.go

@@ -0,0 +1,49 @@
+package constants
+
+import "github.com/qdm12/gluetun/internal/models"
+
+func GetAllServers() (allServers models.AllServers) {
+	return models.AllServers{
+		Version: 1, // used for migration of the top level scheme
+		Cyberghost: models.CyberghostServers{
+			Version:   1,          // model version
+			Timestamp: 1598236838, // latest takes precedence
+			Servers:   CyberghostServers(),
+		},
+		Mullvad: models.MullvadServers{
+			Version:   1,
+			Timestamp: 1598236838,
+			Servers:   MullvadServers(),
+		},
+		Nordvpn: models.NordvpnServers{
+			Version:   1,
+			Timestamp: 1598236838,
+			Servers:   NordvpnServers(),
+		},
+		Pia: models.PiaServers{
+			Version:   1,
+			Timestamp: 1598236838,
+			Servers:   PIAServers(),
+		},
+		Purevpn: models.PurevpnServers{
+			Version:   1,
+			Timestamp: 1598236838,
+			Servers:   PurevpnServers(),
+		},
+		Surfshark: models.SurfsharkServers{
+			Version:   1,
+			Timestamp: 1598236838,
+			Servers:   SurfsharkServers(),
+		},
+		Vyprvpn: models.VyprvpnServers{
+			Version:   1,
+			Timestamp: 1598236838,
+			Servers:   VyprvpnServers(),
+		},
+		Windscribe: models.WindscribeServers{
+			Version:   1,
+			Timestamp: 1598236838,
+			Servers:   WindscribeServers(),
+		},
+	}
+}

internal/constants/servers_test.go

@@ -0,0 +1,58 @@
+package constants
+
+import (
+	"crypto/md5" //nolint:gosec
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/qdm12/gluetun/internal/models"
+	"github.com/stretchr/testify/assert"
+)
+
+func digestServerModelVersion(t *testing.T, server interface{}, version uint16) string { //nolint:unparam
+	bytes, err := json.Marshal(server)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bytes = append(bytes, []byte(fmt.Sprintf("%d", version))...)
+	arr := md5.Sum(bytes) //nolint:gosec
+	return base64.RawStdEncoding.EncodeToString(arr[:])
+}
+
+func Test_versions(t *testing.T) {
+	t.Parallel()
+	allServers := GetAllServers()
+	assert.Equal(t, "e8eLGRpb1sNX8mDNPOjA6g", digestServerModelVersion(t, models.CyberghostServer{}, allServers.Cyberghost.Version))
+	assert.Equal(t, "Dk7lO6AbS46VdHJKQb5Wxg", digestServerModelVersion(t, models.MullvadServer{}, allServers.Mullvad.Version))
+	assert.Equal(t, "fjzfUqJH0KvetGRdZYEtOg", digestServerModelVersion(t, models.NordvpnServer{}, allServers.Nordvpn.Version))
+	assert.Equal(t, "gYO+bJZCtQvxVk2dTi5d5Q", digestServerModelVersion(t, models.PIAServer{}, allServers.Pia.Version))
+	assert.Equal(t, "EZ/SBXQOCS/iJU7A9yc7vg", digestServerModelVersion(t, models.PurevpnServer{}, allServers.Purevpn.Version))
+	assert.Equal(t, "7yfMpHwzRpEngA/6nYsNag", digestServerModelVersion(t, models.SurfsharkServer{}, allServers.Surfshark.Version))
+	assert.Equal(t, "7yfMpHwzRpEngA/6nYsNag", digestServerModelVersion(t, models.VyprvpnServer{}, allServers.Vyprvpn.Version))
+	assert.Equal(t, "7yfMpHwzRpEngA/6nYsNag", digestServerModelVersion(t, models.WindscribeServer{}, allServers.Windscribe.Version))
+}
+
+func digestServersTimestamp(t *testing.T, servers interface{}, timestamp int64) string { //nolint:unparam
+	bytes, err := json.Marshal(servers)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bytes = append(bytes, []byte(fmt.Sprintf("%d", timestamp))...)
+	arr := md5.Sum(bytes) //nolint:gosec
+	return base64.RawStdEncoding.EncodeToString(arr[:])
+}
+
+func Test_timestamps(t *testing.T) {
+	t.Parallel()
+	allServers := GetAllServers()
+	assert.Equal(t, "lZa+3P5DGuo9VXlsXsW5Jw", digestServersTimestamp(t, allServers.Cyberghost.Servers, allServers.Cyberghost.Timestamp))
+	assert.Equal(t, "cK5eeY2KU+doigSAonCfVQ", digestServersTimestamp(t, allServers.Mullvad.Servers, allServers.Mullvad.Timestamp))
+	assert.Equal(t, "ZfMT6wXJJBAT0fOqx3TuOA", digestServersTimestamp(t, allServers.Nordvpn.Servers, allServers.Nordvpn.Timestamp))
+	assert.Equal(t, "JGzrjRLM5MlWUjAkjpqKWw", digestServersTimestamp(t, allServers.Pia.Servers, allServers.Pia.Timestamp))
+	assert.Equal(t, "IW1gWNvYTSRDxpAv4kwmzg", digestServersTimestamp(t, allServers.Purevpn.Servers, allServers.Purevpn.Timestamp))
+	assert.Equal(t, "f934tXGfEVeNGT3TUdnpxw", digestServersTimestamp(t, allServers.Surfshark.Servers, allServers.Surfshark.Timestamp))
+	assert.Equal(t, "wwkmrCGEW06x7ze8+FO2hg", digestServersTimestamp(t, allServers.Vyprvpn.Servers, allServers.Vyprvpn.Timestamp))
+	assert.Equal(t, "jT4WjRKNpYojILLJWzGRRw", digestServersTimestamp(t, allServers.Windscribe.Servers, allServers.Windscribe.Timestamp))
+}

internal/constants/splash.go

@@ -2,9 +2,9 @@ package constants
 
 const (
 	// Announcement is a message announcement
-	Announcement = "Video of the Git history of Gluetun (2020 is crazy): https://youtu.be/khipOYJtGJ0"
+	Announcement = "Persistent server IP addresses at /gluetun/servers.json, please BIND MOUNT"
 	// AnnouncementExpiration is the expiration date of the announcement in format yyyy-mm-dd
-	AnnouncementExpiration = "2020-07-30"
+	AnnouncementExpiration = "2020-09-30"
 )
 
 const (

internal/models/server.go

@@ -0,0 +1,52 @@
+package models
+
+import "net"
+
+type PIAServer struct {
+	IPs    []net.IP `json:"ips"`
+	Region string   `json:"region"`
+}
+
+type MullvadServer struct {
+	IPs     []net.IP `json:"ips"`
+	Country string   `json:"country"`
+	City    string   `json:"city"`
+	ISP     string   `json:"isp"`
+	Owned   bool     `json:"owned"`
+}
+
+type WindscribeServer struct {
+	Region string   `json:"region"`
+	IPs    []net.IP `json:"ips"`
+}
+
+type SurfsharkServer struct {
+	Region string   `json:"region"`
+	IPs    []net.IP `json:"ips"`
+}
+
+type CyberghostServer struct {
+	Region string   `json:"region"`
+	Group  string   `json:"group"`
+	IPs    []net.IP `json:"ips"`
+}
+
+type VyprvpnServer struct {
+	Region string   `json:"region"`
+	IPs    []net.IP `json:"ips"`
+}
+
+type NordvpnServer struct { //nolint:maligned
+	Region string `json:"region"`
+	Number uint16 `json:"number"`
+	IP     net.IP `json:"ip"`
+	TCP    bool   `json:"tcp"`
+	UDP    bool   `json:"udp"`
+}
+
+type PurevpnServer struct {
+	Region  string   `json:"region"`
+	Country string   `json:"country"`
+	City    string   `json:"city"`
+	IPs     []net.IP `json:"ips"`
+}

internal/models/servers.go

@@ -1,52 +1,54 @@
 package models
 
-import "net"
+type AllServers struct {
-
+	Version    uint16            `json:"version"`
-type PIAServer struct {
+	Cyberghost CyberghostServers `json:"cyberghost"`
-	IPs    []net.IP
+	Mullvad    MullvadServers    `json:"mullvad"`
-	Region string
+	Nordvpn    NordvpnServers    `json:"nordvpn"`
+	Pia        PiaServers        `json:"pia"`
+	Purevpn    PurevpnServers    `json:"purevpn"`
+	Surfshark  SurfsharkServers  `json:"surfshark"`
+	Vyprvpn    VyprvpnServers    `json:"vyprvpn"`
+	Windscribe WindscribeServers `json:"windscribe"`
 }
 
-type MullvadServer struct {
+type CyberghostServers struct {
-	IPs     []net.IP
+	Version   uint16             `json:"version"`
-	Country string
+	Timestamp int64              `json:"timestamp"`
-	City    string
+	Servers   []CyberghostServer `json:"servers"`
-	ISP     string
-	Owned   bool
 }
-
+type MullvadServers struct {
-type WindscribeServer struct {
+	Version   uint16          `json:"version"`
-	Region string
+	Timestamp int64           `json:"timestamp"`
-	IPs    []net.IP
+	Servers   []MullvadServer `json:"servers"`
 }
-
+type NordvpnServers struct {
-type SurfsharkServer struct {
+	Version   uint16          `json:"version"`
-	Region string
+	Timestamp int64           `json:"timestamp"`
-	IPs    []net.IP
+	Servers   []NordvpnServer `json:"servers"`
 }
-
+type PiaServers struct {
-type CyberghostServer struct {
+	Version   uint16      `json:"version"`
-	Region string
+	Timestamp int64       `json:"timestamp"`
-	Group  string
+	Servers   []PIAServer `json:"servers"`
-	IPs    []net.IP
 }
-
+type PurevpnServers struct {
-type VyprvpnServer struct {
+	Version   uint16          `json:"version"`
-	Region string
+	Timestamp int64           `json:"timestamp"`
-	IPs    []net.IP
+	Servers   []PurevpnServer `json:"purevpn"`
 }
-
+type SurfsharkServers struct {
-type NordvpnServer struct { //nolint:maligned
+	Version   uint16            `json:"version"`
-	Region string
+	Timestamp int64             `json:"timestamp"`
-	Number uint16
+	Servers   []SurfsharkServer `json:"servers"`
-	IP     net.IP
-	TCP    bool
-	UDP    bool
 }
-
+type VyprvpnServers struct {
-type PurevpnServer struct {
+	Version   uint16          `json:"version"`
-	Region  string
+	Timestamp int64           `json:"timestamp"`
-	Country string
+	Servers   []VyprvpnServer `json:"servers"`
-	City    string
+}
-	IPs     []net.IP
+type WindscribeServers struct {
+	Version   uint16             `json:"version"`
+	Timestamp int64              `json:"timestamp"`
+	Servers   []WindscribeServer `json:"servers"`
 }

internal/openvpn/loop.go

@@ -34,8 +34,9 @@ type looper struct {
 	portForwarded      uint16
 	portForwardedMutex sync.RWMutex
 	// Fixed parameters
-	uid int
+	uid        int
-	gid int
+	gid        int
+	allServers models.AllServers
 	// Configurators
 	conf Configurator
 	fw   firewall.Configurator
@@ -51,7 +52,7 @@ type looper struct {
 }
 
 func NewLooper(provider models.VPNProvider, settings settings.OpenVPN,
-	uid, gid int,
+	uid, gid int, allServers models.AllServers,
 	conf Configurator, fw firewall.Configurator,
 	logger logging.Logger, client network.Client, fileManager files.FileManager,
 	streamMerger command.StreamMerger, fatalOnError func(err error)) Looper {
@@ -60,6 +61,7 @@ func NewLooper(provider models.VPNProvider, settings settings.OpenVPN,
 		settings:           settings,
 		uid:                uid,
 		gid:                gid,
+		allServers:         allServers,
 		conf:               conf,
 		fw:                 fw,
 		logger:             logger.WithPrefix("openvpn: "),
@@ -99,7 +101,7 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) {
 
 	for ctx.Err() == nil {
 		settings := l.GetSettings()
-		providerConf := provider.New(l.provider)
+		providerConf := provider.New(l.provider, l.allServers)
 		connections, err := providerConf.GetOpenVPNConnections(settings.Provider.ServerSelection)
 		if err != nil {
 			l.fatalOnError(err)

internal/provider/cyberghost.go

@@ -9,10 +9,14 @@ import (
 	"github.com/qdm12/golibs/network"
 )
 
-type cyberghost struct{}
+type cyberghost struct {
+	servers []models.CyberghostServer
+}
 
-func newCyberghost() *cyberghost {
+func newCyberghost(servers []models.CyberghostServer) *cyberghost {
-	return &cyberghost{}
+	return &cyberghost{
+		servers: servers,
+	}
 }
 
 func (c *cyberghost) filterServers(region, group string) (servers []models.CyberghostServer) {

internal/provider/mullvad.go

@@ -9,10 +9,14 @@ import (
 	"github.com/qdm12/golibs/network"
 )
 
-type mullvad struct{}
+type mullvad struct {
+	servers []models.MullvadServer
+}
 
-func newMullvad() *mullvad {
+func newMullvad(servers []models.MullvadServer) *mullvad {
-	return &mullvad{}
+	return &mullvad{
+		servers: servers,
+	}
 }
 
 func (m *mullvad) filterServers(country, city, isp string) (servers []models.MullvadServer) {

internal/provider/nordvpn.go

@@ -9,10 +9,14 @@ import (
 	"github.com/qdm12/golibs/network"
 )
 
-type nordvpn struct{}
+type nordvpn struct {
+	servers []models.NordvpnServer
+}
 
-func newNordvpn() *nordvpn {
+func newNordvpn(servers []models.NordvpnServer) *nordvpn {
-	return &nordvpn{}
+	return &nordvpn{
+		servers: servers,
+	}
 }
 
 func (n *nordvpn) filterServers(region string, protocol models.NetworkProtocol, number uint16) (servers []models.NordvpnServer) {

internal/provider/pia.go

@@ -14,12 +14,14 @@ import (
 )
 
 type pia struct {
-	random random.Random
+	random  random.Random
+	servers []models.PIAServer
 }
 
-func newPrivateInternetAccess() *pia {
+func newPrivateInternetAccess(servers []models.PIAServer) *pia {
 	return &pia{
-		random: random.NewRandom(),
+		random:  random.NewRandom(),
+		servers: servers,
 	}
 }
 

internal/provider/provider.go

@@ -13,24 +13,24 @@ type Provider interface {
 	GetPortForward(client network.Client) (port uint16, err error)
 }
 
-func New(provider models.VPNProvider) Provider {
+func New(provider models.VPNProvider, allServers models.AllServers) Provider {
 	switch provider {
 	case constants.PrivateInternetAccess:
-		return newPrivateInternetAccess()
+		return newPrivateInternetAccess(allServers.Pia.Servers)
 	case constants.Mullvad:
-		return newMullvad()
+		return newMullvad(allServers.Mullvad.Servers)
 	case constants.Windscribe:
-		return newWindscribe()
+		return newWindscribe(allServers.Windscribe.Servers)
 	case constants.Surfshark:
-		return newSurfshark()
+		return newSurfshark(allServers.Surfshark.Servers)
 	case constants.Cyberghost:
-		return newCyberghost()
+		return newCyberghost(allServers.Cyberghost.Servers)
 	case constants.Vyprvpn:
-		return newVyprvpn()
+		return newVyprvpn(allServers.Vyprvpn.Servers)
 	case constants.Nordvpn:
-		return newNordvpn()
+		return newNordvpn(allServers.Nordvpn.Servers)
 	case constants.Purevpn:
-		return newPurevpn()
+		return newPurevpn(allServers.Purevpn.Servers)
 	default:
 		return nil // should never occur
 	}

internal/provider/purevpn.go

@@ -9,10 +9,14 @@ import (
 	"github.com/qdm12/golibs/network"
 )
 
-type purevpn struct{}
+type purevpn struct {
+	servers []models.PurevpnServer
+}
 
-func newPurevpn() *purevpn {
+func newPurevpn(servers []models.PurevpnServer) *purevpn {
-	return &purevpn{}
+	return &purevpn{
+		servers: servers,
+	}
 }
 
 func (p *purevpn) filterServers(region, country, city string) (servers []models.PurevpnServer) {

internal/provider/surfshark.go

@@ -9,10 +9,14 @@ import (
 	"github.com/qdm12/golibs/network"
 )
 
-type surfshark struct{}
+type surfshark struct {
+	servers []models.SurfsharkServer
+}
 
-func newSurfshark() *surfshark {
+func newSurfshark(servers []models.SurfsharkServer) *surfshark {
-	return &surfshark{}
+	return &surfshark{
+		servers: servers,
+	}
 }
 
 func (s *surfshark) filterServers(region string) (servers []models.SurfsharkServer) {

internal/provider/vyprvpn.go

@@ -9,10 +9,14 @@ import (
 	"github.com/qdm12/golibs/network"
 )
 
-type vyprvpn struct{}
+type vyprvpn struct {
+	servers []models.VyprvpnServer
+}
 
-func newVyprvpn() *vyprvpn {
+func newVyprvpn(servers []models.VyprvpnServer) *vyprvpn {
-	return &vyprvpn{}
+	return &vyprvpn{
+		servers: servers,
+	}
 }
 
 func (v *vyprvpn) filterServers(region string) (servers []models.VyprvpnServer) {

internal/provider/windscribe.go

@@ -9,10 +9,14 @@ import (
 	"github.com/qdm12/golibs/network"
 )
 
-type windscribe struct{}
+type windscribe struct {
+	servers []models.WindscribeServer
+}
 
-func newWindscribe() *windscribe {
+func newWindscribe(servers []models.WindscribeServer) *windscribe {
-	return &windscribe{}
+	return &windscribe{
+		servers: servers,
+	}
 }
 
 func (w *windscribe) filterServers(region string) (servers []models.WindscribeServer) {

internal/storage/merge.go

@@ -0,0 +1,68 @@
+package storage
+
+import (
+	"time"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+func getUnixTimeDifference(unix1, unix2 int64) (difference time.Duration) {
+	difference = time.Unix(unix1, 0).Sub(time.Unix(unix2, 0))
+	if difference < 0 {
+		difference = -difference
+	}
+	return difference.Truncate(time.Second)
+}
+
+func (s *storage) mergeServers(hardcoded, persistent models.AllServers) (merged models.AllServers) {
+	merged.Version = hardcoded.Version
+	merged.Cyberghost = hardcoded.Cyberghost
+	if persistent.Cyberghost.Timestamp > hardcoded.Cyberghost.Timestamp {
+		s.logger.Info("Using Cyberghost servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Cyberghost.Timestamp, hardcoded.Cyberghost.Timestamp))
+		merged.Cyberghost = persistent.Cyberghost
+	}
+	merged.Mullvad = hardcoded.Mullvad
+	if persistent.Mullvad.Timestamp > hardcoded.Mullvad.Timestamp {
+		s.logger.Info("Using Mullvad servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Mullvad.Timestamp, hardcoded.Mullvad.Timestamp))
+		merged.Mullvad = persistent.Mullvad
+	}
+	merged.Nordvpn = hardcoded.Nordvpn
+	if persistent.Nordvpn.Timestamp > hardcoded.Nordvpn.Timestamp {
+		s.logger.Info("Using Nordvpn servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Nordvpn.Timestamp, hardcoded.Nordvpn.Timestamp))
+		merged.Nordvpn = persistent.Nordvpn
+	}
+	merged.Pia = hardcoded.Pia
+	if persistent.Pia.Timestamp > hardcoded.Pia.Timestamp {
+		s.logger.Info("Using Private Internet Access servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Pia.Timestamp, hardcoded.Pia.Timestamp))
+		merged.Pia = persistent.Pia
+	}
+	merged.Purevpn = hardcoded.Purevpn
+	if persistent.Purevpn.Timestamp > hardcoded.Purevpn.Timestamp {
+		s.logger.Info("Using Purevpn servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Purevpn.Timestamp, hardcoded.Purevpn.Timestamp))
+		merged.Purevpn = persistent.Purevpn
+	}
+	merged.Surfshark = hardcoded.Surfshark
+	if persistent.Surfshark.Timestamp > hardcoded.Surfshark.Timestamp {
+		s.logger.Info("Using Surfshark servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Surfshark.Timestamp, hardcoded.Surfshark.Timestamp))
+		merged.Surfshark = persistent.Surfshark
+	}
+	merged.Vyprvpn = hardcoded.Vyprvpn
+	if persistent.Vyprvpn.Timestamp > hardcoded.Vyprvpn.Timestamp {
+		s.logger.Info("Using Vyprvpn servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Vyprvpn.Timestamp, hardcoded.Vyprvpn.Timestamp))
+		merged.Vyprvpn = persistent.Vyprvpn
+	}
+	merged.Windscribe = hardcoded.Windscribe
+	if persistent.Windscribe.Timestamp > hardcoded.Windscribe.Timestamp {
+		s.logger.Info("Using Windscribe servers from file (%s more recent)",
+			getUnixTimeDifference(persistent.Windscribe.Timestamp, hardcoded.Windscribe.Timestamp))
+		merged.Windscribe = persistent.Windscribe
+	}
+	return merged
+}

internal/storage/storage.go

@@ -0,0 +1,29 @@
+package storage
+
+import (
+	"io/ioutil"
+	"os"
+
+	"github.com/qdm12/gluetun/internal/models"
+	"github.com/qdm12/golibs/logging"
+)
+
+type Storage interface {
+	SyncServers(hardcodedServers models.AllServers) (allServers models.AllServers, err error)
+}
+
+type storage struct {
+	osStat    func(name string) (os.FileInfo, error)
+	readFile  func(filename string) (data []byte, err error)
+	writeFile func(filename string, data []byte, perm os.FileMode) error
+	logger    logging.Logger
+}
+
+func New(logger logging.Logger) Storage {
+	return &storage{
+		osStat:    os.Stat,
+		readFile:  ioutil.ReadFile,
+		writeFile: ioutil.WriteFile,
+		logger:    logger.WithPrefix("storage: "),
+	}
+}

internal/storage/sync.go

@@ -0,0 +1,72 @@
+package storage
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+const (
+	jsonFilepath = "/gluetun/servers.json"
+)
+
+func countServers(allServers models.AllServers) int {
+	return len(allServers.Cyberghost.Servers) +
+		len(allServers.Mullvad.Servers) +
+		len(allServers.Nordvpn.Servers) +
+		len(allServers.Pia.Servers) +
+		len(allServers.Purevpn.Servers) +
+		len(allServers.Surfshark.Servers) +
+		len(allServers.Vyprvpn.Servers) +
+		len(allServers.Windscribe.Servers)
+}
+
+func (s *storage) SyncServers(hardcodedServers models.AllServers) (allServers models.AllServers, err error) {
+	// Eventually read file
+	var serversOnFile models.AllServers
+	_, err = s.osStat(jsonFilepath)
+	if err == nil {
+		serversOnFile, err = s.readFromFile()
+		if err != nil {
+			return allServers, err
+		}
+	} else if !os.IsNotExist(err) {
+		return allServers, err
+	}
+
+	// Merge data from file and hardcoded
+	s.logger.Info("Merging by most recent %d hardcoded servers and %d servers read from %s",
+		countServers(hardcodedServers), countServers(serversOnFile), jsonFilepath)
+	allServers = s.mergeServers(hardcodedServers, serversOnFile)
+
+	// Eventually write file
+	if reflect.DeepEqual(serversOnFile, allServers) {
+		return allServers, nil
+	}
+	return allServers, s.flushToFile(allServers)
+}
+
+func (s *storage) readFromFile() (servers models.AllServers, err error) {
+	bytes, err := s.readFile(jsonFilepath)
+	if err != nil {
+		return servers, err
+	}
+	if err := json.Unmarshal(bytes, &servers); err != nil {
+		return servers, err
+	}
+	return servers, nil
+}
+
+func (s *storage) flushToFile(servers models.AllServers) error {
+	bytes, err := json.MarshalIndent(servers, "", "  ")
+	if err != nil {
+		return fmt.Errorf("cannot write to file: %w", err)
+	}
+	if err := s.writeFile(jsonFilepath, bytes, 0644); err != nil {
+		return err
+	}
+	return nil
+}