chore(all): move sub-packages to internal/provider

internal/provider/surfshark/updater/api.go

@@ -0,0 +1,76 @@
+package surfshark
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/qdm12/gluetun/internal/provider/surfshark/servers"
+)
+
+// Note: no multi-hop and some servers are missing from their API.
+func addServersFromAPI(ctx context.Context, client *http.Client,
+	hts hostToServer) (err error) {
+	data, err := fetchAPI(ctx, client)
+	if err != nil {
+		return err
+	}
+
+	locationData := servers.LocationData()
+	hostToLocation := hostToLocation(locationData)
+
+	const tcp, udp = true, true
+	for _, serverData := range data {
+		locationData := hostToLocation[serverData.Host] // TODO remove in v4
+		retroLoc := locationData.RetroLoc               // empty string if the host has no retro-compatible region
+		hts.add(serverData.Host, serverData.Region, serverData.Country,
+			serverData.Location, retroLoc, tcp, udp)
+	}
+
+	return nil
+}
+
+var (
+	ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK")
+)
+
+type serverData struct {
+	Host     string `json:"connectionName"`
+	Region   string `json:"region"`
+	Country  string `json:"country"`
+	Location string `json:"location"`
+}
+
+func fetchAPI(ctx context.Context, client *http.Client) (
+	servers []serverData, err error) {
+	const url = "https://my.surfshark.com/vpn/api/v4/server/clusters"
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	response, err := client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%w: %d %s", ErrHTTPStatusCodeNotOK,
+			response.StatusCode, response.Status)
+	}
+
+	decoder := json.NewDecoder(response.Body)
+	if err := decoder.Decode(&servers); err != nil {
+		return nil, fmt.Errorf("failed unmarshaling response body: %w", err)
+	}
+
+	if err := response.Body.Close(); err != nil {
+		return nil, err
+	}
+
+	return servers, nil
+}

internal/provider/surfshark/updater/api_test.go

@@ -0,0 +1,169 @@
+package surfshark
+
+import (
+	"context"
+	"errors"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/qdm12/gluetun/internal/constants/vpn"
+	"github.com/qdm12/gluetun/internal/models"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_addServersFromAPI(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		hts            hostToServer
+		responseStatus int
+		responseBody   io.ReadCloser
+		expected       hostToServer
+		err            error
+	}{
+		"fetch API error": {
+			responseStatus: http.StatusNoContent,
+			err:            errors.New("HTTP status code not OK: 204 No Content"),
+		},
+		"success": {
+			hts: hostToServer{
+				"existinghost": {Hostname: "existinghost"},
+			},
+			responseStatus: http.StatusOK,
+			responseBody: ioutil.NopCloser(strings.NewReader(`[
+				{"connectionName":"host1","region":"region1","country":"country1","location":"location1"},
+				{"connectionName":"host2","region":"region2","country":"country1","location":"location2"}
+			]`)),
+			expected: map[string]models.Server{
+				"existinghost": {Hostname: "existinghost"},
+				"host1": {
+					VPN:      vpn.OpenVPN,
+					Region:   "region1",
+					Country:  "country1",
+					City:     "location1",
+					Hostname: "host1",
+					TCP:      true,
+					UDP:      true,
+				},
+				"host2": {
+					VPN:      vpn.OpenVPN,
+					Region:   "region2",
+					Country:  "country1",
+					City:     "location2",
+					Hostname: "host2",
+					TCP:      true,
+					UDP:      true,
+				}},
+		},
+	}
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := context.Background()
+
+			client := &http.Client{
+				Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+					assert.Equal(t, http.MethodGet, r.Method)
+					assert.Equal(t, r.URL.String(), "https://my.surfshark.com/vpn/api/v4/server/clusters")
+					return &http.Response{
+						StatusCode: testCase.responseStatus,
+						Status:     http.StatusText(testCase.responseStatus),
+						Body:       testCase.responseBody,
+					}, nil
+				}),
+			}
+
+			err := addServersFromAPI(ctx, client, testCase.hts)
+
+			assert.Equal(t, testCase.expected, testCase.hts)
+			if testCase.err != nil {
+				require.Error(t, err)
+				assert.Equal(t, testCase.err.Error(), err.Error())
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func Test_fetchAPI(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		responseStatus int
+		responseBody   io.ReadCloser
+		data           []serverData
+		err            error
+	}{
+		"http response status not ok": {
+			responseStatus: http.StatusNoContent,
+			err:            errors.New("HTTP status code not OK: 204 No Content"),
+		},
+		"nil body": {
+			responseStatus: http.StatusOK,
+			err:            errors.New("failed unmarshaling response body: EOF"),
+		},
+		"no server": {
+			responseStatus: http.StatusOK,
+			responseBody:   ioutil.NopCloser(strings.NewReader(`[]`)),
+			data:           []serverData{},
+		},
+		"success": {
+			responseStatus: http.StatusOK,
+			responseBody: ioutil.NopCloser(strings.NewReader(`[
+				{"connectionName":"host1","region":"region1","country":"country1","location":"location1"},
+				{"connectionName":"host2","region":"region2","country":"country1","location":"location2"}
+			]`)),
+			data: []serverData{
+				{
+					Region:   "region1",
+					Country:  "country1",
+					Location: "location1",
+					Host:     "host1",
+				},
+				{
+					Region:   "region2",
+					Country:  "country1",
+					Location: "location2",
+					Host:     "host2",
+				},
+			},
+		},
+	}
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := context.Background()
+
+			client := &http.Client{
+				Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+					assert.Equal(t, http.MethodGet, r.Method)
+					assert.Equal(t, r.URL.String(), "https://my.surfshark.com/vpn/api/v4/server/clusters")
+					return &http.Response{
+						StatusCode: testCase.responseStatus,
+						Status:     http.StatusText(testCase.responseStatus),
+						Body:       testCase.responseBody,
+					}, nil
+				}),
+			}
+
+			data, err := fetchAPI(ctx, client)
+
+			assert.Equal(t, testCase.data, data)
+			if testCase.err != nil {
+				require.Error(t, err)
+				assert.Equal(t, testCase.err.Error(), err.Error())
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

internal/provider/surfshark/updater/hosttoserver.go

@@ -0,0 +1,58 @@
+package surfshark
+
+import (
+	"net"
+
+	"github.com/qdm12/gluetun/internal/constants/vpn"
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+type hostToServer map[string]models.Server
+
+func (hts hostToServer) add(host, region, country, city, retroLoc string, tcp, udp bool) {
+	server, ok := hts[host]
+	if !ok {
+		server.VPN = vpn.OpenVPN
+		server.Hostname = host
+		server.Region = region
+		server.Country = country
+		server.City = city
+		server.RetroLoc = retroLoc
+	}
+	if tcp {
+		server.TCP = tcp
+	}
+	if udp {
+		server.UDP = udp
+	}
+	hts[host] = server
+}
+
+func (hts hostToServer) toHostsSlice() (hosts []string) {
+	hosts = make([]string, 0, len(hts))
+	for host := range hts {
+		hosts = append(hosts, host)
+	}
+	return hosts
+}
+
+func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) {
+	for host, IPs := range hostToIPs {
+		server := hts[host]
+		server.IPs = IPs
+		hts[host] = server
+	}
+	for host, server := range hts {
+		if len(server.IPs) == 0 {
+			delete(hts, host)
+		}
+	}
+}
+
+func (hts hostToServer) toServersSlice() (servers []models.Server) {
+	servers = make([]models.Server, 0, len(hts))
+	for _, server := range hts {
+		servers = append(servers, server)
+	}
+	return servers
+}

internal/provider/surfshark/updater/location.go

@@ -0,0 +1,31 @@
+package surfshark
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/qdm12/gluetun/internal/provider/surfshark/servers"
+)
+
+var (
+	errHostnameNotFound = errors.New("hostname not found in hostname to location mapping")
+)
+
+func getHostInformation(host string, hostnameToLocation map[string]servers.ServerLocation) (
+	data servers.ServerLocation, err error) {
+	locationData, ok := hostnameToLocation[host]
+	if !ok {
+		return locationData, fmt.Errorf("%w: %s", errHostnameNotFound, host)
+	}
+
+	return locationData, nil
+}
+
+func hostToLocation(locationData []servers.ServerLocation) (
+	hostToLocation map[string]servers.ServerLocation) {
+	hostToLocation = make(map[string]servers.ServerLocation, len(locationData))
+	for _, data := range locationData {
+		hostToLocation[data.Hostname] = data
+	}
+	return hostToLocation
+}

internal/provider/surfshark/updater/remaining.go

@@ -0,0 +1,21 @@
+package surfshark
+
+import (
+	"github.com/qdm12/gluetun/internal/provider/surfshark/servers"
+)
+
+// getRemainingServers finds extra servers not found in the API or in the ZIP file.
+func getRemainingServers(hts hostToServer) {
+	locationData := servers.LocationData()
+	hostnameToLocationLeft := hostToLocation(locationData)
+	for _, hostnameDone := range hts.toHostsSlice() {
+		delete(hostnameToLocationLeft, hostnameDone)
+	}
+
+	for hostname, locationData := range hostnameToLocationLeft {
+		// we assume the server supports TCP and UDP
+		const tcp, udp = true, true
+		hts.add(hostname, locationData.Region, locationData.Country,
+			locationData.City, locationData.RetroLoc, tcp, udp)
+	}
+}

internal/provider/surfshark/updater/resolve.go

@@ -0,0 +1,33 @@
+package surfshark
+
+import (
+	"context"
+	"net"
+	"time"
+
+	"github.com/qdm12/gluetun/internal/updater/resolver"
+)
+
+func resolveHosts(ctx context.Context, presolver resolver.Parallel,
+	hosts []string, minServers int) (hostToIPs map[string][]net.IP,
+	warnings []string, err error) {
+	const (
+		maxFailRatio    = 0.1
+		maxDuration     = 20 * time.Second
+		betweenDuration = time.Second
+		maxNoNew        = 2
+		maxFails        = 2
+	)
+	settings := resolver.ParallelSettings{
+		MaxFailRatio: maxFailRatio,
+		MinFound:     minServers,
+		Repeat: resolver.RepeatSettings{
+			MaxDuration:     maxDuration,
+			BetweenDuration: betweenDuration,
+			MaxNoNew:        maxNoNew,
+			MaxFails:        maxFails,
+			SortIPs:         true,
+		},
+	}
+	return presolver.Resolve(ctx, hosts, settings)
+}

internal/provider/surfshark/updater/resolve_test.go

@@ -0,0 +1,57 @@
+package surfshark
+
+import (
+	"context"
+	"errors"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/golang/mock/gomock"
+	"github.com/qdm12/gluetun/internal/updater/resolver"
+	"github.com/qdm12/gluetun/internal/updater/resolver/mock_resolver"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_resolveHosts(t *testing.T) {
+	t.Parallel()
+	ctrl := gomock.NewController(t)
+
+	ctx := context.Background()
+	presolver := mock_resolver.NewMockParallel(ctrl)
+	hosts := []string{"host1", "host2"}
+	const minServers = 10
+
+	expectedHostToIPs := map[string][]net.IP{
+		"host1": {{1, 2, 3, 4}},
+		"host2": {{2, 3, 4, 5}},
+	}
+	expectedWarnings := []string{"warning1", "warning2"}
+	expectedErr := errors.New("dummy")
+
+	const (
+		maxFailRatio    = 0.1
+		maxDuration     = 20 * time.Second
+		betweenDuration = time.Second
+		maxNoNew        = 2
+		maxFails        = 2
+	)
+	expectedSettings := resolver.ParallelSettings{
+		MaxFailRatio: maxFailRatio,
+		MinFound:     minServers,
+		Repeat: resolver.RepeatSettings{
+			MaxDuration:     maxDuration,
+			BetweenDuration: betweenDuration,
+			MaxNoNew:        maxNoNew,
+			MaxFails:        maxFails,
+			SortIPs:         true,
+		},
+	}
+	presolver.EXPECT().Resolve(ctx, hosts, expectedSettings).
+		Return(expectedHostToIPs, expectedWarnings, expectedErr)
+
+	hostToIPs, warnings, err := resolveHosts(ctx, presolver, hosts, minServers)
+	assert.Equal(t, expectedHostToIPs, hostToIPs)
+	assert.Equal(t, expectedWarnings, warnings)
+	assert.Equal(t, expectedErr, err)
+}

internal/provider/surfshark/updater/roundtrip_test.go

@@ -0,0 +1,9 @@
+package surfshark
+
+import "net/http"
+
+type roundTripFunc func(r *http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
+	return f(r)
+}

internal/provider/surfshark/updater/servers.go

@@ -0,0 +1,56 @@
+// Package surfshark contains code to obtain the server information
+// for the Surshark provider.
+package surfshark
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/qdm12/gluetun/internal/models"
+	"github.com/qdm12/gluetun/internal/updater/resolver"
+	"github.com/qdm12/gluetun/internal/updater/unzip"
+)
+
+var (
+	ErrNotEnoughServers = errors.New("not enough servers found")
+)
+
+func GetServers(ctx context.Context, unzipper unzip.Unzipper,
+	client *http.Client, presolver resolver.Parallel, minServers int) (
+	servers []models.Server, warnings []string, err error) {
+	hts := make(hostToServer)
+
+	err = addServersFromAPI(ctx, client, hts)
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot fetch server information from API: %w", err)
+	}
+
+	warnings, err = addOpenVPNServersFromZip(ctx, unzipper, hts)
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot get OpenVPN ZIP file: %w", err)
+	}
+
+	getRemainingServers(hts)
+
+	hosts := hts.toHostsSlice()
+	hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers)
+	warnings = append(warnings, newWarnings...)
+	if err != nil {
+		return nil, warnings, err
+	}
+
+	hts.adaptWithIPs(hostToIPs)
+
+	servers = hts.toServersSlice()
+
+	if len(servers) < minServers {
+		return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
+			ErrNotEnoughServers, len(servers), minServers)
+	}
+
+	sortServers(servers)
+
+	return servers, warnings, nil
+}

internal/provider/surfshark/updater/sort.go

@@ -0,0 +1,22 @@
+package surfshark
+
+import (
+	"sort"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+func sortServers(servers []models.Server) {
+	sort.Slice(servers, func(i, j int) bool {
+		if servers[i].Region == servers[j].Region {
+			if servers[i].Country == servers[j].Country {
+				if servers[i].City == servers[j].City {
+					return servers[i].Hostname < servers[j].Hostname
+				}
+				return servers[i].City < servers[j].City
+			}
+			return servers[i].Country < servers[j].Country
+		}
+		return servers[i].Region < servers[j].Region
+	})
+}

internal/provider/surfshark/updater/sort_test.go

@@ -0,0 +1,42 @@
+package surfshark
+
+import (
+	"testing"
+
+	"github.com/qdm12/gluetun/internal/models"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_sortServers(t *testing.T) {
+	t.Parallel()
+	testCases := map[string]struct {
+		initialServers []models.Server
+		sortedServers  []models.Server
+	}{
+		"no server": {},
+		"sorted servers": {
+			initialServers: []models.Server{
+				{Region: "Z", Country: "B", City: "A", Hostname: "A"},
+				{Country: "B", City: "A", Hostname: "A"},
+				{Country: "A", City: "A", Hostname: "B"},
+				{Country: "A", City: "A", Hostname: "A"},
+				{Country: "A", City: "B", Hostname: "A"},
+			},
+			sortedServers: []models.Server{
+				{Country: "A", City: "A", Hostname: "A"},
+				{Country: "A", City: "A", Hostname: "B"},
+				{Country: "A", City: "B", Hostname: "A"},
+				{Country: "B", City: "A", Hostname: "A"},
+				{Region: "Z", Country: "B", City: "A", Hostname: "A"},
+			},
+		},
+	}
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			sortServers(testCase.initialServers)
+			assert.Equal(t, testCase.sortedServers, testCase.initialServers)
+		})
+	}
+}

internal/provider/surfshark/updater/zip.go

@@ -0,0 +1,72 @@
+package surfshark
+
+import (
+	"context"
+	"strings"
+
+	"github.com/qdm12/gluetun/internal/provider/surfshark/servers"
+	"github.com/qdm12/gluetun/internal/updater/openvpn"
+	"github.com/qdm12/gluetun/internal/updater/unzip"
+)
+
+func addOpenVPNServersFromZip(ctx context.Context,
+	unzipper unzip.Unzipper, hts hostToServer) (
+	warnings []string, err error) {
+	const url = "https://my.surfshark.com/vpn/api/v1/server/configurations"
+	contents, err := unzipper.FetchAndExtract(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+
+	hostnamesDone := hts.toHostsSlice()
+	hostnamesDoneSet := make(map[string]struct{}, len(hostnamesDone))
+	for _, hostname := range hostnamesDone {
+		hostnamesDoneSet[hostname] = struct{}{}
+	}
+
+	locationData := servers.LocationData()
+	hostToLocation := hostToLocation(locationData)
+
+	for fileName, content := range contents {
+		if !strings.HasSuffix(fileName, ".ovpn") {
+			continue // not an OpenVPN file
+		}
+
+		host, warning, err := openvpn.ExtractHost(content)
+		if warning != "" {
+			warnings = append(warnings, warning)
+		}
+		if err != nil {
+			// treat error as warning and go to next file
+			warning := err.Error() + " in " + fileName
+			warnings = append(warnings, warning)
+			continue
+		}
+
+		_, ok := hostnamesDoneSet[host]
+		if ok {
+			continue // already done in API
+		}
+
+		tcp, udp, err := openvpn.ExtractProto(content)
+		if err != nil {
+			// treat error as warning and go to next file
+			warning := err.Error() + " in " + fileName
+			warnings = append(warnings, warning)
+			continue
+		}
+
+		data, err := getHostInformation(host, hostToLocation)
+		if err != nil {
+			// treat error as warning and go to next file
+			warning := err.Error()
+			warnings = append(warnings, warning)
+			continue
+		}
+
+		hts.add(host, data.Region, data.Country, data.City,
+			data.RetroLoc, tcp, udp)
+	}
+
+	return warnings, nil
+}