feat(fastestvpn): update servers data using API instead of zip file

- Add city filter
- More dynamic to servers updates on fastestvpn's end
- Update servers data

internal/provider/fastestvpn/updater/api.go

@@ -0,0 +1,129 @@
+package updater
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/qdm12/gluetun/internal/provider/common"
+)
+
+type apiServer struct {
+	country  string
+	city     string
+	hostname string
+}
+
+var (
+	ErrDataMalformed = errors.New("data is malformed")
+)
+
+const apiURL = "https://support.fastestvpn.com/wp-admin/admin-ajax.php"
+
+// The API URL and requests are shamelessly taken from network operations
+// done on the page https://support.fastestvpn.com/vpn-servers/
+func fetchAPIServers(ctx context.Context, client *http.Client, protocol string) (
+	servers []apiServer, err error) {
+	form := url.Values{
+		"action":   []string{"vpn_servers"},
+		"protocol": []string{protocol},
+	}
+	body := strings.NewReader(form.Encode())
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, body)
+	if err != nil {
+		return nil, fmt.Errorf("creating request: %w", err)
+	}
+	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	// request.Header.Set("User-Agent", "curl/8.9.0")
+	// request.Header.Set("Accept", "*/*")
+
+	response, err := client.Do(request)
+	if err != nil {
+		return nil, fmt.Errorf("sending request: %w", err)
+	}
+
+	if response.StatusCode != http.StatusOK {
+		_ = response.Body.Close()
+		return nil, fmt.Errorf("%w: %d", common.ErrHTTPStatusCodeNotOK, response.StatusCode)
+	}
+
+	data, err := io.ReadAll(response.Body)
+	if err != nil {
+		_ = response.Body.Close()
+		return nil, fmt.Errorf("reading response body: %w", err)
+	}
+
+	err = response.Body.Close()
+	if err != nil {
+		return nil, fmt.Errorf("closing response body: %w", err)
+	}
+
+	const usualMaxNumber = 100
+	servers = make([]apiServer, 0, usualMaxNumber)
+
+	for {
+		trBlock := getNextTRBlock(data)
+		if trBlock == nil {
+			break
+		}
+		data = data[len(trBlock):]
+
+		var server apiServer
+
+		const numberOfTDBlocks = 3
+		for i := 0; i < numberOfTDBlocks; i++ {
+			tdBlock := getNextTDBlock(trBlock)
+			if tdBlock == nil {
+				return nil, fmt.Errorf("%w: expected 3 <td> blocks in <tr> block %q",
+					ErrDataMalformed, string(trBlock))
+			}
+			trBlock = trBlock[len(tdBlock):]
+
+			const startToken, endToken = "<td>", "</td>"
+			tdBlockData := string(tdBlock[len(startToken) : len(tdBlock)-len(endToken)])
+			const countryIndex, cityIndex, hostnameIndex = 0, 1, 2
+			switch i {
+			case countryIndex:
+				server.country = tdBlockData
+			case cityIndex:
+				server.city = tdBlockData
+			case hostnameIndex:
+				server.hostname = tdBlockData
+			}
+		}
+		servers = append(servers, server)
+	}
+
+	return servers, nil
+}
+
+func getNextTRBlock(data []byte) (trBlock []byte) {
+	const startToken, endToken = "<tr>", "</tr>"
+	return getNextBlock(data, startToken, endToken)
+}
+
+func getNextTDBlock(data []byte) (tdBlock []byte) {
+	const startToken, endToken = "<td>", "</td>"
+	return getNextBlock(data, startToken, endToken)
+}
+
+func getNextBlock(data []byte, startToken, endToken string) (nextBlock []byte) {
+	i := bytes.Index(data, []byte(startToken))
+	if i == -1 {
+		return nil
+	}
+
+	nextBlock = data[i:]
+	i = bytes.Index(nextBlock[len(startToken):], []byte(endToken))
+	if i == -1 {
+		return nil
+	}
+	nextBlock = nextBlock[:i+len(startToken)+len(endToken)]
+	return nextBlock
+}

internal/provider/fastestvpn/updater/api_test.go

@@ -0,0 +1,164 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/qdm12/gluetun/internal/provider/common"
+	"github.com/stretchr/testify/assert"
+)
+
+type roundTripFunc func(r *http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
+	return f(r)
+}
+
+func Test_fechAPIServers(t *testing.T) {
+	t.Parallel()
+
+	errTest := errors.New("test error")
+
+	testCases := map[string]struct {
+		ctx            context.Context
+		protocol       string
+		requestBody    string
+		responseStatus int
+		responseBody   io.ReadCloser
+		transportErr   error
+		servers        []apiServer
+		errWrapped     error
+		errMessage     string
+	}{
+		"transport_error": {
+			ctx:            context.Background(),
+			protocol:       "tcp",
+			requestBody:    "action=vpn_servers&protocol=tcp",
+			responseStatus: http.StatusOK,
+			transportErr:   errTest,
+			errWrapped:     errTest,
+			errMessage: `sending request: Post ` +
+				`"https://support.fastestvpn.com/wp-admin/admin-ajax.php": ` +
+				`test error`,
+		},
+		"not_found_status_code": {
+			ctx:            context.Background(),
+			protocol:       "tcp",
+			requestBody:    "action=vpn_servers&protocol=tcp",
+			responseStatus: http.StatusNotFound,
+			errWrapped:     common.ErrHTTPStatusCodeNotOK,
+			errMessage:     "HTTP status code not OK: 404",
+		},
+		"empty_data": {
+			ctx:            context.Background(),
+			protocol:       "tcp",
+			requestBody:    "action=vpn_servers&protocol=tcp",
+			responseStatus: http.StatusOK,
+			responseBody:   io.NopCloser(strings.NewReader("")),
+			servers:        []apiServer{},
+		},
+		"single_server": {
+			ctx:            context.Background(),
+			protocol:       "tcp",
+			requestBody:    "action=vpn_servers&protocol=tcp",
+			responseStatus: http.StatusOK,
+			responseBody: io.NopCloser(strings.NewReader(
+				"irrelevant<tr><td>Australia</td><td>Sydney</td>" +
+					"<td>au-stream.jumptoserver.com</td></tr>irrelevant")),
+			servers: []apiServer{
+				{country: "Australia", city: "Sydney", hostname: "au-stream.jumptoserver.com"},
+			},
+		},
+		"two_servers": {
+			ctx:            context.Background(),
+			protocol:       "tcp",
+			requestBody:    "action=vpn_servers&protocol=tcp",
+			responseStatus: http.StatusOK,
+			responseBody: io.NopCloser(strings.NewReader(
+				"<tr><td>Australia</td><td>Sydney</td><td>au-stream.jumptoserver.com</td></tr>" +
+					"<tr><td>Australia</td><td>Sydney</td><td>au-01.jumptoserver.com</td></tr>")),
+			servers: []apiServer{
+				{country: "Australia", city: "Sydney", hostname: "au-stream.jumptoserver.com"},
+				{country: "Australia", city: "Sydney", hostname: "au-01.jumptoserver.com"},
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			client := &http.Client{
+				Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+					assert.Equal(t, apiURL, r.URL.String())
+					requestBody, err := io.ReadAll(r.Body)
+					assert.NoError(t, err)
+					assert.Equal(t, testCase.requestBody, string(requestBody))
+					if testCase.transportErr != nil {
+						return nil, testCase.transportErr
+					}
+					return &http.Response{
+						StatusCode: testCase.responseStatus,
+						Body:       testCase.responseBody,
+					}, nil
+				}),
+			}
+
+			entries, err := fetchAPIServers(testCase.ctx, client, testCase.protocol)
+
+			assert.ErrorIs(t, err, testCase.errWrapped)
+			if testCase.errWrapped != nil {
+				assert.EqualError(t, err, testCase.errMessage)
+			}
+			assert.Equal(t, testCase.servers, entries)
+		})
+	}
+}
+
+func Test_getNextBlock(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		data       string
+		startToken string
+		endToken   string
+		nextBlock  []byte
+	}{
+		"empty_data": {
+			startToken: "<a>",
+			endToken:   "</a>",
+		},
+		"start_token_not_found": {
+			data:       "test</a>",
+			startToken: "<a>",
+			endToken:   "</a>",
+		},
+		"end_token_not_found": {
+			data:       "<a>test",
+			startToken: "<a>",
+			endToken:   "</a>",
+		},
+		"block_found": {
+			data:       "xy<a>test</a><a>test2</a>zx",
+			startToken: "<a>",
+			endToken:   "</a>",
+			nextBlock:  []byte("<a>test</a>"),
+		},
+	}
+
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			nextBlock := getNextBlock([]byte(testCase.data), testCase.startToken, testCase.endToken)
+
+			assert.Equal(t, testCase.nextBlock, nextBlock)
+		})
+	}
+}

internal/provider/fastestvpn/updater/filename.go

@@ -1,39 +0,0 @@
-package updater
-
-import (
-	"errors"
-	"fmt"
-	"regexp"
-	"strings"
-)
-
-var errFilenameNoProtocolSuffix = errors.New("filename does not have a protocol suffix")
-
-var trailNumberExp = regexp.MustCompile(`[0-9]+$`)
-
-func parseFilename(fileName string) (
-	country string, tcp, udp bool, err error,
-) {
-	const (
-		tcpSuffix = "-tcp.ovpn"
-		udpSuffix = "-udp.ovpn"
-	)
-	var suffix string
-	switch {
-	case strings.HasSuffix(strings.ToLower(fileName), tcpSuffix):
-		suffix = tcpSuffix
-		tcp = true
-	case strings.HasSuffix(strings.ToLower(fileName), udpSuffix):
-		suffix = udpSuffix
-		udp = true
-	default:
-		return "", false, false, fmt.Errorf("%w: %s",
-			errFilenameNoProtocolSuffix, fileName)
-	}
-
-	countryWithNumber := strings.TrimSuffix(fileName, suffix)
-	number := trailNumberExp.FindString(countryWithNumber)
-	country = countryWithNumber[:len(countryWithNumber)-len(number)]
-
-	return country, tcp, udp, nil
-}

internal/provider/fastestvpn/updater/hosttoserver.go

@@ -9,12 +9,19 @@ import (
 
 type hostToServer map[string]models.Server
 
-func (hts hostToServer) add(host, country string, tcp, udp bool) {
+func (hts hostToServer) add(host, country, city string, tcp, udp bool) {
 	server, ok := hts[host]
 	if !ok {
 		server.VPN = vpn.OpenVPN
 		server.Hostname = host
 		server.Country = country
+		server.City = city
+	}
+	if city != "" {
+		// some servers are listed without the city although
+		// they are also listed with the city described, so update
+		// the city field.
+		server.City = city
 	}
 	if tcp {
 		server.TCP = true

internal/provider/fastestvpn/updater/servers.go

@@ -4,48 +4,26 @@ import (
 	"context"
 	"fmt"
 	"sort"
-	"strings"
 
 	"github.com/qdm12/gluetun/internal/models"
 	"github.com/qdm12/gluetun/internal/provider/common"
-	"github.com/qdm12/gluetun/internal/updater/openvpn"
 )
 
 func (u *Updater) FetchServers(ctx context.Context, minServers int) (
 	servers []models.Server, err error) {
-	const url = "https://support.fastestvpn.com/download/fastestvpn_ovpn"
-	contents, err := u.unzipper.FetchAndExtract(ctx, url)
-	if err != nil {
-		return nil, err
-	} else if len(contents) < minServers {
-		return nil, fmt.Errorf("%w: %d and expected at least %d",
-			common.ErrNotEnoughServers, len(contents), minServers)
-	}
-
+	protocols := []string{"tcp", "udp"}
 	hts := make(hostToServer)
 
-	for fileName, content := range contents {
-		if !strings.HasSuffix(fileName, ".ovpn") {
-			continue // not an OpenVPN file
-		}
-
-		country, tcp, udp, err := parseFilename(fileName)
+	for _, protocol := range protocols {
+		apiServers, err := fetchAPIServers(ctx, u.client, protocol)
 		if err != nil {
-			u.warner.Warn(err.Error())
-			continue
+			return nil, fmt.Errorf("fetching %s servers from API: %w", protocol, err)
 		}
-
-		host, warning, err := openvpn.ExtractHost(content)
-		if warning != "" {
-			u.warner.Warn(warning)
+		for _, apiServer := range apiServers {
+			tcp := protocol == "tcp"
+			udp := protocol == "udp"
+			hts.add(apiServer.hostname, apiServer.country, apiServer.city, tcp, udp)
 		}
-		if err != nil {
-			// treat error as warning and go to next file
-			u.warner.Warn(err.Error() + " in " + fileName)
-			continue
-		}
-
-		hts.add(host, country, tcp, udp)
 	}
 
 	if len(hts) < minServers {

internal/provider/fastestvpn/updater/updater.go

@@ -1,19 +1,21 @@
 package updater
 
 import (
+	"net/http"
+
 	"github.com/qdm12/gluetun/internal/provider/common"
 )
 
 type Updater struct {
-	unzipper         common.Unzipper
+	client           *http.Client
 	parallelResolver common.ParallelResolver
 	warner           common.Warner
 }
 
-func New(unzipper common.Unzipper, warner common.Warner,
+func New(client *http.Client, warner common.Warner,
 	parallelResolver common.ParallelResolver) *Updater {
 	return &Updater{
-		unzipper:         unzipper,
+		client:           client,
 		parallelResolver: parallelResolver,
 		warner:           warner,
 	}