IVPN server data update code and ISP filter (#578)

- Use IVPN's HTTP API instead of their .zip file
- Unit tests for API and GetServers
- Paves the way for Wireguard
- Update server information for IVPN
- Add `ISP` filter for IVPN
This commit is contained in:
Quentin McGaw
2021-08-22 20:11:56 -07:00
committed by GitHub
parent b69dcb62e3
commit c348343b22
17 changed files with 711 additions and 614 deletions

View File

@@ -0,0 +1,66 @@
package ivpn
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
)
var (
errBuildRequest = errors.New("cannot build HTTP request")
errDoRequest = errors.New("failed doing HTTP request")
errHTTPStatusCodeNotOK = errors.New("HTTP status code not OK")
errUnmarshalResponseBody = errors.New("failed unmarshaling response body")
errCloseBody = errors.New("failed closing HTTP body")
)
type apiData struct {
Servers []apiServer `json:"servers"`
}
type apiServer struct {
Hostnames apiHostnames `json:"hostnames"`
IsActive bool `json:"is_active"`
Country string `json:"country"`
City string `json:"city"`
ISP string `json:"isp"`
}
type apiHostnames struct {
OpenVPN string `json:"openvpn"`
}
func fetchAPI(ctx context.Context, client *http.Client) (
data apiData, err error) {
const url = "https://api.ivpn.net/v4/servers/stats"
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return data, fmt.Errorf("%w: %s", errBuildRequest, err)
}
response, err := client.Do(request)
if err != nil {
return data, fmt.Errorf("%w: %s", errDoRequest, err)
}
if response.StatusCode != http.StatusOK {
_ = response.Body.Close()
return data, fmt.Errorf("%w: %d %s",
errHTTPStatusCodeNotOK, response.StatusCode, response.Status)
}
decoder := json.NewDecoder(response.Body)
if err := decoder.Decode(&data); err != nil {
_ = response.Body.Close()
return data, fmt.Errorf("%w: %s", errUnmarshalResponseBody, err)
}
if err := response.Body.Close(); err != nil {
return data, fmt.Errorf("%w: %s", errCloseBody, err)
}
return data, nil
}

View File

@@ -0,0 +1,96 @@
package ivpn
import (
"context"
"errors"
"io"
"io/ioutil"
"net/http"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_fetchAPI(t *testing.T) {
t.Parallel()
testCases := map[string]struct {
responseStatus int
responseBody io.ReadCloser
data apiData
err error
}{
"http response status not ok": {
responseStatus: http.StatusNoContent,
err: errors.New("HTTP status code not OK: 204 No Content"),
},
"nil body": {
responseStatus: http.StatusOK,
err: errors.New("failed unmarshaling response body: EOF"),
},
"no server": {
responseStatus: http.StatusOK,
responseBody: ioutil.NopCloser(strings.NewReader(`{}`)),
},
"success": {
responseStatus: http.StatusOK,
responseBody: ioutil.NopCloser(strings.NewReader(`{"servers":[
{"country":"Country1","city":"City A","isp":"xyz","is_active":true,"hostnames":{"openvpn":"hosta"}},
{"country":"Country2","city":"City B","isp":"abc","is_active":false,"hostnames":{"openvpn":"hostb"}}
]}`)),
data: apiData{
Servers: []apiServer{
{
Country: "Country1",
City: "City A",
IsActive: true,
ISP: "xyz",
Hostnames: apiHostnames{
OpenVPN: "hosta",
},
},
{
Country: "Country2",
City: "City B",
ISP: "abc",
Hostnames: apiHostnames{
OpenVPN: "hostb",
},
},
},
},
},
}
for name, testCase := range testCases {
testCase := testCase
t.Run(name, func(t *testing.T) {
t.Parallel()
ctx := context.Background()
client := &http.Client{
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
assert.Equal(t, http.MethodGet, r.Method)
assert.Equal(t, r.URL.String(), "https://api.ivpn.net/v4/servers/stats")
return &http.Response{
StatusCode: testCase.responseStatus,
Status: http.StatusText(testCase.responseStatus),
Body: testCase.responseBody,
}, nil
}),
}
data, err := fetchAPI(ctx, client)
assert.Equal(t, testCase.data, data)
if testCase.err != nil {
require.Error(t, err)
assert.Equal(t, testCase.err.Error(), err.Error())
} else {
assert.NoError(t, err)
}
})
}
}

View File

@@ -1,16 +0,0 @@
package ivpn
import (
"strings"
)
func parseFilename(fileName string) (country, city string) {
const suffix = ".ovpn"
fileName = strings.TrimSuffix(fileName, suffix)
parts := strings.Split(fileName, "-")
country = strings.ReplaceAll(parts[0], "_", " ")
if len(parts) > 1 {
city = strings.ReplaceAll(parts[1], "_", " ")
}
return country, city
}

View File

@@ -1,41 +0,0 @@
package ivpn
import (
"testing"
"github.com/stretchr/testify/assert"
)
func Test_parseFilename(t *testing.T) {
t.Parallel()
testCases := map[string]struct {
fileName string
country string
city string
}{
"empty filename": {},
"country only": {
fileName: "Country.ovpn",
country: "Country",
},
"country and city": {
fileName: "Country-City.ovpn",
country: "Country",
city: "City",
},
"composite country and city": {
fileName: "Coun_try-Ci_ty.ovpn",
country: "Coun try",
city: "Ci ty",
},
}
for name, testCase := range testCases {
testCase := testCase
t.Run(name, func(t *testing.T) {
t.Parallel()
country, city := parseFilename(testCase.fileName)
assert.Equal(t, testCase.country, country)
assert.Equal(t, testCase.city, city)
})
}
}

View File

@@ -1,58 +0,0 @@
package ivpn
import (
"net"
"sort"
"github.com/qdm12/gluetun/internal/models"
)
type hostToServer map[string]models.IvpnServer
func (hts hostToServer) add(host, country, city string, tcp, udp bool) {
server, ok := hts[host]
if !ok {
server.Hostname = host
server.Country = country
server.City = city
}
if tcp {
server.TCP = tcp
}
if udp {
server.UDP = udp
}
hts[host] = server
}
func (hts hostToServer) toHostsSlice() (hosts []string) {
hosts = make([]string, 0, len(hts))
for host := range hts {
hosts = append(hosts, host)
}
sort.Slice(hosts, func(i, j int) bool {
return hosts[i] < hosts[j]
})
return hosts
}
func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) {
for host, IPs := range hostToIPs {
server := hts[host]
server.IPs = IPs
hts[host] = server
}
for host, server := range hts {
if len(server.IPs) == 0 {
delete(hts, host)
}
}
}
func (hts hostToServer) toServersSlice() (servers []models.IvpnServer) {
servers = make([]models.IvpnServer, 0, len(hts))
for _, server := range hts {
servers = append(servers, server)
}
return servers
}

View File

@@ -1,211 +0,0 @@
package ivpn
import (
"net"
"testing"
"github.com/qdm12/gluetun/internal/models"
"github.com/stretchr/testify/assert"
)
func Test_hostToServer_add(t *testing.T) {
t.Parallel()
testCases := map[string]struct {
initialHTS hostToServer
host string
country string
city string
tcp bool
udp bool
expectedHTS hostToServer
}{
"empty host to server": {
initialHTS: hostToServer{},
host: "host",
country: "country",
city: "city",
tcp: true,
udp: true,
expectedHTS: hostToServer{
"host": {
Hostname: "host",
Country: "country",
City: "city",
TCP: true,
UDP: true,
},
},
},
"add server": {
initialHTS: hostToServer{
"existing host": {},
},
host: "host",
country: "country",
city: "city",
tcp: true,
udp: true,
expectedHTS: hostToServer{
"existing host": {},
"host": models.IvpnServer{
Hostname: "host",
Country: "country",
City: "city",
TCP: true,
UDP: true,
},
},
},
"extend existing server": {
initialHTS: hostToServer{
"host": models.IvpnServer{
Hostname: "host",
Country: "country",
City: "city",
TCP: true,
},
},
host: "host",
country: "country",
city: "city",
tcp: false,
udp: true,
expectedHTS: hostToServer{
"host": models.IvpnServer{
Hostname: "host",
Country: "country",
City: "city",
TCP: true,
UDP: true,
},
},
},
}
for name, testCase := range testCases {
testCase := testCase
t.Run(name, func(t *testing.T) {
t.Parallel()
testCase.initialHTS.add(testCase.host, testCase.country, testCase.city, testCase.tcp, testCase.udp)
assert.Equal(t, testCase.expectedHTS, testCase.initialHTS)
})
}
}
func Test_hostToServer_toHostsSlice(t *testing.T) {
t.Parallel()
testCases := map[string]struct {
hts hostToServer
hosts []string
}{
"empty host to server": {
hts: hostToServer{},
hosts: []string{},
},
"single host": {
hts: hostToServer{
"A": {},
},
hosts: []string{"A"},
},
"multiple hosts": {
hts: hostToServer{
"A": {},
"B": {},
},
hosts: []string{"A", "B"},
},
}
for name, testCase := range testCases {
testCase := testCase
t.Run(name, func(t *testing.T) {
t.Parallel()
hosts := testCase.hts.toHostsSlice()
assert.ElementsMatch(t, testCase.hosts, hosts)
})
}
}
func Test_hostToServer_adaptWithIPs(t *testing.T) {
t.Parallel()
testCases := map[string]struct {
initialHTS hostToServer
hostToIPs map[string][]net.IP
expectedHTS hostToServer
}{
"create server": {
initialHTS: hostToServer{},
hostToIPs: map[string][]net.IP{
"A": {{1, 2, 3, 4}},
},
expectedHTS: hostToServer{
"A": models.IvpnServer{
IPs: []net.IP{{1, 2, 3, 4}},
},
},
},
"add IPs to existing server": {
initialHTS: hostToServer{
"A": models.IvpnServer{
Country: "country",
},
},
hostToIPs: map[string][]net.IP{
"A": {{1, 2, 3, 4}},
},
expectedHTS: hostToServer{
"A": models.IvpnServer{
Country: "country",
IPs: []net.IP{{1, 2, 3, 4}},
},
},
},
"remove server without IP": {
initialHTS: hostToServer{
"A": models.IvpnServer{
Country: "country",
},
},
hostToIPs: map[string][]net.IP{},
expectedHTS: hostToServer{},
},
}
for name, testCase := range testCases {
testCase := testCase
t.Run(name, func(t *testing.T) {
t.Parallel()
testCase.initialHTS.adaptWithIPs(testCase.hostToIPs)
assert.Equal(t, testCase.expectedHTS, testCase.initialHTS)
})
}
}
func Test_hostToServer_toServersSlice(t *testing.T) {
t.Parallel()
testCases := map[string]struct {
hts hostToServer
servers []models.IvpnServer
}{
"empty host to server": {
hts: hostToServer{},
servers: []models.IvpnServer{},
},
"multiple servers": {
hts: hostToServer{
"A": {Country: "A"},
"B": {Country: "B"},
},
servers: []models.IvpnServer{
{Country: "A"},
{Country: "B"},
},
},
}
for name, testCase := range testCases {
testCase := testCase
t.Run(name, func(t *testing.T) {
t.Parallel()
servers := testCase.hts.toServersSlice()
assert.ElementsMatch(t, testCase.servers, servers)
})
}
}

View File

@@ -0,0 +1,9 @@
package ivpn
import "net/http"
type roundTripFunc func(r *http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
return f(r)
}

View File

@@ -6,78 +6,64 @@ import (
"context"
"errors"
"fmt"
"strings"
"net/http"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/openvpn"
"github.com/qdm12/gluetun/internal/updater/resolver"
"github.com/qdm12/gluetun/internal/updater/unzip"
)
var ErrNotEnoughServers = errors.New("not enough servers found")
var (
ErrFetchAPI = errors.New("failed fetching API")
ErrNotEnoughServers = errors.New("not enough servers found")
)
func GetServers(ctx context.Context, unzipper unzip.Unzipper,
func GetServers(ctx context.Context, client *http.Client,
presolver resolver.Parallel, minServers int) (
servers []models.IvpnServer, warnings []string, err error) {
const url = "https://www.ivpn.net/releases/config/ivpn-openvpn-config.zip"
contents, err := unzipper.FetchAndExtract(ctx, url)
data, err := fetchAPI(ctx, client)
if err != nil {
return nil, nil, err
} else if len(contents) < minServers {
return nil, nil, fmt.Errorf("%w: %s", ErrFetchAPI, err)
}
hosts := make([]string, 0, len(data.Servers))
for _, serverData := range data.Servers {
host := serverData.Hostnames.OpenVPN
if host == "" {
continue // Wireguard
}
hosts = append(hosts, host)
}
if len(hosts) < minServers {
return nil, nil, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(contents), minServers)
ErrNotEnoughServers, len(hosts), minServers)
}
hts := make(hostToServer)
for fileName, content := range contents {
if !strings.HasSuffix(fileName, ".ovpn") {
continue // not an OpenVPN file
}
tcp, udp, err := openvpn.ExtractProto(content)
if err != nil {
// treat error as warning and go to next file
warning := err.Error() + ": in " + fileName
warnings = append(warnings, warning)
continue
}
host, warning, err := openvpn.ExtractHost(content)
if warning != "" {
warnings = append(warnings, warning)
}
if err != nil {
// treat error as warning and go to next file
warning := err.Error() + " in " + fileName
warnings = append(warnings, warning)
continue
}
country, city := parseFilename(fileName)
hts.add(host, country, city, tcp, udp)
}
if len(hts) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(hts), minServers)
}
hosts := hts.toHostsSlice()
hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers)
warnings = append(warnings, newWarnings...)
hostToIPs, warnings, err := resolveHosts(ctx, presolver, hosts, minServers)
if err != nil {
return nil, warnings, err
}
hts.adaptWithIPs(hostToIPs)
servers = make([]models.IvpnServer, 0, len(hosts))
for _, serverData := range data.Servers {
host := serverData.Hostnames.OpenVPN
if serverData.Hostnames.OpenVPN == "" {
continue // Wireguard
}
servers = hts.toServersSlice()
if len(servers) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(servers), minServers)
server := models.IvpnServer{
Country: serverData.Country,
City: serverData.City,
ISP: serverData.ISP,
Hostname: serverData.Hostnames.OpenVPN,
// TCP is not supported
UDP: true,
IPs: hostToIPs[host],
}
servers = append(servers, server)
}
sortServers(servers)

View File

@@ -3,27 +3,30 @@ package ivpn
import (
"context"
"errors"
"io/ioutil"
"net"
"net/http"
"strings"
"testing"
"github.com/golang/mock/gomock"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/resolver"
"github.com/qdm12/gluetun/internal/updater/resolver/mock_resolver"
"github.com/qdm12/gluetun/internal/updater/unzip/mock_unzip"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_GetServers(t *testing.T) {
t.Parallel()
testCases := map[string]struct {
// Inputs
minServers int
// Unzip
unzipContents map[string][]byte
unzipErr error
// From API
responseBody string
responseStatus int
// Resolution
expectResolve bool
@@ -38,47 +41,15 @@ func Test_GetServers(t *testing.T) {
warnings []string
err error
}{
"unzipper error": {
unzipErr: errors.New("dummy"),
err: errors.New("dummy"),
},
"not enough unzip contents": {
minServers: 1,
unzipContents: map[string][]byte{},
err: errors.New("not enough servers found: 0 and expected at least 1"),
},
"no openvpn file": {
minServers: 1,
unzipContents: map[string][]byte{"somefile.txt": {}},
err: errors.New("not enough servers found: 0 and expected at least 1"),
},
"invalid proto": {
minServers: 1,
unzipContents: map[string][]byte{"badproto.ovpn": []byte(`proto invalid`)},
warnings: []string{"unknown protocol: invalid: in badproto.ovpn"},
err: errors.New("not enough servers found: 0 and expected at least 1"),
},
"no host": {
minServers: 1,
unzipContents: map[string][]byte{"nohost.ovpn": []byte(``)},
warnings: []string{"remote host not found in nohost.ovpn"},
err: errors.New("not enough servers found: 0 and expected at least 1"),
},
"multiple hosts": {
minServers: 1,
unzipContents: map[string][]byte{
"MultiHosts.ovpn": []byte("remote hosta\nremote hostb"),
},
expectResolve: true,
hostsToResolve: []string{"hosta"},
resolveSettings: getResolveSettings(1),
warnings: []string{"only using the first host \"hosta\" and discarding 1 other hosts"},
err: errors.New("not enough servers found: 0 and expected at least 1"),
"http response error": {
responseStatus: http.StatusNoContent,
err: errors.New("failed fetching API: HTTP status code not OK: 204 No Content"),
},
"resolve error": {
unzipContents: map[string][]byte{
"config.ovpn": []byte("remote hosta"),
},
responseBody: `{"servers":[
{"hostnames":{"openvpn":"hosta"}}
]}`,
responseStatus: http.StatusOK,
expectResolve: true,
hostsToResolve: []string{"hosta"},
resolveSettings: getResolveSettings(0),
@@ -87,12 +58,22 @@ func Test_GetServers(t *testing.T) {
warnings: []string{"resolve warning"},
err: errors.New("dummy"),
},
"not enough servers": {
minServers: 2,
responseBody: `{"servers":[
{"hostnames":{"openvpn":"hosta"}}
]}`,
responseStatus: http.StatusOK,
err: errors.New("not enough servers found: 1 and expected at least 2"),
},
"success": {
minServers: 1,
unzipContents: map[string][]byte{
"Country1-City_A.ovpn": []byte("remote hosta"),
"Country2-City_B.ovpn": []byte("remote hostb"),
},
responseBody: `{"servers":[
{"country":"Country1","city":"City A","hostnames":{"openvpn":"hosta"}},
{"country":"Country2","city":"City B","hostnames":{"openvpn":"hostb"}},
{"country":"Country3","city":"City C","hostnames":{"wireguard":"hostc"}}
]}`,
responseStatus: http.StatusOK,
expectResolve: true,
hostsToResolve: []string{"hosta", "hostb"},
resolveSettings: getResolveSettings(1),
@@ -116,10 +97,17 @@ func Test_GetServers(t *testing.T) {
ctx := context.Background()
unzipper := mock_unzip.NewMockUnzipper(ctrl)
const zipURL = "https://www.ivpn.net/releases/config/ivpn-openvpn-config.zip"
unzipper.EXPECT().FetchAndExtract(ctx, zipURL).
Return(testCase.unzipContents, testCase.unzipErr)
client := &http.Client{
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
assert.Equal(t, http.MethodGet, r.Method)
assert.Equal(t, r.URL.String(), "https://api.ivpn.net/v4/servers/stats")
return &http.Response{
StatusCode: testCase.responseStatus,
Status: http.StatusText(testCase.responseStatus),
Body: ioutil.NopCloser(strings.NewReader(testCase.responseBody)),
}, nil
}),
}
presolver := mock_resolver.NewMockParallel(ctrl)
if testCase.expectResolve {
@@ -127,7 +115,7 @@ func Test_GetServers(t *testing.T) {
Return(testCase.hostToIPs, testCase.resolveWarnings, testCase.resolveErr)
}
servers, warnings, err := GetServers(ctx, unzipper, presolver, testCase.minServers)
servers, warnings, err := GetServers(ctx, client, presolver, testCase.minServers)
assert.Equal(t, testCase.servers, servers)
assert.Equal(t, testCase.warnings, warnings)