Obtain PIA v4 server information from API (#257)

- Obtain CN for port forwarding https verification
- Obtain for each server if they support port forwarding
- Obtain for each server their IP address for openvpn UDP and openvpn TCP (one for each)
- Updater program updated to use API
- Hardcoded values updated for PIA v3 and v4 servers
- Clearer separation between pia v3 and v4
- Fixes #250

internal/updater/openvpn.go

@@ -1,7 +1,6 @@
 package updater
 
 import (
-	"net"
 	"strings"
 )
 
@@ -15,18 +14,6 @@ func extractRemoteLinesFromOpenvpn(content []byte) (remoteLines []string) {
 	return remoteLines
 }
 
-func extractIPsFromRemoteLines(remoteLines []string) (ips []net.IP) {
-	for _, remoteLine := range remoteLines {
-		fields := strings.Fields(remoteLine)
-		ip := net.ParseIP(fields[1])
-		if ip == nil { // not an IP address
-			continue
-		}
-		ips = append(ips, ip)
-	}
-	return ips
-}
-
 func extractHostnamesFromRemoteLines(remoteLines []string) (hostnames []string) {
 	for _, remoteLine := range remoteLines {
 		fields := strings.Fields(remoteLine)

internal/updater/piav3.go

@@ -11,40 +11,6 @@ import (
 	"github.com/qdm12/gluetun/internal/models"
 )
 
-func (u *updater) updatePIA() (err error) {
-	const zipURL = "https://www.privateinternetaccess.com/openvpn/openvpn-ip-nextgen.zip"
-	contents, err := fetchAndExtractFiles(zipURL)
-	if err != nil {
-		return err
-	}
-	servers := make([]models.PIAServer, 0, len(contents))
-	for fileName, content := range contents {
-		remoteLines := extractRemoteLinesFromOpenvpn(content)
-		if len(remoteLines) == 0 {
-			return fmt.Errorf("cannot find any remote lines in %s", fileName)
-		}
-		IPs := extractIPsFromRemoteLines(remoteLines)
-		if len(IPs) == 0 {
-			return fmt.Errorf("cannot find any IP addresses in %s", fileName)
-		}
-		region := strings.TrimSuffix(fileName, ".ovpn")
-		server := models.PIAServer{
-			Region: region,
-			IPs:    uniqueSortedIPs(IPs),
-		}
-		servers = append(servers, server)
-	}
-	sort.Slice(servers, func(i, j int) bool {
-		return servers[i].Region < servers[j].Region
-	})
-	if u.options.Stdout {
-		u.println(stringifyPIAServers(servers))
-	}
-	u.servers.Pia.Timestamp = u.timeNow().Unix()
-	u.servers.Pia.Servers = servers
-	return nil
-}
-
 func (u *updater) updatePIAOld(ctx context.Context) (err error) {
 	const zipURL = "https://www.privateinternetaccess.com/openvpn/openvpn.zip"
 	contents, err := fetchAndExtractFiles(zipURL)
@@ -54,8 +20,8 @@ func (u *updater) updatePIAOld(ctx context.Context) (err error) {
 	const maxGoroutines = 10
 	guard := make(chan struct{}, maxGoroutines)
 	errors := make(chan error)
-	serversCh := make(chan models.PIAServer)
-	servers := make([]models.PIAServer, 0, len(contents))
+	serversCh := make(chan models.PIAOldServer)
+	servers := make([]models.PIAOldServer, 0, len(contents))
 	ctx, cancel := context.WithCancel(ctx)
 	wg := &sync.WaitGroup{}
 	defer func() {
@@ -76,7 +42,7 @@ func (u *updater) updatePIAOld(ctx context.Context) (err error) {
 		}
 		region := strings.TrimSuffix(fileName, ".ovpn")
 		wg.Add(1)
-		go resolvePIAHostname(ctx, wg, region, hosts, u.lookupIP, errors, serversCh, guard)
+		go resolvePIAv3Hostname(ctx, wg, region, hosts, u.lookupIP, errors, serversCh, guard)
 	}
 	for range contents {
 		select {
@@ -97,9 +63,9 @@ func (u *updater) updatePIAOld(ctx context.Context) (err error) {
 	return nil
 }
 
-func resolvePIAHostname(ctx context.Context, wg *sync.WaitGroup,
+func resolvePIAv3Hostname(ctx context.Context, wg *sync.WaitGroup,
 	region string, hosts []string, lookupIP lookupIPFunc,
-	errors chan<- error, serversCh chan<- models.PIAServer, guard chan struct{}) {
+	errors chan<- error, serversCh chan<- models.PIAOldServer, guard chan struct{}) {
 	guard <- struct{}{}
 	defer func() {
 		<-guard
@@ -117,26 +83,15 @@ func resolvePIAHostname(ctx context.Context, wg *sync.WaitGroup,
 		}
 		IPs = append(IPs, newIPs...)
 	}
-	serversCh <- models.PIAServer{
+	serversCh <- models.PIAOldServer{
 		Region: region,
 		IPs:    uniqueSortedIPs(IPs),
 	}
 }
 
-func stringifyPIAServers(servers []models.PIAServer) (s string) {
-	s = "func PIAServers() []models.PIAServer {\n"
-	s += "	return []models.PIAServer{\n"
-	for _, server := range servers {
-		s += "		" + server.String() + ",\n"
-	}
-	s += "	}\n"
-	s += "}"
-	return s
-}
-
-func stringifyPIAOldServers(servers []models.PIAServer) (s string) {
-	s = "func PIAOldServers() []models.PIAServer {\n"
-	s += "	return []models.PIAServer{\n"
+func stringifyPIAOldServers(servers []models.PIAOldServer) (s string) {
+	s = "func PIAOldServers() []models.PIAOldServer {\n"
+	s += "	return []models.PIAOldServer{\n"
 	for _, server := range servers {
 		s += "		" + server.String() + ",\n"
 	}

internal/updater/piav4.go

@@ -0,0 +1,100 @@
+package updater
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"sort"
+	"strings"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+func (u *updater) updatePIA() (err error) {
+	const url = "https://serverlist.piaservers.net/vpninfo/servers/v4"
+	response, err := u.httpGet(url)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	b, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return err
+	} else if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("%s: %s", response.Status, strings.ReplaceAll(string(b), "\n", ""))
+	}
+
+	// remove key/signature at the bottom
+	i := bytes.IndexRune(b, '\n')
+	b = b[:i]
+
+	var data struct {
+		Regions []struct {
+			Name        string `json:"name"`
+			PortForward bool   `json:"port_forward"`
+			Servers     struct {
+				UDP []struct {
+					IP net.IP `json:"ip"`
+					CN string `json:"cn"`
+				} `json:"ovpnudp"`
+				TCP []struct {
+					IP net.IP `json:"ip"`
+					CN string `json:"cn"`
+				} `json:"ovpntcp"`
+			} `json:"servers"`
+		} `json:"regions"`
+	}
+	if err := json.Unmarshal(b, &data); err != nil {
+		return err
+	}
+	servers := make([]models.PIAServer, 0, len(data.Regions))
+	for _, region := range data.Regions {
+		server := models.PIAServer{
+			Region:      region.Name,
+			PortForward: region.PortForward,
+		}
+		for _, udpServer := range region.Servers.UDP {
+			if len(server.OpenvpnUDP.CN) > 0 && server.OpenvpnUDP.CN != udpServer.CN {
+				return fmt.Errorf("CN is different for UDP for region %q: %q and %q", region.Name, server.OpenvpnUDP.CN, udpServer.CN)
+			}
+			if udpServer.IP != nil {
+				server.OpenvpnUDP.IPs = append(server.OpenvpnUDP.IPs, udpServer.IP)
+			}
+		}
+		for _, tcpServer := range region.Servers.TCP {
+			if len(server.OpenvpnTCP.CN) > 0 && server.OpenvpnTCP.CN != tcpServer.CN {
+				return fmt.Errorf("CN is different for TCP for region %q: %q and %q", region.Name, server.OpenvpnTCP.CN, tcpServer.CN)
+			}
+			if tcpServer.IP != nil {
+				server.OpenvpnTCP.IPs = append(server.OpenvpnTCP.IPs, tcpServer.IP)
+			}
+		}
+		if server.OpenvpnTCP.CN != server.OpenvpnUDP.CN {
+			return fmt.Errorf("not the same: %q, %q", server.OpenvpnTCP.CN, server.OpenvpnUDP.CN)
+		}
+		servers = append(servers, server)
+	}
+	sort.Slice(servers, func(i, j int) bool {
+		return servers[i].Region < servers[j].Region
+	})
+	if u.options.Stdout {
+		u.println(stringifyPIAServers(servers))
+	}
+	u.servers.Pia.Timestamp = u.timeNow().Unix()
+	u.servers.Pia.Servers = servers
+	return nil
+}
+
+func stringifyPIAServers(servers []models.PIAServer) (s string) {
+	s = "func PIAServers() []models.PIAServer {\n"
+	s += "	return []models.PIAServer{\n"
+	for _, server := range servers {
+		s += "		" + server.String() + ",\n"
+	}
+	s += "	}\n"
+	s += "}"
+	return s
+}