feat: SlickVPN Support (#961)

- `internal/updater/html` package
- Add unit tests for slickvpn updating code
- Change shared html package to be more share-able
- Split html utilities in multiple files
- Fix processing .ovpn files with prefix space

Authored by @Rohaq 
Co-authored-by: Quentin McGaw <quentin.mcgaw@gmail.com>

internal/updater/html/attribute.go

@@ -0,0 +1,12 @@
+package html
+
+import "golang.org/x/net/html"
+
+func Attribute(node *html.Node, key string) (value string) {
+	for _, attribute := range node.Attr {
+		if attribute.Key == key {
+			return attribute.Val
+		}
+	}
+	return ""
+}

internal/updater/html/bfs.go

@@ -0,0 +1,43 @@
+package html
+
+import (
+	"container/list"
+	"fmt"
+
+	"golang.org/x/net/html"
+)
+
+// BFS returns the node matching the match function and nil
+// if no node is found.
+func BFS(rootNode *html.Node, match MatchFunc) (node *html.Node) {
+	visited := make(map[*html.Node]struct{})
+	queue := list.New()
+	_ = queue.PushBack(rootNode)
+
+	for queue.Len() > 0 {
+		listElement := queue.Front()
+		node, ok := queue.Remove(listElement).(*html.Node)
+		if !ok {
+			panic(fmt.Sprintf("linked list has bad type %T", listElement.Value))
+		}
+
+		if node == nil {
+			continue
+		}
+
+		if _, ok := visited[node]; ok {
+			continue
+		}
+		visited[node] = struct{}{}
+
+		if match(node) {
+			return node
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			_ = queue.PushBack(child)
+		}
+	}
+
+	return nil
+}

internal/updater/html/css.go

@@ -0,0 +1,22 @@
+package html
+
+import (
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+func HasClassStrings(node *html.Node, classStrings ...string) (match bool) {
+	targetClasses := make(map[string]struct{}, len(classStrings))
+	for _, classString := range classStrings {
+		targetClasses[classString] = struct{}{}
+	}
+
+	classAttribute := Attribute(node, "class")
+	classes := strings.Fields(classAttribute)
+	for _, class := range classes {
+		delete(targetClasses, class)
+	}
+
+	return len(targetClasses) == 0
+}

internal/updater/html/errors.go

@@ -0,0 +1,27 @@
+package html
+
+import (
+	"bytes"
+	"fmt"
+
+	"golang.org/x/net/html"
+)
+
+func WrapError(sentinelError error, node *html.Node) error {
+	return fmt.Errorf("%w: in HTML code: %s",
+		sentinelError, mustRenderHTML(node))
+}
+
+func WrapWarning(warning string, node *html.Node) string {
+	return fmt.Sprintf("%s: in HTML code: %s",
+		warning, mustRenderHTML(node))
+}
+
+func mustRenderHTML(node *html.Node) (rendered string) {
+	stringBuffer := bytes.NewBufferString("")
+	err := html.Render(stringBuffer, node)
+	if err != nil {
+		panic(err)
+	}
+	return stringBuffer.String()
+}

internal/updater/html/match.go

@@ -0,0 +1,43 @@
+package html
+
+import (
+	"golang.org/x/net/html"
+)
+
+type MatchFunc func(node *html.Node) (match bool)
+
+func MatchID(id string) MatchFunc {
+	return func(node *html.Node) (match bool) {
+		if node == nil {
+			return false
+		}
+
+		return Attribute(node, "id") == id
+	}
+}
+
+func MatchData(data string) MatchFunc {
+	return func(node *html.Node) (match bool) {
+		return node != nil && node.Type == html.ElementNode && node.Data == data
+	}
+}
+
+func DirectChild(parent *html.Node,
+	matchFunc MatchFunc) (child *html.Node) {
+	for child := parent.FirstChild; child != nil; child = child.NextSibling {
+		if matchFunc(child) {
+			return child
+		}
+	}
+	return nil
+}
+
+func DirectChildren(parent *html.Node,
+	matchFunc MatchFunc) (children []*html.Node) {
+	for child := parent.FirstChild; child != nil; child = child.NextSibling {
+		if matchFunc(child) {
+			children = append(children, child)
+		}
+	}
+	return children
+}

internal/updater/openvpn/extract.go

@@ -76,6 +76,7 @@ func ExtractIPs(b []byte) (ips []net.IP, err error) {
 func extractRemoteHosts(content []byte, rejectIP, rejectDomain bool) (hosts []string) {
 	lines := strings.Split(string(content), "\n")
 	for _, line := range lines {
+		line = strings.TrimSpace(line)
 		if !strings.HasPrefix(line, "remote ") {
 			continue
 		}

internal/updater/openvpn/multifetch.go

@@ -8,8 +8,8 @@ import (
 // FetchMultiFiles fetches multiple Openvpn files in parallel and
 // parses them to extract each of their host. A mapping from host to
 // URL is returned.
-func FetchMultiFiles(ctx context.Context, client *http.Client, urls []string) (
-	hostToURL map[string]string, err error) {
+func FetchMultiFiles(ctx context.Context, client *http.Client, urls []string,
+	failEarly bool) (hostToURL map[string]string, errors []error) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -22,14 +22,14 @@ func FetchMultiFiles(ctx context.Context, client *http.Client, urls []string) (
 
 	results := make(chan Result)
 	defer close(results)
-	errors := make(chan error)
-	defer close(errors)
+	errorsCh := make(chan error)
+	defer close(errorsCh)
 
 	for _, url := range urls {
 		go func(url string) {
 			host, err := FetchFile(ctx, client, url)
 			if err != nil {
-				errors <- err
+				errorsCh <- err
 				return
 			}
 			results <- Result{
@@ -41,19 +41,26 @@ func FetchMultiFiles(ctx context.Context, client *http.Client, urls []string) (
 
 	for range urls {
 		select {
-		case newErr := <-errors:
-			if err == nil { // only assign to the first error
-				err = newErr
-				cancel() // stop other operations, this will trigger other errors we ignore
-			}
 		case result := <-results:
 			hostToURL[result.host] = result.url
+		case err := <-errorsCh:
+			if !failEarly {
+				errors = append(errors, err)
+				break
+			}
+
+			if len(errors) == 0 {
+				errors = []error{err} // keep only the first error
+				// stop other operations, this will trigger other errors we ignore
+				cancel()
+			}
 		}
 	}
 
-	if err != nil {
-		return nil, err
+	if len(errors) > 0 && failEarly {
+		// we don't care about the result found
+		return nil, errors
 	}
 
-	return hostToURL, nil
+	return hostToURL, errors
 }