From f1ad6f6bc534608ca63c4696106bd051fdae789f Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Wed, 15 Sep 2021 18:02:22 +0530 Subject: [PATCH 001/147] add payload, variable, helper support to matchers --- v2/pkg/operators/matchers/match.go | 13 ++++++++++++- v2/pkg/operators/matchers/match_test.go | 12 ++++++------ v2/pkg/protocols/dns/operators.go | 2 +- v2/pkg/protocols/file/operators.go | 2 +- v2/pkg/protocols/headless/operators.go | 2 +- v2/pkg/protocols/http/build_request.go | 5 +++-- v2/pkg/protocols/http/http.go | 1 + v2/pkg/protocols/http/operators.go | 2 +- v2/pkg/protocols/http/request.go | 1 + v2/pkg/protocols/network/network.go | 5 +++-- v2/pkg/protocols/network/operators.go | 2 +- v2/pkg/protocols/network/request.go | 2 ++ v2/pkg/protocols/offlinehttp/operators.go | 2 +- 13 files changed, 34 insertions(+), 17 deletions(-) diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 2b31386c4..4ab467740 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -3,6 +3,8 @@ package matchers import ( "encoding/hex" "strings" + + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" ) // MatchStatusCode matches a status code check against a corpus @@ -38,9 +40,18 @@ func (m *Matcher) MatchSize(length int) bool { } // MatchWords matches a word check against a corpus. -func (m *Matcher) MatchWords(corpus string) bool { +func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) bool { // Iterate over all the words accepted as valid for i, word := range m.Words { + if dynamicValues == nil { + dynamicValues = make(map[string]interface{}) + } + + var err error + word, err = expressions.Evaluate(word, dynamicValues) + if err != nil { + continue + } // Continue if the word doesn't match if !strings.Contains(corpus, word) { // If we are in an AND request and a match failed, diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index d6c7dc8a0..6a2d0b858 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -9,23 +9,23 @@ import ( func TestANDCondition(t *testing.T) { m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}} - matched := m.MatchWords("a b") + matched := m.MatchWords("a b", nil) require.True(t, matched, "Could not match valid AND condition") - matched = m.MatchWords("b") + matched = m.MatchWords("b", nil) require.False(t, matched, "Could match invalid AND condition") } func TestORCondition(t *testing.T) { m := &Matcher{condition: ORCondition, Words: []string{"a", "b"}} - matched := m.MatchWords("a b") + matched := m.MatchWords("a b", nil) require.True(t, matched, "Could not match valid OR condition") - matched = m.MatchWords("b") + matched = m.MatchWords("b", nil) require.True(t, matched, "Could not match valid OR condition") - matched = m.MatchWords("c") + matched = m.MatchWords("c", nil) require.False(t, matched, "Could match invalid OR condition") } @@ -34,6 +34,6 @@ func TestHexEncoding(t *testing.T) { err := m.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := m.MatchWords("PING") + matched := m.MatchWords("PING", nil) require.True(t, matched, "Could not match valid Hex condition") } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 1120aff81..a60a6afb0 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -32,7 +32,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(types.ToString(item)))) case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(types.ToString(item))) + return matcher.Result(matcher.MatchWords(types.ToString(item), nil)) case matchers.RegexMatcher: return matcher.Result(matcher.MatchRegex(types.ToString(item))) case matchers.BinaryMatcher: diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 4b8af5053..7ea7e7dfa 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -30,7 +30,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(itemStr))) case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr)) + return matcher.Result(matcher.MatchWords(itemStr, nil)) case matchers.RegexMatcher: return matcher.Result(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index d0a87aca6..a9a015a7d 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -28,7 +28,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(itemStr))) case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr)) + return matcher.Result(matcher.MatchWords(itemStr, nil)) case matchers.RegexMatcher: return matcher.Result(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 23797e887..b48ac085b 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -34,6 +34,7 @@ type generatedRequest struct { meta map[string]interface{} pipelinedClient *rawhttp.PipelineClient request *retryablehttp.Request + dynamicValues map[string]interface{} } // Make creates a http request for the provided input. @@ -135,7 +136,7 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st if err != nil { return nil, err } - return &generatedRequest{request: request, meta: generatorValues, original: r.request}, nil + return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues}, nil } // makeHTTPRequestFromRaw creates a *http.Request from a raw request @@ -196,7 +197,7 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest return nil, err } - return &generatedRequest{request: request, meta: generatorValues, original: r.request}, nil + return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues}, nil } // fillRequest fills various headers in the request with values diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index d3fa89dab..b8d45cba8 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -130,6 +130,7 @@ type Request struct { generator *generators.Generator // optional, only enabled when using payloads httpClient *retryablehttp.Client rawhttpClient *rawhttp.Client + dynamicValues map[string]interface{} // description: | // CookieReuse is an optional setting that enables cookie reuse for diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 81320de8f..c7cd2948c 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -33,7 +33,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(item))) case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(item)) + return matcher.Result(matcher.MatchWords(item, r.dynamicValues)) case matchers.RegexMatcher: return matcher.Result(matcher.MatchRegex(item)) case matchers.BinaryMatcher: diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 4a884b917..4c5cfbc9f 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -216,6 +216,7 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp return err } + r.dynamicValues = request.dynamicValues // Check if hosts just keep erroring if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.Check(reqURL) { break diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 1178d8f7b..802fcbca1 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -65,8 +65,9 @@ type Request struct { generator *generators.Generator attackType generators.Type // cache any variables that may be needed for operation. - dialer *fastdialer.Dialer - options *protocols.ExecuterOptions + dialer *fastdialer.Dialer + options *protocols.ExecuterOptions + dynamicValues map[string]interface{} } type addressKV struct { diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index c658dddca..f97add516 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -28,7 +28,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(itemStr))) case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr)) + return matcher.Result(matcher.MatchWords(itemStr, r.dynamicValues)) case matchers.RegexMatcher: return matcher.Result(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 98f9e744d..7d7f51563 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" ) @@ -83,6 +84,7 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin conn net.Conn err error ) + r.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { hostname = host diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index 3acdbf4fa..82e08052f 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -29,7 +29,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(item))) case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(item)) + return matcher.Result(matcher.MatchWords(item, nil)) case matchers.RegexMatcher: return matcher.Result(matcher.MatchRegex(item)) case matchers.BinaryMatcher: From 10cc08c758f2e6464eb5557234dabca8c0c3d6c2 Mon Sep 17 00:00:00 2001 From: Revblock Date: Fri, 17 Sep 2021 13:20:09 +0100 Subject: [PATCH 002/147] Add severity label to GitHub reporter --- v2/pkg/reporting/trackers/github/github.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 89f6dceb7..cad7c09ae 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -2,6 +2,7 @@ package github import ( "context" + "fmt" "net/url" "golang.org/x/oauth2" @@ -57,11 +58,12 @@ func New(options *Options) (*Integration, error) { func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) description := format.MarkdownDescription(event) + severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) req := &github.IssueRequest{ Title: &summary, Body: &description, - Labels: &[]string{i.options.IssueLabel}, + Labels: &[]string{i.options.IssueLabel, severityLabel}, Assignees: &[]string{i.options.Username}, } _, _, err := i.client.Issues.Create(context.Background(), i.options.Owner, i.options.ProjectName, req) From a4be346e4df86bf48cacbc874ea68c186c94b15c Mon Sep 17 00:00:00 2001 From: Scott Bamforth Date: Fri, 17 Sep 2021 21:08:48 +0100 Subject: [PATCH 003/147] Add severity label to Gitlab/JIRA reporter --- v2/pkg/reporting/trackers/gitlab/gitlab.go | 6 +++++- v2/pkg/reporting/trackers/jira/jira.go | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index 33d4fff01..30e5d33c8 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -1,6 +1,8 @@ package gitlab import ( + "fmt" + "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" "github.com/xanzy/go-gitlab" @@ -48,12 +50,14 @@ func New(options *Options) (*Integration, error) { func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) description := format.MarkdownDescription(event) + severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) _, _, err := i.client.Issues.CreateIssue(i.options.ProjectName, &gitlab.CreateIssueOptions{ Title: &summary, Description: &description, - Labels: gitlab.Labels{i.options.IssueLabel}, + Labels: gitlab.Labels{i.options.IssueLabel, severityLabel}, AssigneeIDs: []int{i.userID}, }) + return err } diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 1bfe2edd5..337ba3f5d 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -60,6 +60,7 @@ func New(options *Options) (*Integration, error) { // CreateNewIssue creates a new issue in the tracker func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { summary := format.Summary(event) + severityLabel := fmt.Sprintf("Severity:%s", event.Info.SeverityHolder.Severity.String()) fields := &jira.IssueFields{ Assignee: &jira.User{AccountID: i.options.AccountID}, @@ -68,6 +69,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { Type: jira.IssueType{Name: i.options.IssueType}, Project: jira.Project{Key: i.options.ProjectName}, Summary: summary, + Labels: []string{severityLabel}, } // On-prem version of Jira server does not use AccountID if !i.options.Cloud { @@ -77,6 +79,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { Type: jira.IssueType{Name: i.options.IssueType}, Project: jira.Project{Key: i.options.ProjectName}, Summary: summary, + Labels: []string{severityLabel}, } } From 4e7f1eef3709c0feb5e7ca791e53558f36c86aff Mon Sep 17 00:00:00 2001 From: Scott Bamforth Date: Fri, 17 Sep 2021 21:20:23 +0100 Subject: [PATCH 004/147] Fix formatting --- v2/pkg/reporting/trackers/gitlab/gitlab.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index 10760aae7..c02479662 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -5,7 +5,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" - "github.com/xanzy/go-gitlab" + "github.com/xanzy/go-gitlab" ) // Integration is a client for an issue tracker integration From 76b86ba2a09fd37a4a67abdaae5458333c8a3ab6 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 19 Sep 2021 16:26:47 +0530 Subject: [PATCH 005/147] disk cleanups --- v2/cmd/nuclei/main.go | 2 +- v2/internal/runner/runner.go | 8 ++++---- v2/pkg/reporting/exporters/es/elasticsearch.go | 2 +- .../exporters/{disk/disk.go => markdown/markdown.go} | 6 +++--- v2/pkg/reporting/exporters/sarif/sarif.go | 2 +- v2/pkg/reporting/reporting.go | 10 +++++----- v2/pkg/types/types.go | 4 ++-- 7 files changed, 17 insertions(+), 17 deletions(-) rename v2/pkg/reporting/exporters/{disk/disk.go => markdown/markdown.go} (92%) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 23fd7b1db..46cd25093 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -89,7 +89,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.NoTimestamp, "no-timestamp", "nts", false, "don't display timestamp metadata in CLI output"), flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"), - flagSet.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"), + flagSet.StringVarP(&options.MarkdownExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"), flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index e2f821134..4d8b7cf58 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -34,7 +34,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -262,12 +262,12 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error) } file.Close() } - if options.DiskExportDirectory != "" { + if options.MarkdownExportDirectory != "" { if reportingOptions != nil { - reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory} + reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory} } else { reportingOptions = &reporting.Options{} - reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory} + reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory} } } if options.SarifExport != "" { diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index e14f1a894..cf625eb66 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -81,7 +81,7 @@ func New(option *Options) (*Exporter, error) { return ei, nil } -// Export exports a passed result event to disk +// Export exports a passed result event to elasticsearch func (i *Exporter) Export(event *output.ResultEvent) error { // creating a request req, err := http.NewRequest(http.MethodPost, i.url, nil) diff --git a/v2/pkg/reporting/exporters/disk/disk.go b/v2/pkg/reporting/exporters/markdown/markdown.go similarity index 92% rename from v2/pkg/reporting/exporters/disk/disk.go rename to v2/pkg/reporting/exporters/markdown/markdown.go index c1fb92a45..5261b85fa 100644 --- a/v2/pkg/reporting/exporters/disk/disk.go +++ b/v2/pkg/reporting/exporters/markdown/markdown.go @@ -1,4 +1,4 @@ -package disk +package markdown import ( "bytes" @@ -22,7 +22,7 @@ type Options struct { Directory string `yaml:"directory"` } -// New creates a new disk exporter integration client based on options. +// New creates a new markdown exporter integration client based on options. func New(options *Options) (*Exporter, error) { directory := options.Directory if options.Directory == "" { @@ -36,7 +36,7 @@ func New(options *Options) (*Exporter, error) { return &Exporter{options: options, directory: directory}, nil } -// Export exports a passed result event to disk +// Export exports a passed result event to markdown func (i *Exporter) Export(event *output.ResultEvent) error { summary := format.Summary(event) description := format.MarkdownDescription(event) diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index 9c69dda3e..6395ee319 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -33,7 +33,7 @@ type Options struct { File string `yaml:"file"` } -// New creates a new disk exporter integration client based on options. +// New creates a new sarif exporter integration client based on options. func New(options *Options) (*Exporter, error) { report, err := sarif.New(sarif.Version210) if err != nil { diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index e50ec90e0..92c162f8f 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -10,7 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/dedupe" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/es" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github" @@ -30,8 +30,8 @@ type Options struct { Gitlab *gitlab.Options `yaml:"gitlab"` // Jira contains configuration options for Jira Issue Tracker Jira *jira.Options `yaml:"jira"` - // DiskExporter contains configuration options for Disk Exporter Module - DiskExporter *disk.Options `yaml:"disk"` + // MarkdownExporter contains configuration options for Markdown Exporter Module + MarkdownExporter *markdown.Options `yaml:"markdown"` // SarifExporter contains configuration options for Sarif Exporter Module SarifExporter *sarif.Options `yaml:"sarif"` // ElasticsearchExporter contains configuration options for Elasticsearch Exporter Module @@ -128,8 +128,8 @@ func New(options *Options, db string) (*Client, error) { } client.trackers = append(client.trackers, tracker) } - if options.DiskExporter != nil { - exporter, err := disk.New(options.DiskExporter) + if options.MarkdownExporter != nil { + exporter, err := markdown.New(options.MarkdownExporter) if err != nil { return nil, errors.Wrap(err, "could not create exporting client") } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index dafb2f3e7..5918ac41a 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -57,8 +57,8 @@ type Options struct { ReportingDB string // ReportingConfig is the config file for nuclei reporting module ReportingConfig string - // DiskExportDirectory is the directory to export reports in markdown on disk to - DiskExportDirectory string + // MarkdownExportDirectory is the directory to export reports in markdown format + MarkdownExportDirectory string // SarifExport is the file to export sarif output format to SarifExport string // ResolversFile is a file containing resolvers for nuclei. From b34d9507e34475604a798c5f4fdff67dee810ca0 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sat, 25 Sep 2021 04:38:57 +0530 Subject: [PATCH 006/147] Added version tagging to docker release --- .github/workflows/dockerhub-push.yml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index f201dc1b8..b6b7ecd96 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -9,26 +9,30 @@ jobs: docker: runs-on: ubuntu-latest steps: - - - name: Checkout + - name: Git Checkout uses: actions/checkout@v2 - - - name: Set up QEMU + + - name: Get Github tag + id: meta + run: | + echo "::set-output name=tag::$(git describe --tags --abbrev=0)" + + - name: Set up QEMU uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx + + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub + + - name: Login to DockerHub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - - - name: Build and push + + - name: Build and push uses: docker/build-push-action@v2 with: context: . platforms: linux/amd64,linux/arm64,linux/arm push: true - tags: projectdiscovery/nuclei:latest \ No newline at end of file + tags: projectdiscovery/nuclei:latest,projectdiscovery/nuclei:${{ steps.meta.outputs.tag }} \ No newline at end of file From 1a3e071fc2835fa0f465bd70172c565de913c10c Mon Sep 17 00:00:00 2001 From: sandeep Date: Sat, 25 Sep 2021 05:07:58 +0530 Subject: [PATCH 007/147] Update dockerhub-push.yml --- .github/workflows/dockerhub-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index b6b7ecd96..cebe5b7c1 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -15,7 +15,7 @@ jobs: - name: Get Github tag id: meta run: | - echo "::set-output name=tag::$(git describe --tags --abbrev=0)" + echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/nuclei/releases/latest" | jq -r .tag_name)" - name: Set up QEMU uses: docker/setup-qemu-action@v1 From cedc391485f394b9a5ad634dd78e619af299e4ae Mon Sep 17 00:00:00 2001 From: sandeep Date: Sat, 25 Sep 2021 16:13:07 +0530 Subject: [PATCH 008/147] Replacing go get with go install --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7936aed39..cc4b0e0e1 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp # Install Nuclei ```sh -GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei +go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest ``` **More installation [methods can be found here](https://nuclei.projectdiscovery.io/nuclei/get-started/).** From 1bf023abc223fb0cf274dad48730d4d6dbd69db0 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 25 Sep 2021 21:40:38 +0530 Subject: [PATCH 009/147] Disk report should include matcher or extractor name --- v2/pkg/reporting/exporters/disk/disk.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/v2/pkg/reporting/exporters/disk/disk.go b/v2/pkg/reporting/exporters/disk/disk.go index c1fb92a45..561fa9a17 100644 --- a/v2/pkg/reporting/exporters/disk/disk.go +++ b/v2/pkg/reporting/exporters/disk/disk.go @@ -45,6 +45,11 @@ func (i *Exporter) Export(event *output.ResultEvent) error { filenameBuilder.WriteString(event.TemplateID) filenameBuilder.WriteString("-") filenameBuilder.WriteString(strings.ReplaceAll(strings.ReplaceAll(event.Matched, "/", "_"), ":", "_")) + if event.MatcherName != "" { + filenameBuilder.WriteString(event.MatcherName) + } else if event.ExtractorName != "" { + filenameBuilder.WriteString(event.ExtractorName) + } filenameBuilder.WriteString(".md") finalFilename := sanitizeFilename(filenameBuilder.String()) From 0397068dd089617bafcaee58b2c90e3b704c2e22 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 26 Sep 2021 07:22:00 +0200 Subject: [PATCH 010/147] Fixing extended dns edge cases --- v2/pkg/protocols/dns/dns.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 3881d1a5a..4c2b8904f 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -128,6 +128,14 @@ func (r *Request) Make(domain string) (*dns.Msg, error) { q.Qclass = r.class q.Qtype = r.question req.Question = append(req.Question, q) + + req.SetEdns0(4096, false) + + switch r.question { + case dns.TypeTXT: + req.AuthenticatedData = true + } + return req, nil } From ce205ae8a0b5b5a31398c16080ccc667ab8ce975 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 26 Sep 2021 07:56:35 +0200 Subject: [PATCH 011/147] Adding support for installed chrome browser --- v2/cmd/nuclei/main.go | 1 + v2/pkg/protocols/headless/engine/engine.go | 9 +++++++++ v2/pkg/types/types.go | 2 ++ 3 files changed, 12 insertions(+) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 23fd7b1db..4f5171d06 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -139,6 +139,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"), flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"), flagSet.BoolVar(&options.ShowBrowser, "show-browser", false, "show the browser on the screen when running templates with headless mode"), + flagSet.BoolVar(&options.UseInstalledChrome, "use-installed-chrome", false, "use installed browser"), ) createGroup(flagSet, "debug", "Debug", diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index 073743840..a506cbda4 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -34,6 +34,7 @@ func New(options *types.Options) (*Browser, error) { return nil, errors.Wrap(err, "could not create temporary directory") } previouspids := findChromeProcesses() + chromeLauncher := launcher.New(). Leakless(false). Set("disable-gpu", "true"). @@ -49,6 +50,14 @@ func New(options *types.Options) (*Browser, error) { Delete("use-mock-keychain"). UserDataDir(dataStore) + if options.UseInstalledChrome { + if chromePath, hasChrome := launcher.LookPath(); hasChrome { + chromeLauncher.Bin(chromePath) + } else { + return nil, errors.New("the chrome browser is not installed") + } + } + if options.ShowBrowser { chromeLauncher = chromeLauncher.Headless(false) } else { diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index dafb2f3e7..3e5bcb5b4 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -103,6 +103,8 @@ type Options struct { Headless bool // ShowBrowser specifies whether the show the browser in headless mode ShowBrowser bool + // UseInstalledChrome skips chrome install and use local instance + UseInstalledChrome bool // SytemResolvers enables override of nuclei's DNS client opting to use system resolver stack. SystemResolvers bool // Metrics enables display of metrics via an http endpoint From df0c4e2361d076cf6a56d26e5b4cc03404f5e3b4 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 26 Sep 2021 07:56:54 +0200 Subject: [PATCH 012/147] updating dockerfile to support headless with alpine --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7a7431ac7..2ba73cfd4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.17.1-alpine as build-env -RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei +RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest FROM alpine:3.14 -RUN apk add --no-cache bind-tools ca-certificates +RUN apk add --no-cache bind-tools ca-certificates chromium COPY --from=build-env /go/bin/nuclei /usr/local/bin/nuclei ENTRYPOINT ["nuclei"] From 50fa95747a1f6af627d206cd10eeb39548a6da51 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 26 Sep 2021 08:31:24 +0200 Subject: [PATCH 013/147] Removing unwanted bytes operation for unsafe requests --- v2/pkg/protocols/http/raw/raw.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index ea23db738..348db9e3e 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -28,9 +28,6 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { Headers: make(map[string]string), } if unsafe { - request = strings.ReplaceAll(request, "\\0", "\x00") - request = strings.ReplaceAll(request, "\\r", "\r") - request = strings.ReplaceAll(request, "\\n", "\n") rawRequest.UnsafeRawBytes = []byte(request) } reader := bufio.NewReader(strings.NewReader(request)) From 30fb8aacaa078f38bb97699a50fd8f9c66efa4b0 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 26 Sep 2021 17:32:19 +0530 Subject: [PATCH 014/147] docker workflow improvements --- .github/workflows/dockerhub-push.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index cebe5b7c1..cc6775e82 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -1,8 +1,10 @@ name: 🌥 Docker Push on: - release: - types: [published] + workflow_run: + workflows: ["🎉 Release Binary"] + types: + - completed workflow_dispatch: jobs: From 5cd9ce03df10e72472ab21c6782f8f0f19ce41c9 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 26 Sep 2021 18:03:18 +0530 Subject: [PATCH 015/147] version update --- v2/pkg/catalog/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 76160a2d7..2eb49569f 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -25,7 +25,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.2` +const Version = `2.5.3-dev` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() From 74253459a81be02026afb9fe78ed136fada73ad7 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 26 Sep 2021 18:20:05 +0530 Subject: [PATCH 016/147] misc update --- v2/pkg/reporting/exporters/markdown/markdown.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/v2/pkg/reporting/exporters/markdown/markdown.go b/v2/pkg/reporting/exporters/markdown/markdown.go index c31b2eeb9..3e779ea1e 100644 --- a/v2/pkg/reporting/exporters/markdown/markdown.go +++ b/v2/pkg/reporting/exporters/markdown/markdown.go @@ -45,10 +45,16 @@ func (i *Exporter) Export(event *output.ResultEvent) error { filenameBuilder.WriteString(event.TemplateID) filenameBuilder.WriteString("-") filenameBuilder.WriteString(strings.ReplaceAll(strings.ReplaceAll(event.Matched, "/", "_"), ":", "_")) + + var suffix string if event.MatcherName != "" { - filenameBuilder.WriteString(event.MatcherName) + suffix = event.MatcherName } else if event.ExtractorName != "" { - filenameBuilder.WriteString(event.ExtractorName) + suffix = event.ExtractorName + } + if suffix != "" { + filenameBuilder.WriteRune('-') + filenameBuilder.WriteString(event.MatcherName) } filenameBuilder.WriteString(".md") finalFilename := sanitizeFilename(filenameBuilder.String()) From 71e4396bd41c36fe0b582d97f5bcd3a0e0550558 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 26 Sep 2021 22:33:12 +0200 Subject: [PATCH 017/147] Adding support to preserve input order --- v2/cmd/nuclei/main.go | 1 + v2/go.mod | 4 ++++ v2/go.sum | 10 ++++++++++ v2/internal/runner/processor.go | 21 +++++++++++++++++---- v2/internal/runner/runner.go | 29 +++++++++++++++++++++++++++++ v2/pkg/types/types.go | 2 ++ 6 files changed, 63 insertions(+), 4 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 46cd25093..269c1151e 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -133,6 +133,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"), flagSet.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "stop processing HTTP requests after the first match (may break template/workflow logic)"), + flagSet.BoolVar(&options.Stream, "stream", false, "Stream mode - start elaborating without sorting the input"), ) createGroup(flagSet, "headless", "Headless", diff --git a/v2/go.mod b/v2/go.mod index 3f5502466..41e38d700 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -31,6 +31,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e + github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 github.com/projectdiscovery/goflags v0.0.7 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa @@ -72,6 +73,8 @@ require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.1.6 // indirect + github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -99,6 +102,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect + github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c // indirect github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect diff --git a/v2/go.sum b/v2/go.sum index d908b8fe3..ca470ec0b 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -74,6 +74,10 @@ github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkE github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bloom/v3 v3.0.1 h1:Inlf0YXbgehxVjMPmCGv86iMCKMGPPrPSHtBF5yRHwA= +github.com/bits-and-blooms/bloom/v3 v3.0.1/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= @@ -341,6 +345,12 @@ github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= +github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew= +github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ= +github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4 h1:nnQAETrQgAwnlhjs3akTsTtOKSmXvLHAUL3GHx7RU44= +github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c h1:KDmCXhLLnS/Gc1VDyTxxamRzc8OmHCm1X+f8WQoaTRs= +github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index f24055ccf..9b7c6dd65 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -11,7 +11,7 @@ import ( func (r *Runner) processTemplateWithList(template *templates.Template) bool { results := &atomic.Bool{} wg := sizedwaitgroup.New(r.options.BulkSize) - r.hostMap.Scan(func(k, _ []byte) error { + processItem := func(k, _ []byte) error { URL := string(k) // Skip if the host has had errors @@ -29,7 +29,13 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool { results.CAS(false, match) }(URL) return nil - }) + } + if r.options.Stream { + r.hostMapStream.Scan(processItem) + } else { + r.hostMap.Scan(processItem) + } + wg.Wait() return results.Load() } @@ -39,7 +45,7 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool { results := &atomic.Bool{} wg := sizedwaitgroup.New(r.options.BulkSize) - r.hostMap.Scan(func(k, _ []byte) error { + processItem := func(k, _ []byte) error { URL := string(k) // Skip if the host has had errors @@ -53,7 +59,14 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool { results.CAS(false, match) }(URL) return nil - }) + } + + if r.options.Stream { + r.hostMapStream.Scan(processItem) + } else { + r.hostMap.Scan(processItem) + } + wg.Wait() return results.Load() } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 4d8b7cf58..6c23c2868 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -16,6 +16,8 @@ import ( "go.uber.org/ratelimit" "gopkg.in/yaml.v2" + "github.com/projectdiscovery/filekv" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/nuclei/v2/internal/colorizer" @@ -45,6 +47,7 @@ import ( // Runner is a client for running the enumeration process. type Runner struct { hostMap *hybrid.HybridMap + hostMapStream *filekv.FileDB output output.Writer interactsh *interactsh.Client inputCount int64 @@ -119,6 +122,20 @@ func New(options *types.Options) (*Runner, error) { } runner.hostMap = hm + if options.Stream { + fkvOptions := filekv.DefaultOptions + if tmpFileName, err := fileutil.GetTempFileName(); err != nil { + return nil, errors.Wrap(err, "could not create temporary input file") + } else { + fkvOptions.Path = tmpFileName + } + fkv, err := filekv.Open(fkvOptions) + if err != nil { + return nil, errors.Wrap(err, "could not create temporary unsorted input file") + } + runner.hostMapStream = fkv + } + runner.inputCount = 0 dupeCount := 0 @@ -138,6 +155,9 @@ func New(options *types.Options) (*Runner, error) { runner.inputCount++ // nolint:errcheck // ignoring error runner.hostMap.Set(url, nil) + if options.Stream { + runner.hostMapStream.Set([]byte(url), nil) + } } } @@ -158,6 +178,9 @@ func New(options *types.Options) (*Runner, error) { runner.inputCount++ // nolint:errcheck // ignoring error runner.hostMap.Set(url, nil) + if options.Stream { + runner.hostMapStream.Set([]byte(url), nil) + } } } @@ -180,6 +203,9 @@ func New(options *types.Options) (*Runner, error) { runner.inputCount++ // nolint:errcheck // ignoring error runner.hostMap.Set(url, nil) + if options.Stream { + runner.hostMapStream.Set([]byte(url), nil) + } } input.Close() } @@ -290,6 +316,9 @@ func (r *Runner) Close() { if r.projectFile != nil { r.projectFile.Close() } + if r.options.Stream { + r.hostMapStream.Close() + } protocolinit.Close() } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 5918ac41a..c6b029638 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -140,6 +140,8 @@ type Options struct { Stdin bool // StopAtFirstMatch stops processing template at first full match (this may break chained requests) StopAtFirstMatch bool + // Stream the input without sorting + Stream bool // NoMeta disables display of metadata for the matches NoMeta bool // NoTimestamp disables display of timestamp for the matcher From fc868aba4a0346a325612242ebd547fdf26e63f0 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 26 Sep 2021 22:36:44 +0200 Subject: [PATCH 018/147] misc --- v2/internal/runner/processor.go | 4 ++-- v2/internal/runner/runner.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 9b7c6dd65..60264591b 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -31,7 +31,7 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool { return nil } if r.options.Stream { - r.hostMapStream.Scan(processItem) + _ = r.hostMapStream.Scan(processItem) } else { r.hostMap.Scan(processItem) } @@ -62,7 +62,7 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool { } if r.options.Stream { - r.hostMapStream.Scan(processItem) + _ = r.hostMapStream.Scan(processItem) } else { r.hostMap.Scan(processItem) } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 6c23c2868..60a7ab6ba 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -156,7 +156,7 @@ func New(options *types.Options) (*Runner, error) { // nolint:errcheck // ignoring error runner.hostMap.Set(url, nil) if options.Stream { - runner.hostMapStream.Set([]byte(url), nil) + _ = runner.hostMapStream.Set([]byte(url), nil) } } } @@ -179,7 +179,7 @@ func New(options *types.Options) (*Runner, error) { // nolint:errcheck // ignoring error runner.hostMap.Set(url, nil) if options.Stream { - runner.hostMapStream.Set([]byte(url), nil) + _ = runner.hostMapStream.Set([]byte(url), nil) } } } @@ -204,7 +204,7 @@ func New(options *types.Options) (*Runner, error) { // nolint:errcheck // ignoring error runner.hostMap.Set(url, nil) if options.Stream { - runner.hostMapStream.Set([]byte(url), nil) + _ = runner.hostMapStream.Set([]byte(url), nil) } } input.Close() From 4be6b3cc963ad04574a8b5e36f66bd0c3e922e32 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 29 Sep 2021 19:43:46 +0300 Subject: [PATCH 019/147] [feature] Add coloring to debug information #999 [WIP] TODO: * if there are multiple matchers, make sure the response is only displayed once, with all the matching values colored * remove code duplication from the request.go files --- v2/pkg/operators/matchers/match.go | 33 ++++---- v2/pkg/operators/matchers/match_test.go | 30 +++++--- v2/pkg/operators/matchers/matchers.go | 8 ++ v2/pkg/operators/operators.go | 20 +++-- v2/pkg/protocols/dns/operators.go | 20 ++--- v2/pkg/protocols/dns/operators_test.go | 20 +++-- v2/pkg/protocols/dns/request.go | 58 +++++++++++--- v2/pkg/protocols/file/operators.go | 16 ++-- v2/pkg/protocols/file/operators_test.go | 15 ++-- v2/pkg/protocols/file/request.go | 76 ++++++++++++++----- v2/pkg/protocols/headless/operators.go | 16 ++-- v2/pkg/protocols/headless/request.go | 56 +++++++++++--- v2/pkg/protocols/http/operators.go | 23 +++--- v2/pkg/protocols/http/operators_test.go | 15 ++-- v2/pkg/protocols/http/request.go | 63 +++++++++++---- v2/pkg/protocols/network/operators.go | 16 ++-- v2/pkg/protocols/network/operators_test.go | 15 ++-- v2/pkg/protocols/network/request.go | 58 ++++++++++---- v2/pkg/protocols/offlinehttp/operators.go | 21 ++--- .../protocols/offlinehttp/operators_test.go | 15 ++-- v2/pkg/protocols/offlinehttp/request.go | 3 +- v2/pkg/protocols/protocols.go | 6 +- 22 files changed, 401 insertions(+), 202 deletions(-) diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 401110cfb..00c01d1ed 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -40,7 +40,8 @@ func (m *Matcher) MatchSize(length int) bool { } // MatchWords matches a word check against a corpus. -func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) bool { +func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) { + var matchedWords []string // Iterate over all the words accepted as valid for i, word := range m.Words { if dynamicValues == nil { @@ -57,7 +58,7 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { - return false + return false, []string{} } // Continue with the flow since it's an OR Condition. continue @@ -65,19 +66,21 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true + return true, []string{word} } + matchedWords = append(matchedWords, word) + // If we are at the end of the words, return with true if len(m.Words)-1 == i { - return true + return true, matchedWords } } - return false + return false, []string{} } // MatchRegex matches a regex check against a corpus -func (m *Matcher) MatchRegex(corpus string) bool { +func (m *Matcher) MatchRegex(corpus string) (bool, []string) { // Iterate over all the regexes accepted as valid for i, regex := range m.regexCompiled { // Continue if the regex doesn't match @@ -85,7 +88,7 @@ func (m *Matcher) MatchRegex(corpus string) bool { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { - return false + return false, []string{} } // Continue with the flow since it's an OR Condition. continue @@ -93,19 +96,19 @@ func (m *Matcher) MatchRegex(corpus string) bool { // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true + return true, regex.FindAllString(corpus, -1) } // If we are at the end of the regex, return with true if len(m.regexCompiled)-1 == i { - return true + return true, []string{corpus} } } - return false + return false, []string{} } // MatchBinary matches a binary check against a corpus -func (m *Matcher) MatchBinary(corpus string) bool { +func (m *Matcher) MatchBinary(corpus string) (bool, []string) { // Iterate over all the words accepted as valid for i, binary := range m.Binary { // Continue if the word doesn't match @@ -114,7 +117,7 @@ func (m *Matcher) MatchBinary(corpus string) bool { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { - return false + return false, []string{} } // Continue with the flow since it's an OR Condition. continue @@ -122,15 +125,15 @@ func (m *Matcher) MatchBinary(corpus string) bool { // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true + return true, []string{string(hexa)} } // If we are at the end of the words, return with true if len(m.Binary)-1 == i { - return true + return true, []string{string(hexa)} } } - return false + return false, []string{} } // MatchDSL matches on a generic map result diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index 6a2d0b858..d5161ca61 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -9,24 +9,29 @@ import ( func TestANDCondition(t *testing.T) { m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}} - matched := m.MatchWords("a b", nil) - require.True(t, matched, "Could not match valid AND condition") + isMatched, matched := m.MatchWords("a b", nil) + require.True(t, isMatched, "Could not match valid AND condition") + require.Equal(t, m.Words, matched) - matched = m.MatchWords("b", nil) - require.False(t, matched, "Could match invalid AND condition") + isMatched, matched = m.MatchWords("b", nil) + require.False(t, isMatched, "Could match invalid AND condition") + require.Equal(t, []string{}, matched) } func TestORCondition(t *testing.T) { m := &Matcher{condition: ORCondition, Words: []string{"a", "b"}} - matched := m.MatchWords("a b", nil) - require.True(t, matched, "Could not match valid OR condition") + isMatched, matched := m.MatchWords("a b", nil) + require.True(t, isMatched, "Could not match valid OR condition") + require.Equal(t, []string{"a"}, matched) - matched = m.MatchWords("b", nil) - require.True(t, matched, "Could not match valid OR condition") + isMatched, matched = m.MatchWords("b", nil) + require.True(t, isMatched, "Could not match valid OR condition") + require.Equal(t, []string{"b"}, matched) - matched = m.MatchWords("c", nil) - require.False(t, matched, "Could match invalid OR condition") + isMatched, matched = m.MatchWords("c", nil) + require.False(t, isMatched, "Could match invalid OR condition") + require.Equal(t, []string{}, matched) } func TestHexEncoding(t *testing.T) { @@ -34,6 +39,7 @@ func TestHexEncoding(t *testing.T) { err := m.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := m.MatchWords("PING", nil) - require.True(t, matched, "Could not match valid Hex condition") + isMatched, matched := m.MatchWords("PING", nil) + require.True(t, isMatched, "Could not match valid Hex condition") + require.Equal(t, m.Words, matched) } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index a0d176477..88ee413e1 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -165,6 +165,14 @@ func (m *Matcher) Result(data bool) bool { return data } +// ResultWithMatchedSnippet returns true and the matched snippet, or false and an empty string +func (m *Matcher) ResultWithMatchedSnippet(data bool, matchedSnippet []string) (bool, []string) { + if m.Negative { + return !data, []string{} + } + return data, matchedSnippet +} + // GetType returns the type of the matcher func (m *Matcher) GetType() MatcherType { return m.matcherType diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 493eddf06..23ed85978 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -100,7 +100,7 @@ func (r *Result) Merge(result *Result) { } // MatchFunc performs matching operation for a matcher on model and returns true or false. -type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool +type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) // ExtractFunc performs extracting operation for an extractor on model and returns true or false. type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} @@ -138,21 +138,19 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac for _, matcher := range r.Matchers { // Check if the matcher matched - if !match(data, matcher) { - // If the condition is AND we haven't matched, try next request. - if matcherCondition == matchers.ANDCondition { - if len(result.DynamicValues) > 0 { - return result, true - } - return nil, false - } - } else { + if isMatch, _ := match(data, matcher); isMatch { // If the matcher has matched, and it's an OR // write the first output then move to next matcher. if matcherCondition == matchers.ORCondition && matcher.Name != "" { result.Matches[matcher.Name] = struct{}{} } + matches = true + } else if matcherCondition == matchers.ANDCondition { + if len(result.DynamicValues) > 0 { + return result, true + } + return nil, false } } @@ -162,7 +160,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac return result, true } - // Don't print if we have matchers and they have not matched, regardless of extractor + // Don't print if we have matchers, and they have not matched, regardless of extractor if len(r.Matchers) > 0 && !matches { return nil, false } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index cda725d33..3b86e234b 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -14,7 +14,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "": @@ -23,24 +23,24 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } switch matcher.GetType() { - case matchers.StatusMatcher: - return matcher.Result(matcher.MatchStatusCode(item.(int))) + case matchers.StatusMatcher: // TODO is this correct? + return matcher.Result(matcher.MatchStatusCode(item.(int))), []string{} case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(types.ToString(item)))) + return matcher.Result(matcher.MatchSize(len(types.ToString(item)))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(types.ToString(item), nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(types.ToString(item), nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(types.ToString(item))) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(types.ToString(item))) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(types.ToString(item))) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(types.ToString(item))) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index 286dd582f..d249f1953 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -87,8 +87,9 @@ func TestDNSOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatch, matched := request.Match(event, matcher) + require.True(t, isMatch, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("rcode", func(t *testing.T) { @@ -100,8 +101,9 @@ func TestDNSOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile rcode matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid rcode response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid rcode response") + require.Equal(t, []string{}, matched) }) t.Run("negative", func(t *testing.T) { @@ -114,8 +116,9 @@ func TestDNSOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -127,8 +130,9 @@ func TestDNSOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index ae3041f33..f6487603a 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -2,9 +2,13 @@ package dns import ( "net/url" + "strings" + "github.com/logrusorgru/aurora" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" ) @@ -48,27 +52,57 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused r.options.Output.Request(r.options.TemplateID, domain, "dns", err) gologger.Verbose().Msgf("[%s] Sent DNS request to %s", r.options.TemplateID, domain) - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain) - gologger.Print().Msgf("%s", resp.String()) - } outputEvent := r.responseToDSLMap(compiledRequest, resp, input, input) for k, v := range previous { outputEvent[k] = v } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) - } - } + event := createEvent(r, domain, resp.String(), outputEvent) + callback(event) return nil } +// TODO extract duplicated code +func createEvent(request *Request, domain string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent { + debugResponse := func(data string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain) + gologger.Print().Msgf("%s", data) + } + } + + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + if request.CompiledOperators != nil { + + matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + isMatch, matched := request.Match(data, matcher) + var result string = response + + if len(matched) != 0 { + if !request.options.Options.NoColor { + colorizer := aurora.NewAurora(true) + for _, currentMatch := range matched { + result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + } + } + debugResponse(result) + } + + return isMatch, matched + } + + result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.Results = request.MakeResultEvent(event) + } + } else { + debugResponse(response) + } + return event +} + // isURL tests a string to determine if it is a well-structured url or not. func isURL(toTest string) bool { if _, err := url.ParseRequestURI(toTest); err != nil { diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 743dd7d13..11a804cab 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -13,7 +13,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "data", "": @@ -22,23 +22,23 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(itemStr))) + return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr, nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index 7c5be06d9..7b985f86b 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -72,8 +72,9 @@ func TestFileOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -86,8 +87,9 @@ func TestFileOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -99,8 +101,9 @@ func TestFileOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index d272e256b..fe2342dc6 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -3,13 +3,17 @@ package file import ( "io/ioutil" "os" + "strings" + "github.com/logrusorgru/aurora" "github.com/pkg/errors" + "github.com/remeh/sizedwaitgroup" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" - "github.com/remeh/sizedwaitgroup" ) var _ protocols.Request = &Request{} @@ -21,50 +25,40 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused err := r.getInputPaths(input, func(data string) { wg.Add() - go func(data string) { + go func(filePath string) { defer wg.Done() - file, err := os.Open(data) + file, err := os.Open(filePath) if err != nil { - gologger.Error().Msgf("Could not open file path %s: %s\n", data, err) + gologger.Error().Msgf("Could not open file path %s: %s\n", filePath, err) return } defer file.Close() stat, err := file.Stat() if err != nil { - gologger.Error().Msgf("Could not stat file path %s: %s\n", data, err) + gologger.Error().Msgf("Could not stat file path %s: %s\n", filePath, err) return } if stat.Size() >= int64(r.MaxSize) { - gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", data) + gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", filePath) return } buffer, err := ioutil.ReadAll(file) if err != nil { - gologger.Error().Msgf("Could not read file path %s: %s\n", data, err) + gologger.Error().Msgf("Could not read file path %s: %s\n", filePath, err) return } dataStr := tostring.UnsafeToString(buffer) - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, data) - gologger.Print().Msgf("%s", dataStr) - } - gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, data) - outputEvent := r.responseToDSLMap(dataStr, input, data) + + gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, filePath) + outputEvent := r.responseToDSLMap(dataStr, input, filePath) for k, v := range previous { outputEvent[k] = v } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) - } - } + event := createEvent(r, filePath, dataStr, outputEvent) callback(event) }(data) }) @@ -77,3 +71,43 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused r.options.Progress.IncrementRequests() return nil } + +// TODO extract duplicated code +func createEvent(request *Request, filePath string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent { + debugResponse := func(data string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath) + gologger.Print().Msgf("%s", data) + } + } + + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + if request.CompiledOperators != nil { + + matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + isMatch, matched := request.Match(data, matcher) + var result = response + + if len(matched) != 0 { + if !request.options.Options.NoColor { + colorizer := aurora.NewAurora(true) + for _, currentMatch := range matched { + result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + } + } + debugResponse(result) + } + + return isMatch, matched + } + + result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.Results = request.MakeResultEvent(event) + } + } else { + debugResponse(response) + } + return event +} diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index c384ee26a..1053acccb 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -11,7 +11,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "resp", "": @@ -20,23 +20,23 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(itemStr))) + return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr, nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 54345d468..095b7afdf 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -5,8 +5,11 @@ import ( "strings" "time" + "github.com/logrusorgru/aurora" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" ) @@ -62,19 +65,48 @@ func (r *Request) ExecuteWithResults(input string, metadata, previous output.Int outputEvent[k] = v } - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Headless response for %s", r.options.TemplateID, input) - gologger.Print().Msgf("%s", respBody) - } + event := createEvent(r, input, respBody, outputEvent) - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) - } - } callback(event) return nil } + +// TODO extract duplicated code +func createEvent(request *Request, input string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent { + debugResponse := func(data string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input) + gologger.Print().Msgf("%s", data) + } + } + + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + if request.CompiledOperators != nil { + + matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + isMatch, matched := request.Match(data, matcher) + var result = response + + if len(matched) != 0 { + if !request.options.Options.NoColor { + colorizer := aurora.NewAurora(true) + for _, currentMatch := range matched { + result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + } + } + debugResponse(result) + } + + return isMatch, matched + } + + result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.Results = request.MakeResultEvent(event) + } + } else { + debugResponse(response) + } + return event +} diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index a836fba7b..b35bae8e4 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -1,6 +1,7 @@ package http import ( + "fmt" "net/http" "strings" "time" @@ -13,35 +14,35 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { item, ok := getMatchPart(matcher.Part, data) if !ok { - return false + return false, []string{} } switch matcher.GetType() { case matchers.StatusMatcher: statusCode, ok := data["status_code"] if !ok { - return false + return false, []string{} } status, ok := statusCode.(int) if !ok { - return false + return false, []string{} } - return matcher.Result(matcher.MatchStatusCode(status)) + return matcher.Result(matcher.MatchStatusCode(status)), []string{fmt.Sprintf("HTTP/1.0 %d", status), fmt.Sprintf("HTTP/1.1 %d", status)} case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(item))) + return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(item, r.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, r.dynamicValues)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 6c24f236a..509312ea1 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -84,8 +84,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -98,8 +99,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -111,8 +113,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index d8a2c98dc..a6c493d69 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -12,11 +12,13 @@ import ( "sync" "time" + "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/remeh/sizedwaitgroup" "go.uber.org/multierr" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" @@ -424,12 +426,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ } } - // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, formedURL) - gologger.Print().Msgf("%s", string(redirectedResponse)) - } - // if nuclei-project is enabled store the response if not previously done if r.options.ProjectFile != nil && !fromcache { if err := r.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil { @@ -467,20 +463,55 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ } } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - var ok bool - event.OperatorsResult, ok = r.CompiledOperators.Execute(finalEvent, r.Match, r.Extract) - if ok && event.OperatorsResult != nil { - event.OperatorsResult.PayloadValues = request.meta - event.Results = r.MakeResultEvent(event) - } - event.InternalEvent = outputEvent - } + event := createEvent(r, formedURL, outputEvent, string(redirectedResponse), finalEvent, request) + callback(event) return nil } +// TODO extract duplicated code +func createEvent(request *Request, formedURL string, outputEvent output.InternalEvent, response string, finalEvent output.InternalEvent, generatedRequest *generatedRequest) *output.InternalWrappedEvent { + debugResponse := func(data string) { + // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL) + gologger.Print().Msgf("%s", data) + } + } + + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + if request.CompiledOperators != nil { + + matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + isMatch, matched := request.Match(data, matcher) + //var result = data["response"].(string) + var result = response + + if len(matched) != 0 { + if !request.options.Options.NoColor { + colorizer := aurora.NewAurora(true) + for _, currentMatch := range matched { + result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + } + } + debugResponse(result) + } + + return isMatch, matched + } + + result, ok := request.CompiledOperators.Execute(finalEvent, matcher, request.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.OperatorsResult.PayloadValues = generatedRequest.meta + event.Results = request.MakeResultEvent(event) + } + } else { + debugResponse(response) + } + return event +} + // setCustomHeaders sets the custom headers for generated request func (r *Request) setCustomHeaders(req *generatedRequest) { for k, v := range r.customHeaders { diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 75e71dda7..9bb996bca 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -11,7 +11,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "": @@ -20,23 +20,23 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(itemStr))) + return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr, r.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, r.dynamicValues)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index 4c8ea3028..474ac0dec 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -70,8 +70,9 @@ func TestNetworkOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -84,8 +85,9 @@ func TestNetworkOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -97,8 +99,9 @@ func TestNetworkOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 61abdf3fc..e64a292bd 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -9,8 +9,11 @@ import ( "strings" "time" + "github.com/logrusorgru/aurora" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" @@ -187,11 +190,6 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin } responseBuilder.Write(final[:n]) - if r.options.Options.Debug || r.options.Options.DebugResponse { - responseOutput := responseBuilder.String() - gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s\nHex: %s", responseOutput, hex.EncodeToString([]byte(responseOutput))) - } outputEvent := r.responseToDSLMap(reqBuilder.String(), string(final[:n]), responseBuilder.String(), input, actualAddress) outputEvent["ip"] = r.dialer.GetDialedIP(hostname) for k, v := range previous { @@ -206,14 +204,7 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin event := &output.InternalWrappedEvent{InternalEvent: outputEvent} if interactURL == "" { - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.OperatorsResult.PayloadValues = payloads - event.Results = r.MakeResultEvent(event) - } - } + event := createEvent(r, actualAddress, responseBuilder.String(), outputEvent, event, payloads) callback(event) } else if r.options.Interactsh != nil { r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ @@ -227,6 +218,47 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin return nil } +// TODO extract duplicated code +func createEvent(request *Request, actualAddress string, response string, outputEvent output.InternalEvent, event *output.InternalWrappedEvent, payloads map[string]interface{}) *output.InternalWrappedEvent { + debugResponse := func(data string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress) + gologger.Print().Msgf("%s\nHex: %s", response, hex.EncodeToString([]byte(response))) + } + } + + if request.CompiledOperators != nil { + + matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + isMatch, matched := request.Match(data, matcher) + //var result = data["response"].(string) + var result = response + + if len(matched) != 0 { + if !request.options.Options.NoColor { + colorizer := aurora.NewAurora(true) + for _, currentMatch := range matched { + result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + } + } + debugResponse(result) + } + + return isMatch, matched + } + + result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.OperatorsResult.PayloadValues = payloads + event.Results = request.MakeResultEvent(event) + } + } else { + debugResponse(response) + } + return event +} + // getAddress returns the address of the host to make request to func getAddress(toTest string) (string, error) { if strings.Contains(toTest, "://") { diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index ace976856..c5a7352da 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -1,6 +1,7 @@ package offlinehttp import ( + "fmt" "net/http" "strings" "time" @@ -13,31 +14,31 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { item, ok := getMatchPart(matcher.Part, data) if !ok { - return false + return false, []string{} } switch matcher.GetType() { case matchers.StatusMatcher: statusCode, ok := data["status_code"] if !ok { - return false + return false, []string{} } - return matcher.Result(matcher.MatchStatusCode(statusCode.(int))) + return matcher.Result(matcher.MatchStatusCode(statusCode.(int))), []string{fmt.Sprintf("HTTP/1.0 %d", statusCode), fmt.Sprintf("HTTP/1.1 %d", statusCode)} case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(item))) + return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(item, nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index 56facee78..eba24203e 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -76,8 +76,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -90,8 +91,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -103,8 +105,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 27b10e9db..9404180c1 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -8,11 +8,12 @@ import ( "strings" "github.com/pkg/errors" + "github.com/remeh/sizedwaitgroup" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" - "github.com/remeh/sizedwaitgroup" ) var _ protocols.Request = &Request{} diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 9ca38333e..2e03434fd 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -74,8 +74,10 @@ type Request interface { // condition matching. So, two requests can be sent and their match can // be evaluated from the third request by using the IDs for both requests. GetID() string - // Match performs matching operation for a matcher on model and returns true or false. - Match(data map[string]interface{}, matcher *matchers.Matcher) bool + // Match performs matching operation for a matcher on model and returns: + // true and a list of matched snippets if the matcher type is supports it + // otherwise false and an empty string slice + Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) // Extract performs extracting operation for an extractor on model and returns true or false. Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. From 8fdc9a7164e3ce7671d0f1cf505ed12777577eae Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 29 Sep 2021 19:44:37 +0300 Subject: [PATCH 020/147] Minor optimization: short-circuit the updateTemplates logic if template update was not requested --- v2/internal/runner/update.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 2b4cc06d6..3fc26131f 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -49,6 +49,10 @@ var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) // If the path exists but does not contain the latest version of public templates, // the new version is downloaded from GitHub to the templates' directory, overwriting the old content. func (r *Runner) updateTemplates() error { + if r.options.NoUpdateTemplates && !r.options.UpdateTemplates { + return nil + } + home, err := os.UserHomeDir() if err != nil { return err @@ -72,9 +76,6 @@ func (r *Runner) updateTemplates() error { r.templatesConfig = currentConfig } - if r.options.NoUpdateTemplates && !r.options.UpdateTemplates { - return nil - } client.InitNucleiVersion(config.Version) r.fetchLatestVersionsFromGithub(configDir) // also fetch the latest versions From d10dfb550e2f6a62ca246aea8eebd8596aa73c71 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 30 Sep 2021 19:07:59 +0300 Subject: [PATCH 021/147] Implicitly set the headless flag if template validation was requested, in order to correctly validate headless templates instead of complaining about "cannot create template executer" --- v2/internal/runner/options.go | 1 + v2/pkg/types/types.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index cba1af5bf..eb57d94aa 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -95,6 +95,7 @@ func validateOptions(options *types.Options) error { } if options.Validate { + options.Headless = true // required for correct validation of headless templates validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 5918ac41a..6e1b54565 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -103,7 +103,7 @@ type Options struct { Headless bool // ShowBrowser specifies whether the show the browser in headless mode ShowBrowser bool - // SytemResolvers enables override of nuclei's DNS client opting to use system resolver stack. + // SystemResolvers enables override of nuclei's DNS client opting to use system resolver stack. SystemResolvers bool // Metrics enables display of metrics via an http endpoint Metrics bool From 3abffe1f5d82b823f4501b5b0892cdfc6e3cd2bd Mon Sep 17 00:00:00 2001 From: Sullo Date: Thu, 30 Sep 2021 13:22:53 -0400 Subject: [PATCH 022/147] =?UTF-8?q?Only=20add=20a=20Host=20header=20automa?= =?UTF-8?q?tically=20if=20the=20raw=20request=20is=20*not*=20marked=20unsa?= =?UTF-8?q?fe.=20This=20will=20prevent=20nuclei=20from=20adding=20it=20whe?= =?UTF-8?q?n=20the=20template=20author=20doesn=E2=80=99t=20want=20it=20and?= =?UTF-8?q?=20uses=20unsafe:=20true=20Templates=20using=20raw=20requests?= =?UTF-8?q?=20without=20Host=20header=20were=20modified=20in=20https://git?= =?UTF-8?q?hub.com/projectdiscovery/nuclei-templates/pull/2784?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- v2/pkg/protocols/http/raw/raw.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 348db9e3e..582cd1351 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -110,9 +110,9 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { } rawRequest.FullURL = fmt.Sprintf("%s://%s%s", parsedURL.Scheme, strings.TrimSpace(hostURL), rawRequest.Path) - // If raw request doesn't have a Host header - // this will be generated from the parsed baseURL - if rawRequest.Headers["Host"] == "" { + // If raw request doesn't have a Host header and isn't marked unsafe, + // this will generate the Host header from the parsed baseURL + if !unsafe && rawRequest.Headers["Host"] == "" { rawRequest.Headers["Host"] = hostURL } From dccd47ed191506b88bd21442be40a7df223a7259 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 30 Sep 2021 20:36:39 +0300 Subject: [PATCH 023/147] [feature] Add coloring to debug information #999 [WIP] Return the matched content in case of multiple regex/binary matchers (+test) --- v2/pkg/operators/matchers/match.go | 13 ++++++-- v2/pkg/operators/matchers/match_test.go | 40 +++++++++++++++++++++---- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 00c01d1ed..7877660f5 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -81,6 +81,7 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} // MatchRegex matches a regex check against a corpus func (m *Matcher) MatchRegex(corpus string) (bool, []string) { + var matchedRegexes []string // Iterate over all the regexes accepted as valid for i, regex := range m.regexCompiled { // Continue if the regex doesn't match @@ -94,14 +95,17 @@ func (m *Matcher) MatchRegex(corpus string) (bool, []string) { continue } + currentMatches := regex.FindAllString(corpus, -1) // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true, regex.FindAllString(corpus, -1) + return true, currentMatches } + matchedRegexes = append(matchedRegexes, currentMatches...) + // If we are at the end of the regex, return with true if len(m.regexCompiled)-1 == i { - return true, []string{corpus} + return true, matchedRegexes } } return false, []string{} @@ -109,6 +113,7 @@ func (m *Matcher) MatchRegex(corpus string) (bool, []string) { // MatchBinary matches a binary check against a corpus func (m *Matcher) MatchBinary(corpus string) (bool, []string) { + var matchedBinary []string // Iterate over all the words accepted as valid for i, binary := range m.Binary { // Continue if the word doesn't match @@ -128,9 +133,11 @@ func (m *Matcher) MatchBinary(corpus string) (bool, []string) { return true, []string{string(hexa)} } + matchedBinary = append(matchedBinary, string(hexa)) + // If we are at the end of the words, return with true if len(m.Binary)-1 == i { - return true, []string{string(hexa)} + return true, matchedBinary } } return false, []string{} diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index d5161ca61..4c2f9bc0c 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -6,15 +6,29 @@ import ( "github.com/stretchr/testify/require" ) -func TestANDCondition(t *testing.T) { +func TestWordANDCondition(t *testing.T) { m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}} isMatched, matched := m.MatchWords("a b", nil) - require.True(t, isMatched, "Could not match valid AND condition") + require.True(t, isMatched, "Could not match words with valid AND condition") require.Equal(t, m.Words, matched) isMatched, matched = m.MatchWords("b", nil) - require.False(t, isMatched, "Could match invalid AND condition") + require.False(t, isMatched, "Could match words with invalid AND condition") + require.Equal(t, []string{}, matched) +} + +func TestRegexANDCondition(t *testing.T) { + m := &Matcher{Type: "regex", Condition: "and", Regex: []string{"[a-z]{3}", "\\d{2}"}} + err := m.CompileMatchers() + require.Nil(t, err) + + isMatched, matched := m.MatchRegex("abc abcd 123") + require.True(t, isMatched, "Could not match regex with valid AND condition") + require.Equal(t, []string{"abc", "abc", "12"}, matched) + + isMatched, matched = m.MatchRegex("bc 1") + require.False(t, isMatched, "Could match regex with invalid AND condition") require.Equal(t, []string{}, matched) } @@ -22,15 +36,29 @@ func TestORCondition(t *testing.T) { m := &Matcher{condition: ORCondition, Words: []string{"a", "b"}} isMatched, matched := m.MatchWords("a b", nil) - require.True(t, isMatched, "Could not match valid OR condition") + require.True(t, isMatched, "Could not match valid word OR condition") require.Equal(t, []string{"a"}, matched) isMatched, matched = m.MatchWords("b", nil) - require.True(t, isMatched, "Could not match valid OR condition") + require.True(t, isMatched, "Could not match valid word OR condition") require.Equal(t, []string{"b"}, matched) isMatched, matched = m.MatchWords("c", nil) - require.False(t, isMatched, "Could match invalid OR condition") + require.False(t, isMatched, "Could match invalid word OR condition") + require.Equal(t, []string{}, matched) +} + +func TestRegexOrCondition(t *testing.T) { + m := &Matcher{Type: "regex", Condition: "or", Regex: []string{"[a-z]{3}", "\\d{2}"}} + err := m.CompileMatchers() + require.Nil(t, err) + + isMatched, matched := m.MatchRegex("ab 123") + require.True(t, isMatched, "Could not match valid regex OR condition") + require.Equal(t, []string{"12"}, matched) + + isMatched, matched = m.MatchRegex("bc 1") + require.False(t, isMatched, "Could match invalid regex OR condition") require.Equal(t, []string{}, matched) } From aa78b58866795027de1befb26ab7a9ff41621f60 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 30 Sep 2021 21:05:12 +0300 Subject: [PATCH 024/147] [feature] Add coloring to debug information #999 [WIP] * if there are multiple matchers, the response is only displayed once, with all the matching values colored --- v2/pkg/protocols/dns/request.go | 22 +++++++++------------- v2/pkg/protocols/file/request.go | 22 +++++++++------------- v2/pkg/protocols/headless/request.go | 22 +++++++++------------- v2/pkg/protocols/http/request.go | 26 ++++++++++---------------- v2/pkg/protocols/network/request.go | 22 ++++++++-------------- 5 files changed, 45 insertions(+), 69 deletions(-) diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index f6487603a..3c511728c 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -65,28 +65,20 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused // TODO extract duplicated code func createEvent(request *Request, domain string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent { - debugResponse := func(data string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain) - gologger.Print().Msgf("%s", data) - } - } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if request.CompiledOperators != nil { + var responseToDump = response + if request.CompiledOperators != nil { matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { isMatch, matched := request.Match(data, matcher) - var result string = response if len(matched) != 0 { if !request.options.Options.NoColor { colorizer := aurora.NewAurora(true) for _, currentMatch := range matched { - result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + responseToDump = strings.ReplaceAll(responseToDump, currentMatch, colorizer.Green(currentMatch).String()) } } - debugResponse(result) } return isMatch, matched @@ -97,9 +89,13 @@ func createEvent(request *Request, domain string, response string, outputEvent o event.OperatorsResult = result event.Results = request.MakeResultEvent(event) } - } else { - debugResponse(response) } + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain) + gologger.Print().Msgf("%s", responseToDump) + } + return event } diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index fe2342dc6..35cedcd50 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -74,28 +74,20 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused // TODO extract duplicated code func createEvent(request *Request, filePath string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent { - debugResponse := func(data string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath) - gologger.Print().Msgf("%s", data) - } - } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if request.CompiledOperators != nil { + var responseToDump = response + if request.CompiledOperators != nil { matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { isMatch, matched := request.Match(data, matcher) - var result = response if len(matched) != 0 { if !request.options.Options.NoColor { colorizer := aurora.NewAurora(true) for _, currentMatch := range matched { - result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + responseToDump = strings.ReplaceAll(responseToDump, currentMatch, colorizer.Green(currentMatch).String()) } } - debugResponse(result) } return isMatch, matched @@ -106,8 +98,12 @@ func createEvent(request *Request, filePath string, response string, outputEvent event.OperatorsResult = result event.Results = request.MakeResultEvent(event) } - } else { - debugResponse(response) } + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath) + gologger.Print().Msgf("%s", responseToDump) + } + return event } diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 095b7afdf..cf9a550eb 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -73,28 +73,20 @@ func (r *Request) ExecuteWithResults(input string, metadata, previous output.Int // TODO extract duplicated code func createEvent(request *Request, input string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent { - debugResponse := func(data string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input) - gologger.Print().Msgf("%s", data) - } - } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if request.CompiledOperators != nil { + var responseToDump = response + if request.CompiledOperators != nil { matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { isMatch, matched := request.Match(data, matcher) - var result = response if len(matched) != 0 { if !request.options.Options.NoColor { colorizer := aurora.NewAurora(true) for _, currentMatch := range matched { - result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + responseToDump = strings.ReplaceAll(responseToDump, currentMatch, colorizer.Green(currentMatch).String()) } } - debugResponse(result) } return isMatch, matched @@ -105,8 +97,12 @@ func createEvent(request *Request, input string, response string, outputEvent ou event.OperatorsResult = result event.Results = request.MakeResultEvent(event) } - } else { - debugResponse(response) } + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input) + gologger.Print().Msgf("%s", responseToDump) + } + return event } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index a6c493d69..99fe43c47 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -471,30 +471,19 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ // TODO extract duplicated code func createEvent(request *Request, formedURL string, outputEvent output.InternalEvent, response string, finalEvent output.InternalEvent, generatedRequest *generatedRequest) *output.InternalWrappedEvent { - debugResponse := func(data string) { - // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL) - gologger.Print().Msgf("%s", data) - } - } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if request.CompiledOperators != nil { + var responseToDump = response + if request.CompiledOperators != nil { matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { isMatch, matched := request.Match(data, matcher) - //var result = data["response"].(string) - var result = response - if len(matched) != 0 { if !request.options.Options.NoColor { colorizer := aurora.NewAurora(true) for _, currentMatch := range matched { - result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + responseToDump = strings.ReplaceAll(responseToDump, currentMatch, colorizer.Green(currentMatch).String()) } } - debugResponse(result) } return isMatch, matched @@ -506,9 +495,14 @@ func createEvent(request *Request, formedURL string, outputEvent output.Internal event.OperatorsResult.PayloadValues = generatedRequest.meta event.Results = request.MakeResultEvent(event) } - } else { - debugResponse(response) } + + // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL) + gologger.Print().Msgf("%s", responseToDump) + } + return event } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index e64a292bd..2be7a9955 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -220,28 +220,18 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin // TODO extract duplicated code func createEvent(request *Request, actualAddress string, response string, outputEvent output.InternalEvent, event *output.InternalWrappedEvent, payloads map[string]interface{}) *output.InternalWrappedEvent { - debugResponse := func(data string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s\nHex: %s", response, hex.EncodeToString([]byte(response))) - } - } + var responseToDump = response if request.CompiledOperators != nil { - matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { isMatch, matched := request.Match(data, matcher) - //var result = data["response"].(string) - var result = response - if len(matched) != 0 { if !request.options.Options.NoColor { colorizer := aurora.NewAurora(true) for _, currentMatch := range matched { - result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + responseToDump = strings.ReplaceAll(responseToDump, currentMatch, colorizer.Green(currentMatch).String()) } } - debugResponse(result) } return isMatch, matched @@ -253,9 +243,13 @@ func createEvent(request *Request, actualAddress string, response string, output event.OperatorsResult.PayloadValues = payloads event.Results = request.MakeResultEvent(event) } - } else { - debugResponse(response) } + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress) + gologger.Print().Msgf("%s\nHex: %s", response, hex.EncodeToString([]byte(responseToDump))) + } + return event } From 4f8fb0e516a7e64e9a8db3ce7c89d128acd66718 Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 1 Oct 2021 00:40:13 +0530 Subject: [PATCH 025/147] Interactsh server update --- README.md | 2 +- v2/cmd/nuclei/main.go | 2 +- v2/pkg/protocols/common/interactsh/interactsh.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cc4b0e0e1..b68d2e700 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ CONFIGURATIONS: INTERACTSH: -no-interactsh disable interactsh server for OOB testing - -interactsh-url string interactsh server url for self-hosted instance (default "https://interact.sh") + -interactsh-url string interactsh server url for self-hosted instance (default "https://interactsh.com") -interactsh-token string authentication token for self-hosted interactsh server -interactions-cache-size int number of requests to keep in the interactions cache (default 5000) -interactions-eviction int number of seconds to wait before evicting requests from cache (default 60) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 46cd25093..cae40155d 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -109,7 +109,7 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "interactsh", "interactsh", flagSet.BoolVar(&options.NoInteractsh, "no-interactsh", false, "disable interactsh server for OOB testing"), - flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "interactsh server url for self-hosted instance"), + flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interactsh.com", "interactsh server url for self-hosted instance"), flagSet.StringVar(&options.InteractshToken, "interactsh-token", "", "authentication token for self-hosted interactsh server"), flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"), flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"), diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 1c3310365..0d11c7af1 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -256,7 +256,7 @@ func (c *Client) RequestEvent(interactshURL string, data *RequestData) { // HasMatchers returns true if an operator has interactsh part // matchers or extractors. // -// Used by requests to show result or not depending on presence of interact.sh +// Used by requests to show result or not depending on presence of interactsh.com // data part matchers. func HasMatchers(op *operators.Operators) bool { if op == nil { From a00083e2215734c5105b41b7716d7e31240cee79 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 1 Oct 2021 14:24:45 +0300 Subject: [PATCH 026/147] [feature] Add coloring to debug information #999 * remove some of the code duplication with a different approach --- v2/pkg/operators/operators.go | 8 ++-- .../response_highligther.go | 25 ++++++++++ v2/pkg/protocols/dns/request.go | 37 ++++----------- v2/pkg/protocols/file/request.go | 38 ++++----------- v2/pkg/protocols/headless/request.go | 42 +++++------------ v2/pkg/protocols/http/request.go | 46 ++++++------------- v2/pkg/protocols/network/request.go | 40 +++++----------- v2/pkg/protocols/offlinehttp/request.go | 21 ++++++--- v2/pkg/workflows/execute_test.go | 7 +-- 9 files changed, 103 insertions(+), 161 deletions(-) create mode 100644 v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 23ed85978..4262c7deb 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -64,7 +64,7 @@ type Result struct { // Extracted is true if any result type values were extracted Extracted bool // Matches is a map of matcher names that we matched - Matches map[string]struct{} + Matches map[string][]string // Extracts contains all the data extracted from inputs Extracts map[string][]string // OutputExtracts is the list of extracts to be displayed on screen. @@ -111,7 +111,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac var matches bool result := &Result{ - Matches: make(map[string]struct{}), + Matches: make(map[string][]string), Extracts: make(map[string][]string), DynamicValues: make(map[string]interface{}), } @@ -138,11 +138,11 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac for _, matcher := range r.Matchers { // Check if the matcher matched - if isMatch, _ := match(data, matcher); isMatch { + if isMatch, matched := match(data, matcher); isMatch { // If the matcher has matched, and it's an OR // write the first output then move to next matcher. if matcherCondition == matchers.ORCondition && matcher.Name != "" { - result.Matches[matcher.Name] = struct{}{} + result.Matches[matcher.Name] = matched } matches = true diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go new file mode 100644 index 000000000..fa168e26c --- /dev/null +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go @@ -0,0 +1,25 @@ +package responsehighlighter + +import ( + "strings" + + "github.com/logrusorgru/aurora" + + "github.com/projectdiscovery/nuclei/v2/pkg/operators" +) + +func Highlight(operatorResult *operators.Result, response string, noColor bool) string { + result := response + if operatorResult != nil && !noColor { + for _, matches := range operatorResult.Matches { + if len(matches) > 0 { + colorizer := aurora.NewAurora(true) + for _, currentMatch := range matches { + result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + } + } + } + } + + return result +} diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 3c511728c..ee5217b56 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -2,15 +2,13 @@ package dns import ( "net/url" - "strings" - "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" ) var _ protocols.Request = &Request{} @@ -57,45 +55,28 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused outputEvent[k] = v } - event := createEvent(r, domain, resp.String(), outputEvent) + event := createEvent(r, outputEvent) + + if r.options.Options.Debug || r.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, resp.String(), r.options.Options.NoColor)) + } callback(event) return nil } -// TODO extract duplicated code -func createEvent(request *Request, domain string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent { +func createEvent(request *Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent { event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - var responseToDump = response if request.CompiledOperators != nil { - matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - isMatch, matched := request.Match(data, matcher) - - if len(matched) != 0 { - if !request.options.Options.NoColor { - colorizer := aurora.NewAurora(true) - for _, currentMatch := range matched { - responseToDump = strings.ReplaceAll(responseToDump, currentMatch, colorizer.Green(currentMatch).String()) - } - } - } - - return isMatch, matched - } - - result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract) + result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract) if ok && result != nil { event.OperatorsResult = result event.Results = request.MakeResultEvent(event) } } - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain) - gologger.Print().Msgf("%s", responseToDump) - } - return event } diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 35cedcd50..7a36b04e4 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -3,16 +3,14 @@ package file import ( "io/ioutil" "os" - "strings" - "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/remeh/sizedwaitgroup" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" ) @@ -58,7 +56,13 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused outputEvent[k] = v } - event := createEvent(r, filePath, dataStr, outputEvent) + event := createEvent(r, outputEvent) + + if r.options.Options.Debug || r.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, filePath) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, dataStr, r.options.Options.NoColor)) + } + callback(event) }(data) }) @@ -72,38 +76,16 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused return nil } -// TODO extract duplicated code -func createEvent(request *Request, filePath string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent { +func createEvent(request *Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent { event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - var responseToDump = response if request.CompiledOperators != nil { - matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - isMatch, matched := request.Match(data, matcher) - - if len(matched) != 0 { - if !request.options.Options.NoColor { - colorizer := aurora.NewAurora(true) - for _, currentMatch := range matched { - responseToDump = strings.ReplaceAll(responseToDump, currentMatch, colorizer.Green(currentMatch).String()) - } - } - } - - return isMatch, matched - } - - result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract) + result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract) if ok && result != nil { event.OperatorsResult = result event.Results = request.MakeResultEvent(event) } } - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath) - gologger.Print().Msgf("%s", responseToDump) - } - return event } diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index cf9a550eb..2ce8cf7ee 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -5,13 +5,12 @@ import ( "strings" "time" - "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" ) var _ protocols.Request = &Request{} @@ -55,54 +54,37 @@ func (r *Request) ExecuteWithResults(input string, metadata, previous output.Int gologger.Print().Msgf("%s", reqBuilder.String()) } - var respBody string + var responseBody string html, err := page.Page().Element("html") if err == nil { - respBody, _ = html.HTML() + responseBody, _ = html.HTML() } - outputEvent := r.responseToDSLMap(respBody, reqBuilder.String(), input, input) + outputEvent := r.responseToDSLMap(responseBody, reqBuilder.String(), input, input) for k, v := range out { outputEvent[k] = v } - event := createEvent(r, input, respBody, outputEvent) + event := createEvent(r, outputEvent) + + if r.options.Options.Debug || r.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Headless response for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, r.options.Options.NoColor)) + } callback(event) return nil } -// TODO extract duplicated code -func createEvent(request *Request, input string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent { +func createEvent(request *Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent { event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - var responseToDump = response if request.CompiledOperators != nil { - matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - isMatch, matched := request.Match(data, matcher) - - if len(matched) != 0 { - if !request.options.Options.NoColor { - colorizer := aurora.NewAurora(true) - for _, currentMatch := range matched { - responseToDump = strings.ReplaceAll(responseToDump, currentMatch, colorizer.Green(currentMatch).String()) - } - } - } - - return isMatch, matched - } - - result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract) + result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract) if ok && result != nil { event.OperatorsResult = result event.Results = request.MakeResultEvent(event) } } - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input) - gologger.Print().Msgf("%s", responseToDump) - } - return event } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 99fe43c47..d3405fdba 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -12,16 +12,15 @@ import ( "sync" "time" - "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/remeh/sizedwaitgroup" "go.uber.org/multierr" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" @@ -463,44 +462,25 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ } } - event := createEvent(r, formedURL, outputEvent, string(redirectedResponse), finalEvent, request) + event := createEvent(r, outputEvent, finalEvent, request) + + if r.options.Options.Debug || r.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, formedURL) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, string(redirectedResponse), r.options.Options.NoColor)) + } callback(event) return nil } -// TODO extract duplicated code -func createEvent(request *Request, formedURL string, outputEvent output.InternalEvent, response string, finalEvent output.InternalEvent, generatedRequest *generatedRequest) *output.InternalWrappedEvent { +func createEvent(request *Request, outputEvent output.InternalEvent, finalEvent output.InternalEvent, generatedRequest *generatedRequest) *output.InternalWrappedEvent { event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - var responseToDump = response - if request.CompiledOperators != nil { - matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - isMatch, matched := request.Match(data, matcher) - if len(matched) != 0 { - if !request.options.Options.NoColor { - colorizer := aurora.NewAurora(true) - for _, currentMatch := range matched { - responseToDump = strings.ReplaceAll(responseToDump, currentMatch, colorizer.Green(currentMatch).String()) - } - } - } - - return isMatch, matched - } - - result, ok := request.CompiledOperators.Execute(finalEvent, matcher, request.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.OperatorsResult.PayloadValues = generatedRequest.meta - event.Results = request.MakeResultEvent(event) - } - } - - // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL) - gologger.Print().Msgf("%s", responseToDump) + result, ok := request.CompiledOperators.Execute(finalEvent, request.Match, request.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.OperatorsResult.PayloadValues = generatedRequest.meta + event.Results = request.MakeResultEvent(event) } return event diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 2be7a9955..bb4540cbf 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -9,15 +9,14 @@ import ( "strings" "time" - "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" ) @@ -190,7 +189,8 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin } responseBuilder.Write(final[:n]) - outputEvent := r.responseToDSLMap(reqBuilder.String(), string(final[:n]), responseBuilder.String(), input, actualAddress) + response := responseBuilder.String() + outputEvent := r.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress) outputEvent["ip"] = r.dialer.GetDialedIP(hostname) for k, v := range previous { outputEvent[k] = v @@ -204,7 +204,7 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin event := &output.InternalWrappedEvent{InternalEvent: outputEvent} if interactURL == "" { - event := createEvent(r, actualAddress, responseBuilder.String(), outputEvent, event, payloads) + event := createEvent(r, outputEvent, event, payloads) callback(event) } else if r.options.Interactsh != nil { r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ @@ -215,29 +215,18 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin ExtractFunc: r.Extract, }) } + + if r.options.Options.Debug || r.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress) + gologger.Print().Msgf("%s\nHex: %s", response, responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), r.options.Options.NoColor)) + } + return nil } -// TODO extract duplicated code -func createEvent(request *Request, actualAddress string, response string, outputEvent output.InternalEvent, event *output.InternalWrappedEvent, payloads map[string]interface{}) *output.InternalWrappedEvent { - var responseToDump = response - +func createEvent(request *Request, outputEvent output.InternalEvent, event *output.InternalWrappedEvent, payloads map[string]interface{}) *output.InternalWrappedEvent { if request.CompiledOperators != nil { - matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - isMatch, matched := request.Match(data, matcher) - if len(matched) != 0 { - if !request.options.Options.NoColor { - colorizer := aurora.NewAurora(true) - for _, currentMatch := range matched { - responseToDump = strings.ReplaceAll(responseToDump, currentMatch, colorizer.Green(currentMatch).String()) - } - } - } - - return isMatch, matched - } - - result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract) + result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract) if ok && result != nil { event.OperatorsResult = result event.OperatorsResult.PayloadValues = payloads @@ -245,11 +234,6 @@ func createEvent(request *Request, actualAddress string, response string, output } } - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s\nHex: %s", response, hex.EncodeToString([]byte(responseToDump))) - } - return event } diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 9404180c1..18d0e3822 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -11,6 +11,7 @@ import ( "github.com/remeh/sizedwaitgroup" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" @@ -85,13 +86,7 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused } for _, operator := range r.compiledOperators { - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - var ok bool - - event.OperatorsResult, ok = operator.Execute(outputEvent, r.Match, r.Extract) - if ok && event.OperatorsResult != nil { - event.Results = r.MakeResultEvent(event) - } + event := createEvent(outputEvent, operator, r) callback(event) } }(data) @@ -106,6 +101,18 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused return nil } +func createEvent(outputEvent map[string]interface{}, operator *operators.Operators, r *Request) *output.InternalWrappedEvent { + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + + result, ok := operator.Execute(outputEvent, r.Match, r.Extract) + if ok && event.OperatorsResult != nil { + event.OperatorsResult = result + event.Results = r.MakeResultEvent(event) + } + + return event +} + // headersToString converts http headers to string func headersToString(headers http.Header) string { builder := &strings.Builder{} diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/workflows/execute_test.go index 26f7c2b99..6d9ab6a09 100644 --- a/v2/pkg/workflows/execute_test.go +++ b/v2/pkg/workflows/execute_test.go @@ -3,12 +3,13 @@ package workflows import ( "testing" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/stretchr/testify/require" ) func TestWorkflowsSimple(t *testing.T) { @@ -106,7 +107,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{ - Matches: map[string]struct{}{"tomcat": {}}, + Matches: map[string][]string{"tomcat": {}}, Extracts: map[string][]string{}, }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, @@ -134,7 +135,7 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{ - Matches: map[string]struct{}{"tomcat": {}}, + Matches: map[string][]string{"tomcat": {}}, Extracts: map[string][]string{}, }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, From 898c9431b5ec11992b8561600d6b8f92ba86cfa5 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 1 Oct 2021 14:30:04 +0300 Subject: [PATCH 027/147] [feature] Add coloring to debug information #999 * corrected/renamed receivers from one character names to human-readable format --- v2/pkg/operators/operators.go | 30 +-- v2/pkg/protocols/dns/dns.go | 38 ++-- v2/pkg/protocols/dns/operators.go | 22 +- v2/pkg/protocols/dns/request.go | 34 +-- v2/pkg/protocols/file/file.go | 36 ++-- v2/pkg/protocols/file/find.go | 35 +-- v2/pkg/protocols/file/operators.go | 22 +- v2/pkg/protocols/file/request.go | 26 +-- v2/pkg/protocols/headless/headless.go | 16 +- v2/pkg/protocols/headless/operators.go | 22 +- v2/pkg/protocols/headless/request.go | 38 ++-- v2/pkg/protocols/http/cluster.go | 16 +- v2/pkg/protocols/http/http.go | 76 +++---- v2/pkg/protocols/http/operators.go | 24 +-- v2/pkg/protocols/http/request.go | 228 ++++++++++---------- v2/pkg/protocols/http/request_generator.go | 9 +- v2/pkg/protocols/network/network.go | 40 ++-- v2/pkg/protocols/network/operators.go | 24 +-- v2/pkg/protocols/network/request.go | 102 ++++----- v2/pkg/protocols/offlinehttp/find.go | 14 +- v2/pkg/protocols/offlinehttp/offlinehttp.go | 11 +- v2/pkg/protocols/offlinehttp/operators.go | 22 +- v2/pkg/protocols/offlinehttp/request.go | 24 +-- 23 files changed, 455 insertions(+), 454 deletions(-) diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 4262c7deb..86f1611b9 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -32,19 +32,19 @@ type Operators struct { } // Compile compiles the operators as well as their corresponding matchers and extractors -func (r *Operators) Compile() error { - if r.MatchersCondition != "" { - r.matchersCondition = matchers.ConditionTypes[r.MatchersCondition] +func (operators *Operators) Compile() error { + if operators.MatchersCondition != "" { + operators.matchersCondition = matchers.ConditionTypes[operators.MatchersCondition] } else { - r.matchersCondition = matchers.ORCondition + operators.matchersCondition = matchers.ORCondition } - for _, matcher := range r.Matchers { + for _, matcher := range operators.Matchers { if err := matcher.CompileMatchers(); err != nil { return errors.Wrap(err, "could not compile matcher") } } - for _, extractor := range r.Extractors { + for _, extractor := range operators.Extractors { if err := extractor.CompileExtractors(); err != nil { return errors.Wrap(err, "could not compile extractor") } @@ -53,8 +53,8 @@ func (r *Operators) Compile() error { } // GetMatchersCondition returns the condition for the matchers -func (r *Operators) GetMatchersCondition() matchers.ConditionType { - return r.matchersCondition +func (operators *Operators) GetMatchersCondition() matchers.ConditionType { + return operators.matchersCondition } // Result is a result structure created from operators running on data. @@ -106,8 +106,8 @@ type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) (boo type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // Execute executes the operators on data and returns a result structure -func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc) (*Result, bool) { - matcherCondition := r.GetMatchersCondition() +func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc) (*Result, bool) { + matcherCondition := operators.GetMatchersCondition() var matches bool result := &Result{ @@ -117,7 +117,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac } // Start with the extractors first and evaluate them. - for _, extractor := range r.Extractors { + for _, extractor := range operators.Extractors { var extractorResults []string for match := range extract(data, extractor) { @@ -136,7 +136,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac } } - for _, matcher := range r.Matchers { + for _, matcher := range operators.Matchers { // Check if the matcher matched if isMatch, matched := match(data, matcher); isMatch { // If the matcher has matched, and it's an OR @@ -161,7 +161,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac } // Don't print if we have matchers, and they have not matched, regardless of extractor - if len(r.Matchers) > 0 && !matches { + if len(operators.Matchers) > 0 && !matches { return nil, false } // Write a final string of output if matcher type is @@ -173,11 +173,11 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac } // ExecuteInternalExtractors executes internal dynamic extractors -func (r *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} { +func (operators *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} { dynamicValues := make(map[string]interface{}) // Start with the extractors first and evaluate them. - for _, extractor := range r.Extractors { + for _, extractor := range operators.Extractors { if !extractor.Internal { continue } diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 4c2b8904f..ff8d907c9 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -75,42 +75,42 @@ type Request struct { } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { +func (request *Request) Compile(options *protocols.ExecuterOptions) error { // Create a dns client for the class client, err := dnsclientpool.Get(options.Options, &dnsclientpool.Configuration{ - Retries: r.Retries, + Retries: request.Retries, }) if err != nil { return errors.Wrap(err, "could not get dns client") } - r.dnsClient = client + request.dnsClient = client - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } - r.class = classToInt(r.Class) - r.options = options - r.question = questionTypeToInt(r.Type) + request.class = classToInt(request.Class) + request.options = options + request.question = questionTypeToInt(request.Type) return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } // Make returns the request to be sent for the protocol -func (r *Request) Make(domain string) (*dns.Msg, error) { - if r.question != dns.TypePTR && net.ParseIP(domain) != nil { +func (request *Request) Make(domain string) (*dns.Msg, error) { + if request.question != dns.TypePTR && net.ParseIP(domain) != nil { return nil, errors.New("cannot use IP address as DNS input") } domain = dns.Fqdn(domain) @@ -118,20 +118,20 @@ func (r *Request) Make(domain string) (*dns.Msg, error) { // Build a request on the specified URL req := new(dns.Msg) req.Id = dns.Id() - req.RecursionDesired = r.Recursion + req.RecursionDesired = request.Recursion var q dns.Question - final := replacer.Replace(r.Name, map[string]interface{}{"FQDN": domain}) + final := replacer.Replace(request.Name, map[string]interface{}{"FQDN": domain}) q.Name = dns.Fqdn(final) - q.Qclass = r.class - q.Qtype = r.question + q.Qclass = request.class + q.Qtype = request.question req.Question = append(req.Question, q) req.SetEdns0(4096, false) - switch r.question { + switch request.question { case dns.TypeTXT: req.AuthenticatedData = true } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 3b86e234b..22da3a11a 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -14,7 +14,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "": @@ -44,7 +44,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { part := extractor.Part switch part { case "body", "all": @@ -67,7 +67,7 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext } // responseToDSLMap converts a DNS response to a map for use in DSL matching -func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent { +func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent { data := make(output.InternalEvent, 11) // Some data regarding the request metadata @@ -103,14 +103,14 @@ func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) out rawData := resp.String() data["raw"] = rawData - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath + data["template-id"] = request.options.TemplateID + data["template-info"] = request.options.TemplateInfo + data["template-path"] = request.options.TemplatePath return data } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { if len(wrapped.OperatorsResult.DynamicValues) > 0 { return nil } @@ -119,25 +119,25 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu // If we have multiple matchers with names, write each of them separately. if len(wrapped.OperatorsResult.Matches) > 0 { for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) data.MatcherName = k results = append(results, data) } } else if len(wrapped.OperatorsResult.Extracts) > 0 { for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) data.ExtractedResults = v data.ExtractorName = k results = append(results, data) } } else { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) results = append(results, data) } return results } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index ee5217b56..00a31efe4 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -14,7 +14,7 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // Parse the URL and return domain if URL. var domain string if isURL(input) { @@ -24,42 +24,42 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused } // Compile each request for the template based on the URL - compiledRequest, err := r.Make(domain) + compiledRequest, err := request.Make(domain) if err != nil { - r.options.Output.Request(r.options.TemplateID, domain, "dns", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not build request") } - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", r.options.TemplateID, domain) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", request.options.TemplateID, domain) gologger.Print().Msgf("%s", compiledRequest.String()) } // Send the request to the target servers - resp, err := r.dnsClient.Do(compiledRequest) + resp, err := request.dnsClient.Do(compiledRequest) if err != nil { - r.options.Output.Request(r.options.TemplateID, domain, "dns", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + request.options.Progress.IncrementFailedRequestsBy(1) } if resp == nil { return errors.Wrap(err, "could not send dns request") } - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() - r.options.Output.Request(r.options.TemplateID, domain, "dns", err) - gologger.Verbose().Msgf("[%s] Sent DNS request to %s", r.options.TemplateID, domain) + request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + gologger.Verbose().Msgf("[%s] Sent DNS request to %s", request.options.TemplateID, domain) - outputEvent := r.responseToDSLMap(compiledRequest, resp, input, input) + outputEvent := request.responseToDSLMap(compiledRequest, resp, input, input) for k, v := range previous { outputEvent[k] = v } - event := createEvent(r, outputEvent) + event := createEvent(request, outputEvent) - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, resp.String(), r.options.Options.NoColor)) + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, resp.String(), request.options.Options.NoColor)) } callback(event) diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go index 2d1e64393..e61965025 100644 --- a/v2/pkg/protocols/file/file.go +++ b/v2/pkg/protocols/file/file.go @@ -56,54 +56,54 @@ type Request struct { var defaultDenylist = []string{".3g2", ".3gp", ".7z", ".apk", ".arj", ".avi", ".axd", ".bmp", ".css", ".csv", ".deb", ".dll", ".doc", ".drv", ".eot", ".exe", ".flv", ".gif", ".gifv", ".gz", ".h264", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lock", ".m4a", ".m4v", ".map", ".mkv", ".mov", ".mp3", ".mp4", ".mpeg", ".mpg", ".msi", ".ogg", ".ogm", ".ogv", ".otf", ".pdf", ".pkg", ".png", ".ppt", ".psd", ".rar", ".rm", ".rpm", ".svg", ".swf", ".sys", ".tar.gz", ".tar", ".tif", ".tiff", ".ttf", ".vob", ".wav", ".webm", ".wmv", ".woff", ".woff2", ".xcf", ".xls", ".xlsx", ".zip"} // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators +func (request *Request) Compile(options *protocols.ExecuterOptions) error { + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } // By default, use 5 MB as max size to read. - if r.MaxSize == 0 { - r.MaxSize = 5 * 1024 * 1024 + if request.MaxSize == 0 { + request.MaxSize = 5 * 1024 * 1024 } - r.options = options + request.options = options - r.extensions = make(map[string]struct{}) - r.extensionDenylist = make(map[string]struct{}) + request.extensions = make(map[string]struct{}) + request.extensionDenylist = make(map[string]struct{}) - for _, extension := range r.Extensions { + for _, extension := range request.Extensions { if extension == "all" { - r.allExtensions = true + request.allExtensions = true } else { if !strings.HasPrefix(extension, ".") { extension = "." + extension } - r.extensions[extension] = struct{}{} + request.extensions[extension] = struct{}{} } } for _, extension := range defaultDenylist { if !strings.HasPrefix(extension, ".") { extension = "." + extension } - r.extensionDenylist[extension] = struct{}{} + request.extensionDenylist[extension] = struct{}{} } - for _, extension := range r.ExtensionDenylist { + for _, extension := range request.ExtensionDenylist { if !strings.HasPrefix(extension, ".") { extension = "." + extension } - r.extensionDenylist[extension] = struct{}{} + request.extensionDenylist[extension] = struct{}{} } return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } diff --git a/v2/pkg/protocols/file/find.go b/v2/pkg/protocols/file/find.go index ec1cc2585..f4deaa100 100644 --- a/v2/pkg/protocols/file/find.go +++ b/v2/pkg/protocols/file/find.go @@ -7,50 +7,51 @@ import ( "github.com/karrick/godirwalk" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" ) // getInputPaths parses the specified input paths and returns a compiled // list of finished absolute paths to the files evaluating any allowlist, denylist, // glob, file or folders, etc. -func (r *Request) getInputPaths(target string, callback func(string)) error { +func (request *Request) getInputPaths(target string, callback func(string)) error { processed := make(map[string]struct{}) // Template input includes a wildcard - if strings.Contains(target, "*") && !r.NoRecursive { - if err := r.findGlobPathMatches(target, processed, callback); err != nil { + if strings.Contains(target, "*") && !request.NoRecursive { + if err := request.findGlobPathMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find glob matches") } return nil } // Template input is either a file or a directory - file, err := r.findFileMatches(target, processed, callback) + file, err := request.findFileMatches(target, processed, callback) if err != nil { return errors.Wrap(err, "could not find file") } if file { return nil } - if r.NoRecursive { + if request.NoRecursive { return nil // we don't process dirs in no-recursive mode } // Recursively walk down the Templates directory and run all // the template file checks - if err := r.findDirectoryMatches(target, processed, callback); err != nil { + if err := request.findDirectoryMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find directory matches") } return nil } // findGlobPathMatches returns the matched files from a glob path -func (r *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error { +func (request *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error { matches, err := filepath.Glob(absPath) if err != nil { return errors.Errorf("wildcard found, but unable to glob: %s\n", err) } for _, match := range matches { - if !r.validatePath(match) { + if !request.validatePath(match) { continue } if _, ok := processed[match]; !ok { @@ -63,7 +64,7 @@ func (r *Request) findGlobPathMatches(absPath string, processed map[string]struc // findFileMatches finds if a path is an absolute file. If the path // is a file, it returns true otherwise false with no errors. -func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) { +func (request *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) { info, err := os.Stat(absPath) if err != nil { return false, err @@ -72,7 +73,7 @@ func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, return false, nil } if _, ok := processed[absPath]; !ok { - if !r.validatePath(absPath) { + if !request.validatePath(absPath) { return false, nil } processed[absPath] = struct{}{} @@ -82,7 +83,7 @@ func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, } // findDirectoryMatches finds matches for templates from a directory -func (r *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { +func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { err := godirwalk.Walk(absPath, &godirwalk.Options{ Unsorted: true, ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { @@ -92,7 +93,7 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru if d.IsDir() { return nil } - if !r.validatePath(path) { + if !request.validatePath(path) { return nil } if _, ok := processed[path]; !ok { @@ -106,17 +107,17 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru } // validatePath validates a file path for blacklist and whitelist options -func (r *Request) validatePath(item string) bool { +func (request *Request) validatePath(item string) bool { extension := filepath.Ext(item) - if len(r.extensions) > 0 { - if _, ok := r.extensions[extension]; ok { + if len(request.extensions) > 0 { + if _, ok := request.extensions[extension]; ok { return true - } else if !r.allExtensions { + } else if !request.allExtensions { return false } } - if _, ok := r.extensionDenylist[extension]; ok { + if _, ok := request.extensionDenylist[extension]; ok { gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, extension) return false } diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 11a804cab..c0c5cd20c 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -13,7 +13,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "data", "": @@ -42,7 +42,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { partString := extractor.Part switch partString { case "body", "all", "data", "": @@ -65,21 +65,21 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext } // responseToDSLMap converts a DNS response to a map for use in DSL matching -func (r *Request) responseToDSLMap(raw, host, matched string) output.InternalEvent { +func (request *Request) responseToDSLMap(raw, host, matched string) output.InternalEvent { data := make(output.InternalEvent, 5) // Some data regarding the request metadata data["path"] = host data["matched"] = matched data["raw"] = raw - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath + data["template-id"] = request.options.TemplateID + data["template-info"] = request.options.TemplateInfo + data["template-path"] = request.options.TemplatePath return data } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { if len(wrapped.OperatorsResult.DynamicValues) > 0 { return nil } @@ -88,19 +88,19 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu // If we have multiple matchers with names, write each of them separately. if len(wrapped.OperatorsResult.Matches) > 0 { for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) data.MatcherName = k results = append(results, data) } } else if len(wrapped.OperatorsResult.Extracts) > 0 { for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) data.ExtractedResults = v data.ExtractorName = k results = append(results, data) } } else { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) results = append(results, data) } raw, ok := wrapped.InternalEvent["raw"] @@ -133,7 +133,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu return results } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 7a36b04e4..3118a8466 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -17,10 +17,10 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - wg := sizedwaitgroup.New(r.options.Options.BulkSize) +func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + wg := sizedwaitgroup.New(request.options.Options.BulkSize) - err := r.getInputPaths(input, func(data string) { + err := request.getInputPaths(input, func(data string) { wg.Add() go func(filePath string) { @@ -38,7 +38,7 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused gologger.Error().Msgf("Could not stat file path %s: %s\n", filePath, err) return } - if stat.Size() >= int64(r.MaxSize) { + if stat.Size() >= int64(request.MaxSize) { gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", filePath) return } @@ -50,17 +50,17 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused } dataStr := tostring.UnsafeToString(buffer) - gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, filePath) - outputEvent := r.responseToDSLMap(dataStr, input, filePath) + gologger.Verbose().Msgf("[%s] Sent FILE request to %s", request.options.TemplateID, filePath) + outputEvent := request.responseToDSLMap(dataStr, input, filePath) for k, v := range previous { outputEvent[k] = v } - event := createEvent(r, outputEvent) + event := createEvent(request, outputEvent) - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, filePath) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, dataStr, r.options.Options.NoColor)) + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, dataStr, request.options.Options.NoColor)) } callback(event) @@ -68,11 +68,11 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused }) wg.Wait() if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "file", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "file", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() return nil } diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 8af250b2b..43f62f437 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -32,24 +32,24 @@ type Step struct { } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators +func (request *Request) Compile(options *protocols.ExecuterOptions) error { + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } - r.options = options + request.options = options return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index 1053acccb..fc4eac43c 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -11,7 +11,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "resp", "": @@ -40,7 +40,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { partString := extractor.Part switch partString { case "body", "resp", "": @@ -63,7 +63,7 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext } // responseToDSLMap converts a DNS response to a map for use in DSL matching -func (r *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent { +func (request *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent { data := make(output.InternalEvent, 5) // Some data regarding the request metadata @@ -71,14 +71,14 @@ func (r *Request) responseToDSLMap(resp, req, host, matched string) output.Inter data["matched"] = matched data["req"] = req data["data"] = resp - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath + data["template-id"] = request.options.TemplateID + data["template-info"] = request.options.TemplateInfo + data["template-path"] = request.options.TemplatePath return data } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { if len(wrapped.OperatorsResult.DynamicValues) > 0 { return nil } @@ -87,25 +87,25 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu // If we have multiple matchers with names, write each of them separately. if len(wrapped.OperatorsResult.Matches) > 0 { for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) data.MatcherName = k results = append(results, data) } } else if len(wrapped.OperatorsResult.Extracts) > 0 { for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) data.ExtractedResults = v data.ExtractorName = k results = append(results, data) } } else { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) results = append(results, data) } return results } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 2ce8cf7ee..62734e879 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -16,38 +16,38 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { - instance, err := r.options.Browser.NewInstance() +func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { + instance, err := request.options.Browser.NewInstance() if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "headless", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer instance.Close() parsed, err := url.Parse(input) if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "headless", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } - out, page, err := instance.Run(parsed, r.Steps, time.Duration(r.options.Options.PageTimeout)*time.Second) + out, page, err := instance.Run(parsed, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second) if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "headless", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer page.Close() - r.options.Output.Request(r.options.TemplateID, input, "headless", nil) - r.options.Progress.IncrementRequests() + request.options.Output.Request(request.options.TemplateID, input, "headless", nil) + request.options.Progress.IncrementRequests() gologger.Verbose().Msgf("Sent Headless request to %s", input) reqBuilder := &strings.Builder{} - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped Headless request for %s", r.options.TemplateID, input) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input) - for _, act := range r.Steps { + for _, act := range request.Steps { reqBuilder.WriteString(act.String()) reqBuilder.WriteString("\n") } @@ -59,16 +59,16 @@ func (r *Request) ExecuteWithResults(input string, metadata, previous output.Int if err == nil { responseBody, _ = html.HTML() } - outputEvent := r.responseToDSLMap(responseBody, reqBuilder.String(), input, input) + outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), input, input) for k, v := range out { outputEvent[k] = v } - event := createEvent(r, outputEvent) + event := createEvent(request, outputEvent) - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Headless response for %s", r.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, r.options.Options.NoColor)) + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor)) } callback(event) diff --git a/v2/pkg/protocols/http/cluster.go b/v2/pkg/protocols/http/cluster.go index bdfa07358..8c1674be3 100644 --- a/v2/pkg/protocols/http/cluster.go +++ b/v2/pkg/protocols/http/cluster.go @@ -9,20 +9,20 @@ import ( // This used by the clustering engine to decide whether two requests // are similar enough to be considered one and can be checked by // just adding the matcher/extractors for the request and the correct IDs. -func (r *Request) CanCluster(other *Request) bool { - if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe || r.ReqCondition || r.Name != "" { +func (request *Request) CanCluster(other *Request) bool { + if len(request.Payloads) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.ReqCondition || request.Name != "" { return false } - if r.Method != other.Method || - r.MaxRedirects != other.MaxRedirects || - r.CookieReuse != other.CookieReuse || - r.Redirects != other.Redirects { + if request.Method != other.Method || + request.MaxRedirects != other.MaxRedirects || + request.CookieReuse != other.CookieReuse || + request.Redirects != other.Redirects { return false } - if !compare.StringSlice(r.Path, other.Path) { + if !compare.StringSlice(request.Path, other.Path) { return false } - if !compare.StringMap(r.Headers, other.Headers) { + if !compare.StringMap(request.Headers, other.Headers) { return false } return true diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 5558e7035..85bec5c56 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -170,21 +170,21 @@ type Request struct { } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { +func (request *Request) Compile(options *protocols.ExecuterOptions) error { connectionConfiguration := &httpclientpool.Configuration{ - Threads: r.Threads, - MaxRedirects: r.MaxRedirects, - FollowRedirects: r.Redirects, - CookieReuse: r.CookieReuse, + Threads: request.Threads, + MaxRedirects: request.MaxRedirects, + FollowRedirects: request.Redirects, + CookieReuse: request.CookieReuse, } // if the headers contain "Connection" we need to disable the automatic keep alive of the standard library - if _, hasConnectionHeader := r.Headers["Connection"]; hasConnectionHeader { + if _, hasConnectionHeader := request.Headers["Connection"]; hasConnectionHeader { connectionConfiguration.Connection = &httpclientpool.ConnectionConfiguration{DisableKeepAlive: false} } @@ -192,76 +192,76 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if err != nil { return errors.Wrap(err, "could not get dns client") } - r.customHeaders = make(map[string]string) - r.httpClient = client - r.options = options - for _, option := range r.options.Options.CustomHeaders { + request.customHeaders = make(map[string]string) + request.httpClient = client + request.options = options + for _, option := range request.options.Options.CustomHeaders { parts := strings.SplitN(option, ":", 2) if len(parts) != 2 { continue } - r.customHeaders[parts[0]] = strings.TrimSpace(parts[1]) + request.customHeaders[parts[0]] = strings.TrimSpace(parts[1]) } - if r.Body != "" && !strings.Contains(r.Body, "\r\n") { - r.Body = strings.ReplaceAll(r.Body, "\n", "\r\n") + if request.Body != "" && !strings.Contains(request.Body, "\r\n") { + request.Body = strings.ReplaceAll(request.Body, "\n", "\r\n") } - if len(r.Raw) > 0 { - for i, raw := range r.Raw { + if len(request.Raw) > 0 { + for i, raw := range request.Raw { if !strings.Contains(raw, "\r\n") { - r.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n") + request.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n") } } - r.rawhttpClient = httpclientpool.GetRawHTTP(options.Options) + request.rawhttpClient = httpclientpool.GetRawHTTP(options.Options) } - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if compileErr := compiled.Compile(); compileErr != nil { return errors.Wrap(compileErr, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } - if len(r.Payloads) > 0 { - attackType := r.AttackType + if len(request.Payloads) > 0 { + attackType := request.AttackType if attackType == "" { attackType = "sniper" } - r.attackType = generators.StringToType[attackType] + request.attackType = generators.StringToType[attackType] // Resolve payload paths if they are files. - for name, payload := range r.Payloads { + for name, payload := range request.Payloads { payloadStr, ok := payload.(string) if ok { final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) if resolveErr != nil { return errors.Wrap(resolveErr, "could not read payload file") } - r.Payloads[name] = final + request.Payloads[name] = final } } - r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath) + request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) if err != nil { return errors.Wrap(err, "could not parse payloads") } } - r.options = options - r.totalRequests = r.Requests() + request.options = options + request.totalRequests = request.Requests() return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { - if r.generator != nil { - payloadRequests := r.generator.NewIterator().Total() * len(r.Raw) +func (request *Request) Requests() int { + if request.generator != nil { + payloadRequests := request.generator.NewIterator().Total() * len(request.Raw) return payloadRequests } - if len(r.Raw) > 0 { - requests := len(r.Raw) - if requests == 1 && r.RaceNumberRequests != 0 { - requests *= r.RaceNumberRequests + if len(request.Raw) > 0 { + requests := len(request.Raw) + if requests == 1 && request.RaceNumberRequests != 0 { + requests *= request.RaceNumberRequests } return requests } - return len(r.Path) + return len(request.Path) } diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index b35bae8e4..83978acc9 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -14,7 +14,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { item, ok := getMatchPart(matcher.Part, data) if !ok { return false, []string{} @@ -34,7 +34,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: - return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, r.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, request.dynamicValues)) case matchers.RegexMatcher: return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item)) case matchers.BinaryMatcher: @@ -46,7 +46,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { item, ok := getMatchPart(extractor.Part, data) if !ok { return nil @@ -87,7 +87,7 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) { } // responseToDSLMap converts an HTTP response to a map for use in DSL matching -func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { +func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies())) for k, v := range extra { data[k] = v @@ -108,14 +108,14 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data["content_length"] = resp.ContentLength data["all_headers"] = headers data["duration"] = duration.Seconds() - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath + data["template-id"] = request.options.TemplateID + data["template-info"] = request.options.TemplateInfo + data["template-path"] = request.options.TemplatePath return data } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { if len(wrapped.OperatorsResult.DynamicValues) > 0 && !wrapped.OperatorsResult.Matched { return nil } @@ -125,25 +125,25 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu // If we have multiple matchers with names, write each of them separately. if len(wrapped.OperatorsResult.Matches) > 0 { for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) data.MatcherName = k results = append(results, data) } } else if len(wrapped.OperatorsResult.Extracts) > 0 { for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) data.ExtractedResults = v data.ExtractorName = k results = append(results, data) } } else { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) results = append(results, data) } return results } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index d3405fdba..3206a5f54 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -31,52 +31,52 @@ import ( const defaultMaxWorkers = 150 // executeRaceRequest executes race condition request for a URL -func (r *Request) executeRaceRequest(reqURL string, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - var requests []*generatedRequest +func (request *Request) executeRaceRequest(reqURL string, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + var generatedRequests []*generatedRequest // Requests within race condition should be dumped once and the output prefilled to allow DSL language to work // This will introduce a delay and will populate in hacky way the field "request" of outputEvent - generator := r.newGenerator() + generator := request.newGenerator() requestForDump, err := generator.Make(reqURL, nil, "") if err != nil { return err } - r.setCustomHeaders(requestForDump) + request.setCustomHeaders(requestForDump) dumpedRequest, err := dump(requestForDump, reqURL) if err != nil { return err } - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL) gologger.Print().Msgf("%s", string(dumpedRequest)) } previous["request"] = string(dumpedRequest) // Pre-Generate requests - for i := 0; i < r.RaceNumberRequests; i++ { - generator := r.newGenerator() - request, err := generator.Make(reqURL, nil, "") + for i := 0; i < request.RaceNumberRequests; i++ { + generator := request.newGenerator() + generatedRequest, err := generator.Make(reqURL, nil, "") if err != nil { return err } - requests = append(requests, request) + generatedRequests = append(generatedRequests, generatedRequest) } wg := sync.WaitGroup{} var requestErr error mutex := &sync.Mutex{} - for i := 0; i < r.RaceNumberRequests; i++ { + for i := 0; i < request.RaceNumberRequests; i++ { wg.Add(1) go func(httpRequest *generatedRequest) { defer wg.Done() - err := r.executeRequest(reqURL, httpRequest, previous, false, callback, 0) + err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) } mutex.Unlock() - }(requests[i]) - r.options.Progress.IncrementRequests() + }(generatedRequests[i]) + request.options.Progress.IncrementRequests() } wg.Wait() @@ -84,47 +84,47 @@ func (r *Request) executeRaceRequest(reqURL string, previous output.InternalEven } // executeRaceRequest executes parallel requests for a template -func (r *Request) executeParallelHTTP(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error { - generator := r.newGenerator() +func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error { + generator := request.newGenerator() // Workers that keeps enqueuing new requests - maxWorkers := r.Threads + maxWorkers := request.Threads swg := sizedwaitgroup.New(maxWorkers) var requestErr error mutex := &sync.Mutex{} for { - request, err := generator.Make(reqURL, dynamicValues, "") + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "") if err == io.EOF { break } if err != nil { - r.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } swg.Add() go func(httpRequest *generatedRequest) { defer swg.Done() - r.options.RateLimiter.Take() + request.options.RateLimiter.Take() previous := make(map[string]interface{}) - err := r.executeRequest(reqURL, httpRequest, previous, false, callback, 0) + err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) } mutex.Unlock() - }(request) - r.options.Progress.IncrementRequests() + }(generatedHttpRequest) + request.options.Progress.IncrementRequests() } swg.Wait() return requestErr } // executeTurboHTTP executes turbo http request for a URL -func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - generator := r.newGenerator() +func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + generator := request.newGenerator() // need to extract the target from the url URL, err := url.Parse(reqURL) @@ -135,13 +135,13 @@ func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output pipeOptions := rawhttp.DefaultPipelineOptions pipeOptions.Host = URL.Host pipeOptions.MaxConnections = 1 - if r.PipelineConcurrentConnections > 0 { - pipeOptions.MaxConnections = r.PipelineConcurrentConnections + if request.PipelineConcurrentConnections > 0 { + pipeOptions.MaxConnections = request.PipelineConcurrentConnections } - if r.PipelineRequestsPerConnection > 0 { - pipeOptions.MaxPendingRequests = r.PipelineRequestsPerConnection + if request.PipelineRequestsPerConnection > 0 { + pipeOptions.MaxPendingRequests = request.PipelineRequestsPerConnection } - pipeclient := rawhttp.NewPipelineClient(pipeOptions) + pipeClient := rawhttp.NewPipelineClient(pipeOptions) // defaultMaxWorkers should be a sufficient value to keep queues always full maxWorkers := defaultMaxWorkers @@ -154,106 +154,106 @@ func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output var requestErr error mutex := &sync.Mutex{} for { - request, err := generator.Make(reqURL, dynamicValues, "") + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "") if err == io.EOF { break } if err != nil { - r.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } - request.pipelinedClient = pipeclient + generatedHttpRequest.pipelinedClient = pipeClient swg.Add() go func(httpRequest *generatedRequest) { defer swg.Done() - err := r.executeRequest(reqURL, httpRequest, previous, false, callback, 0) + err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) } mutex.Unlock() - }(request) - r.options.Progress.IncrementRequests() + }(generatedHttpRequest) + request.options.Progress.IncrementRequests() } swg.Wait() return requestErr } // ExecuteWithResults executes the final request on a URL -func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // verify if pipeline was requested - if r.Pipeline { - return r.executeTurboHTTP(reqURL, dynamicValues, previous, callback) + if request.Pipeline { + return request.executeTurboHTTP(reqURL, dynamicValues, previous, callback) } // verify if a basic race condition was requested - if r.Race && r.RaceNumberRequests > 0 { - return r.executeRaceRequest(reqURL, previous, callback) + if request.Race && request.RaceNumberRequests > 0 { + return request.executeRaceRequest(reqURL, previous, callback) } // verify if parallel elaboration was requested - if r.Threads > 0 { - return r.executeParallelHTTP(reqURL, dynamicValues, callback) + if request.Threads > 0 { + return request.executeParallelHTTP(reqURL, dynamicValues, callback) } - generator := r.newGenerator() + generator := request.newGenerator() requestCount := 1 var requestErr error for { - hasInteractMarkers := interactsh.HasMatchers(r.CompiledOperators) + hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) var interactURL string - if r.options.Interactsh != nil && hasInteractMarkers { - interactURL = r.options.Interactsh.URL() + if request.options.Interactsh != nil && hasInteractMarkers { + interactURL = request.options.Interactsh.URL() } - request, err := generator.Make(reqURL, dynamicValues, interactURL) + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, interactURL) if err == io.EOF { break } if err != nil { - r.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } - r.dynamicValues = request.dynamicValues + request.dynamicValues = generatedHttpRequest.dynamicValues // Check if hosts just keep erroring - if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.Check(reqURL) { + if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { break } var gotOutput bool - r.options.RateLimiter.Take() - err = r.executeRequest(reqURL, request, previous, hasInteractMarkers, func(event *output.InternalWrappedEvent) { + request.options.RateLimiter.Take() + err = request.executeRequest(reqURL, generatedHttpRequest, previous, hasInteractMarkers, func(event *output.InternalWrappedEvent) { // Add the extracts to the dynamic values if any. if event.OperatorsResult != nil { gotOutput = true dynamicValues = generators.MergeMaps(dynamicValues, event.OperatorsResult.DynamicValues) } - if hasInteractMarkers && r.options.Interactsh != nil { - r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ - MakeResultFunc: r.MakeResultEvent, + if hasInteractMarkers && request.options.Interactsh != nil { + request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ + MakeResultFunc: request.MakeResultEvent, Event: event, - Operators: r.CompiledOperators, - MatchFunc: r.Match, - ExtractFunc: r.Extract, + Operators: request.CompiledOperators, + MatchFunc: request.Match, + ExtractFunc: request.Extract, }) } else { callback(event) } }, requestCount) if err != nil { - if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.CheckError(err) { - r.options.HostErrorsCache.MarkFailed(reqURL) + if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.CheckError(err) { + request.options.HostErrorsCache.MarkFailed(reqURL) } requestErr = err } requestCount++ - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() - if (request.original.options.Options.StopAtFirstMatch || r.StopAtFirstMatch) && gotOutput { - r.options.Progress.IncrementErrorsBy(int64(generator.Total())) + if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput { + request.options.Progress.IncrementErrorsBy(int64(generator.Total())) break } } @@ -263,12 +263,12 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp const drainReqSize = int64(8 * 1024) // executeRequest executes the actual generated request and returns error if occurred -func (r *Request) executeRequest(reqURL string, request *generatedRequest, previous output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error { - r.setCustomHeaders(request) +func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previous output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error { + request.setCustomHeaders(generatedRequest) var ( resp *http.Response - fromcache bool + fromCache bool dumpedRequest []byte err error ) @@ -276,75 +276,75 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ var formedURL string var hostname string timeStart := time.Now() - if request.original.Pipeline { - if request.rawRequest != nil { - formedURL = request.rawRequest.FullURL + if generatedRequest.original.Pipeline { + if generatedRequest.rawRequest != nil { + formedURL = generatedRequest.rawRequest.FullURL if parsed, parseErr := url.Parse(formedURL); parseErr == nil { hostname = parsed.Host } - resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data))) - } else if request.request != nil { - resp, err = request.pipelinedClient.Dor(request.request) + resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data))) + } else if generatedRequest.request != nil { + resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request) } - } else if request.original.Unsafe && request.rawRequest != nil { - formedURL = request.rawRequest.FullURL + } else if generatedRequest.original.Unsafe && generatedRequest.rawRequest != nil { + formedURL = generatedRequest.rawRequest.FullURL if parsed, parseErr := url.Parse(formedURL); parseErr == nil { hostname = parsed.Host } - options := request.original.rawhttpClient.Options - options.FollowRedirects = r.Redirects - options.CustomRawBytes = request.rawRequest.UnsafeRawBytes - resp, err = request.original.rawhttpClient.DoRawWithOptions(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)), options) + options := generatedRequest.original.rawhttpClient.Options + options.FollowRedirects = request.Redirects + options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes + resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), options) } else { - hostname = request.request.URL.Host - formedURL = request.request.URL.String() + hostname = generatedRequest.request.URL.Host + formedURL = generatedRequest.request.URL.String() // if nuclei-project is available check if the request was already sent previously - if r.options.ProjectFile != nil { + if request.options.ProjectFile != nil { // if unavailable fail silently - fromcache = true - resp, err = r.options.ProjectFile.Get(dumpedRequest) + fromCache = true + resp, err = request.options.ProjectFile.Get(dumpedRequest) if err != nil { - fromcache = false + fromCache = false } } if resp == nil { - resp, err = r.httpClient.Do(request.request) + resp, err = request.httpClient.Do(generatedRequest.request) } } // For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function - if !request.original.Race { + if !generatedRequest.original.Race { var dumpError error - dumpedRequest, dumpError = dump(request, reqURL) + dumpedRequest, dumpError = dump(generatedRequest, reqURL) if dumpError != nil { return dumpError } - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL) gologger.Print().Msgf("%s", string(dumpedRequest)) } } if err != nil { // rawhttp doesn't support draining response bodies. - if resp != nil && resp.Body != nil && request.rawRequest == nil { + if resp != nil && resp.Body != nil && generatedRequest.rawRequest == nil { _, _ = io.CopyN(ioutil.Discard, resp.Body, drainReqSize) resp.Body.Close() } - r.options.Output.Request(r.options.TemplateID, formedURL, "http", err) - r.options.Progress.IncrementErrorsBy(1) + request.options.Output.Request(request.options.TemplateID, formedURL, "http", err) + request.options.Progress.IncrementErrorsBy(1) // If we have interactsh markers and request times out, still send // a callback event so in case we receive an interaction, correlation is possible. if hasInteractMarkers { - outputEvent := r.responseToDSLMap(&http.Response{}, reqURL, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, request.meta) + outputEvent := request.responseToDSLMap(&http.Response{}, reqURL, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { + if request.CompiledOperators != nil { event.InternalEvent = outputEvent } callback(event) @@ -358,8 +358,8 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ resp.Body.Close() }() - gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", r.options.TemplateID, formedURL) - r.options.Output.Request(r.options.TemplateID, formedURL, "http", err) + gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", request.options.TemplateID, formedURL) + request.options.Output.Request(request.options.TemplateID, formedURL, "http", err) duration := time.Since(timeStart) @@ -372,8 +372,8 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ // If the status code is HTTP 101, we should not proceed with reading body. if resp.StatusCode != http.StatusSwitchingProtocols { var bodyReader io.Reader - if r.MaxSize != 0 { - bodyReader = io.LimitReader(resp.Body, int64(r.MaxSize)) + if request.MaxSize != 0 { + bodyReader = io.LimitReader(resp.Body, int64(request.MaxSize)) } else { bodyReader = resp.Body } @@ -381,7 +381,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ if err != nil { // Ignore body read due to server misconfiguration errors if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") { - gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", r.options.TemplateID, formedURL, err.Error()) + gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", request.options.TemplateID, formedURL, err.Error()) } else if !stringsutil.ContainsAny(err.Error(), "unexpected EOF", "user canceled") { // ignore EOF and random error return errors.Wrap(err, "could not read http body") } @@ -426,22 +426,22 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ } // if nuclei-project is enabled store the response if not previously done - if r.options.ProjectFile != nil && !fromcache { - if err := r.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil { + if request.options.ProjectFile != nil && !fromCache { + if err := request.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil { return errors.Wrap(err, "could not store in project file") } } matchedURL := reqURL - if request.rawRequest != nil && request.rawRequest.FullURL != "" { - matchedURL = request.rawRequest.FullURL + if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { + matchedURL = generatedRequest.rawRequest.FullURL } - if request.request != nil { - matchedURL = request.request.URL.String() + if generatedRequest.request != nil { + matchedURL = generatedRequest.request.URL.String() } finalEvent := make(output.InternalEvent) - outputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta) + outputEvent := request.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, generatedRequest.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } @@ -454,7 +454,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ finalEvent[k] = v } // Add to history the current request number metadata if asked by the user. - if r.ReqCondition { + if request.ReqCondition { for k, v := range outputEvent { key := fmt.Sprintf("%s_%d", k, requestCount) previous[key] = v @@ -462,11 +462,11 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ } } - event := createEvent(r, outputEvent, finalEvent, request) + event := createEvent(request, outputEvent, finalEvent, generatedRequest) - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, formedURL) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, string(redirectedResponse), r.options.Options.NoColor)) + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, string(redirectedResponse), request.options.Options.NoColor)) } callback(event) @@ -487,8 +487,8 @@ func createEvent(request *Request, outputEvent output.InternalEvent, finalEvent } // setCustomHeaders sets the custom headers for generated request -func (r *Request) setCustomHeaders(req *generatedRequest) { - for k, v := range r.customHeaders { +func (request *Request) setCustomHeaders(req *generatedRequest) { + for k, v := range request.customHeaders { if req.rawRequest != nil { req.rawRequest.Headers[k] = v } else { diff --git a/v2/pkg/protocols/http/request_generator.go b/v2/pkg/protocols/http/request_generator.go index 236225921..90c5d0fbd 100644 --- a/v2/pkg/protocols/http/request_generator.go +++ b/v2/pkg/protocols/http/request_generator.go @@ -19,11 +19,11 @@ type requestGenerator struct { } // newGenerator creates a new request generator instance -func (r *Request) newGenerator() *requestGenerator { - generator := &requestGenerator{request: r, options: r.options} +func (request *Request) newGenerator() *requestGenerator { + generator := &requestGenerator{request: request, options: request.options} - if len(r.Payloads) > 0 { - generator.payloadIterator = r.generator.NewIterator() + if len(request.Payloads) > 0 { + generator.payloadIterator = request.generator.NewIterator() } return generator } @@ -63,7 +63,6 @@ func (r *requestGenerator) nextValue() (value string, payloads map[string]interf } } - if len(r.request.Raw) > 0 && r.currentIndex < len(r.request.Raw) { if r.payloadIterator != nil { payload, ok := r.payloadIterator.Value() diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 3ad431ed0..08ae41fcd 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -114,17 +114,17 @@ type Input struct { } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { +func (request *Request) Compile(options *protocols.ExecuterOptions) error { var shouldUseTLS bool var err error - r.options = options - for _, address := range r.Address { + request.options = options + for _, address := range request.Address { // check if the connection should be encrypted if strings.HasPrefix(address, "tls://") { shouldUseTLS = true @@ -135,13 +135,13 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if portErr != nil { return errors.Wrap(portErr, "could not parse address") } - r.addresses = append(r.addresses, addressKV{ip: addressHost, port: addressPort, tls: shouldUseTLS}) + request.addresses = append(request.addresses, addressKV{ip: addressHost, port: addressPort, tls: shouldUseTLS}) } else { - r.addresses = append(r.addresses, addressKV{ip: address, tls: shouldUseTLS}) + request.addresses = append(request.addresses, addressKV{ip: address, tls: shouldUseTLS}) } } // Pre-compile any input dsl functions before executing the request. - for _, input := range r.Inputs { + for _, input := range request.Inputs { if input.Type != "" { continue } @@ -150,25 +150,25 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } } - if len(r.Payloads) > 0 { - attackType := r.AttackType + if len(request.Payloads) > 0 { + attackType := request.AttackType if attackType == "" { attackType = "sniper" } - r.attackType = generators.StringToType[attackType] + request.attackType = generators.StringToType[attackType] // Resolve payload paths if they are files. - for name, payload := range r.Payloads { + for name, payload := range request.Payloads { payloadStr, ok := payload.(string) if ok { final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) if resolveErr != nil { return errors.Wrap(resolveErr, "could not read payload file") } - r.Payloads[name] = final + request.Payloads[name] = final } } - r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath) + request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) if err != nil { return errors.Wrap(err, "could not parse payloads") } @@ -179,19 +179,19 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if err != nil { return errors.Wrap(err, "could not get network client") } - r.dialer = client + request.dialer = client - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { - return len(r.Address) +func (request *Request) Requests() int { + return len(request.Address) } diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 9bb996bca..547622aed 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -11,7 +11,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "": @@ -28,7 +28,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, r.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, request.dynamicValues)) case matchers.RegexMatcher: return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: @@ -40,7 +40,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { partString := extractor.Part switch partString { case "body", "all", "": @@ -63,7 +63,7 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext } // responseToDSLMap converts a DNS response to a map for use in DSL matching -func (r *Request) responseToDSLMap(req, resp, raw, host, matched string) output.InternalEvent { +func (request *Request) responseToDSLMap(req, resp, raw, host, matched string) output.InternalEvent { data := make(output.InternalEvent, 6) // Some data regarding the request metadata @@ -72,14 +72,14 @@ func (r *Request) responseToDSLMap(req, resp, raw, host, matched string) output. data["request"] = req data["data"] = resp // Data is the last bytes read data["raw"] = raw // Raw is the full transaction data for network - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath + data["template-id"] = request.options.TemplateID + data["template-info"] = request.options.TemplateInfo + data["template-path"] = request.options.TemplatePath return data } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { if len(wrapped.OperatorsResult.DynamicValues) > 0 { return nil } @@ -88,25 +88,25 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu // If we have multiple matchers with names, write each of them separately. if len(wrapped.OperatorsResult.Matches) > 0 { for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) data.MatcherName = k results = append(results, data) } } else if len(wrapped.OperatorsResult.Extracts) > 0 { for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) data.ExtractedResults = v data.ExtractorName = k results = append(results, data) } } else { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) results = append(results, data) } return results } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index bb4540cbf..6d99b2ded 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -24,15 +24,15 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { address, err := getAddress(input) if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not get address from url") } - for _, kv := range r.addresses { + for _, kv := range request.addresses { actualAddress := replacer.Replace(kv.ip, map[string]interface{}{"Hostname": address}) if kv.port != "" { if strings.Contains(address, ":") { @@ -41,7 +41,7 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused actualAddress = net.JoinHostPort(actualAddress, kv.port) } - if err := r.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil { + if err := request.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil { gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err) continue } @@ -50,71 +50,71 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused } // executeAddress executes the request for an address -func (r *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { if !strings.Contains(actualAddress, ":") { err := errors.New("no port provided in network protocol request") - r.options.Output.Request(r.options.TemplateID, address, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return err } - if r.generator != nil { - iterator := r.generator.NewIterator() + if request.generator != nil { + iterator := request.generator.NewIterator() for { value, ok := iterator.Value() if !ok { break } - if err := r.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } } else { value := make(map[string]interface{}) - if err := r.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } return nil } -func (r *Request) executeRequestWithPayloads(actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeRequestWithPayloads(actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var ( hostname string conn net.Conn err error ) - r.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) + request.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { hostname = host } if shouldUseTLS { - conn, err = r.dialer.DialTLS(context.Background(), "tcp", actualAddress) + conn, err = request.dialer.DialTLS(context.Background(), "tcp", actualAddress) } else { - conn, err = r.dialer.Dial(context.Background(), "tcp", actualAddress) + conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress) } if err != nil { - r.options.Output.Request(r.options.TemplateID, address, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server request") } defer conn.Close() - _ = conn.SetReadDeadline(time.Now().Add(time.Duration(r.options.Options.Timeout) * time.Second)) + _ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second)) - hasInteractMarkers := interactsh.HasMatchers(r.CompiledOperators) + hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) var interactURL string - if r.options.Interactsh != nil && hasInteractMarkers { - interactURL = r.options.Interactsh.URL() + if request.options.Interactsh != nil && hasInteractMarkers { + interactURL = request.options.Interactsh.URL() } responseBuilder := &strings.Builder{} reqBuilder := &strings.Builder{} inputEvents := make(map[string]interface{}) - for _, input := range r.Inputs { + for _, input := range request.Inputs { var data []byte switch input.Type { @@ -122,28 +122,28 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin data, err = hex.DecodeString(input.Data) default: if interactURL != "" { - input.Data = r.options.Interactsh.ReplaceMarkers(input.Data, interactURL) + input.Data = request.options.Interactsh.ReplaceMarkers(input.Data, interactURL) } data = []byte(input.Data) } if err != nil { - r.options.Output.Request(r.options.TemplateID, address, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } reqBuilder.Grow(len(input.Data)) finalData, dataErr := expressions.EvaluateByte(data, payloads) if dataErr != nil { - r.options.Output.Request(r.options.TemplateID, address, "network", dataErr) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", dataErr) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } reqBuilder.Write(finalData) if _, err := conn.Write(finalData); err != nil { - r.options.Output.Request(r.options.TemplateID, address, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } @@ -158,40 +158,40 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin } // Run any internal extractors for the request here and add found values to map. - if r.CompiledOperators != nil { - values := r.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{input.Name: bufferStr}, r.Extract) + if request.CompiledOperators != nil { + values := request.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{input.Name: bufferStr}, request.Extract) for k, v := range values { payloads[k] = v } } } } - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() - if r.options.Options.Debug || r.options.Options.DebugRequests { + if request.options.Options.Debug || request.options.Options.DebugRequests { requestOutput := reqBuilder.String() - gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", r.options.TemplateID, actualAddress) + gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", request.options.TemplateID, actualAddress) gologger.Print().Msgf("%s\nHex: %s", requestOutput, hex.EncodeToString([]byte(requestOutput))) } - r.options.Output.Request(r.options.TemplateID, actualAddress, "network", err) + request.options.Output.Request(request.options.TemplateID, actualAddress, "network", err) gologger.Verbose().Msgf("Sent TCP request to %s", actualAddress) bufferSize := 1024 - if r.ReadSize != 0 { - bufferSize = r.ReadSize + if request.ReadSize != 0 { + bufferSize = request.ReadSize } final := make([]byte, bufferSize) n, err := conn.Read(final) if err != nil && err != io.EOF { - r.options.Output.Request(r.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplateID, address, "network", err) return errors.Wrap(err, "could not read from server") } responseBuilder.Write(final[:n]) response := responseBuilder.String() - outputEvent := r.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress) - outputEvent["ip"] = r.dialer.GetDialedIP(hostname) + outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress) + outputEvent["ip"] = request.dialer.GetDialedIP(hostname) for k, v := range previous { outputEvent[k] = v } @@ -204,21 +204,21 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin event := &output.InternalWrappedEvent{InternalEvent: outputEvent} if interactURL == "" { - event := createEvent(r, outputEvent, event, payloads) + event := createEvent(request, outputEvent, event, payloads) callback(event) - } else if r.options.Interactsh != nil { - r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ - MakeResultFunc: r.MakeResultEvent, + } else if request.options.Interactsh != nil { + request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ + MakeResultFunc: request.MakeResultEvent, Event: event, - Operators: r.CompiledOperators, - MatchFunc: r.Match, - ExtractFunc: r.Extract, + Operators: request.CompiledOperators, + MatchFunc: request.Match, + ExtractFunc: request.Extract, }) } - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s\nHex: %s", response, responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), r.options.Options.NoColor)) + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress) + gologger.Print().Msgf("%s\nHex: %s", response, responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor)) } return nil diff --git a/v2/pkg/protocols/offlinehttp/find.go b/v2/pkg/protocols/offlinehttp/find.go index ee918ad57..2c876dab4 100644 --- a/v2/pkg/protocols/offlinehttp/find.go +++ b/v2/pkg/protocols/offlinehttp/find.go @@ -12,19 +12,19 @@ import ( // getInputPaths parses the specified input paths and returns a compiled // list of finished absolute paths to the files evaluating any allowlist, denylist, // glob, file or folders, etc. -func (r *Request) getInputPaths(target string, callback func(string)) error { +func (request *Request) getInputPaths(target string, callback func(string)) error { processed := make(map[string]struct{}) // Template input includes a wildcard if strings.Contains(target, "*") { - if err := r.findGlobPathMatches(target, processed, callback); err != nil { + if err := request.findGlobPathMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find glob matches") } return nil } // Template input is either a file or a directory - file, err := r.findFileMatches(target, processed, callback) + file, err := request.findFileMatches(target, processed, callback) if err != nil { return errors.Wrap(err, "could not find file") } @@ -34,14 +34,14 @@ func (r *Request) getInputPaths(target string, callback func(string)) error { // Recursively walk down the Templates directory and run all // the template file checks - if err := r.findDirectoryMatches(target, processed, callback); err != nil { + if err := request.findDirectoryMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find directory matches") } return nil } // findGlobPathMatches returns the matched files from a glob path -func (r *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error { +func (request *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error { matches, err := filepath.Glob(absPath) if err != nil { return errors.Errorf("wildcard found, but unable to glob: %s\n", err) @@ -60,7 +60,7 @@ func (r *Request) findGlobPathMatches(absPath string, processed map[string]struc // findFileMatches finds if a path is an absolute file. If the path // is a file, it returns true otherwise false with no errors. -func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) { +func (request *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) { info, err := os.Stat(absPath) if err != nil { return false, err @@ -79,7 +79,7 @@ func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, } // findDirectoryMatches finds matches for templates from a directory -func (r *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { +func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { err := godirwalk.Walk(absPath, &godirwalk.Options{ Unsorted: true, ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { diff --git a/v2/pkg/protocols/offlinehttp/offlinehttp.go b/v2/pkg/protocols/offlinehttp/offlinehttp.go index c315c3372..85b669320 100644 --- a/v2/pkg/protocols/offlinehttp/offlinehttp.go +++ b/v2/pkg/protocols/offlinehttp/offlinehttp.go @@ -2,6 +2,7 @@ package offlinehttp import ( "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" ) @@ -13,23 +14,23 @@ type Request struct { } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { +func (request *Request) GetID() string { return "" } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { +func (request *Request) Compile(options *protocols.ExecuterOptions) error { for _, operator := range options.Operators { if err := operator.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.compiledOperators = append(r.compiledOperators, operator) + request.compiledOperators = append(request.compiledOperators, operator) } - r.options = options + request.options = options return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index c5a7352da..6c9bdc138 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -14,7 +14,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { item, ok := getMatchPart(matcher.Part, data) if !ok { return false, []string{} @@ -42,7 +42,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { item, ok := getMatchPart(extractor.Part, data) if !ok { return nil @@ -79,7 +79,7 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) { } // responseToDSLMap converts an HTTP response to a map for use in DSL matching -func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { +func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies())) for k, v := range extra { data[k] = v @@ -101,14 +101,14 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r } data["all_headers"] = headers data["duration"] = duration.Seconds() - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath + data["template-id"] = request.options.TemplateID + data["template-info"] = request.options.TemplateInfo + data["template-path"] = request.options.TemplatePath return data } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { if len(wrapped.OperatorsResult.DynamicValues) > 0 { return nil } @@ -117,25 +117,25 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu // If we have multiple matchers with names, write each of them separately. if len(wrapped.OperatorsResult.Matches) > 0 { for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) data.MatcherName = k results = append(results, data) } } else if len(wrapped.OperatorsResult.Extracts) > 0 { for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) data.ExtractedResults = v data.ExtractorName = k results = append(results, data) } } else { - data := r.makeResultEventItem(wrapped) + data := request.makeResultEventItem(wrapped) results = append(results, data) } return results } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 18d0e3822..2d7da28b1 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -22,10 +22,10 @@ var _ protocols.Request = &Request{} const maxSize = 5 * 1024 * 1024 // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - wg := sizedwaitgroup.New(r.options.Options.BulkSize) +func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + wg := sizedwaitgroup.New(request.options.Options.BulkSize) - err := r.getInputPaths(input, func(data string) { + err := request.getInputPaths(input, func(data string) { wg.Add() go func(data string) { @@ -61,11 +61,11 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused return } - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped offline-http request for %s", r.options.TemplateID, data) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped offline-http request for %s", request.options.TemplateID, data) gologger.Print().Msgf("%s", dataStr) } - gologger.Verbose().Msgf("[%s] Sent OFFLINE-HTTP request to %s", r.options.TemplateID, data) + gologger.Verbose().Msgf("[%s] Sent OFFLINE-HTTP request to %s", request.options.TemplateID, data) dumpedResponse, err := httputil.DumpResponse(resp, true) if err != nil { @@ -79,25 +79,25 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused return } - outputEvent := r.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), headersToString(resp.Header), 0, nil) + outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), headersToString(resp.Header), 0, nil) outputEvent["ip"] = "" for k, v := range previous { outputEvent[k] = v } - for _, operator := range r.compiledOperators { - event := createEvent(outputEvent, operator, r) + for _, operator := range request.compiledOperators { + event := createEvent(outputEvent, operator, request) callback(event) } }(data) }) wg.Wait() if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "file", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "file", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() return nil } From b8a2b2a1c487c55760e88156cd65711d1dad92d6 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 1 Oct 2021 15:10:17 +0300 Subject: [PATCH 028/147] [feature] Add coloring to debug information #999 * fix nil pointer dereference --- v2/pkg/protocols/http/request.go | 12 +++++++----- v2/pkg/protocols/offlinehttp/request.go | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 3206a5f54..aba8d62e7 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -476,11 +476,13 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate func createEvent(request *Request, outputEvent output.InternalEvent, finalEvent output.InternalEvent, generatedRequest *generatedRequest) *output.InternalWrappedEvent { event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - result, ok := request.CompiledOperators.Execute(finalEvent, request.Match, request.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.OperatorsResult.PayloadValues = generatedRequest.meta - event.Results = request.MakeResultEvent(event) + if request.CompiledOperators != nil { + result, ok := request.CompiledOperators.Execute(finalEvent, request.Match, request.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.OperatorsResult.PayloadValues = generatedRequest.meta + event.Results = request.MakeResultEvent(event) + } } return event diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 2d7da28b1..c5e419262 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -101,13 +101,15 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review return nil } -func createEvent(outputEvent map[string]interface{}, operator *operators.Operators, r *Request) *output.InternalWrappedEvent { +func createEvent(outputEvent map[string]interface{}, operator *operators.Operators, request *Request) *output.InternalWrappedEvent { event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - result, ok := operator.Execute(outputEvent, r.Match, r.Extract) - if ok && event.OperatorsResult != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) + if request.compiledOperators != nil { + result, ok := operator.Execute(outputEvent, request.Match, request.Extract) + if ok && event.OperatorsResult != nil { + event.OperatorsResult = result + event.Results = request.MakeResultEvent(event) + } } return event From 08cd7a4ba72f64fb9861783c9777eeeada443b52 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 1 Oct 2021 16:52:38 +0300 Subject: [PATCH 029/147] [feature] Add coloring to debug information #999 * extracted common logic and made sure that all requests implement the same interface --- .../helpers/eventcreator/eventcreator.go | 26 +++++++++++++++++++ v2/pkg/protocols/dns/dns.go | 4 +++ v2/pkg/protocols/dns/operators.go | 2 +- v2/pkg/protocols/dns/request.go | 17 ++---------- v2/pkg/protocols/file/operators.go | 5 ++++ v2/pkg/protocols/file/request.go | 17 ++---------- v2/pkg/protocols/headless/operators.go | 5 ++++ v2/pkg/protocols/headless/request.go | 17 ++---------- v2/pkg/protocols/http/operators.go | 5 ++++ v2/pkg/protocols/http/request.go | 26 +++++-------------- v2/pkg/protocols/network/operators.go | 5 ++++ v2/pkg/protocols/network/request.go | 21 +++++---------- v2/pkg/protocols/offlinehttp/operators.go | 5 ++++ v2/pkg/protocols/offlinehttp/request.go | 22 +++------------- v2/pkg/protocols/protocols.go | 4 +++ 15 files changed, 82 insertions(+), 99 deletions(-) create mode 100644 v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go diff --git a/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go new file mode 100644 index 000000000..6dbd400b2 --- /dev/null +++ b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go @@ -0,0 +1,26 @@ +package eventcreator + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +func CreateEvent(request protocols.Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent { + return CreateEventWithAdditionalOptions(request, outputEvent, func(internalWrappedEvent *output.InternalWrappedEvent) {}) +} + +func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent { + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + for _, compiledOperator := range request.GetCompiledOperators() { + if compiledOperator != nil { + result, ok := compiledOperator.Execute(outputEvent, request.Match, request.Extract) + if ok && result != nil { + event.OperatorsResult = result + addAdditionalOptions(event) + event.Results = request.MakeResultEvent(event) + } + } + } + + return event +} diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index ff8d907c9..22930bf6f 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -74,6 +74,10 @@ type Request struct { Recursion bool `yaml:"recursion,omitempty" jsonschema:"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results"` } +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + // GetID returns the unique ID of the request if any. func (request *Request) GetID() string { return request.ID diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 22da3a11a..2eef27ded 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -27,7 +27,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat } switch matcher.GetType() { - case matchers.StatusMatcher: // TODO is this correct? + case matchers.StatusMatcher: return matcher.Result(matcher.MatchStatusCode(item.(int))), []string{} case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(types.ToString(item)))), []string{} diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 00a31efe4..1e4ad5d03 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" ) @@ -55,7 +56,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review outputEvent[k] = v } - event := createEvent(request, outputEvent) + event := eventcreator.CreateEvent(request, outputEvent) if request.options.Options.Debug || request.options.Options.DebugResponse { gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain) @@ -66,20 +67,6 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review return nil } -func createEvent(request *Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent { - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - - if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = request.MakeResultEvent(event) - } - } - - return event -} - // isURL tests a string to determine if it is a well-structured url or not. func isURL(toTest string) bool { if _, err := url.ParseRequestURI(toTest); err != nil { diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index c0c5cd20c..4e252b711 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -6,6 +6,7 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -133,6 +134,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) [] return results } +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 3118a8466..4b2c03d6c 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" ) @@ -56,7 +57,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review outputEvent[k] = v } - event := createEvent(request, outputEvent) + event := eventcreator.CreateEvent(request, outputEvent) if request.options.Options.Debug || request.options.Options.DebugResponse { gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath) @@ -75,17 +76,3 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review request.options.Progress.IncrementRequests() return nil } - -func createEvent(request *Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent { - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - - if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = request.MakeResultEvent(event) - } - } - - return event -} diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index fc4eac43c..fad89761f 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -4,6 +4,7 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -105,6 +106,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) [] return results } +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 62734e879..c675165d6 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" ) @@ -64,7 +65,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp outputEvent[k] = v } - event := createEvent(request, outputEvent) + event := eventcreator.CreateEvent(request, outputEvent) if request.options.Options.Debug || request.options.Options.DebugResponse { gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input) @@ -74,17 +75,3 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp callback(event) return nil } - -func createEvent(request *Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent { - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - - if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = request.MakeResultEvent(event) - } - } - - return event -} diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 83978acc9..c1fbedb0d 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -7,6 +7,7 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -143,6 +144,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) [] return results } +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index aba8d62e7..5b1cb9dc6 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -20,6 +20,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" @@ -263,7 +264,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou const drainReqSize = int64(8 * 1024) // executeRequest executes the actual generated request and returns error if occurred -func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previous output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error { +func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error { request.setCustomHeaders(generatedRequest) var ( @@ -447,7 +448,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse) - for k, v := range previous { + for k, v := range previousEvent { finalEvent[k] = v } for k, v := range outputEvent { @@ -457,12 +458,14 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if request.ReqCondition { for k, v := range outputEvent { key := fmt.Sprintf("%s_%d", k, requestCount) - previous[key] = v + previousEvent[key] = v finalEvent[key] = v } } - event := createEvent(request, outputEvent, finalEvent, generatedRequest) + event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, func(internalWrappedEvent *output.InternalWrappedEvent) { + internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta + }) if request.options.Options.Debug || request.options.Options.DebugResponse { gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL) @@ -473,21 +476,6 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate return nil } -func createEvent(request *Request, outputEvent output.InternalEvent, finalEvent output.InternalEvent, generatedRequest *generatedRequest) *output.InternalWrappedEvent { - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - - if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(finalEvent, request.Match, request.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.OperatorsResult.PayloadValues = generatedRequest.meta - event.Results = request.MakeResultEvent(event) - } - } - - return event -} - // setCustomHeaders sets the custom headers for generated request func (request *Request) setCustomHeaders(req *generatedRequest) { for k, v := range request.customHeaders { diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 547622aed..b5e14a721 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -4,6 +4,7 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -106,6 +107,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) [] return results } +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 6d99b2ded..a76907dff 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" @@ -202,11 +203,14 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input outputEvent[k] = v } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + var event *output.InternalWrappedEvent if interactURL == "" { - event := createEvent(request, outputEvent, event, payloads) + event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, func(wrappedEvent *output.InternalWrappedEvent) { + wrappedEvent.OperatorsResult.PayloadValues = payloads + }) callback(event) } else if request.options.Interactsh != nil { + event = &output.InternalWrappedEvent{InternalEvent: outputEvent} request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ MakeResultFunc: request.MakeResultEvent, Event: event, @@ -224,19 +228,6 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input return nil } -func createEvent(request *Request, outputEvent output.InternalEvent, event *output.InternalWrappedEvent, payloads map[string]interface{}) *output.InternalWrappedEvent { - if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(outputEvent, request.Match, request.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.OperatorsResult.PayloadValues = payloads - event.Results = request.MakeResultEvent(event) - } - } - - return event -} - // getAddress returns the address of the host to make request to func getAddress(toTest string) (string, error) { if strings.Contains(toTest, "://") { diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index 6c9bdc138..e7e4a3c74 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -7,6 +7,7 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -135,6 +136,10 @@ func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) [] return results } +func (request *Request) GetCompiledOperators() []*operators.Operators { + return request.compiledOperators +} + func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index c5e419262..438ed0324 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -11,9 +11,9 @@ import ( "github.com/remeh/sizedwaitgroup" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" ) @@ -85,10 +85,8 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review outputEvent[k] = v } - for _, operator := range request.compiledOperators { - event := createEvent(outputEvent, operator, request) - callback(event) - } + event := eventcreator.CreateEvent(request, outputEvent) + callback(event) }(data) }) wg.Wait() @@ -101,20 +99,6 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review return nil } -func createEvent(outputEvent map[string]interface{}, operator *operators.Operators, request *Request) *output.InternalWrappedEvent { - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - - if request.compiledOperators != nil { - result, ok := operator.Execute(outputEvent, request.Match, request.Extract) - if ok && event.OperatorsResult != nil { - event.OperatorsResult = result - event.Results = request.MakeResultEvent(event) - } - } - - return event -} - // headersToString converts http headers to string func headersToString(headers http.Header) string { builder := &strings.Builder{} diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 2e03434fd..f18311fd3 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -82,6 +82,10 @@ type Request interface { Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error + // MakeResultEvent creates a result event from internal wrapped event + MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent + // GetCompiledOperators returns a list of the compiled operators + GetCompiledOperators() []*operators.Operators } // OutputEventCallback is a callback event for any results found during scanning. From b55f59ec26bb1b17eff916323ca731fd5ce6e945 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 1 Oct 2021 18:23:06 +0300 Subject: [PATCH 030/147] Show the command line for the integration/regression tests when the debug option is enabled. * minor code cleanup and code duplication removal --- v2/cmd/functional-test/main.go | 5 +- v2/cmd/integration-test/dns.go | 2 +- v2/cmd/integration-test/http.go | 102 +++++++++++++-------------- v2/cmd/integration-test/network.go | 4 +- v2/cmd/nuclei/main.go | 7 ++ v2/internal/testutils/integration.go | 58 +++++++-------- 6 files changed, 91 insertions(+), 87 deletions(-) diff --git a/v2/cmd/functional-test/main.go b/v2/cmd/functional-test/main.go index bc95f20b5..fc96fde80 100644 --- a/v2/cmd/functional-test/main.go +++ b/v2/cmd/functional-test/main.go @@ -15,6 +15,7 @@ import ( ) var ( + debug = os.Getenv("DEBUG") == "true" success = aurora.Green("[✓]").String() failed = aurora.Red("[✘]").String() errored = false @@ -65,11 +66,11 @@ func runIndividualTestCase(testcase string) error { if len(parts) > 1 { finalArgs = parts[1:] } - mainOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*mainNucleiBinary, finalArgs) + mainOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*mainNucleiBinary, debug, finalArgs) if err != nil { return errors.Wrap(err, "could not run nuclei main test") } - devOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*devNucleiBinary, finalArgs) + devOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*devNucleiBinary, debug, finalArgs) if err != nil { return errors.Wrap(err, "could not run nuclei dev test") } diff --git a/v2/cmd/integration-test/dns.go b/v2/cmd/integration-test/dns.go index 7661f48f9..3e1ae8146 100644 --- a/v2/cmd/integration-test/dns.go +++ b/v2/cmd/integration-test/dns.go @@ -14,7 +14,7 @@ type dnsBasic struct{} func (h *dnsBasic) Execute(filePath string) error { var routerErr error - results, err := testutils.RunNucleiAndGetResults(filePath, "one.one.one.one", debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug) if err != nil { return err } diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 8188c27af..6d43c0cc7 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -47,16 +47,16 @@ type httpGetHeaders struct{} // Execute executes a test case and returns an error if occurred func (h *httpGetHeaders) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) if strings.EqualFold(r.Header.Get("test"), "nuclei") { fmt.Fprintf(w, "This is test headers matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -71,16 +71,16 @@ type httpGetQueryString struct{} // Execute executes a test case and returns an error if occurred func (h *httpGetQueryString) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") { fmt.Fprintf(w, "This is test querystring matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -95,18 +95,18 @@ type httpGetRedirects struct{} // Execute executes a test case and returns an error if occurred func (h *httpGetRedirects) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) http.Redirect(w, r, "/redirected", http.StatusFound) - })) - router.GET("/redirected", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + }) + router.GET("/redirected", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) fmt.Fprintf(w, "This is test redirects matcher text") - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -121,14 +121,14 @@ type httpGet struct{} // Execute executes a test case and returns an error if occurred func (h *httpGet) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -145,7 +145,7 @@ func (h *httpPostBody) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err @@ -154,11 +154,11 @@ func (h *httpPostBody) Execute(filePath string) error { if strings.EqualFold(r.Form.Get("username"), "test") && strings.EqualFold(r.Form.Get("password"), "nuclei") { fmt.Fprintf(w, "This is test post-body matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -178,7 +178,7 @@ func (h *httpPostJSONBody) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) type doc struct { @@ -193,11 +193,11 @@ func (h *httpPostJSONBody) Execute(filePath string) error { if strings.EqualFold(obj.Username, "test") && strings.EqualFold(obj.Password, "nuclei") { fmt.Fprintf(w, "This is test post-json-body matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -217,7 +217,7 @@ func (h *httpPostMultipartBody) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) if err := r.ParseMultipartForm(1 * 1024); err != nil { routerErr = err @@ -236,11 +236,11 @@ func (h *httpPostMultipartBody) Execute(filePath string) error { if strings.EqualFold(password[0], "nuclei") && strings.EqualFold(file[0].Filename, "username") { fmt.Fprintf(w, "This is test post-multipart matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -260,7 +260,7 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err @@ -269,17 +269,17 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error { if strings.EqualFold(r.Form.Get("testing"), "parameter") { fmt.Fprintf(w, "Token: 'nuclei'") } - })) - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + }) + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) if strings.EqualFold(r.URL.Query().Get("username"), "nuclei") { fmt.Fprintf(w, "Test is test-dynamic-extractor-raw matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -299,16 +299,16 @@ func (h *httpRawGetQuery) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") { fmt.Fprintf(w, "Test is test raw-get-query-matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -328,15 +328,15 @@ func (h *httpRawGet) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) fmt.Fprintf(w, "Test is test raw-get-matcher text") - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -356,7 +356,7 @@ func (h *httpRawPayload) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err @@ -368,11 +368,11 @@ func (h *httpRawPayload) Execute(filePath string) error { if strings.EqualFold(r.Form.Get("username"), "test") && (strings.EqualFold(r.Form.Get("password"), "nuclei") || strings.EqualFold(r.Form.Get("password"), "guest")) { fmt.Fprintf(w, "Test is raw-payload matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -392,7 +392,7 @@ func (h *httpRawPostBody) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err @@ -401,11 +401,11 @@ func (h *httpRawPostBody) Execute(filePath string) error { if strings.EqualFold(r.Form.Get("username"), "test") && strings.EqualFold(r.Form.Get("password"), "nuclei") { fmt.Fprintf(w, "Test is test raw-post-body-matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -425,7 +425,7 @@ func (h *httpRawCookieReuse) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err @@ -434,8 +434,8 @@ func (h *httpRawCookieReuse) Execute(filePath string) error { if strings.EqualFold(r.Form.Get("testing"), "parameter") { http.SetCookie(w, &http.Cookie{Name: "nuclei", Value: "test"}) } - })) - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + }) + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err @@ -450,11 +450,11 @@ func (h *httpRawCookieReuse) Execute(filePath string) error { if strings.EqualFold(cookie.Value, "test") { fmt.Fprintf(w, "Test is test-cookie-reuse matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -479,7 +479,7 @@ func (h *httpRawUnsafeRequest) Execute(filePath string) error { }) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, "http://"+ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "http://"+ts.URL, debug) if err != nil { return err } @@ -499,18 +499,18 @@ func (h *httpRequestCondition) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.GET("/200", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/200", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) w.WriteHeader(200) - })) - router.GET("/400", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + }) + router.GET("/400", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { httpDebugRequestDump(r) w.WriteHeader(400) - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index 5bb0fe519..a377a1c0a 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -32,7 +32,7 @@ func (h *networkBasic) Execute(filePath string) error { }) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -75,7 +75,7 @@ func (h *networkMultiStep) Execute(filePath string) error { }) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index cae40155d..e40a6dadb 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -188,3 +188,10 @@ func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags currentFlag.Group(groupName) } } + +/* +HacktoberFest update: Below, you can find our ticket recommendations. Tasks with the "good first issue" label are suitable for first time contributors. If you have other ideas, or need help with getting started, join our Discord channel or reach out to @forgedhallpass. + +https://github.com/issues?q=is%3Aopen+is%3Aissue+user%3Aprojectdiscovery+label%3AHacktoberfest + +*/ diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index bb5d557f1..f2c398869 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -2,6 +2,7 @@ package testutils import ( "errors" + "fmt" "net" "os" "os/exec" @@ -9,20 +10,36 @@ import ( "strings" ) -// RunNucleiAndGetResults returns a list of results for a template -func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { - cmd := exec.Command("./nuclei", "-t", template, "-target", url, "-silent") +// RunNucleiTemplateAndGetResults returns a list of results for a template +func RunNucleiTemplateAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { + return runNucleiAndGetResults(true, template, url, debug, extra...) +} + +// RunNucleiWorkflowAndGetResults returns a list of results for a workflow +func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { + return runNucleiAndGetResults(false, template, url, debug, extra...) +} + +func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) { + var templateOrWorkflowFlag string + if isTemplate { + templateOrWorkflowFlag = "-t" + } else { + templateOrWorkflowFlag = "-w" + } + + cmd := exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-silent") if debug { - cmd = exec.Command("./nuclei", "-t", template, "-target", url, "-debug") + cmd = exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-debug") cmd.Stderr = os.Stderr + fmt.Println(cmd.String()) } cmd.Args = append(cmd.Args, extra...) - data, err := cmd.Output() if err != nil { return nil, err } - parts := []string{} + var parts []string items := strings.Split(string(data), "\n") for _, i := range items { if i != "" { @@ -35,9 +52,11 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ( var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded[^:]*: (\d+)`) // RunNucleiBinaryAndGetLoadedTemplates returns a list of results for a template -func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, args []string) (string, error) { +func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, debug bool, args []string) (string, error) { cmd := exec.Command(nucleiBinary, args...) - + if debug { + fmt.Println(cmd.String()) + } data, err := cmd.CombinedOutput() if err != nil { return "", err @@ -49,29 +68,6 @@ func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, args []string) (s return matches[0][1], nil } -// RunNucleiWorkflowAndGetResults returns a list of results for a workflow -func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { - cmd := exec.Command("./nuclei", "-w", template, "-target", url, "-silent") - if debug { - cmd = exec.Command("./nuclei", "-w", template, "-target", url, "-debug") - cmd.Stderr = os.Stderr - } - cmd.Args = append(cmd.Args, extra...) - - data, err := cmd.Output() - if err != nil { - return nil, err - } - parts := []string{} - items := strings.Split(string(data), "\n") - for _, i := range items { - if i != "" { - parts = append(parts, i) - } - } - return parts, nil -} - // TestCase is a single integration test case type TestCase interface { // Execute executes a test case and returns any errors if occurred From 68f3fb51e5a2944115649a78e891e4af4b858bb2 Mon Sep 17 00:00:00 2001 From: nothinux Date: Sat, 2 Oct 2021 14:30:40 +0700 Subject: [PATCH 031/147] add minor version to HTTP/2 & HTTP/3 --- v2/pkg/protocols/offlinehttp/read_response.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/v2/pkg/protocols/offlinehttp/read_response.go b/v2/pkg/protocols/offlinehttp/read_response.go index 0869a5643..2c8528733 100644 --- a/v2/pkg/protocols/offlinehttp/read_response.go +++ b/v2/pkg/protocols/offlinehttp/read_response.go @@ -4,13 +4,22 @@ import ( "bufio" "errors" "net/http" + "regexp" "strings" ) // readResponseFromString reads a raw http response from a string. func readResponseFromString(data string) (*http.Response, error) { var final string + noMinor := regexp.MustCompile(`HTTP\/[0-9] `) + if strings.HasPrefix(data, "HTTP/") { + // Go could not parse http version with no minor version + if noMinor.MatchString(data) { + // add minor version + data = strings.Replace(data, "HTTP/2", "HTTP/2.0", 1) + data = strings.Replace(data, "HTTP/3", "HTTP/3.0", 1) + } final = data } else { lastIndex := strings.LastIndex(data, "HTTP/") @@ -18,6 +27,11 @@ func readResponseFromString(data string) (*http.Response, error) { return nil, errors.New("malformed raw http response") } final = data[lastIndex:] // choose last http/ in case of it being later. + + if noMinor.MatchString(final) { + final = strings.ReplaceAll(final, "HTTP/2", "HTTP/2.0") + final = strings.ReplaceAll(final, "HTTP/3", "HTTP/3.0") + } } return http.ReadResponse(bufio.NewReader(strings.NewReader(final)), nil) } From d08d9d2dc07839f76f2c0a60d10580aa7f9bd5b4 Mon Sep 17 00:00:00 2001 From: nothinux Date: Sat, 2 Oct 2021 14:31:18 +0700 Subject: [PATCH 032/147] add test for response without http minor version --- .../offlinehttp/read_response_test.go | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/v2/pkg/protocols/offlinehttp/read_response_test.go b/v2/pkg/protocols/offlinehttp/read_response_test.go index cf29ba80b..e7bd7c5c6 100644 --- a/v2/pkg/protocols/offlinehttp/read_response_test.go +++ b/v2/pkg/protocols/offlinehttp/read_response_test.go @@ -28,6 +28,62 @@ Cache-Control: public, max-age=600 Content-Type: text/html Server: Google Frontend + + + +Firing Range + + +

Version 0.48

+

What is the Firing Range?

+

+ + +` + resp, err := readResponseFromString(data) + require.Nil(t, err, "could not read response from string") + + respData, err := ioutil.ReadAll(resp.Body) + require.Nil(t, err, "could not read response body") + require.Equal(t, expectedBody, string(respData), "could not get correct parsed body") + require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") + }) + + t.Run("response-http2-without-minor-version", func(t *testing.T) { + data := `HTTP/2 200 OK +Age: 0 +Cache-Control: public, max-age=600 +Content-Type: text/html +Server: Google Frontend + + + + +Firing Range + + +

Version 0.48

+

What is the Firing Range?

+

+ + +` + resp, err := readResponseFromString(data) + require.Nil(t, err, "could not read response from string") + + respData, err := ioutil.ReadAll(resp.Body) + require.Nil(t, err, "could not read response body") + require.Equal(t, expectedBody, string(respData), "could not get correct parsed body") + require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") + }) + + t.Run("response-http2-with-minor-version", func(t *testing.T) { + data := `HTTP/2.0 200 OK +Age: 0 +Cache-Control: public, max-age=600 +Content-Type: text/html +Server: Google Frontend + @@ -82,4 +138,39 @@ Server: Google Frontend require.Equal(t, expectedBody, string(respData), "could not get correct parsed body") require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") }) + + t.Run("request-response-without-minor-version", func(t *testing.T) { + data := `GET http://public-firing-range.appspot.com/ HTTP/1.1 +Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 +Accept-Encoding: gzip, deflate +Upgrade-Insecure-Requests: 1 +User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 + +HTTP/2 200 OK +Age: 0 +Cache-Control: public, max-age=600 +Content-Type: text/html +Server: Google Frontend + + + + +Firing Range + + +

Version 0.48

+

What is the Firing Range?

+

+ + +` + resp, err := readResponseFromString(data) + require.Nil(t, err, "could not read response from string") + + respData, err := ioutil.ReadAll(resp.Body) + require.Nil(t, err, "could not read response body") + require.Equal(t, expectedBody, string(respData), "could not get correct parsed body") + require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") + }) + } From 0c2b79bbf5ca396310909a2d13fd82c3f92986af Mon Sep 17 00:00:00 2001 From: nothinux Date: Sat, 2 Oct 2021 15:05:03 +0700 Subject: [PATCH 033/147] using table driven tests --- .../offlinehttp/read_response_test.go | 104 +++++++----------- 1 file changed, 40 insertions(+), 64 deletions(-) diff --git a/v2/pkg/protocols/offlinehttp/read_response_test.go b/v2/pkg/protocols/offlinehttp/read_response_test.go index e7bd7c5c6..80086a0cb 100644 --- a/v2/pkg/protocols/offlinehttp/read_response_test.go +++ b/v2/pkg/protocols/offlinehttp/read_response_test.go @@ -18,11 +18,15 @@ func TestReadResponseFromString(t *testing.T) {

What is the Firing Range?

- ` - t.Run("response", func(t *testing.T) { - data := `HTTP/1.1 200 OK + tests := []struct { + name string + data string + }{ + { + name: "response", + data: `HTTP/1.1 200 OK Age: 0 Cache-Control: public, max-age=600 Content-Type: text/html @@ -38,19 +42,11 @@ Server: Google Frontend

What is the Firing Range?

- -` - resp, err := readResponseFromString(data) - require.Nil(t, err, "could not read response from string") - - respData, err := ioutil.ReadAll(resp.Body) - require.Nil(t, err, "could not read response body") - require.Equal(t, expectedBody, string(respData), "could not get correct parsed body") - require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") - }) - - t.Run("response-http2-without-minor-version", func(t *testing.T) { - data := `HTTP/2 200 OK +`, + }, + { + name: "response-http2-without-minor-version", + data: `HTTP/2 200 OK Age: 0 Cache-Control: public, max-age=600 Content-Type: text/html @@ -66,19 +62,11 @@ Server: Google Frontend

What is the Firing Range?

- -` - resp, err := readResponseFromString(data) - require.Nil(t, err, "could not read response from string") - - respData, err := ioutil.ReadAll(resp.Body) - require.Nil(t, err, "could not read response body") - require.Equal(t, expectedBody, string(respData), "could not get correct parsed body") - require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") - }) - - t.Run("response-http2-with-minor-version", func(t *testing.T) { - data := `HTTP/2.0 200 OK +`, + }, + { + name: "response-http2-with-minor-version", + data: `HTTP/2.0 200 OK Age: 0 Cache-Control: public, max-age=600 Content-Type: text/html @@ -94,19 +82,11 @@ Server: Google Frontend

What is the Firing Range?

- -` - resp, err := readResponseFromString(data) - require.Nil(t, err, "could not read response from string") - - respData, err := ioutil.ReadAll(resp.Body) - require.Nil(t, err, "could not read response body") - require.Equal(t, expectedBody, string(respData), "could not get correct parsed body") - require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") - }) - - t.Run("request-response", func(t *testing.T) { - data := `GET http://public-firing-range.appspot.com/ HTTP/1.1 +`, + }, + { + name: "request-response", + data: `GET http://public-firing-range.appspot.com/ HTTP/1.1 Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 Accept-Encoding: gzip, deflate Upgrade-Insecure-Requests: 1 @@ -128,19 +108,11 @@ Server: Google Frontend

What is the Firing Range?

- -` - resp, err := readResponseFromString(data) - require.Nil(t, err, "could not read response from string") - - respData, err := ioutil.ReadAll(resp.Body) - require.Nil(t, err, "could not read response body") - require.Equal(t, expectedBody, string(respData), "could not get correct parsed body") - require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") - }) - - t.Run("request-response-without-minor-version", func(t *testing.T) { - data := `GET http://public-firing-range.appspot.com/ HTTP/1.1 +`, + }, + { + name: "request-response-without-minor-version", + data: `GET http://public-firing-range.appspot.com/ HTTP/1.1 Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 Accept-Encoding: gzip, deflate Upgrade-Insecure-Requests: 1 @@ -162,15 +134,19 @@ Server: Google Frontend

What is the Firing Range?

- -` - resp, err := readResponseFromString(data) - require.Nil(t, err, "could not read response from string") +`, + }, + } - respData, err := ioutil.ReadAll(resp.Body) - require.Nil(t, err, "could not read response body") - require.Equal(t, expectedBody, string(respData), "could not get correct parsed body") - require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") - }) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp, err := readResponseFromString(tt.data) + require.Nil(t, err, "could not read response from string") + respData, err := ioutil.ReadAll(resp.Body) + require.Nil(t, err, "could not read response body") + require.Equal(t, expectedBody, string(respData), "could not get correct parsed body") + require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") + }) + } } From e2349b1d2df3a7175cfac5417a68cff01da38647 Mon Sep 17 00:00:00 2001 From: nothinux Date: Sat, 2 Oct 2021 22:10:05 +0700 Subject: [PATCH 034/147] add test live response --- .../offlinehttp/read_response_test.go | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/v2/pkg/protocols/offlinehttp/read_response_test.go b/v2/pkg/protocols/offlinehttp/read_response_test.go index 80086a0cb..4a75d7263 100644 --- a/v2/pkg/protocols/offlinehttp/read_response_test.go +++ b/v2/pkg/protocols/offlinehttp/read_response_test.go @@ -2,7 +2,10 @@ package offlinehttp import ( "io/ioutil" + "net/http" + "net/http/httputil" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -149,4 +152,26 @@ Server: Google Frontend require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") }) } + + t.Run("test-live-response-with-content-length", func(t *testing.T) { + client := &http.Client{ + Timeout: 3 * time.Second, + } + + data, err := client.Get("https://golang.org/doc/install") + require.Nil(t, err, "could not dial url") + defer data.Body.Close() + + b, err := httputil.DumpResponse(data, true) + require.Nil(t, err, "could not dump response") + + respData, err := readResponseFromString(string(b)) + require.Nil(t, err, "could not read response from string") + + _, err = ioutil.ReadAll(respData.Body) + require.Nil(t, err, "could not read response body") + + require.Equal(t, "Google Frontend", respData.Header.Get("Server"), "could not get correct headers") + + }) } From 27cc0d61bcd37b319759f32c72ffbe6a3db83654 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 4 Oct 2021 04:14:15 +0530 Subject: [PATCH 035/147] misc update --- v2/cmd/nuclei/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 4f5171d06..cf5f611d6 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -139,7 +139,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"), flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"), flagSet.BoolVar(&options.ShowBrowser, "show-browser", false, "show the browser on the screen when running templates with headless mode"), - flagSet.BoolVar(&options.UseInstalledChrome, "use-installed-chrome", false, "use installed browser"), + flagSet.BoolVar(&options.UseInstalledChrome, "system-chrome", false, "Use local installed chrome browser instead of nuclei installed"), ) createGroup(flagSet, "debug", "Debug", From 407e5cbfeb075549566960a5858fcfb0a1e711d3 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 4 Oct 2021 15:31:14 +0200 Subject: [PATCH 036/147] Adding support for custom resolvers in DNS templates --- v2/pkg/protocols/dns/dns.go | 12 +++++++++--- v2/pkg/protocols/dns/dnsclientpool/clientpool.go | 9 +++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 4c2b8904f..e4e7833e9 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -72,6 +72,8 @@ type Request struct { // description: | // Recursion determines if resolver should recurse all records to get fresh results. Recursion bool `yaml:"recursion,omitempty" jsonschema:"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results"` + // Resolvers to use for the dns requests + Resolvers []string `yaml:"resolvers,omitempty" jsonschema:"title=Resolvers,description=Define resolvers to use within the template"` } // GetID returns the unique ID of the request if any. @@ -81,10 +83,14 @@ func (r *Request) GetID() string { // Compile compiles the protocol request for further execution. func (r *Request) Compile(options *protocols.ExecuterOptions) error { - // Create a dns client for the class - client, err := dnsclientpool.Get(options.Options, &dnsclientpool.Configuration{ + dnsClientOptions := &dnsclientpool.Configuration{ Retries: r.Retries, - }) + } + if len(r.Resolvers) > 0 { + dnsClientOptions.Resolvers = r.Resolvers + } + // Create a dns client for the class + client, err := dnsclientpool.Get(options.Options, dnsClientOptions) if err != nil { return errors.Wrap(err, "could not get dns client") } diff --git a/v2/pkg/protocols/dns/dnsclientpool/clientpool.go b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go index 999f2a2c9..46308c9fe 100644 --- a/v2/pkg/protocols/dns/dnsclientpool/clientpool.go +++ b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go @@ -44,21 +44,24 @@ func Init(options *types.Options) error { type Configuration struct { // Retries contains the retries for the dns client Retries int + // Resolvers contains the specific per request resolvers + Resolvers []string } // Hash returns the hash of the configuration to allow client pooling func (c *Configuration) Hash() string { builder := &strings.Builder{} - builder.Grow(8) builder.WriteString("r") builder.WriteString(strconv.Itoa(c.Retries)) + builder.WriteString("l") + builder.WriteString(strings.Join(c.Resolvers, "")) hash := builder.String() return hash } // Get creates or gets a client for the protocol based on custom configuration func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) { - if !(configuration.Retries > 1) { + if !(configuration.Retries > 1) && len(configuration.Resolvers) == 0 { return normalClient, nil } hash := configuration.Hash() @@ -72,6 +75,8 @@ func Get(options *types.Options, configuration *Configuration) (*retryabledns.Cl resolvers := defaultResolvers if options.ResolversFile != "" { resolvers = options.InternalResolversList + } else if len(configuration.Resolvers) > 0 { + resolvers = configuration.Resolvers } client := retryabledns.New(resolvers, configuration.Retries) From eb2dee22f19f7fe389f8c423cf65b4d9ffc0f2cd Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 4 Oct 2021 20:40:57 +0000 Subject: [PATCH 037/147] Auto Generate Syntax Docs + JSONSchema [Mon Oct 4 20:40:57 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 13 +++++++++++++ nuclei-jsonschema.json | 8 ++++++++ v2/pkg/templates/templates_doc.go | 7 ++++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 86191bdba..bb1ad86bd 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -2026,6 +2026,19 @@ Recursion determines if resolver should recurse all records to get fresh results

+
+ +resolvers []string + +
+
+ +Resolvers to use for the dns requests + +
+ +
+ diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 28c3e2de0..20da43727 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -373,6 +373,14 @@ "type": "boolean", "title": "recurse all servers", "description": "Recursion determines if resolver should recurse all records to get fresh results" + }, + "resolvers": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Resolvers", + "description": "Define resolvers to use within the template" } }, "additionalProperties": false, diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index ce06e5b62..e7758c96f 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -680,7 +680,7 @@ func init() { FieldName: "dns", }, } - DNSRequestDoc.Fields = make([]encoder.Doc, 9) + DNSRequestDoc.Fields = make([]encoder.Doc, 10) DNSRequestDoc.Fields[0].Name = "matchers" DNSRequestDoc.Fields[0].Type = "[]matchers.Matcher" DNSRequestDoc.Fields[0].Note = "" @@ -753,6 +753,11 @@ func init() { DNSRequestDoc.Fields[8].Note = "" DNSRequestDoc.Fields[8].Description = "Recursion determines if resolver should recurse all records to get fresh results." DNSRequestDoc.Fields[8].Comments[encoder.LineComment] = "Recursion determines if resolver should recurse all records to get fresh results." + DNSRequestDoc.Fields[9].Name = "resolvers" + DNSRequestDoc.Fields[9].Type = "[]string" + DNSRequestDoc.Fields[9].Note = "" + DNSRequestDoc.Fields[9].Description = "Resolvers to use for the dns requests" + DNSRequestDoc.Fields[9].Comments[encoder.LineComment] = " Resolvers to use for the dns requests" FILERequestDoc.Type = "file.Request" FILERequestDoc.Comments[encoder.LineComment] = " Request contains a File matching mechanism for local disk operations." From 79de2d82cb73a8fc28fe921fcbf7762367e53546 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Tue, 5 Oct 2021 13:56:02 +0300 Subject: [PATCH 038/147] [feature] Add coloring to debug information #999 * adding status color support for all existing HTTP versions --- .../response_highligther.go | 12 ++++++++++ v2/pkg/protocols/dns/operators.go | 6 ++++- v2/pkg/protocols/http/operators.go | 22 +++++++++++++------ v2/pkg/protocols/offlinehttp/operators.go | 18 ++++++++++++--- 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go index fa168e26c..02ca8afc7 100644 --- a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go @@ -1,6 +1,7 @@ package responsehighlighter import ( + "fmt" "strings" "github.com/logrusorgru/aurora" @@ -23,3 +24,14 @@ func Highlight(operatorResult *operators.Result, response string, noColor bool) return result } + +func CreateHTTPStatusMatcherSnippets(statusCode int) []string { + httpVersions := []string{"0.9", "1.0", "1.1", "2", "2.0", "3", "3.0"} + var matcherValues = make([]string, 0, len(httpVersions)) + + for _, httpVersion := range httpVersions { + matcherValues = append(matcherValues, fmt.Sprintf("HTTP/%s %d", httpVersion, statusCode)) + } + + return matcherValues +} diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 2eef27ded..cebc02a10 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -28,7 +28,11 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat switch matcher.GetType() { case matchers.StatusMatcher: - return matcher.Result(matcher.MatchStatusCode(item.(int))), []string{} + statusCode, ok := item.(int) + if !ok { + return false, []string{} + } + return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{} case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(types.ToString(item)))), []string{} case matchers.WordsMatcher: diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index c1fbedb0d..781b845ad 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -1,7 +1,6 @@ package http import ( - "fmt" "net/http" "strings" "time" @@ -11,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -23,15 +23,11 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat switch matcher.GetType() { case matchers.StatusMatcher: - statusCode, ok := data["status_code"] + statusCode, ok := getStatusCode(data) if !ok { return false, []string{} } - status, ok := statusCode.(int) - if !ok { - return false, []string{} - } - return matcher.Result(matcher.MatchStatusCode(status)), []string{fmt.Sprintf("HTTP/1.0 %d", status), fmt.Sprintf("HTTP/1.1 %d", status)} + return matcher.Result(matcher.MatchStatusCode(statusCode)), responsehighlighter.CreateHTTPStatusMatcherSnippets(statusCode) case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: @@ -46,6 +42,18 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat return false, []string{} } +func getStatusCode(data map[string]interface{}) (int, bool) { + statusCodeValue, ok := data["status_code"] + if !ok { + return 0, false + } + statusCode, ok := statusCodeValue.(int) + if !ok { + return 0, false + } + return statusCode, true +} + // Extract performs extracting operation for an extractor on model and returns true or false. func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { item, ok := getMatchPart(extractor.Part, data) diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index e7e4a3c74..d8fd842df 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -1,7 +1,6 @@ package offlinehttp import ( - "fmt" "net/http" "strings" "time" @@ -11,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -23,11 +23,11 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat switch matcher.GetType() { case matchers.StatusMatcher: - statusCode, ok := data["status_code"] + statusCode, ok := getStatusCode(data) if !ok { return false, []string{} } - return matcher.Result(matcher.MatchStatusCode(statusCode.(int))), []string{fmt.Sprintf("HTTP/1.0 %d", statusCode), fmt.Sprintf("HTTP/1.1 %d", statusCode)} + return matcher.Result(matcher.MatchStatusCode(statusCode)), responsehighlighter.CreateHTTPStatusMatcherSnippets(statusCode) case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: @@ -42,6 +42,18 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat return false, []string{} } +func getStatusCode(data map[string]interface{}) (int, bool) { + statusCodeValue, ok := data["status_code"] + if !ok { + return 0, false + } + statusCode, ok := statusCodeValue.(int) + if !ok { + return 0, false + } + return statusCode, true +} + // Extract performs extracting operation for an extractor on model and returns true or false. func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { item, ok := getMatchPart(extractor.Part, data) From 4d345138810eb05a058cb2c782b28ef5bb6f151b Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Tue, 5 Oct 2021 16:44:18 +0300 Subject: [PATCH 039/147] Fix for Binary matcher always returns true if the input cannot be decoded as hex #1085 --- v2/pkg/operators/matchers/match.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 7877660f5..8da6f6a56 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "strings" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" ) @@ -117,7 +118,14 @@ func (m *Matcher) MatchBinary(corpus string) (bool, []string) { // Iterate over all the words accepted as valid for i, binary := range m.Binary { // Continue if the word doesn't match - hexa, _ := hex.DecodeString(binary) + hexa, err := hex.DecodeString(binary) + if err != nil { + gologger.Error().Msgf("Could not hex encode the given binary matcher value: '%s'", binary) + if m.condition == ANDCondition { + return false, []string{} + } + continue + } if !strings.Contains(corpus, string(hexa)) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. From 404f8ebb2795aadc4f10069c28a1932c74d6613f Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Tue, 5 Oct 2021 16:49:55 +0300 Subject: [PATCH 040/147] Fixes:Binary matcher always returns true if the input cannot be decoded as hex #1085 * changed the Log to Warn instead of Error, so it wouldn't show up in the output without the verbose flag --- v2/pkg/operators/matchers/match.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 8da6f6a56..fff0055de 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -120,7 +120,7 @@ func (m *Matcher) MatchBinary(corpus string) (bool, []string) { // Continue if the word doesn't match hexa, err := hex.DecodeString(binary) if err != nil { - gologger.Error().Msgf("Could not hex encode the given binary matcher value: '%s'", binary) + gologger.Warning().Msgf("Could not hex encode the given binary matcher value: '%s'", binary) if m.condition == ANDCondition { return false, []string{} } From ed3b80b5d727a6c155e0750349d34139d29c11f0 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Tue, 5 Oct 2021 22:02:09 +0300 Subject: [PATCH 041/147] If debug is enabled, show matchers that are not named also #1084 --- v2/pkg/operators/operators.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 86f1611b9..e88e34814 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -1,6 +1,8 @@ package operators import ( + "strconv" + "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" @@ -136,14 +138,10 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc } } - for _, matcher := range operators.Matchers { - // Check if the matcher matched + for matcherIndex, matcher := range operators.Matchers { if isMatch, matched := match(data, matcher); isMatch { - // If the matcher has matched, and it's an OR - // write the first output then move to next matcher. - if matcherCondition == matchers.ORCondition && matcher.Name != "" { - result.Matches[matcher.Name] = matched - } + matcherName := getMatcherName(matcher, matcherIndex) + result.Matches[matcherName] = matched matches = true } else if matcherCondition == matchers.ANDCondition { @@ -172,6 +170,14 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc return nil, false } +func getMatcherName(matcher *matchers.Matcher, matcherIndex int) string { + if matcher.Name != "" { + return matcher.Name + } else { + return matcher.Type + "-" + strconv.Itoa(matcherIndex) + } +} + // ExecuteInternalExtractors executes internal dynamic extractors func (operators *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} { dynamicValues := make(map[string]interface{}) From 85447ca49d583216f5a790750ebad262a76fdd6a Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Tue, 5 Oct 2021 22:04:09 +0300 Subject: [PATCH 042/147] Fixes: Do not show all matcher names for OR conditions #1082 --- v2/pkg/operators/operators.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index e88e34814..5643152b1 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -144,6 +144,10 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc result.Matches[matcherName] = matched matches = true + + if matcherCondition == matchers.ORCondition { + break + } } else if matcherCondition == matchers.ANDCondition { if len(result.DynamicValues) > 0 { return result, true From 49291cc937a732138641999e74a8b1d534be8eb3 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Tue, 5 Oct 2021 22:14:18 +0300 Subject: [PATCH 043/147] If debug is enabled, show matchers that are not named also #1084 * integration test fix --- v2/cmd/integration-test/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index a377a1c0a..04cc70136 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -82,7 +82,7 @@ func (h *networkMultiStep) Execute(filePath string) error { if routerErr != nil { return routerErr } - if len(results) != 1 { + if len(results) != 3 { return errIncorrectResultsCount(results) } return nil From 76e952ebd296aaa63316367f92a74063593b8aff Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 6 Oct 2021 21:21:40 +0300 Subject: [PATCH 044/147] Removing duplicate request dumping from integration tests, since nuclei already does it if they are started in debug mode --- v2/cmd/integration-test/http.go | 29 ----------------------------- v2/cmd/integration-test/workflow.go | 4 ---- 2 files changed, 33 deletions(-) diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 6d43c0cc7..28b52d7c1 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -7,7 +7,6 @@ import ( "net" "net/http" "net/http/httptest" - "net/http/httputil" "strings" "github.com/julienschmidt/httprouter" @@ -34,21 +33,12 @@ var httpTestcases = map[string]testutils.TestCase{ "http/request-condition-new.yaml": &httpRequestCondition{}, } -func httpDebugRequestDump(r *http.Request) { - if debug { - if dump, err := httputil.DumpRequest(r, true); err == nil { - fmt.Printf("\nRequest dump: \n%s\n\n", string(dump)) - } - } -} - type httpGetHeaders struct{} // Execute executes a test case and returns an error if occurred func (h *httpGetHeaders) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if strings.EqualFold(r.Header.Get("test"), "nuclei") { fmt.Fprintf(w, "This is test headers matcher text") } @@ -72,7 +62,6 @@ type httpGetQueryString struct{} func (h *httpGetQueryString) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") { fmt.Fprintf(w, "This is test querystring matcher text") } @@ -96,11 +85,9 @@ type httpGetRedirects struct{} func (h *httpGetRedirects) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) http.Redirect(w, r, "/redirected", http.StatusFound) }) router.GET("/redirected", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test redirects matcher text") }) ts := httptest.NewServer(router) @@ -122,7 +109,6 @@ type httpGet struct{} func (h *httpGet) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) @@ -146,7 +132,6 @@ func (h *httpPostBody) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err return @@ -179,8 +164,6 @@ func (h *httpPostJSONBody) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) - type doc struct { Username string `json:"username"` Password string `json:"password"` @@ -218,7 +201,6 @@ func (h *httpPostMultipartBody) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseMultipartForm(1 * 1024); err != nil { routerErr = err return @@ -261,7 +243,6 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err return @@ -271,7 +252,6 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error { } }) router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if strings.EqualFold(r.URL.Query().Get("username"), "nuclei") { fmt.Fprintf(w, "Test is test-dynamic-extractor-raw matcher text") } @@ -300,7 +280,6 @@ func (h *httpRawGetQuery) Execute(filePath string) error { var routerErr error router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") { fmt.Fprintf(w, "Test is test raw-get-query-matcher text") } @@ -329,8 +308,6 @@ func (h *httpRawGet) Execute(filePath string) error { var routerErr error router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) - fmt.Fprintf(w, "Test is test raw-get-matcher text") }) ts := httptest.NewServer(router) @@ -357,7 +334,6 @@ func (h *httpRawPayload) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err return @@ -393,7 +369,6 @@ func (h *httpRawPostBody) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err return @@ -426,7 +401,6 @@ func (h *httpRawCookieReuse) Execute(filePath string) error { var routerErr error router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err return @@ -436,7 +410,6 @@ func (h *httpRawCookieReuse) Execute(filePath string) error { } }) router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) if err := r.ParseForm(); err != nil { routerErr = err return @@ -500,11 +473,9 @@ func (h *httpRequestCondition) Execute(filePath string) error { var routerErr error router.GET("/200", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) w.WriteHeader(200) }) router.GET("/400", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) w.WriteHeader(400) }) ts := httptest.NewServer(router) diff --git a/v2/cmd/integration-test/workflow.go b/v2/cmd/integration-test/workflow.go index 08d6e1385..5f39b4ebf 100644 --- a/v2/cmd/integration-test/workflow.go +++ b/v2/cmd/integration-test/workflow.go @@ -23,7 +23,6 @@ type workflowBasic struct{} func (h *workflowBasic) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) @@ -45,7 +44,6 @@ type workflowConditionMatched struct{} func (h *workflowConditionMatched) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) @@ -67,7 +65,6 @@ type workflowConditionUnmatch struct{} func (h *workflowConditionUnmatch) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) @@ -89,7 +86,6 @@ type workflowMatcherName struct{} func (h *workflowMatcherName) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) From b5c161d2670418ab6ee13a0994f677dfb4cc9204 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 6 Oct 2021 21:53:03 +0300 Subject: [PATCH 045/147] [feature] Add coloring to debug information #999 * extracted common MakeResultEvent logic and added the MakeResultEventItem method to the Request interface --- .../helpers/eventcreator/eventcreator.go | 2 +- v2/pkg/protocols/dns/operators.go | 28 ++-------------- v2/pkg/protocols/file/operators.go | 29 +++-------------- v2/pkg/protocols/headless/operators.go | 28 ++-------------- v2/pkg/protocols/http/operators.go | 29 ++--------------- v2/pkg/protocols/network/operators.go | 28 ++-------------- v2/pkg/protocols/offlinehttp/operators.go | 28 ++-------------- v2/pkg/protocols/protocols.go | 32 ++++++++++++++++++- 8 files changed, 52 insertions(+), 152 deletions(-) diff --git a/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go index 6dbd400b2..73e3af13c 100644 --- a/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go +++ b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go @@ -17,7 +17,7 @@ func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent out if ok && result != nil { event.OperatorsResult = result addAdditionalOptions(event) - event.Results = request.MakeResultEvent(event) + event.Results = append(event.Results, request.MakeResultEvent(event)...) } } } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index cebc02a10..7c9aaa4df 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -115,33 +116,10 @@ func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched strin // MakeResultEvent creates a result event from internal wrapped event func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := request.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := request.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := request.makeResultEventItem(wrapped) - results = append(results, data) - } - return results + return protocols.MakeDefaultResultEvent(request, wrapped) } -func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 4e252b711..6c7e3b6d2 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -81,33 +82,13 @@ func (request *Request) responseToDSLMap(raw, host, matched string) output.Inter // MakeResultEvent creates a result event from internal wrapped event func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) + results := protocols.MakeDefaultResultEvent(request, wrapped) - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := request.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := request.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := request.makeResultEventItem(wrapped) - results = append(results, data) - } raw, ok := wrapped.InternalEvent["raw"] if !ok { return results } + rawStr, ok := raw.(string) if !ok { return results @@ -138,7 +119,7 @@ func (request *Request) GetCompiledOperators() []*operators.Operators { return []*operators.Operators{request.CompiledOperators} } -func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), @@ -146,7 +127,7 @@ func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent Type: "file", Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), - Host: types.ToString(wrapped.InternalEvent["matched"]), + Host: types.ToString(wrapped.InternalEvent["host"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Response: types.ToString(wrapped.InternalEvent["raw"]), Timestamp: time.Now(), diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index fad89761f..0d76d421b 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -80,37 +81,14 @@ func (request *Request) responseToDSLMap(resp, req, host, matched string) output // MakeResultEvent creates a result event from internal wrapped event func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := request.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := request.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := request.makeResultEventItem(wrapped) - results = append(results, data) - } - return results + return protocols.MakeDefaultResultEvent(request, wrapped) } func (request *Request) GetCompiledOperators() []*operators.Operators { return []*operators.Operators{request.CompiledOperators} } -func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 781b845ad..a53081945 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -125,38 +126,14 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw // MakeResultEvent creates a result event from internal wrapped event func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 && !wrapped.OperatorsResult.Matched { - return nil - } - - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := request.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := request.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := request.makeResultEventItem(wrapped) - results = append(results, data) - } - return results + return protocols.MakeDefaultResultEvent(request, wrapped) } func (request *Request) GetCompiledOperators() []*operators.Operators { return []*operators.Operators{request.CompiledOperators} } -func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index b5e14a721..a1e459173 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -81,37 +82,14 @@ func (request *Request) responseToDSLMap(req, resp, raw, host, matched string) o // MakeResultEvent creates a result event from internal wrapped event func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := request.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := request.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := request.makeResultEventItem(wrapped) - results = append(results, data) - } - return results + return protocols.MakeDefaultResultEvent(request, wrapped) } func (request *Request) GetCompiledOperators() []*operators.Operators { return []*operators.Operators{request.CompiledOperators} } -func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index d8fd842df..7789b7e4b 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -122,37 +123,14 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw // MakeResultEvent creates a result event from internal wrapped event func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := request.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := request.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := request.makeResultEventItem(wrapped) - results = append(results, data) - } - return results + return protocols.MakeDefaultResultEvent(request, wrapped) } func (request *Request) GetCompiledOperators() []*operators.Operators { return request.compiledOperators } -func (request *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index f18311fd3..d77ba147e 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -82,7 +82,9 @@ type Request interface { Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error - // MakeResultEvent creates a result event from internal wrapped event + // MakeResultEventItem creates a result event from internal wrapped event. Intended to be used by MakeResultEventItem internally + MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent + // MakeResultEvent creates a flat list of result events from an internal wrapped event, based on successful matchers and extracted data MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent // GetCompiledOperators returns a list of the compiled operators GetCompiledOperators() []*operators.Operators @@ -90,3 +92,31 @@ type Request interface { // OutputEventCallback is a callback event for any results found during scanning. type OutputEventCallback func(result *output.InternalWrappedEvent) + +func MakeDefaultResultEvent(request Request, wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + if len(wrapped.OperatorsResult.DynamicValues) > 0 && !wrapped.OperatorsResult.Matched { + return nil + } + + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for matcherNames := range wrapped.OperatorsResult.Matches { + data := request.MakeResultEventItem(wrapped) + data.MatcherName = matcherNames + results = append(results, data) + } + } else if len(wrapped.OperatorsResult.Extracts) > 0 { + for k, v := range wrapped.OperatorsResult.Extracts { + data := request.MakeResultEventItem(wrapped) + data.ExtractorName = k + data.ExtractedResults = v + results = append(results, data) + } + } else { + data := request.MakeResultEventItem(wrapped) + results = append(results, data) + } + return results +} From ce13bf34d09b60d29819b9f9fa41033f9dfa889e Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 7 Oct 2021 01:40:49 +0530 Subject: [PATCH 046/147] Added check for unresolved variables --- .../protocols/common/expressions/variables.go | 32 +++++++++++++++++ .../common/expressions/variables_test.go | 23 ++++++++++++ v2/pkg/protocols/http/http.go | 3 ++ v2/pkg/protocols/http/request.go | 36 +++++++++++-------- v2/pkg/protocols/network/request.go | 4 +++ 5 files changed, 84 insertions(+), 14 deletions(-) create mode 100644 v2/pkg/protocols/common/expressions/variables.go create mode 100644 v2/pkg/protocols/common/expressions/variables_test.go diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go new file mode 100644 index 000000000..01a1e2b00 --- /dev/null +++ b/v2/pkg/protocols/common/expressions/variables.go @@ -0,0 +1,32 @@ +package expressions + +import ( + "errors" + "regexp" + "strings" +) + +var unresolvedVariablesRegex = regexp.MustCompile(`\{\{(.+)\}\}`) + +// ContainsUnresolvedVariables returns an error with variable names if the passed +// input contains unresolved {{}} variables. +func ContainsUnresolvedVariables(data string) error { + matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) + if len(matches) == 0 { + return nil + } + errorString := &strings.Builder{} + errorString.WriteString("unresolved variables found: ") + + for i, match := range matches { + if len(match) < 2 { + continue + } + errorString.WriteString(match[1]) + if i != len(matches)-1 { + errorString.WriteString(",") + } + } + errorMessage := errorString.String() + return errors.New(errorMessage) +} diff --git a/v2/pkg/protocols/common/expressions/variables_test.go b/v2/pkg/protocols/common/expressions/variables_test.go new file mode 100644 index 000000000..55c35fffb --- /dev/null +++ b/v2/pkg/protocols/common/expressions/variables_test.go @@ -0,0 +1,23 @@ +package expressions + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUnresolvedVariablesCheck(t *testing.T) { + tests := []struct { + data string + err error + }{ + {"{{test}}", errors.New("unresolved variables found: test")}, + {"{{test}}/{{another}}", errors.New("unresolved variables found: test,another")}, + {"test", nil}, + } + for _, test := range tests { + err := ContainsUnresolvedVariables(test.data) + require.Equal(t, test.err, err, "could not get unresolved variables") + } +} diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 5558e7035..d79b99a25 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -167,6 +167,9 @@ type Request struct { // description: | // StopAtFirstMatch stops the execution of the requests and template as soon as a match is found. StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop the execution after a match is found"` + // description: | + // SkipVariablesCheck skips the check for unresolved variables in request + SkipVariablesCheck bool `yaml:"skip-variables-check,omitempty" jsonschema:"title=skip variable checks,description=Skips the check for unresolved variables in request"` } // GetID returns the unique ID of the request if any. diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index d8a2c98dc..0ee185e2b 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -19,6 +19,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" @@ -272,6 +273,27 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ err error ) + // For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function + if !request.original.Race { + var dumpError error + dumpedRequest, dumpError = dump(request, reqURL) + if dumpError != nil { + return dumpError + } + dumpedRequestString := string(dumpedRequest) + + // Check if are there any unresolved variables. If yes, skip unless overriden by user. + if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !r.SkipVariablesCheck { + gologger.Warning().Msgf("Could not make http request for %s: %v\n", reqURL, varErr) + return nil + } + + if r.options.Options.Debug || r.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) + gologger.Print().Msgf("%s") + } + } + var formedURL string var hostname string timeStart := time.Now() @@ -310,20 +332,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ resp, err = r.httpClient.Do(request.request) } } - - // For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function - if !request.original.Race { - var dumpError error - dumpedRequest, dumpError = dump(request, reqURL) - if dumpError != nil { - return dumpError - } - - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) - gologger.Print().Msgf("%s", string(dumpedRequest)) - } - } if err != nil { // rawhttp doesn't support draining response bodies. if resp != nil && resp.Body != nil && request.rawRequest == nil { diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 61abdf3fc..0fa6e6cca 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -139,6 +139,10 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin } reqBuilder.Write(finalData) + if varErr := expressions.ContainsUnresolvedVariables(string(finalData)); varErr != nil { + gologger.Warning().Msgf("Could not make network request for %s: %v\n", actualAddress, varErr) + return nil + } if _, err := conn.Write(finalData); err != nil { r.options.Output.Request(r.options.TemplateID, address, "network", err) r.options.Progress.IncrementFailedRequestsBy(1) From 92857497f35a8a4343aa4b9d143e9f0b06567c20 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 7 Oct 2021 01:48:10 +0530 Subject: [PATCH 047/147] Fixed variable check regex --- v2/pkg/protocols/common/expressions/variables.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go index 01a1e2b00..5bacaf60d 100644 --- a/v2/pkg/protocols/common/expressions/variables.go +++ b/v2/pkg/protocols/common/expressions/variables.go @@ -6,7 +6,7 @@ import ( "strings" ) -var unresolvedVariablesRegex = regexp.MustCompile(`\{\{(.+)\}\}`) +var unresolvedVariablesRegex = regexp.MustCompile(`\{\{([^}]+)\}\}["'\)\}]*`) // ContainsUnresolvedVariables returns an error with variable names if the passed // input contains unresolved {{}} variables. From 247ede7170cd1bab2a6666d25275562801cb1016 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 7 Oct 2021 01:48:48 +0530 Subject: [PATCH 048/147] Misc --- v2/pkg/protocols/http/request.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 0ee185e2b..3dbb87d3d 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -290,7 +290,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ if r.options.Options.Debug || r.options.Options.DebugRequests { gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) - gologger.Print().Msgf("%s") + gologger.Print().Msgf("%s", dumpedRequestString) } } From 0541040e6b7396e460c9ff6dbbe653bd9ba07813 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 7 Oct 2021 04:41:31 +0530 Subject: [PATCH 049/147] Added templateID to error message --- v2/pkg/protocols/http/request.go | 2 +- v2/pkg/protocols/network/request.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 3dbb87d3d..95deacc3a 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -284,7 +284,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ // Check if are there any unresolved variables. If yes, skip unless overriden by user. if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !r.SkipVariablesCheck { - gologger.Warning().Msgf("Could not make http request for %s: %v\n", reqURL, varErr) + gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", r.options.TemplateID, reqURL, varErr) return nil } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 0fa6e6cca..aad3c60ef 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -140,7 +140,7 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin reqBuilder.Write(finalData) if varErr := expressions.ContainsUnresolvedVariables(string(finalData)); varErr != nil { - gologger.Warning().Msgf("Could not make network request for %s: %v\n", actualAddress, varErr) + gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", r.options.TemplateID, actualAddress, varErr) return nil } if _, err := conn.Write(finalData); err != nil { From 4669ee22ce2526dc698a2503da951dbd49414429 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 7 Oct 2021 04:50:34 +0530 Subject: [PATCH 050/147] Misc changes to http request execution with variables --- v2/pkg/protocols/http/request.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 95deacc3a..0111de425 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -243,6 +243,10 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp callback(event) } }, requestCount) + // If a variable is unresolved, skip all further requests + if err == errStopExecution { + break + } if err != nil { if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.CheckError(err) { r.options.HostErrorsCache.MarkFailed(reqURL) @@ -252,8 +256,8 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp requestCount++ r.options.Progress.IncrementRequests() + // If this was a match and we want to stop at first match, skip all further requests. if (request.original.options.Options.StopAtFirstMatch || r.StopAtFirstMatch) && gotOutput { - r.options.Progress.IncrementErrorsBy(int64(generator.Total())) break } } @@ -262,6 +266,8 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp const drainReqSize = int64(8 * 1024) +var errStopExecution = errors.New("stop execution due to unresolved variables") + // executeRequest executes the actual generated request and returns error if occurred func (r *Request) executeRequest(reqURL string, request *generatedRequest, previous output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error { r.setCustomHeaders(request) @@ -285,7 +291,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ // Check if are there any unresolved variables. If yes, skip unless overriden by user. if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !r.SkipVariablesCheck { gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", r.options.TemplateID, reqURL, varErr) - return nil + return errStopExecution } if r.options.Options.Debug || r.options.Options.DebugRequests { From 514d6f94ecd5264628d958d6fc73b7df458bb0a0 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 7 Oct 2021 05:35:32 +0530 Subject: [PATCH 051/147] Added support for urlencoded variable variation --- v2/pkg/protocols/common/expressions/variables.go | 2 +- v2/pkg/protocols/common/expressions/variables_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go index 5bacaf60d..5026cef0e 100644 --- a/v2/pkg/protocols/common/expressions/variables.go +++ b/v2/pkg/protocols/common/expressions/variables.go @@ -6,7 +6,7 @@ import ( "strings" ) -var unresolvedVariablesRegex = regexp.MustCompile(`\{\{([^}]+)\}\}["'\)\}]*`) +var unresolvedVariablesRegex = regexp.MustCompile(`(?:%7[B|b]|\{){2}([^}]+)(?:%7[D|d]|\}){2}["'\)\}]*`) // ContainsUnresolvedVariables returns an error with variable names if the passed // input contains unresolved {{}} variables. diff --git a/v2/pkg/protocols/common/expressions/variables_test.go b/v2/pkg/protocols/common/expressions/variables_test.go index 55c35fffb..61d071351 100644 --- a/v2/pkg/protocols/common/expressions/variables_test.go +++ b/v2/pkg/protocols/common/expressions/variables_test.go @@ -15,6 +15,7 @@ func TestUnresolvedVariablesCheck(t *testing.T) { {"{{test}}", errors.New("unresolved variables found: test")}, {"{{test}}/{{another}}", errors.New("unresolved variables found: test,another")}, {"test", nil}, + {"%7b%7btest%7d%7d", errors.New("unresolved variables found: test")}, } for _, test := range tests { err := ContainsUnresolvedVariables(test.data) From 2ef034721f9ad425f21b76d86a156e2f96cfaf40 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 7 Oct 2021 14:23:38 +0530 Subject: [PATCH 052/147] interactsh client update --- v2/go.mod | 7 +- v2/go.sum | 390 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 388 insertions(+), 9 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 3f5502466..853143f55 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -34,7 +34,7 @@ require ( github.com/projectdiscovery/goflags v0.0.7 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa - github.com/projectdiscovery/interactsh v0.0.4 + github.com/projectdiscovery/interactsh v0.0.5 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a @@ -84,14 +84,14 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.0.0 // indirect - github.com/google/uuid v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect github.com/itchyny/timefmt-go v0.1.3 // indirect github.com/jasonlvhit/gocron v0.0.1 // indirect github.com/karlseguin/ccache/v2 v2.0.8 // indirect - github.com/klauspost/compress v1.13.4 // indirect + github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/mattn/go-isatty v0.0.13 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -99,6 +99,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect + github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca // indirect github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect diff --git a/v2/go.sum b/v2/go.sum index d908b8fe3..95a544eeb 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -31,13 +31,21 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= +github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Ice3man543/nvd v1.0.8 h1:2CBEgOxyWAkQocnnmEMmRtVPWooPRvcuHFLWj48EM4c= github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -48,13 +56,23 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c h1:oJsq4z4xKgZWWOhrSZuLZ5KyYfRFytddLL1E5+psfIY= github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= @@ -65,15 +83,29 @@ github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= @@ -82,6 +114,8 @@ github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaq github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I= github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -89,17 +123,38 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= +github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= +github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/errors v1.8.6/go.mod h1:hOm5fabihW+xEyY1kuypGwqT+Vt7rafg04ytBtIpeIQ= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= +github.com/cockroachdb/pebble v0.0.0-20210827150156-ff43a5880feb/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= +github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/redact v1.1.1/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= @@ -109,10 +164,12 @@ github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= @@ -120,33 +177,76 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/eggsampler/acme/v3 v3.2.1 h1:Lfsrg3M2zt00QRnizOFzdpSfsS9oDvPsGrodXS/w1KI= github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI= github.com/go-rod/rod v0.101.7 h1:kbI5CNvcRhf7feybBln4xDutsM0mbsF0ENNZfKcF6WA= github.com/go-rod/rod v0.101.7/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= @@ -179,6 +279,7 @@ github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -209,32 +310,70 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs= github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= +github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o= github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg= @@ -243,15 +382,25 @@ github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrn github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA= @@ -260,13 +409,25 @@ github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLM github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= +github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= +github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= +github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= -github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -277,25 +438,48 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= +github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -304,34 +488,70 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= +github.com/onsi/ginkgo v1.16.2 h1:HFB2fbVIlhIfCfOW81bZFbiC/RvnpXSdhbF2/DJr134= +github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= +github.com/onsi/gomega v1.12.0 h1:p4oGGk2M2UJc0wWN4lHFvIB71lxsh0T/UiKCCgFADY8= +github.com/onsi/gomega v1.12.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/owenrumney/go-sarif v1.0.11 h1:7k4TLSi6h3vAozSECjO0arcQoeUNDMgvA7LDac95sJo= github.com/owenrumney/go-sarif v1.0.11/go.mod h1:hTBFbxU7GuVRUvwMx+eStp9M/Oun4xHCS3vqpPvket8= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwaFH1jvtOo5ndhTQgoA349ozhX+1dc4b6tbaPnBOA= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= @@ -339,8 +559,11 @@ github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydK github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= +github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= +github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca h1:xT//ApxoeQRbt9GgL/122688bhGy9hur8YG0Qh69k5I= +github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= @@ -348,11 +571,14 @@ github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTt github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= +github.com/projectdiscovery/hmap v0.0.2-0.20210727180307-d63d35146e97/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= +github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f/go.mod h1:RLM8b1z2HEq74u5AXN1Lbvfq+1BZWpnTQJcwLnMLA54= github.com/projectdiscovery/hmap v0.0.2-0.20210917073634-bfb0e9c03800/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= -github.com/projectdiscovery/interactsh v0.0.4 h1:3BtCZrrTovGYiqdFktXJ4NxKAQFvUvzcEI5pJIuShM8= github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= +github.com/projectdiscovery/interactsh v0.0.5 h1:7NtFTbbn1Aty3RnB95DXaGw445mSXXbCAPOX5RXWTSU= +github.com/projectdiscovery/interactsh v0.0.5/go.mod h1:dB/c1A9I2trIHfMbU/wNzxQkGara0UE33zDtGcdBh+U= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= @@ -383,29 +609,63 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil/v3 v3.21.7 h1:PnTqQamUjwEDSgn+nBGu0qSDV/CfvyiR/gwTH3i7HTU= github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -413,11 +673,17 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -441,23 +707,36 @@ github.com/tklauser/go-sysconf v0.3.7 h1:HT7h4+536gjqeq1ZIJPgOl1rg1XFatQGVZWp7Py github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4= github.com/tklauser/numcpus v0.2.3 h1:nQ0QYpiritP6ViFhrKYsiv6VVxOpum2Gks5GhnJbS/8= github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM= github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY= github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0= @@ -475,39 +754,67 @@ github.com/ysmood/gson v0.6.4/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3R github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw= github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zclconf/go-cty v1.8.4 h1:pwhhz5P+Fjxse7S7UriBrMu6AUJSZM5pKqGem1PjGAs= github.com/zclconf/go-cty v1.8.4/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -515,6 +822,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20210826195003-46c773283d9d/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -527,29 +836,42 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -570,10 +892,13 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -595,27 +920,38 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -629,6 +965,7 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -642,7 +979,9 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= @@ -658,16 +997,23 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -677,6 +1023,8 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -685,6 +1033,8 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -701,17 +1051,21 @@ golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -729,6 +1083,7 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -737,11 +1092,13 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -766,10 +1123,17 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -778,6 +1142,7 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -792,18 +1157,27 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nRnFU= gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DSo0nK9U+c0Y= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -814,6 +1188,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -821,7 +1196,10 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= From 12b6b2ca89cc13c1587f7d7a34ea0c0a44cb25fb Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 7 Oct 2021 12:36:27 +0200 Subject: [PATCH 053/147] Add support for CLI payload variables --- v2/go.mod | 5 +-- v2/go.sum | 10 ++++-- v2/internal/runner/options.go | 9 +++++ .../protocols/common/expressions/variables.go | 24 ++++++++++++++ v2/pkg/protocols/http/http.go | 33 +++++++++++++++++++ v2/pkg/protocols/http/request.go | 7 ---- v2/pkg/protocols/network/network.go | 23 +++++++++++++ v2/pkg/types/types.go | 14 ++++++++ 8 files changed, 114 insertions(+), 11 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 3f5502466..7c0bd3c7f 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -31,11 +31,11 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e - github.com/projectdiscovery/goflags v0.0.7 + github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 + github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/interactsh v0.0.4 - github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a github.com/projectdiscovery/retryablehttp-go v1.0.2 @@ -102,6 +102,7 @@ require ( github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect + github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/tklauser/go-sysconf v0.3.7 // indirect github.com/tklauser/numcpus v0.2.3 // indirect diff --git a/v2/go.sum b/v2/go.sum index d908b8fe3..ccbd26248 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -341,8 +341,14 @@ github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= +github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= +github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/goflags v0.0.7-0.20211007101748-dd9c32bab303 h1:5ownIWK6jZYdg/pM+H5wfC4zF0rOvDeHtejbpScZt8I= +github.com/projectdiscovery/goflags v0.0.7-0.20211007101748-dd9c32bab303/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= +github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 h1:b7zDUSsgN5f4/IlhKF6RVGsp/NkHIuty0o1YjzAMKUs= +github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= @@ -365,8 +371,8 @@ github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= -github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 h1:SNtAiRRrJtDJJDroaa/bFXt/Tix2LA6+rHRib0ORlJQ= -github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df h1:CvTNAUD5JbLMqpMFoGNgfk2gOcN0NC57ICu0+oK84vs= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY= github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index eb57d94aa..e993fedb9 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" @@ -61,6 +62,14 @@ func ParseOptions(options *types.Options) { // Load the resolvers if user asked for them loadResolvers(options) + // removes all cli variables containing payloads and add them to the internal struct + for key, value := range options.Vars.AsMap() { + if fileutil.FileExists(value.(string)) { + options.Vars.Del(key) + options.AddVarPayload(key, value) + } + } + err := protocolinit.Init(options) if err != nil { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go index 5026cef0e..48ac546ed 100644 --- a/v2/pkg/protocols/common/expressions/variables.go +++ b/v2/pkg/protocols/common/expressions/variables.go @@ -30,3 +30,27 @@ func ContainsUnresolvedVariables(data string) error { errorMessage := errorString.String() return errors.New(errorMessage) } + +func ContainsVariablesWithNames(data string, names map[string]interface{}) error { + matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) + if len(matches) == 0 { + return nil + } + errorString := &strings.Builder{} + errorString.WriteString("unresolved variables with values found: ") + + for i, match := range matches { + if len(match) < 2 { + continue + } + matchName := match[1] + if _, ok := names[matchName]; !ok { + errorString.WriteString(matchName) + if i != len(matches)-1 { + errorString.WriteString(",") + } + } + } + errorMessage := errorString.String() + return errors.New(errorMessage) +} diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index d79b99a25..2671c039f 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -1,12 +1,15 @@ package http import ( + "fmt" "strings" "github.com/pkg/errors" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/rawhttp" @@ -225,6 +228,36 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { r.CompiledOperators = compiled } + // Resolve payload paths from vars if they exists + for name, payload := range r.options.Options.VarsPayload() { + payloadStr, ok := payload.(string) + // check if inputs contains the payload + var hasPayloadName bool + // search for markers in all request parts + var inputs []string + inputs = append(inputs, r.Method, r.Body) + inputs = append(inputs, r.Raw...) + for k, v := range r.customHeaders { + inputs = append(inputs, fmt.Sprintf("%s: %s", k, v)) + } + for k, v := range r.Headers { + inputs = append(inputs, fmt.Sprintf("%s: %s", k, v)) + } + + for _, input := range inputs { + if expressions.ContainsVariablesWithNames(input, map[string]interface{}{name: payload}) == nil { + hasPayloadName = true + break + } + } + if ok && hasPayloadName && fileutil.FileExists(payloadStr) { + if r.Payloads == nil { + r.Payloads = make(map[string]interface{}) + } + r.Payloads[name] = payloadStr + } + } + if len(r.Payloads) > 0 { attackType := r.AttackType if attackType == "" { diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 0111de425..e4966f36a 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -19,7 +19,6 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" @@ -288,12 +287,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ } dumpedRequestString := string(dumpedRequest) - // Check if are there any unresolved variables. If yes, skip unless overriden by user. - if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !r.SkipVariablesCheck { - gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", r.options.TemplateID, reqURL, varErr) - return errStopExecution - } - if r.options.Options.Debug || r.options.Options.DebugRequests { gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) gologger.Print().Msgf("%s", dumpedRequestString) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 3ad431ed0..68eca713d 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" @@ -150,6 +151,28 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } } + // Resolve payload paths from vars if they exists + for name, payload := range r.options.Options.VarsPayload() { + payloadStr, ok := payload.(string) + // check if inputs contains the payload + var hasPayloadName bool + for _, input := range r.Inputs { + if input.Type != "" { + continue + } + if expressions.ContainsVariablesWithNames(input.Data, map[string]interface{}{name: payload}) == nil { + hasPayloadName = true + break + } + } + if ok && hasPayloadName && fileutil.FileExists(payloadStr) { + if r.Payloads == nil { + r.Payloads = make(map[string]interface{}) + } + r.Payloads[name] = payloadStr + } + } + if len(r.Payloads) > 0 { attackType := r.AttackType if attackType == "" { diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 91db2b91b..b2b6e34e2 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -23,6 +23,8 @@ type Options struct { CustomHeaders goflags.StringSlice // Vars is the list of custom global vars Vars goflags.RuntimeMap + // vars to use as iterative payload + varsPayload map[string]interface{} // Severities filters templates based on their severity and only run the matching ones. Severities severity.Severities // Author filters templates based on their author and only run the matching ones. @@ -159,3 +161,15 @@ type Options struct { // EnvironmentVariables enables support for environment variables EnvironmentVariables bool } + +func (options *Options) AddVarPayload(key string, value interface{}) { + if options.varsPayload == nil { + options.varsPayload = make(map[string]interface{}) + } + + options.varsPayload[key] = value +} + +func (options *Options) VarsPayload() map[string]interface{} { + return options.varsPayload +} From 571f6b8c4cf8983b0f7e54fd9a7eb4c52f8072d8 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 7 Oct 2021 12:39:22 +0200 Subject: [PATCH 054/147] misc --- v2/go.mod | 3 +-- v2/go.sum | 6 +----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 542393da2..7b3300fed 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -36,6 +36,7 @@ require ( github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/interactsh v0.0.5 + github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a github.com/projectdiscovery/retryablehttp-go v1.0.2 @@ -99,11 +100,9 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect - github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca // indirect github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect - github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/tklauser/go-sysconf v0.3.7 // indirect github.com/tklauser/numcpus v0.2.3 // indirect diff --git a/v2/go.sum b/v2/go.sum index 2a7541c01..2031719e4 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -562,13 +562,9 @@ github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/i github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= +github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= -github.com/projectdiscovery/goflags v0.0.7-0.20211007101748-dd9c32bab303 h1:5ownIWK6jZYdg/pM+H5wfC4zF0rOvDeHtejbpScZt8I= -github.com/projectdiscovery/goflags v0.0.7-0.20211007101748-dd9c32bab303/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= -github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca h1:xT//ApxoeQRbt9GgL/122688bhGy9hur8YG0Qh69k5I= -github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= -github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 h1:b7zDUSsgN5f4/IlhKF6RVGsp/NkHIuty0o1YjzAMKUs= github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= From fc31688f697e759fd57abf1993683dc4dcb4d06e Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 7 Oct 2021 12:40:18 +0200 Subject: [PATCH 055/147] fixing lint errors --- v2/internal/runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index e993fedb9..745f15d02 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -65,7 +65,7 @@ func ParseOptions(options *types.Options) { // removes all cli variables containing payloads and add them to the internal struct for key, value := range options.Vars.AsMap() { if fileutil.FileExists(value.(string)) { - options.Vars.Del(key) + _ = options.Vars.Del(key) options.AddVarPayload(key, value) } } From 7651edfa62912671575687cf0c22054fc194077a Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 7 Oct 2021 13:02:15 +0200 Subject: [PATCH 056/147] misc --- v2/pkg/protocols/http/request.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index e4966f36a..0111de425 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -19,6 +19,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" @@ -287,6 +288,12 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ } dumpedRequestString := string(dumpedRequest) + // Check if are there any unresolved variables. If yes, skip unless overriden by user. + if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !r.SkipVariablesCheck { + gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", r.options.TemplateID, reqURL, varErr) + return errStopExecution + } + if r.options.Options.Debug || r.options.Options.DebugRequests { gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) gologger.Print().Msgf("%s", dumpedRequestString) From 92e410fdb5f6ec3d7c8469b768071d41d7f566e4 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 7 Oct 2021 19:38:31 +0530 Subject: [PATCH 057/147] Added variable check to dns protocol --- v2/pkg/protocols/dns/request.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index ae3041f33..25721aff8 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -7,6 +7,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" ) var _ protocols.Request = &Request{} @@ -29,9 +30,14 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused return errors.Wrap(err, "could not build request") } + requestString := compiledRequest.String() + if varErr := expressions.ContainsUnresolvedVariables(string(requestString)); varErr != nil { + gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", r.options.TemplateID, domain, varErr) + return nil + } if r.options.Options.Debug || r.options.Options.DebugRequests { gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", r.options.TemplateID, domain) - gologger.Print().Msgf("%s", compiledRequest.String()) + gologger.Print().Msgf("%s", requestString) } // Send the request to the target servers From 8d6193ef1f1a3cfd2c24b37cc50418fe07acba49 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 7 Oct 2021 19:40:16 +0530 Subject: [PATCH 058/147] Misc --- v2/pkg/protocols/dns/request.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 25721aff8..9c72d9d85 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -31,7 +31,7 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused } requestString := compiledRequest.String() - if varErr := expressions.ContainsUnresolvedVariables(string(requestString)); varErr != nil { + if varErr := expressions.ContainsUnresolvedVariables(requestString); varErr != nil { gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", r.options.TemplateID, domain, varErr) return nil } From 63ac86e60b3abb92c4dc24cd43262cbba29f2ad9 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 7 Oct 2021 19:43:38 +0530 Subject: [PATCH 059/147] Updated go.mod/sum --- v2/go.mod | 7 +- v2/go.sum | 390 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 388 insertions(+), 9 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 3f5502466..853143f55 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -34,7 +34,7 @@ require ( github.com/projectdiscovery/goflags v0.0.7 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa - github.com/projectdiscovery/interactsh v0.0.4 + github.com/projectdiscovery/interactsh v0.0.5 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a @@ -84,14 +84,14 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.0.0 // indirect - github.com/google/uuid v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect github.com/itchyny/timefmt-go v0.1.3 // indirect github.com/jasonlvhit/gocron v0.0.1 // indirect github.com/karlseguin/ccache/v2 v2.0.8 // indirect - github.com/klauspost/compress v1.13.4 // indirect + github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/mattn/go-isatty v0.0.13 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -99,6 +99,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect + github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca // indirect github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect diff --git a/v2/go.sum b/v2/go.sum index d908b8fe3..95a544eeb 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -31,13 +31,21 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= +github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Ice3man543/nvd v1.0.8 h1:2CBEgOxyWAkQocnnmEMmRtVPWooPRvcuHFLWj48EM4c= github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -48,13 +56,23 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c h1:oJsq4z4xKgZWWOhrSZuLZ5KyYfRFytddLL1E5+psfIY= github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= @@ -65,15 +83,29 @@ github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= @@ -82,6 +114,8 @@ github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaq github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I= github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -89,17 +123,38 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= +github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= +github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/errors v1.8.6/go.mod h1:hOm5fabihW+xEyY1kuypGwqT+Vt7rafg04ytBtIpeIQ= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= +github.com/cockroachdb/pebble v0.0.0-20210827150156-ff43a5880feb/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= +github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/redact v1.1.1/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= @@ -109,10 +164,12 @@ github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= @@ -120,33 +177,76 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/eggsampler/acme/v3 v3.2.1 h1:Lfsrg3M2zt00QRnizOFzdpSfsS9oDvPsGrodXS/w1KI= github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI= github.com/go-rod/rod v0.101.7 h1:kbI5CNvcRhf7feybBln4xDutsM0mbsF0ENNZfKcF6WA= github.com/go-rod/rod v0.101.7/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= @@ -179,6 +279,7 @@ github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -209,32 +310,70 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs= github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= +github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o= github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg= @@ -243,15 +382,25 @@ github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrn github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA= @@ -260,13 +409,25 @@ github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLM github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= +github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= +github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= +github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= -github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -277,25 +438,48 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= +github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -304,34 +488,70 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= +github.com/onsi/ginkgo v1.16.2 h1:HFB2fbVIlhIfCfOW81bZFbiC/RvnpXSdhbF2/DJr134= +github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= +github.com/onsi/gomega v1.12.0 h1:p4oGGk2M2UJc0wWN4lHFvIB71lxsh0T/UiKCCgFADY8= +github.com/onsi/gomega v1.12.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/owenrumney/go-sarif v1.0.11 h1:7k4TLSi6h3vAozSECjO0arcQoeUNDMgvA7LDac95sJo= github.com/owenrumney/go-sarif v1.0.11/go.mod h1:hTBFbxU7GuVRUvwMx+eStp9M/Oun4xHCS3vqpPvket8= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwaFH1jvtOo5ndhTQgoA349ozhX+1dc4b6tbaPnBOA= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= @@ -339,8 +559,11 @@ github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydK github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= +github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= +github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca h1:xT//ApxoeQRbt9GgL/122688bhGy9hur8YG0Qh69k5I= +github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= @@ -348,11 +571,14 @@ github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTt github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= +github.com/projectdiscovery/hmap v0.0.2-0.20210727180307-d63d35146e97/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= +github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f/go.mod h1:RLM8b1z2HEq74u5AXN1Lbvfq+1BZWpnTQJcwLnMLA54= github.com/projectdiscovery/hmap v0.0.2-0.20210917073634-bfb0e9c03800/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= -github.com/projectdiscovery/interactsh v0.0.4 h1:3BtCZrrTovGYiqdFktXJ4NxKAQFvUvzcEI5pJIuShM8= github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= +github.com/projectdiscovery/interactsh v0.0.5 h1:7NtFTbbn1Aty3RnB95DXaGw445mSXXbCAPOX5RXWTSU= +github.com/projectdiscovery/interactsh v0.0.5/go.mod h1:dB/c1A9I2trIHfMbU/wNzxQkGara0UE33zDtGcdBh+U= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= @@ -383,29 +609,63 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil/v3 v3.21.7 h1:PnTqQamUjwEDSgn+nBGu0qSDV/CfvyiR/gwTH3i7HTU= github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -413,11 +673,17 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -441,23 +707,36 @@ github.com/tklauser/go-sysconf v0.3.7 h1:HT7h4+536gjqeq1ZIJPgOl1rg1XFatQGVZWp7Py github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4= github.com/tklauser/numcpus v0.2.3 h1:nQ0QYpiritP6ViFhrKYsiv6VVxOpum2Gks5GhnJbS/8= github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM= github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY= github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0= @@ -475,39 +754,67 @@ github.com/ysmood/gson v0.6.4/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3R github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw= github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zclconf/go-cty v1.8.4 h1:pwhhz5P+Fjxse7S7UriBrMu6AUJSZM5pKqGem1PjGAs= github.com/zclconf/go-cty v1.8.4/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -515,6 +822,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20210826195003-46c773283d9d/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -527,29 +836,42 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -570,10 +892,13 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -595,27 +920,38 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -629,6 +965,7 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -642,7 +979,9 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= @@ -658,16 +997,23 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -677,6 +1023,8 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -685,6 +1033,8 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -701,17 +1051,21 @@ golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -729,6 +1083,7 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -737,11 +1092,13 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -766,10 +1123,17 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -778,6 +1142,7 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -792,18 +1157,27 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nRnFU= gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DSo0nK9U+c0Y= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -814,6 +1188,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -821,7 +1196,10 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= From 5b3b5b82800ffe0335131b94c7ad337de3b9d00d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 7 Oct 2021 19:44:22 +0530 Subject: [PATCH 060/147] misc --- v2/go.mod | 101 ++++++++++++++++++++++++++---------------------------- 1 file changed, 49 insertions(+), 52 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 853143f55..41f3f8955 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -3,38 +3,75 @@ module github.com/projectdiscovery/nuclei/v2 go 1.17 require ( + git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect github.com/Ice3man543/nvd v1.0.8 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible + github.com/PuerkitoBio/goquery v1.6.0 // indirect + github.com/StackExchange/wmi v1.2.1 // indirect github.com/akrylysov/pogreb v0.10.1 // indirect github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c + github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/andybalholm/cascadia v1.1.0 // indirect github.com/andygrunwald/go-jira v1.14.0 github.com/antchfx/htmlquery v1.2.3 + github.com/antchfx/xpath v1.1.6 // indirect github.com/apex/log v1.9.0 github.com/blang/semver v3.5.1+incompatible github.com/bluele/gcache v0.0.2 github.com/c4milo/unpackit v0.1.0 // indirect + github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/corpix/uarand v0.1.1 + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/dsnet/compress v0.0.1 // indirect + github.com/eggsampler/acme/v3 v3.2.1 // indirect + github.com/fatih/structs v1.1.0 // indirect + github.com/go-ole/go-ole v1.2.5 // indirect github.com/go-rod/rod v0.101.7 + github.com/golang-jwt/jwt v3.2.1+incompatible // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github v17.0.0+incompatible + github.com/google/go-querystring v1.0.0 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/gosuri/uilive v0.0.4 // indirect github.com/gosuri/uiprogress v0.0.1 // indirect + github.com/hashicorp/go-cleanhttp v0.5.1 // indirect + github.com/hashicorp/go-retryablehttp v0.6.8 // indirect + github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect github.com/itchyny/gojq v0.12.4 + github.com/itchyny/timefmt-go v0.1.3 // indirect + github.com/jasonlvhit/gocron v0.0.1 // indirect github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible + github.com/karlseguin/ccache/v2 v2.0.8 // indirect github.com/karrick/godirwalk v1.16.1 + github.com/klauspost/compress v1.13.6 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible + github.com/mattn/go-isatty v0.0.13 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/miekg/dns v1.1.43 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/olekukonko/tablewriter v0.0.5 github.com/owenrumney/go-sarif v1.0.11 github.com/pkg/errors v0.9.1 + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/clistats v0.0.8 + github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e + github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca // indirect github.com/projectdiscovery/goflags v0.0.7 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/interactsh v0.0.5 + github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect + github.com/projectdiscovery/mapcidr v0.0.8 // indirect + github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a @@ -42,6 +79,7 @@ require ( github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/yamldoc-go v1.0.2 github.com/remeh/sizedwaitgroup v1.0.0 + github.com/rivo/uniseg v0.2.0 // indirect github.com/rs/xid v1.3.0 github.com/segmentio/ksuid v1.0.4 github.com/shirou/gopsutil/v3 v3.21.7 @@ -50,10 +88,19 @@ require ( github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible + github.com/tklauser/go-sysconf v0.3.7 // indirect + github.com/tklauser/numcpus v0.2.3 // indirect + github.com/trivago/tgo v1.0.7 // indirect + github.com/ulikunitz/xz v0.5.10 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 github.com/xanzy/go-gitlab v0.50.3 + github.com/yl2chen/cidranger v1.0.2 // indirect + github.com/ysmood/goob v0.3.0 // indirect github.com/ysmood/gson v0.6.4 // indirect github.com/ysmood/leakless v0.7.0 // indirect + github.com/zclconf/go-cty v1.8.4 // indirect + go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 go.uber.org/multierr v1.7.0 go.uber.org/ratelimit v0.2.0 @@ -61,60 +108,10 @@ require ( golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect golang.org/x/text v0.3.7 - google.golang.org/appengine v1.6.7 // indirect - gopkg.in/yaml.v2 v2.4.0 -) - -require ( - git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect - github.com/PuerkitoBio/goquery v1.6.0 // indirect - github.com/StackExchange/wmi v1.2.1 // indirect - github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect - github.com/andybalholm/cascadia v1.1.0 // indirect - github.com/antchfx/xpath v1.1.6 // indirect - github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/dsnet/compress v0.0.1 // indirect - github.com/eggsampler/acme/v3 v3.2.1 // indirect - github.com/fatih/structs v1.1.0 // indirect - github.com/go-ole/go-ole v1.2.5 // indirect - github.com/golang-jwt/jwt v3.2.1+incompatible // indirect - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-querystring v1.0.0 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/hashicorp/go-cleanhttp v0.5.1 // indirect - github.com/hashicorp/go-retryablehttp v0.6.8 // indirect - github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect - github.com/itchyny/timefmt-go v0.1.3 // indirect - github.com/jasonlvhit/gocron v0.0.1 // indirect - github.com/karlseguin/ccache/v2 v2.0.8 // indirect - github.com/klauspost/compress v1.13.6 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect - github.com/mattn/go-isatty v0.0.13 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect - github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect - github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca // indirect - github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect - github.com/projectdiscovery/mapcidr v0.0.8 // indirect - github.com/projectdiscovery/networkpolicy v0.0.1 // indirect - github.com/rivo/uniseg v0.2.0 // indirect - github.com/tklauser/go-sysconf v0.3.7 // indirect - github.com/tklauser/numcpus v0.2.3 // indirect - github.com/trivago/tgo v1.0.7 // indirect - github.com/ulikunitz/xz v0.5.10 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/yl2chen/cidranger v1.0.2 // indirect - github.com/ysmood/goob v0.3.0 // indirect - github.com/zclconf/go-cty v1.8.4 // indirect - go.etcd.io/bbolt v1.3.6 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect + gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) From c4b69f59916319d6178d3356109ae670f3d10a6e Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 7 Oct 2021 21:03:00 +0530 Subject: [PATCH 061/147] interactsh update --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 853143f55..a0b3beb4c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -34,7 +34,7 @@ require ( github.com/projectdiscovery/goflags v0.0.7 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa - github.com/projectdiscovery/interactsh v0.0.5 + github.com/projectdiscovery/interactsh v0.0.6 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a diff --git a/v2/go.sum b/v2/go.sum index 95a544eeb..e4d6e79d4 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -577,8 +577,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210917073634-bfb0e9c03800/go.mod h1: github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= -github.com/projectdiscovery/interactsh v0.0.5 h1:7NtFTbbn1Aty3RnB95DXaGw445mSXXbCAPOX5RXWTSU= -github.com/projectdiscovery/interactsh v0.0.5/go.mod h1:dB/c1A9I2trIHfMbU/wNzxQkGara0UE33zDtGcdBh+U= +github.com/projectdiscovery/interactsh v0.0.6 h1:z2nVuc2FDRtOVjoYySiG0by/JA1Dwmgzb2rBFJEFR/c= +github.com/projectdiscovery/interactsh v0.0.6/go.mod h1:dB/c1A9I2trIHfMbU/wNzxQkGara0UE33zDtGcdBh+U= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= From 100cd41548bd9beb00e249a9bf33f89d6dec995b Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 7 Oct 2021 21:11:02 +0530 Subject: [PATCH 062/147] mod update --- v2/go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/v2/go.sum b/v2/go.sum index 31fbfb80a..f8a8360e2 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -562,6 +562,7 @@ github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/i github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= +github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca h1:xT//ApxoeQRbt9GgL/122688bhGy9hur8YG0Qh69k5I= github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= @@ -593,6 +594,7 @@ github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df h1:CvTNAUD5JbLMqpMFoGNgfk2gOcN0NC57ICu0+oK84vs= github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY= From bdc38a49a664c8ba32f2eb23e32f2fb43cdef8b8 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 7 Oct 2021 21:28:54 +0530 Subject: [PATCH 063/147] misc update --- v2/go.mod | 4 ++-- v2/go.sum | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index a0b3beb4c..471e12c5c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -31,7 +31,8 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e - github.com/projectdiscovery/goflags v0.0.7 + github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 + github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/interactsh v0.0.6 @@ -99,7 +100,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect - github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca // indirect github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect diff --git a/v2/go.sum b/v2/go.sum index f8a8360e2..e85a7fce4 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -562,7 +562,6 @@ github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/i github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= -github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca h1:xT//ApxoeQRbt9GgL/122688bhGy9hur8YG0Qh69k5I= github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= @@ -594,9 +593,8 @@ github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 h1:SNtAiRRrJtDJJDroaa/bFXt/Tix2LA6+rHRib0ORlJQ= github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= -github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df h1:CvTNAUD5JbLMqpMFoGNgfk2gOcN0NC57ICu0+oK84vs= -github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY= github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= From 0a9704ebd8d5f06bb1562bcb41db07926462bb7d Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 7 Oct 2021 20:54:12 +0300 Subject: [PATCH 064/147] [feature] Add coloring to debug information #999 * code-review fixes --- v2/internal/runner/update.go | 89 +++++++++++-------- v2/pkg/catalog/config/config.go | 3 +- .../helpers/eventcreator/eventcreator.go | 2 + .../response_highligther.go | 2 +- 4 files changed, 57 insertions(+), 39 deletions(-) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 3fc26131f..567d2577f 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -48,11 +48,7 @@ var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) // // If the path exists but does not contain the latest version of public templates, // the new version is downloaded from GitHub to the templates' directory, overwriting the old content. -func (r *Runner) updateTemplates() error { - if r.options.NoUpdateTemplates && !r.options.UpdateTemplates { - return nil - } - +func (r *Runner) updateTemplates() error { // TODO this method does more than just update templates. Should be refactored. home, err := os.UserHomeDir() if err != nil { return err @@ -76,6 +72,10 @@ func (r *Runner) updateTemplates() error { r.templatesConfig = currentConfig } + if r.options.NoUpdateTemplates && !r.options.UpdateTemplates { + return nil + } + client.InitNucleiVersion(config.Version) r.fetchLatestVersionsFromGithub(configDir) // also fetch the latest versions @@ -122,63 +122,78 @@ func (r *Runner) updateTemplates() error { return nil } - // Get the configuration currently on disk. - verText := r.templatesConfig.TemplateVersion - indices := reVersion.FindStringIndex(verText) - if indices == nil { - return fmt.Errorf("invalid release found with tag %s", err) - } - if indices[0] > 0 { - verText = verText[indices[0]:] - } - - oldVersion, err := semver.Make(verText) + latestVersion, currentVersion, err := getVersions(r) if err != nil { return err } - version, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion) - if err != nil { - return err - } - - if version.EQ(oldVersion) { + if latestVersion.EQ(currentVersion) { if r.options.UpdateTemplates { gologger.Info().Msgf("No new updates found for nuclei templates") } return config.WriteConfiguration(r.templatesConfig) } - if version.GT(oldVersion) { - gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String()) + if err := updateTemplates(latestVersion, currentVersion, r, ctx); err != nil { + return err + } + return nil +} + +func updateTemplates(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error { + if latestVersion.GT(currentVersion) { + gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", currentVersion, latestVersion.String()) gologger.Info().Msgf("Downloading latest release...") - if r.options.TemplatesDirectory != "" { - r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory + if runner.options.TemplatesDirectory != "" { + runner.templatesConfig.TemplatesDirectory = runner.options.TemplatesDirectory } - r.templatesConfig.TemplateVersion = version.String() + runner.templatesConfig.TemplateVersion = latestVersion.String() - gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) + gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", latestVersion.String(), runner.templatesConfig.TemplatesDirectory) - asset, err := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiTemplatesLatestVersion) + asset, err := runner.getLatestReleaseFromGithub(runner.templatesConfig.NucleiTemplatesLatestVersion) if err != nil { return err } - if _, err := r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()); err != nil { + if _, err := runner.downloadReleaseAndUnzip(ctx, latestVersion.String(), asset.GetZipballURL()); err != nil { return err } - if err := config.WriteConfiguration(r.templatesConfig); err != nil { + if err := config.WriteConfiguration(runner.templatesConfig); err != nil { return err } - gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", version.String()) + gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", latestVersion.String()) } return nil } +func getVersions(runner *Runner) (semver.Version, semver.Version, error) { + // Get the configuration currently on disk. + verText := runner.templatesConfig.TemplateVersion + indices := reVersion.FindStringIndex(verText) + if indices == nil { + return semver.Version{}, semver.Version{}, fmt.Errorf("invalid release found with tag %s", verText) + } + if indices[0] > 0 { + verText = verText[indices[0]:] + } + + currentVersion, err := semver.Make(verText) + if err != nil { + return semver.Version{}, semver.Version{}, err + } + + latestVersion, err := semver.Parse(runner.templatesConfig.NucleiTemplatesLatestVersion) + if err != nil { + return semver.Version{}, semver.Version{}, err + } + return latestVersion, currentVersion, nil +} + // readInternalConfigurationFile reads the internal configuration file for nuclei func (r *Runner) readInternalConfigurationFile(home, configDir string) error { templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename) - if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) { + if _, statErr := os.Stat(templatesConfigFile); os.IsExist(statErr) { configuration, readErr := config.ReadConfiguration() if readErr != nil { return readErr @@ -211,9 +226,9 @@ func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool { // getLatestReleaseFromGithub returns the latest release from GitHub func (r *Runner) getLatestReleaseFromGithub(latestTag string) (*github.RepositoryRelease, error) { - client := github.NewClient(nil) + gitHubClient := github.NewClient(nil) - release, _, err := client.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag) + release, _, err := gitHubClient.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag) if err != nil { return nil, err } @@ -362,7 +377,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat for templatePath, templateChecksums := range templateChecksumsMap { _, ok := results.checksums[templatePath] if !ok && templateChecksums[0] == templateChecksums[1] { - os.Remove(templatePath) + _ = os.Remove(templatePath) results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(templatePath, r.templatesConfig.TemplatesDirectory), string(os.PathSeparator))) } } @@ -457,7 +472,7 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st // fetchLatestVersionsFromGithub fetches the latest versions of nuclei repos from GitHub // -// This fetches latest nuclei/templates/ignore from https://version-check.nuclei.sh/versions +// This fetches the latest nuclei/templates/ignore from https://version-check.nuclei.sh/versions // If you want to disable this automatic update check, use -nut flag. func (r *Runner) fetchLatestVersionsFromGithub(configDir string) { versions, err := client.GetLatestNucleiTemplatesVersion() diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 2eb49569f..41ce53613 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -6,8 +6,9 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" "gopkg.in/yaml.v2" + + "github.com/projectdiscovery/gologger" ) // Config contains the internal nuclei engine configuration diff --git a/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go index 73e3af13c..7f59b0894 100644 --- a/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go +++ b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go @@ -5,10 +5,12 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" ) +// CreateEvent wraps the outputEvent with the result of the operators defined on the request func CreateEvent(request protocols.Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent { return CreateEventWithAdditionalOptions(request, outputEvent, func(internalWrappedEvent *output.InternalWrappedEvent) {}) } +// CreateEventWithAdditionalOptions wraps the outputEvent with the result of the operators defined on the request and enables extending the resulting event with additional attributes or values. func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent { event := &output.InternalWrappedEvent{InternalEvent: outputEvent} for _, compiledOperator := range request.GetCompiledOperators() { diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go index 02ca8afc7..6d185dbdb 100644 --- a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go @@ -12,9 +12,9 @@ import ( func Highlight(operatorResult *operators.Result, response string, noColor bool) string { result := response if operatorResult != nil && !noColor { + colorizer := aurora.NewAurora(true) for _, matches := range operatorResult.Matches { if len(matches) > 0 { - colorizer := aurora.NewAurora(true) for _, currentMatch := range matches { result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) } From 52b599ab5dd03a03c96a0bf6295ed1af925d44aa Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 7 Oct 2021 22:09:08 +0300 Subject: [PATCH 065/147] [feature] Add coloring to debug information #999 * fixed typo in the name of the file --- .../{response_highligther.go => response_highlighter.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename v2/pkg/protocols/common/helpers/responsehighlighter/{response_highligther.go => response_highlighter.go} (100%) diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go similarity index 100% rename from v2/pkg/protocols/common/helpers/responsehighlighter/response_highligther.go rename to v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go From f96168fc2c20c83e4b17e26a3ae0fe336a151c50 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 8 Oct 2021 20:17:37 +0300 Subject: [PATCH 066/147] [feature] Add coloring to debug information #999 * Reverted the bug introduced by mistake (checking file exists using os.IsExist vs !os.IsNotExist) --- v2/internal/runner/update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 567d2577f..217f34008 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -193,7 +193,7 @@ func getVersions(runner *Runner) (semver.Version, semver.Version, error) { // readInternalConfigurationFile reads the internal configuration file for nuclei func (r *Runner) readInternalConfigurationFile(home, configDir string) error { templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename) - if _, statErr := os.Stat(templatesConfigFile); os.IsExist(statErr) { + if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) { configuration, readErr := config.ReadConfiguration() if readErr != nil { return readErr From fc66195b28d09efe1eb5d28c94e2cbfd78e14c26 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 8 Oct 2021 20:18:00 +0300 Subject: [PATCH 067/147] [feature] Add coloring to debug information #999 * Code review changes --- .../response_highlighter.go | 18 ++++++++---------- v2/pkg/protocols/http/operators.go | 2 +- v2/pkg/protocols/offlinehttp/operators.go | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go index 6d185dbdb..1cb914e97 100644 --- a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go @@ -1,7 +1,7 @@ package responsehighlighter import ( - "fmt" + "strconv" "strings" "github.com/logrusorgru/aurora" @@ -9,10 +9,11 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" ) +var colorizer = aurora.NewAurora(true) + func Highlight(operatorResult *operators.Result, response string, noColor bool) string { result := response if operatorResult != nil && !noColor { - colorizer := aurora.NewAurora(true) for _, matches := range operatorResult.Matches { if len(matches) > 0 { for _, currentMatch := range matches { @@ -25,13 +26,10 @@ func Highlight(operatorResult *operators.Result, response string, noColor bool) return result } -func CreateHTTPStatusMatcherSnippets(statusCode int) []string { - httpVersions := []string{"0.9", "1.0", "1.1", "2", "2.0", "3", "3.0"} - var matcherValues = make([]string, 0, len(httpVersions)) - - for _, httpVersion := range httpVersions { - matcherValues = append(matcherValues, fmt.Sprintf("HTTP/%s %d", httpVersion, statusCode)) +func CreateStatusCodeSnippet(response string, statusCode int) string { + if strings.HasPrefix(response, "HTTP/") { + strStatusCode := strconv.Itoa(statusCode) + return response[:strings.Index(response, strStatusCode)+len(strStatusCode)] } - - return matcherValues + return "" } diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index a53081945..6a8bcfad1 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -28,7 +28,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat if !ok { return false, []string{} } - return matcher.Result(matcher.MatchStatusCode(statusCode)), responsehighlighter.CreateHTTPStatusMatcherSnippets(statusCode) + return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{responsehighlighter.CreateStatusCodeSnippet(data["response"].(string), statusCode)} case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index 7789b7e4b..730aa357d 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -28,7 +28,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat if !ok { return false, []string{} } - return matcher.Result(matcher.MatchStatusCode(statusCode)), responsehighlighter.CreateHTTPStatusMatcherSnippets(statusCode) + return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{responsehighlighter.CreateStatusCodeSnippet(data["response"].(string), statusCode)} case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: From 3e8a0af36fb249476ac43816fc8a929766b80e8a Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Fri, 8 Oct 2021 22:24:29 +0300 Subject: [PATCH 068/147] Add exclude severity filter --- v2/cmd/nuclei/main.go | 1 + v2/internal/runner/runner.go | 1 + v2/pkg/catalog/loader/filter/tag_filter.go | 54 ++++++++++++------- .../catalog/loader/filter/tag_filter_test.go | 10 ++++ v2/pkg/catalog/loader/loader.go | 22 ++++---- v2/pkg/types/types.go | 2 + 6 files changed, 60 insertions(+), 30 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index cc4a2c2b4..83b13fc6b 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -71,6 +71,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"), flagSet.VarP(&options.Severities, "impact", "severity", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), + flagSet.VarP(&options.ExcludeSeverities, "exclude-impact", "exclude-severity", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 4d8b7cf58..e87903f14 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -344,6 +344,7 @@ func (r *Runner) RunEnumeration() error { IncludeTemplates: r.options.IncludeTemplates, Authors: r.options.Author, Severities: r.options.Severities, + ExcludeSeverities: r.options.ExcludeSeverities, IncludeTags: r.options.IncludeTags, TemplatesDirectory: r.options.TemplatesDirectory, Catalog: r.catalog, diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index bfc75bd17..ed8722c90 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -9,11 +9,12 @@ import ( // TagFilter is used to filter nuclei templates for tag based execution type TagFilter struct { - allowedTags map[string]struct{} - severities map[severity.Severity]struct{} - authors map[string]struct{} - block map[string]struct{} - matchAllows map[string]struct{} + allowedTags map[string]struct{} + severities map[severity.Severity]struct{} + excludeSeverities map[severity.Severity]struct{} + authors map[string]struct{} + block map[string]struct{} + matchAllows map[string]struct{} } // ErrExcluded is returned for excluded templates @@ -54,15 +55,21 @@ func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templa } func isSeverityMatch(tagFilter *TagFilter, templateSeverity severity.Severity) bool { - if len(tagFilter.severities) == 0 || templateSeverity == severity.Undefined { + if (len(tagFilter.excludeSeverities) == 0 && len(tagFilter.severities) == 0) || templateSeverity == severity.Undefined { return true } - if _, ok := tagFilter.severities[templateSeverity]; ok { - return true + included := true + if len(tagFilter.severities) > 0 { + _, included = tagFilter.severities[templateSeverity] } - return false + excluded := false + if len(tagFilter.excludeSeverities) > 0 { + _, excluded = tagFilter.excludeSeverities[templateSeverity] + } + + return included && !excluded } func isAuthorMatch(tagFilter *TagFilter, templateAuthors []string) bool { @@ -110,23 +117,25 @@ func isTagMatch(tagFilter *TagFilter, templateTags []string) bool { } type Config struct { - Tags []string - ExcludeTags []string - Authors []string - Severities severity.Severities - IncludeTags []string + Tags []string + ExcludeTags []string + Authors []string + Severities severity.Severities + ExcludeSeverities severity.Severities + IncludeTags []string } // New returns a tag filter for nuclei tag based execution // -// It takes into account Tags, Severities, Authors, IncludeTags, ExcludeTags. +// It takes into account Tags, Severities, ExcludeSeverities, Authors, IncludeTags, ExcludeTags. func New(config *Config) *TagFilter { filter := &TagFilter{ - allowedTags: make(map[string]struct{}), - authors: make(map[string]struct{}), - severities: make(map[severity.Severity]struct{}), - block: make(map[string]struct{}), - matchAllows: make(map[string]struct{}), + allowedTags: make(map[string]struct{}), + authors: make(map[string]struct{}), + severities: make(map[severity.Severity]struct{}), + excludeSeverities: make(map[severity.Severity]struct{}), + block: make(map[string]struct{}), + matchAllows: make(map[string]struct{}), } for _, tag := range config.ExcludeTags { for _, val := range splitCommaTrim(tag) { @@ -140,6 +149,11 @@ func New(config *Config) *TagFilter { filter.severities[tag] = struct{}{} } } + for _, tag := range config.ExcludeSeverities { + if _, ok := filter.excludeSeverities[tag]; !ok { + filter.excludeSeverities[tag] = struct{}{} + } + } for _, tag := range config.Authors { for _, val := range splitCommaTrim(tag) { if _, ok := filter.authors[val]; !ok { diff --git a/v2/pkg/catalog/loader/filter/tag_filter_test.go b/v2/pkg/catalog/loader/filter/tag_filter_test.go index 8acc9a356..22d18b189 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter_test.go +++ b/v2/pkg/catalog/loader/filter/tag_filter_test.go @@ -73,6 +73,16 @@ func TestTagBasedFilter(t *testing.T) { matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil) require.True(t, matched, "could not get correct match") }) + t.Run("match-exclude-severity", func(t *testing.T) { + filter := New(&Config{ + ExcludeSeverities: severity.Severities{severity.Low}, + }) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil) + require.True(t, matched, "could not get correct match") + + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + require.False(t, matched, "could not get correct match") + }) t.Run("match-exclude-with-tags", func(t *testing.T) { filter := New(&Config{ Tags: []string{"tag"}, diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index fee382447..eac22b4ae 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -19,11 +19,12 @@ type Config struct { ExcludeTemplates []string IncludeTemplates []string - Tags []string - ExcludeTags []string - Authors []string - Severities severity.Severities - IncludeTags []string + Tags []string + ExcludeTags []string + Authors []string + Severities severity.Severities + ExcludeSeverities severity.Severities + IncludeTags []string Catalog *catalog.Catalog ExecutorOptions protocols.ExecuterOptions @@ -49,11 +50,12 @@ func New(config *Config) (*Store, error) { store := &Store{ config: config, tagFilter: filter.New(&filter.Config{ - Tags: config.Tags, - ExcludeTags: config.ExcludeTags, - Authors: config.Authors, - Severities: config.Severities, - IncludeTags: config.IncludeTags, + Tags: config.Tags, + ExcludeTags: config.ExcludeTags, + Authors: config.Authors, + Severities: config.Severities, + ExcludeSeverities: config.ExcludeSeverities, + IncludeTags: config.IncludeTags, }), pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{ IncludedTemplates: config.IncludeTemplates, diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 91db2b91b..1b80c4ed6 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -25,6 +25,8 @@ type Options struct { Vars goflags.RuntimeMap // Severities filters templates based on their severity and only run the matching ones. Severities severity.Severities + // ExcludeSeverities specifies severities to exclude + ExcludeSeverities severity.Severities // Author filters templates based on their author and only run the matching ones. Author goflags.NormalizedStringSlice // IncludeTags includes specified tags to be run even while being in denylist From 61a8ba80b4cf2fd65bdaa466c91e4ef8977b6c61 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 8 Oct 2021 22:44:40 +0300 Subject: [PATCH 069/147] If debug is enabled, show matchers that are not named also #1084 * start the indexing of the matcher names from 1 instead of 0 --- v2/pkg/operators/operators.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 5643152b1..3444341bb 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -178,7 +178,7 @@ func getMatcherName(matcher *matchers.Matcher, matcherIndex int) string { if matcher.Name != "" { return matcher.Name } else { - return matcher.Type + "-" + strconv.Itoa(matcherIndex) + return matcher.Type + "-" + strconv.Itoa(matcherIndex+1) // making the index start from 1 to be more readable } } From a86c843be5e26c8af635051f502c3127ca935407 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 8 Oct 2021 22:46:53 +0300 Subject: [PATCH 070/147] [issue] Do not show all matcher names for OR conditions #1082 * removing the short circuit in case of matchers with OR operator, because it is intended in cases like technology detection templates etc. --- v2/pkg/operators/operators.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 3444341bb..32f265b8f 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -144,10 +144,6 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc result.Matches[matcherName] = matched matches = true - - if matcherCondition == matchers.ORCondition { - break - } } else if matcherCondition == matchers.ANDCondition { if len(result.DynamicValues) > 0 { return result, true From 4e54a61b653e9a4b3bacfaface449d64bceb55e9 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sat, 9 Oct 2021 08:38:42 +0200 Subject: [PATCH 071/147] feat: expose WaitVisible method from the rod library --- v2/pkg/protocols/headless/engine/action.go | 4 ++++ v2/pkg/protocols/headless/engine/page_actions.go | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/v2/pkg/protocols/headless/engine/action.go b/v2/pkg/protocols/headless/engine/action.go index 0be71fff8..5f97217b2 100644 --- a/v2/pkg/protocols/headless/engine/action.go +++ b/v2/pkg/protocols/headless/engine/action.go @@ -50,6 +50,8 @@ const ( ActionDebug // ActionSleep executes a sleep for a specified duration ActionSleep + // ActionWaitVisible waits until an element appears. + ActionWaitVisible ) // ActionStringToAction converts an action from string to internal representation @@ -75,6 +77,7 @@ var ActionStringToAction = map[string]ActionType{ "keyboard": ActionKeyboard, "debug": ActionDebug, "sleep": ActionSleep, + "waitvisible": ActionWaitVisible, } // ActionToActionString converts an action from internal representation to string @@ -100,6 +103,7 @@ var ActionToActionString = map[ActionType]string{ ActionKeyboard: "keyboard", ActionDebug: "debug", ActionSleep: "sleep", + ActionWaitVisible: "waitvisible", } // Action is an action taken by the browser to reach a navigation diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index 2ec2365b9..bc4239b76 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -67,6 +67,8 @@ func (p *Page) ExecuteActions(baseURL *url.URL, actions []*Action) (map[string]s err = p.DebugAction(act, outData) case ActionSleep: err = p.SleepAction(act, outData) + case ActionWaitVisible: + err = p.WaitVisible(act, outData) default: continue } @@ -83,6 +85,15 @@ type requestRule struct { Args map[string]string } +// WaitVisible waits until an element appears. +func (p *Page) WaitVisible(act *Action, out map[string]string) error { + element, err := p.pageElementBy(act.Data) + if err != nil { + return err + } + return element.WaitVisible() +} + // ActionAddHeader executes a AddHeader action. func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error { in := act.GetArg("part") From 3386c1c5bd0417607d1f31a2ca13f12305474751 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sat, 9 Oct 2021 12:26:18 +0200 Subject: [PATCH 072/147] chore: add new test --- .../protocols/headless/engine/page_actions.go | 5 ++- .../headless/engine/page_actions_test.go | 38 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index bc4239b76..0f69c8d23 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -91,7 +91,10 @@ func (p *Page) WaitVisible(act *Action, out map[string]string) error { if err != nil { return err } - return element.WaitVisible() + if err = element.WaitVisible(); err != nil { + return errors.Wrap(err, "could not wait element") + } + return nil } // ActionAddHeader executes a AddHeader action. diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 658c32908..3f803309c 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -278,3 +278,41 @@ func TestActionHeadersChange(t *testing.T) { require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") } + +func TestActionWaitVisible(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + page.Page().MustElement("button").MustWaitVisible() +} From d06cd671e22e826dd34a9caf65ff8a0a04593d52 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sat, 9 Oct 2021 12:29:08 +0200 Subject: [PATCH 073/147] chore: remove .MustWaitVisible to .MustVisible --- v2/pkg/protocols/headless/engine/page_actions_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 3f803309c..ca17f98e5 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -314,5 +314,5 @@ func TestActionWaitVisible(t *testing.T) { require.Nil(t, err, "could not run page actions") defer page.Close() - page.Page().MustElement("button").MustWaitVisible() + page.Page().MustElement("button").MustVisible() } From bd14b0f2ac5c5061c665d3ec2cbf50b3532beb42 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sat, 9 Oct 2021 13:00:22 +0200 Subject: [PATCH 074/147] chore: remove block --- v2/pkg/protocols/headless/engine/page_actions_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index ca17f98e5..3a333d1ad 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -297,7 +297,7 @@ func TestActionWaitVisible(t *testing.T) { `) })) From 6ebeaf84a38f1676990f19669a06d0052842bc0b Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sat, 9 Oct 2021 13:18:43 +0200 Subject: [PATCH 075/147] feat: add missin actions tests --- .../headless/engine/page_actions_test.go | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 658c32908..5971ab1d0 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -278,3 +278,67 @@ func TestActionHeadersChange(t *testing.T) { require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") } + +func TestActionScreenshot(t *testing.T) { + +} + +func TestActionTimeInput(t *testing.T) { + +} + +func TestActionSelectInput(t *testing.T) { + +} + +func TestActionFilesInput(t *testing.T) { + +} + +func TestActionWaitLoad(t *testing.T) { + +} + +func TestActionGetResource(t *testing.T) { + +} + +func TestActionExtract(t *testing.T) { + +} + +func TestActionSetMethod(t *testing.T) { + +} + +func TestActionAddHeader(t *testing.T) { + +} + +func TestActionSetHeader(t *testing.T) { + +} + +func TestActionDeleteHeader(t *testing.T) { + +} + +func TestActionSetBody(t *testing.T) { + +} + +func TestActionWaitEvent(t *testing.T) { + +} + +func TestActionKeyboard(t *testing.T) { + +} + +func TestActionDebug(t *testing.T) { + +} + +func TestActionSleep(t *testing.T) { + +} From 2c88370b35d8cce6f9a25170f7feda2fff364048 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 9 Oct 2021 18:53:17 +0530 Subject: [PATCH 076/147] Made the version validation a bit better --- v2/pkg/protocols/offlinehttp/read_response.go | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/v2/pkg/protocols/offlinehttp/read_response.go b/v2/pkg/protocols/offlinehttp/read_response.go index 2c8528733..f864b5ab8 100644 --- a/v2/pkg/protocols/offlinehttp/read_response.go +++ b/v2/pkg/protocols/offlinehttp/read_response.go @@ -8,19 +8,14 @@ import ( "strings" ) +var noMinor = regexp.MustCompile(`HTTP\/([0-9]) `) + // readResponseFromString reads a raw http response from a string. func readResponseFromString(data string) (*http.Response, error) { var final string - noMinor := regexp.MustCompile(`HTTP\/[0-9] `) if strings.HasPrefix(data, "HTTP/") { - // Go could not parse http version with no minor version - if noMinor.MatchString(data) { - // add minor version - data = strings.Replace(data, "HTTP/2", "HTTP/2.0", 1) - data = strings.Replace(data, "HTTP/3", "HTTP/3.0", 1) - } - final = data + final = addMinorVersionToHTTP(data) } else { lastIndex := strings.LastIndex(data, "HTTP/") if lastIndex == -1 { @@ -28,10 +23,22 @@ func readResponseFromString(data string) (*http.Response, error) { } final = data[lastIndex:] // choose last http/ in case of it being later. - if noMinor.MatchString(final) { - final = strings.ReplaceAll(final, "HTTP/2", "HTTP/2.0") - final = strings.ReplaceAll(final, "HTTP/3", "HTTP/3.0") - } + final = addMinorVersionToHTTP(final) } return http.ReadResponse(bufio.NewReader(strings.NewReader(final)), nil) } + +// addMinorVersionToHTTP tries to add a minor version to http status header +// fixing the compatibility issue with go standard library. +func addMinorVersionToHTTP(data string) string { + matches := noMinor.FindAllStringSubmatch(data, 1) + if len(matches) == 0 { + return data + } + if len(matches[0]) < 2 { + return data + } + replacedVersion := strings.Replace(matches[0][0], matches[0][1], matches[0][1]+".0", 1) + data = strings.Replace(data, matches[0][0], replacedVersion, 1) + return data +} From aacbb28052856a23ca4cf086ba066ab88397c3de Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 9 Oct 2021 19:46:23 +0530 Subject: [PATCH 077/147] Removed sniper + made batteringram default + misc --- .../common/expressions/expressions.go | 42 +++++++------------ .../protocols/common/generators/generators.go | 32 ++++++++------ .../common/generators/generators_test.go | 7 ++-- 3 files changed, 38 insertions(+), 43 deletions(-) diff --git a/v2/pkg/protocols/common/expressions/expressions.go b/v2/pkg/protocols/common/expressions/expressions.go index 7f27b88a3..5a4ba2f3a 100644 --- a/v2/pkg/protocols/common/expressions/expressions.go +++ b/v2/pkg/protocols/common/expressions/expressions.go @@ -18,6 +18,21 @@ var templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}["'\)\}]*`) // The provided keys from finalValues will be used as variable names // for substitution inside the expression. func Evaluate(data string, base map[string]interface{}) (string, error) { + return evaluate(data, base) +} + +// EvaluateByte checks if the match contains a dynamic variable, for each +// found one we will check if it's an expression and can +// be compiled, it will be evaluated and the results will be returned. +// +// The provided keys from finalValues will be used as variable names +// for substitution inside the expression. +func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) { + finalData, err := evaluate(string(data), base) + return []byte(finalData), err +} + +func evaluate(data string, base map[string]interface{}) (string, error) { data = replacer.Replace(data, base) dynamicValues := make(map[string]interface{}) @@ -37,30 +52,3 @@ func Evaluate(data string, base map[string]interface{}) (string, error) { // Replacer dynamic values if any in raw request and parse it return replacer.Replace(data, dynamicValues), nil } - -// EvaluateByte checks if the match contains a dynamic variable, for each -// found one we will check if it's an expression and can -// be compiled, it will be evaluated and the results will be returned. -// -// The provided keys from finalValues will be used as variable names -// for substitution inside the expression. -func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) { - final := replacer.Replace(string(data), base) - - dynamicValues := make(map[string]interface{}) - for _, match := range templateExpressionRegex.FindAllString(final, -1) { - expr := generators.TrimDelimiters(match) - - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) - if err != nil { - continue - } - result, err := compiled.Evaluate(base) - if err != nil { - continue - } - dynamicValues[expr] = result - } - // Replacer dynamic values if any in raw request and parse it - return []byte(replacer.Replace(final, dynamicValues)), nil -} diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index dcca16f55..07c72d200 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -2,6 +2,8 @@ package generators +import "github.com/pkg/errors" + // Generator is the generator struct for generating payloads type Generator struct { Type Type @@ -12,8 +14,8 @@ type Generator struct { type Type int const ( - // Sniper replaces each variable with values at a time. - Sniper Type = iota + 1 + // Sniper replaces one iteration of the payload with a value. + BatteringRam Type = iota + 1 // PitchFork replaces variables with positional value from multiple wordlists PitchFork // ClusterBomb replaces variables with all possible combinations of values @@ -22,9 +24,9 @@ const ( // StringToType is a table for conversion of attack type from string. var StringToType = map[string]Type{ - "sniper": Sniper, - "pitchfork": PitchFork, - "clusterbomb": ClusterBomb, + "batteringram": BatteringRam, + "pitchfork": PitchFork, + "clusterbomb": ClusterBomb, } // New creates a new generator structure for payload generation @@ -41,6 +43,12 @@ func New(payloads map[string]interface{}, payloadType Type, templatePath string) generator.Type = payloadType generator.payloads = compiled + // Validate the sniper/batteringram payload set + if payloadType == BatteringRam { + if len(payloads) != 1 { + return nil, errors.New("sniper/batteringram must have single payload set") + } + } return generator, nil } @@ -87,7 +95,7 @@ func (i *Iterator) Remaining() int { func (i *Iterator) Total() int { count := 0 switch i.Type { - case Sniper: + case BatteringRam: for _, p := range i.payloads { count += len(p.values) } @@ -110,19 +118,19 @@ func (i *Iterator) Total() int { // Value returns the next value for an iterator func (i *Iterator) Value() (map[string]interface{}, bool) { switch i.Type { - case Sniper: - return i.sniperValue() + case BatteringRam: + return i.batteringRamValue() case PitchFork: return i.pitchforkValue() case ClusterBomb: return i.clusterbombValue() default: - return i.sniperValue() + return i.batteringRamValue() } } -// sniperValue returns a list of all payloads for the iterator -func (i *Iterator) sniperValue() (map[string]interface{}, bool) { +// batteringRamValue returns a list of all payloads for the iterator +func (i *Iterator) batteringRamValue() (map[string]interface{}, bool) { values := make(map[string]interface{}, 1) currentIndex := i.msbIterator @@ -132,7 +140,7 @@ func (i *Iterator) sniperValue() (map[string]interface{}, bool) { if i.msbIterator == len(i.payloads) { return nil, false } - return i.sniperValue() + return i.batteringRamValue() } values[payload.name] = payload.value() payload.incrementPosition() diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index a19f53114..de37ca62d 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -6,11 +6,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestSniperGenerator(t *testing.T) { +func TestBatteringRamGenerator(t *testing.T) { usernames := []string{"admin", "password"} - moreUsernames := []string{"login", "test"} - generator, err := New(map[string]interface{}{"username": usernames, "aliases": moreUsernames}, Sniper, "") + generator, err := New(map[string]interface{}{"username": usernames}, BatteringRam, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -22,7 +21,7 @@ func TestSniperGenerator(t *testing.T) { } count++ } - require.Equal(t, len(usernames)+len(moreUsernames), count, "could not get correct sniper counts") + require.Equal(t, len(usernames), count, "could not get correct batteringram counts") } func TestPitchforkGenerator(t *testing.T) { From 3584a4b12a8cab8595b60d3370ff87dfc00024d8 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 9 Oct 2021 19:53:12 +0530 Subject: [PATCH 078/147] Added validation for invalid payload attack type --- v2/pkg/protocols/http/http.go | 9 +++++++-- v2/pkg/protocols/network/network.go | 7 ++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 5558e7035..201804c96 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -1,6 +1,7 @@ package http import ( + "fmt" "strings" "github.com/pkg/errors" @@ -225,9 +226,13 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if len(r.Payloads) > 0 { attackType := r.AttackType if attackType == "" { - attackType = "sniper" + attackType = "batteringram" + } + var ok bool + r.attackType, ok = generators.StringToType[attackType] + if !ok { + return fmt.Errorf("invalid attack type provided: %s", attackType) } - r.attackType = generators.StringToType[attackType] // Resolve payload paths if they are files. for name, payload := range r.Payloads { diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 3ad431ed0..70b818876 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -1,6 +1,7 @@ package network import ( + "fmt" "net" "strings" @@ -155,7 +156,11 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if attackType == "" { attackType = "sniper" } - r.attackType = generators.StringToType[attackType] + var ok bool + r.attackType, ok = generators.StringToType[attackType] + if !ok { + return fmt.Errorf("invalid attack type provided: %s", attackType) + } // Resolve payload paths if they are files. for name, payload := range r.Payloads { From df1c133f68f43421473443c3403373b2db0cc314 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 9 Oct 2021 19:53:37 +0530 Subject: [PATCH 079/147] Misc --- v2/pkg/protocols/network/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 70b818876..244041c5d 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -154,7 +154,7 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if len(r.Payloads) > 0 { attackType := r.AttackType if attackType == "" { - attackType = "sniper" + attackType = "batteringram" } var ok bool r.attackType, ok = generators.StringToType[attackType] From 1b8ad856d65a5a9204e39eb53cf1ea13e58b1945 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sat, 9 Oct 2021 14:28:15 +0000 Subject: [PATCH 080/147] Auto Generate Syntax Docs + JSONSchema [Sat Oct 9 14:28:15 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 13 +++++++++++++ nuclei-jsonschema.json | 5 +++++ v2/pkg/templates/templates_doc.go | 7 ++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index bb1ad86bd..691c25af7 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -1192,6 +1192,19 @@ StopAtFirstMatch stops the execution of the requests and template as soon as a m
+
+ +skip-variables-check bool + +
+
+ +SkipVariablesCheck skips the check for unresolved variables in request + +
+ +
+ diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 20da43727..1578630eb 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -720,6 +720,11 @@ "type": "boolean", "title": "stop at first match", "description": "Stop the execution after a match is found" + }, + "skip-variables-check": { + "type": "boolean", + "title": "skip variable checks", + "description": "Skips the check for unresolved variables in request" } }, "additionalProperties": false, diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index e7758c96f..30ac25af7 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -270,7 +270,7 @@ func init() { FieldName: "requests", }, } - HTTPRequestDoc.Fields = make([]encoder.Doc, 25) + HTTPRequestDoc.Fields = make([]encoder.Doc, 26) HTTPRequestDoc.Fields[0].Name = "matchers" HTTPRequestDoc.Fields[0].Type = "[]matchers.Matcher" HTTPRequestDoc.Fields[0].Note = "" @@ -437,6 +437,11 @@ func init() { HTTPRequestDoc.Fields[24].Note = "" HTTPRequestDoc.Fields[24].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found." HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found." + HTTPRequestDoc.Fields[25].Name = "skip-variables-check" + HTTPRequestDoc.Fields[25].Type = "bool" + HTTPRequestDoc.Fields[25].Note = "" + HTTPRequestDoc.Fields[25].Description = "SkipVariablesCheck skips the check for unresolved variables in request" + HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request" MATCHERSMatcherDoc.Type = "matchers.Matcher" MATCHERSMatcherDoc.Comments[encoder.LineComment] = " Matcher is used to match a part in the output from a protocol." From cb32a7b5e23f0ed2868e4660100f4d798770e3f3 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sat, 9 Oct 2021 20:00:33 +0530 Subject: [PATCH 081/147] misc update --- v2/go.mod | 1 - v2/go.sum | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index de33b8d0b..471e12c5c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -100,7 +100,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect - github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca // indirect github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect diff --git a/v2/go.sum b/v2/go.sum index 707b0fc69..e85a7fce4 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -562,9 +562,9 @@ github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/i github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= -github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca h1:xT//ApxoeQRbt9GgL/122688bhGy9hur8YG0Qh69k5I= github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= -github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= +github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= +github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 h1:b7zDUSsgN5f4/IlhKF6RVGsp/NkHIuty0o1YjzAMKUs= github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= From 7e12774543f7198355f12b752809e637639b1f0a Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sat, 9 Oct 2021 17:08:00 +0200 Subject: [PATCH 082/147] chore: add helper method --- .../headless/engine/page_actions_test.go | 71 +++++-------------- 1 file changed, 19 insertions(+), 52 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 5971ab1d0..a6be422ca 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -14,16 +14,22 @@ import ( "github.com/stretchr/testify/require" ) -func TestActionNavigate(t *testing.T) { +func setUp(t *testing.T) (*Browser, *Instance, error) { + t.Helper() _ = protocolstate.Init(&types.Options{}) browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") - defer browser.Close() instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") + return browser, instance, err +} + +func TestActionNavigate(t *testing.T) { + browser, instance, err := setUp(t) + defer browser.Close() defer instance.Close() + require.Nil(t, err, "could not create browser instance") ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -50,13 +56,9 @@ func TestActionNavigate(t *testing.T) { } func TestActionScript(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") + browser, instance, err := setUp(t) defer browser.Close() - - instance, err := browser.NewInstance() + defer instance.Close() require.Nil(t, err, "could not create browser instance") t.Run("run-and-results", func(t *testing.T) { @@ -119,13 +121,9 @@ func TestActionScript(t *testing.T) { } func TestActionClick(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") + browser, instance, err := setUp(t) defer browser.Close() - - instance, err := browser.NewInstance() + defer instance.Close() require.Nil(t, err, "could not create browser instance") ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -159,13 +157,9 @@ func TestActionClick(t *testing.T) { } func TestActionRightClick(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") + browser, instance, err := setUp(t) defer browser.Close() - - instance, err := browser.NewInstance() + defer instance.Close() require.Nil(t, err, "could not create browser instance") ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -207,13 +201,9 @@ func TestActionRightClick(t *testing.T) { } func TestActionTextInput(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") + browser, instance, err := setUp(t) defer browser.Close() - - instance, err := browser.NewInstance() + defer instance.Close() require.Nil(t, err, "could not create browser instance") ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -248,13 +238,9 @@ func TestActionTextInput(t *testing.T) { } func TestActionHeadersChange(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") + browser, instance, err := setUp(t) defer browser.Close() - - instance, err := browser.NewInstance() + defer instance.Close() require.Nil(t, err, "could not create browser instance") ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -280,65 +266,46 @@ func TestActionHeadersChange(t *testing.T) { } func TestActionScreenshot(t *testing.T) { - } func TestActionTimeInput(t *testing.T) { - } func TestActionSelectInput(t *testing.T) { - } func TestActionFilesInput(t *testing.T) { - } func TestActionWaitLoad(t *testing.T) { - } func TestActionGetResource(t *testing.T) { - } func TestActionExtract(t *testing.T) { - } func TestActionSetMethod(t *testing.T) { - } func TestActionAddHeader(t *testing.T) { - -} - -func TestActionSetHeader(t *testing.T) { - } func TestActionDeleteHeader(t *testing.T) { - } func TestActionSetBody(t *testing.T) { - } func TestActionWaitEvent(t *testing.T) { - } func TestActionKeyboard(t *testing.T) { - } func TestActionDebug(t *testing.T) { - } func TestActionSleep(t *testing.T) { - } From 4383d8557fb7ca0477bb6ec5892fa3426cae453f Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sat, 9 Oct 2021 17:30:45 +0200 Subject: [PATCH 083/147] chore: add timeout test --- .../protocols/headless/engine/page_actions.go | 2 +- .../headless/engine/page_actions_test.go | 91 +++++++++++++------ 2 files changed, 64 insertions(+), 29 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index 0f69c8d23..ad4eec626 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -89,7 +89,7 @@ type requestRule struct { func (p *Page) WaitVisible(act *Action, out map[string]string) error { element, err := p.pageElementBy(act.Data) if err != nil { - return err + return errors.Wrap(err, "could not find element") } if err = element.WaitVisible(); err != nil { return errors.Wrap(err, "could not wait element") diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 3a333d1ad..5c23fdee4 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -280,39 +280,74 @@ func TestActionHeadersChange(t *testing.T) { } func TestActionWaitVisible(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Run("wait for an element being visible", func(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` - - Nuclei Test Page - - - - `) - })) - defer ts.Close() + + Nuclei Test Page + + + + `) + })) + defer ts.Close() - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, - } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() - page.Page().MustElement("button").MustVisible() + page.Page().MustElement("button").MustVisible() + }) + + t.Run("timeout because of element not visible", func(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + } + _, _, err = instance.Run(parsed, actions, 2*time.Second) + require.Error(t, err) + require.Contains(t, err.Error(), "could not wait element") + }) } From 6d88f26b1fcc5389a0d00e85bcd7118d27054515 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sat, 9 Oct 2021 23:19:07 +0200 Subject: [PATCH 084/147] chore: add tests --- .../headless/engine/page_actions_test.go | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index fa3148aa1..016e735b7 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -273,6 +273,41 @@ func TestActionTimeInput(t *testing.T) { } func TestActionSelectInput(t *testing.T) { + browser, instance, err := setUp(t) + defer browser.Close() + defer instance.Close() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "select", Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + el := page.Page().MustElement("select") + require.Equal(t, "Test2", el.MustText(), "could not get input change value") } func TestActionFilesInput(t *testing.T) { @@ -380,6 +415,43 @@ func TestActionWaitEvent(t *testing.T) { } func TestActionKeyboard(t *testing.T) { + browser, instance, err := setUp(t) + defer browser.Close() + defer instance.Close() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "click", Data: map[string]string{"selector": "select"}}, + {ActionType: "keyboard", Data: map[string]string{"keys": "Test 2"}}, + {ActionType: "keyboard", Data: map[string]string{"keys": "enter"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + el := page.Page().MustElement("select") + require.Equal(t, "Test2", el.MustText(), "could not get input change value") } func TestActionDebug(t *testing.T) { From 2b1ab54a6d01237d6f688e8e86426090fa038622 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sun, 10 Oct 2021 08:29:58 +0200 Subject: [PATCH 085/147] chore: add tests --- .../engine/C:\\fakepath\\test1\\test.png" | Bin 0 -> 145393 bytes .../headless/engine/page_actions_test.go | 97 ++++++++++++++++++ v2/pkg/protocols/headless/engine/test.png | Bin 0 -> 145393 bytes 3 files changed, 97 insertions(+) create mode 100755 "v2/pkg/protocols/headless/engine/C:\\fakepath\\test1\\test.png" create mode 100755 v2/pkg/protocols/headless/engine/test.png diff --git "a/v2/pkg/protocols/headless/engine/C:\\fakepath\\test1\\test.png" "b/v2/pkg/protocols/headless/engine/C:\\fakepath\\test1\\test.png" new file mode 100755 index 0000000000000000000000000000000000000000..9dc043d6cb4d556e75cb3ec13013834eac1fe8da GIT binary patch literal 145393 zcmeI5cU)8F*TCaOaZsz`La?H^04oAT1k|b+6_7d*kYPno5wHx|LoM36(4Zi@PF#S3 zfNUx*wu(%FIM`eE3VF{%3j{Ly$JrBi{O<;5WR`a^!&FV+0k^>Y z;qr}DLM=b{^Kd4S{>`vNp0C(?jH1u;1q;_umhpVSMG?w$o-f!WKpD^T1@rkR#O7Qd zE|W)6MsPg~CsvOg?lO$m3w|4kW&rzqCT55o79Jk%waV5!_|b<`X})q6r|TH`MggV8 z_VwwK&K;@SH8jj##qo4#-l}~*VeOxu3uj(jv@K^Q-6+6uxlO~n3BkgO4mOJI?~c zsqT$$={-lT<-{>s-_B65Yc>p^2dV`$SG?LA)|OwO*jaIH{4ANJaQD^s4&>e6<1J(2 zEo5EztxmKh^=*X7wh-Movjz`dASN=+WR_4KX@(i*o5s{O-qmB=5un9rUE6Thyy|vf zur&j0;Ah)ZR@&25uco4+Lc4BNog-Ht+Z39sYj2W~Eu;PXoY1x%JHmHl?y_LY(j6?W z<%;jRFQUZk$dv8szNzBVL%qdg|1v9iGk|p1zK)`Nw$1 zic83iy0WE;6~7%f&)%W2FT+ivmXr^-f8oPD`6D=@^`F~Nzk+^)G|i;qmuG8x;( zlHE=nrE*{z+46%vwP$6EU)1uNFI~FiGo>PC_opK-Rd+O>No3qr+8ly{7J;kUCRDCZO z$Jq;}XW!lycq^f+F2=}Bp>-pz=E3thawX@#H*bG`GSDc~zoW!mExs&0A-Pc);pY@Cf=C0*KO)q9joje zw{PFhzh}YQ%8n8HDTfzrySXKKy`av+aJT$}DJd!OwM*{ZSbut2)VpK+GRAI-j-A&- zL(Tb%_J3T}_+nlZ#GtIPn_Wwl$^Dx;F-LCSy0vhoJ)Vq2pPXJ>{>pX)481bZ*li+4C~WmWYiza-96T-(D2F=&TH!6_*zI?U7#0t zw+*7xIA|2F&k+#}A0xKh-@fcuG5>`#W57s_;5F0cIl>MNYgCmO&Esc(cfX=OD6)9* zn^moOdyUiGrhogRpM6iInsW-|Q$HD#g6pxdtKOQ%m}ouRe_=`Rqi;5QEX{8oqm%U5 zmER&muKc=+j|@pfpy1zY@A$sHd(VUzm;m7E#Rx>USYS{)oc ze){=o;#vmJg{-$g#(Ev^*hSLe5{=-cQHI^^pAOLD9cM}(yjUAaR}|NJxRKezuxNgz zTwZ9AsnWP(b)8kNj;C|`N#})H_BonHQ{(e*c1y_Qlm|HLzA*zksWXpHQhcH2)aYIl z@z(S<hVB9H+OyuMC7r_|G)<}GcU8q3S}6#X!hOO$t_ zm9MTT*F{$cUwr^+NRfZq(!5(OV6yZBfqowF)Eg&Yfj+CWCGP>D>?tiCo#`y+7#a_{}nl{nk(3niQUOv>QJ`Pr5)*N|j$D(CDRg_Vr1 zHZ@I6%?)mI{C@pf%&_6z?=zH~9QvREs@=F^9zyB{s% z7nDc~-Ip3sAr-Xi!8e%-ZM$FXg1h0bg(4(XTM(Wao~!G#K*ihO%`Nq))&k?KTk6+V zeR}eB6r@aZxTH{Dh8N}2^XfvOtq`W3&zA{S;YRu*xJF>N;gR~=6t8;>y zpv5SE%rCjSTzBuYGs&7pewReO6zq=>hlt8m-wO+c=(-Udy>iEn9Z}7%H#hmvyA9p_ zKY=keLo{T8XJQ*3^B;g}Mk@(Tl}Lp`=Y1gSCiy4F$4ts-%2Wyn4Gn#BS3?pK$sVVU zR6WDG!YCihVvi%aI@bFtUS6rp4bgLg>l*l&L{MqI&OIIZ&P>L(ZQI=8I;F%qrtuwR zqRA(Poi5iqGrOp?pFTd@2M*doYpyGjhu$PP@#qNtE?MK8r4Y@VXb<7vHN++`8TDDQ zwoO5eTV#tYx81pWm!CoYIq7SzFpmp&|F%rYFzvD>xOQtK)NoPihI6yM>K*It599{x z>?!ipbMmTpYKtM(NJ&j?dUt%%O}H?XHfD~ENJwnsruQ9-7cVw1xuASg{n~~Dq+l9! zG?vOiW{3XeQnI@kDS%X(Frz{_3M#lobx`C2YJ8JlJ=DjE)Vn)_B2v0|6%~q%YEh75 zBIE^&ygetUc1jthy&(RPOz1Yh*0!G>&#)_xv8s;9h?>9IW6I-$%#HwNlt7P<=!BUo zUf7oVX;dh6)r8G*49u2@Dxrxc`&k!j@-bCt&(6*&hwLog3`qFX8FkiFn!B2!oKpMm zlX7Nhb*r>DBu%mapv9;?Eqcswaz(g~8uT^l6_bP&S3zyuM2qdLy6gH$I#)FLcA&qv zZk(M-y0PHx;9!Yvk7DrN?j!m8qTnU`P0$(5kU6yLmA9C`&DEFSdW$PX_cX(rt8+tM zGMfUJyC0fXy}9kRY|mQSb!TReYXe>Mwnm8ViHSm56_KVzD@i9*7u%HaSUwtBpdY}p zkmj!n^u!c^VYaEOt5?u1vs7kv@LpgA-8f_{ycHU|?$*eJ^kHLWpZ}xBaA^3Q^cYHv zN;TPoZlR$qdSOnC&T8M7jxrxYpr~BAuSJeGGs8OGo$9`z6v8b1si0o0dYh(6%Ed*H zP>g7z={~ZVXJ$y{Xfq3pLrR}~t+uq;y5K;d%1X1%F;I8Nb#j8Vjb2?{@9IL&uF4V5 zIOz{rOe^A$*VH9Dr#y3Nz9w7>otbLVi3$9BjeLLF;P2E~8D!tVG`7+=gJ$u)e25I> zxbZ?OSCCrw%EoPu!t#A77cHxEOoEIGq3rP2?o0KUny%uvGW=!nK2M#T=u~L1^|q5L z{n*SJxO8=SS9IiUP>I=%9d~zvqf}|J5YE*$Kk`a0?|eeDN4U6?f|%C+>0{P|z0V)# zWozj9srs$F0p8MXse$UIn5;4E0o21?#!dl_t;x=eT<3jI?!b-33u+aNbMCF}av3o( zPoDu9PVX{iTvPBn%jTn9X z-o2$zrBkS4eh;lA7#elZL#{9@?qv{G52Sv^wH3>o6g;G^U%x(C+D)m)26FU8e+Bz# z&+gp0W6or>l|xEed?rXM{LcO&?_o*x3Ctd?t{$s}4bO$0eT9zX%}r+7TT{({A0>FG zA*!c2P|4&zPnhze<1diX`SQy5&)kFqfS0MyRKI)wbK2!^uPvNzv)M<+M2lt+=yycR ztmx(m=7MBMLgQ*7Mi1S0f)ux?%JY)r-jIarP}0Sy=b&B|Lcq+rM~60L9COTsnHQm( zmQ2=oA4TdUyop7+bgsd>#G?mt?&asiw9i~=RdsR1q*aIAb5@uXtZ?jVRiTmcGEezj zTGAvX-^RwqmSAnEcANBu@@x-DJ>C5p+qX}7?Cc)0)oqM{+&s?!k!M?TdAl-7a zmy)y7Iaq!+*W^pNi>YNmhvg5Ab5P3`<0@EZTv332mhig1XTOqT3SimxavC-#wwyfy(^k3e7ZO*`-Hdhj%}0IYwi(=4tf z$F5pdTEyn9Nu>D0@-(k7sxfmK^x0w^i}`G}(VYRt<l@lN!P>90)Rg?listJDTQ!Y{?Rm_@12?kS;4y*F*s< zQ&M1os#(FT(n!#%FWBp*AOn@Lpdzfb$eO;ktFt;p&#V@bz9!!<4DAQdWC#{OzpG_d zOWHE!IkuwIM(7KSieqi5G(c?`amk~fKE95&s$R>0Hi8sGQ&(*>|2Dd_l9=yd$lZ`t zY@r4GL0$d9t7xN)`6mKGjPnozx}4c4NIJJWg21nK9*XHvSMAkC*LGMcj{j-{kBXIy2~_wg_v6r zP)2(@0|vB(S39J;?j~fNnz|&X^JaC}k55ls?SUR##?FKWYnZz){UMdUDul(Qlu?H7 zTIZJf4j7+prWrtWseBu01_^$-Lwkc|U7;L6YoH1X)@}x!=@>zG>ay~WfoyXVS_#*l zmR!An(~|MukwxvE8fZx~VW@TCebPz6QA_{A#-S5D55>Cj3of8elhln>Zy6(GxwWU3 z-XjiUomRW*f=DyH{d(3#R%B@y4Z{pbdnLlmPr_1;oz0KCp%ZO{U@M2WR0Diohv+3V zS7k4Oa8jjhf#@)K8Ij8#o(g*G40%E1V*sr9W{ImD?N%@>9M>oYG*IXRVSweVAJ3p5kcT5{XOCn|L} z_*%ZE+QK--T{Js8yAm1^7&u&l;fn#ZU9RT~^_ZQx#-lo+(WR%CB)f;D%edeN7|&2+ zj?|h%FJ}6h=a`0~t?LI%|AO=)LXa&~B8$gk#je9rg>+4*U4DM zOWHU`s|mUWCvbMc&URxOxjDgv_Vq9^e3a8(9g^S=Yj`q9lxu$Z{pMgj6X*owASC%2 z@LmADd{K<`AG1#FNYfdzrldLcTw4TKTYUIctPDnG+U3P?2ezM{3PQl>HgAFTE1CP! zL`zFcJCcNZ&S&_^lQBSbp|K;>)XTpgA7k|4$*-;tKvxPCCVkIIp@^L>X0+~7N$2FR zlxJUn#wyr?E|s}Tij04+Fkxv73-fbS8hCCVStLeVR?=V&yYI#MxwbIFAWZ=&YGfU= zr-;lbcH7s6ueXQZ!2p`}0|gH&M+byJ$u#n_+WD#}+&EVPy4BPR%5%IrTdK_tyFEQE z7zh#Fppv^?Q_~A-wh?%9{l~AxRJQ;}P0%L@j%p89gDx#Ny6O&D<*i?nC|hi^m(0jp zg8dX7?O+ictzOQn@KLlgb2oeIB;`HWyKU!Ak2wlA=D zSP9aSL&@1Tr-_mfO!Jbmc%!Zy6mCsc z#4zR*bcME!WDd1 zSsaBnkAYa!0ucLr%qb^RC|-=wZAbO{Vr4rAn9#)O3tJZT+g~?VlQCygqepf7`20cq%6wPNPdCglEq5vI$sZQZPN>J31dl> zS`rdVgU%2pt;>bvk8C0Hy*t_whcsa#bCIkSP76wAz`c{fI_aO`HEFOKjw`kxfdyU6 z)OdR29#}BhempDt1FS$HhcB1_Q^2(BH*ZS7Z5v@aK)wjMDh(+f)Ktj?UkDh>%GdMBET$;h z$1iyT4|7ru&2^R$!`tr9mP-eFLIfD<0Y=xdvrK`2J3DRmgbk02|O!;_Th)TX! zO=OOQVNmMjZ}-wfPtG=fF}fN$ncho4CNC<`;8_?WCJ#%z{}mb0uk8xxsSId*r7RrO z+g+w7iv{>XVcrdl(Vw*Nx(}D2-^iE1wULfv+qJEeCdihgED2dUbJmLeTVaak2@bms zi$cwZ`_&=8ULva^>T=V`TzkMXZ>&o{>@lj5ky?-$K&Tgp9QGYDv!-@;YoP&G; z1H+g2D{pxYU$l=?^z%jvJ9qz72Mb0r%I%tOf zH8Z#{lZu6UYZYH(q^|zYlgDz+fhYdrM>u$%_m7m&K4>4V!38If599;+;BEno4~!4) zbYT9(_`vwU_~32@mVGtje*Y{fhPEP()hra#-e2xC) zY60XM`Q{1-oIpN44?YC!two5RL|)MCj()XCl*k+&E_46lud(2Hp8x+F+7J2R5LW#v z|hju;^DIAIL{P3>*aH1Nq>9MZXI9KtB3m;2lA0n-F ztI3C4eLkUO_`Hi7`#$jaVZ8tTEuKgF^s&O}8TmjyIC0UZLq3p?J{CAVBOm_@d^~5E zum9`c0-!Jd7ktSn3jffd|ImNj>7Y=M599;+;BEnok1r%XR@W3AC2n>RajqV`50N40%$%`4$JNa1Clf!>z>o!1%!U;7$kTPmB+YkAEya zo)ingdW%BgYmC&@|9SFQ4sVQGDRbINdB9L;AGFT^4eS!~fqbx2F`z*{kdFZx*d^ox z`CzAFK!bcB9|JV7OUMWE!A`}12Khif254ZH{>yy)5+^S}EFw^17MI93e=>-;J#!+l z2-nBoXP-qu%m3oH0Ds?masD`X{Y1`&`BRJbLHqCm1o=QdkPq$_!1%!U;7$kjniwA# z9~d9pEr9WX@xh&rf4Y7M*jtN`yAgRow>$dPCQ-IK`%|j3xaiX%AIL`^3!I++MLvv*jw%iM zy8!>{JYBkPqZz zfChF6`9MC{sTj~8AIQf54eS!~fqbx2F`z*{kdFZx*d^rSALV0pP0>-}1Q!wK>QNgO zUKr{Su3HKJ`9eJJGHj^LSdY;@XdhM{hRVnX@-dVN)??%Y`C#Q?sEm9dA48d7Jw`r| z4^|$A%E$-uF_ekE6x&tEtnE_pE{de{1EdUE*BZ2e{!Zk61> zRWc_WC}`jvIP^1vJ5R?_+d#^4~Keb7D}u;^DIAIL{P3>*aH1Nq>9MZXI9 zKtB3m;2VF z{LS(4q*w?xrKM2#8Y6Y}f1W&c(B=QjTO|`i;OED!#0Le$kOSlc`51!4phM&X`4|Mm zkOSlc`51x(%i-Z)FWw#aEda8DtgyhrCL$lm2OAVD3giR%V8MY+L_YqSk6+^C1&HAU zO3dOC`Q}e%Gva^(x3W$gFa`$+?SuB=fJMIw`9MDUVc;MjAIJv>Ec#W*2lCMm0|x>5 zKt4EN(XT>2kdJ;CI0(oG^1%U%e${`G4*`2?5n=$57j(O$Uu_cYHwNr`GZ(j_W-$kw zi1tA~*q~rhARovF3l3}|^6_Q!aVpOJUEgm3(AQtKuhGBgU$z9#qCh^74;CEQMC1ec zV1t51fqZ-pe2BExttL)ceLkUO_`Hi7SsXM(#I2hD(XE;K}X0x=Ep%m3^_nPkdGlq3_3(UkdHw?3^_nPJ_|lv@?u8x_CafUa`?|| z{f28#gj-SLR@6g6c+es8fqV=CV#opVfqV=>V$dP-fqV=CV#opVfqV=>V$dP-fqV=C zV#opVfqV=>V$dP-fqV=C;*Wy|)?={f6zbNEr9v!KN255)=wNsFc{gPExBy%ue~xm7Pu8PZbi*b#efF+Kt2X&V3&{& zgxYIdF-F3K7;L(3j}SH`hE+*a>MN6XK=6VH^rjC`}^yys9DU#CZhe&er!;%D3A~2 zg9QgR5&1wq*q~rhARovF3l3}|@_~G?LBXOxK9CO<9N0wU<4fV=mpFL=;$DFgv$#aQ z`IFgPO%(sdt&&*{fuBE-)f{Xx+6V2!1_g@(`9MBca9|UW59EUl3Kj+Ofqbywz$PLe z$Ojt~EDGcU`C!3;P5d|b@EBjdgwt;U{*8NKH3+^xw^$Ldw-zCm5_v(lJNnfoQ6f`t ztLD#PtL9!;aQqMY2mQklj6apg2l9b@aJK-)2gV0?I$s7q_u7}@%8HS2`$6tUEIjY@1NyX#w<6A_lH|?v*5rcA|J>H8x$-Go3sPED??V{m=S$bxqMxLV}BkbM>eV z3oj5Ew`z9bx-Bx=2l>GG!1y3}fZGH4Kt8y!LG;K6@_~GCw*bZm#s_ygFn?lvV0>VF zaJRtU8XqosF(Z1X0c(14_|I(phQnk4x01%Kq`A{bp&%c~2lBz)0vI0{AKdA{{E6{_ z@$t{c#|MXlbP8n}ow{z#_JMat7LmmP8rUV|1NmU5VnBm@e8GH#TaOt|Tz#xNNO1a< zbs*zb(E0=HbCA$JXdez(^sA5$`Uxj=iAN??J5Reb#g98@*D&zzC o=!b!WfP5ey9I)tDefE3|lbCfvP{VNZXmZQc^{VTF*XkepKVj3MK>z>% literal 0 HcmV?d00001 diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 016e735b7..c8b111024 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" "strings" "testing" "time" @@ -267,9 +268,73 @@ func TestActionHeadersChange(t *testing.T) { } func TestActionScreenshot(t *testing.T) { + os.Remove("/tmp/test1.png") + browser, instance, err := setUp(t) + defer browser.Close() + defer instance.Close() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + Nuclei Test Page + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "screenshot", Data: map[string]string{"to": "/tmp/test1"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page() + require.FileExists(t, "/tmp/test1.png", el, "could not get screenshot file") } func TestActionTimeInput(t *testing.T) { + browser, instance, err := setUp(t) + defer browser.Close() + defer instance.Close() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + Nuclei Test Page + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "time", Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + require.Equal(t, "2006-01-02", el.MustText(), "could not get input time value") } func TestActionSelectInput(t *testing.T) { @@ -311,6 +376,38 @@ func TestActionSelectInput(t *testing.T) { } func TestActionFilesInput(t *testing.T) { + browser, instance, err := setUp(t) + defer browser.Close() + defer instance.Close() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + Nuclei Test Page + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "files", Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + require.Equal(t, "C:\\fakepath\\test1.pdf", el.MustText(), "could not get input file") } func TestActionWaitLoad(t *testing.T) { diff --git a/v2/pkg/protocols/headless/engine/test.png b/v2/pkg/protocols/headless/engine/test.png new file mode 100755 index 0000000000000000000000000000000000000000..9dc043d6cb4d556e75cb3ec13013834eac1fe8da GIT binary patch literal 145393 zcmeI5cU)8F*TCaOaZsz`La?H^04oAT1k|b+6_7d*kYPno5wHx|LoM36(4Zi@PF#S3 zfNUx*wu(%FIM`eE3VF{%3j{Ly$JrBi{O<;5WR`a^!&FV+0k^>Y z;qr}DLM=b{^Kd4S{>`vNp0C(?jH1u;1q;_umhpVSMG?w$o-f!WKpD^T1@rkR#O7Qd zE|W)6MsPg~CsvOg?lO$m3w|4kW&rzqCT55o79Jk%waV5!_|b<`X})q6r|TH`MggV8 z_VwwK&K;@SH8jj##qo4#-l}~*VeOxu3uj(jv@K^Q-6+6uxlO~n3BkgO4mOJI?~c zsqT$$={-lT<-{>s-_B65Yc>p^2dV`$SG?LA)|OwO*jaIH{4ANJaQD^s4&>e6<1J(2 zEo5EztxmKh^=*X7wh-Movjz`dASN=+WR_4KX@(i*o5s{O-qmB=5un9rUE6Thyy|vf zur&j0;Ah)ZR@&25uco4+Lc4BNog-Ht+Z39sYj2W~Eu;PXoY1x%JHmHl?y_LY(j6?W z<%;jRFQUZk$dv8szNzBVL%qdg|1v9iGk|p1zK)`Nw$1 zic83iy0WE;6~7%f&)%W2FT+ivmXr^-f8oPD`6D=@^`F~Nzk+^)G|i;qmuG8x;( zlHE=nrE*{z+46%vwP$6EU)1uNFI~FiGo>PC_opK-Rd+O>No3qr+8ly{7J;kUCRDCZO z$Jq;}XW!lycq^f+F2=}Bp>-pz=E3thawX@#H*bG`GSDc~zoW!mExs&0A-Pc);pY@Cf=C0*KO)q9joje zw{PFhzh}YQ%8n8HDTfzrySXKKy`av+aJT$}DJd!OwM*{ZSbut2)VpK+GRAI-j-A&- zL(Tb%_J3T}_+nlZ#GtIPn_Wwl$^Dx;F-LCSy0vhoJ)Vq2pPXJ>{>pX)481bZ*li+4C~WmWYiza-96T-(D2F=&TH!6_*zI?U7#0t zw+*7xIA|2F&k+#}A0xKh-@fcuG5>`#W57s_;5F0cIl>MNYgCmO&Esc(cfX=OD6)9* zn^moOdyUiGrhogRpM6iInsW-|Q$HD#g6pxdtKOQ%m}ouRe_=`Rqi;5QEX{8oqm%U5 zmER&muKc=+j|@pfpy1zY@A$sHd(VUzm;m7E#Rx>USYS{)oc ze){=o;#vmJg{-$g#(Ev^*hSLe5{=-cQHI^^pAOLD9cM}(yjUAaR}|NJxRKezuxNgz zTwZ9AsnWP(b)8kNj;C|`N#})H_BonHQ{(e*c1y_Qlm|HLzA*zksWXpHQhcH2)aYIl z@z(S<hVB9H+OyuMC7r_|G)<}GcU8q3S}6#X!hOO$t_ zm9MTT*F{$cUwr^+NRfZq(!5(OV6yZBfqowF)Eg&Yfj+CWCGP>D>?tiCo#`y+7#a_{}nl{nk(3niQUOv>QJ`Pr5)*N|j$D(CDRg_Vr1 zHZ@I6%?)mI{C@pf%&_6z?=zH~9QvREs@=F^9zyB{s% z7nDc~-Ip3sAr-Xi!8e%-ZM$FXg1h0bg(4(XTM(Wao~!G#K*ihO%`Nq))&k?KTk6+V zeR}eB6r@aZxTH{Dh8N}2^XfvOtq`W3&zA{S;YRu*xJF>N;gR~=6t8;>y zpv5SE%rCjSTzBuYGs&7pewReO6zq=>hlt8m-wO+c=(-Udy>iEn9Z}7%H#hmvyA9p_ zKY=keLo{T8XJQ*3^B;g}Mk@(Tl}Lp`=Y1gSCiy4F$4ts-%2Wyn4Gn#BS3?pK$sVVU zR6WDG!YCihVvi%aI@bFtUS6rp4bgLg>l*l&L{MqI&OIIZ&P>L(ZQI=8I;F%qrtuwR zqRA(Poi5iqGrOp?pFTd@2M*doYpyGjhu$PP@#qNtE?MK8r4Y@VXb<7vHN++`8TDDQ zwoO5eTV#tYx81pWm!CoYIq7SzFpmp&|F%rYFzvD>xOQtK)NoPihI6yM>K*It599{x z>?!ipbMmTpYKtM(NJ&j?dUt%%O}H?XHfD~ENJwnsruQ9-7cVw1xuASg{n~~Dq+l9! zG?vOiW{3XeQnI@kDS%X(Frz{_3M#lobx`C2YJ8JlJ=DjE)Vn)_B2v0|6%~q%YEh75 zBIE^&ygetUc1jthy&(RPOz1Yh*0!G>&#)_xv8s;9h?>9IW6I-$%#HwNlt7P<=!BUo zUf7oVX;dh6)r8G*49u2@Dxrxc`&k!j@-bCt&(6*&hwLog3`qFX8FkiFn!B2!oKpMm zlX7Nhb*r>DBu%mapv9;?Eqcswaz(g~8uT^l6_bP&S3zyuM2qdLy6gH$I#)FLcA&qv zZk(M-y0PHx;9!Yvk7DrN?j!m8qTnU`P0$(5kU6yLmA9C`&DEFSdW$PX_cX(rt8+tM zGMfUJyC0fXy}9kRY|mQSb!TReYXe>Mwnm8ViHSm56_KVzD@i9*7u%HaSUwtBpdY}p zkmj!n^u!c^VYaEOt5?u1vs7kv@LpgA-8f_{ycHU|?$*eJ^kHLWpZ}xBaA^3Q^cYHv zN;TPoZlR$qdSOnC&T8M7jxrxYpr~BAuSJeGGs8OGo$9`z6v8b1si0o0dYh(6%Ed*H zP>g7z={~ZVXJ$y{Xfq3pLrR}~t+uq;y5K;d%1X1%F;I8Nb#j8Vjb2?{@9IL&uF4V5 zIOz{rOe^A$*VH9Dr#y3Nz9w7>otbLVi3$9BjeLLF;P2E~8D!tVG`7+=gJ$u)e25I> zxbZ?OSCCrw%EoPu!t#A77cHxEOoEIGq3rP2?o0KUny%uvGW=!nK2M#T=u~L1^|q5L z{n*SJxO8=SS9IiUP>I=%9d~zvqf}|J5YE*$Kk`a0?|eeDN4U6?f|%C+>0{P|z0V)# zWozj9srs$F0p8MXse$UIn5;4E0o21?#!dl_t;x=eT<3jI?!b-33u+aNbMCF}av3o( zPoDu9PVX{iTvPBn%jTn9X z-o2$zrBkS4eh;lA7#elZL#{9@?qv{G52Sv^wH3>o6g;G^U%x(C+D)m)26FU8e+Bz# z&+gp0W6or>l|xEed?rXM{LcO&?_o*x3Ctd?t{$s}4bO$0eT9zX%}r+7TT{({A0>FG zA*!c2P|4&zPnhze<1diX`SQy5&)kFqfS0MyRKI)wbK2!^uPvNzv)M<+M2lt+=yycR ztmx(m=7MBMLgQ*7Mi1S0f)ux?%JY)r-jIarP}0Sy=b&B|Lcq+rM~60L9COTsnHQm( zmQ2=oA4TdUyop7+bgsd>#G?mt?&asiw9i~=RdsR1q*aIAb5@uXtZ?jVRiTmcGEezj zTGAvX-^RwqmSAnEcANBu@@x-DJ>C5p+qX}7?Cc)0)oqM{+&s?!k!M?TdAl-7a zmy)y7Iaq!+*W^pNi>YNmhvg5Ab5P3`<0@EZTv332mhig1XTOqT3SimxavC-#wwyfy(^k3e7ZO*`-Hdhj%}0IYwi(=4tf z$F5pdTEyn9Nu>D0@-(k7sxfmK^x0w^i}`G}(VYRt<l@lN!P>90)Rg?listJDTQ!Y{?Rm_@12?kS;4y*F*s< zQ&M1os#(FT(n!#%FWBp*AOn@Lpdzfb$eO;ktFt;p&#V@bz9!!<4DAQdWC#{OzpG_d zOWHE!IkuwIM(7KSieqi5G(c?`amk~fKE95&s$R>0Hi8sGQ&(*>|2Dd_l9=yd$lZ`t zY@r4GL0$d9t7xN)`6mKGjPnozx}4c4NIJJWg21nK9*XHvSMAkC*LGMcj{j-{kBXIy2~_wg_v6r zP)2(@0|vB(S39J;?j~fNnz|&X^JaC}k55ls?SUR##?FKWYnZz){UMdUDul(Qlu?H7 zTIZJf4j7+prWrtWseBu01_^$-Lwkc|U7;L6YoH1X)@}x!=@>zG>ay~WfoyXVS_#*l zmR!An(~|MukwxvE8fZx~VW@TCebPz6QA_{A#-S5D55>Cj3of8elhln>Zy6(GxwWU3 z-XjiUomRW*f=DyH{d(3#R%B@y4Z{pbdnLlmPr_1;oz0KCp%ZO{U@M2WR0Diohv+3V zS7k4Oa8jjhf#@)K8Ij8#o(g*G40%E1V*sr9W{ImD?N%@>9M>oYG*IXRVSweVAJ3p5kcT5{XOCn|L} z_*%ZE+QK--T{Js8yAm1^7&u&l;fn#ZU9RT~^_ZQx#-lo+(WR%CB)f;D%edeN7|&2+ zj?|h%FJ}6h=a`0~t?LI%|AO=)LXa&~B8$gk#je9rg>+4*U4DM zOWHU`s|mUWCvbMc&URxOxjDgv_Vq9^e3a8(9g^S=Yj`q9lxu$Z{pMgj6X*owASC%2 z@LmADd{K<`AG1#FNYfdzrldLcTw4TKTYUIctPDnG+U3P?2ezM{3PQl>HgAFTE1CP! zL`zFcJCcNZ&S&_^lQBSbp|K;>)XTpgA7k|4$*-;tKvxPCCVkIIp@^L>X0+~7N$2FR zlxJUn#wyr?E|s}Tij04+Fkxv73-fbS8hCCVStLeVR?=V&yYI#MxwbIFAWZ=&YGfU= zr-;lbcH7s6ueXQZ!2p`}0|gH&M+byJ$u#n_+WD#}+&EVPy4BPR%5%IrTdK_tyFEQE z7zh#Fppv^?Q_~A-wh?%9{l~AxRJQ;}P0%L@j%p89gDx#Ny6O&D<*i?nC|hi^m(0jp zg8dX7?O+ictzOQn@KLlgb2oeIB;`HWyKU!Ak2wlA=D zSP9aSL&@1Tr-_mfO!Jbmc%!Zy6mCsc z#4zR*bcME!WDd1 zSsaBnkAYa!0ucLr%qb^RC|-=wZAbO{Vr4rAn9#)O3tJZT+g~?VlQCygqepf7`20cq%6wPNPdCglEq5vI$sZQZPN>J31dl> zS`rdVgU%2pt;>bvk8C0Hy*t_whcsa#bCIkSP76wAz`c{fI_aO`HEFOKjw`kxfdyU6 z)OdR29#}BhempDt1FS$HhcB1_Q^2(BH*ZS7Z5v@aK)wjMDh(+f)Ktj?UkDh>%GdMBET$;h z$1iyT4|7ru&2^R$!`tr9mP-eFLIfD<0Y=xdvrK`2J3DRmgbk02|O!;_Th)TX! zO=OOQVNmMjZ}-wfPtG=fF}fN$ncho4CNC<`;8_?WCJ#%z{}mb0uk8xxsSId*r7RrO z+g+w7iv{>XVcrdl(Vw*Nx(}D2-^iE1wULfv+qJEeCdihgED2dUbJmLeTVaak2@bms zi$cwZ`_&=8ULva^>T=V`TzkMXZ>&o{>@lj5ky?-$K&Tgp9QGYDv!-@;YoP&G; z1H+g2D{pxYU$l=?^z%jvJ9qz72Mb0r%I%tOf zH8Z#{lZu6UYZYH(q^|zYlgDz+fhYdrM>u$%_m7m&K4>4V!38If599;+;BEno4~!4) zbYT9(_`vwU_~32@mVGtje*Y{fhPEP()hra#-e2xC) zY60XM`Q{1-oIpN44?YC!two5RL|)MCj()XCl*k+&E_46lud(2Hp8x+F+7J2R5LW#v z|hju;^DIAIL{P3>*aH1Nq>9MZXI9KtB3m;2lA0n-F ztI3C4eLkUO_`Hi7`#$jaVZ8tTEuKgF^s&O}8TmjyIC0UZLq3p?J{CAVBOm_@d^~5E zum9`c0-!Jd7ktSn3jffd|ImNj>7Y=M599;+;BEnok1r%XR@W3AC2n>RajqV`50N40%$%`4$JNa1Clf!>z>o!1%!U;7$kTPmB+YkAEya zo)ingdW%BgYmC&@|9SFQ4sVQGDRbINdB9L;AGFT^4eS!~fqbx2F`z*{kdFZx*d^ox z`CzAFK!bcB9|JV7OUMWE!A`}12Khif254ZH{>yy)5+^S}EFw^17MI93e=>-;J#!+l z2-nBoXP-qu%m3oH0Ds?masD`X{Y1`&`BRJbLHqCm1o=QdkPq$_!1%!U;7$kjniwA# z9~d9pEr9WX@xh&rf4Y7M*jtN`yAgRow>$dPCQ-IK`%|j3xaiX%AIL`^3!I++MLvv*jw%iM zy8!>{JYBkPqZz zfChF6`9MC{sTj~8AIQf54eS!~fqbx2F`z*{kdFZx*d^rSALV0pP0>-}1Q!wK>QNgO zUKr{Su3HKJ`9eJJGHj^LSdY;@XdhM{hRVnX@-dVN)??%Y`C#Q?sEm9dA48d7Jw`r| z4^|$A%E$-uF_ekE6x&tEtnE_pE{de{1EdUE*BZ2e{!Zk61> zRWc_WC}`jvIP^1vJ5R?_+d#^4~Keb7D}u;^DIAIL{P3>*aH1Nq>9MZXI9 zKtB3m;2VF z{LS(4q*w?xrKM2#8Y6Y}f1W&c(B=QjTO|`i;OED!#0Le$kOSlc`51!4phM&X`4|Mm zkOSlc`51x(%i-Z)FWw#aEda8DtgyhrCL$lm2OAVD3giR%V8MY+L_YqSk6+^C1&HAU zO3dOC`Q}e%Gva^(x3W$gFa`$+?SuB=fJMIw`9MDUVc;MjAIJv>Ec#W*2lCMm0|x>5 zKt4EN(XT>2kdJ;CI0(oG^1%U%e${`G4*`2?5n=$57j(O$Uu_cYHwNr`GZ(j_W-$kw zi1tA~*q~rhARovF3l3}|^6_Q!aVpOJUEgm3(AQtKuhGBgU$z9#qCh^74;CEQMC1ec zV1t51fqZ-pe2BExttL)ceLkUO_`Hi7SsXM(#I2hD(XE;K}X0x=Ep%m3^_nPkdGlq3_3(UkdHw?3^_nPJ_|lv@?u8x_CafUa`?|| z{f28#gj-SLR@6g6c+es8fqV=CV#opVfqV=>V$dP-fqV=CV#opVfqV=>V$dP-fqV=C zV#opVfqV=>V$dP-fqV=C;*Wy|)?={f6zbNEr9v!KN255)=wNsFc{gPExBy%ue~xm7Pu8PZbi*b#efF+Kt2X&V3&{& zgxYIdF-F3K7;L(3j}SH`hE+*a>MN6XK=6VH^rjC`}^yys9DU#CZhe&er!;%D3A~2 zg9QgR5&1wq*q~rhARovF3l3}|@_~G?LBXOxK9CO<9N0wU<4fV=mpFL=;$DFgv$#aQ z`IFgPO%(sdt&&*{fuBE-)f{Xx+6V2!1_g@(`9MBca9|UW59EUl3Kj+Ofqbywz$PLe z$Ojt~EDGcU`C!3;P5d|b@EBjdgwt;U{*8NKH3+^xw^$Ldw-zCm5_v(lJNnfoQ6f`t ztLD#PtL9!;aQqMY2mQklj6apg2l9b@aJK-)2gV0?I$s7q_u7}@%8HS2`$6tUEIjY@1NyX#w<6A_lH|?v*5rcA|J>H8x$-Go3sPED??V{m=S$bxqMxLV}BkbM>eV z3oj5Ew`z9bx-Bx=2l>GG!1y3}fZGH4Kt8y!LG;K6@_~GCw*bZm#s_ygFn?lvV0>VF zaJRtU8XqosF(Z1X0c(14_|I(phQnk4x01%Kq`A{bp&%c~2lBz)0vI0{AKdA{{E6{_ z@$t{c#|MXlbP8n}ow{z#_JMat7LmmP8rUV|1NmU5VnBm@e8GH#TaOt|Tz#xNNO1a< zbs*zb(E0=HbCA$JXdez(^sA5$`Uxj=iAN??J5Reb#g98@*D&zzC o=!b!WfP5ey9I)tDefE3|lbCfvP{VNZXmZQc^{VTF*XkepKVj3MK>z>% literal 0 HcmV?d00001 From e53f54b53f58839eed50c8600f6910f40a4cada8 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sun, 10 Oct 2021 08:30:32 +0200 Subject: [PATCH 086/147] chore: remove files --- .../engine/C:\\fakepath\\test1\\test.png" | Bin 145393 -> 0 bytes v2/pkg/protocols/headless/engine/test.png | Bin 145393 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100755 "v2/pkg/protocols/headless/engine/C:\\fakepath\\test1\\test.png" delete mode 100755 v2/pkg/protocols/headless/engine/test.png diff --git "a/v2/pkg/protocols/headless/engine/C:\\fakepath\\test1\\test.png" "b/v2/pkg/protocols/headless/engine/C:\\fakepath\\test1\\test.png" deleted file mode 100755 index 9dc043d6cb4d556e75cb3ec13013834eac1fe8da..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 145393 zcmeI5cU)8F*TCaOaZsz`La?H^04oAT1k|b+6_7d*kYPno5wHx|LoM36(4Zi@PF#S3 zfNUx*wu(%FIM`eE3VF{%3j{Ly$JrBi{O<;5WR`a^!&FV+0k^>Y z;qr}DLM=b{^Kd4S{>`vNp0C(?jH1u;1q;_umhpVSMG?w$o-f!WKpD^T1@rkR#O7Qd zE|W)6MsPg~CsvOg?lO$m3w|4kW&rzqCT55o79Jk%waV5!_|b<`X})q6r|TH`MggV8 z_VwwK&K;@SH8jj##qo4#-l}~*VeOxu3uj(jv@K^Q-6+6uxlO~n3BkgO4mOJI?~c zsqT$$={-lT<-{>s-_B65Yc>p^2dV`$SG?LA)|OwO*jaIH{4ANJaQD^s4&>e6<1J(2 zEo5EztxmKh^=*X7wh-Movjz`dASN=+WR_4KX@(i*o5s{O-qmB=5un9rUE6Thyy|vf zur&j0;Ah)ZR@&25uco4+Lc4BNog-Ht+Z39sYj2W~Eu;PXoY1x%JHmHl?y_LY(j6?W z<%;jRFQUZk$dv8szNzBVL%qdg|1v9iGk|p1zK)`Nw$1 zic83iy0WE;6~7%f&)%W2FT+ivmXr^-f8oPD`6D=@^`F~Nzk+^)G|i;qmuG8x;( zlHE=nrE*{z+46%vwP$6EU)1uNFI~FiGo>PC_opK-Rd+O>No3qr+8ly{7J;kUCRDCZO z$Jq;}XW!lycq^f+F2=}Bp>-pz=E3thawX@#H*bG`GSDc~zoW!mExs&0A-Pc);pY@Cf=C0*KO)q9joje zw{PFhzh}YQ%8n8HDTfzrySXKKy`av+aJT$}DJd!OwM*{ZSbut2)VpK+GRAI-j-A&- zL(Tb%_J3T}_+nlZ#GtIPn_Wwl$^Dx;F-LCSy0vhoJ)Vq2pPXJ>{>pX)481bZ*li+4C~WmWYiza-96T-(D2F=&TH!6_*zI?U7#0t zw+*7xIA|2F&k+#}A0xKh-@fcuG5>`#W57s_;5F0cIl>MNYgCmO&Esc(cfX=OD6)9* zn^moOdyUiGrhogRpM6iInsW-|Q$HD#g6pxdtKOQ%m}ouRe_=`Rqi;5QEX{8oqm%U5 zmER&muKc=+j|@pfpy1zY@A$sHd(VUzm;m7E#Rx>USYS{)oc ze){=o;#vmJg{-$g#(Ev^*hSLe5{=-cQHI^^pAOLD9cM}(yjUAaR}|NJxRKezuxNgz zTwZ9AsnWP(b)8kNj;C|`N#})H_BonHQ{(e*c1y_Qlm|HLzA*zksWXpHQhcH2)aYIl z@z(S<hVB9H+OyuMC7r_|G)<}GcU8q3S}6#X!hOO$t_ zm9MTT*F{$cUwr^+NRfZq(!5(OV6yZBfqowF)Eg&Yfj+CWCGP>D>?tiCo#`y+7#a_{}nl{nk(3niQUOv>QJ`Pr5)*N|j$D(CDRg_Vr1 zHZ@I6%?)mI{C@pf%&_6z?=zH~9QvREs@=F^9zyB{s% z7nDc~-Ip3sAr-Xi!8e%-ZM$FXg1h0bg(4(XTM(Wao~!G#K*ihO%`Nq))&k?KTk6+V zeR}eB6r@aZxTH{Dh8N}2^XfvOtq`W3&zA{S;YRu*xJF>N;gR~=6t8;>y zpv5SE%rCjSTzBuYGs&7pewReO6zq=>hlt8m-wO+c=(-Udy>iEn9Z}7%H#hmvyA9p_ zKY=keLo{T8XJQ*3^B;g}Mk@(Tl}Lp`=Y1gSCiy4F$4ts-%2Wyn4Gn#BS3?pK$sVVU zR6WDG!YCihVvi%aI@bFtUS6rp4bgLg>l*l&L{MqI&OIIZ&P>L(ZQI=8I;F%qrtuwR zqRA(Poi5iqGrOp?pFTd@2M*doYpyGjhu$PP@#qNtE?MK8r4Y@VXb<7vHN++`8TDDQ zwoO5eTV#tYx81pWm!CoYIq7SzFpmp&|F%rYFzvD>xOQtK)NoPihI6yM>K*It599{x z>?!ipbMmTpYKtM(NJ&j?dUt%%O}H?XHfD~ENJwnsruQ9-7cVw1xuASg{n~~Dq+l9! zG?vOiW{3XeQnI@kDS%X(Frz{_3M#lobx`C2YJ8JlJ=DjE)Vn)_B2v0|6%~q%YEh75 zBIE^&ygetUc1jthy&(RPOz1Yh*0!G>&#)_xv8s;9h?>9IW6I-$%#HwNlt7P<=!BUo zUf7oVX;dh6)r8G*49u2@Dxrxc`&k!j@-bCt&(6*&hwLog3`qFX8FkiFn!B2!oKpMm zlX7Nhb*r>DBu%mapv9;?Eqcswaz(g~8uT^l6_bP&S3zyuM2qdLy6gH$I#)FLcA&qv zZk(M-y0PHx;9!Yvk7DrN?j!m8qTnU`P0$(5kU6yLmA9C`&DEFSdW$PX_cX(rt8+tM zGMfUJyC0fXy}9kRY|mQSb!TReYXe>Mwnm8ViHSm56_KVzD@i9*7u%HaSUwtBpdY}p zkmj!n^u!c^VYaEOt5?u1vs7kv@LpgA-8f_{ycHU|?$*eJ^kHLWpZ}xBaA^3Q^cYHv zN;TPoZlR$qdSOnC&T8M7jxrxYpr~BAuSJeGGs8OGo$9`z6v8b1si0o0dYh(6%Ed*H zP>g7z={~ZVXJ$y{Xfq3pLrR}~t+uq;y5K;d%1X1%F;I8Nb#j8Vjb2?{@9IL&uF4V5 zIOz{rOe^A$*VH9Dr#y3Nz9w7>otbLVi3$9BjeLLF;P2E~8D!tVG`7+=gJ$u)e25I> zxbZ?OSCCrw%EoPu!t#A77cHxEOoEIGq3rP2?o0KUny%uvGW=!nK2M#T=u~L1^|q5L z{n*SJxO8=SS9IiUP>I=%9d~zvqf}|J5YE*$Kk`a0?|eeDN4U6?f|%C+>0{P|z0V)# zWozj9srs$F0p8MXse$UIn5;4E0o21?#!dl_t;x=eT<3jI?!b-33u+aNbMCF}av3o( zPoDu9PVX{iTvPBn%jTn9X z-o2$zrBkS4eh;lA7#elZL#{9@?qv{G52Sv^wH3>o6g;G^U%x(C+D)m)26FU8e+Bz# z&+gp0W6or>l|xEed?rXM{LcO&?_o*x3Ctd?t{$s}4bO$0eT9zX%}r+7TT{({A0>FG zA*!c2P|4&zPnhze<1diX`SQy5&)kFqfS0MyRKI)wbK2!^uPvNzv)M<+M2lt+=yycR ztmx(m=7MBMLgQ*7Mi1S0f)ux?%JY)r-jIarP}0Sy=b&B|Lcq+rM~60L9COTsnHQm( zmQ2=oA4TdUyop7+bgsd>#G?mt?&asiw9i~=RdsR1q*aIAb5@uXtZ?jVRiTmcGEezj zTGAvX-^RwqmSAnEcANBu@@x-DJ>C5p+qX}7?Cc)0)oqM{+&s?!k!M?TdAl-7a zmy)y7Iaq!+*W^pNi>YNmhvg5Ab5P3`<0@EZTv332mhig1XTOqT3SimxavC-#wwyfy(^k3e7ZO*`-Hdhj%}0IYwi(=4tf z$F5pdTEyn9Nu>D0@-(k7sxfmK^x0w^i}`G}(VYRt<l@lN!P>90)Rg?listJDTQ!Y{?Rm_@12?kS;4y*F*s< zQ&M1os#(FT(n!#%FWBp*AOn@Lpdzfb$eO;ktFt;p&#V@bz9!!<4DAQdWC#{OzpG_d zOWHE!IkuwIM(7KSieqi5G(c?`amk~fKE95&s$R>0Hi8sGQ&(*>|2Dd_l9=yd$lZ`t zY@r4GL0$d9t7xN)`6mKGjPnozx}4c4NIJJWg21nK9*XHvSMAkC*LGMcj{j-{kBXIy2~_wg_v6r zP)2(@0|vB(S39J;?j~fNnz|&X^JaC}k55ls?SUR##?FKWYnZz){UMdUDul(Qlu?H7 zTIZJf4j7+prWrtWseBu01_^$-Lwkc|U7;L6YoH1X)@}x!=@>zG>ay~WfoyXVS_#*l zmR!An(~|MukwxvE8fZx~VW@TCebPz6QA_{A#-S5D55>Cj3of8elhln>Zy6(GxwWU3 z-XjiUomRW*f=DyH{d(3#R%B@y4Z{pbdnLlmPr_1;oz0KCp%ZO{U@M2WR0Diohv+3V zS7k4Oa8jjhf#@)K8Ij8#o(g*G40%E1V*sr9W{ImD?N%@>9M>oYG*IXRVSweVAJ3p5kcT5{XOCn|L} z_*%ZE+QK--T{Js8yAm1^7&u&l;fn#ZU9RT~^_ZQx#-lo+(WR%CB)f;D%edeN7|&2+ zj?|h%FJ}6h=a`0~t?LI%|AO=)LXa&~B8$gk#je9rg>+4*U4DM zOWHU`s|mUWCvbMc&URxOxjDgv_Vq9^e3a8(9g^S=Yj`q9lxu$Z{pMgj6X*owASC%2 z@LmADd{K<`AG1#FNYfdzrldLcTw4TKTYUIctPDnG+U3P?2ezM{3PQl>HgAFTE1CP! zL`zFcJCcNZ&S&_^lQBSbp|K;>)XTpgA7k|4$*-;tKvxPCCVkIIp@^L>X0+~7N$2FR zlxJUn#wyr?E|s}Tij04+Fkxv73-fbS8hCCVStLeVR?=V&yYI#MxwbIFAWZ=&YGfU= zr-;lbcH7s6ueXQZ!2p`}0|gH&M+byJ$u#n_+WD#}+&EVPy4BPR%5%IrTdK_tyFEQE z7zh#Fppv^?Q_~A-wh?%9{l~AxRJQ;}P0%L@j%p89gDx#Ny6O&D<*i?nC|hi^m(0jp zg8dX7?O+ictzOQn@KLlgb2oeIB;`HWyKU!Ak2wlA=D zSP9aSL&@1Tr-_mfO!Jbmc%!Zy6mCsc z#4zR*bcME!WDd1 zSsaBnkAYa!0ucLr%qb^RC|-=wZAbO{Vr4rAn9#)O3tJZT+g~?VlQCygqepf7`20cq%6wPNPdCglEq5vI$sZQZPN>J31dl> zS`rdVgU%2pt;>bvk8C0Hy*t_whcsa#bCIkSP76wAz`c{fI_aO`HEFOKjw`kxfdyU6 z)OdR29#}BhempDt1FS$HhcB1_Q^2(BH*ZS7Z5v@aK)wjMDh(+f)Ktj?UkDh>%GdMBET$;h z$1iyT4|7ru&2^R$!`tr9mP-eFLIfD<0Y=xdvrK`2J3DRmgbk02|O!;_Th)TX! zO=OOQVNmMjZ}-wfPtG=fF}fN$ncho4CNC<`;8_?WCJ#%z{}mb0uk8xxsSId*r7RrO z+g+w7iv{>XVcrdl(Vw*Nx(}D2-^iE1wULfv+qJEeCdihgED2dUbJmLeTVaak2@bms zi$cwZ`_&=8ULva^>T=V`TzkMXZ>&o{>@lj5ky?-$K&Tgp9QGYDv!-@;YoP&G; z1H+g2D{pxYU$l=?^z%jvJ9qz72Mb0r%I%tOf zH8Z#{lZu6UYZYH(q^|zYlgDz+fhYdrM>u$%_m7m&K4>4V!38If599;+;BEno4~!4) zbYT9(_`vwU_~32@mVGtje*Y{fhPEP()hra#-e2xC) zY60XM`Q{1-oIpN44?YC!two5RL|)MCj()XCl*k+&E_46lud(2Hp8x+F+7J2R5LW#v z|hju;^DIAIL{P3>*aH1Nq>9MZXI9KtB3m;2lA0n-F ztI3C4eLkUO_`Hi7`#$jaVZ8tTEuKgF^s&O}8TmjyIC0UZLq3p?J{CAVBOm_@d^~5E zum9`c0-!Jd7ktSn3jffd|ImNj>7Y=M599;+;BEnok1r%XR@W3AC2n>RajqV`50N40%$%`4$JNa1Clf!>z>o!1%!U;7$kTPmB+YkAEya zo)ingdW%BgYmC&@|9SFQ4sVQGDRbINdB9L;AGFT^4eS!~fqbx2F`z*{kdFZx*d^ox z`CzAFK!bcB9|JV7OUMWE!A`}12Khif254ZH{>yy)5+^S}EFw^17MI93e=>-;J#!+l z2-nBoXP-qu%m3oH0Ds?masD`X{Y1`&`BRJbLHqCm1o=QdkPq$_!1%!U;7$kjniwA# z9~d9pEr9WX@xh&rf4Y7M*jtN`yAgRow>$dPCQ-IK`%|j3xaiX%AIL`^3!I++MLvv*jw%iM zy8!>{JYBkPqZz zfChF6`9MC{sTj~8AIQf54eS!~fqbx2F`z*{kdFZx*d^rSALV0pP0>-}1Q!wK>QNgO zUKr{Su3HKJ`9eJJGHj^LSdY;@XdhM{hRVnX@-dVN)??%Y`C#Q?sEm9dA48d7Jw`r| z4^|$A%E$-uF_ekE6x&tEtnE_pE{de{1EdUE*BZ2e{!Zk61> zRWc_WC}`jvIP^1vJ5R?_+d#^4~Keb7D}u;^DIAIL{P3>*aH1Nq>9MZXI9 zKtB3m;2VF z{LS(4q*w?xrKM2#8Y6Y}f1W&c(B=QjTO|`i;OED!#0Le$kOSlc`51!4phM&X`4|Mm zkOSlc`51x(%i-Z)FWw#aEda8DtgyhrCL$lm2OAVD3giR%V8MY+L_YqSk6+^C1&HAU zO3dOC`Q}e%Gva^(x3W$gFa`$+?SuB=fJMIw`9MDUVc;MjAIJv>Ec#W*2lCMm0|x>5 zKt4EN(XT>2kdJ;CI0(oG^1%U%e${`G4*`2?5n=$57j(O$Uu_cYHwNr`GZ(j_W-$kw zi1tA~*q~rhARovF3l3}|^6_Q!aVpOJUEgm3(AQtKuhGBgU$z9#qCh^74;CEQMC1ec zV1t51fqZ-pe2BExttL)ceLkUO_`Hi7SsXM(#I2hD(XE;K}X0x=Ep%m3^_nPkdGlq3_3(UkdHw?3^_nPJ_|lv@?u8x_CafUa`?|| z{f28#gj-SLR@6g6c+es8fqV=CV#opVfqV=>V$dP-fqV=CV#opVfqV=>V$dP-fqV=C zV#opVfqV=>V$dP-fqV=C;*Wy|)?={f6zbNEr9v!KN255)=wNsFc{gPExBy%ue~xm7Pu8PZbi*b#efF+Kt2X&V3&{& zgxYIdF-F3K7;L(3j}SH`hE+*a>MN6XK=6VH^rjC`}^yys9DU#CZhe&er!;%D3A~2 zg9QgR5&1wq*q~rhARovF3l3}|@_~G?LBXOxK9CO<9N0wU<4fV=mpFL=;$DFgv$#aQ z`IFgPO%(sdt&&*{fuBE-)f{Xx+6V2!1_g@(`9MBca9|UW59EUl3Kj+Ofqbywz$PLe z$Ojt~EDGcU`C!3;P5d|b@EBjdgwt;U{*8NKH3+^xw^$Ldw-zCm5_v(lJNnfoQ6f`t ztLD#PtL9!;aQqMY2mQklj6apg2l9b@aJK-)2gV0?I$s7q_u7}@%8HS2`$6tUEIjY@1NyX#w<6A_lH|?v*5rcA|J>H8x$-Go3sPED??V{m=S$bxqMxLV}BkbM>eV z3oj5Ew`z9bx-Bx=2l>GG!1y3}fZGH4Kt8y!LG;K6@_~GCw*bZm#s_ygFn?lvV0>VF zaJRtU8XqosF(Z1X0c(14_|I(phQnk4x01%Kq`A{bp&%c~2lBz)0vI0{AKdA{{E6{_ z@$t{c#|MXlbP8n}ow{z#_JMat7LmmP8rUV|1NmU5VnBm@e8GH#TaOt|Tz#xNNO1a< zbs*zb(E0=HbCA$JXdez(^sA5$`Uxj=iAN??J5Reb#g98@*D&zzC o=!b!WfP5ey9I)tDefE3|lbCfvP{VNZXmZQc^{VTF*XkepKVj3MK>z>% diff --git a/v2/pkg/protocols/headless/engine/test.png b/v2/pkg/protocols/headless/engine/test.png deleted file mode 100755 index 9dc043d6cb4d556e75cb3ec13013834eac1fe8da..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 145393 zcmeI5cU)8F*TCaOaZsz`La?H^04oAT1k|b+6_7d*kYPno5wHx|LoM36(4Zi@PF#S3 zfNUx*wu(%FIM`eE3VF{%3j{Ly$JrBi{O<;5WR`a^!&FV+0k^>Y z;qr}DLM=b{^Kd4S{>`vNp0C(?jH1u;1q;_umhpVSMG?w$o-f!WKpD^T1@rkR#O7Qd zE|W)6MsPg~CsvOg?lO$m3w|4kW&rzqCT55o79Jk%waV5!_|b<`X})q6r|TH`MggV8 z_VwwK&K;@SH8jj##qo4#-l}~*VeOxu3uj(jv@K^Q-6+6uxlO~n3BkgO4mOJI?~c zsqT$$={-lT<-{>s-_B65Yc>p^2dV`$SG?LA)|OwO*jaIH{4ANJaQD^s4&>e6<1J(2 zEo5EztxmKh^=*X7wh-Movjz`dASN=+WR_4KX@(i*o5s{O-qmB=5un9rUE6Thyy|vf zur&j0;Ah)ZR@&25uco4+Lc4BNog-Ht+Z39sYj2W~Eu;PXoY1x%JHmHl?y_LY(j6?W z<%;jRFQUZk$dv8szNzBVL%qdg|1v9iGk|p1zK)`Nw$1 zic83iy0WE;6~7%f&)%W2FT+ivmXr^-f8oPD`6D=@^`F~Nzk+^)G|i;qmuG8x;( zlHE=nrE*{z+46%vwP$6EU)1uNFI~FiGo>PC_opK-Rd+O>No3qr+8ly{7J;kUCRDCZO z$Jq;}XW!lycq^f+F2=}Bp>-pz=E3thawX@#H*bG`GSDc~zoW!mExs&0A-Pc);pY@Cf=C0*KO)q9joje zw{PFhzh}YQ%8n8HDTfzrySXKKy`av+aJT$}DJd!OwM*{ZSbut2)VpK+GRAI-j-A&- zL(Tb%_J3T}_+nlZ#GtIPn_Wwl$^Dx;F-LCSy0vhoJ)Vq2pPXJ>{>pX)481bZ*li+4C~WmWYiza-96T-(D2F=&TH!6_*zI?U7#0t zw+*7xIA|2F&k+#}A0xKh-@fcuG5>`#W57s_;5F0cIl>MNYgCmO&Esc(cfX=OD6)9* zn^moOdyUiGrhogRpM6iInsW-|Q$HD#g6pxdtKOQ%m}ouRe_=`Rqi;5QEX{8oqm%U5 zmER&muKc=+j|@pfpy1zY@A$sHd(VUzm;m7E#Rx>USYS{)oc ze){=o;#vmJg{-$g#(Ev^*hSLe5{=-cQHI^^pAOLD9cM}(yjUAaR}|NJxRKezuxNgz zTwZ9AsnWP(b)8kNj;C|`N#})H_BonHQ{(e*c1y_Qlm|HLzA*zksWXpHQhcH2)aYIl z@z(S<hVB9H+OyuMC7r_|G)<}GcU8q3S}6#X!hOO$t_ zm9MTT*F{$cUwr^+NRfZq(!5(OV6yZBfqowF)Eg&Yfj+CWCGP>D>?tiCo#`y+7#a_{}nl{nk(3niQUOv>QJ`Pr5)*N|j$D(CDRg_Vr1 zHZ@I6%?)mI{C@pf%&_6z?=zH~9QvREs@=F^9zyB{s% z7nDc~-Ip3sAr-Xi!8e%-ZM$FXg1h0bg(4(XTM(Wao~!G#K*ihO%`Nq))&k?KTk6+V zeR}eB6r@aZxTH{Dh8N}2^XfvOtq`W3&zA{S;YRu*xJF>N;gR~=6t8;>y zpv5SE%rCjSTzBuYGs&7pewReO6zq=>hlt8m-wO+c=(-Udy>iEn9Z}7%H#hmvyA9p_ zKY=keLo{T8XJQ*3^B;g}Mk@(Tl}Lp`=Y1gSCiy4F$4ts-%2Wyn4Gn#BS3?pK$sVVU zR6WDG!YCihVvi%aI@bFtUS6rp4bgLg>l*l&L{MqI&OIIZ&P>L(ZQI=8I;F%qrtuwR zqRA(Poi5iqGrOp?pFTd@2M*doYpyGjhu$PP@#qNtE?MK8r4Y@VXb<7vHN++`8TDDQ zwoO5eTV#tYx81pWm!CoYIq7SzFpmp&|F%rYFzvD>xOQtK)NoPihI6yM>K*It599{x z>?!ipbMmTpYKtM(NJ&j?dUt%%O}H?XHfD~ENJwnsruQ9-7cVw1xuASg{n~~Dq+l9! zG?vOiW{3XeQnI@kDS%X(Frz{_3M#lobx`C2YJ8JlJ=DjE)Vn)_B2v0|6%~q%YEh75 zBIE^&ygetUc1jthy&(RPOz1Yh*0!G>&#)_xv8s;9h?>9IW6I-$%#HwNlt7P<=!BUo zUf7oVX;dh6)r8G*49u2@Dxrxc`&k!j@-bCt&(6*&hwLog3`qFX8FkiFn!B2!oKpMm zlX7Nhb*r>DBu%mapv9;?Eqcswaz(g~8uT^l6_bP&S3zyuM2qdLy6gH$I#)FLcA&qv zZk(M-y0PHx;9!Yvk7DrN?j!m8qTnU`P0$(5kU6yLmA9C`&DEFSdW$PX_cX(rt8+tM zGMfUJyC0fXy}9kRY|mQSb!TReYXe>Mwnm8ViHSm56_KVzD@i9*7u%HaSUwtBpdY}p zkmj!n^u!c^VYaEOt5?u1vs7kv@LpgA-8f_{ycHU|?$*eJ^kHLWpZ}xBaA^3Q^cYHv zN;TPoZlR$qdSOnC&T8M7jxrxYpr~BAuSJeGGs8OGo$9`z6v8b1si0o0dYh(6%Ed*H zP>g7z={~ZVXJ$y{Xfq3pLrR}~t+uq;y5K;d%1X1%F;I8Nb#j8Vjb2?{@9IL&uF4V5 zIOz{rOe^A$*VH9Dr#y3Nz9w7>otbLVi3$9BjeLLF;P2E~8D!tVG`7+=gJ$u)e25I> zxbZ?OSCCrw%EoPu!t#A77cHxEOoEIGq3rP2?o0KUny%uvGW=!nK2M#T=u~L1^|q5L z{n*SJxO8=SS9IiUP>I=%9d~zvqf}|J5YE*$Kk`a0?|eeDN4U6?f|%C+>0{P|z0V)# zWozj9srs$F0p8MXse$UIn5;4E0o21?#!dl_t;x=eT<3jI?!b-33u+aNbMCF}av3o( zPoDu9PVX{iTvPBn%jTn9X z-o2$zrBkS4eh;lA7#elZL#{9@?qv{G52Sv^wH3>o6g;G^U%x(C+D)m)26FU8e+Bz# z&+gp0W6or>l|xEed?rXM{LcO&?_o*x3Ctd?t{$s}4bO$0eT9zX%}r+7TT{({A0>FG zA*!c2P|4&zPnhze<1diX`SQy5&)kFqfS0MyRKI)wbK2!^uPvNzv)M<+M2lt+=yycR ztmx(m=7MBMLgQ*7Mi1S0f)ux?%JY)r-jIarP}0Sy=b&B|Lcq+rM~60L9COTsnHQm( zmQ2=oA4TdUyop7+bgsd>#G?mt?&asiw9i~=RdsR1q*aIAb5@uXtZ?jVRiTmcGEezj zTGAvX-^RwqmSAnEcANBu@@x-DJ>C5p+qX}7?Cc)0)oqM{+&s?!k!M?TdAl-7a zmy)y7Iaq!+*W^pNi>YNmhvg5Ab5P3`<0@EZTv332mhig1XTOqT3SimxavC-#wwyfy(^k3e7ZO*`-Hdhj%}0IYwi(=4tf z$F5pdTEyn9Nu>D0@-(k7sxfmK^x0w^i}`G}(VYRt<l@lN!P>90)Rg?listJDTQ!Y{?Rm_@12?kS;4y*F*s< zQ&M1os#(FT(n!#%FWBp*AOn@Lpdzfb$eO;ktFt;p&#V@bz9!!<4DAQdWC#{OzpG_d zOWHE!IkuwIM(7KSieqi5G(c?`amk~fKE95&s$R>0Hi8sGQ&(*>|2Dd_l9=yd$lZ`t zY@r4GL0$d9t7xN)`6mKGjPnozx}4c4NIJJWg21nK9*XHvSMAkC*LGMcj{j-{kBXIy2~_wg_v6r zP)2(@0|vB(S39J;?j~fNnz|&X^JaC}k55ls?SUR##?FKWYnZz){UMdUDul(Qlu?H7 zTIZJf4j7+prWrtWseBu01_^$-Lwkc|U7;L6YoH1X)@}x!=@>zG>ay~WfoyXVS_#*l zmR!An(~|MukwxvE8fZx~VW@TCebPz6QA_{A#-S5D55>Cj3of8elhln>Zy6(GxwWU3 z-XjiUomRW*f=DyH{d(3#R%B@y4Z{pbdnLlmPr_1;oz0KCp%ZO{U@M2WR0Diohv+3V zS7k4Oa8jjhf#@)K8Ij8#o(g*G40%E1V*sr9W{ImD?N%@>9M>oYG*IXRVSweVAJ3p5kcT5{XOCn|L} z_*%ZE+QK--T{Js8yAm1^7&u&l;fn#ZU9RT~^_ZQx#-lo+(WR%CB)f;D%edeN7|&2+ zj?|h%FJ}6h=a`0~t?LI%|AO=)LXa&~B8$gk#je9rg>+4*U4DM zOWHU`s|mUWCvbMc&URxOxjDgv_Vq9^e3a8(9g^S=Yj`q9lxu$Z{pMgj6X*owASC%2 z@LmADd{K<`AG1#FNYfdzrldLcTw4TKTYUIctPDnG+U3P?2ezM{3PQl>HgAFTE1CP! zL`zFcJCcNZ&S&_^lQBSbp|K;>)XTpgA7k|4$*-;tKvxPCCVkIIp@^L>X0+~7N$2FR zlxJUn#wyr?E|s}Tij04+Fkxv73-fbS8hCCVStLeVR?=V&yYI#MxwbIFAWZ=&YGfU= zr-;lbcH7s6ueXQZ!2p`}0|gH&M+byJ$u#n_+WD#}+&EVPy4BPR%5%IrTdK_tyFEQE z7zh#Fppv^?Q_~A-wh?%9{l~AxRJQ;}P0%L@j%p89gDx#Ny6O&D<*i?nC|hi^m(0jp zg8dX7?O+ictzOQn@KLlgb2oeIB;`HWyKU!Ak2wlA=D zSP9aSL&@1Tr-_mfO!Jbmc%!Zy6mCsc z#4zR*bcME!WDd1 zSsaBnkAYa!0ucLr%qb^RC|-=wZAbO{Vr4rAn9#)O3tJZT+g~?VlQCygqepf7`20cq%6wPNPdCglEq5vI$sZQZPN>J31dl> zS`rdVgU%2pt;>bvk8C0Hy*t_whcsa#bCIkSP76wAz`c{fI_aO`HEFOKjw`kxfdyU6 z)OdR29#}BhempDt1FS$HhcB1_Q^2(BH*ZS7Z5v@aK)wjMDh(+f)Ktj?UkDh>%GdMBET$;h z$1iyT4|7ru&2^R$!`tr9mP-eFLIfD<0Y=xdvrK`2J3DRmgbk02|O!;_Th)TX! zO=OOQVNmMjZ}-wfPtG=fF}fN$ncho4CNC<`;8_?WCJ#%z{}mb0uk8xxsSId*r7RrO z+g+w7iv{>XVcrdl(Vw*Nx(}D2-^iE1wULfv+qJEeCdihgED2dUbJmLeTVaak2@bms zi$cwZ`_&=8ULva^>T=V`TzkMXZ>&o{>@lj5ky?-$K&Tgp9QGYDv!-@;YoP&G; z1H+g2D{pxYU$l=?^z%jvJ9qz72Mb0r%I%tOf zH8Z#{lZu6UYZYH(q^|zYlgDz+fhYdrM>u$%_m7m&K4>4V!38If599;+;BEno4~!4) zbYT9(_`vwU_~32@mVGtje*Y{fhPEP()hra#-e2xC) zY60XM`Q{1-oIpN44?YC!two5RL|)MCj()XCl*k+&E_46lud(2Hp8x+F+7J2R5LW#v z|hju;^DIAIL{P3>*aH1Nq>9MZXI9KtB3m;2lA0n-F ztI3C4eLkUO_`Hi7`#$jaVZ8tTEuKgF^s&O}8TmjyIC0UZLq3p?J{CAVBOm_@d^~5E zum9`c0-!Jd7ktSn3jffd|ImNj>7Y=M599;+;BEnok1r%XR@W3AC2n>RajqV`50N40%$%`4$JNa1Clf!>z>o!1%!U;7$kTPmB+YkAEya zo)ingdW%BgYmC&@|9SFQ4sVQGDRbINdB9L;AGFT^4eS!~fqbx2F`z*{kdFZx*d^ox z`CzAFK!bcB9|JV7OUMWE!A`}12Khif254ZH{>yy)5+^S}EFw^17MI93e=>-;J#!+l z2-nBoXP-qu%m3oH0Ds?masD`X{Y1`&`BRJbLHqCm1o=QdkPq$_!1%!U;7$kjniwA# z9~d9pEr9WX@xh&rf4Y7M*jtN`yAgRow>$dPCQ-IK`%|j3xaiX%AIL`^3!I++MLvv*jw%iM zy8!>{JYBkPqZz zfChF6`9MC{sTj~8AIQf54eS!~fqbx2F`z*{kdFZx*d^rSALV0pP0>-}1Q!wK>QNgO zUKr{Su3HKJ`9eJJGHj^LSdY;@XdhM{hRVnX@-dVN)??%Y`C#Q?sEm9dA48d7Jw`r| z4^|$A%E$-uF_ekE6x&tEtnE_pE{de{1EdUE*BZ2e{!Zk61> zRWc_WC}`jvIP^1vJ5R?_+d#^4~Keb7D}u;^DIAIL{P3>*aH1Nq>9MZXI9 zKtB3m;2VF z{LS(4q*w?xrKM2#8Y6Y}f1W&c(B=QjTO|`i;OED!#0Le$kOSlc`51!4phM&X`4|Mm zkOSlc`51x(%i-Z)FWw#aEda8DtgyhrCL$lm2OAVD3giR%V8MY+L_YqSk6+^C1&HAU zO3dOC`Q}e%Gva^(x3W$gFa`$+?SuB=fJMIw`9MDUVc;MjAIJv>Ec#W*2lCMm0|x>5 zKt4EN(XT>2kdJ;CI0(oG^1%U%e${`G4*`2?5n=$57j(O$Uu_cYHwNr`GZ(j_W-$kw zi1tA~*q~rhARovF3l3}|^6_Q!aVpOJUEgm3(AQtKuhGBgU$z9#qCh^74;CEQMC1ec zV1t51fqZ-pe2BExttL)ceLkUO_`Hi7SsXM(#I2hD(XE;K}X0x=Ep%m3^_nPkdGlq3_3(UkdHw?3^_nPJ_|lv@?u8x_CafUa`?|| z{f28#gj-SLR@6g6c+es8fqV=CV#opVfqV=>V$dP-fqV=CV#opVfqV=>V$dP-fqV=C zV#opVfqV=>V$dP-fqV=C;*Wy|)?={f6zbNEr9v!KN255)=wNsFc{gPExBy%ue~xm7Pu8PZbi*b#efF+Kt2X&V3&{& zgxYIdF-F3K7;L(3j}SH`hE+*a>MN6XK=6VH^rjC`}^yys9DU#CZhe&er!;%D3A~2 zg9QgR5&1wq*q~rhARovF3l3}|@_~G?LBXOxK9CO<9N0wU<4fV=mpFL=;$DFgv$#aQ z`IFgPO%(sdt&&*{fuBE-)f{Xx+6V2!1_g@(`9MBca9|UW59EUl3Kj+Ofqbywz$PLe z$Ojt~EDGcU`C!3;P5d|b@EBjdgwt;U{*8NKH3+^xw^$Ldw-zCm5_v(lJNnfoQ6f`t ztLD#PtL9!;aQqMY2mQklj6apg2l9b@aJK-)2gV0?I$s7q_u7}@%8HS2`$6tUEIjY@1NyX#w<6A_lH|?v*5rcA|J>H8x$-Go3sPED??V{m=S$bxqMxLV}BkbM>eV z3oj5Ew`z9bx-Bx=2l>GG!1y3}fZGH4Kt8y!LG;K6@_~GCw*bZm#s_ygFn?lvV0>VF zaJRtU8XqosF(Z1X0c(14_|I(phQnk4x01%Kq`A{bp&%c~2lBz)0vI0{AKdA{{E6{_ z@$t{c#|MXlbP8n}ow{z#_JMat7LmmP8rUV|1NmU5VnBm@e8GH#TaOt|Tz#xNNO1a< zbs*zb(E0=HbCA$JXdez(^sA5$`Uxj=iAN??J5Reb#g98@*D&zzC o=!b!WfP5ey9I)tDefE3|lbCfvP{VNZXmZQc^{VTF*XkepKVj3MK>z>% From 3cd1b70bc018b5b6b447b741fe874237e6fec774 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sun, 10 Oct 2021 08:36:44 +0200 Subject: [PATCH 087/147] chore: fix screenshot path --- v2/pkg/protocols/headless/engine/page_actions_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index c8b111024..0a8285840 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -268,7 +268,6 @@ func TestActionHeadersChange(t *testing.T) { } func TestActionScreenshot(t *testing.T) { - os.Remove("/tmp/test1.png") browser, instance, err := setUp(t) defer browser.Close() defer instance.Close() @@ -291,7 +290,7 @@ func TestActionScreenshot(t *testing.T) { actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, - {ActionType: "screenshot", Data: map[string]string{"to": "/tmp/test1"}}, + {ActionType: "screenshot", Data: map[string]string{"to": "test"}}, } _, page, err := instance.Run(parsed, actions, 20*time.Second) require.Nil(t, err, "could not run page actions") @@ -299,7 +298,8 @@ func TestActionScreenshot(t *testing.T) { require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") el := page.Page() - require.FileExists(t, "/tmp/test1.png", el, "could not get screenshot file") + require.FileExists(t, "test.png", el, "could not get screenshot file") + os.Remove("test.png") } func TestActionTimeInput(t *testing.T) { From 62bfdf30b4484d4d1c08d4691d0356841fb32dfe Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sun, 10 Oct 2021 13:17:35 +0200 Subject: [PATCH 088/147] chore: add more tests --- .../protocols/headless/engine/page_actions.go | 2 +- .../headless/engine/page_actions_test.go | 134 ++++++++++++++++++ 2 files changed, 135 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index ad4eec626..d9cb6451e 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -46,7 +46,7 @@ func (p *Page) ExecuteActions(baseURL *url.URL, actions []*Action) (map[string]s case ActionGetResource: err = p.GetResource(act, outData) case ActionExtract: - err = p.SelectInputElement(act, outData) + err = p.ExtractElement(act, outData) case ActionWaitEvent: err = p.WaitEvent(act, outData) case ActionFilesInput: diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 0a8285840..3331f4003 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -411,15 +411,149 @@ func TestActionFilesInput(t *testing.T) { } func TestActionWaitLoad(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + el := page.Page().MustElement("button") + style, err := el.Attribute("style") + require.Equal(t, "color: red;", string(*style), "could not get color") } func TestActionGetResource(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "getresource", Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, + } + out, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, len(out["src"]), 3159, "could not find resource") } func TestActionExtract(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "extract", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, + } + out, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "Wait for me!", out["extract"], "could not extract text") } func TestActionSetMethod(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "setmethod", Data: map[string]string{"part": "x", "method": "SET"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "SET", page.rules[0].Args["method"], "could not find resource") } func TestActionAddHeader(t *testing.T) { From 8502785a924a79365a8d8ee778c5ca08a5609d23 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sun, 10 Oct 2021 17:18:53 +0200 Subject: [PATCH 089/147] chore: add more tests --- .../headless/engine/page_actions_test.go | 141 ++++++++++++++---- 1 file changed, 114 insertions(+), 27 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 3331f4003..2e03fc8f6 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -16,19 +16,13 @@ import ( "github.com/stretchr/testify/require" ) -func setUp(t *testing.T) (*Browser, *Instance, error) { - t.Helper() +func TestActionNavigate(t *testing.T) { _ = protocolstate.Init(&types.Options{}) browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") instance, err := browser.NewInstance() - return browser, instance, err -} - -func TestActionNavigate(t *testing.T) { - browser, instance, err := setUp(t) defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") @@ -58,7 +52,12 @@ func TestActionNavigate(t *testing.T) { } func TestActionScript(t *testing.T) { - browser, instance, err := setUp(t) + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + + instance, err := browser.NewInstance() defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") @@ -123,7 +122,12 @@ func TestActionScript(t *testing.T) { } func TestActionClick(t *testing.T) { - browser, instance, err := setUp(t) + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + + instance, err := browser.NewInstance() defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") @@ -159,7 +163,12 @@ func TestActionClick(t *testing.T) { } func TestActionRightClick(t *testing.T) { - browser, instance, err := setUp(t) + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + + instance, err := browser.NewInstance() defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") @@ -203,7 +212,12 @@ func TestActionRightClick(t *testing.T) { } func TestActionTextInput(t *testing.T) { - browser, instance, err := setUp(t) + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + + instance, err := browser.NewInstance() defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") @@ -240,7 +254,12 @@ func TestActionTextInput(t *testing.T) { } func TestActionHeadersChange(t *testing.T) { - browser, instance, err := setUp(t) + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + + instance, err := browser.NewInstance() defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") @@ -268,7 +287,12 @@ func TestActionHeadersChange(t *testing.T) { } func TestActionScreenshot(t *testing.T) { - browser, instance, err := setUp(t) + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + + instance, err := browser.NewInstance() defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") @@ -303,7 +327,12 @@ func TestActionScreenshot(t *testing.T) { } func TestActionTimeInput(t *testing.T) { - browser, instance, err := setUp(t) + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + + instance, err := browser.NewInstance() defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") @@ -338,7 +367,12 @@ func TestActionTimeInput(t *testing.T) { } func TestActionSelectInput(t *testing.T) { - browser, instance, err := setUp(t) + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + + instance, err := browser.NewInstance() defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") @@ -376,7 +410,12 @@ func TestActionSelectInput(t *testing.T) { } func TestActionFilesInput(t *testing.T) { - browser, instance, err := setUp(t) + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + + instance, err := browser.NewInstance() defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") @@ -557,7 +596,12 @@ func TestActionSetMethod(t *testing.T) { } func TestActionAddHeader(t *testing.T) { - browser, instance, err := setUp(t) + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + + instance, err := browser.NewInstance() defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") @@ -585,14 +629,18 @@ func TestActionAddHeader(t *testing.T) { } func TestActionDeleteHeader(t *testing.T) { - browser, instance, err := setUp(t) + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + + instance, err := browser.NewInstance() defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Test1") == "Hello" && r.Header.Get("Test2") == "" { - fmt.Println(r.Header) fmt.Fprintln(w, `header deleted`) } })) @@ -616,7 +664,12 @@ func TestActionDeleteHeader(t *testing.T) { } func TestActionSetBody(t *testing.T) { - browser, instance, err := setUp(t) + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + + instance, err := browser.NewInstance() defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") @@ -642,11 +695,13 @@ func TestActionSetBody(t *testing.T) { require.Equal(t, "hello", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") } -func TestActionWaitEvent(t *testing.T) { -} - func TestActionKeyboard(t *testing.T) { - browser, instance, err := setUp(t) + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + + instance, err := browser.NewInstance() defer browser.Close() defer instance.Close() require.Nil(t, err, "could not create browser instance") @@ -685,10 +740,42 @@ func TestActionKeyboard(t *testing.T) { require.Equal(t, "Test2", el.MustText(), "could not get input change value") } -func TestActionDebug(t *testing.T) { -} - func TestActionSleep(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "sleep", Data: map[string]string{"duration": "2"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + defer page.Close() + + require.Nil(t, err, "could not run page actions") + require.True(t, page.Page().MustElement("button").MustVisible(), "could not get button") } func TestActionWaitVisible(t *testing.T) { From 322a6213c6cf727bba249adc067fdf3313abbfee Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Sun, 10 Oct 2021 17:33:18 +0200 Subject: [PATCH 090/147] chore: refactor tests --- .../headless/engine/page_actions_test.go | 63 ++++++++++--------- 1 file changed, 35 insertions(+), 28 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 2e03fc8f6..bad20e372 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -21,11 +21,11 @@ func TestActionNavigate(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -56,11 +56,11 @@ func TestActionScript(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() t.Run("run-and-results", func(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -126,11 +126,11 @@ func TestActionClick(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -167,11 +167,11 @@ func TestActionRightClick(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -216,11 +216,11 @@ func TestActionTextInput(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -258,11 +258,11 @@ func TestActionHeadersChange(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Test") == "Hello" { @@ -291,11 +291,11 @@ func TestActionScreenshot(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -331,11 +331,11 @@ func TestActionTimeInput(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -371,11 +371,11 @@ func TestActionSelectInput(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -414,11 +414,11 @@ func TestActionFilesInput(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -458,6 +458,7 @@ func TestActionWaitLoad(t *testing.T) { instance, err := browser.NewInstance() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -498,6 +499,7 @@ func TestActionGetResource(t *testing.T) { instance, err := browser.NewInstance() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -535,6 +537,7 @@ func TestActionExtract(t *testing.T) { instance, err := browser.NewInstance() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -570,6 +573,7 @@ func TestActionSetMethod(t *testing.T) { instance, err := browser.NewInstance() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -600,11 +604,11 @@ func TestActionAddHeader(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Test") == "Hello" { @@ -633,11 +637,11 @@ func TestActionDeleteHeader(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Test1") == "Hello" && r.Header.Get("Test2") == "" { @@ -668,11 +672,11 @@ func TestActionSetBody(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := ioutil.ReadAll(r.Body) @@ -700,11 +704,11 @@ func TestActionKeyboard(t *testing.T) { browser, err := New(&types.Options{ShowBrowser: false}) require.Nil(t, err, "could not create browser") + defer browser.Close() instance, err := browser.NewInstance() - defer browser.Close() - defer instance.Close() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -749,6 +753,7 @@ func TestActionSleep(t *testing.T) { instance, err := browser.NewInstance() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -788,6 +793,7 @@ func TestActionWaitVisible(t *testing.T) { instance, err := browser.NewInstance() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` @@ -826,6 +832,7 @@ func TestActionWaitVisible(t *testing.T) { instance, err := browser.NewInstance() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` From 80140661540dac4da44b86265903b02db178a489 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 05:03:56 +0000 Subject: [PATCH 091/147] chore(deps): bump golang from 1.17.1-alpine to 1.17.2-alpine Bumps golang from 1.17.1-alpine to 1.17.2-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2ba73cfd4..7d3467d34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.1-alpine as build-env +FROM golang:1.17.2-alpine as build-env RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest FROM alpine:3.14 From 435ec5cd5d24f90dd2608791943eaa651e81f8cf Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Tue, 12 Oct 2021 20:06:55 +0300 Subject: [PATCH 092/147] Do not show AND matcher information in the command line output if debug is not enabled #1081 --- v2/pkg/operators/operators.go | 13 ++- v2/pkg/protocols/common/clusterer/executer.go | 4 +- .../helpers/eventcreator/eventcreator.go | 9 +- .../protocols/common/interactsh/interactsh.go | 2 +- v2/pkg/protocols/dns/operators.go | 67 +++++++-------- v2/pkg/protocols/dns/operators_test.go | 8 +- v2/pkg/protocols/dns/request.go | 4 +- v2/pkg/protocols/file/operators.go | 22 +++-- v2/pkg/protocols/file/operators_test.go | 83 +++++++++++++++---- v2/pkg/protocols/file/request.go | 2 +- v2/pkg/protocols/headless/operators.go | 22 +++-- v2/pkg/protocols/headless/request.go | 2 +- v2/pkg/protocols/http/operators.go | 4 +- v2/pkg/protocols/http/operators_test.go | 2 +- v2/pkg/protocols/http/request.go | 2 +- v2/pkg/protocols/network/operators.go | 24 +++--- v2/pkg/protocols/network/operators_test.go | 2 +- v2/pkg/protocols/network/request.go | 2 +- v2/pkg/protocols/offlinehttp/operators.go | 4 +- .../protocols/offlinehttp/operators_test.go | 2 +- v2/pkg/protocols/offlinehttp/request.go | 2 +- 21 files changed, 165 insertions(+), 117 deletions(-) diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 32f265b8f..af13b4d93 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -108,7 +108,7 @@ type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) (boo type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // Execute executes the operators on data and returns a result structure -func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc) (*Result, bool) { +func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc, isDebug bool) (*Result, bool) { matcherCondition := operators.GetMatchersCondition() var matches bool @@ -140,9 +140,14 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc for matcherIndex, matcher := range operators.Matchers { if isMatch, matched := match(data, matcher); isMatch { - matcherName := getMatcherName(matcher, matcherIndex) - result.Matches[matcherName] = matched - + if isDebug { // matchers without an explicit name or with AND condition should only be made visible if debug is enabled + matcherName := getMatcherName(matcher, matcherIndex) + result.Matches[matcherName] = matched + } else { // if it's a "named" matcher with OR condition, then display it + if matcherCondition == matchers.ORCondition && matcher.Name != "" { + result.Matches[matcher.Name] = matched + } + } matches = true } else if matcherCondition == matchers.ANDCondition { if len(result.DynamicValues) > 0 { diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index ac523c8d6..c060f341d 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -67,7 +67,7 @@ func (e *Executer) Execute(input string) (bool, error) { dynamicValues := make(map[string]interface{}) err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { - result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) + result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) if matched && result != nil { event.OperatorsResult = result event.InternalEvent["template-id"] = operator.templateID @@ -98,7 +98,7 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve dynamicValues := make(map[string]interface{}) err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { - result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) + result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) if matched && result != nil { event.OperatorsResult = result event.InternalEvent["template-id"] = operator.templateID diff --git a/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go index 7f59b0894..c94eb15a3 100644 --- a/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go +++ b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go @@ -6,16 +6,17 @@ import ( ) // CreateEvent wraps the outputEvent with the result of the operators defined on the request -func CreateEvent(request protocols.Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent { - return CreateEventWithAdditionalOptions(request, outputEvent, func(internalWrappedEvent *output.InternalWrappedEvent) {}) +func CreateEvent(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool) *output.InternalWrappedEvent { + return CreateEventWithAdditionalOptions(request, outputEvent, isResponseDebug, func(internalWrappedEvent *output.InternalWrappedEvent) {}) } // CreateEventWithAdditionalOptions wraps the outputEvent with the result of the operators defined on the request and enables extending the resulting event with additional attributes or values. -func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent { +func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool, + addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent { event := &output.InternalWrappedEvent{InternalEvent: outputEvent} for _, compiledOperator := range request.GetCompiledOperators() { if compiledOperator != nil { - result, ok := compiledOperator.Execute(outputEvent, request.Match, request.Extract) + result, ok := compiledOperator.Execute(outputEvent, request.Match, request.Extract, isResponseDebug) if ok && result != nil { event.OperatorsResult = result addAdditionalOptions(event) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 0d11c7af1..5c2914ff5 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -145,7 +145,7 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse - result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc) + result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, false) if !matched || result == nil { return false // if we don't match, return } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 7c9aaa4df..b9f0454b3 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -73,45 +73,20 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto // responseToDSLMap converts a DNS response to a map for use in DSL matching func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent { - data := make(output.InternalEvent, 11) - - // Some data regarding the request metadata - data["host"] = host - data["matched"] = matched - data["request"] = req.String() - - data["rcode"] = resp.Rcode - buffer := &bytes.Buffer{} - for _, question := range resp.Question { - buffer.WriteString(question.String()) + return output.InternalEvent{ + "host": host, + "matched": matched, + "request": req.String(), + "rcode": resp.Rcode, + "question": questionToString(resp.Question), + "extra": rrToString(resp.Extra), + "answer": rrToString(resp.Answer), + "ns": rrToString(resp.Ns), + "raw": resp.String(), + "template-id": request.options.TemplateID, + "template-info": request.options.TemplateInfo, + "template-path": request.options.TemplatePath, } - data["question"] = buffer.String() - buffer.Reset() - - for _, extra := range resp.Extra { - buffer.WriteString(extra.String()) - } - data["extra"] = buffer.String() - buffer.Reset() - - for _, answer := range resp.Answer { - buffer.WriteString(answer.String()) - } - data["answer"] = buffer.String() - buffer.Reset() - - for _, ns := range resp.Ns { - buffer.WriteString(ns.String()) - } - data["ns"] = buffer.String() - buffer.Reset() - - rawData := resp.String() - data["raw"] = rawData - data["template-id"] = request.options.TemplateID - data["template-info"] = request.options.TemplateInfo - data["template-path"] = request.options.TemplatePath - return data } // MakeResultEvent creates a result event from internal wrapped event @@ -134,3 +109,19 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent } return data } + +func rrToString(resourceRecords []dns.RR) string { // TODO rewrite with generics when available + buffer := &bytes.Buffer{} + for _, resourceRecord := range resourceRecords { + buffer.WriteString(resourceRecord.String()) + } + return buffer.String() +} + +func questionToString(resourceRecords []dns.Question) string { + buffer := &bytes.Buffer{} + for _, resourceRecord := range resourceRecords { + buffer.WriteString(resourceRecord.String()) + } + return buffer.String() +} diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index d249f1953..c8d06124a 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -236,13 +236,15 @@ func TestDNSMakeResult(t *testing.T) { event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") finalEvent := &output.InternalWrappedEvent{InternalEvent: event} if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract) + result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) } } require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results") - require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results") - require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") + resultEvent := finalEvent.Results[0] + require.Equal(t, "test", resultEvent.MatcherName, "could not get correct matcher name of results") + require.Equal(t, "1.1.1.1", resultEvent.ExtractedResults[0], "could not get correct extracted results") + require.Equal(t, "one.one.one.one", resultEvent.Matched, "could not get matched value") } diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 28904289b..28a3309b0 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -8,9 +8,9 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" ) var _ protocols.Request = &Request{} @@ -62,7 +62,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review outputEvent[k] = v } - event := eventcreator.CreateEvent(request, outputEvent) + event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) if request.options.Options.Debug || request.options.Options.DebugResponse { gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain) diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 6c7e3b6d2..c0d95c5b0 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -66,18 +66,16 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto return nil } -// responseToDSLMap converts a DNS response to a map for use in DSL matching -func (request *Request) responseToDSLMap(raw, host, matched string) output.InternalEvent { - data := make(output.InternalEvent, 5) - - // Some data regarding the request metadata - data["path"] = host - data["matched"] = matched - data["raw"] = raw - data["template-id"] = request.options.TemplateID - data["template-info"] = request.options.TemplateInfo - data["template-path"] = request.options.TemplatePath - return data +// responseToDSLMap converts a file response to a map for use in DSL matching +func (request *Request) responseToDSLMap(raw, inputFilePath, matchedFileName string) output.InternalEvent { + return output.InternalEvent{ + "path": inputFilePath, + "matched": matchedFileName, + "raw": raw, + "template-id": request.options.TemplateID, + "template-info": request.options.TemplateInfo, + "template-path": request.options.TemplatePath, + } } // MakeResultEvent creates a result event from internal wrapped event diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index 7b985f86b..b03d641b5 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -159,7 +159,62 @@ func TestFileOperatorExtract(t *testing.T) { }) } -func TestFileMakeResult(t *testing.T) { +func TestFileMakeResultWithOrMatcher(t *testing.T) { + expectedValue := []string{"1.1.1.1"} + namedMatcherName := "test" + + finalEvent := testFileMakeResultOperators(t, "or") + require.Equal(t, namedMatcherName, finalEvent.Results[0].MatcherName) + require.Equal(t, expectedValue, finalEvent.OperatorsResult.Matches[namedMatcherName], "could not get matched value") +} + +func TestFileMakeResultWithAndMatcher(t *testing.T) { + finalEvent := testFileMakeResultOperators(t, "and") + require.Equal(t, "", finalEvent.Results[0].MatcherName) + require.Empty(t, finalEvent.OperatorsResult.Matches) +} + +func testFileMakeResultOperators(t *testing.T, matcherCondition string) *output.InternalWrappedEvent { + expectedValue := []string{"1.1.1.1"} + namedMatcherName := "test" + matcher := []*matchers.Matcher{ + { + Part: "raw", + Type: "word", + Words: expectedValue, + }, + { + Name: namedMatcherName, + Part: "raw", + Type: "word", + Words: expectedValue, + }, + } + + expectedValues := map[string][]string{ + "word-1": expectedValue, + namedMatcherName: expectedValue, + } + + finalEvent := testFileMakeResult(t, matcher, matcherCondition, true) + for matcherName, matchedValues := range expectedValues { + var matchesOne = false + for i := 0; i <= len(expectedValue); i++ { + resultEvent := finalEvent.Results[i] + if matcherName == resultEvent.MatcherName { + matchesOne = true + } + } + require.True(t, matchesOne) + require.Equal(t, matchedValues, finalEvent.OperatorsResult.Matches[matcherName], "could not get matched value") + } + + finalEvent = testFileMakeResult(t, matcher, matcherCondition, false) + require.Equal(t, 1, len(finalEvent.Results)) + return finalEvent +} + +func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondition string, isDebug bool) *output.InternalWrappedEvent { options := testutils.DefaultOptions testutils.Init(options) @@ -171,12 +226,8 @@ func TestFileMakeResult(t *testing.T) { Extensions: []string{"*", ".lock"}, ExtensionDenylist: []string{".go"}, Operators: operators.Operators{ - Matchers: []*matchers.Matcher{{ - Name: "test", - Part: "raw", - Type: "word", - Words: []string{"1.1.1.1"}, - }}, + MatchersCondition: matcherCondition, + Matchers: matchers, Extractors: []*extractors.Extractor{{ Part: "raw", Type: "regex", @@ -191,20 +242,24 @@ func TestFileMakeResult(t *testing.T) { err := request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") - resp := "test-data\r\n1.1.1.1\r\n" - event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") + matchedFileName := "test.txt" + fileContent := "test-data\r\n1.1.1.1\r\n" + + event := request.responseToDSLMap(fileContent, "/tmp", matchedFileName) require.Len(t, event, 6, "could not get correct number of items in dsl map") - require.Equal(t, resp, event["raw"], "could not get correct resp") + require.Equal(t, fileContent, event["raw"], "could not get correct resp") finalEvent := &output.InternalWrappedEvent{InternalEvent: event} if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract) + result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, isDebug) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) } } - require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results") - require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results") - require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") + resultEvent := finalEvent.Results[0] + require.Equal(t, "1.1.1.1", resultEvent.ExtractedResults[0], "could not get correct extracted results") + require.Equal(t, matchedFileName, resultEvent.Matched, "could not get matched value") + + return finalEvent } diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 4b2c03d6c..61bb87185 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -57,7 +57,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review outputEvent[k] = v } - event := eventcreator.CreateEvent(request, outputEvent) + event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) if request.options.Options.Debug || request.options.Options.DebugResponse { gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath) diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index 0d76d421b..81a81a6ab 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -64,19 +64,17 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto return nil } -// responseToDSLMap converts a DNS response to a map for use in DSL matching +// responseToDSLMap converts a headless response to a map for use in DSL matching func (request *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent { - data := make(output.InternalEvent, 5) - - // Some data regarding the request metadata - data["host"] = host - data["matched"] = matched - data["req"] = req - data["data"] = resp - data["template-id"] = request.options.TemplateID - data["template-info"] = request.options.TemplateInfo - data["template-path"] = request.options.TemplatePath - return data + return output.InternalEvent{ + "host": host, + "matched": matched, + "req": req, + "data": resp, + "template-id": request.options.TemplateID, + "template-info": request.options.TemplateInfo, + "template-path": request.options.TemplatePath, + } } // MakeResultEvent creates a result event from internal wrapped event diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index c675165d6..55394ed7f 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -65,7 +65,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp outputEvent[k] = v } - event := eventcreator.CreateEvent(request, outputEvent) + event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) if request.options.Options.Debug || request.options.Options.DebugResponse { gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input) diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 6a8bcfad1..c91ee0ec1 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -97,8 +97,8 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) { } // responseToDSLMap converts an HTTP response to a map for use in DSL matching -func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { - data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies())) +func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) output.InternalEvent { + data := make(output.InternalEvent, 12+len(extra)+len(resp.Header)+len(resp.Cookies())) for k, v := range extra { data[k] = v } diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 509312ea1..c9d3b795d 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -262,7 +262,7 @@ func TestHTTPMakeResult(t *testing.T) { event["ip"] = "192.169.1.1" finalEvent := &output.InternalWrappedEvent{InternalEvent: event} if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract) + result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 73245bbfb..979e0eae2 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -477,7 +477,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } } - event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, func(internalWrappedEvent *output.InternalWrappedEvent) { + event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index a1e459173..8e0330dc0 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -64,20 +64,18 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto return nil } -// responseToDSLMap converts a DNS response to a map for use in DSL matching +// responseToDSLMap converts a network response to a map for use in DSL matching func (request *Request) responseToDSLMap(req, resp, raw, host, matched string) output.InternalEvent { - data := make(output.InternalEvent, 6) - - // Some data regarding the request metadata - data["host"] = host - data["matched"] = matched - data["request"] = req - data["data"] = resp // Data is the last bytes read - data["raw"] = raw // Raw is the full transaction data for network - data["template-id"] = request.options.TemplateID - data["template-info"] = request.options.TemplateInfo - data["template-path"] = request.options.TemplatePath - return data + return output.InternalEvent{ + "host": host, + "matched": matched, + "request": req, + "data": resp, // Data is the last bytes read + "raw": raw, // Raw is the full transaction data for network + "template-id": request.options.TemplateID, + "template-info": request.options.TemplateInfo, + "template-path": request.options.TemplatePath, + } } // MakeResultEvent creates a result event from internal wrapped event diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index 474ac0dec..bf577d7e7 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -192,7 +192,7 @@ func TestNetworkMakeResult(t *testing.T) { finalEvent := &output.InternalWrappedEvent{InternalEvent: event} event["ip"] = "192.168.1.1" if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract) + result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 0b07a31f3..ba996da8c 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -209,7 +209,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input var event *output.InternalWrappedEvent if interactURL == "" { - event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, func(wrappedEvent *output.InternalWrappedEvent) { + event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { wrappedEvent.OperatorsResult.PayloadValues = payloads }) callback(event) diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index 730aa357d..59d23470d 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -93,8 +93,8 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) { } // responseToDSLMap converts an HTTP response to a map for use in DSL matching -func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { - data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies())) +func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) output.InternalEvent { + data := make(output.InternalEvent, 12+len(extra)+len(resp.Header)+len(resp.Cookies())) for k, v := range extra { data[k] = v } diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index eba24203e..2f4b91fd5 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -204,7 +204,7 @@ func TestHTTPMakeResult(t *testing.T) { event["ip"] = "192.169.1.1" finalEvent := &output.InternalWrappedEvent{InternalEvent: event} for _, operator := range request.compiledOperators { - result, ok := operator.Execute(event, request.Match, request.Extract) + result, ok := operator.Execute(event, request.Match, request.Extract, false) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 438ed0324..1eb4bfc57 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -85,7 +85,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review outputEvent[k] = v } - event := eventcreator.CreateEvent(request, outputEvent) + event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) callback(event) }(data) }) From d88ed67d2c1e040d55bac0235a070c96405de2ff Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Tue, 12 Oct 2021 20:18:53 +0300 Subject: [PATCH 093/147] Do not show AND matcher information in the command line output if debug is not enabled #1081 * integration test fix --- v2/cmd/integration-test/network.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index 04cc70136..8c738537d 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -82,7 +82,14 @@ func (h *networkMultiStep) Execute(filePath string) error { if routerErr != nil { return routerErr } - if len(results) != 3 { + + var expectedResultsSize int + if debug { + expectedResultsSize = 3 + } else { + expectedResultsSize = 1 + } + if len(results) != expectedResultsSize { return errIncorrectResultsCount(results) } return nil From a9cb83ff2786ac241b64cb9189d4d2ce34a2afd2 Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 13 Oct 2021 13:19:00 +0530 Subject: [PATCH 094/147] misc sniper cleanups --- SYNTAX-REFERENCE.md | 8 ++++---- nuclei-jsonschema.json | 4 ++-- v2/pkg/protocols/common/generators/generators.go | 6 +++--- v2/pkg/protocols/http/http.go | 6 +++--- v2/pkg/protocols/network/network.go | 6 +++--- v2/pkg/templates/templates_doc.go | 8 ++++---- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 691c25af7..825e9fdc2 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -823,14 +823,14 @@ in a combined manner allowing multirequest based matchers. Attack is the type of payload combinations to perform. -Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates +Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates permutations and combinations for all payloads. Valid values: - - sniper + - batteringram - pitchfork @@ -2312,14 +2312,14 @@ host: Attack is the type of payload combinations to perform. -Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates +Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates permutations and combinations for all payloads. Valid values: - - sniper + - batteringram - pitchfork diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 1578630eb..14c578e08 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -606,7 +606,7 @@ }, "attack": { "enum": [ - "sniper", + "batteringram", "pitchfork", "clusterbomb" ], @@ -777,7 +777,7 @@ }, "attack": { "enum": [ - "sniper", + "batteringram", "pitchfork", "clusterbomb" ], diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 07c72d200..a63fbcb2b 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -14,7 +14,7 @@ type Generator struct { type Type int const ( - // Sniper replaces one iteration of the payload with a value. + // Batteringram replaces same payload into all of the defined payload positions at once. BatteringRam Type = iota + 1 // PitchFork replaces variables with positional value from multiple wordlists PitchFork @@ -43,10 +43,10 @@ func New(payloads map[string]interface{}, payloadType Type, templatePath string) generator.Type = payloadType generator.payloads = compiled - // Validate the sniper/batteringram payload set + // Validate the batteringram payload set if payloadType == BatteringRam { if len(payloads) != 1 { - return nil, errors.New("sniper/batteringram must have single payload set") + return nil, errors.New("batteringram must have single payload set") } } return generator, nil diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 69738e367..f89e41f7b 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -46,13 +46,13 @@ type Request struct { // description: | // Attack is the type of payload combinations to perform. // - // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates + // batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. // values: - // - "sniper" + // - "batteringram" // - "pitchfork" // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` + AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Method is the HTTP Request Method. // values: diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index b7de7b920..45b7b67bd 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -35,13 +35,13 @@ type Request struct { // description: | // Attack is the type of payload combinations to perform. // - // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates + // Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. // values: - // - "sniper" + // - "batteringram" // - "pitchfork" // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` + AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Payloads contains any payloads for the current request. // diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 30ac25af7..0a634b0b8 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -317,10 +317,10 @@ func init() { HTTPRequestDoc.Fields[7].Name = "attack" HTTPRequestDoc.Fields[7].Type = "string" HTTPRequestDoc.Fields[7].Note = "" - HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nBatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." HTTPRequestDoc.Fields[7].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." HTTPRequestDoc.Fields[7].Values = []string{ - "sniper", + "batteringram", "pitchfork", "clusterbomb", } @@ -854,10 +854,10 @@ func init() { NETWORKRequestDoc.Fields[2].Name = "attack" NETWORKRequestDoc.Fields[2].Type = "string" NETWORKRequestDoc.Fields[2].Note = "" - NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nBatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." NETWORKRequestDoc.Fields[2].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." NETWORKRequestDoc.Fields[2].Values = []string{ - "sniper", + "batteringram", "pitchfork", "clusterbomb", } From fe03cd8e597a9f3434f0192c3b25f4eedbeb33fb Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 13 Oct 2021 07:59:09 +0000 Subject: [PATCH 095/147] Auto Generate Syntax Docs + JSONSchema [Wed Oct 13 07:59:09 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 2 +- v2/pkg/templates/templates_doc.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 825e9fdc2..19f59a3ae 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -823,7 +823,7 @@ in a combined manner allowing multirequest based matchers. Attack is the type of payload combinations to perform. -Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates +batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates permutations and combinations for all payloads. diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 0a634b0b8..27436ea90 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -317,7 +317,7 @@ func init() { HTTPRequestDoc.Fields[7].Name = "attack" HTTPRequestDoc.Fields[7].Type = "string" HTTPRequestDoc.Fields[7].Note = "" - HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nBatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nbatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." HTTPRequestDoc.Fields[7].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." HTTPRequestDoc.Fields[7].Values = []string{ "batteringram", From c0ef41904897fa7ee06831040955931c12ad4a28 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 13 Oct 2021 20:08:10 +0300 Subject: [PATCH 096/147] Add/Expose Headless action that waits until an element appears #1096 * add timeout and pollTime options --- .../protocols/headless/engine/page_actions.go | 77 ++++++++++++++-- .../headless/engine/page_actions_test.go | 89 ++++++++----------- 2 files changed, 108 insertions(+), 58 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index ad4eec626..fc6e821ea 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -1,6 +1,7 @@ package engine import ( + "context" "io/ioutil" "net" "net/url" @@ -11,6 +12,7 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/proto" + "github.com/go-rod/rod/lib/utils" "github.com/pkg/errors" "github.com/segmentio/ksuid" "github.com/valyala/fasttemplate" @@ -46,7 +48,7 @@ func (p *Page) ExecuteActions(baseURL *url.URL, actions []*Action) (map[string]s case ActionGetResource: err = p.GetResource(act, outData) case ActionExtract: - err = p.SelectInputElement(act, outData) + err = p.ExtractElement(act, outData) case ActionWaitEvent: err = p.WaitEvent(act, outData) case ActionFilesInput: @@ -85,18 +87,83 @@ type requestRule struct { Args map[string]string } +const elementDidNotAppearMessage = "Element did not appear in the given amount of time" + // WaitVisible waits until an element appears. func (p *Page) WaitVisible(act *Action, out map[string]string) error { - element, err := p.pageElementBy(act.Data) + timeout, err := getTimeout(act) if err != nil { - return errors.Wrap(err, "could not find element") + return errors.Wrap(err, "Wrong timeout given") } - if err = element.WaitVisible(); err != nil { - return errors.Wrap(err, "could not wait element") + + pollTime, err := getPollTime(act) + if err != nil { + return errors.Wrap(err, "Wrong polling time given") } + + element, _ := p.Sleeper(pollTime, timeout). + Timeout(timeout). + pageElementBy(act.Data) + + if element != nil { + if err := element.WaitVisible(); err != nil { + return errors.Wrap(err, elementDidNotAppearMessage) + } + } else { + return errors.New(elementDidNotAppearMessage) + } + return nil } +func (p *Page) Sleeper(pollTimeout, timeout time.Duration) *Page { + page := *p + page.page = page.Page().Sleeper(func() utils.Sleeper { + return createBackOffSleeper(pollTimeout, timeout) + }) + return &page +} + +func (p *Page) Timeout(timeout time.Duration) *Page { + page := *p + page.page = page.Page().Timeout(timeout) + return &page +} + +func createBackOffSleeper(pollTimeout, timeout time.Duration) utils.Sleeper { + backoffSleeper := utils.BackoffSleeper(pollTimeout, timeout, func(duration time.Duration) time.Duration { + return duration + }) + + return func(ctx context.Context) error { + if ctx.Err() != nil { + return ctx.Err() + } + + return backoffSleeper(ctx) + } +} + +func getTimeout(act *Action) (time.Duration, error) { + return geTimeParameter(act, "timeout", 3, time.Second) +} + +func getPollTime(act *Action) (time.Duration, error) { + return geTimeParameter(act, "pollTime", 100, time.Millisecond) +} + +func geTimeParameter(act *Action, parameterName string, defaultValue time.Duration, duration time.Duration) (time.Duration, error) { + pollTimeString := act.GetArg(parameterName) + if pollTimeString == "" { + return defaultValue * duration, nil + } + timeout, err := strconv.Atoi(pollTimeString) + if err != nil { + return time.Duration(0), err + } + return time.Duration(timeout) * duration, nil +} + // ActionAddHeader executes a AddHeader action. func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error { in := act.GetArg("part") diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 5c23fdee4..d5cb56350 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -9,9 +9,10 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/stretchr/testify/require" ) func TestActionNavigate(t *testing.T) { @@ -281,17 +282,24 @@ func TestActionHeadersChange(t *testing.T) { func TestActionWaitVisible(t *testing.T) { t.Run("wait for an element being visible", func(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + testWaitVisible(t, 2*time.Second, func(page *Page, err error) { + require.Nil(t, err, "could not run page actions") - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() + page.Page().MustElement("button").MustVisible() + page.Close() + }) + }) - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") + t.Run("timeout because of element not visible", func(t *testing.T) { + testWaitVisible(t, time.Second/2, func(page *Page, err error) { + require.Error(t, err) + require.Contains(t, err.Error(), "Element did not appear in the given amount of time") + }) + }) +} - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` +func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page, err error)) { + response := ` Nuclei Test Page @@ -300,54 +308,29 @@ func TestActionWaitVisible(t *testing.T) { - `) - })) - defer ts.Close() + ` - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + _ = protocolstate.Init(&types.Options{}) - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, - } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() - page.Page().MustElement("button").MustVisible() - }) + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") - t.Run("timeout because of element not visible", func(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, response) + })) + defer ts.Close() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, - } - _, _, err = instance.Run(parsed, actions, 2*time.Second) - require.Error(t, err) - require.Contains(t, err.Error(), "could not wait element") - }) + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + } + _, page, err := instance.Run(parsed, actions, timeout) + assert(page, err) } From 8ee4ed2c1770cba8067c86694b925d60eb741200 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Wed, 13 Oct 2021 20:29:28 +0200 Subject: [PATCH 097/147] chore: fix build --- .../protocols/headless/engine/page_actions_test.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 86ac93c4f..dcace31d5 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -719,10 +719,7 @@ func TestActionKeyboard(t *testing.T) { Nuclei Test Page - + `) })) @@ -734,15 +731,14 @@ func TestActionKeyboard(t *testing.T) { actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, - {ActionType: "click", Data: map[string]string{"selector": "select"}}, - {ActionType: "keyboard", Data: map[string]string{"keys": "Test 2"}}, - {ActionType: "keyboard", Data: map[string]string{"keys": "enter"}}, + {ActionType: "click", Data: map[string]string{"selector": "input"}}, + {ActionType: "keyboard", Data: map[string]string{"keys": "Test2"}}, } _, page, err := instance.Run(parsed, actions, 20*time.Second) require.Nil(t, err, "could not run page actions") defer page.Close() - el := page.Page().MustElement("select") + el := page.Page().MustElement("input") require.Equal(t, "Test2", el.MustText(), "could not get input change value") } From 53281660bebd681406e094048d5a8681780c96ef Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Wed, 13 Oct 2021 20:45:04 +0200 Subject: [PATCH 098/147] chore: refactor tests --- .../headless/engine/page_actions_test.go | 100 ++++++++---------- 1 file changed, 44 insertions(+), 56 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index dcace31d5..ae5d16e49 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -53,6 +53,42 @@ func TestActionNavigate(t *testing.T) { } func TestActionScript(t *testing.T) { + t.Run("run-and-results", func(t *testing.T) { + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, + } + testActionScript(t, actions, func(page *Page, out map[string]string, err error) { + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + require.Equal(t, "some-data", out["test"], "could not run js and get results correctly") + }) + }) + + t.Run("hook", func(t *testing.T) { + actions := []*Action{ + {ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, + } + testActionScript(t, actions, func(page *Page, out map[string]string, err error) { + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook") + }) + }) +} + +func testActionScript(t *testing.T, actions []*Action, assert func(page *Page, out map[string]string, err error)) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + ` + _ = protocolstate.Init(&types.Options{}) browser, err := New(&types.Options{ShowBrowser: false}) @@ -61,65 +97,17 @@ func TestActionScript(t *testing.T) { instance, err := browser.NewInstance() require.Nil(t, err, "could not create browser instance") - defer instance.Close() - t.Run("run-and-results", func(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, response) + })) + defer ts.Close() - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, - } - out, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - require.Equal(t, "some-data", out["test"], "could not run js and get results correctly") - }) - - t.Run("hook", func(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - Nuclei Test Page - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - - actions := []*Action{ - {ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, - } - out, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook") - }) + out, page, err := instance.Run(parsed, actions, 2*time.Second) + assert(page, out, err) } func TestActionClick(t *testing.T) { From 88dc16c910cc55a138fcaba153a44f285ed4f684 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 14 Oct 2021 00:17:24 +0200 Subject: [PATCH 099/147] Improving gbk detection coverage --- v2/go.mod | 6 +++++- v2/go.sum | 11 +++++++++++ v2/pkg/protocols/http/request.go | 3 ++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 471e12c5c..c4a34a797 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -40,7 +40,7 @@ require ( github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a github.com/projectdiscovery/retryablehttp-go v1.0.2 - github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 + github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d github.com/projectdiscovery/yamldoc-go v1.0.2 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 @@ -73,6 +73,7 @@ require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.1.6 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -86,6 +87,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/css v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect @@ -95,6 +97,7 @@ require ( github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/mattn/go-isatty v0.0.13 // indirect + github.com/microcosm-cc/bluemonday v1.0.15 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -104,6 +107,7 @@ require ( github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect + github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/tklauser/go-sysconf v0.3.7 // indirect github.com/tklauser/numcpus v0.2.3 // indirect github.com/trivago/tgo v1.0.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index e85a7fce4..557a11f89 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -96,11 +96,14 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -320,6 +323,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -468,6 +473,8 @@ github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go. github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY= +github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -609,6 +616,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d h1:YBYwsm8MrSp9t7mLehyqGwUKZWB08fG+YRePQRo5iFw= +github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d/go.mod h1:JK4F9ACNPgO+Lbm80khX2q1ABInBMbwIOmbsEE61Sn4= github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -647,6 +656,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 979e0eae2..bd209d7f2 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -429,7 +429,8 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate redirectedResponse = bytes.ReplaceAll(redirectedResponse, dataOrig, data) // Decode gbk response content-types - if contentType := strings.ToLower(resp.Header.Get("Content-Type")); contentType != "" && (strings.Contains(contentType, "gbk") || strings.Contains(contentType, "gb2312")) { + // gb18030 supersedes gb2312 + if contentType := strings.ToLower(resp.Header.Get("Content-Type")); stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030") { dumpedResponse, err = decodegbk(dumpedResponse) if err != nil { return errors.Wrap(err, "could not gbk decode") From 4d14063cbed11948609236257b307f9df672ed90 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 14 Oct 2021 17:35:32 +0300 Subject: [PATCH 100/147] Add missing headless actions tests #1102 * test refactor --- .../headless/engine/page_actions_test.go | 817 ++++++------------ 1 file changed, 286 insertions(+), 531 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index ae5d16e49..b756507cc 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -18,18 +18,8 @@ import ( ) func TestActionNavigate(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + t.Parallel() + response := ` Nuclei Test Page @@ -37,49 +27,19 @@ func TestActionNavigate(t *testing.T) {

Nuclei Test

- `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}} - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + }) } func TestActionScript(t *testing.T) { - t.Run("run-and-results", func(t *testing.T) { - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, - } - testActionScript(t, actions, func(page *Page, out map[string]string, err error) { - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - require.Equal(t, "some-data", out["test"], "could not run js and get results correctly") - }) - }) + t.Parallel() - t.Run("hook", func(t *testing.T) { - actions := []*Action{ - {ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, - } - testActionScript(t, actions, func(page *Page, out map[string]string, err error) { - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook") - }) - }) -} - -func testActionScript(t *testing.T, actions []*Action, assert func(page *Page, out map[string]string, err error)) { response := ` @@ -89,285 +49,205 @@ func testActionScript(t *testing.T, actions []*Action, assert func(page *Page, o ` - _ = protocolstate.Init(&types.Options{}) + timeout := 2 * time.Second - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() + t.Run("run-and-results", func(t *testing.T) { + t.Parallel() + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, + } + testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + require.Equal(t, "some-data", out["test"], "could not run js and get results correctly") + }) + }) - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, response) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - - out, page, err := instance.Run(parsed, actions, 2*time.Second) - assert(page, out, err) + t.Run("hook", func(t *testing.T) { + t.Parallel() + actions := []*Action{ + {ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, + } + testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook") + }) + }) } func TestActionClick(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Parallel() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "click", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("button") - val := el.MustAttribute("a") - require.Equal(t, "ok", *val, "could not click button") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("button") + val := el.MustAttribute("a") + require.Equal(t, "ok", *val, "could not click button") + }) } func TestActionRightClick(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Parallel() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + + + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "rightclick", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("button") - val := el.MustAttribute("a") - require.Equal(t, "ok", *val, "could not click button") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("button") + val := el.MustAttribute("a") + require.Equal(t, "ok", *val, "could not click button") + }) } func TestActionTextInput(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Parallel() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "text", Data: map[string]string{"selector": "input", "value": "test"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("input") - val := el.MustAttribute("event") - require.Equal(t, "input-change", *val, "could not get input change") - require.Equal(t, "test", el.MustText(), "could not get input change value") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + val := el.MustAttribute("event") + require.Equal(t, "input-change", *val, "could not get input change") + require.Equal(t, "test", el.MustText(), "could not get input change value") + }) } func TestActionHeadersChange(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Test") == "Hello" { - fmt.Fprintln(w, `found`) - } - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + t.Parallel() actions := []*Action{ {ActionType: "setheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + handler := func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test") == "Hello" { + _, _ = fmt.Fprintln(w, `found`) + } + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + }) } func TestActionScreenshot(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Parallel() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "screenshot", Data: map[string]string{"to": "test"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page() - require.FileExists(t, "test.png", el, "could not get screenshot file") - os.Remove("test.png") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page() + require.FileExists(t, "test.png", el, "could not get screenshot file") + _ = os.Remove("test.png") + }) } func TestActionTimeInput(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Parallel() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "time", Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("input") - require.Equal(t, "2006-01-02", el.MustText(), "could not get input time value") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + require.Equal(t, "2006-01-02", el.MustText(), "could not get input time value") + }) } func TestActionSelectInput(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Parallel() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page @@ -378,79 +258,51 @@ func TestActionSelectInput(t *testing.T) { - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "select", Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - el := page.Page().MustElement("select") - require.Equal(t, "Test2", el.MustText(), "could not get input change value") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + el := page.Page().MustElement("select") + require.Equal(t, "Test2", el.MustText(), "could not get input change value") + }) } func TestActionFilesInput(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Parallel() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "files", Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("input") - require.Equal(t, "C:\\fakepath\\test1.pdf", el.MustText(), "could not get input file") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + require.Equal(t, "C:\\fakepath\\test1.pdf", el.MustText(), "could not get input file") + }) } func TestActionWaitLoad(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Parallel() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page @@ -459,40 +311,26 @@ func TestActionWaitLoad(t *testing.T) { - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - el := page.Page().MustElement("button") - style, err := el.Attribute("style") - require.Nil(t, err) - require.Equal(t, "color: red;", *style, "could not get color") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + el := page.Page().MustElement("button") + style, attributeErr := el.Attribute("style") + require.Nil(t, attributeErr) + require.Equal(t, "color: red;", *style, "could not get color") + }) } func TestActionGetResource(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Parallel() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page @@ -500,148 +338,85 @@ func TestActionGetResource(t *testing.T) { - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "getresource", Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, } - out, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, len(out["src"]), 3159, "could not find resource") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, len(out["src"]), 3159, "could not find resource") + }) } func TestActionExtract(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Parallel() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "extract", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, } - out, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Wait for me!", out["extract"], "could not extract text") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Wait for me!", out["extract"], "could not extract text") + }) } func TestActionSetMethod(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Parallel() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "setmethod", Data: map[string]string{"part": "x", "method": "SET"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "SET", page.rules[0].Args["method"], "could not find resource") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "SET", page.rules[0].Args["method"], "could not find resource") + }) } func TestActionAddHeader(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Test") == "Hello" { - fmt.Fprintln(w, `found`) - } - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + t.Parallel() actions := []*Action{ {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + handler := func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test") == "Hello" { + _, _ = fmt.Fprintln(w, `found`) + } + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + }) } func TestActionDeleteHeader(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Test1") == "Hello" && r.Header.Get("Test2") == "" { - fmt.Fprintln(w, `header deleted`) - } - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + t.Parallel() actions := []*Action{ {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, @@ -650,58 +425,43 @@ func TestActionDeleteHeader(t *testing.T) { {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "header deleted", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not delete header correctly") + handler := func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test1") == "Hello" && r.Header.Get("Test2") == "" { + _, _ = fmt.Fprintln(w, `header deleted`) + } + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "header deleted", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not delete header correctly") + }) } func TestActionSetBody(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := ioutil.ReadAll(r.Body) - fmt.Fprintln(w, string(body)) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + t.Parallel() actions := []*Action{ {ActionType: "setbody", Data: map[string]string{"part": "request", "body": "hello"}}, {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "hello", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + handler := func(w http.ResponseWriter, r *http.Request) { + body, _ := ioutil.ReadAll(r.Body) + _, _ = fmt.Fprintln(w, string(body)) + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "hello", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + }) } func TestActionKeyboard(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Parallel() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page @@ -709,12 +469,7 @@ func TestActionKeyboard(t *testing.T) { - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, @@ -722,72 +477,17 @@ func TestActionKeyboard(t *testing.T) { {ActionType: "click", Data: map[string]string{"selector": "input"}}, {ActionType: "keyboard", Data: map[string]string{"keys": "Test2"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - el := page.Page().MustElement("input") - require.Equal(t, "Test2", el.MustText(), "could not get input change value") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + el := page.Page().MustElement("input") + require.Equal(t, "Test2", el.MustText(), "could not get input change value") + }) } func TestActionSleep(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) + t.Parallel() - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "sleep", Data: map[string]string{"duration": "2"}}, - } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - defer page.Close() - - require.Nil(t, err, "could not run page actions") - require.True(t, page.Page().MustElement("button").MustVisible(), "could not get button") -} - -func TestActionWaitVisible(t *testing.T) { - t.Run("wait for an element being visible", func(t *testing.T) { - testWaitVisible(t, 2*time.Second, func(page *Page, err error) { - require.Nil(t, err, "could not run page actions") - - page.Page().MustElement("button").MustVisible() - page.Close() - }) - }) - - t.Run("timeout because of element not visible", func(t *testing.T) { - testWaitVisible(t, time.Second/2, func(page *Page, err error) { - require.Error(t, err) - require.Contains(t, err.Error(), "Element did not appear in the given amount of time") - }) - }) -} - -func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page, err error)) { response := ` @@ -799,6 +499,63 @@ func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page ` + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "sleep", Data: map[string]string{"duration": "2"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.True(t, page.Page().MustElement("button").MustVisible(), "could not get button") + }) +} + +func TestActionWaitVisible(t *testing.T) { + t.Parallel() + + response := ` + + + Nuclei Test Page + + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + } + + t.Run("wait for an element being visible", func(t *testing.T) { + t.Parallel() + testHeadlessSimpleResponse(t, response, actions, 2*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + + page.Page().MustElement("button").MustVisible() + }) + }) + + t.Run("timeout because of element not visible", func(t *testing.T) { + t.Parallel() + testHeadlessSimpleResponse(t, response, actions, time.Second/2, func(page *Page, err error, out map[string]string) { + require.Error(t, err) + require.Contains(t, err.Error(), "Element did not appear in the given amount of time") + }) + }) +} + +func testHeadlessSimpleResponse(t *testing.T, response string, actions []*Action, timeout time.Duration, assert func(page *Page, pageErr error, out map[string]string)) { + t.Helper() + testHeadless(t, actions, timeout, func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintln(w, response) + }, assert) +} + +func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handler func(w http.ResponseWriter, r *http.Request), assert func(page *Page, pageErr error, extractedData map[string]string)) { + t.Helper() _ = protocolstate.Init(&types.Options{}) browser, err := New(&types.Options{ShowBrowser: false}) @@ -807,19 +564,17 @@ func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page instance, err := browser.NewInstance() require.Nil(t, err, "could not create browser instance") + defer instance.Close() - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, response) - })) + ts := httptest.NewServer(http.HandlerFunc(handler)) defer ts.Close() parsed, err := url.Parse(ts.URL) require.Nil(t, err, "could not parse URL") + extractedData, page, err := instance.Run(parsed, actions, timeout) + assert(page, err, extractedData) - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + if page != nil { + page.Close() } - _, page, err := instance.Run(parsed, actions, timeout) - assert(page, err) } From 7d6386449d300015c0903ef4949c72745c4682a5 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 14 Oct 2021 17:38:56 +0300 Subject: [PATCH 101/147] Revert "Add missing headless actions tests #1102" This reverts commit 4d14063cbed11948609236257b307f9df672ed90. --- .../headless/engine/page_actions_test.go | 1333 ++++++++++------- 1 file changed, 789 insertions(+), 544 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index b756507cc..ae5d16e49 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -18,544 +18,6 @@ import ( ) func TestActionNavigate(t *testing.T) { - t.Parallel() - response := ` - - - Nuclei Test Page - - -

Nuclei Test

- - ` - - actions := []*Action{{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}} - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - }) -} - -func TestActionScript(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - Nuclei Test Page - - ` - - timeout := 2 * time.Second - - t.Run("run-and-results", func(t *testing.T) { - t.Parallel() - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, - } - testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - require.Equal(t, "some-data", out["test"], "could not run js and get results correctly") - }) - }) - - t.Run("hook", func(t *testing.T) { - t.Parallel() - actions := []*Action{ - {ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, - } - testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook") - }) - }) -} - -func TestActionClick(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - Nuclei Test Page - - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "click", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking - } - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("button") - val := el.MustAttribute("a") - require.Equal(t, "ok", *val, "could not click button") - }) -} - -func TestActionRightClick(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - Nuclei Test Page - - - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "rightclick", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking - } - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("button") - val := el.MustAttribute("a") - require.Equal(t, "ok", *val, "could not click button") - }) -} - -func TestActionTextInput(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - Nuclei Test Page - - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "text", Data: map[string]string{"selector": "input", "value": "test"}}, - } - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("input") - val := el.MustAttribute("event") - require.Equal(t, "input-change", *val, "could not get input change") - require.Equal(t, "test", el.MustText(), "could not get input change value") - }) -} - -func TestActionHeadersChange(t *testing.T) { - t.Parallel() - - actions := []*Action{ - {ActionType: "setheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - } - - handler := func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Test") == "Hello" { - _, _ = fmt.Fprintln(w, `found`) - } - } - - testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") - }) -} - -func TestActionScreenshot(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - Nuclei Test Page - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "screenshot", Data: map[string]string{"to": "test"}}, - } - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page() - require.FileExists(t, "test.png", el, "could not get screenshot file") - _ = os.Remove("test.png") - }) -} - -func TestActionTimeInput(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - Nuclei Test Page - - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "time", Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, - } - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("input") - require.Equal(t, "2006-01-02", el.MustText(), "could not get input time value") - }) -} - -func TestActionSelectInput(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - - - - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "select", Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, - } - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - el := page.Page().MustElement("select") - require.Equal(t, "Test2", el.MustText(), "could not get input change value") - }) -} - -func TestActionFilesInput(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - Nuclei Test Page - - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "files", Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, - } - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("input") - require.Equal(t, "C:\\fakepath\\test1.pdf", el.MustText(), "could not get input file") - }) -} - -func TestActionWaitLoad(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - - - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - } - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - el := page.Page().MustElement("button") - style, attributeErr := el.Attribute("style") - require.Nil(t, attributeErr) - require.Equal(t, "color: red;", *style, "could not get color") - }) -} - -func TestActionGetResource(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - - - - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "getresource", Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, - } - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, len(out["src"]), 3159, "could not find resource") - }) -} - -func TestActionExtract(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "extract", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, - } - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "Wait for me!", out["extract"], "could not extract text") - }) -} - -func TestActionSetMethod(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "setmethod", Data: map[string]string{"part": "x", "method": "SET"}}, - } - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "SET", page.rules[0].Args["method"], "could not find resource") - }) -} - -func TestActionAddHeader(t *testing.T) { - t.Parallel() - - actions := []*Action{ - {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - } - - handler := func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Test") == "Hello" { - _, _ = fmt.Fprintln(w, `found`) - } - } - - testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") - }) -} - -func TestActionDeleteHeader(t *testing.T) { - t.Parallel() - - actions := []*Action{ - {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, - {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}}, - {ActionType: "deleteheader", Data: map[string]string{"part": "request", "key": "Test2"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - } - - handler := func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Test1") == "Hello" && r.Header.Get("Test2") == "" { - _, _ = fmt.Fprintln(w, `header deleted`) - } - } - - testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "header deleted", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not delete header correctly") - }) -} - -func TestActionSetBody(t *testing.T) { - t.Parallel() - - actions := []*Action{ - {ActionType: "setbody", Data: map[string]string{"part": "request", "body": "hello"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - } - - handler := func(w http.ResponseWriter, r *http.Request) { - body, _ := ioutil.ReadAll(r.Body) - _, _ = fmt.Fprintln(w, string(body)) - } - - testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.Equal(t, "hello", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") - }) -} - -func TestActionKeyboard(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - - - - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "click", Data: map[string]string{"selector": "input"}}, - {ActionType: "keyboard", Data: map[string]string{"keys": "Test2"}}, - } - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - el := page.Page().MustElement("input") - require.Equal(t, "Test2", el.MustText(), "could not get input change value") - }) -} - -func TestActionSleep(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - - - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "sleep", Data: map[string]string{"duration": "2"}}, - } - - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - require.True(t, page.Page().MustElement("button").MustVisible(), "could not get button") - }) -} - -func TestActionWaitVisible(t *testing.T) { - t.Parallel() - - response := ` - - - Nuclei Test Page - - - - ` - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, - } - - t.Run("wait for an element being visible", func(t *testing.T) { - t.Parallel() - testHeadlessSimpleResponse(t, response, actions, 2*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") - - page.Page().MustElement("button").MustVisible() - }) - }) - - t.Run("timeout because of element not visible", func(t *testing.T) { - t.Parallel() - testHeadlessSimpleResponse(t, response, actions, time.Second/2, func(page *Page, err error, out map[string]string) { - require.Error(t, err) - require.Contains(t, err.Error(), "Element did not appear in the given amount of time") - }) - }) -} - -func testHeadlessSimpleResponse(t *testing.T, response string, actions []*Action, timeout time.Duration, assert func(page *Page, pageErr error, out map[string]string)) { - t.Helper() - testHeadless(t, actions, timeout, func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprintln(w, response) - }, assert) -} - -func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handler func(w http.ResponseWriter, r *http.Request), assert func(page *Page, pageErr error, extractedData map[string]string)) { - t.Helper() _ = protocolstate.Init(&types.Options{}) browser, err := New(&types.Options{ShowBrowser: false}) @@ -566,15 +28,798 @@ func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handle require.Nil(t, err, "could not create browser instance") defer instance.Close() - ts := httptest.NewServer(http.HandlerFunc(handler)) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + +

Nuclei Test

+ + `) + })) defer ts.Close() parsed, err := url.Parse(ts.URL) require.Nil(t, err, "could not parse URL") - extractedData, page, err := instance.Run(parsed, actions, timeout) - assert(page, err, extractedData) - if page != nil { - page.Close() - } + actions := []*Action{{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}} + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") +} + +func TestActionScript(t *testing.T) { + t.Run("run-and-results", func(t *testing.T) { + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, + } + testActionScript(t, actions, func(page *Page, out map[string]string, err error) { + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + require.Equal(t, "some-data", out["test"], "could not run js and get results correctly") + }) + }) + + t.Run("hook", func(t *testing.T) { + actions := []*Action{ + {ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, + } + testActionScript(t, actions, func(page *Page, out map[string]string, err error) { + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook") + }) + }) +} + +func testActionScript(t *testing.T, actions []*Action, assert func(page *Page, out map[string]string, err error)) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + ` + + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, response) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + out, page, err := instance.Run(parsed, actions, 2*time.Second) + assert(page, out, err) +} + +func TestActionClick(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + Nuclei Test Page + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "click", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("button") + val := el.MustAttribute("a") + require.Equal(t, "ok", *val, "could not click button") +} + +func TestActionRightClick(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + Nuclei Test Page + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "rightclick", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("button") + val := el.MustAttribute("a") + require.Equal(t, "ok", *val, "could not click button") +} + +func TestActionTextInput(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + Nuclei Test Page + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "text", Data: map[string]string{"selector": "input", "value": "test"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + val := el.MustAttribute("event") + require.Equal(t, "input-change", *val, "could not get input change") + require.Equal(t, "test", el.MustText(), "could not get input change value") +} + +func TestActionHeadersChange(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test") == "Hello" { + fmt.Fprintln(w, `found`) + } + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "setheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") +} + +func TestActionScreenshot(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + Nuclei Test Page + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "screenshot", Data: map[string]string{"to": "test"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page() + require.FileExists(t, "test.png", el, "could not get screenshot file") + os.Remove("test.png") +} + +func TestActionTimeInput(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + Nuclei Test Page + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "time", Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + require.Equal(t, "2006-01-02", el.MustText(), "could not get input time value") +} + +func TestActionSelectInput(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "select", Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + el := page.Page().MustElement("select") + require.Equal(t, "Test2", el.MustText(), "could not get input change value") +} + +func TestActionFilesInput(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + Nuclei Test Page + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "files", Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + require.Equal(t, "C:\\fakepath\\test1.pdf", el.MustText(), "could not get input file") +} + +func TestActionWaitLoad(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + el := page.Page().MustElement("button") + style, err := el.Attribute("style") + require.Nil(t, err) + require.Equal(t, "color: red;", *style, "could not get color") +} + +func TestActionGetResource(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "getresource", Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, + } + out, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, len(out["src"]), 3159, "could not find resource") +} + +func TestActionExtract(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "extract", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, + } + out, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "Wait for me!", out["extract"], "could not extract text") +} + +func TestActionSetMethod(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "setmethod", Data: map[string]string{"part": "x", "method": "SET"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "SET", page.rules[0].Args["method"], "could not find resource") +} + +func TestActionAddHeader(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test") == "Hello" { + fmt.Fprintln(w, `found`) + } + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") +} + +func TestActionDeleteHeader(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test1") == "Hello" && r.Header.Get("Test2") == "" { + fmt.Fprintln(w, `header deleted`) + } + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, + {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}}, + {ActionType: "deleteheader", Data: map[string]string{"part": "request", "key": "Test2"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "header deleted", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not delete header correctly") +} + +func TestActionSetBody(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := ioutil.ReadAll(r.Body) + fmt.Fprintln(w, string(body)) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "setbody", Data: map[string]string{"part": "request", "body": "hello"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + require.Equal(t, "hello", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") +} + +func TestActionKeyboard(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "click", Data: map[string]string{"selector": "input"}}, + {ActionType: "keyboard", Data: map[string]string{"keys": "Test2"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + require.Nil(t, err, "could not run page actions") + defer page.Close() + + el := page.Page().MustElement("input") + require.Equal(t, "Test2", el.MustText(), "could not get input change value") +} + +func TestActionSleep(t *testing.T) { + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, ` + + + Nuclei Test Page + + + + `) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "sleep", Data: map[string]string{"duration": "2"}}, + } + _, page, err := instance.Run(parsed, actions, 20*time.Second) + defer page.Close() + + require.Nil(t, err, "could not run page actions") + require.True(t, page.Page().MustElement("button").MustVisible(), "could not get button") +} + +func TestActionWaitVisible(t *testing.T) { + t.Run("wait for an element being visible", func(t *testing.T) { + testWaitVisible(t, 2*time.Second, func(page *Page, err error) { + require.Nil(t, err, "could not run page actions") + + page.Page().MustElement("button").MustVisible() + page.Close() + }) + }) + + t.Run("timeout because of element not visible", func(t *testing.T) { + testWaitVisible(t, time.Second/2, func(page *Page, err error) { + require.Error(t, err) + require.Contains(t, err.Error(), "Element did not appear in the given amount of time") + }) + }) +} + +func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page, err error)) { + response := ` + + + Nuclei Test Page + + + + ` + + _ = protocolstate.Init(&types.Options{}) + + browser, err := New(&types.Options{ShowBrowser: false}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, response) + })) + defer ts.Close() + + parsed, err := url.Parse(ts.URL) + require.Nil(t, err, "could not parse URL") + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + } + _, page, err := instance.Run(parsed, actions, timeout) + assert(page, err) } From 9b7a105766789c1da3f3f32352158ed251264d24 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 14 Oct 2021 19:11:07 +0300 Subject: [PATCH 102/147] Headless tests refactor - test fix (#1120) * Add missing headless actions tests #1102 * test refactor (removed parallel test execution) --- .../headless/engine/page_actions_test.go | 788 ++++++------------ 1 file changed, 250 insertions(+), 538 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index ae5d16e49..902b978ac 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -18,18 +18,7 @@ import ( ) func TestActionNavigate(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page @@ -37,29 +26,36 @@ func TestActionNavigate(t *testing.T) {

Nuclei Test

- `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}} - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + }) } func TestActionScript(t *testing.T) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + ` + + timeout := 2 * time.Second + t.Run("run-and-results", func(t *testing.T) { actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, } - testActionScript(t, actions, func(page *Page, out map[string]string, err error) { + testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") require.Equal(t, "some-data", out["test"], "could not run js and get results correctly") }) @@ -72,302 +68,167 @@ func TestActionScript(t *testing.T) { {ActionType: "waitload"}, {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, } - testActionScript(t, actions, func(page *Page, out map[string]string, err error) { + testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook") }) }) } -func testActionScript(t *testing.T, actions []*Action, assert func(page *Page, out map[string]string, err error)) { +func TestActionClick(t *testing.T) { response := ` - - Nuclei Test Page - - Nuclei Test Page - - ` - - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, response) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - - out, page, err := instance.Run(parsed, actions, 2*time.Second) - assert(page, out, err) -} - -func TestActionClick(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "click", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("button") - val := el.MustAttribute("a") - require.Equal(t, "ok", *val, "could not click button") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("button") + val := el.MustAttribute("a") + require.Equal(t, "ok", *val, "could not click button") + }) } func TestActionRightClick(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + + + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "rightclick", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("button") - val := el.MustAttribute("a") - require.Equal(t, "ok", *val, "could not click button") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("button") + val := el.MustAttribute("a") + require.Equal(t, "ok", *val, "could not click button") + }) } func TestActionTextInput(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "text", Data: map[string]string{"selector": "input", "value": "test"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("input") - val := el.MustAttribute("event") - require.Equal(t, "input-change", *val, "could not get input change") - require.Equal(t, "test", el.MustText(), "could not get input change value") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + val := el.MustAttribute("event") + require.Equal(t, "input-change", *val, "could not get input change") + require.Equal(t, "test", el.MustText(), "could not get input change value") + }) } func TestActionHeadersChange(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Test") == "Hello" { - fmt.Fprintln(w, `found`) - } - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - actions := []*Action{ {ActionType: "setheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + handler := func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test") == "Hello" { + _, _ = fmt.Fprintln(w, `found`) + } + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + }) } func TestActionScreenshot(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "screenshot", Data: map[string]string{"to": "test"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page() - require.FileExists(t, "test.png", el, "could not get screenshot file") - os.Remove("test.png") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page() + require.FileExists(t, "test.png", el, "could not get screenshot file") + _ = os.Remove("test.png") + }) } func TestActionTimeInput(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "time", Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("input") - require.Equal(t, "2006-01-02", el.MustText(), "could not get input time value") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + require.Equal(t, "2006-01-02", el.MustText(), "could not get input time value") + }) } func TestActionSelectInput(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page @@ -378,79 +239,47 @@ func TestActionSelectInput(t *testing.T) { - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "select", Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - el := page.Page().MustElement("select") - require.Equal(t, "Test2", el.MustText(), "could not get input change value") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + el := page.Page().MustElement("select") + require.Equal(t, "Test2", el.MustText(), "could not get input change value") + }) } func TestActionFilesInput(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + + Nuclei Test Page + + Nuclei Test Page + + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, {ActionType: "files", Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("input") - require.Equal(t, "C:\\fakepath\\test1.pdf", el.MustText(), "could not get input file") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + require.Equal(t, "C:\\fakepath\\test1.pdf", el.MustText(), "could not get input file") + }) } func TestActionWaitLoad(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page @@ -459,40 +288,24 @@ func TestActionWaitLoad(t *testing.T) { - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - el := page.Page().MustElement("button") - style, err := el.Attribute("style") - require.Nil(t, err) - require.Equal(t, "color: red;", *style, "could not get color") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + el := page.Page().MustElement("button") + style, attributeErr := el.Attribute("style") + require.Nil(t, attributeErr) + require.Equal(t, "color: red;", *style, "could not get color") + }) } func TestActionGetResource(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page @@ -500,149 +313,78 @@ func TestActionGetResource(t *testing.T) { - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "getresource", Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, } - out, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, len(out["src"]), 3159, "could not find resource") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, len(out["src"]), 3159, "could not find resource") + }) } func TestActionExtract(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "extract", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, } - out, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "Wait for me!", out["extract"], "could not extract text") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Wait for me!", out["extract"], "could not extract text") + }) } func TestActionSetMethod(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "setmethod", Data: map[string]string{"part": "x", "method": "SET"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "SET", page.rules[0].Args["method"], "could not find resource") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "SET", page.rules[0].Args["method"], "could not find resource") + }) } func TestActionAddHeader(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Test") == "Hello" { - fmt.Fprintln(w, `found`) - } - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - actions := []*Action{ {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + handler := func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test") == "Hello" { + _, _ = fmt.Fprintln(w, `found`) + } + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + }) } func TestActionDeleteHeader(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Test1") == "Hello" && r.Header.Get("Test2") == "" { - fmt.Fprintln(w, `header deleted`) - } - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - actions := []*Action{ {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}}, @@ -650,58 +392,39 @@ func TestActionDeleteHeader(t *testing.T) { {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "header deleted", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not delete header correctly") + handler := func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test1") == "Hello" && r.Header.Get("Test2") == "" { + _, _ = fmt.Fprintln(w, `header deleted`) + } + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "header deleted", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not delete header correctly") + }) } func TestActionSetBody(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := ioutil.ReadAll(r.Body) - fmt.Fprintln(w, string(body)) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - actions := []*Action{ {ActionType: "setbody", Data: map[string]string{"part": "request", "body": "hello"}}, {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - require.Equal(t, "hello", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + handler := func(w http.ResponseWriter, r *http.Request) { + body, _ := ioutil.ReadAll(r.Body) + _, _ = fmt.Fprintln(w, string(body)) + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "hello", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + }) } func TestActionKeyboard(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` + response := ` Nuclei Test Page @@ -709,12 +432,7 @@ func TestActionKeyboard(t *testing.T) { - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") + ` actions := []*Action{ {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, @@ -722,72 +440,15 @@ func TestActionKeyboard(t *testing.T) { {ActionType: "click", Data: map[string]string{"selector": "input"}}, {ActionType: "keyboard", Data: map[string]string{"keys": "Test2"}}, } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - el := page.Page().MustElement("input") - require.Equal(t, "Test2", el.MustText(), "could not get input change value") + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + el := page.Page().MustElement("input") + require.Equal(t, "Test2", el.MustText(), "could not get input change value") + }) } func TestActionSleep(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - defer instance.Close() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "sleep", Data: map[string]string{"duration": "2"}}, - } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - defer page.Close() - - require.Nil(t, err, "could not run page actions") - require.True(t, page.Page().MustElement("button").MustVisible(), "could not get button") -} - -func TestActionWaitVisible(t *testing.T) { - t.Run("wait for an element being visible", func(t *testing.T) { - testWaitVisible(t, 2*time.Second, func(page *Page, err error) { - require.Nil(t, err, "could not run page actions") - - page.Page().MustElement("button").MustVisible() - page.Close() - }) - }) - - t.Run("timeout because of element not visible", func(t *testing.T) { - testWaitVisible(t, time.Second/2, func(page *Page, err error) { - require.Error(t, err) - require.Contains(t, err.Error(), "Element did not appear in the given amount of time") - }) - }) -} - -func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page, err error)) { response := ` @@ -799,6 +460,59 @@ func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page ` + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "sleep", Data: map[string]string{"duration": "2"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.True(t, page.Page().MustElement("button").MustVisible(), "could not get button") + }) +} + +func TestActionWaitVisible(t *testing.T) { + response := ` + + + Nuclei Test Page + + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + } + + t.Run("wait for an element being visible", func(t *testing.T) { + testHeadlessSimpleResponse(t, response, actions, 2*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + + page.Page().MustElement("button").MustVisible() + }) + }) + + t.Run("timeout because of element not visible", func(t *testing.T) { + testHeadlessSimpleResponse(t, response, actions, time.Second/2, func(page *Page, err error, out map[string]string) { + require.Error(t, err) + require.Contains(t, err.Error(), "Element did not appear in the given amount of time") + }) + }) +} + +func testHeadlessSimpleResponse(t *testing.T, response string, actions []*Action, timeout time.Duration, assert func(page *Page, pageErr error, out map[string]string)) { + t.Helper() + testHeadless(t, actions, timeout, func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintln(w, response) + }, assert) +} + +func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handler func(w http.ResponseWriter, r *http.Request), assert func(page *Page, pageErr error, extractedData map[string]string)) { + t.Helper() _ = protocolstate.Init(&types.Options{}) browser, err := New(&types.Options{ShowBrowser: false}) @@ -807,19 +521,17 @@ func testWaitVisible(t *testing.T, timeout time.Duration, assert func(page *Page instance, err := browser.NewInstance() require.Nil(t, err, "could not create browser instance") + defer instance.Close() - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, response) - })) + ts := httptest.NewServer(http.HandlerFunc(handler)) defer ts.Close() parsed, err := url.Parse(ts.URL) require.Nil(t, err, "could not parse URL") + extractedData, page, err := instance.Run(parsed, actions, timeout) + assert(page, err, extractedData) - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + if page != nil { + page.Close() } - _, page, err := instance.Run(parsed, actions, timeout) - assert(page, err) } From 736a530bdec6571d5f348ef65bd1ec1d197bc62e Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 14 Oct 2021 22:26:01 +0530 Subject: [PATCH 103/147] Added self-contained http requests implementation --- v2/internal/runner/processor.go | 21 ++++++++++++ v2/internal/runner/runner.go | 4 ++- v2/pkg/protocols/http/build_request.go | 47 +++++++++++++++++++++++++- v2/pkg/protocols/http/http.go | 3 ++ v2/pkg/protocols/http/raw/raw.go | 6 ++-- 5 files changed, 77 insertions(+), 4 deletions(-) diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index f24055ccf..3f611b280 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -7,6 +7,27 @@ import ( "go.uber.org/atomic" ) +// processSelfContainedTemplates execute a self-contained template. +func (r *Runner) processSelfContainedTemplates(template *templates.Template) bool { + match, err := template.Executer.Execute("") + if err != nil { + gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) + } + return match +} + +func httpRequestIsSelfContained(template *templates.Template) bool { + if len(template.RequestsHTTP) == 0 { + return false + } + for _, request := range template.RequestsHTTP { + if request.SelfContained { + return true + } + } + return false +} + // processTemplateWithList execute a template against the list of user provided targets func (r *Runner) processTemplateWithList(template *templates.Template) bool { results := &atomic.Bool{} diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index e87903f14..35f5788bf 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -509,7 +509,9 @@ func (r *Runner) RunEnumeration() error { go func(template *templates.Template) { defer wgtemplates.Done() - if len(template.Workflows) > 0 { + if httpRequestIsSelfContained(template) { + results.CAS(false, r.processSelfContainedTemplates(template)) + } else if len(template.Workflows) > 0 { results.CAS(false, r.processWorkflowWithList(template)) } else { results.CAS(false, r.processTemplateWithList(template)) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index ee3e42ca5..bbab4f3a2 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -1,6 +1,7 @@ package http import ( + "bufio" "context" "fmt" "io" @@ -41,6 +42,9 @@ type generatedRequest struct { // Make creates a http request for the provided input. // It returns io.EOF as error when all the requests have been exhausted. func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) { + if r.request.SelfContained { + return r.makeSelfContainedRequest(dynamicValues, interactURL) + } // We get the next payload for the request. data, payloads, ok := r.nextValue() if !ok { @@ -55,8 +59,8 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa data, parsed = baseURLWithTemplatePrefs(data, parsed) - trailingSlash := false isRawRequest := len(r.request.Raw) > 0 + trailingSlash := false if !isRawRequest && strings.HasSuffix(parsed.Path, "/") && strings.Contains(data, "{{BaseURL}}/") { trailingSlash = true } @@ -80,6 +84,47 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL) } +func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) { + // We get the next payload for the request. + data, payloads, ok := r.nextValue() + if !ok { + return nil, io.EOF + } + ctx := context.Background() + + values := generators.CopyMap(dynamicValues) + if !r.options.Options.Vars.IsEmpty() { + values = generators.MergeMaps(values, r.options.Options.Vars.AsMap()) + } + if r.options.Options.EnvironmentVariables { + values = generators.MergeMaps(generators.EnvVars(), values) + } + + isRawRequest := len(r.request.Raw) > 0 + + // If data contains \n it's a raw request, process it like raw. Else + // continue with the template based request flow. + if isRawRequest { + // Get the hostname from the URL section to build the request. + reader := bufio.NewReader(strings.NewReader(data)) + s, err := reader.ReadString('\n') + if err != nil { + return nil, fmt.Errorf("could not read request: %s", err) + } + + parts := strings.Split(s, " ") + if len(parts) < 3 { + return nil, fmt.Errorf("malformed request supplied") + } + parsed, err := url.Parse(parts[1]) + if err != nil { + return nil, fmt.Errorf("could not parse request URL: %s", err) + } + return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL) + } + return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL) +} + // Total returns the total number of requests for the generator func (r *requestGenerator) Total() int { if r.payloadIterator != nil { diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 1763f6ba9..760f1bc14 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -137,6 +137,9 @@ type Request struct { rawhttpClient *rawhttp.Client dynamicValues map[string]interface{} + // description: | + // Self Contained marks HTTP Requests for the template as self-contained + SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark http requests as self-contained,description=Mark HTTP Requests for the template as self-contained"` // description: | // CookieReuse is an optional setting that enables cookie reuse for // all requests defined in raw section. diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 582cd1351..8d1b302dc 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -90,7 +90,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { return nil, fmt.Errorf("could not parse request URL: %s", parseErr) } - rawRequest.Path = parts[1] + rawRequest.Path = parsed.Path rawRequest.Headers["Host"] = parsed.Host } else if len(parts) > 1 { rawRequest.Path = parts[1] @@ -104,7 +104,9 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { if strings.HasSuffix(parsedURL.Path, "/") && strings.HasPrefix(rawRequest.Path, "/") { parsedURL.Path = strings.TrimSuffix(parsedURL.Path, "/") } - rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path) + if parsedURL.Path != rawRequest.Path { + rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path) + } if strings.HasSuffix(rawRequest.Path, "//") { rawRequest.Path = strings.TrimSuffix(rawRequest.Path, "/") } From 0e3c6561031f90956540963472f269c0a5518408 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 15 Oct 2021 13:55:50 +0530 Subject: [PATCH 104/147] Added curl command to http request report (#1107) * Added curl command to http request report --- v2/go.mod | 1 + v2/go.sum | 2 ++ v2/pkg/output/output.go | 4 +++- v2/pkg/protocols/http/operators.go | 1 + v2/pkg/protocols/http/request.go | 12 ++++++++++++ v2/pkg/reporting/format/format.go | 7 +++++++ v2/pkg/reporting/trackers/jira/jira.go | 7 +++++++ 7 files changed, 33 insertions(+), 1 deletion(-) diff --git a/v2/go.mod b/v2/go.mod index 471e12c5c..ba180bc2a 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -117,4 +117,5 @@ require ( google.golang.org/protobuf v1.27.1 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + moul.io/http2curl v1.0.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index e85a7fce4..98a29c7f1 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1199,6 +1199,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= +moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 8f42646cf..ebe8e3925 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -90,7 +90,9 @@ type ResultEvent struct { Timestamp time.Time `json:"timestamp"` // Interaction is the full details of interactsh interaction. Interaction *server.Interaction `json:"interaction,omitempty"` - + // CURLCommand is an optional curl command to reproduce the request + // Only applicable if the report is for HTTP. + CURLCommand string `json:"curl_command,omitempty"` FileToIndexPosition map[string]int `json:"-"` } diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index c91ee0ec1..9ae939cf6 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -147,6 +147,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), + CURLCommand: types.ToString(wrapped.InternalEvent["curl-command"]), } return data } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 979e0eae2..4a1466633 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -15,6 +15,7 @@ import ( "github.com/pkg/errors" "github.com/remeh/sizedwaitgroup" "go.uber.org/multierr" + "moul.io/http2curl" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -373,6 +374,16 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate resp.Body.Close() }() + var curlCommand string + if !request.Unsafe && resp != nil && generatedRequest.request != nil { + bodyBytes, _ := generatedRequest.request.BodyBytes() + resp.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) + command, _ := http2curl.GetCurlCommand(resp.Request) + if err == nil && command != nil { + curlCommand = command.String() + } + } + gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", request.options.TemplateID, formedURL) request.options.Output.Request(request.options.TemplateID, formedURL, "http", err) @@ -460,6 +471,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } + outputEvent["curl-command"] = curlCommand outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse) for k, v := range previousEvent { diff --git a/v2/pkg/reporting/format/format.go b/v2/pkg/reporting/format/format.go index e1a0f8995..4f9d3e526 100644 --- a/v2/pkg/reporting/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -131,6 +131,13 @@ func MarkdownDescription(event *output.ResultEvent) string { // TODO remove the } } } + builder.WriteString("\n") + + if event.CURLCommand != "" { + builder.WriteString("\n**CURL Command**\n```\n") + builder.WriteString(event.CURLCommand) + builder.WriteString("\n```") + } builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei %s](https://github.com/projectdiscovery/nuclei)", config.Version)) data := builder.String() diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 337ba3f5d..8f56d71ed 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -244,6 +244,13 @@ func jiraFormatDescription(event *output.ResultEvent) string { // TODO remove th } } } + builder.WriteString("\n") + + if event.CURLCommand != "" { + builder.WriteString("\n*CURL Command*\n{code}\n") + builder.WriteString(event.CURLCommand) + builder.WriteString("\n{code}") + } builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei v%s](https://github.com/projectdiscovery/nuclei)", config.Version)) data := builder.String() return data From 54b314019483431b82a7b05a8531eea19ee87bd4 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Fri, 15 Oct 2021 14:51:55 +0200 Subject: [PATCH 105/147] fix: add validation for github and gitblab options --- v2/pkg/reporting/trackers/github/github.go | 23 ++++++++++++++++++++++ v2/pkg/reporting/trackers/gitlab/gitlab.go | 21 ++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 89f6dceb7..4ef624498 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -36,6 +36,10 @@ type Options struct { // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { + err := validateOptions(options) + if err != nil { + return nil, errors.Wrap(err, "could not parse config") + } ctx := context.Background() ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: options.Token}, @@ -53,6 +57,25 @@ func New(options *Options) (*Integration, error) { return &Integration{client: client, options: options}, nil } +func validateOptions(options *Options) error { + if options.Username == "" { + return errors.New("Username name is mandatory") + } + if options.Owner == "" { + return errors.New("Owner name is mandatory") + } + if options.Token == "" { + return errors.New("Token name is mandatory") + } + if options.ProjectName == "" { + return errors.New("ProjectName name is mandatory") + } + if options.IssueLabel == "" { + return errors.New("IssueLabel name is mandatory") + } + return nil +} + // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index 33d4fff01..fe4b19546 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -1,6 +1,7 @@ package gitlab import ( + "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" "github.com/xanzy/go-gitlab" @@ -29,6 +30,10 @@ type Options struct { // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { + err := validateOptions(options) + if err != nil { + return nil, errors.Wrap(err, "could not parse config") + } gitlabOpts := []gitlab.ClientOptionFunc{} if options.BaseURL != "" { gitlabOpts = append(gitlabOpts, gitlab.WithBaseURL(options.BaseURL)) @@ -44,6 +49,22 @@ func New(options *Options) (*Integration, error) { return &Integration{client: git, userID: user.ID, options: options}, nil } +func validateOptions(options *Options) error { + if options.Username == "" { + return errors.New("Username name is mandatory") + } + if options.Token == "" { + return errors.New("Token name is mandatory") + } + if options.ProjectName == "" { + return errors.New("ProjectName name is mandatory") + } + if options.IssueLabel == "" { + return errors.New("IssueLabel name is mandatory") + } + return nil +} + // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) From 61ca557faf8060516a79fe8119d19013baea4dba Mon Sep 17 00:00:00 2001 From: Pedro Lopez Mareque Date: Fri, 15 Oct 2021 14:59:16 +0200 Subject: [PATCH 106/147] chore: improve doc --- v2/pkg/reporting/trackers/github/github.go | 10 +++++----- v2/pkg/reporting/trackers/gitlab/gitlab.go | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 1349499d0..c794fc1e4 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -24,15 +24,15 @@ type Integration struct { type Options struct { // BaseURL is the optional self-hosted github application url BaseURL string `yaml:"base-url"` - // Username is the username of the github user + // Username is the mandatory username of the github user Username string `yaml:"username"` - // Owner is the owner name of the repository for issues. + // Owner is the mandatory owner name of the repository for issues. Owner string `yaml:"owner"` - // Token is the token for github account. + // Token is the mandatory token for github account. Token string `yaml:"token"` - // ProjectName is the name of the repository. + // ProjectName is the mandatory name of the repository. ProjectName string `yaml:"project-name"` - // IssueLabel is the label of the created issue type + // IssueLabel is the mandatory label of the created issue type IssueLabel string `yaml:"issue-label"` } diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index bf1e7c43b..cc1c8bcce 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -21,13 +21,13 @@ type Integration struct { type Options struct { // BaseURL is the optional self-hosted gitlab application url BaseURL string `yaml:"base-url"` - // Username is the username of the gitlab user + // Username is the mandatory username of the gitlab user Username string `yaml:"username"` - // Token is the token for gitlab account. + // Token is the mandatory token for gitlab account. Token string `yaml:"token"` - // ProjectName is the name of the repository. + // ProjectName is the mandatory name of the repository. ProjectName string `yaml:"project-name"` - // IssueLabel is the label of the created issue type + // IssueLabel is the mandatory label of the created issue type IssueLabel string `yaml:"issue-label"` } From e7d632a726a66bfac93bb4b859376d3d9dcba5d0 Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 15 Oct 2021 18:17:00 +0200 Subject: [PATCH 107/147] moving gbk detection code into helper function --- v2/pkg/protocols/http/request.go | 2 +- v2/pkg/protocols/http/utils.go | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index bd209d7f2..c8781c19e 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -430,7 +430,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate // Decode gbk response content-types // gb18030 supersedes gb2312 - if contentType := strings.ToLower(resp.Header.Get("Content-Type")); stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030") { + if isContentTypeGbk(resp.Header.Get("Content-Type")) { dumpedResponse, err = decodegbk(dumpedResponse) if err != nil { return errors.Wrap(err, "could not gbk decode") diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index bb12b5a66..89c35a233 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/rawhttp" + "github.com/projectdiscovery/stringsutil" "golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/transform" ) @@ -135,3 +136,9 @@ func decodegbk(s []byte) ([]byte, error) { } return d, nil } + +// isContentTypeGbk checks if the content-type header is gbk +func isContentTypeGbk(contentType string) bool { + contentType = strings.ToLower(contentType) + return stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030") +} From dcc5a2840b99db89ac13025e0741d430ba3ae019 Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 15 Oct 2021 18:45:40 +0200 Subject: [PATCH 108/147] ensures internal keyword has priority over response headers in internal map - fixes #667 --- v2/pkg/protocols/offlinehttp/operators.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index 59d23470d..b5141037b 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -98,6 +98,13 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw for k, v := range extra { data[k] = v } + for _, cookie := range resp.Cookies() { + data[strings.ToLower(cookie.Name)] = cookie.Value + } + for k, v := range resp.Header { + k = strings.ToLower(strings.TrimSpace(k)) + data[k] = strings.Join(v, " ") + } data["path"] = host data["matched"] = matched @@ -106,13 +113,6 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw data["content_length"] = resp.ContentLength data["status_code"] = resp.StatusCode data["body"] = body - for _, cookie := range resp.Cookies() { - data[strings.ToLower(cookie.Name)] = cookie.Value - } - for k, v := range resp.Header { - k = strings.ToLower(strings.TrimSpace(k)) - data[k] = strings.Join(v, " ") - } data["all_headers"] = headers data["duration"] = duration.Seconds() data["template-id"] = request.options.TemplateID From 662fb6315638c7477c0f077ea16efb3580265b86 Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 16 Oct 2021 00:17:33 +0200 Subject: [PATCH 109/147] adding missing decoding on response body --- v2/pkg/protocols/http/request.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index c8781c19e..6f8b187d3 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -439,6 +439,12 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if err != nil { return errors.Wrap(err, "could not gbk decode") } + + // the uncompressed body needs to be decoded to standard utf8 + data, err = decodegbk(data) + if err != nil { + return errors.Wrap(err, "could not gbk decode") + } } // if nuclei-project is enabled store the response if not previously done From b6104527e15ae024fd77a3a89e201b2d58ba7f0d Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 16 Oct 2021 01:05:13 +0200 Subject: [PATCH 110/147] replacing go get with go install --- v2/cmd/functional-test/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh index 030b25443..8f6c635c4 100644 --- a/v2/cmd/functional-test/run.sh +++ b/v2/cmd/functional-test/run.sh @@ -7,7 +7,7 @@ echo 'Building Nuclei binary from current branch' go build -o nuclei_dev ../nuclei echo 'Installing latest release of nuclei' -GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei +GO111MODULE=on go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei echo 'Starting Nuclei functional test' ./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt \ No newline at end of file From fc905ccccc5187f7d0e62b4f77872de36c9be42a Mon Sep 17 00:00:00 2001 From: voidz0r Date: Sun, 17 Oct 2021 01:02:37 +0200 Subject: [PATCH 111/147] Adding unixtime functionality with seconds offset --- v2/pkg/operators/common/dsl/dsl.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 056f4ace7..787cd8ff6 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -224,6 +224,15 @@ var functions = map[string]govaluate.ExpressionFunction{ } return rand.Intn(max-min) + min, nil }, + "unixtime": func(args ...interface{}) (interface{}, error) { + seconds := 0 + if len(args) >= 1 { + seconds = int(args[0].(float64)) + } + now := time.Now() + offset := now.Add(time.Duration(seconds) * time.Second) + return offset.Unix(), nil + }, // Time Functions "waitfor": func(args ...interface{}) (interface{}, error) { seconds := args[0].(float64) From 5f9b2b3df302c586bdaa4357be5f805ab31d84e7 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 17 Oct 2021 11:52:48 +0200 Subject: [PATCH 112/147] Adding support for CLi/Env variables in network requests --- v2/pkg/protocols/common/generators/options.go | 20 +++++++++++++++++++ v2/pkg/protocols/http/build_request.go | 15 ++++---------- v2/pkg/protocols/network/request.go | 6 +++++- 3 files changed, 29 insertions(+), 12 deletions(-) create mode 100644 v2/pkg/protocols/common/generators/options.go diff --git a/v2/pkg/protocols/common/generators/options.go b/v2/pkg/protocols/common/generators/options.go new file mode 100644 index 000000000..868ccef7c --- /dev/null +++ b/v2/pkg/protocols/common/generators/options.go @@ -0,0 +1,20 @@ +package generators + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// BuildPayloadFromOptions returns a map with the payloads provided via CLI +func BuildPayloadFromOptions(options *types.Options) map[string]interface{} { + m := make(map[string]interface{}) + // merge with vars + if !options.Vars.IsEmpty() { + m = MergeMaps(m, options.Vars.AsMap()) + } + + // merge with env vars + if options.EnvironmentVariables { + m = MergeMaps(EnvVars(), m) + } + return m +} diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index ee3e42ca5..e88874aab 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -60,17 +60,10 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa if !isRawRequest && strings.HasSuffix(parsed.Path, "/") && strings.Contains(data, "{{BaseURL}}/") { trailingSlash = true } - values := generators.MergeMaps(dynamicValues, generateVariables(parsed, trailingSlash)) - - // merge with vars - if !r.options.Options.Vars.IsEmpty() { - values = generators.MergeMaps(values, r.options.Options.Vars.AsMap()) - } - - // merge with env vars - if r.options.Options.EnvironmentVariables { - values = generators.MergeMaps(generators.EnvVars(), values) - } + values := generators.MergeMaps( + generators.MergeMaps(dynamicValues, generateVariables(parsed, trailingSlash)), + generators.BuildPayloadFromOptions(r.request.options.Options), + ) // If data contains \n it's a raw request, process it like raw. Else // continue with the template based request flow. diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index ba996da8c..0d5dbccac 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -59,6 +59,8 @@ func (request *Request) executeAddress(actualAddress, address, input string, sho return err } + payloads := generators.BuildPayloadFromOptions(request.options.Options) + if request.generator != nil { iterator := request.generator.NewIterator() @@ -67,12 +69,13 @@ func (request *Request) executeAddress(actualAddress, address, input string, sho if !ok { break } + value = generators.MergeMaps(value, payloads) if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } } else { - value := make(map[string]interface{}) + value := generators.MergeMaps(map[string]interface{}{}, payloads) if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } @@ -86,6 +89,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input conn net.Conn err error ) + request.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { From 8c9d9a77bfe445c9a10076a28d013dfe0fc7ed61 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 17 Oct 2021 18:50:07 +0530 Subject: [PATCH 113/147] Added self-contained to network protocol --- v2/go.mod | 4 ++-- v2/go.sum | 5 ++++- v2/internal/runner/processor.go | 9 +++++++-- v2/internal/runner/runner.go | 2 +- v2/pkg/protocols/network/network.go | 3 +++ v2/pkg/protocols/network/request.go | 9 ++++++++- 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 0055a2ff4..e5e74498f 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -65,6 +65,7 @@ require ( golang.org/x/text v0.3.7 google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v2 v2.4.0 + moul.io/http2curl v1.0.0 ) require ( @@ -74,9 +75,9 @@ require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.1.6 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect - github.com/aymerick/douceur v0.2.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -124,5 +125,4 @@ require ( google.golang.org/protobuf v1.27.1 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - moul.io/http2curl v1.0.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 845521196..b95089c7e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -325,6 +325,7 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= @@ -402,6 +403,7 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= @@ -621,7 +623,6 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjB github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d h1:YBYwsm8MrSp9t7mLehyqGwUKZWB08fG+YRePQRo5iFw= github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d/go.mod h1:JK4F9ACNPgO+Lbm80khX2q1ABInBMbwIOmbsEE61Sn4= @@ -680,8 +681,10 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 1e57536e4..a20cf0909 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -16,8 +16,8 @@ func (r *Runner) processSelfContainedTemplates(template *templates.Template) boo return match } -func httpRequestIsSelfContained(template *templates.Template) bool { - if len(template.RequestsHTTP) == 0 { +func isRequestSelfContained(template *templates.Template) bool { + if len(template.RequestsNetwork) == 0 && len(template.RequestsHTTP) == 0 { return false } for _, request := range template.RequestsHTTP { @@ -25,6 +25,11 @@ func httpRequestIsSelfContained(template *templates.Template) bool { return true } } + for _, request := range template.RequestsNetwork { + if request.SelfContained { + return true + } + } return false } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 015ca5eba..042971782 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -538,7 +538,7 @@ func (r *Runner) RunEnumeration() error { go func(template *templates.Template) { defer wgtemplates.Done() - if httpRequestIsSelfContained(template) { + if isRequestSelfContained(template) { results.CAS(false, r.processSelfContainedTemplates(template)) } else if len(template.Workflows) > 0 { results.CAS(false, r.processWorkflowWithList(template)) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index c6849eb64..98d7e80fd 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -60,6 +60,9 @@ type Request struct { // examples: // - value: "2048" ReadSize int `yaml:"read-size,omitempty" jsonschema:"title=size of network response to read,description=Size of response to read at the end. Default is 1024 bytes"` + // description: | + // Self Contained marks Network Requests for the template as self-contained + SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark network requests as self-contained,description=Mark Network Requests for the template as self-contained"` // Operators for the current request go here. operators.Operators `yaml:",inline,omitempty"` diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index ba996da8c..a1ac12333 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -26,7 +26,14 @@ var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - address, err := getAddress(input) + var address string + var err error + + if !request.SelfContained { + address, err = getAddress(input) + } else { + address = "" + } if err != nil { request.options.Output.Request(request.options.TemplateID, input, "network", err) request.options.Progress.IncrementFailedRequestsBy(1) From c8edd4ce4a37ec1eee9a376bfd3f3f772b67a911 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 18 Oct 2021 09:32:38 +0200 Subject: [PATCH 114/147] Fixing http proxy not working in headless browser --- v2/pkg/protocols/headless/engine/engine.go | 1 + v2/pkg/protocols/headless/engine/http_client.go | 8 ++++++++ v2/pkg/protocols/headless/engine/page.go | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index a506cbda4..0d604dfa9 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -72,6 +72,7 @@ func New(options *types.Options) (*Browser, error) { } browser := rod.New().ControlURL(launcherURL) + browser.MustIgnoreCertErrors(true) if browserErr := browser.Connect(); browserErr != nil { return nil, browserErr } diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index c9ec6e0ce..b4747d541 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -3,6 +3,7 @@ package engine import ( "crypto/tls" "net/http" + "net/url" "time" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" @@ -22,5 +23,12 @@ func newhttpClient(options *types.Options) *http.Client { InsecureSkipVerify: true, }, } + + if options.ProxyURL != "" { + if proxyURL, err := url.Parse(options.ProxyURL); err == nil { + transport.Proxy = http.ProxyURL(proxyURL) + } + } + return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second} } diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index 0ae5f0ac7..afd59fd29 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -8,7 +8,7 @@ import ( "github.com/go-rod/rod/lib/proto" ) -// Page is a single page in an isolated browser instanace +// Page is a single page in an isolated browser instance type Page struct { page *rod.Page rules []requestRule From 83924e203d7cda225dbc7258b949934053dbd2fb Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 18 Oct 2021 09:36:22 +0200 Subject: [PATCH 115/147] fixing object assignment --- v2/pkg/protocols/headless/engine/engine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index 0d604dfa9..a53ea16d9 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -72,7 +72,7 @@ func New(options *types.Options) (*Browser, error) { } browser := rod.New().ControlURL(launcherURL) - browser.MustIgnoreCertErrors(true) + browser = browser.MustIgnoreCertErrors(true) if browserErr := browser.Connect(); browserErr != nil { return nil, browserErr } From fbea7535ca41b59445e2c938e0943377c2fa14c1 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 18 Oct 2021 09:38:43 +0200 Subject: [PATCH 116/147] removing unrecognized routine --- v2/pkg/protocols/headless/engine/engine.go | 1 - 1 file changed, 1 deletion(-) diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index a53ea16d9..a506cbda4 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -72,7 +72,6 @@ func New(options *types.Options) (*Browser, error) { } browser := rod.New().ControlURL(launcherURL) - browser = browser.MustIgnoreCertErrors(true) if browserErr := browser.Connect(); browserErr != nil { return nil, browserErr } From a48e5f52e26f1b7b5feaff738d9ec2e78a26a53e Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 18 Oct 2021 14:25:25 +0200 Subject: [PATCH 117/147] Adding support for interactsh in http payloads --- v2/pkg/protocols/http/build_request.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index e88874aab..f7d975e3f 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -20,6 +20,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" ) @@ -48,6 +49,14 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa } ctx := context.Background() + if interactURL != "" { + data = r.options.Interactsh.ReplaceMarkers(data, interactURL) + + for payloadName, payloadValue := range payloads { + payloads[payloadName] = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), interactURL) + } + } + parsed, err := url.Parse(baseURL) if err != nil { return nil, err From eac57fb229745ea1aa8f0732f7993737db028ef8 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 18 Oct 2021 18:39:32 +0530 Subject: [PATCH 118/147] Fixed host header not working in raw request --- v2/pkg/protocols/http/raw/raw.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 8d1b302dc..2b3d44af5 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -91,7 +91,9 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { } rawRequest.Path = parsed.Path - rawRequest.Headers["Host"] = parsed.Host + if _, ok := rawRequest.Headers["Host"]; !ok { + rawRequest.Headers["Host"] = parsed.Host + } } else if len(parts) > 1 { rawRequest.Path = parts[1] } From 03852417328252b0aa69819ebf3bf26278f1a195 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 18 Oct 2021 19:48:47 +0530 Subject: [PATCH 119/147] Misc --- v2/pkg/protocols/http/build_request.go | 28 ++++++++---- v2/pkg/protocols/http/request.go | 3 ++ v2/pkg/protocols/network/request.go | 3 ++ v2/pkg/web/controllers/targets/targets.go | 55 +++++++++++++++++++++++ 4 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 v2/pkg/web/controllers/targets/targets.go diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 8dcecf7d6..06384a240 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -39,6 +39,16 @@ type generatedRequest struct { dynamicValues map[string]interface{} } +func (g *generatedRequest) URL() string { + if g.request != nil { + return g.request.URL.String() + } + if g.rawRequest != nil { + return g.rawRequest.FullURL + } + return "" +} + // Make creates a http request for the provided input. // It returns io.EOF as error when all the requests have been exhausted. func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) { @@ -85,14 +95,6 @@ func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]int } ctx := context.Background() - values := generators.CopyMap(dynamicValues) - if !r.options.Options.Vars.IsEmpty() { - values = generators.MergeMaps(values, r.options.Options.Vars.AsMap()) - } - if r.options.Options.EnvironmentVariables { - values = generators.MergeMaps(generators.EnvVars(), values) - } - isRawRequest := len(r.request.Raw) > 0 // If data contains \n it's a raw request, process it like raw. Else @@ -113,8 +115,18 @@ func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]int if err != nil { return nil, fmt.Errorf("could not parse request URL: %s", err) } + values := generators.MergeMaps( + generators.MergeMaps(dynamicValues, generateVariables(parsed, false)), + generators.BuildPayloadFromOptions(r.request.options.Options), + ) + return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL) } + values := generators.MergeMaps( + dynamicValues, + generators.BuildPayloadFromOptions(r.request.options.Options), + ) + return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL) } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 27a66a8af..d40689d50 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -101,6 +101,9 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output. if err == io.EOF { break } + if reqURL == "" { + reqURL = generatedHttpRequest.URL() + } if err != nil { request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 4acea63fd..567575254 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -48,6 +48,9 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } actualAddress = net.JoinHostPort(actualAddress, kv.port) } + if input != "" { + input = actualAddress + } if err := request.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil { gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err) diff --git a/v2/pkg/web/controllers/targets/targets.go b/v2/pkg/web/controllers/targets/targets.go new file mode 100644 index 000000000..bae2560dc --- /dev/null +++ b/v2/pkg/web/controllers/targets/targets.go @@ -0,0 +1,55 @@ +package targets + +import ( + "io" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// TargetsRepository stores targets on disk and provides +// functions to access them and add/update/delete them. +type TargetsRepository struct { + root string +} + +// NewTargetsRepository creates a new target repository +func NewTargetsRepository(directory string) *TargetsRepository { + return &TargetsRepository{root: directory} +} + +// Get returns an io.Reader for a targetPath +func (t *TargetsRepository) Get(targetPath string) (io.Reader, error) { + targetPathFinal := filepath.Join(t.root, targetPath) + return os.Open(targetPathFinal) +} + +// Add adds a target to the disk. If append is specified, the +// target is updated with the new input +func (t *TargetsRepository) Add(targetPath string, data io.Reader, append bool) error { + targetPathFinal := filepath.Join(t.root, targetPath) + + var file *os.File + var err error + if append { + file, err = os.OpenFile(targetPathFinal, os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModePerm) + } else { + file, err = os.Create(targetPathFinal) + } + if err != nil { + return errors.Wrap(err, "could not create target") + } + defer file.Close() + + if _, err = io.Copy(file, data); err != nil { + return errors.Wrap(err, "could not write target data") + } + return nil +} + +// Delete deletes a target with a targetPath +func (t *TargetsRepository) Delete(targetPath string) error { + targetPathFinal := filepath.Join(t.root, targetPath) + return os.Remove(targetPathFinal) +} From b01dd81d84965680ff70099cd49d407c54885a8d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 18 Oct 2021 19:50:28 +0530 Subject: [PATCH 120/147] Misc --- v2/pkg/web/controllers/targets/targets.go | 55 ----------------------- 1 file changed, 55 deletions(-) delete mode 100644 v2/pkg/web/controllers/targets/targets.go diff --git a/v2/pkg/web/controllers/targets/targets.go b/v2/pkg/web/controllers/targets/targets.go deleted file mode 100644 index bae2560dc..000000000 --- a/v2/pkg/web/controllers/targets/targets.go +++ /dev/null @@ -1,55 +0,0 @@ -package targets - -import ( - "io" - "os" - "path/filepath" - - "github.com/pkg/errors" -) - -// TargetsRepository stores targets on disk and provides -// functions to access them and add/update/delete them. -type TargetsRepository struct { - root string -} - -// NewTargetsRepository creates a new target repository -func NewTargetsRepository(directory string) *TargetsRepository { - return &TargetsRepository{root: directory} -} - -// Get returns an io.Reader for a targetPath -func (t *TargetsRepository) Get(targetPath string) (io.Reader, error) { - targetPathFinal := filepath.Join(t.root, targetPath) - return os.Open(targetPathFinal) -} - -// Add adds a target to the disk. If append is specified, the -// target is updated with the new input -func (t *TargetsRepository) Add(targetPath string, data io.Reader, append bool) error { - targetPathFinal := filepath.Join(t.root, targetPath) - - var file *os.File - var err error - if append { - file, err = os.OpenFile(targetPathFinal, os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModePerm) - } else { - file, err = os.Create(targetPathFinal) - } - if err != nil { - return errors.Wrap(err, "could not create target") - } - defer file.Close() - - if _, err = io.Copy(file, data); err != nil { - return errors.Wrap(err, "could not write target data") - } - return nil -} - -// Delete deletes a target with a targetPath -func (t *TargetsRepository) Delete(targetPath string) error { - targetPathFinal := filepath.Join(t.root, targetPath) - return os.Remove(targetPathFinal) -} From 6e7318bcbac85e83ce3bf273f6d3ddcfacec00d0 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 18 Oct 2021 20:34:26 +0530 Subject: [PATCH 121/147] Misc output fix --- v2/pkg/protocols/http/request.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index d40689d50..1a619c179 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -164,6 +164,9 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous if err == io.EOF { break } + if reqURL == "" { + reqURL = generatedHttpRequest.URL() + } if err != nil { request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err @@ -219,6 +222,9 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou if err == io.EOF { break } + if reqURL == "" { + reqURL = generatedHttpRequest.URL() + } if err != nil { request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err From 2e26b0fa49c5158b43897fee1500c7a7e39c6cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20L=C3=B3pez=20Mareque?= Date: Mon, 18 Oct 2021 20:45:46 +0200 Subject: [PATCH 122/147] chore: fix pr comments --- v2/cmd/nuclei/issue-tracker-config.yaml | 6 +++- .../reporting/exporters/es/elasticsearch.go | 27 ++++++++++++++-- v2/pkg/reporting/trackers/github/github.go | 27 ++++++++++------ v2/pkg/reporting/trackers/gitlab/gitlab.go | 25 +++++++++------ v2/pkg/reporting/trackers/jira/jira.go | 32 +++++++++++++++++-- 5 files changed, 93 insertions(+), 24 deletions(-) diff --git a/v2/cmd/nuclei/issue-tracker-config.yaml b/v2/cmd/nuclei/issue-tracker-config.yaml index fd3cab274..acc991dfe 100644 --- a/v2/cmd/nuclei/issue-tracker-config.yaml +++ b/v2/cmd/nuclei/issue-tracker-config.yaml @@ -19,6 +19,8 @@ # project-name: "" # # issue-label is the label of the created issue type # issue-label: "" +# # severity-as-label set the sevetiry as the label of the created issue type +# severity-as-label: false # gitlab contains configuration options for gitlab issue tracker #gitlab: @@ -32,6 +34,8 @@ # project-id: "" # # issue-label is the label of the created issue type # issue-label: "" +# # severity-as-label set the sevetiry as the label of the created issue type +# severity-as-label: false # jira contains configuration options for jira issue tracker #jira: @@ -67,4 +71,4 @@ # # Username for the elasticsearch instance # username: test # # Password is the password for elasticsearch instance -# password: test \ No newline at end of file +# password: test diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index cf625eb66..6753b0cf0 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -49,6 +49,10 @@ type Exporter struct { // New creates and returns a new exporter for elasticsearch func New(option *Options) (*Exporter, error) { var ei *Exporter + err := validateOptions(options) + if err != nil { + return nil, errors.New("could not parse config") + } client := &http.Client{ Timeout: 5 * time.Second, @@ -81,6 +85,25 @@ func New(option *Options) (*Exporter, error) { return ei, nil } +func validateOptions(options *Options) error { + if options.IP == "" { + return errors.New("IP name is mandatory") + } + if options.Port == 0 { + return errors.New("Port name is mandatory") + } + if options.Username == "" { + return errors.New("Username name is mandatory") + } + if options.Password == "" { + return errors.New("Password name is mandatory") + } + if options.IndexName == "" { + return errors.New("IndexName name is mandatory") + } + return nil +} + // Export exports a passed result event to elasticsearch func (i *Exporter) Export(event *output.ResultEvent) error { // creating a request @@ -105,9 +128,9 @@ func (i *Exporter) Export(event *output.ResultEvent) error { res, err := i.elasticsearch.Do(req) if err != nil { - return err + return err } - + b, err = ioutil.ReadAll(res.Body) if err != nil { return errors.New(err.Error() + "error thrown by elasticsearch " + string(b)) diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index c794fc1e4..bf9ed0765 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -22,18 +22,21 @@ type Integration struct { // Options contains the configuration options for github issue tracker client type Options struct { - // BaseURL is the optional self-hosted github application url + // BaseURL (optional) is the self-hosted github application url BaseURL string `yaml:"base-url"` - // Username is the mandatory username of the github user + // Username (mandatory) is the username of the github user Username string `yaml:"username"` - // Owner is the mandatory owner name of the repository for issues. + // Owner (manadatory)is the owner name of the repository for issues. Owner string `yaml:"owner"` - // Token is the mandatory token for github account. + // Token (mandatory) is the token for github account. Token string `yaml:"token"` - // ProjectName is the mandatory name of the repository. + // ProjectName (mandatory) name of the repository. ProjectName string `yaml:"project-name"` - // IssueLabel is the mandatory label of the created issue type + // IssueLabel (optional) it the label of the created issue type IssueLabel string `yaml:"issue-label"` + // SeverityAsLabel (optional) send the severity as the label of the created + // issue. + SeverityAsLabel bool `yaml:"severity-as-label"` } // New creates a new issue tracker integration client based on options. @@ -72,9 +75,6 @@ func validateOptions(options *Options) error { if options.ProjectName == "" { return errors.New("ProjectName name is mandatory") } - if options.IssueLabel == "" { - return errors.New("IssueLabel name is mandatory") - } return nil } @@ -82,12 +82,19 @@ func validateOptions(options *Options) error { func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) description := format.MarkdownDescription(event) + labels := []string{} severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) + if i.options.SeverityAsLabel && severityLabel != "" { + labels = append(labels, severityLabel) + } + if label := i.options.IssueLabel; label != "" { + labels = append(labels, label) + } req := &github.IssueRequest{ Title: &summary, Body: &description, - Labels: &[]string{i.options.IssueLabel, severityLabel}, + Labels: &labels, Assignees: &[]string{i.options.Username}, } _, _, err := i.client.Issues.Create(context.Background(), i.options.Owner, i.options.ProjectName, req) diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index cc1c8bcce..a12fe07cd 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -19,16 +19,19 @@ type Integration struct { // Options contains the configuration options for gitlab issue tracker client type Options struct { - // BaseURL is the optional self-hosted gitlab application url + // BaseURL (optional) is the self-hosted gitlab application url BaseURL string `yaml:"base-url"` - // Username is the mandatory username of the gitlab user + // Username (mandatory) is the username of the gitlab user Username string `yaml:"username"` - // Token is the mandatory token for gitlab account. + // Token (mandatory) is the token for gitlab account. Token string `yaml:"token"` - // ProjectName is the mandatory name of the repository. + // ProjectName (mandatory) is the name of the repository. ProjectName string `yaml:"project-name"` - // IssueLabel is the mandatory label of the created issue type + // IssueLabel (mandatory) is the label of the created issue type IssueLabel string `yaml:"issue-label"` + // SeverityAsLabel (optional) send the severity as the label of the created + // issue. + SeverityAsLabel bool `yaml:"severity-as-label"` } // New creates a new issue tracker integration client based on options. @@ -62,9 +65,6 @@ func validateOptions(options *Options) error { if options.ProjectName == "" { return errors.New("ProjectName name is mandatory") } - if options.IssueLabel == "" { - return errors.New("IssueLabel name is mandatory") - } return nil } @@ -72,12 +72,19 @@ func validateOptions(options *Options) error { func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) description := format.MarkdownDescription(event) + labels := []string{} severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) + if i.options.SeverityAsLabel && severityLabel != "" { + labels = append(labels, severityLabel) + } + if label := i.options.IssueLabel; label != "" { + labels = append(labels, label) + } _, _, err := i.client.Issues.CreateIssue(i.options.ProjectName, &gitlab.CreateIssueOptions{ Title: &summary, Description: &description, - Labels: gitlab.Labels{i.options.IssueLabel, severityLabel}, + Labels: labels, AssigneeIDs: []int{i.userID}, }) diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 8f56d71ed..befcfb9de 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -2,6 +2,7 @@ package jira import ( "bytes" + "errors" "fmt" "io/ioutil" "strings" @@ -42,6 +43,10 @@ type Options struct { // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { + err := validateOptions(options) + if err != nil { + return nil, errors.New("could not parse config") + } username := options.Email if !options.Cloud { username = options.AccountID @@ -57,10 +62,33 @@ func New(options *Options) (*Integration, error) { return &Integration{jira: jiraClient, options: options}, nil } +func validateOptions(options *Options) error { + if options.URL == "" { + return errors.New("URL name is mandatory") + } + if options.AccountID == "" { + return errors.New("AccountID name is mandatory") + } + if options.Email == "" { + return errors.New("Email name is mandatory") + } + if options.ProjectName == "" { + return errors.New("ProjectName name is mandatory") + } + return nil +} + // CreateNewIssue creates a new issue in the tracker func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { summary := format.Summary(event) - severityLabel := fmt.Sprintf("Severity:%s", event.Info.SeverityHolder.Severity.String()) + labels := []string{} + severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) + if severityLabel != "" { + labels = append(labels, severityLabel) + } + if label := i.options.IssueType; label != "" { + labels = append(labels, label) + } fields := &jira.IssueFields{ Assignee: &jira.User{AccountID: i.options.AccountID}, @@ -69,7 +97,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { Type: jira.IssueType{Name: i.options.IssueType}, Project: jira.Project{Key: i.options.ProjectName}, Summary: summary, - Labels: []string{severityLabel}, + Labels: labels, } // On-prem version of Jira server does not use AccountID if !i.options.Cloud { From 9734297d2d713f5ad825593fe70df6d6d7c31edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20L=C3=B3pez=20Mareque?= Date: Mon, 18 Oct 2021 20:48:23 +0200 Subject: [PATCH 123/147] chore: fix pr comment --- v2/pkg/reporting/trackers/github/github.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index bf9ed0765..a4b43984a 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -26,15 +26,15 @@ type Options struct { BaseURL string `yaml:"base-url"` // Username (mandatory) is the username of the github user Username string `yaml:"username"` - // Owner (manadatory)is the owner name of the repository for issues. + // Owner (manadatory) is the owner name of the repository for issues. Owner string `yaml:"owner"` // Token (mandatory) is the token for github account. Token string `yaml:"token"` - // ProjectName (mandatory) name of the repository. + // ProjectName (mandatory) is the name of the repository. ProjectName string `yaml:"project-name"` - // IssueLabel (optional) it the label of the created issue type + // IssueLabel (optional) is the label of the created issue type IssueLabel string `yaml:"issue-label"` - // SeverityAsLabel (optional) send the severity as the label of the created + // SeverityAsLabel (optional) sends the severity as the label of the created // issue. SeverityAsLabel bool `yaml:"severity-as-label"` } From befb79decb8a3aa74788b58d08abab2d931701f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20L=C3=B3pez=20Mareque?= Date: Mon, 18 Oct 2021 20:49:12 +0200 Subject: [PATCH 124/147] chore: fix pr comment --- v2/pkg/reporting/trackers/gitlab/gitlab.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index a12fe07cd..b0d28d0dc 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -29,7 +29,7 @@ type Options struct { ProjectName string `yaml:"project-name"` // IssueLabel (mandatory) is the label of the created issue type IssueLabel string `yaml:"issue-label"` - // SeverityAsLabel (optional) send the severity as the label of the created + // SeverityAsLabel (optional) sends the severity as the label of the created // issue. SeverityAsLabel bool `yaml:"severity-as-label"` } From 12cf05127184a60af9a09942b80133967300d068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20L=C3=B3pez=20Mareque?= Date: Mon, 18 Oct 2021 20:52:35 +0200 Subject: [PATCH 125/147] chore: fix pr comment --- v2/pkg/reporting/trackers/jira/jira.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index befcfb9de..7f84d7611 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -23,21 +23,21 @@ type Integration struct { // Options contains the configuration options for jira client type Options struct { - // Cloud value is set to true when Jira cloud is used + // Cloud value (optional) is set to true when Jira cloud is used Cloud bool `yaml:"cloud"` - // UpdateExisting value if true, the existing opened issue is updated + // UpdateExisting value (optional) if true, the existing opened issue is updated UpdateExisting bool `yaml:"update-existing"` - // URL is the URL of the jira server + // URL (mandatory) is the URL of the jira server URL string `yaml:"url"` - // AccountID is the accountID of the jira user. + // AccountID (mandatory) is the accountID of the jira user. AccountID string `yaml:"account-id"` - // Email is the email of the user for jira instance + // Email (mandatory) is the email of the user for jira instance Email string `yaml:"email"` - // Token is the token for jira instance. + // Token (mandatory) is the token for jira instance. Token string `yaml:"token"` - // ProjectName is the name of the project. + // ProjectName (mandatory) is the name of the project. ProjectName string `yaml:"project-name"` - // IssueType is the name of the created issue type + // IssueType (optional) is the name of the created issue type IssueType string `yaml:"issue-type"` } @@ -72,6 +72,9 @@ func validateOptions(options *Options) error { if options.Email == "" { return errors.New("Email name is mandatory") } + if options.Token == "" { + return errors.New("Token name is mandatory") + } if options.ProjectName == "" { return errors.New("ProjectName name is mandatory") } From 5a03acb2efa5dd721b27d99c7cff54e07dd390f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20L=C3=B3pez=20Mareque?= Date: Mon, 18 Oct 2021 20:54:30 +0200 Subject: [PATCH 126/147] chore: fix pr comment --- v2/pkg/reporting/exporters/es/elasticsearch.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index 6753b0cf0..208953586 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -18,19 +18,19 @@ import ( // Options contains necessary options required for elasticsearch communicaiton type Options struct { - // IP for elasticsearch instance + // IP (mandatory) for elasticsearch instance IP string `yaml:"ip"` - // Port is the port of elasticsearch instance + // Port (mandatory) is the port of elasticsearch instance Port int `yaml:"port"` - // SSL enables ssl for elasticsearch connection + // SSL (optional) enables ssl for elasticsearch connection SSL bool `yaml:"ssl"` - // SSLVerification disables SSL verification for elasticsearch + // SSLVerification (optional) disables SSL verification for elasticsearch SSLVerification bool `yaml:"ssl-verification"` - // Username for the elasticsearch instance + // Username (mandatory) for the elasticsearch instance Username string `yaml:"username"` - // Password is the password for elasticsearch instance + // Password (mandatory) is the password for elasticsearch instance Password string `yaml:"password"` - // IndexName is the name of the elasticsearch index + // IndexName (mandatory) is the name of the elasticsearch index IndexName string `yaml:"index-name"` } From 17ce2848e63f3ec89cb45ebc324f3049be76f7c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20L=C3=B3pez=20Mareque?= Date: Mon, 18 Oct 2021 20:57:21 +0200 Subject: [PATCH 127/147] chore: fix pr comment --- v2/cmd/nuclei/issue-tracker-config.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/v2/cmd/nuclei/issue-tracker-config.yaml b/v2/cmd/nuclei/issue-tracker-config.yaml index acc991dfe..37c2532bd 100644 --- a/v2/cmd/nuclei/issue-tracker-config.yaml +++ b/v2/cmd/nuclei/issue-tracker-config.yaml @@ -7,7 +7,7 @@ # github contains configuration options for github issue tracker #github: -# # base-url is the optional self-hosted github application url +# # base-url (optional) is the self-hosted github application url # base-url: "" # # username is the username of the github user # username: "" @@ -17,14 +17,14 @@ # token: "" # # project-name is the name of the repository. # project-name: "" -# # issue-label is the label of the created issue type +# # issue-label (optional) is the label of the created issue type # issue-label: "" -# # severity-as-label set the sevetiry as the label of the created issue type +# # severity-as-label (optional) sets the sevetiry as the label of the created issue type # severity-as-label: false # gitlab contains configuration options for gitlab issue tracker #gitlab: -# # base-url is the optional self-hosted gitlab application url +# # base-url (optional) is the self-hosted gitlab application url # base-url: "" # # username is the username of the gitlab user # username: "" @@ -32,16 +32,16 @@ # token: "" # # project-id is the ID of the repository. # project-id: "" -# # issue-label is the label of the created issue type +# # issue-label (optional) is the label of the created issue type # issue-label: "" -# # severity-as-label set the sevetiry as the label of the created issue type +# # severity-as-label (optional) sets the sevetiry as the label of the created issue type # severity-as-label: false # jira contains configuration options for jira issue tracker #jira: -# # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used +# # cloud (optional) is the boolean which tells if Jira instance is running in the cloud or on-prem version is used # cloud: true -# # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created +# # update-existing (optional) is the boolean which tells if the existing, opened issue should be updated or new one should be created # update-existing: false # # URL is the jira application url # url: "" @@ -64,9 +64,9 @@ # port: 9200 # # IndexName is the name of the elasticsearch index # index-name: nuclei -# # SSL enables ssl for elasticsearch connection +# # SSL (optional) enables ssl for elasticsearch connection # ssl: false -# # SSLVerification disables SSL verification for elasticsearch +# # SSLVerification (optional) disables SSL verification for elasticsearch # ssl-verification: false # # Username for the elasticsearch instance # username: test From 2b9bd7e9c3953c63dc967461f8542b2d9bc3c640 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 19 Oct 2021 01:26:21 +0530 Subject: [PATCH 128/147] Uniform JSON output --- v2/pkg/output/output.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index ebe8e3925..b90be29c3 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -59,15 +59,15 @@ type InternalWrappedEvent struct { // ResultEvent is a wrapped result event for a single nuclei output. type ResultEvent struct { // TemplateID is the ID of the template for the result. - TemplateID string `json:"templateID"` + TemplateID string `json:"template-id"` // TemplatePath is the path of template TemplatePath string `json:"-"` // Info contains information block of the template for the result. Info model.Info `json:"info,inline"` // MatcherName is the name of the matcher matched if any. - MatcherName string `json:"matcher_name,omitempty"` + MatcherName string `json:"matcher-name,omitempty"` // ExtractorName is the name of the extractor matched if any. - ExtractorName string `json:"extractor_name,omitempty"` + ExtractorName string `json:"extractor-name,omitempty"` // Type is the type of the result event. Type string `json:"type"` // Host is the host input on which match was found. @@ -75,9 +75,9 @@ type ResultEvent struct { // Path is the path input on which match was found. Path string `json:"path,omitempty"` // Matched contains the matched input in its transformed form. - Matched string `json:"matched,omitempty"` + Matched string `json:"matched-at,omitempty"` // ExtractedResults contains the extraction result from the inputs. - ExtractedResults []string `json:"extracted_results,omitempty"` + ExtractedResults []string `json:"extracted-results,omitempty"` // Request is the optional, dumped request for the match. Request string `json:"request,omitempty"` // Response is the optional, dumped response for the match. @@ -92,7 +92,7 @@ type ResultEvent struct { Interaction *server.Interaction `json:"interaction,omitempty"` // CURLCommand is an optional curl command to reproduce the request // Only applicable if the report is for HTTP. - CURLCommand string `json:"curl_command,omitempty"` + CURLCommand string `json:"curl-command,omitempty"` FileToIndexPosition map[string]int `json:"-"` } From d022804d6160c6d0d9a257a5f373a216a9238da0 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 19 Oct 2021 02:31:25 +0530 Subject: [PATCH 129/147] release workflow fix --- .github/workflows/release-binary.yml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index c0748e870..3a9983470 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -1,30 +1,26 @@ name: 🎉 Release Binary on: create: - tags: - - v* workflow_dispatch: jobs: release: runs-on: ubuntu-latest + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + steps: - - - name: "Check out code" - uses: actions/checkout@v2 + - uses: actions/checkout@v2 with: fetch-depth: 0 - - - name: "Set up Go" - uses: actions/setup-go@v2 + + - uses: actions/setup-go@v2 with: go-version: 1.17 - - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - name: "Create release on GitHub" - uses: goreleaser/goreleaser-action@v2 + + - uses: goreleaser/goreleaser-action@v2 with: args: "release --rm-dist" version: latest - workdir: v2/ \ No newline at end of file + workdir: v2/ + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file From 8df46e17745e197a315ccc5434ee2e40d72485b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20L=C3=B3pez=20Mareque?= Date: Tue, 19 Oct 2021 17:15:58 +0200 Subject: [PATCH 130/147] chore: fix pr comment --- v2/cmd/nuclei/issue-tracker-config.yaml | 2 +- .../reporting/exporters/es/elasticsearch.go | 29 ++++++++++------- v2/pkg/reporting/trackers/github/github.go | 21 ++++++++----- v2/pkg/reporting/trackers/gitlab/gitlab.go | 21 ++++++++----- v2/pkg/reporting/trackers/jira/jira.go | 31 ++++++++++++------- 5 files changed, 67 insertions(+), 37 deletions(-) diff --git a/v2/cmd/nuclei/issue-tracker-config.yaml b/v2/cmd/nuclei/issue-tracker-config.yaml index 37c2532bd..5364db5f5 100644 --- a/v2/cmd/nuclei/issue-tracker-config.yaml +++ b/v2/cmd/nuclei/issue-tracker-config.yaml @@ -7,7 +7,7 @@ # github contains configuration options for github issue tracker #github: -# # base-url (optional) is the self-hosted github application url +# # base-url (optional) is the self-hosted github application url # base-url: "" # # username is the username of the github user # username: "" diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index 208953586..f6c69cf6b 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strings" "time" "encoding/base64" @@ -18,19 +19,19 @@ import ( // Options contains necessary options required for elasticsearch communicaiton type Options struct { - // IP (mandatory) for elasticsearch instance + // IP for elasticsearch instance IP string `yaml:"ip"` - // Port (mandatory) is the port of elasticsearch instance + // Port is the port of elasticsearch instance Port int `yaml:"port"` // SSL (optional) enables ssl for elasticsearch connection SSL bool `yaml:"ssl"` // SSLVerification (optional) disables SSL verification for elasticsearch SSLVerification bool `yaml:"ssl-verification"` - // Username (mandatory) for the elasticsearch instance + // Username for the elasticsearch instance Username string `yaml:"username"` - // Password (mandatory) is the password for elasticsearch instance + // Password is the password for elasticsearch instance Password string `yaml:"password"` - // IndexName (mandatory) is the name of the elasticsearch index + // IndexName is the name of the elasticsearch index IndexName string `yaml:"index-name"` } @@ -49,7 +50,7 @@ type Exporter struct { // New creates and returns a new exporter for elasticsearch func New(option *Options) (*Exporter, error) { var ei *Exporter - err := validateOptions(options) + err := validateOptions(option) if err != nil { return nil, errors.New("could not parse config") } @@ -86,21 +87,27 @@ func New(option *Options) (*Exporter, error) { } func validateOptions(options *Options) error { + errs := []string{} if options.IP == "" { - return errors.New("IP name is mandatory") + errs = append(errs, "IP") } if options.Port == 0 { - return errors.New("Port name is mandatory") + errs = append(errs, "Port") } if options.Username == "" { - return errors.New("Username name is mandatory") + errs = append(errs, "Username") } if options.Password == "" { - return errors.New("Password name is mandatory") + errs = append(errs, "Password") } if options.IndexName == "" { - return errors.New("IndexName name is mandatory") + errs = append(errs, "IndexName") } + + if len(errs) > 0 { + return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) + } + return nil } diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index a4b43984a..c1adbfb86 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/url" + "strings" "golang.org/x/oauth2" @@ -24,13 +25,13 @@ type Integration struct { type Options struct { // BaseURL (optional) is the self-hosted github application url BaseURL string `yaml:"base-url"` - // Username (mandatory) is the username of the github user + // Username is the username of the github user Username string `yaml:"username"` // Owner (manadatory) is the owner name of the repository for issues. Owner string `yaml:"owner"` - // Token (mandatory) is the token for github account. + // Token is the token for github account. Token string `yaml:"token"` - // ProjectName (mandatory) is the name of the repository. + // ProjectName is the name of the repository. ProjectName string `yaml:"project-name"` // IssueLabel (optional) is the label of the created issue type IssueLabel string `yaml:"issue-label"` @@ -63,18 +64,24 @@ func New(options *Options) (*Integration, error) { } func validateOptions(options *Options) error { + errs := []string{} if options.Username == "" { - return errors.New("Username name is mandatory") + errs = append(errs, "Username") } if options.Owner == "" { - return errors.New("Owner name is mandatory") + errs = append(errs, "Owner") } if options.Token == "" { - return errors.New("Token name is mandatory") + errs = append(errs, "Token") } if options.ProjectName == "" { - return errors.New("ProjectName name is mandatory") + errs = append(errs, "ProjectName") } + + if len(errs) > 0 { + return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) + } + return nil } diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index b0d28d0dc..fbce367b7 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -2,6 +2,7 @@ package gitlab import ( "fmt" + "strings" "github.com/pkg/errors" @@ -21,13 +22,13 @@ type Integration struct { type Options struct { // BaseURL (optional) is the self-hosted gitlab application url BaseURL string `yaml:"base-url"` - // Username (mandatory) is the username of the gitlab user + // Username is the username of the gitlab user Username string `yaml:"username"` - // Token (mandatory) is the token for gitlab account. + // Token is the token for gitlab account. Token string `yaml:"token"` - // ProjectName (mandatory) is the name of the repository. + // ProjectName is the name of the repository. ProjectName string `yaml:"project-name"` - // IssueLabel (mandatory) is the label of the created issue type + // IssueLabel is the label of the created issue type IssueLabel string `yaml:"issue-label"` // SeverityAsLabel (optional) sends the severity as the label of the created // issue. @@ -56,15 +57,21 @@ func New(options *Options) (*Integration, error) { } func validateOptions(options *Options) error { + errs := []string{} if options.Username == "" { - return errors.New("Username name is mandatory") + errs = append(errs, "Username") } if options.Token == "" { - return errors.New("Token name is mandatory") + errs = append(errs, "Token") } if options.ProjectName == "" { - return errors.New("ProjectName name is mandatory") + errs = append(errs, "ProjectName") } + + if len(errs) > 0 { + return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) + } + return nil } diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 7f84d7611..d1552286e 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -27,18 +27,21 @@ type Options struct { Cloud bool `yaml:"cloud"` // UpdateExisting value (optional) if true, the existing opened issue is updated UpdateExisting bool `yaml:"update-existing"` - // URL (mandatory) is the URL of the jira server + // URL is the URL of the jira server URL string `yaml:"url"` - // AccountID (mandatory) is the accountID of the jira user. + // AccountID is the accountID of the jira user. AccountID string `yaml:"account-id"` - // Email (mandatory) is the email of the user for jira instance + // Email is the email of the user for jira instance Email string `yaml:"email"` - // Token (mandatory) is the token for jira instance. + // Token is the token for jira instance. Token string `yaml:"token"` - // ProjectName (mandatory) is the name of the project. + // ProjectName is the name of the project. ProjectName string `yaml:"project-name"` // IssueType (optional) is the name of the created issue type IssueType string `yaml:"issue-type"` + // SeverityAsLabel (optional) sends the severity as the label of the created + // issue. + SeverityAsLabel bool `yaml:"severity-as-label"` } // New creates a new issue tracker integration client based on options. @@ -63,21 +66,27 @@ func New(options *Options) (*Integration, error) { } func validateOptions(options *Options) error { + errs := []string{} if options.URL == "" { - return errors.New("URL name is mandatory") + errs = append(errs, "URL") } if options.AccountID == "" { - return errors.New("AccountID name is mandatory") + errs = append(errs, "AccountID") } if options.Email == "" { - return errors.New("Email name is mandatory") + errs = append(errs, "Email") } if options.Token == "" { - return errors.New("Token name is mandatory") + errs = append(errs, "Token") } if options.ProjectName == "" { - return errors.New("ProjectName name is mandatory") + errs = append(errs, "ProjectName") } + + if len(errs) > 0 { + return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) + } + return nil } @@ -86,7 +95,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { summary := format.Summary(event) labels := []string{} severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) - if severityLabel != "" { + if i.options.SeverityAsLabel && severityLabel != "" { labels = append(labels, severityLabel) } if label := i.options.IssueType; label != "" { From 2aade725b71f05861a444d35ea42f47c1335df21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20L=C3=B3pez=20Mareque?= Date: Tue, 19 Oct 2021 17:17:45 +0200 Subject: [PATCH 131/147] chore: fix pr comment --- v2/pkg/reporting/exporters/es/elasticsearch.go | 2 +- v2/pkg/reporting/trackers/github/github.go | 2 +- v2/pkg/reporting/trackers/gitlab/gitlab.go | 2 +- v2/pkg/reporting/trackers/jira/jira.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index f6c69cf6b..784e587d0 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -52,7 +52,7 @@ func New(option *Options) (*Exporter, error) { var ei *Exporter err := validateOptions(option) if err != nil { - return nil, errors.New("could not parse config") + return nil, err } client := &http.Client{ diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index c1adbfb86..672f62ede 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -44,7 +44,7 @@ type Options struct { func New(options *Options) (*Integration, error) { err := validateOptions(options) if err != nil { - return nil, errors.Wrap(err, "could not parse config") + return nil, err } ctx := context.Background() ts := oauth2.StaticTokenSource( diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index fbce367b7..35922970b 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -39,7 +39,7 @@ type Options struct { func New(options *Options) (*Integration, error) { err := validateOptions(options) if err != nil { - return nil, errors.Wrap(err, "could not parse config") + return nil, err } gitlabOpts := []gitlab.ClientOptionFunc{} if options.BaseURL != "" { diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index d1552286e..3237e929b 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -48,7 +48,7 @@ type Options struct { func New(options *Options) (*Integration, error) { err := validateOptions(options) if err != nil { - return nil, errors.New("could not parse config") + return nil, err } username := options.Email if !options.Cloud { From 612ab61df4e3cb064ef76465cc3e813c04ba9b83 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 19 Oct 2021 21:29:18 +0530 Subject: [PATCH 132/147] Reworked self-contained requests to template --- v2/pkg/protocols/http/http.go | 5 +++-- v2/pkg/protocols/network/network.go | 5 +++-- v2/pkg/templates/compile.go | 15 +++++++++++++++ v2/pkg/templates/templates.go | 4 ++++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 760f1bc14..ce3036c52 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -138,8 +138,9 @@ type Request struct { dynamicValues map[string]interface{} // description: | - // Self Contained marks HTTP Requests for the template as self-contained - SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark http requests as self-contained,description=Mark HTTP Requests for the template as self-contained"` + // SelfContained specifies if the request is self contained. + SelfContained bool `yaml:"-" json:"-"` + // description: | // CookieReuse is an optional setting that enables cookie reuse for // all requests defined in raw section. diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 98d7e80fd..a5274eff6 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -60,9 +60,10 @@ type Request struct { // examples: // - value: "2048" ReadSize int `yaml:"read-size,omitempty" jsonschema:"title=size of network response to read,description=Size of response to read at the end. Default is 1024 bytes"` + // description: | - // Self Contained marks Network Requests for the template as self-contained - SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark network requests as self-contained,description=Mark Network Requests for the template as self-contained"` + // SelfContained specifies if the request is self contained. + SelfContained bool `yaml:"-" json:"-"` // Operators for the current request go here. operators.Operators `yaml:",inline,omitempty"` diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index d7f545bc3..bf4e79b18 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -144,6 +144,21 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute } template.Path = filePath + template.parseSelfContainedRequests() + parsedTemplatesCache.Store(filePath, template, err) return template, nil } + +// parseSelfContainedRequests parses the self contained template requests. +func (t *Template) parseSelfContainedRequests() { + if !t.SelfContained { + return + } + for _, request := range t.RequestsHTTP { + request.SelfContained = true + } + for _, request := range t.RequestsNetwork { + request.SelfContained = true + } +} diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index b6f2491ac..90f716add 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -62,6 +62,10 @@ type Template struct { workflows.Workflow `yaml:",inline,omitempty" jsonschema:"title=workflows to run,description=Workflows to run for the template"` CompiledWorkflow *workflows.Workflow `yaml:"-" json:"-" jsonschema:"-"` + // description: | + // Self Contained marks Network Requests for the template as self-contained + SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"` + // TotalRequests is the total number of requests for the template. TotalRequests int `yaml:"-" json:"-"` // Executer is the actual template executor for running template requests From 55eab08faa216ed6e17d00765b7a30eab3b5f8fc Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 19 Oct 2021 21:33:17 +0530 Subject: [PATCH 133/147] Slight change to formatting --- v2/pkg/protocols/network/request.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 567575254..acbbac9af 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -29,10 +29,10 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review var address string var err error - if !request.SelfContained { - address, err = getAddress(input) - } else { + if request.SelfContained { address = "" + } else { + address, err = getAddress(input) } if err != nil { request.options.Output.Request(request.options.TemplateID, input, "network", err) From de011585561ee59a51cd5753a291c31497268e83 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 19 Oct 2021 21:52:40 +0530 Subject: [PATCH 134/147] Removed non-used code --- v2/internal/runner/processor.go | 17 ----------------- v2/internal/runner/runner.go | 2 +- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index a20cf0909..6b14b5f91 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -16,23 +16,6 @@ func (r *Runner) processSelfContainedTemplates(template *templates.Template) boo return match } -func isRequestSelfContained(template *templates.Template) bool { - if len(template.RequestsNetwork) == 0 && len(template.RequestsHTTP) == 0 { - return false - } - for _, request := range template.RequestsHTTP { - if request.SelfContained { - return true - } - } - for _, request := range template.RequestsNetwork { - if request.SelfContained { - return true - } - } - return false -} - // processTemplateWithList execute a template against the list of user provided targets func (r *Runner) processTemplateWithList(template *templates.Template) bool { results := &atomic.Bool{} diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 042971782..b3f806952 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -538,7 +538,7 @@ func (r *Runner) RunEnumeration() error { go func(template *templates.Template) { defer wgtemplates.Done() - if isRequestSelfContained(template) { + if template.SelfContained { results.CAS(false, r.processSelfContainedTemplates(template)) } else if len(template.Workflows) > 0 { results.CAS(false, r.processWorkflowWithList(template)) From c4e5fa49dd3fc4a3d8beead126c5c817484238b7 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 19 Oct 2021 22:17:44 +0530 Subject: [PATCH 135/147] Added integration tests for http and network --- integration_tests/http/self-contained.yaml | 18 ++++++++++ integration_tests/network/self-contained.yaml | 16 +++++++++ v2/cmd/integration-test/http.go | 33 +++++++++++++++++++ v2/cmd/integration-test/network.go | 32 ++++++++++++++++-- v2/internal/testutils/integration.go | 8 +++-- 5 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 integration_tests/http/self-contained.yaml create mode 100644 integration_tests/network/self-contained.yaml diff --git a/integration_tests/http/self-contained.yaml b/integration_tests/http/self-contained.yaml new file mode 100644 index 000000000..4ecacbbd9 --- /dev/null +++ b/integration_tests/http/self-contained.yaml @@ -0,0 +1,18 @@ +id: example-self-contained-input + +info: + name: example-self-contained + author: pd-team + severity: info + +self-contained: true +requests: + - raw: + - | + GET http://localhost:5431/ HTTP/1.1 + Host: {{Hostname}} + + matchers: + - type: word + words: + - This is self-contained response \ No newline at end of file diff --git a/integration_tests/network/self-contained.yaml b/integration_tests/network/self-contained.yaml new file mode 100644 index 000000000..fad3e2ac8 --- /dev/null +++ b/integration_tests/network/self-contained.yaml @@ -0,0 +1,16 @@ +id: example-self-contained-input + +info: + name: example-self-contained + author: pd-team + severity: info + +self-contained: true +network: + - host: + - "localhost:5431" + + matchers: + - type: word + words: + - "Authentication successful" \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 28b52d7c1..c9645f805 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -31,6 +31,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{}, "http/request-condition.yaml": &httpRequestCondition{}, "http/request-condition-new.yaml": &httpRequestCondition{}, + "http/self-contained.yaml": &httpRequestSelContained{}, } type httpGetHeaders struct{} @@ -493,3 +494,35 @@ func (h *httpRequestCondition) Execute(filePath string) error { } return nil } + +type httpRequestSelContained struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpRequestSelContained) Execute(filePath string) error { + router := httprouter.New() + var routerErr error + + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.Write([]byte("This is self-contained response")) + }) + server := &http.Server{ + Addr: "localhost:5431", + Handler: router, + } + go func() { + _ = server.ListenAndServe() + }() + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug) + if err != nil { + return err + } + if routerErr != nil { + return routerErr + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index 8c738537d..1eaff7c27 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -7,9 +7,10 @@ import ( ) var networkTestcases = map[string]testutils.TestCase{ - "network/basic.yaml": &networkBasic{}, - "network/hex.yaml": &networkBasic{}, - "network/multi-step.yaml": &networkMultiStep{}, + "network/basic.yaml": &networkBasic{}, + "network/hex.yaml": &networkBasic{}, + "network/multi-step.yaml": &networkMultiStep{}, + "network/self-contained.yaml": &networkRequestSelContained{}, } type networkBasic struct{} @@ -94,3 +95,28 @@ func (h *networkMultiStep) Execute(filePath string) error { } return nil } + +type networkRequestSelContained struct{} + +// Execute executes a test case and returns an error if occurred +func (h *networkRequestSelContained) Execute(filePath string) error { + var routerErr error + + ts := testutils.NewTCPServer(func(conn net.Conn) { + defer conn.Close() + + _, _ = conn.Write([]byte("Authentication successful")) + }, 5431) + defer ts.Close() + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug) + if err != nil { + return err + } + if routerErr != nil { + return routerErr + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index f2c398869..8c5491d28 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -81,10 +81,14 @@ type TCPServer struct { } // NewTCPServer creates a new TCP server from a handler -func NewTCPServer(handler func(conn net.Conn)) *TCPServer { +func NewTCPServer(handler func(conn net.Conn), port ...int) *TCPServer { server := &TCPServer{} - l, err := net.Listen("tcp", "127.0.0.1:0") + var gotPort int + if len(port) > 0 { + gotPort = port[0] + } + l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", gotPort)) if err != nil { panic(err) } From ec424e01e4a5bd1602854769531a9559b3045687 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 19 Oct 2021 22:20:29 +0530 Subject: [PATCH 136/147] MIsc --- v2/cmd/integration-test/http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index c9645f805..5b43f360e 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -503,7 +503,7 @@ func (h *httpRequestSelContained) Execute(filePath string) error { var routerErr error router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - w.Write([]byte("This is self-contained response")) + _, _ = w.Write([]byte("This is self-contained response")) }) server := &http.Server{ Addr: "localhost:5431", From 5383040a83000fe937c401df58e39f04e4653b88 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 19 Oct 2021 22:28:48 +0530 Subject: [PATCH 137/147] Misc --- v2/pkg/templates/templates.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 90f716add..330e87bab 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -63,7 +63,7 @@ type Template struct { CompiledWorkflow *workflows.Workflow `yaml:"-" json:"-" jsonschema:"-"` // description: | - // Self Contained marks Network Requests for the template as self-contained + // Self Contained marks Requests for the template as self-contained SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"` // TotalRequests is the total number of requests for the template. From 8e8249e6a5b8020004ceec1658ec820ca8032abe Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 20 Oct 2021 16:39:11 +0530 Subject: [PATCH 138/147] Added multiple short flags --- v2/cmd/nuclei/main.go | 72 +++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 44 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 477332596..c5170f201 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -53,43 +53,33 @@ on extensive configurability, massive extensibility and ease of use.`) ) createGroup(flagSet, "templates", "Templates", - flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), - flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"), - flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflows to run"), - - flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run newly added templates only"), + flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"), + flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "workflow or workflow directory paths to include in the scan"), flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), + flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), ) createGroup(flagSet, "filters", "Filtering", flagSet.NormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"), - flagSet.NormalizedStringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list + flagSet.NormalizedStringSliceVarP(&options.IncludeTags, "include-tags", "itags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list flagSet.NormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"), - - flagSet.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "templates to be executed even if they are excluded either by default or configuration"), - flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"), - - flagSet.VarP(&options.Severities, "impact", "severity", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), - flagSet.VarP(&options.ExcludeSeverities, "exclude-impact", "exclude-severity", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), - flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"), + flagSet.StringSliceVarP(&options.IncludeTemplates, "include-templates", "it", []string{}, "templates to be executed even if they are excluded either by default or configuration"), + flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), + flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), + flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), + flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) createGroup(flagSet, "output", "Output", flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"), - flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"), - flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), - flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"), flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"), - flagSet.BoolVar(&options.JSON, "json", false, "write output in JSONL(ines) format"), flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"), - flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"), flagSet.BoolVarP(&options.NoTimestamp, "no-timestamp", "nts", false, "don't display timestamp metadata in CLI output"), flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"), - flagSet.StringVarP(&options.MarkdownExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"), flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"), ) @@ -97,42 +87,37 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "configs", "Configurations", flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"), flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking - flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom headers in header:value format"), - flagSet.RuntimeMapVarP(&options.Vars, "var", "V", []string{}, "custom vars in var=value format"), - flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"), - flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"), + flagSet.BoolVarP(&options.SystemResolvers, "system-resolvers", "sr", false, "use system DNS resolving as error fallback"), flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"), - flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "enable environment variables support"), + flagSet.BoolVarP(&options.EnvironmentVariables, "env-vars", "ev", false, "enable environment variables to be used in template"), ) createGroup(flagSet, "interactsh", "interactsh", - flagSet.BoolVar(&options.NoInteractsh, "no-interactsh", false, "disable interactsh server for OOB testing"), - flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interactsh.com", "interactsh server url for self-hosted instance"), - flagSet.StringVar(&options.InteractshToken, "interactsh-token", "", "authentication token for self-hosted interactsh server"), + flagSet.StringVarP(&options.InteractshURL, "interactsh-server", "iserver", "https://interactsh.com", "interactsh server url for self-hosted instance"), + flagSet.StringVarP(&options.InteractshToken, "interactsh-token", "itoken", "", "authentication token for self-hosted interactsh server"), flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"), flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"), flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"), flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"), + flagSet.BoolVarP(&options.NoInteractsh, "no-interactsh", "ni", false, "disable interactsh server for OAST testing, exclude OAST based templates"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"), flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"), - flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"), + flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"), ) createGroup(flagSet, "optimization", "Optimizations", flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"), flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"), - flagSet.IntVar(&options.MaxHostError, "max-host-error", 30, "max errors for a host before skipping from scan"), - + flagSet.IntVarP(&options.MaxHostError, "max-host-error", "mhe", 30, "max errors for a host before skipping from scan"), flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"), flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"), - flagSet.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "stop processing HTTP requests after the first match (may break template/workflow logic)"), flagSet.BoolVar(&options.Stream, "stream", false, "Stream mode - start elaborating without sorting the input"), ) @@ -140,8 +125,8 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "headless", "Headless", flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"), flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"), - flagSet.BoolVar(&options.ShowBrowser, "show-browser", false, "show the browser on the screen when running templates with headless mode"), - flagSet.BoolVar(&options.UseInstalledChrome, "system-chrome", false, "Use local installed chrome browser instead of nuclei installed"), + flagSet.BoolVarP(&options.ShowBrowser, "show-browser", "sb", false, "show the browser on the screen when running templates with headless mode"), + flagSet.BoolVarP(&options.UseInstalledChrome, "system-chrome", "sc", false, "Use local installed chrome browser instead of nuclei installed"), ) createGroup(flagSet, "debug", "Debug", @@ -153,27 +138,26 @@ on extensive configurability, massive extensibility and ease of use.`) TODO should auto-set the HTTP_PROXY variable for the process? */ flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), - - flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"), - + flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), + flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), + flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"), flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"), ) createGroup(flagSet, "update", "Update", - flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei to the latest released version"), - flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update the community templates to latest released version"), - flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "do not check for nuclei-templates updates"), - flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default nuclei-templates directory"), + flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei engine to the latest released version"), + flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update nuclei-templates to latest released version"), + flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default directory to install nuclei-templates"), + flagSet.BoolVarP(&options.NoUpdateTemplates, "disable-update-check", "duc", false, "disable automatic nuclei/templates update check"), ) createGroup(flagSet, "stats", "Statistics", flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"), - flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to an output file in JSONL(ines) format"), + flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "write statistics data to an output file in JSONL(ines) format"), flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"), - - flagSet.BoolVar(&options.Metrics, "metrics", false, "expose nuclei metrics on a port"), - flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"), + flagSet.BoolVarP(&options.Metrics, "metrics", "m", false, "expose nuclei metrics on a port"), + flagSet.IntVarP(&options.MetricsPort, "metrics-port", "mp", 9092, "port to expose nuclei metrics on"), ) _ = flagSet.Parse() From 51f3ea5c88090c2484ad06ca9455483c9e9a5844 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 20 Oct 2021 20:13:40 +0530 Subject: [PATCH 139/147] Added constant for static ports in integration tests --- v2/cmd/integration-test/http.go | 2 +- v2/cmd/integration-test/network.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 5b43f360e..c3e23a7a0 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -506,7 +506,7 @@ func (h *httpRequestSelContained) Execute(filePath string) error { _, _ = w.Write([]byte("This is self-contained response")) }) server := &http.Server{ - Addr: "localhost:5431", + Addr: fmt.Sprintf("localhost:%d", defaultStaticPort), Handler: router, } go func() { diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index 1eaff7c27..e170b07b9 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -13,6 +13,8 @@ var networkTestcases = map[string]testutils.TestCase{ "network/self-contained.yaml": &networkRequestSelContained{}, } +const defaultStaticPort = 5431 + type networkBasic struct{} // Execute executes a test case and returns an error if occurred @@ -106,7 +108,7 @@ func (h *networkRequestSelContained) Execute(filePath string) error { defer conn.Close() _, _ = conn.Write([]byte("Authentication successful")) - }, 5431) + }, defaultStaticPort) defer ts.Close() results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug) if err != nil { From b8628a6e5b5eafbbdcd383be94df993d7be77ff1 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 20 Oct 2021 20:17:00 +0530 Subject: [PATCH 140/147] fixed comment --- v2/pkg/protocols/http/build_request.go | 7 +++---- v2/pkg/protocols/http/http.go | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 06384a240..08c459a5e 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -95,10 +95,10 @@ func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]int } ctx := context.Background() - isRawRequest := len(r.request.Raw) > 0 + isRawRequest := r.request.isRaw() - // If data contains \n it's a raw request, process it like raw. Else - // continue with the template based request flow. + // If the request is a raw request, get the URL from the request + // header and use it to make the request. if isRawRequest { // Get the hostname from the URL section to build the request. reader := bufio.NewReader(strings.NewReader(data)) @@ -126,7 +126,6 @@ func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]int dynamicValues, generators.BuildPayloadFromOptions(r.request.options.Options), ) - return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL) } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index ce3036c52..304f1a497 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -184,6 +184,10 @@ func (request *Request) GetID() string { return request.ID } +func (request *Request) isRaw() bool { + return len(request.Raw) > 0 +} + // Compile compiles the protocol request for further execution. func (request *Request) Compile(options *protocols.ExecuterOptions) error { connectionConfiguration := &httpclientpool.Configuration{ From 36edde22410779777d44e8cca186df5292765a5d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 20 Oct 2021 20:26:40 +0530 Subject: [PATCH 141/147] Added comment for old logic --- v2/pkg/protocols/http/build_request.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 08c459a5e..82dd94775 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -70,6 +70,11 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa data, parsed = baseURLWithTemplatePrefs(data, parsed) isRawRequest := len(r.request.Raw) > 0 + + // If the request is not a raw request, and the URL input path is suffixed with + // a trailing slash, and out Input URL is also suffixed with a trailing slash, + // mark trailingSlash bool as true which will be later used during variable generation + // to generate correct path removed slash which would otherwise generate // invalid sequence. trailingSlash := false if !isRawRequest && strings.HasSuffix(parsed.Path, "/") && strings.Contains(data, "{{BaseURL}}/") { trailingSlash = true From bb6486910fe63d892ae169d083023be85fa77eea Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 20 Oct 2021 20:27:36 +0530 Subject: [PATCH 142/147] typo --- v2/pkg/protocols/http/build_request.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 82dd94775..fa9be67c9 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -72,7 +72,7 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa isRawRequest := len(r.request.Raw) > 0 // If the request is not a raw request, and the URL input path is suffixed with - // a trailing slash, and out Input URL is also suffixed with a trailing slash, + // a trailing slash, and our Input URL is also suffixed with a trailing slash, // mark trailingSlash bool as true which will be later used during variable generation // to generate correct path removed slash which would otherwise generate // invalid sequence. trailingSlash := false From a6c5dbf7fbd52b9d83c16c8c0a5a9ed0314d3ad9 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 20 Oct 2021 20:29:28 +0530 Subject: [PATCH 143/147] added todo --- v2/pkg/protocols/http/build_request.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index fa9be67c9..cbeb1db69 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -75,10 +75,12 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa // a trailing slash, and our Input URL is also suffixed with a trailing slash, // mark trailingSlash bool as true which will be later used during variable generation // to generate correct path removed slash which would otherwise generate // invalid sequence. + // TODO: Figure out a cleaner way to do this sanitization. trailingSlash := false if !isRawRequest && strings.HasSuffix(parsed.Path, "/") && strings.Contains(data, "{{BaseURL}}/") { trailingSlash = true } + values := generators.MergeMaps( generators.MergeMaps(dynamicValues, generateVariables(parsed, trailingSlash)), generators.BuildPayloadFromOptions(r.request.options.Options), From 379233c6325992b722621a23c97f04ef5af577d4 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 20 Oct 2021 15:25:55 +0000 Subject: [PATCH 144/147] Auto Generate Syntax Docs + JSONSchema [Wed Oct 20 15:25:55 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 13 +++++++++++++ nuclei-jsonschema.json | 5 +++++ v2/pkg/templates/templates_doc.go | 7 ++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 19f59a3ae..30ed998fb 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -230,6 +230,19 @@ Workflows is a list of workflows to execute for a template.
+
+ +self-contained bool + +
+
+ +Self Contained marks Requests for the template as self-contained + +
+ +
+ diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 14c578e08..56253b7ed 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -911,6 +911,11 @@ "type": "array", "title": "list of workflows to execute", "description": "List of workflows to execute for template" + }, + "self-contained": { + "type": "boolean", + "title": "mark requests as self-contained", + "description": "Mark Requests for the template as self-contained" } }, "additionalProperties": false, diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 27436ea90..0b19e675d 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -31,7 +31,7 @@ func init() { TemplateDoc.Type = "Template" TemplateDoc.Comments[encoder.LineComment] = " Template is a YAML input file which defines all the requests and" TemplateDoc.Description = "Template is a YAML input file which defines all the requests and\n other metadata for a template." - TemplateDoc.Fields = make([]encoder.Doc, 8) + TemplateDoc.Fields = make([]encoder.Doc, 9) TemplateDoc.Fields[0].Name = "id" TemplateDoc.Fields[0].Type = "string" TemplateDoc.Fields[0].Note = "" @@ -84,6 +84,11 @@ func init() { TemplateDoc.Fields[7].Note = "" TemplateDoc.Fields[7].Description = "Workflows is a list of workflows to execute for a template." TemplateDoc.Fields[7].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template." + TemplateDoc.Fields[8].Name = "self-contained" + TemplateDoc.Fields[8].Type = "bool" + TemplateDoc.Fields[8].Note = "" + TemplateDoc.Fields[8].Description = "Self Contained marks Requests for the template as self-contained" + TemplateDoc.Fields[8].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained" MODELInfoDoc.Type = "model.Info" MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template" From fa79a16a9d261b46f743b7773dc1d1304dcfba7d Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 21 Oct 2021 05:56:15 +0530 Subject: [PATCH 145/147] reverting #1017 --- v2/internal/runner/options.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 745f15d02..59791fe63 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -22,11 +22,6 @@ func ParseOptions(options *types.Options) { // Check if stdin pipe was given options.Stdin = hasStdin() - // if VerboseVerbose is set, it implicitly enables the Verbose option as well - if options.VerboseVerbose { - options.Verbose = true - } - // Read the inputs and configure the logging configureOutput(options) @@ -127,7 +122,7 @@ func isValidURL(urlString string) bool { // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output - if options.Verbose || options.VerboseVerbose { + if options.Verbose { gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) } if options.Debug { From dd311897fcbb61c46a6722de51a740f4599b56c6 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 21 Oct 2021 06:05:09 +0530 Subject: [PATCH 146/147] misc flag update --- v2/cmd/nuclei/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index c5170f201..d2f8ebb9f 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -141,7 +141,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), - flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"), + flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"), ) From f260ed6da78e2f8295c82d00c13b6c54f09c9970 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 21 Oct 2021 06:17:18 +0530 Subject: [PATCH 147/147] readme + version update --- README.md | 75 +++++++++++++++++---------------- v2/pkg/catalog/config/config.go | 2 +- 2 files changed, 40 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 750e2a147..bfa68972e 100644 --- a/README.md +++ b/README.md @@ -90,30 +90,29 @@ TARGET: -l, -list string path to file containing a list of target URLs/hosts to scan (one per line) TEMPLATES: - -tl list all available templates -t, -templates string[] template or template directory paths to include in the scan - -w, -workflows string[] list of workflows to run - -nt, -new-templates run newly added templates only + -nt, -new-templates run only new templates added in latest nuclei-templates release + -w, -workflows string[] workflow or workflow directory paths to include in the scan -validate validate the passed templates to nuclei + -tl list all available templates FILTERING: - -tags string[] execute a subset of templates that contain the provided tags - -include-tags string[] tags from the default deny list that permit executing more intrusive templates - -etags, -exclude-tags string[] exclude templates with the provided tags - -include-templates string[] templates to be executed even if they are excluded either by default or configuration - -exclude-templates, -exclude string[] template or template directory paths to exclude - -severity, -impact value[] Templates to run based on severity. Possible values: info, low, medium, high, critical - -author string[] execute templates that are (co-)created by the specified authors + -tags string[] execute a subset of templates that contain the provided tags + -etags, -exclude-tags string[] exclude templates with the provided tags + -itags, -include-tags string[] tags from the default deny list that permit executing more intrusive templates + -et, -exclude-templates string[] template or template directory paths to exclude + -it, -include-templates string[] templates to be executed even if they are excluded either by default or configuration + -s, -severity value[] Templates to run based on severity. Possible values - info,low,medium,high,critical + -es, -exclude-severity value[] Templates to exclude based on severity. Possible values - info,low,medium,high,critical + -a, -author string[] execute templates that are (co-)created by the specified authors OUTPUT: -o, -output string output file to write found issues/vulnerabilities -silent display findings only - -v, -verbose show verbose output - -vv display extra verbose information -nc, -no-color disable output content coloring (ANSI escape codes) -json write output in JSONL(ines) format -irr, -include-rr include request/response pairs in the JSONL output (for findings only) - -nm, -no-meta don't display match metadata in CLI output + -nm, -no-meta don't display match metadata -nts, -no-timestamp don't display timestamp metadata in CLI output -rdb, -report-db string local nuclei reporting database (always use this to persist report data) -me, -markdown-export string directory to export results in markdown format @@ -125,37 +124,39 @@ CONFIGURATIONS: -H, -header string[] custom headers in header:value format -V, -var value custom vars in var=value format -r, -resolvers string file containing resolver list for nuclei - -system-resolvers use system DNS resolving as error fallback + -sr, -system-resolvers use system DNS resolving as error fallback -passive enable passive HTTP response processing mode - -env-vars enable environment variables support + -ev, -env-vars enable environment variables to be used in template INTERACTSH: - -no-interactsh disable interactsh server for OOB testing - -interactsh-url string interactsh server url for self-hosted instance (default "https://interactsh.com") - -interactsh-token string authentication token for self-hosted interactsh server - -interactions-cache-size int number of requests to keep in the interactions cache (default 5000) - -interactions-eviction int number of seconds to wait before evicting requests from cache (default 60) - -interactions-poll-duration int number of seconds to wait before each interaction poll request (default 5) - -interactions-cooldown-period int extra time for interaction polling before exiting (default 5) + -iserver, -interactsh-server string interactsh server url for self-hosted instance (default "https://interactsh.com") + -itoken, -interactsh-token string authentication token for self-hosted interactsh server + -interactions-cache-size int number of requests to keep in the interactions cache (default 5000) + -interactions-eviction int number of seconds to wait before evicting requests from cache (default 60) + -interactions-poll-duration int number of seconds to wait before each interaction poll request (default 5) + -interactions-cooldown-period int extra time for interaction polling before exiting (default 5) + -ni, -no-interactsh disable interactsh server for OAST testing, exclude OAST based templates RATE-LIMIT: -rl, -rate-limit int maximum number of requests to send per second (default 150) -rlm, -rate-limit-minute int maximum number of requests to send per minute -bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25) - -c, -concurrency int maximum number of templates to be executed in parallel (default 10) + -c, -concurrency int maximum number of templates to be executed in parallel (default 25) OPTIMIZATIONS: -timeout int time to wait in seconds before timeout (default 5) -retries int number of times to retry a failed request (default 1) - -max-host-error int max errors for a host before skipping from scan (default 30) + -mhe, -max-host-error int max errors for a host before skipping from scan (default 30) -project use a project folder to avoid sending same request multiple times - -project-path string set a specific project path (default "$TMPDIR/") + -project-path string set a specific project path -spm, -stop-at-first-path stop processing HTTP requests after the first match (may break template/workflow logic) + -stream Stream mode - start elaborating without sorting the input HEADLESS: - -headless enable templates that require headless browser support - -page-timeout int seconds to wait for each page in headless mode (default 20) - -show-browser show the browser on the screen when running templates with headless mode + -headless enable templates that require headless browser support + -page-timeout int seconds to wait for each page in headless mode (default 20) + -sb, -show-browser show the browser on the screen when running templates with headless mode + -sc, -system-chrome Use local installed chrome browser instead of nuclei installed DEBUG: -debug show all requests and responses @@ -163,22 +164,24 @@ DEBUG: -debug-resp show all received responses -proxy, -proxy-url string URL of the HTTP proxy server -proxy-socks-url string URL of the SOCKS proxy server - -trace-log string file to write sent requests trace log + -tlog, -trace-log string file to write sent requests trace log -version show nuclei version + -v, -verbose show verbose output + -vv display extra verbose information -tv, -templates-version shows the version of the installed nuclei-templates UPDATE: - -update update nuclei to the latest released version - -ut, -update-templates update the community templates to latest released version - -nut, -no-update-templates do not check for nuclei-templates updates - -ud, -update-directory string overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates") + -update update nuclei engine to the latest released version + -ut, -update-templates update nuclei-templates to latest released version + -ud, -update-directory string overwrite the default directory to install nuclei-templates + -duc, -disable-update-check disable automatic nuclei/templates update check STATISTICS: -stats display statistics about the running scan - -stats-json write statistics data to an output file in JSONL(ines) format + -sj, -stats-json write statistics data to an output file in JSONL(ines) format -si, -stats-interval int number of seconds to wait between showing a statistics update (default 5) - -metrics expose nuclei metrics on a port - -metrics-port int port to expose nuclei metrics on (default 9092) + -m, -metrics expose nuclei metrics on a port + -mp, -metrics-port int port to expose nuclei metrics on (default 9092) ``` ### Running Nuclei diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 41ce53613..8a996b62d 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -26,7 +26,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.3-dev` +const Version = `2.5.3` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir()